diff --git a/Cargo.lock b/Cargo.lock index 497b79f7..55d8239a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1025,9 +1025,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -1753,7 +1753,7 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint", + "num-bigint 0.4.8", "num-traits", "rusticata-macros", ] @@ -2456,9 +2456,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "fixedbitset" @@ -3380,9 +3380,9 @@ checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" [[package]] name = "hickory-net" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1" dependencies = [ "async-trait", "bitflags 2.13.1", @@ -3421,9 +3421,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d" dependencies = [ "bitflags 2.13.1", "data-encoding", @@ -3444,9 +3444,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f" dependencies = [ "cfg-if", "futures-util", @@ -3934,18 +3934,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "internal-russh-num-bigint" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" -dependencies = [ - "num-integer", - "num-traits", - "rand 0.10.2", - "rand_core 0.10.1", -] - [[package]] name = "ipconfig" version = "0.3.4" @@ -5882,6 +5870,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", + "rand 0.10.2", + "rand_core 0.10.1", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -6465,9 +6465,9 @@ dependencies = [ [[package]] name = "pageant" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791" +checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce" dependencies = [ "base16ct", "byteorder", @@ -7774,9 +7774,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.63.1" +version = "0.63.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031" +checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd" dependencies = [ "aes 0.9.3", "bitflags 2.13.1", @@ -7803,13 +7803,12 @@ dependencies = [ "hex-literal", "hmac", "inout 0.2.2", - "internal-russh-num-bigint", "keccak", "log", "md5", "ml-kem", "module-lattice", - "num-bigint", + "num-bigint 0.5.1", "p256", "p384", "p521", @@ -10076,9 +10075,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -10141,9 +10140,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -10160,7 +10159,7 @@ dependencies = [ "chrono", "futures", "log", - "num-bigint", + "num-bigint 0.4.8", "pem 3.0.6", "proc-macro2", "rcgen", @@ -10968,8 +10967,11 @@ dependencies = [ "jiff", "libc", "mdns-sd", + "noq", + "noq-proto", "parking_lot", "prefix-trie 0.10.1", + "proptest", "rand 0.10.2", "reqwest", "rstest", @@ -12378,13 +12380,14 @@ dependencies = [ [[package]] name = "wnaf" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" dependencies = [ "ff", "group", "hybrid-array", + "primefield", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 58105c53..6e51616b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,6 +144,8 @@ rcgen = "0.14" # Networking iroh = { version = "1.1.0" } +noq = { version = "1.2.0" } +noq-proto = { version = "1.2.0" } iroh-relay = { version = "1.1.0", default-features = false, features = [ "server", ] } diff --git a/crates/tunnet-agent/src/accept.rs b/crates/tunnet-agent/src/accept.rs index 81a6ada2..732412f5 100644 --- a/crates/tunnet-agent/src/accept.rs +++ b/crates/tunnet-agent/src/accept.rs @@ -13,11 +13,11 @@ use tunnet_common::ws::ClientMsg; use tunnet_common::{RECORDING_ALPN, SEND_ALPN, TUNNEL_ALPN}; use tunnet_core::Docs; use tunnet_core::direct::{ - AUTH_ALPN, AuthCache, DOCS_ALPN, DocsMembership, FirewallEngine, GOSSIP_ALPN, - SharedAuthServerContext, SpoofTracker, run_auth_server, + AUTH_ALPN, AuthCache, DOCS_ALPN, DocsMembership, GOSSIP_ALPN, SharedAuthServerContext, + SpoofTracker, run_auth_server, }; use tunnet_core::stream::{StreamHandler, StreamProtocolHandler, TUNNEL_STREAM_ALPN}; -use tunnet_core::{AclEngine, ConnPool, RoutingTable, SendManager, SignedClient}; +use tunnet_core::{AclEngine, ConnPool, PolicyRuntime, RoutingTable, SendManager, SignedClient}; use uuid::Uuid; use crate::actors::dataplane::PublishedPlane; @@ -30,6 +30,7 @@ pub struct AcceptDeps { pub endpoint: iroh::Endpoint, pub routes: RoutingTable, pub acl: AclEngine, + pub runtime: PolicyRuntime, pub metrics: AgentMetrics, pub tun: PublishedPlane, pub stream_handler: StreamHandler, @@ -43,9 +44,9 @@ pub struct AcceptDeps { pub auth_server_ctx: Option, pub state_dir: PathBuf, pub docs: HashMap, - pub firewalls: HashMap, pub spoofs: HashMap, pub dgram_pool: ConnPool, + pub bufs: Arc, pub agent_gossip: Option, pub shared_docs: Option, pub ingress: IngressRegistry, @@ -57,12 +58,13 @@ pub fn spawn(deps: AcceptDeps) -> Router { tun: deps.tun, routes: deps.routes.clone(), acl: deps.acl.clone(), - firewalls: deps.firewalls, + runtime: deps.runtime.clone(), spoofs: deps.spoofs, dgram_pool: deps.dgram_pool, + bufs: deps.bufs, metrics: deps.metrics, - direct_auth: deps.direct_auth.clone(), ingress: deps.ingress, + direct_auth: deps.direct_auth.clone(), }; let stream = StreamProtocolHandler::new(deps.stream_handler); let auth = AuthHandler { @@ -115,12 +117,13 @@ struct TunnelHandler { tun: PublishedPlane, routes: RoutingTable, acl: AclEngine, - firewalls: HashMap, + runtime: PolicyRuntime, spoofs: HashMap, dgram_pool: ConnPool, + bufs: Arc, metrics: AgentMetrics, - direct_auth: Option, ingress: IngressRegistry, + direct_auth: Option, } impl fmt::Debug for TunnelHandler { @@ -148,22 +151,24 @@ impl ProtocolHandler for TunnelHandler { let tun = self.tun.clone(); let routes = self.routes.clone(); let acl = self.acl.clone(); - let firewalls = self.firewalls.clone(); + let runtime = self.runtime.clone(); let spoofs = self.spoofs.clone(); let dgram_pool = self.dgram_pool.clone(); + let bufs = self.bufs.clone(); let metrics = self.metrics.clone(); - let direct_auth = self.direct_auth.clone(); + let auth = self.direct_auth.clone(); async move { serve_tunnel_connection(InboundDeps { conn, tun, routes, + runtime, acl, - firewalls, spoofs, pool: Some(dgram_pool), + bufs, metrics, - direct_auth, + auth, }) .await; } diff --git a/crates/tunnet-agent/src/actors/dataplane.rs b/crates/tunnet-agent/src/actors/dataplane.rs index 796af4f3..095185eb 100644 --- a/crates/tunnet-agent/src/actors/dataplane.rs +++ b/crates/tunnet-agent/src/actors/dataplane.rs @@ -184,6 +184,11 @@ impl DataPlaneActor { self.peer_dns_active.store(false, Ordering::SeqCst); self.up = false; self.status.set_up(false); + self.status.set_outbound_alive(false); + // NOTE: `restarting` is deliberately left alone here: teardown runs + // on every stop including crashes, and a crash must keep reporting + // `restarting` until the next successful bring-up clears it. + // `do_bring_down` clears it for intentional shutdowns. } async fn do_bring_up( @@ -276,12 +281,6 @@ impl DataPlaneActor { } crate::forward::ensure_exit_nat(self.node.routes.is_exit_node()); - let firewalls: std::collections::HashMap<_, _> = self - .node - .direct - .iter() - .map(|(id, rt)| (*id, rt.firewall.clone())) - .collect(); // The outbound loop's unexpected end is abnormal: report it so // supervision restarts us. Shutdown ends it via abort (the // generation token is already cancelled then, so no report fires). @@ -289,14 +288,20 @@ impl DataPlaneActor { .generation_cancel .clone() .expect("generation token published above"); + // Shared tunnel packet resources for this generation: pooled buffers and + // the runtime sweeper (tied to the generation token — no leaked tasks + // across bring-up cycles). + let packet_pool = tunnet_common::packet::PacketPool::new(128); + self.node.policy.spawn_sweeper(exit_gen.clone()); let exit_weak = self_ref.clone(); let outbound = crate::dataplane::spawn_outbound(crate::dataplane::OutboundSpawn { tun, routes: self.node.routes.clone(), pool: self.node.tunnel_pool.clone(), - acl: self.node.acl.clone(), - firewalls, + runtime: self.node.policy.clone(), metrics: self.metrics.clone(), + bufs: packet_pool, + meter: self.node.tunnel_pool.cloud_relay_meter(), mtu: self.cfg.mtu, on_unexpected_end: Box::new(move || { if !exit_gen.is_cancelled() @@ -309,6 +314,38 @@ impl DataPlaneActor { self.outbound = Some(outbound); self.up = true; self.status.set_up(true); + self.status.set_restarting(false); + self.status.set_outbound_alive(true); + self.status.set_generation(generation); + // Eager preconnect (keep-alive): dial every known peer NOW so the + // first real packet doesn't pay connection setup (the classic + // first-ping-timeout). Best-effort and bounded: skipped peers are + // still dialed on demand by the pump. Skipped entirely without + // keep-alive. + if self.node.tunnel_pool.keep_alive() { + let pool = self.node.tunnel_pool.clone(); + let routes = self.node.routes.clone(); + let local = pool.endpoint().id(); + tokio::spawn(async move { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(8)); + let mut set = tokio::task::JoinSet::new(); + for peer in routes.peers() { + if peer.endpoint == local { + continue; + } + let Ok(permit) = sem.clone().try_acquire_owned() else { + continue; + }; + let pool = pool.clone(); + let ep = peer.endpoint; + set.spawn(async move { + let _permit = permit; + let _ = pool.get(ep).await; + }); + } + while set.join_next().await.is_some() {} + }); + } let _ = self.events.send(LocalEvent::DataPlaneChanged { up: true }); tracing::info!("data plane up"); Ok(()) @@ -323,6 +360,7 @@ impl DataPlaneActor { // readers also self-remove from the registry on exit. self.ingress.abort_all(); self.teardown().await; + self.status.set_restarting(false); let _ = self.events.send(LocalEvent::DataPlaneChanged { up: false }); tracing::info!("data plane down"); Ok(()) @@ -373,7 +411,13 @@ impl Message for DataPlaneActor { async fn handle(&mut self, _msg: BringUp, ctx: &mut Context) -> Self::Reply { let weak = ctx.actor_ref().downgrade(); - self.do_bring_up(weak).await + let res = self.do_bring_up(weak).await; + if let Err(e) = &res { + // Failed bring-up is Down, not restarting: record the cause. + self.status.set_restarting(false); + self.status.set_last_error(format!("bring-up failed: {e}")); + } + res } } @@ -400,6 +444,13 @@ impl Message for DataPlaneActor { type Reply = (); async fn handle(&mut self, _msg: OutboundExited, _ctx: &mut Context) { + // Publish degraded state BEFORE supervision restarts us, so status + // readers see `restarting` (with the error and restart count) + // instead of a stale healthy `up`. + self.status + .note_restart("outbound TUN loop unexpectedly terminated".into()); + self.status.set_outbound_alive(false); + self.status.set_restarting(true); panic!("outbound TUN loop unexpectedly terminated"); } } @@ -481,6 +532,16 @@ impl DataPlaneControl for ActorDataPlaneControl { self.status.is_up() } + fn data_plane_info(&self) -> tunnet_common::local_api::DataPlaneInfo { + tunnet_common::local_api::DataPlaneInfo { + state: self.status.state().to_string(), + outbound_alive: self.status.outbound_alive(), + restart_count: self.status.restart_count(), + generation: self.status.generation(), + last_error: self.status.last_error(), + } + } + async fn bring_up(&self) -> Result<(), String> { // Kameo flattens `Result` replies into the `ask` error channel. tokio::time::timeout(std::time::Duration::from_secs(30), self.actor.ask(BringUp)) diff --git a/crates/tunnet-agent/src/actors/test_support.rs b/crates/tunnet-agent/src/actors/test_support.rs index 03bbd551..895ae724 100644 --- a/crates/tunnet-agent/src/actors/test_support.rs +++ b/crates/tunnet-agent/src/actors/test_support.rs @@ -44,6 +44,18 @@ pub async fn test_node() -> (tunnet_core::CoreNode, tempfile::TempDir) { ) .await .expect("open send manager"); + let policy = tunnet_core::PolicyRuntime::bootstrap( + &tunnet_common::policy::PolicyBundle::default(), + &std::collections::HashMap::new(), + &tunnet_core::SelfIdentity { + endpoint_hex: identity.endpoint_id_hex(), + ip: "10.9.0.1".parse().unwrap(), + tags: vec![], + network: "test".into(), + }, + true, + false, + ); let node = tunnet_core::CoreNode { identity, persisted: tunnet_core::PersistedState::Direct { networks: vec![] }, @@ -66,6 +78,7 @@ pub async fn test_node() -> (tunnet_core::CoreNode, tempfile::TempDir) { gossip: None, docs_engine: None, presence_tables: Arc::new(std::sync::Mutex::new(HashMap::new())), + policy, }; (node, tmp) } diff --git a/crates/tunnet-agent/src/dataplane.rs b/crates/tunnet-agent/src/dataplane.rs index a9a825f5..dd50a0f7 100644 --- a/crates/tunnet-agent/src/dataplane.rs +++ b/crates/tunnet-agent/src/dataplane.rs @@ -8,7 +8,8 @@ use std::net::Ipv4Addr; use std::sync::Arc; use tun_rs::AsyncDevice; -use tunnet_core::{AclEngine, ConnPool, RoutingTable}; +use tunnet_common::packet::PacketPool; +use tunnet_core::{ConnPool, PolicyRuntime, RoutingTable}; use crate::metrics::AgentMetrics; @@ -16,9 +17,10 @@ pub struct OutboundSpawn { pub tun: Arc, pub routes: RoutingTable, pub pool: ConnPool, - pub acl: AclEngine, - pub firewalls: std::collections::HashMap, + pub runtime: PolicyRuntime, pub metrics: AgentMetrics, + pub bufs: Arc, + pub meter: tunnet_core::CloudRelayMeter, pub mtu: u16, /// Called when the loop ends without shutdown (abnormal service death). pub on_unexpected_end: Box, @@ -29,9 +31,10 @@ pub fn spawn_outbound(spawn: OutboundSpawn) -> tokio::task::JoinHandle<()> { tun, routes, pool, - acl, - firewalls, + runtime, metrics, + bufs, + meter, mtu, on_unexpected_end, } = spawn; @@ -40,9 +43,10 @@ pub fn spawn_outbound(spawn: OutboundSpawn) -> tokio::task::JoinHandle<()> { tun, routes, pool, - acl, - firewalls, + runtime, metrics, + bufs, + meter, mtu, }) .await diff --git a/crates/tunnet-agent/src/dgram_pump.rs b/crates/tunnet-agent/src/dgram_pump.rs index b7345980..39c10b1b 100644 --- a/crates/tunnet-agent/src/dgram_pump.rs +++ b/crates/tunnet-agent/src/dgram_pump.rs @@ -3,9 +3,10 @@ use std::collections::HashMap; use std::sync::Arc; +use tunnet_common::packet::PacketPool; use tunnet_core::ConnPool; -use tunnet_core::direct::{AuthCache, FirewallEngine, SpoofTracker}; -use tunnet_core::{AclEngine, RoutingTable}; +use tunnet_core::direct::SpoofTracker; +use tunnet_core::{AclEngine, PolicyRuntime, RoutingTable}; use uuid::Uuid; use crate::actors::dataplane::PublishedPlane; @@ -24,10 +25,10 @@ pub fn install_dialer_datagram_pump( tun_slot: PublishedPlane, routes: RoutingTable, acl: AclEngine, - firewalls: HashMap, + runtime: PolicyRuntime, spoofs: HashMap, metrics: AgentMetrics, - direct_auth: Option, + bufs: Arc, ingress: IngressRegistry, ) { let pool_for_hook = pool.clone(); @@ -35,10 +36,10 @@ pub fn install_dialer_datagram_pump( let tun_slot = tun_slot.clone(); let routes = routes.clone(); let acl = acl.clone(); - let firewalls = firewalls.clone(); + let runtime = runtime.clone(); let spoofs = spoofs.clone(); let metrics = metrics.clone(); - let direct_auth = direct_auth.clone(); + let bufs = bufs.clone(); let pool = pool_for_hook.clone(); let ingress = ingress.clone(); ingress.force_spawn(peer, async move { @@ -49,12 +50,15 @@ pub fn install_dialer_datagram_pump( conn, tun: tun_slot, routes, + runtime, acl, - firewalls, spoofs, pool: Some(pool), + bufs, metrics, - direct_auth, + // Dialer-side readers have no AuthCache handle; membership + // existence still gates every frame network. + auth: None, }) .await; }); diff --git a/crates/tunnet-agent/src/ingress.rs b/crates/tunnet-agent/src/ingress.rs index 399b6426..9771fe9f 100644 --- a/crates/tunnet-agent/src/ingress.rs +++ b/crates/tunnet-agent/src/ingress.rs @@ -8,9 +8,14 @@ use iroh::EndpointId; use tokio::task::JoinHandle; /// Tracks active TUN ingress tasks per remote endpoint. +/// +/// Each registration carries a generation: a finishing old reader removes +/// its entry ONLY if no newer reader replaced it (otherwise a slow +/// shutdown could unregister a live replacement and leave the peer +/// readerless — or resurrect routing for a dead one). #[derive(Clone, Default)] pub struct IngressRegistry { - readers: Arc>>, + readers: Arc)>>, generation: Arc, } @@ -24,27 +29,29 @@ impl IngressRegistry { self.generation.fetch_add(1, Ordering::SeqCst); } - /// Abort any existing reader and start a new one. + /// Abort any existing reader and start a new one. The old task's exit + /// cleanup cannot remove the new registration (generation-guarded). pub fn force_spawn(&self, peer: EndpointId, fut: F) where F: std::future::Future + Send + 'static, { - if let Some((_, h)) = self.readers.remove(&peer) { + let reader_gen = self.generation.fetch_add(1, Ordering::SeqCst); + if let Some((_, (_, h))) = self.readers.remove(&peer) { h.abort(); } let readers = self.readers.clone(); let handle = tokio::spawn(async move { fut.await; - readers.remove(&peer); + readers.remove_if(&peer, |_, (g, _)| *g == reader_gen); }); - self.readers.insert(peer, handle); + self.readers.insert(peer, (reader_gen, handle)); } pub fn abort_all(&self) { self.bump_generation(); let keys: Vec<_> = self.readers.iter().map(|e| *e.key()).collect(); for k in keys { - if let Some((_, h)) = self.readers.remove(&k) { + if let Some((_, (_, h))) = self.readers.remove(&k) { h.abort(); } } @@ -52,7 +59,7 @@ impl IngressRegistry { #[cfg(test)] pub fn has_reader(&self, peer: EndpointId) -> bool { - self.readers.get(&peer).is_some_and(|h| !h.is_finished()) + self.readers.get(&peer).is_some_and(|h| !h.1.is_finished()) } } @@ -96,6 +103,54 @@ mod tests { }); } + #[test] + fn stale_reader_exit_keeps_new_registration() { + // An old reader that exits NORMALLY after being replaced (e.g. + // tie-break swap: the old connection closes on its own while the + // new reader runs) must not unregister the live replacement. + // (Aborted tasks never run wrapper cleanup — the future is + // dropped — so only normal exits exercise this path.) + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let reg = IngressRegistry::new(); + let mut bytes = [9u8; 32]; + bytes[0] = 4; + let p = iroh::SecretKey::from(bytes).public(); + let (tx_old, rx_old) = tokio::sync::oneshot::channel::<()>(); + let (tx_new, rx_new) = tokio::sync::oneshot::channel::<()>(); + reg.force_spawn(p, async move { + let _ = rx_old.await; + }); + tokio::task::yield_now().await; + assert!(reg.has_reader(p)); + // Concurrent replacement with a newer generation (same + // discipline force_spawn itself uses). + let new_gen = reg.generation.fetch_add(1, Ordering::SeqCst); + let readers = reg.readers.clone(); + let new_handle = tokio::spawn(async move { + let _ = rx_new.await; + readers.remove_if(&p, |_, (g, _)| *g == new_gen); + }); + reg.readers.insert(p, (new_gen, new_handle)); + tokio::task::yield_now().await; + // Old reader now exits normally (its connection closed). + drop(tx_old); + for _ in 0..10 { + tokio::task::yield_now().await; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert!( + reg.has_reader(p), + "old exit cleanup must not remove the new reader" + ); + drop(tx_new); + tokio::task::yield_now().await; + }); + } + #[test] fn abort_all_clears_readers() { let rt = tokio::runtime::Builder::new_current_thread() diff --git a/crates/tunnet-agent/src/main.rs b/crates/tunnet-agent/src/main.rs index b46f6cf4..f2fa34f0 100644 --- a/crates/tunnet-agent/src/main.rs +++ b/crates/tunnet-agent/src/main.rs @@ -17,7 +17,7 @@ mod ingress; mod magic_dns; mod metrics; mod policy_api; -mod qos; +mod pump; mod recorder; mod runtime; #[cfg(unix)] @@ -29,6 +29,7 @@ mod system_dns; mod system_firewall; mod system_info; mod system_routes; +mod tun_fast; mod tun_io; mod underlay; #[cfg(unix)] diff --git a/crates/tunnet-agent/src/metrics.rs b/crates/tunnet-agent/src/metrics.rs index 07f069ac..cf589476 100644 --- a/crates/tunnet-agent/src/metrics.rs +++ b/crates/tunnet-agent/src/metrics.rs @@ -1,11 +1,114 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::time::Duration; -use metrics::{counter, describe_counter, describe_gauge, gauge}; +use metrics::{Counter, Gauge, counter, describe_counter, describe_gauge, gauge}; use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; +/// Cached metric handles: registered once, incremented without per-packet +/// registry lookup. Hot packet/byte counters accumulate in shared atomics +/// (correct across peers — no per-peer gauge overwrite) and flush once a +/// second. Sojourn latency uses a bounded bucket histogram for p50/p95/p99 +/// estimates without per-packet registry work. Drop reasons on hot paths +/// use pre-registered handles selected by a plain match. #[derive(Clone)] pub struct AgentMetrics { handle: PrometheusHandle, + packets_out: Counter, + packets_in: Counter, + bytes_out: Counter, + bytes_in: Counter, + active_conns: Gauge, + queue_packets: Gauge, + queue_bytes: Gauge, + queue_flows: Gauge, + transport_full: Counter, + frames: Counter, + segments: Counter, + pool_hits: Counter, + pool_miss: Counter, + drop_sched_stale: Counter, + drop_sched_peer: Counter, + drop_sched_flow: Counter, + drop_sched_codel: Counter, + drop_policy: Counter, + drop_too_large: Counter, + drop_no_conn: Counter, + drop_other: Counter, + sojourn_p50: Gauge, + sojourn_p95: Gauge, + sojourn_p99: Gauge, + sojourn_avg: Gauge, + hot: Arc, + sojourn: Arc, +} + +#[derive(Default)] +struct HotCounters { + packets_out: AtomicU64, + bytes_out: AtomicU64, + packets_in: AtomicU64, + bytes_in: AtomicU64, + queue_packets: AtomicI64, + queue_bytes: AtomicI64, + queue_flows: AtomicI64, +} + +/// Sojourn buckets (ms): [1, 5, 25, 100, 250, 1000, +inf). p50/p95/p99 are +/// estimated from cumulative counts — cheap and bounded. +const SOJOURN_BOUNDS_MS: [u64; 6] = [1, 5, 25, 100, 250, 1000]; + +struct SojournHist { + buckets: [AtomicU64; 7], + sum_us: AtomicU64, + count: AtomicU64, +} + +impl SojournHist { + fn observe(&self, d: Duration) { + let ms = d.as_millis() as u64; + let mut idx = 6; + for (i, b) in SOJOURN_BOUNDS_MS.iter().enumerate() { + if ms <= *b { + idx = i; + break; + } + } + self.buckets[idx].fetch_add(1, Ordering::Relaxed); + self.sum_us + .fetch_add(d.as_micros() as u64, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + } + + /// Estimate the q-quantile (0 < q <= 1) as a bucket upper bound in ms. + fn quantile_ms(&self, q: f64) -> f64 { + let total: u64 = self.buckets.iter().map(|b| b.load(Ordering::Relaxed)).sum(); + if total == 0 { + return 0.0; + } + let mut cum = 0u64; + for (i, b) in self.buckets.iter().enumerate() { + cum += b.load(Ordering::Relaxed); + if cum as f64 >= q * total as f64 { + return if i < SOJOURN_BOUNDS_MS.len() { + SOJOURN_BOUNDS_MS[i] as f64 + } else { + 2000.0 + }; + } + } + 2000.0 + } +} + +impl Default for SojournHist { + fn default() -> Self { + Self { + buckets: Default::default(), + sum_us: AtomicU64::new(0), + count: AtomicU64::new(0), + } + } } impl AgentMetrics { @@ -15,8 +118,40 @@ impl AgentMetrics { let recorder = PrometheusBuilder::new() .with_recommended_naming(true) .build_recorder(); + Self::from_handle(recorder.handle()) + } + + #[allow(clippy::too_many_lines)] + fn from_handle(handle: PrometheusHandle) -> Self { Self { - handle: recorder.handle(), + handle, + packets_out: counter!("tunnet_packets_total", "direction" => "out"), + packets_in: counter!("tunnet_packets_total", "direction" => "in"), + bytes_out: counter!("tunnet_bytes_total", "direction" => "out"), + bytes_in: counter!("tunnet_bytes_total", "direction" => "in"), + active_conns: gauge!("tunnet_active_connections"), + queue_packets: gauge!("tunnet_sched_queue_packets"), + queue_bytes: gauge!("tunnet_sched_queue_bytes"), + queue_flows: gauge!("tunnet_sched_active_flows"), + transport_full: counter!("tunnet_sched_transport_full_total"), + frames: counter!("tunnet_frames_total"), + segments: counter!("tunnet_segments_total"), + pool_hits: counter!("tunnet_pool_hits_total"), + pool_miss: counter!("tunnet_pool_miss_total"), + drop_sched_stale: counter!("tunnet_sched_drops_total", "reason" => "sched_stale"), + drop_sched_peer: counter!("tunnet_sched_drops_total", "reason" => "sched_peer"), + drop_sched_flow: counter!("tunnet_sched_drops_total", "reason" => "sched_flow_cap"), + drop_sched_codel: counter!("tunnet_sched_drops_total", "reason" => "sched_codel"), + drop_policy: counter!("tunnet_dropped_packets_total", "reason" => "policy_deny"), + drop_too_large: counter!("tunnet_dropped_packets_total", "reason" => "datagram_too_large"), + drop_no_conn: counter!("tunnet_dropped_packets_total", "reason" => "no_connection"), + drop_other: counter!("tunnet_dropped_packets_total", "reason" => "other"), + sojourn_p50: gauge!("tunnet_sojourn_p50_ms"), + sojourn_p95: gauge!("tunnet_sojourn_p95_ms"), + sojourn_p99: gauge!("tunnet_sojourn_p99_ms"), + sojourn_avg: gauge!("tunnet_sojourn_avg_ms"), + hot: Arc::new(HotCounters::default()), + sojourn: Arc::new(SojournHist::default()), } } @@ -28,38 +163,195 @@ impl AgentMetrics { describe_counter!("tunnet_packets_total", "Packets processed by the tunnel"); describe_counter!("tunnet_bytes_total", "Bytes processed by the tunnel"); describe_counter!("tunnet_dropped_packets_total", "Packets dropped"); + describe_counter!( + "tunnet_sched_transport_full_total", + "Transport-full events (scheduler owns drop/retry)" + ); + describe_counter!("tunnet_sched_drops_total", "Scheduler drops by reason"); + describe_counter!("tunnet_frames_total", "Overlay tunnel frames transmitted"); + describe_counter!( + "tunnet_segments_total", + "Overlay tunnel segments transmitted" + ); + describe_counter!("tunnet_reassembly_total", "Reassembly outcomes by result"); + describe_counter!("tunnet_tun_syscalls_total", "TUN syscalls by operation"); + describe_counter!("tunnet_datagrams_total", "QUIC DATAGRAMs by direction"); + describe_counter!("tunnet_pool_hits_total", "Packet pool hits"); + describe_counter!("tunnet_pool_miss_total", "Packet pool misses"); describe_gauge!("tunnet_active_connections", "Live peer connections"); + describe_gauge!( + "tunnet_sched_queue_packets", + "Queued logical packets across all peers" + ); + describe_gauge!("tunnet_sched_queue_bytes", "Queued bytes across all peers"); + describe_gauge!("tunnet_sched_active_flows", "Active flows across all peers"); + describe_gauge!("tunnet_virtual_mtu_bytes", "Configured logical MTU"); + describe_gauge!("tunnet_sojourn_p50_ms", "Queue sojourn p50 estimate"); + describe_gauge!("tunnet_sojourn_p95_ms", "Queue sojourn p95 estimate"); + describe_gauge!("tunnet_sojourn_p99_ms", "Queue sojourn p99 estimate"); + describe_gauge!("tunnet_sojourn_avg_ms", "Queue sojourn mean"); - let upkeep = handle.clone(); + let m = Self::from_handle(handle.clone()); + // Periodic flush of hot counters + prometheus upkeep. + let flush = m.clone(); tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(5)); + let mut interval = tokio::time::interval(Duration::from_secs(1)); loop { interval.tick().await; - upkeep.run_upkeep(); + flush.flush_hot(); + handle.run_upkeep(); } }); - Ok(Self { handle }) + Ok(m) + } + + fn flush_hot(&self) { + let p_out = self.hot.packets_out.swap(0, Ordering::Relaxed); + let b_out = self.hot.bytes_out.swap(0, Ordering::Relaxed); + let p_in = self.hot.packets_in.swap(0, Ordering::Relaxed); + let b_in = self.hot.bytes_in.swap(0, Ordering::Relaxed); + if p_out > 0 { + self.packets_out.increment(p_out); + } + if b_out > 0 { + self.bytes_out.increment(b_out); + } + if p_in > 0 { + self.packets_in.increment(p_in); + } + if b_in > 0 { + self.bytes_in.increment(b_in); + } + self.queue_packets + .set(self.hot.queue_packets.load(Ordering::Relaxed) as f64); + self.queue_bytes + .set(self.hot.queue_bytes.load(Ordering::Relaxed) as f64); + self.queue_flows + .set(self.hot.queue_flows.load(Ordering::Relaxed) as f64); + self.sojourn_p50.set(self.sojourn.quantile_ms(0.50)); + self.sojourn_p95.set(self.sojourn.quantile_ms(0.95)); + self.sojourn_p99.set(self.sojourn.quantile_ms(0.99)); + let n = self.sojourn.count.load(Ordering::Relaxed); + if n > 0 { + self.sojourn_avg + .set(self.sojourn.sum_us.load(Ordering::Relaxed) as f64 / 1000.0 / n as f64); + } } pub fn packets_inc(&self, direction: &'static str) { - counter!("tunnet_packets_total", "direction" => direction).increment(1); + match direction { + "out" => self.hot.packets_out.fetch_add(1, Ordering::Relaxed), + "in" => self.hot.packets_in.fetch_add(1, Ordering::Relaxed), + _ => { + counter!("tunnet_packets_total", "direction" => direction).increment(1); + 0 + } + }; } pub fn bytes_add(&self, direction: &'static str, n: u64) { - counter!("tunnet_bytes_total", "direction" => direction).increment(n); + match direction { + "out" => self.hot.bytes_out.fetch_add(n, Ordering::Relaxed), + "in" => self.hot.bytes_in.fetch_add(n, Ordering::Relaxed), + _ => { + counter!("tunnet_bytes_total", "direction" => direction).increment(n); + 0 + } + }; } + /// Drop counter with pre-registered hot handles (no per-packet lookup). pub fn dropped_inc(&self, reason: &'static str) { - counter!("tunnet_dropped_packets_total", "reason" => reason).increment(1); + match reason { + "sched_stale" | "sched_emergency" => self.drop_sched_stale.increment(1), + "sched_peer_bytes" | "sched_peer_packets" => self.drop_sched_peer.increment(1), + "sched_flow_cap" => self.drop_sched_flow.increment(1), + "sched_codel" => self.drop_sched_codel.increment(1), + "policy_deny" | "policy_deny_in" => self.drop_policy.increment(1), + "datagram_too_large" => self.drop_too_large.increment(1), + "no_connection" => self.drop_no_conn.increment(1), + _ => self.drop_other.increment(1), + } + } + + /// Scheduler drop (same cached handles; reason already scheduler-scoped). + pub fn sched_drop_inc(&self, reason: &'static str) { + self.dropped_inc(reason); + } + + /// Report drained scheduler drop deltas (CoDel/emergency drops observed + /// inside dequeue, which have no enqueue decision site to report them). + pub fn sched_drops_add(&self, codel: u64, emergency: u64) { + if codel > 0 { + self.drop_sched_codel.increment(codel); + } + if emergency > 0 { + self.drop_sched_stale.increment(emergency); + } + } + + /// Aggregate queue levels across all peers (signed deltas, never + /// overwrites another peer's values). + pub fn queue_add(&self, packets: i64, bytes: i64, flows: i64) { + if packets != 0 { + self.hot.queue_packets.fetch_add(packets, Ordering::Relaxed); + } + if bytes != 0 { + self.hot.queue_bytes.fetch_add(bytes, Ordering::Relaxed); + } + if flows != 0 { + self.hot.queue_flows.fetch_add(flows, Ordering::Relaxed); + } + } + + pub fn sched_transport_full_inc(&self) { + self.transport_full.increment(1); + } + + /// Phase 2 telemetry (all cached handles, no per-packet registry work). + pub fn frame_sent_inc(&self, segments: u64) { + self.frames.increment(1); + if segments > 1 { + self.segments.increment(segments); + } + } + + pub fn observe_sojourn(&self, d: Duration) { + self.sojourn.observe(d); + } + + pub fn reassembly_inc(&self, result: &'static str) { + counter!("tunnet_reassembly_total", "result" => result).increment(1); + } + + pub fn tun_syscall_inc(&self, op: &'static str) { + counter!("tunnet_tun_syscalls_total", "op" => op).increment(1); + } + + pub fn datagram_inc(&self, direction: &'static str) { + counter!("tunnet_datagrams_total", "direction" => direction).increment(1); + } + + pub fn pool_hit_miss(&self, hits: u64, misses: u64) { + if hits > 0 { + self.pool_hits.increment(hits); + } + if misses > 0 { + self.pool_miss.increment(misses); + } + } + + pub fn mtu_set(&self, mtu: u64) { + gauge!("tunnet_virtual_mtu_bytes").set(mtu as f64); } pub fn active_conns_inc(&self) { - gauge!("tunnet_active_connections").increment(1.0); + self.active_conns.increment(1.0); } pub fn active_conns_dec(&self) { - gauge!("tunnet_active_connections").decrement(1.0); + self.active_conns.decrement(1.0); } pub fn render(&self) -> String { diff --git a/crates/tunnet-agent/src/pump.rs b/crates/tunnet-agent/src/pump.rs new file mode 100644 index 00000000..170b79e7 --- /dev/null +++ b/crates/tunnet-agent/src/pump.rs @@ -0,0 +1,895 @@ +//! Per-peer outbound pump: FQ-CoDel → tunnel segmenter → Model A transport. +//! +//! Each peer's pump owns one loop over its [`PeerMembershipState`]: +//! +//! ```text +//! scheduler.next() → logical packet (or resume stashed cursor) +//! → single frame (header in headroom, from_owner, no copy) or +//! segments (incremental cursor, pooled staging per segment) +//! → transport.try_send_frame (Model A: space must fit the whole frame; +//! the frame is returned on failure, so stalls never consume bytes) +//! → TransportFull: stash cursor (segmented) or requeue (single), +//! adaptive backoff (notify + RTT/4, no fixed 5 ms, no spin) +//! → NoConnection: requeue whole packet, slow dial, wake on completion +//! → TooLarge: refresh MPS, restart with fresh id (path shrank) +//! ``` +//! +//! One logical packet emits at most [`MAX_SEGMENTS`] DATAGRAMs, so no packet +//! monopolizes the connection for an arbitrary burst (§7). Scheduler and +//! transport never see fragments — only logical packets and frames. + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use tunnet_common::packet::{ + FlowKey, LogicalPacket, MAX_LOGICAL_LEN, MAX_SEGMENTS, MIN_SEGMENT_PAYLOAD, PacketOwner, + PacketPool, SEGMENT_OVERHEAD, SINGLE_OVERHEAD, SegmentHeader, encode_segment_prefix, + encode_single_prefix, +}; +use tunnet_core::peers::{FastSendError, PeerMembershipState, PeerRegistry}; +use tunnet_core::{ConnPool, scheduler::Dequeue}; + +use crate::metrics::AgentMetrics; + +/// Ensure the peer's pump task is running; otherwise wake it. +pub fn ensure_pump( + fast: &Arc, + pool: ConnPool, + metrics: AgentMetrics, + bufs: Arc, + meter: tunnet_core::CloudRelayMeter, +) { + if !fast.pump_running.swap(true, Ordering::AcqRel) { + let ctx = PumpCtx { + fast: fast.clone(), + pool, + metrics, + bufs, + meter, + }; + tokio::spawn(async move { + run_peer_pump(ctx).await; + }); + } else { + fast.notify.notify_one(); + } +} + +struct PumpCtx { + fast: Arc, + pool: ConnPool, + metrics: AgentMetrics, + bufs: Arc, + meter: tunnet_core::CloudRelayMeter, +} + +/// Shared transmit context: one struct instead of eight parameters. +struct Tx<'a> { + fast: &'a Arc, + pool: &'a ConnPool, + metrics: &'a AgentMetrics, + bufs: &'a Arc, + meter: &'a tunnet_core::CloudRelayMeter, + peer: iroh::EndpointId, +} + +/// Mid-packet transmit cursor (§7): stashed only across TransportFull waits. +/// The logical owner is retained untouched; segments encode from borrows, so +/// resume never re-parses and never loses bytes. +/// +/// The cursor tracks the FULL segmentation geometry (plan + packet id + +/// next index), never just a count: any geometry change restarts the +/// logical packet from byte 0 with a fresh id, so old offsets are never +/// reused with a new MPS (§2.1-1). +struct PartialPacket { + packet: Option, + flow: FlowKey, + next_index: usize, + frame_id: u32, + /// Geometry the in-flight segments conform to (count + seg_cap, or + /// Single for a fresh single-frame cursor). + plan: SegmentPlan, + total: usize, + /// Wire bytes sent under the current frame id (preserved across + /// TransportFull resume; reset on restart). Accounted exactly once at + /// completion, so segmented traffic is never double-charged (§2.1-2). + wire_bytes: u64, + /// Network bound at dequeue from the route's membership. Every frame + /// of this packet carries it (§2.2-1). + net: uuid::Uuid, +} + +/// Outcome of re-planning a cursor against current MPS. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Replan { + /// Geometry changed (or first sizing): restart from byte 0, fresh id. + Restarted, + /// Geometry identical: retry the current segment in place (transient + /// TooLarge, no bytes wasted on a redundant restart). + Retry, + /// The path cannot carry this packet at all: drop it. + Impossible, +} + +impl PartialPacket { + fn new(packet: LogicalPacket, flow: FlowKey, fast: &PeerMembershipState) -> Self { + let total = packet.len(); + let plan = plan_for_mps(total, fast.transport.mps.load(Ordering::Relaxed)); + Self { + packet: Some(packet), + flow, + next_index: 0, + frame_id: fast.transport.next_frame_id.fetch_add(1, Ordering::Relaxed), + plan, + total, + wire_bytes: 0, + // Bound at dequeue: every frame of this packet carries this + // network, resolved from the route (§2.2-1). + net: fast.identity.read().network_id, + } + } + + /// Adopt a geometry wholesale: fresh id, offset reset, wire accumulator + /// reset. Old offsets are never reused after this point. + fn adopt(&mut self, plan: SegmentPlan, fast: &PeerMembershipState) { + self.plan = plan; + self.next_index = 0; + self.wire_bytes = 0; + self.frame_id = fast.transport.next_frame_id.fetch_add(1, Ordering::Relaxed); + } + + /// Re-plan against current MPS after a TooLarge (MPS already refreshed + /// by the caller). Compares the COMPLETE geometry — count AND segment + /// capacity AND single/segmented shape — not just the count: an MPS + /// change that keeps the segment count but alters seg_cap still + /// restarts with a fresh id. + fn replan(&mut self, fast: &PeerMembershipState) -> Replan { + let mps = fast.transport.mps.load(Ordering::Relaxed); + match plan_for_mps(self.total, mps) { + SegmentPlan::Impossible => Replan::Impossible, + plan if plan == self.plan => Replan::Retry, + plan => { + self.adopt(plan, fast); + Replan::Restarted + } + } + } +} + +async fn run_peer_pump(ctx: PumpCtx) { + let PumpCtx { + fast, + pool, + metrics, + bufs, + meter, + } = ctx; + let peer = fast.identity.read().endpoint; + let mut partial: Option = None; + // Unstarted remainder of the current DRR burst (gauges debited at + // dequeue; requeued with re-credit on stall, drained packet by packet). + let mut pending: std::collections::VecDeque = std::collections::VecDeque::new(); + // Ownership epoch at pump start; teardown advances it so this task + // drains and exits instead of parking on a dead generation. + let epoch0 = fast.epoch.load(Ordering::Relaxed); + // Last-seen queue levels for gauge-delta reconciliation (global gauges + // are sums; per-pump deltas keep them correct without overwrites). + let mut last_levels: (i64, i64, i64) = (0, 0, 0); + let reconcile = + |fast: &Arc, metrics: &AgentMetrics, last: &mut (i64, i64, i64)| { + let (p, b, f) = fast.scheduler.lock().levels(); + let (p, b, f) = (p as i64, b as i64, f as i64); + metrics.queue_add(p - last.0, b - last.1, f - last.2); + *last = (p, b, f); + }; + + loop { + // Ownership change (teardown/drop): shed queued packets, zero the + // gauges, and exit. The scheduler contents belong to the old + // generation and must not cross into a new TUN generation. + // (Enqueue/dequeue deltas were emitted as they happened, so the + // peer's live contribution equals its current levels: negate them.) + if fast.epoch.load(Ordering::Relaxed) != epoch0 { + // Return in-flight packets (stashed cursor + burst remainder) + // to the scheduler first, so the clear below reconciles every + // outstanding gauge debit in one place. + if let Some(cur) = partial.take() + && let Some(packet) = cur.packet + { + let flow = packet.flow; + fast.scheduler.lock().requeue_head(flow, packet); + } + while let Some(cur) = pending.pop_front() { + if let Some(packet) = cur.packet { + let flow = packet.flow; + fast.scheduler.lock().requeue_head(flow, packet); + } + } + let (p, b, f) = fast.scheduler.lock().clear(); + metrics.queue_add(-(p as i64), -(b as i64), -(f as i64)); + fast.pump_running.store(false, Ordering::Release); + return; + } + // 1) Resume a stashed cursor, else dequeue the next burst. A burst + // is one DRR service opportunity (all packets the visited flow + // could afford); the pump transmits every packet in order. + if partial.is_none() && pending.is_empty() { + let dequeued = { + let mut sched = fast.scheduler.lock(); + sched.next(Instant::now()) + }; + match dequeued { + Dequeue::Empty => { + reconcile(&fast, &metrics, &mut last_levels); + // Idle: wait for work or exit after a quiet period. + tokio::select! { + _ = fast.notify.notified() => continue, + _ = tokio::time::sleep(Duration::from_millis(50)) => { + if fast.scheduler.lock().is_empty() { + fast.pump_running.store(false, Ordering::Release); + if !fast.scheduler.lock().is_empty() + && !fast.pump_running.swap(true, Ordering::AcqRel) + { + continue; + } + if fast.scheduler.lock().is_empty() { + // Zero this peer's gauge contribution from + // live levels (deltas were emitted live). + let (p, b, f) = fast.scheduler.lock().levels(); + metrics.queue_add(-(p as i64), -(b as i64), -(f as i64)); + return; + } + } + continue; + } + } + } + Dequeue::Send(burst) => { + for (packet, sample) in burst.packets { + metrics.observe_sojourn(sample.sojourn); + let flow = packet.flow; + metrics.queue_add(-1, -(packet.len() as i64), 0); + pending.push_back(PartialPacket::new(*packet, flow, &fast)); + } + } + } + } + + // Periodic MPS refresh covers silent path changes (plus event-driven + // refresh in the pool's path watcher and TooLarge recovery below). + if fast + .transport + .sends_since_mps_check + .fetch_add(1, Ordering::Relaxed) + >= 512 + { + fast.transport + .sends_since_mps_check + .store(0, Ordering::Relaxed); + fast.refresh_mps(); + } + + // 2) Transmit one cursor to completion, stall, or drop. A stashed + // cursor resumes next iteration; the burst remainder waits in + // `pending` (gauges already debited at dequeue; requeues below + // re-credit). + let mut cur = partial + .take() + .or_else(|| pending.pop_front()) + .expect("cursor"); + let tx = Tx { + fast: &fast, + pool: &pool, + metrics: &metrics, + bufs: &bufs, + meter: &meter, + peer, + }; + match transmit_cursor(&tx, &mut cur).await { + TransmitOut::Done { logical, frames } => { + metrics.frame_sent_inc(frames); + metrics.packets_inc("out"); + metrics.bytes_add("out", logical as u64); + reconcile(&fast, &metrics, &mut last_levels); + } + TransmitOut::Stash => { + partial = Some(cur); + // Return the unstarted remainder to the scheduler head in + // order (gauges re-credited per packet). + requeue_pending(&fast, &metrics, &mut pending); + metrics.sched_transport_full_inc(); + // Adaptive backoff (§0.7): wake on new work or timeout. + tokio::select! { + _ = fast.notify.notified() => {} + _ = tokio::time::sleep(PeerRegistry::backoff_for(&fast.transport)) => {} + } + } + TransmitOut::Wait => { + // Requeue the working packet if transmit left it intact + // (the NoConnection arm usually requeues itself; this + // covers the re-parse-failure path losslessly), then the + // unstarted remainder. Dial was kicked inside. + if let Some(packet) = cur.packet.take() { + let flow = packet.flow; + let len = packet.len() as i64; + fast.scheduler.lock().requeue_head(flow, packet); + metrics.queue_add(1, len, 0); + } + requeue_pending(&fast, &metrics, &mut pending); + tokio::select! { + _ = fast.notify.notified() => {} + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + } + TransmitOut::Dropped(reason) => { + metrics.dropped_inc(reason); + reconcile(&fast, &metrics, &mut last_levels); + } + } + // Report dequeue-side drops (CoDel/emergency inside next()): they + // have no enqueue decision site, so the pump drains them here. + // Deltas partition across lock holders; the sum stays exact. + let deltas = fast.scheduler.lock().drain_drops(); + metrics.sched_drops_add(deltas.codel, deltas.emergency); + } +} + +/// Return unstarted burst packets to the scheduler head, preserving order. +/// The stashed/working cursor is handled separately by the caller. +fn requeue_pending( + fast: &Arc, + metrics: &AgentMetrics, + pending: &mut std::collections::VecDeque, +) { + if pending.is_empty() { + return; + } + let mut sched = fast.scheduler.lock(); + while let Some(cur) = pending.pop_back() { + if let Some(packet) = cur.packet { + let flow = packet.flow; + let len = packet.len() as i64; + sched.requeue_head(flow, packet); + metrics.queue_add(1, len, 0); + } + } +} + +enum TransmitOut { + /// Logical packet fully transmitted (logical bytes, segment frames). + Done { logical: usize, frames: u64 }, + /// Transport full: caller stashes `cur` and backs off. + Stash, + /// Stalled but handled inside (requeued, dial kicked): caller waits. + Wait, + /// Dropped with reason. + Dropped(&'static str), +} + +/// Segmentation plan for a logical packet at one MPS snapshot. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SegmentPlan { + /// Fits in one DATAGRAM (plus 1-byte prefix). + Single, + /// Split into `count` segments of at most `seg_cap` payload bytes. + Segmented { count: usize, seg_cap: usize }, + /// Degenerate path (no useful segment fits). + Impossible, +} + +/// Pure sizing decision (§3, §7): single when it fits, else uniform +/// segments; impossible when the path cannot carry even minimal segments. +fn plan_for_mps(total: usize, mps: usize) -> SegmentPlan { + if total == 0 || total > MAX_LOGICAL_LEN { + return SegmentPlan::Impossible; + } + if total + SINGLE_OVERHEAD <= mps { + return SegmentPlan::Single; + } + match usable_seg_cap(mps) { + Some(seg_cap) => { + let count = total.div_ceil(seg_cap).max(2); + if count > MAX_SEGMENTS { + SegmentPlan::Impossible + } else { + SegmentPlan::Segmented { count, seg_cap } + } + } + None => SegmentPlan::Impossible, + } +} + +/// Transmit one cursor to completion, stall, or drop. At most MAX_SEGMENTS +/// DATAGRAMs per packet — bounded by construction. +/// +/// Fresh cursors adopt the CURRENT geometry wholesale; resumed cursors +/// (next_index > 0) continue their stored geometry even if the path has +/// changed — only a TooLarge re-plans them. Either way old offsets are +/// never mixed with a new MPS: adoption restarts from byte 0. +async fn transmit_cursor(tx: &Tx<'_>, cur: &mut PartialPacket) -> TransmitOut { + if cur.next_index == 0 { + let mps = tx.fast.transport.mps.load(Ordering::Relaxed); + match plan_for_mps(cur.total, mps) { + SegmentPlan::Single => return transmit_single(tx, cur).await, + plan @ SegmentPlan::Segmented { .. } => { + cur.adopt(plan, tx.fast); + return transmit_segmented(tx, cur).await; + } + SegmentPlan::Impossible => { + // Degenerate path: refresh once, then give up if useless. + tx.fast.refresh_mps(); + let mps2 = tx.fast.transport.mps.load(Ordering::Relaxed); + match plan_for_mps(cur.total, mps2) { + SegmentPlan::Single => return transmit_single(tx, cur).await, + plan @ SegmentPlan::Segmented { .. } => { + cur.adopt(plan, tx.fast); + return transmit_segmented(tx, cur).await; + } + SegmentPlan::Impossible => { + return TransmitOut::Dropped("datagram_too_large"); + } + } + } + } + } + debug_assert!( + matches!(cur.plan, SegmentPlan::Segmented { .. }), + "resumed cursors are always segmented (singles never stash)" + ); + transmit_segmented(tx, cur).await +} + +/// Encode a logical packet as one frame: pooled owners prepend the header +/// in headroom (no copy); shared owners stage through a pooled buffer. +async fn transmit_single(tx: &Tx<'_>, cur: &mut PartialPacket) -> TransmitOut { + let Tx { + fast, + pool, + metrics, + bufs, + meter, + peer, + } = tx; + let packet = cur.packet.take().expect("cursor holds packet"); + let total = packet.len(); + let owner = packet.owner; + let frame = match owner { + PacketOwner::Pooled(mut buf) => match buf.header_slot(SINGLE_OVERHEAD) { + Some(slot) => { + encode_single_prefix(slot, cur.net); + Bytes::from_owner(buf) + } + None => { + // No headroom (should not happen): stage a copy. + let src = buf.as_ref().to_vec(); + drop(buf); + stage_single(bufs, cur.net, &src) + } + }, + PacketOwner::Shared(s) => stage_single(bufs, cur.net, &s), + }; + let wire = frame.len(); + match fast.transport.try_send_frame(frame) { + Ok(()) => { + account_sent(fast, cur.flow, total, wire); + if fast.transport.relay.load(Ordering::Relaxed) { + meter.record(wire as u64); + } + TransmitOut::Done { + logical: total, + frames: 1, + } + } + Err((FastSendError::TransportFull, frame)) => { + // Recover the frame, strip the prefix, requeue losslessly. + if let Some(rebuilt) = LogicalPacket::from_shared(strip_single_prefix(frame)) { + let flow = rebuilt.flow; + let len = rebuilt.len() as i64; + fast.scheduler.lock().requeue_head(flow, rebuilt); + metrics.queue_add(1, len, 0); + } else { + metrics.dropped_inc("datagram_too_large"); + } + TransmitOut::Wait + } + Err((FastSendError::NoConnection | FastSendError::Closed, frame)) => { + if let Some(rebuilt) = LogicalPacket::from_shared(strip_single_prefix(frame)) { + let flow = rebuilt.flow; + let len = rebuilt.len() as i64; + fast.scheduler.lock().requeue_head(flow, rebuilt); + metrics.queue_add(1, len, 0); + } else { + // Our own encoding failed to re-parse: count it and still + // kick the dial, since connectivity is suspect anyway. + metrics.dropped_inc("no_connection"); + } + kick_dial(pool, *peer, fast); + TransmitOut::Wait + } + Err((FastSendError::TooLarge, frame)) => { + // Stale MPS: refresh, rebuild the logical packet from the + // recovered frame, and re-route through the cursor (which + // adopts the current geometry wholesale). Boxed: rare path + // (stale MPS on a single frame) that would otherwise close a + // single↔segmented async cycle. + fast.refresh_mps(); + let Some(rebuilt) = LogicalPacket::from_shared(strip_single_prefix(frame)) else { + return TransmitOut::Dropped("datagram_too_large"); + }; + cur.packet = Some(rebuilt); + cur.next_index = 0; + cur.total = cur.packet.as_ref().map(|p| p.len()).unwrap_or(0); + cur.wire_bytes = 0; + return Box::pin(transmit_cursor(tx, cur)).await; + } + } +} + +fn kick_dial(pool: &ConnPool, peer: iroh::EndpointId, fast: &Arc) { + let pool2 = pool.clone(); + let fast2 = fast.clone(); + tokio::spawn(async move { + let _ = pool2.get(peer).await; + fast2.notify.notify_one(); + }); +} + +fn stage_single(pool: &Arc, net: uuid::Uuid, payload: &[u8]) -> Bytes { + let mut buf = pool.acquire(payload.len() + SINGLE_OVERHEAD); + let region = buf.recv_region(payload.len() + SINGLE_OVERHEAD); + encode_single_prefix(&mut region[..SINGLE_OVERHEAD], net); + region[SINGLE_OVERHEAD..].copy_from_slice(payload); + buf.set_len(payload.len() + SINGLE_OVERHEAD); + Bytes::from_owner(buf) +} + +/// Remove the single-frame header (kind + network), returning the payload. +fn strip_single_prefix(frame: Bytes) -> Bytes { + if frame.first() == Some(&tunnet_common::packet::KIND_SINGLE) && frame.len() > SINGLE_OVERHEAD { + frame.slice(SINGLE_OVERHEAD..) + } else { + frame + } +} + +/// Transmit the cursor's remainder segment by segment, encoding +/// incrementally from the retained logical owner under the cursor's STORED +/// geometry. TransportFull stashes (wire accumulator preserved for resume); +/// TooLarge re-plans against fresh MPS (full geometry compare → restart or +/// in-place retry); completion accounts the whole logical packet exactly +/// once with total wire bytes. +async fn transmit_segmented(tx: &Tx<'_>, cur: &mut PartialPacket) -> TransmitOut { + transmit_segmented_budgeted(tx, cur, 0).await +} + +/// Continue a restarted cursor under its newly adopted geometry. +/// Split out so the TooLarge arm stays readable; the restart budget is +/// threaded through to bound flapping paths. +async fn transmit_segmented_restarted( + tx: &Tx<'_>, + cur: &mut PartialPacket, + restarts: u8, +) -> TransmitOut { + transmit_segmented_budgeted(tx, cur, restarts).await +} + +async fn transmit_segmented_budgeted( + tx: &Tx<'_>, + cur: &mut PartialPacket, + mut restarts: u8, +) -> TransmitOut { + let Tx { + fast, + pool, + metrics, + bufs, + meter, + peer, + } = tx; + let (count, seg_cap) = match cur.plan { + SegmentPlan::Segmented { count, seg_cap } => (count, seg_cap), + // Fresh cursors never arrive here (transmit_cursor routes them); + // a TooLarge replan can adopt Single — route out, boxed to break + // the async cycle. + SegmentPlan::Single => return Box::pin(transmit_single(tx, cur)).await, + SegmentPlan::Impossible => return TransmitOut::Dropped("datagram_too_large"), + }; + let mut frames = 0u64; + // Bounded retries on flapping paths (budget shared across restarts). + loop { + if cur.next_index >= count { + // Completion: account the whole logical packet ONCE with total + // wire bytes (never per segment — no double charge). + account_sent(fast, cur.flow, cur.total, cur.wire_bytes as usize); + return TransmitOut::Done { + logical: cur.total, + frames, + }; + } + let i = cur.next_index; + let off = i * seg_cap; + let end = (off + seg_cap).min(cur.total); + if off >= cur.total || end <= off { + return TransmitOut::Dropped("datagram_too_large"); + } + // Encode from a borrow (owner retained for resume/retry). + let Some(packet) = cur.packet.as_ref() else { + return TransmitOut::Dropped("datagram_too_large"); + }; + let payload = &packet.owner.as_bytes()[off..end]; + let mut buf = bufs.acquire(payload.len() + SEGMENT_OVERHEAD); + { + let region = buf.recv_region(payload.len() + SEGMENT_OVERHEAD); + encode_segment_prefix( + &mut region[..SEGMENT_OVERHEAD], + cur.net, + SegmentHeader { + id: cur.frame_id, + index: i as u16, + count: count as u16, + total: cur.total as u16, + }, + ); + region[SEGMENT_OVERHEAD..].copy_from_slice(payload); + buf.set_len(payload.len() + SEGMENT_OVERHEAD); + } + let frame = Bytes::from_owner(buf); + let wire = frame.len(); + match fast.transport.try_send_frame(frame) { + Ok(()) => { + frames += 1; + // Accumulate; accounted once at completion. Preserved + // across TransportFull resume via the stashed cursor. + cur.wire_bytes += wire as u64; + if fast.transport.relay.load(Ordering::Relaxed) { + meter.record(wire as u64); + } + cur.next_index += 1; + } + Err((FastSendError::TransportFull, _)) => { + // Owner intact, accumulator intact: stash the cursor, + // resume after backoff. + return TransmitOut::Stash; + } + Err((FastSendError::NoConnection | FastSendError::Closed, _)) => { + // Owner intact: requeue the whole logical packet (fresh id + // on retry; orphaned prefix expires), then dial. + if let Some(packet) = cur.packet.take() { + let flow = packet.flow; + let len = packet.len() as i64; + fast.scheduler.lock().requeue_head(flow, packet); + metrics.queue_add(1, len, 0); + } else { + metrics.dropped_inc("no_connection"); + } + kick_dial(pool, *peer, fast); + return TransmitOut::Wait; + } + Err((FastSendError::TooLarge, _)) => { + // Stale MPS mid-packet: refresh and re-plan with a FULL + // geometry compare. Changed geometry (count, seg_cap, or + // shape) restarts from byte 0 with a fresh id — old + // offsets are never reused with the new MPS. Identical + // geometry retries the segment in place (transient). + fast.refresh_mps(); + restarts += 1; + if restarts > 2 { + return TransmitOut::Dropped("datagram_too_large"); + } + match cur.replan(fast) { + Replan::Restarted => { + // Adopted the new geometry wholesale (fresh id, + // offset 0, wire accumulator reset); continue under + // it with the remaining restart budget. Boxed: the + // restart cycle would otherwise recurse unboundedly. + return Box::pin(transmit_segmented_restarted(tx, cur, restarts)).await; + } + Replan::Retry => { + // Same geometry: retry this segment in place. + } + Replan::Impossible => { + return TransmitOut::Dropped("datagram_too_large"); + } + } + } + } + } +} + +fn usable_seg_cap(mps: usize) -> Option { + let cap = mps.checked_sub(SEGMENT_OVERHEAD)?; + (cap >= MIN_SEGMENT_PAYLOAD).then_some(cap) +} + +fn account_sent( + fast: &Arc, + flow: FlowKey, + logical_len: usize, + wire_len: usize, +) { + // Debit DRR by logical bytes; wire overhead leans future rounds. + fast.scheduler + .lock() + .account_sent(flow, logical_len, wire_len); +} + +#[cfg(test)] +mod tests { + use super::*; + use tunnet_core::peers::{PeerIdentity, PeerRegistry}; + + fn test_fast() -> (PeerRegistry, Arc) { + let reg = PeerRegistry::new(); + let ep = iroh::SecretKey::generate().public(); + let fast = reg.ensure(Arc::new(PeerIdentity { + endpoint: ep, + endpoint_hex: format!("{ep}"), + hostname: "peer".into(), + ip: std::net::Ipv4Addr::new(10, 0, 0, 2), + tags: vec![], + network_id: uuid::Uuid::nil(), + network_name: "net".into(), + })); + (reg, fast) + } + + fn test_packet(size: usize) -> (LogicalPacket, FlowKey) { + let pool = PacketPool::new(8); + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 443); + let mut raw = Vec::new(); + b.write(&mut raw, &vec![0xABu8; size.saturating_sub(28)]) + .unwrap(); + let mut buf = pool.acquire(raw.len()); + buf.recv_region(raw.len()).copy_from_slice(&raw); + let pkt = LogicalPacket::from_pooled(buf, raw.len()).unwrap(); + let flow = pkt.flow; + (pkt, flow) + } + + #[test] + fn replan_restarts_on_segcap_change_with_same_count() { + // §2.1-1: 2800 bytes needs 3 segments at both MPS 1350 (cap 1323) + // and MPS 1400 (cap 1373) — same count, different geometry. The + // cursor must still restart with a fresh id, never reusing old + // offsets with the new MPS. + let (_reg, fast) = test_fast(); + fast.transport.mps.store(1350, Ordering::Relaxed); + let (pkt, flow) = test_packet(2800); + let mut cur = PartialPacket::new(pkt, flow, &fast); + assert!(matches!( + cur.plan, + SegmentPlan::Segmented { + count: 3, + seg_cap: 1323 + } + )); + // Simulate two sent segments, then a path change (no conn needed: + // replan only reads MPS and mints ids). + cur.next_index = 2; + cur.wire_bytes = 2700; + let old_id = cur.frame_id; + fast.transport.mps.store(1400, Ordering::Relaxed); + assert_eq!(cur.replan(&fast), Replan::Restarted); + assert!(matches!( + cur.plan, + SegmentPlan::Segmented { + count: 3, + seg_cap: 1373 + } + )); + assert_eq!(cur.next_index, 0, "restart from byte 0"); + assert_eq!(cur.wire_bytes, 0, "fresh accounting unit"); + assert_ne!(cur.frame_id, old_id, "fresh packet id"); + } + + #[test] + fn replan_retries_in_place_on_identical_geometry() { + // Spurious TooLarge with unchanged MPS: retry the segment in place + // (same id, same offset), don't waste a redundant restart. + let (_reg, fast) = test_fast(); + fast.transport.mps.store(1350, Ordering::Relaxed); + let (pkt, flow) = test_packet(2800); + let mut cur = PartialPacket::new(pkt, flow, &fast); + cur.next_index = 1; + let old_id = cur.frame_id; + assert_eq!(cur.replan(&fast), Replan::Retry); + assert_eq!(cur.next_index, 1); + assert_eq!(cur.frame_id, old_id); + } + + #[test] + fn replan_handles_shape_transitions() { + let (_reg, fast) = test_fast(); + // Segmented → single (path grew). + fast.transport.mps.store(1350, Ordering::Relaxed); + let (pkt, flow) = test_packet(2800); + let mut cur = PartialPacket::new(pkt, flow, &fast); + assert!(matches!(cur.plan, SegmentPlan::Segmented { .. })); + fast.transport.mps.store(9000, Ordering::Relaxed); + assert_eq!(cur.replan(&fast), Replan::Restarted); + assert_eq!(cur.plan, SegmentPlan::Single); + assert_eq!(cur.next_index, 0); + // Single → segmented (path shrank). + fast.transport.mps.store(9000, Ordering::Relaxed); + let (pkt, flow) = test_packet(1200); + let mut cur = PartialPacket::new(pkt, flow, &fast); + assert_eq!(cur.plan, SegmentPlan::Single); + fast.transport.mps.store(1100, Ordering::Relaxed); + assert_eq!(cur.replan(&fast), Replan::Restarted); + assert!(matches!(cur.plan, SegmentPlan::Segmented { .. })); + // Degenerate path: impossible, caller drops. + fast.transport.mps.store(64, Ordering::Relaxed); + assert_eq!(cur.replan(&fast), Replan::Impossible); + } + + #[test] + fn plan_single_segmented_impossible() { + // Exact fit → single (single overhead is 17: net-bound frames). + assert_eq!(plan_for_mps(1184, 1201), SegmentPlan::Single); + // One byte over → segmented. + assert!(matches!( + plan_for_mps(1185, 1201), + SegmentPlan::Segmented { .. } + )); + // 2800 logical at 1350 MPS → 3 segments of ≤1323. + match plan_for_mps(2800, 1350) { + SegmentPlan::Segmented { count, seg_cap } => { + assert_eq!(count, 3); + assert_eq!(seg_cap, 1350 - SEGMENT_OVERHEAD); + } + other => panic!("expected segmented, got {other:?}"), + } + assert_eq!(plan_for_mps(0, 1350), SegmentPlan::Impossible); + assert_eq!(plan_for_mps(9001, 1500), SegmentPlan::Impossible); + assert_eq!(plan_for_mps(100, 10), SegmentPlan::Impossible); + } + + #[test] + fn plan_path_shrink_grows_count() { + // §3: the same logical packet needs more segments on a smaller path; + // the pump restarts it with a fresh id (tested here via the planner). + let total = 2800; + let before = plan_for_mps(total, 1350); + let after = plan_for_mps(total, 1200); + match (before, after) { + ( + SegmentPlan::Segmented { count: c1, .. }, + SegmentPlan::Segmented { count: c2, .. }, + ) => assert!(c2 >= c1, "shrink must not reduce segments"), + other => panic!("expected segmented plans, got {other:?}"), + } + // 9000 at tiny MPS exceeds the segment cap (20 > 16). + assert_eq!(plan_for_mps(9000, 500), SegmentPlan::Impossible); + } + + #[test] + fn single_encode_round_trip() { + use tunnet_common::packet::SINGLE_OVERHEAD; + let pool = PacketPool::new(8); + let net = uuid::Uuid::from_u128(0x0c); + let payload = vec![0xABu8; 200]; + let frame = stage_single(&pool, net, &payload); + assert_eq!(frame.len(), SINGLE_OVERHEAD + 200); + assert_eq!(frame[0], tunnet_common::packet::KIND_SINGLE); + assert_eq!(&frame[1..SINGLE_OVERHEAD], net.as_bytes()); + let back = strip_single_prefix(frame); + assert_eq!(&back[..], &payload[..]); + } + + #[test] + fn cursor_binds_network_at_dequeue() { + // §2.2-1: every frame of a packet carries the route's membership + // network, captured at dequeue. + let (_reg, fast) = test_fast(); + assert_eq!( + fast.identity.read().network_id, + uuid::Uuid::nil(), + "test membership network" + ); + let (pkt, flow) = test_packet(200); + let cur = PartialPacket::new(pkt, flow, &fast); + assert_eq!(cur.net, uuid::Uuid::nil()); + } +} diff --git a/crates/tunnet-agent/src/qos.rs b/crates/tunnet-agent/src/qos.rs deleted file mode 100644 index 0aa54d7a..00000000 --- a/crates/tunnet-agent/src/qos.rs +++ /dev/null @@ -1,452 +0,0 @@ -//! Per-peer Byte-DRR outbound scheduler for mesh IP datagrams. -//! -//! One canonical QUIC connection per peer; latency / normal / bulk queues feed a -//! single sender. Deficit is counted in **bytes**, not packets. - -use std::collections::VecDeque; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - -use bytes::Bytes; -use dashmap::DashMap; -use iroh::EndpointId; -use parking_lot::Mutex; -use tokio::sync::Notify; -use tunnet_core::ConnPool; - -use crate::metrics::AgentMetrics; -use tunnet_common::packet::{Packet, TcpFlags, Transport}; - -const CAP_LATENCY: usize = 64; -const CAP_NORMAL: usize = 256; -const CAP_BULK: usize = 512; - -/// Packet quantum is one configured TUN MTU. -fn quanta(mtu: usize) -> (usize, usize, usize) { - (8 * mtu, 4 * mtu, mtu) -} - -/// Packets at or above this fraction of MTU are treated as bulk. -fn bulk_threshold(mtu: usize) -> usize { - mtu.saturating_sub(80).max(512) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Class { - Latency, - Normal, - Bulk, -} - -impl Class { - fn quantum(self, mtu: usize) -> usize { - let (lat, norm, bulk) = quanta(mtu); - match self { - Self::Latency => lat, - Self::Normal => norm, - Self::Bulk => bulk, - } - } -} - -/// Classify using validated parser metadata (not backing-buffer length). -pub fn classify(packet: &Packet<'_>, mtu: u16) -> Class { - let mtu = mtu.max(576) as usize; - let bulk = bulk_threshold(mtu); - let size = packet.wire_len; - match packet.transport { - Transport::Icmpv4 { .. } | Transport::Icmpv6 { .. } => Class::Latency, - Transport::Tcp { - flags, - payload_len, - src_port, - dst_port, - .. - } => { - if flags.0 & (TcpFlags::SYN | TcpFlags::FIN | TcpFlags::RST) != 0 { - return Class::Latency; - } - if flags.ack() && payload_len == 0 { - return Class::Latency; - } - if src_port == 53 || dst_port == 53 { - return Class::Latency; - } - if size >= bulk { - return Class::Bulk; - } - Class::Normal - } - Transport::Udp { - src_port, dst_port, .. - } => { - if src_port == 53 || dst_port == 53 { - return Class::Latency; - } - if size >= bulk { - Class::Bulk - } else { - Class::Normal - } - } - Transport::LaterFragment { .. } | Transport::Other { .. } => { - if size >= bulk { - Class::Bulk - } else { - Class::Normal - } - } - } -} - -struct PeerState { - latency: Mutex>, - normal: Mutex>, - bulk: Mutex>, - notify: Notify, - running: AtomicBool, -} - -impl PeerState { - fn new() -> Self { - Self { - latency: Mutex::new(VecDeque::new()), - normal: Mutex::new(VecDeque::new()), - bulk: Mutex::new(VecDeque::new()), - notify: Notify::new(), - running: AtomicBool::new(false), - } - } - - fn queue(&self, class: Class) -> &Mutex> { - match class { - Class::Latency => &self.latency, - Class::Normal => &self.normal, - Class::Bulk => &self.bulk, - } - } - - fn cap(class: Class) -> usize { - match class { - Class::Latency => CAP_LATENCY, - Class::Normal => CAP_NORMAL, - Class::Bulk => CAP_BULK, - } - } - - fn is_empty(&self) -> bool { - self.latency.lock().is_empty() - && self.normal.lock().is_empty() - && self.bulk.lock().is_empty() - } - - /// Enqueue with drop policy: prefer dropping bulk, then normal, before latency. - fn try_enqueue(&self, class: Class, packet: Bytes) -> Result<(), ()> { - { - let mut q = self.queue(class).lock(); - if q.len() < Self::cap(class) { - q.push_back(packet); - return Ok(()); - } - } - // Target class full: try to free bulk first, then normal. - if class != Class::Bulk { - let mut bulk = self.bulk.lock(); - if !bulk.is_empty() { - bulk.pop_front(); - drop(bulk); - let mut q = self.queue(class).lock(); - if q.len() < Self::cap(class) { - q.push_back(packet); - return Ok(()); - } - } - } - if class == Class::Latency { - let mut normal = self.normal.lock(); - if !normal.is_empty() { - normal.pop_front(); - drop(normal); - let mut q = self.latency.lock(); - if q.len() < CAP_LATENCY { - q.push_back(packet); - return Ok(()); - } - } - } - Err(()) - } -} - -/// Fan-out TUN packets to per-peer Byte-DRR senders. -#[derive(Clone)] -pub struct OutboundScheduler { - peers: Arc>>, - pool: ConnPool, - metrics: AgentMetrics, - mtu: u16, -} - -impl OutboundScheduler { - pub fn new(pool: ConnPool, metrics: AgentMetrics, mtu: u16) -> Self { - Self { - peers: Arc::new(DashMap::new()), - pool, - metrics, - mtu: mtu.max(576), - } - } - - pub fn enqueue(&self, peer: EndpointId, class: Class, packet: Bytes) { - let state = self - .peers - .entry(peer) - .or_insert_with(|| Arc::new(PeerState::new())) - .clone(); - - if state.try_enqueue(class, packet).is_err() { - self.metrics.dropped_inc(match class { - Class::Latency => "qos_latency_full", - Class::Normal => "qos_normal_full", - Class::Bulk => "qos_bulk_full", - }); - return; - } - - if !state.running.swap(true, Ordering::AcqRel) { - let pool = self.pool.clone(); - let metrics = self.metrics.clone(); - let peers = self.peers.clone(); - let mtu = self.mtu; - tokio::spawn(async move { - run_peer_sender(peer, state, pool, metrics, mtu).await; - peers.remove(&peer); - }); - } else { - state.notify.notify_one(); - } - } -} - -async fn run_peer_sender( - peer: EndpointId, - state: Arc, - pool: ConnPool, - metrics: AgentMetrics, - mtu: u16, -) { - let mut deficit = [0usize; 3]; // latency, normal, bulk - loop { - let mut sent_any = false; - for (idx, class) in [Class::Latency, Class::Normal, Class::Bulk] - .into_iter() - .enumerate() - { - { - let q = state.queue(class).lock(); - if q.is_empty() { - deficit[idx] = 0; - continue; - } - } - deficit[idx] = deficit[idx].saturating_add(class.quantum(mtu as usize)); - loop { - let packet = { - let mut q = state.queue(class).lock(); - let Some(head) = q.front() else { break }; - if head.len() > deficit[idx] { - break; - } - q.pop_front().expect("front checked") - }; - let n = packet.len(); - deficit[idx] -= n; - sent_any = true; - match pool.send_or_buffer(peer, packet).await { - Ok(()) => { - metrics.packets_inc("out"); - metrics.bytes_add("out", n as u64); - pool.record_bytes_out(peer, n as u64); - } - Err(e) => { - let msg = e.to_string(); - if msg.contains("datagram_too_large") { - metrics.dropped_inc("datagram_too_large"); - } else { - tracing::debug!(%peer, ?e, "send/buffer failed"); - metrics.dropped_inc("send_failed"); - } - } - } - } - } - - if state.is_empty() { - // Wait for more work or exit if idle after notify timeout. - tokio::select! { - _ = state.notify.notified() => {} - _ = tokio::time::sleep(std::time::Duration::from_millis(50)) => { - if state.is_empty() { - state.running.store(false, Ordering::Release); - // Race: packet enqueued after is_empty check. - if !state.is_empty() - && !state.running.swap(true, Ordering::AcqRel) - { - continue; - } - if state.is_empty() { - return; - } - } - } - } - if !sent_any && state.is_empty() { - // Spurious wake with empty queues. - } - } - } -} - -/// Run one Byte-DRR round over in-memory queues (unit tests / saturation checks). -#[cfg(test)] -fn drr_round_drain( - latency: &mut VecDeque, - normal: &mut VecDeque, - bulk: &mut VecDeque, - deficit: &mut [usize; 3], - out: &mut Vec<(Class, usize)>, -) { - for (idx, (class, q)) in [ - (Class::Latency, &mut *latency), - (Class::Normal, &mut *normal), - (Class::Bulk, &mut *bulk), - ] - .into_iter() - .enumerate() - { - if q.is_empty() { - deficit[idx] = 0; - continue; - } - deficit[idx] = deficit[idx].saturating_add(class.quantum(1280)); - while let Some(head) = q.front() { - if head.len() > deficit[idx] { - break; - } - let pkt = q.pop_front().expect("front"); - deficit[idx] -= pkt.len(); - out.push((class, pkt.len())); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn pkt(n: usize) -> Bytes { - Bytes::from(vec![0u8; n]) - } - - #[test] - fn byte_drr_prefers_latency_but_bulk_progresses() { - let mut latency = VecDeque::from([pkt(100), pkt(100), pkt(100)]); - let mut normal: VecDeque = VecDeque::new(); - let mut bulk = VecDeque::from([pkt(1200), pkt(1200), pkt(1200)]); - let mut deficit = [0usize; 3]; - let mut out = Vec::new(); - - drr_round_drain(&mut latency, &mut normal, &mut bulk, &mut deficit, &mut out); - - let lat_bytes: usize = out - .iter() - .filter(|(c, _)| *c == Class::Latency) - .map(|(_, n)| n) - .sum(); - let bulk_bytes: usize = out - .iter() - .filter(|(c, _)| *c == Class::Bulk) - .map(|(_, n)| n) - .sum(); - assert!(lat_bytes >= 300, "latency should drain small packets"); - assert!( - bulk_bytes >= 1200, - "bulk must still send at least one quantum" - ); - assert!(latency.is_empty()); - assert_eq!(bulk.len(), 2); - } - - #[test] - fn empty_queue_zeros_deficit() { - let mut latency: VecDeque = VecDeque::new(); - let mut normal: VecDeque = VecDeque::new(); - let mut bulk: VecDeque = VecDeque::new(); - let mut deficit = [50_000, 0, 0]; - let mut out = Vec::new(); - drr_round_drain(&mut latency, &mut normal, &mut bulk, &mut deficit, &mut out); - assert_eq!(deficit[0], 0); - assert!(out.is_empty()); - } - - #[test] - fn classifies_from_validated_packets() { - let icmp = { - let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) - .icmpv4_echo_request(1, 1); - let mut o = Vec::new(); - b.write(&mut o, &[]).unwrap(); - o - }; - let dns = { - let b = - etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 53); - let mut o = Vec::new(); - b.write(&mut o, &[0; 32]).unwrap(); - o - }; - let large = { - let b = - etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 5201); - let mut o = Vec::new(); - b.write(&mut o, &[0; 1200]).unwrap(); - o - }; - let ordinary = { - let b = - etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 443); - let mut o = Vec::new(); - b.write(&mut o, &[0; 200]).unwrap(); - o - }; - assert_eq!( - classify(&tunnet_common::packet::parse(&icmp).unwrap(), 1280), - Class::Latency - ); - assert_eq!( - classify(&tunnet_common::packet::parse(&dns).unwrap(), 1280), - Class::Latency - ); - assert_eq!( - classify(&tunnet_common::packet::parse(&large).unwrap(), 1280), - Class::Bulk - ); - assert_eq!( - classify(&tunnet_common::packet::parse(&ordinary).unwrap(), 1280), - Class::Normal - ); - - let mut padded = ordinary.clone(); - padded.extend_from_slice(&[0u8; 2000]); - assert_eq!( - classify(&tunnet_common::packet::parse(&padded).unwrap(), 1280), - Class::Normal, - "trailing buffer must not change class" - ); - assert_eq!( - classify(&tunnet_common::packet::parse(&large).unwrap(), 9000), - Class::Normal, - "configured MTU raises bulk threshold" - ); - } -} diff --git a/crates/tunnet-agent/src/runtime.rs b/crates/tunnet-agent/src/runtime.rs index 3d9b2312..e4c0181b 100644 --- a/crates/tunnet-agent/src/runtime.rs +++ b/crates/tunnet-agent/src/runtime.rs @@ -159,7 +159,12 @@ pub async fn run( ( node.self_ipv4, 10u8, - 1280u16, + config_store + .load() + .effective + .tunnel_mtu + .value + .clamp(576, 9000), tunnet_core::load_dns(&node.paths), ) } else { @@ -170,7 +175,12 @@ pub async fn run( .find(|m| m.network_id == network_id) }) .context("cached snapshot missing enrolled network")?; - let effective_mtu = config_store.load().effective.tunnel_mtu.value.max(576); + let effective_mtu = config_store + .load() + .effective + .tunnel_mtu + .value + .clamp(576, 9000); ( membership_snap.assigned_ipv4, membership_snap.prefix, @@ -387,26 +397,24 @@ pub async fn run( let stream_handler = tunnet_core::stream_handler(node.routes.clone()); let dgram_pool = node.tunnel_pool.clone(); - let firewalls: HashMap<_, _> = node - .direct - .iter() - .map(|(id, rt)| (*id, rt.firewall.clone())) - .collect(); let spoofs: HashMap<_, _> = node .direct .iter() .map(|(id, rt)| (*id, rt.spoof_tracker.clone())) .collect(); + // Shared tunnel packet resources: pooled buffers (MTU classes) for TUN + // receives and segment staging. + let packet_pool = tunnet_common::packet::PacketPool::new(128); crate::dgram_pump::install_dialer_datagram_pump( &dgram_pool, published.clone(), node.routes.clone(), node.acl.clone(), - firewalls.clone(), + node.policy.clone(), spoofs.clone(), metrics.clone(), - node.direct_auth.clone(), + packet_pool.clone(), ingress.clone(), ); @@ -515,6 +523,7 @@ pub async fn run( endpoint: node.endpoint.clone(), routes: node.routes.clone(), acl: node.acl.clone(), + runtime: node.policy.clone(), metrics: metrics.clone(), tun: published.clone(), stream_handler, @@ -528,9 +537,9 @@ pub async fn run( auth_server_ctx, state_dir: node.paths.dir.clone(), docs: docs_map, - firewalls, spoofs, dgram_pool: dgram_pool.clone(), + bufs: packet_pool.clone(), agent_gossip: node.gossip.clone(), shared_docs: node.docs_engine.clone(), ingress: ingress.clone(), diff --git a/crates/tunnet-agent/src/ssh_nat.rs b/crates/tunnet-agent/src/ssh_nat.rs index 0dc91f22..d4bc48d2 100644 --- a/crates/tunnet-agent/src/ssh_nat.rs +++ b/crates/tunnet-agent/src/ssh_nat.rs @@ -1,79 +1,65 @@ //! Transparent TCP port NAT for Tunnet SSH (22 ↔ internal listen port). //! -//! Only unfragmented (or first-fragment) TCP packets identified by the shared -//! parser are rewritten. Checksums are recomputed with etherparse. +//! Parse-once: all entry points take already-parsed [`PacketMeta`] — the data +//! plane never parses a packet twice for NAT. Only unfragmented (or +//! first-fragment) TCP packets are rewritten; checksums via etherparse. use std::net::Ipv4Addr; -use tunnet_common::packet::{self, Packet, Transport, set_tcp_ipv4_checksum}; +use tunnet_common::packet::{PacketMeta, SshNatClass, set_tcp_ipv4_checksum}; pub const SSH_EXTERNAL_PORT: u16 = 22; pub const SSH_INTERNAL_PORT: u16 = 30022; -pub fn needs_inbound_rewrite(packet: &[u8], self_ip: Ipv4Addr) -> bool { - let Ok(pkt) = packet::parse(packet) else { - return false; - }; - eligible(&pkt, self_ip, true).is_some() +/// Parse-once outbound check using already-parsed metadata. +/// Gates materialization: only packets actually needing a rewrite take the +/// mutable path (§2.1-7); everything else stays immutable (zero copy). +pub fn needs_outbound_rewrite_with_meta(meta: &PacketMeta, self_ip: Ipv4Addr) -> bool { + matches!(meta.ssh_nat_class(self_ip), SshNatClass::OutboundToExternal) } -pub fn rewrite_inbound(packet: &mut [u8], self_ip: Ipv4Addr) -> bool { - rewrite(packet, self_ip, true) +/// Parse-once outbound rewrite using already-parsed metadata. +/// Returns true when a rewrite was applied (caller must refresh metadata). +pub fn rewrite_outbound_with_meta(packet: &mut [u8], meta: &PacketMeta, self_ip: Ipv4Addr) -> bool { + match meta.ssh_nat_class(self_ip) { + SshNatClass::OutboundToExternal => { + let ip_len = meta.ip_header_len; + if packet.len() < ip_len + 2 { + return false; + } + packet[ip_len] = (SSH_EXTERNAL_PORT >> 8) as u8; + packet[ip_len + 1] = (SSH_EXTERNAL_PORT & 0xff) as u8; + set_tcp_ipv4_checksum(packet, ip_len) + } + _ => false, + } } -pub fn rewrite_outbound(packet: &mut [u8], self_ip: Ipv4Addr) -> bool { - rewrite(packet, self_ip, false) +/// Parse-once inbound check using already-parsed metadata. +pub fn needs_inbound_rewrite_with_meta(meta: &PacketMeta, self_ip: Ipv4Addr) -> bool { + matches!(meta.ssh_nat_class(self_ip), SshNatClass::InboundToInternal) } -fn eligible(pkt: &Packet<'_>, self_ip: Ipv4Addr, inbound: bool) -> Option<(usize, usize)> { - if pkt.fragmentation.is_later() { - return None; - } - let Transport::Tcp { - src_port, - dst_port, - header_len, - .. - } = pkt.transport - else { - return None; - }; - if header_len < 18 { - return None; - } - let ip_len = pkt.ip.header_len(); - if inbound { - if pkt.ip.v4_dst() != Some(self_ip) || dst_port != SSH_EXTERNAL_PORT { - return None; - } - Some((ip_len + 2, SSH_INTERNAL_PORT as usize)) - } else { - if pkt.ip.v4_src() != Some(self_ip) || src_port != SSH_INTERNAL_PORT { - return None; +pub fn rewrite_inbound_with_meta(packet: &mut [u8], meta: &PacketMeta, self_ip: Ipv4Addr) -> bool { + match meta.ssh_nat_class(self_ip) { + SshNatClass::InboundToInternal => { + let ip_len = meta.ip_header_len; + if packet.len() < ip_len + 4 { + return false; + } + packet[ip_len + 2] = (SSH_INTERNAL_PORT >> 8) as u8; + packet[ip_len + 3] = (SSH_INTERNAL_PORT & 0xff) as u8; + set_tcp_ipv4_checksum(packet, ip_len) } - Some((ip_len, SSH_EXTERNAL_PORT as usize)) + _ => false, } } -fn rewrite(packet: &mut [u8], self_ip: Ipv4Addr, inbound: bool) -> bool { - let Ok(pkt) = packet::parse(packet) else { - return false; - }; - let Some((offset, new_port)) = eligible(&pkt, self_ip, inbound) else { - return false; - }; - let ip_len = pkt.ip.header_len(); - let new = new_port as u16; - packet[offset] = (new >> 8) as u8; - packet[offset + 1] = (new & 0xff) as u8; - set_tcp_ipv4_checksum(packet, ip_len) -} - #[cfg(test)] mod tests { use super::*; use etherparse::PacketBuilder; - use tunnet_common::packet::{parse, tcp_ipv4_checksum_of}; + use tunnet_common::packet::{PacketMeta, parse, tcp_ipv4_checksum_of}; fn sample_tcp(src: Ipv4Addr, dst: Ipv4Addr, sport: u16, dport: u16, payload: &[u8]) -> Vec { let b = PacketBuilder::ipv4(src.octets(), dst.octets(), 64).tcp(sport, dport, 1, 1000); @@ -82,13 +68,20 @@ mod tests { out } + fn meta_of(raw: &[u8]) -> PacketMeta { + let pkt = parse(raw).unwrap(); + PacketMeta::from_packet(&pkt) + } + #[test] fn inbound_rewrites_22_to_internal() { let self_ip = Ipv4Addr::new(100, 64, 0, 1); let peer = Ipv4Addr::new(100, 64, 0, 2); let mut p = sample_tcp(peer, self_ip, 45678, 22, b"hello"); let before_payload = p[40..].to_vec(); - assert!(rewrite_inbound(&mut p, self_ip)); + let meta = meta_of(&p); + assert!(needs_inbound_rewrite_with_meta(&meta, self_ip)); + assert!(rewrite_inbound_with_meta(&mut p, &meta, self_ip)); let pkt = parse(&p).unwrap(); assert_eq!(pkt.transport.dst_port(), Some(SSH_INTERNAL_PORT)); assert_eq!(&p[40..], before_payload.as_slice()); @@ -103,7 +96,8 @@ mod tests { let self_ip = Ipv4Addr::new(100, 64, 0, 1); let peer = Ipv4Addr::new(100, 64, 0, 2); let mut p = sample_tcp(self_ip, peer, SSH_INTERNAL_PORT, 45678, &[]); - assert!(rewrite_outbound(&mut p, self_ip)); + let meta = meta_of(&p); + assert!(rewrite_outbound_with_meta(&mut p, &meta, self_ip)); let pkt = parse(&p).unwrap(); assert_eq!(pkt.transport.src_port(), Some(22)); assert_eq!( @@ -117,12 +111,20 @@ mod tests { let self_ip = Ipv4Addr::new(100, 64, 0, 1); let peer = Ipv4Addr::new(100, 64, 0, 2); let mut p = sample_tcp(peer, self_ip, 45678, 443, &[]); - assert!(!rewrite_inbound(&mut p, self_ip)); + let meta = meta_of(&p); + assert!(!needs_inbound_rewrite_with_meta(&meta, self_ip)); + assert!(!rewrite_inbound_with_meta(&mut p, &meta, self_ip)); - let mut later = sample_tcp(peer, self_ip, 45678, 22, &[]); + let later = sample_tcp(peer, self_ip, 45678, 22, &[]); + let mut later = later; later[6] = 0; later[7] = 8; - assert!(!needs_inbound_rewrite(&later, self_ip)); - assert!(!rewrite_inbound(&mut later, self_ip)); + // Later fragment with nonzero offset: parse may fail or classify as + // later fragment; either way no rewrite must happen. + if let Ok(pkt) = parse(&later) { + let meta = PacketMeta::from_packet(&pkt); + assert!(!needs_inbound_rewrite_with_meta(&meta, self_ip)); + assert!(!rewrite_inbound_with_meta(&mut later, &meta, self_ip)); + } } } diff --git a/crates/tunnet-agent/src/tun_fast.rs b/crates/tunnet-agent/src/tun_fast.rs new file mode 100644 index 00000000..f16fa503 --- /dev/null +++ b/crates/tunnet-agent/src/tun_fast.rs @@ -0,0 +1,541 @@ +//! Platform-specific TUN fast paths sharing one packet semantics (§9). +//! +//! Linux: offload + `recv_multiple` into pool-owned batch slots (ownership +//! transferred to logical packets, no per-packet copy) and genuine +//! multi-packet `send_multiple` batches that let GSO coalesce. +//! Windows: Wintun ring drained as bursts into pooled buffers and filled +//! from an explicit pending batch that retains its unsent tail — no silent +//! loss. Ring capacity stays deliberate; bigger rings only mask queueing. +//! +//! All slot sizes derive from the configured virtual MTU (§6): a 2800+ byte +//! logical packet is never truncated by a fixed 2 KiB assumption. + +#[cfg(not(target_os = "linux"))] +use std::collections::VecDeque; +use std::sync::Arc; + +#[cfg(not(target_os = "linux"))] +use bytes::Bytes; +use tun_rs::AsyncDevice; +#[cfg(any(target_os = "linux", test))] +use tunnet_common::packet::PooledBuffer; +use tunnet_common::packet::{LogicalPacket, MAX_LOGICAL_LEN, PacketPool}; + +/// Desired TUN batch depth (starting point; tun-rs `IDEAL_BATCH_SIZE` = 128). +#[cfg(target_os = "linux")] +pub const BATCH_SIZE: usize = tun_rs::IDEAL_BATCH_SIZE; +/// Windows burst budget per readiness wakeup. +pub const BURST_BUDGET: usize = 64; +/// Inbound TUN write batch: packets accumulated per drain iteration (§9). +pub const TUN_WRITE_BATCH: usize = 32; + +/// Slot size for a virtual MTU: payload room plus virtio headroom on Linux. +pub fn slot_cap_for_mtu(mtu: usize) -> usize { + mtu.clamp(576, MAX_LOGICAL_LEN) + 256 +} + +/// Preallocated batch engine for Linux `recv_multiple`. +/// +/// Batch slots are pool-owned buffers with frame headroom intact: on +/// receipt each slot moves wholesale into the logical packet +/// (`from_pooled` — zero copy, and single-frame transmit later prepends +/// its header with no staging copy) and the slot is refilled from the +/// pool. `recv_multiple` writes at offset 0 of the slot's receive area, +/// which starts after the headroom. +#[cfg(target_os = "linux")] +pub struct LinuxBatchEngine { + pub orig: Vec, + bufs: Vec, + sizes: Vec, + pool: Arc, + slot_cap: usize, +} + +/// A pool-owned TUN receive slot. +/// +/// tun-rs `recv_multiple` contract (see `tun-rs/src/platform/linux/device.rs` +/// `handle_virtio_read` / `gso_split`): `as_ref().len()` is the RECEIVE +/// CAPACITY tun-rs checks writes against, and packets land at +/// `as_mut()[offset..]`. The two views therefore differ on purpose: +/// - `AsRef` reports the WHOLE backing storage (headroom + sized area), +/// independent of the current packet length (fresh slots hold no packet +/// yet — reporting the packet view here fails every batch with +/// "overflows bufs element len"); +/// - `AsMut` exposes the headroomed receive area sized by `prepare`. +/// +/// Receipt transfers the whole buffer into a logical packet with ownership +/// and headroom intact. Platform-independent by design so the contract is +/// unit-tested everywhere, not only on Linux. +#[cfg(any(target_os = "linux", test))] +struct BatchSlot(PooledBuffer); + +#[cfg(any(target_os = "linux", test))] +impl BatchSlot { + fn new(pool: &Arc, slot_cap: usize) -> Self { + let mut buf = pool.acquire(slot_cap); + buf.recv_region(slot_cap); + Self(buf) + } + + /// Size the receive area for the next batch. + fn prepare(&mut self, slot_cap: usize) { + self.0.recv_region(slot_cap); + // tun-rs capacity gate: the AsRef view must cover the slot even + // when no packet is stored yet (fresh/recycled buffers). + debug_assert!(self.as_ref().len() >= slot_cap); + } + + fn into_pooled(self) -> PooledBuffer { + self.0 + } +} + +#[cfg(any(target_os = "linux", test))] +impl AsRef<[u8]> for BatchSlot { + fn as_ref(&self) -> &[u8] { + // Same region and length as the AsMut view below: tun-rs + // validates `AsRef::len()` as capacity and writes into `AsMut`, + // so these must never diverge (a packet-length view here fails + // every batch on fresh slots). + self.0.recv_area() + } +} + +#[cfg(any(target_os = "linux", test))] +impl AsMut<[u8]> for BatchSlot { + fn as_mut(&mut self) -> &mut [u8] { + self.0.recv_area_mut() + } +} + +#[cfg(target_os = "linux")] +impl LinuxBatchEngine { + pub fn new(pool: Arc, mtu: usize) -> Self { + let slot_cap = slot_cap_for_mtu(mtu); + let mut bufs = Vec::with_capacity(BATCH_SIZE); + for _ in 0..BATCH_SIZE { + bufs.push(BatchSlot::new(&pool, slot_cap)); + } + Self { + orig: vec![0u8; tun_rs::VIRTIO_NET_HDR_LEN + 65535], + bufs, + sizes: vec![0usize; BATCH_SIZE], + pool, + slot_cap, + } + } + + /// Receive a batch; each packet takes ownership of its slot storage. + /// Reuses preallocated/pooled buffers; no per-packet copy, and the + /// common single-frame path never copies afterwards either. + pub async fn recv_batch(&mut self, dev: &AsyncDevice) -> anyhow::Result> { + for b in &mut self.bufs { + b.prepare(self.slot_cap); + } + let n = dev + .recv_multiple(&mut self.orig, &mut self.bufs, &mut self.sizes, 0) + .await?; + let mut out = Vec::with_capacity(n); + for i in 0..n { + let len = self.sizes[i]; + if len == 0 || len > self.slot_cap { + continue; + } + // Move the pool-owned slot into the packet (zero copy, + // headroom intact); refill the slot from the pool. + let slot = + std::mem::replace(&mut self.bufs[i], BatchSlot::new(&self.pool, self.slot_cap)); + if let Some(p) = LogicalPacket::from_pooled(slot.into_pooled(), len) { + out.push(p); + } + // else: malformed; the fresh replacement slot stays. + } + Ok(out) + } +} + +/// Genuine multi-packet TUN writer for Linux (§9). +/// +/// Accumulates decoded logical packets and flushes with ONE +/// `send_multiple`, letting GSO coalesce same-flow segments into fewer +/// syscalls. GRO state and staging storage are reused across iterations. +/// +/// Staging uses DEDICATED reusable byte buffers with the exact layout the +/// device expects — `[12B virtio header][IP packet]` — never `PooledBuffer` +/// (whose 32-byte tunnel-frame headroom would shift the packet and hand +/// the kernel 32 zero bytes instead of an IPv4 header; that misframing +/// silently black-holed all Linux TUN writes). +#[cfg(target_os = "linux")] +pub struct LinuxTunBatchWriter { + gro: tun_rs::GROTable, + staging: Vec>, + /// Empty buffers retained for reuse (capacity kept, no per-packet alloc + /// after warmup). + free: Vec>, +} + +#[cfg(target_os = "linux")] +impl LinuxTunBatchWriter { + pub fn new() -> Self { + Self { + gro: tun_rs::GROTable::default(), + staging: Vec::with_capacity(TUN_WRITE_BATCH), + free: Vec::new(), + } + } + + /// Stage one packet. Layout is exactly `[VIRTIO header zeros][packet]` + /// so `flush` can pass `VIRTIO_NET_HDR_LEN` as the packet offset: + /// tun-rs reads the IP packet at `buf[offset..]` and encodes the + /// virtio header into `buf[offset-12..offset]` itself. + pub fn push(&mut self, pkt: &[u8]) { + const HDR: usize = tun_rs::VIRTIO_NET_HDR_LEN; + let mut buf = self.free.pop().unwrap_or_default(); + buf.clear(); + buf.resize(HDR, 0); + buf.extend_from_slice(pkt); + debug_assert_eq!(&buf[HDR..], pkt, "packet must start exactly at the offset"); + self.staging.push(buf); + } + + pub fn is_empty(&self) -> bool { + self.staging.is_empty() + } + + /// Flush the staged batch with one `send_multiple` call. Buffers are + /// cleared and retained for the next batch (capacity kept). + pub async fn flush(&mut self, dev: &AsyncDevice) -> anyhow::Result { + if self.staging.is_empty() { + return Ok(0); + } + const HDR: usize = tun_rs::VIRTIO_NET_HDR_LEN; + let n = dev + .send_multiple(&mut self.gro, &mut self.staging, HDR) + .await?; + for mut buf in std::mem::take(&mut self.staging) { + buf.clear(); + if self.free.len() < TUN_WRITE_BATCH * 2 { + self.free.push(buf); + } + } + Ok(n) + } +} + +#[cfg(target_os = "linux")] +impl Default for LinuxTunBatchWriter { + fn default() -> Self { + Self::new() + } +} + +/// Windows burst drain into pooled buffers: after readiness, `try_recv` +/// until WouldBlock/budget. Each packet owns its pooled storage (no copy). +pub async fn windows_recv_burst( + dev: &AsyncDevice, + pool: &Arc, + mtu: usize, + budget: usize, +) -> anyhow::Result> { + let slot_cap = slot_cap_for_mtu(mtu); + let mut out = Vec::with_capacity(budget.min(BURST_BUDGET)); + // Prime with one async recv so we wait only when the ring is empty. + { + let mut buf = pool.acquire(slot_cap); + let n = dev.recv(buf.recv_region(slot_cap)).await?; + if n == 0 { + return Ok(out); + } + if let Some(p) = LogicalPacket::from_pooled(buf, n) { + out.push(p); + } + } + for _ in 1..budget.min(BURST_BUDGET) { + let mut buf = pool.acquire(slot_cap); + match dev.try_recv(buf.recv_region(slot_cap)) { + Ok(0) => break, + Ok(n) => { + if let Some(p) = LogicalPacket::from_pooled(buf, n) { + out.push(p); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e.into()), + } + } + Ok(out) +} + +/// Pending TUN write batch (§9, no silent loss). +/// +/// `drain_pending` fills the device with repeated `try_send`; when it is +/// full it waits once via async `send`, then resumes the SAME batch. The +/// unsent tail is retained in `pending` across waits — ownership is explicit +/// (`Bytes`, no copy) and nothing is silently discarded. +/// +/// Used by the Windows Wintun burst writer and by platforms without GSO +/// batching (same ring discipline everywhere outside Linux, where the GSO +/// writer owns TUN output instead). +#[cfg(not(target_os = "linux"))] +pub struct TunWriteBatch { + pub pending: VecDeque, +} + +#[cfg(not(target_os = "linux"))] +impl TunWriteBatch { + pub fn new() -> Self { + Self { + pending: VecDeque::new(), + } + } + + pub fn push(&mut self, pkt: Bytes) { + self.pending.push_back(pkt); + } + + pub fn is_empty(&self) -> bool { + self.pending.is_empty() + } + + /// Drain as much as the ring accepts right now. Returns the number of + /// packets written; the remainder stays queued. + pub fn drain_pending(&mut self, dev: &AsyncDevice) -> anyhow::Result { + let mut wrote = 0; + while let Some(front) = self.pending.front() { + match dev.try_send(front) { + Ok(_) => { + self.pending.pop_front(); + wrote += 1; + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e.into()), + } + } + Ok(wrote) + } + + /// Drain with one async wait when the ring is full; the tail is retained. + pub async fn drain_or_wait(&mut self, dev: &AsyncDevice) -> anyhow::Result { + let wrote = self.drain_pending(dev)?; + if self.pending.is_empty() { + return Ok(wrote); + } + // Ring full: exactly one async send to wait for space, then resume + // the same batch (no tail loss, no async-send pileup). + if let Some(front) = self.pending.front().cloned() { + dev.send(&front).await?; + self.pending.pop_front(); + Ok(wrote + 1) + } else { + Ok(wrote) + } + } +} + +#[cfg(not(target_os = "linux"))] +impl Default for TunWriteBatch { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const _: () = { + assert!(BURST_BUDGET >= 16 && BURST_BUDGET <= 256); + assert!(TUN_WRITE_BATCH >= 8); + }; + + #[test] + fn slot_cap_tracks_mtu() { + // No fixed 2048 assumption: large logical packets must fit. + assert!(slot_cap_for_mtu(1280) >= 1280); + assert!(slot_cap_for_mtu(2800) >= 2800); + assert!(slot_cap_for_mtu(9000) >= 9000); + assert!(slot_cap_for_mtu(100) >= 576); + assert!(slot_cap_for_mtu(99_999) <= MAX_LOGICAL_LEN + 256); + } + + #[test] + #[cfg(target_os = "linux")] + fn tun_batch_writer_stages_virtio_layout() { + // Regression test for silently misframed Linux TUN writes (kernel + // received zeros instead of IPv4): the staged buffer must be + // exactly [12B virtio zeros][IP packet] so `flush` with offset 12 + // hands tun-rs the packet at the right place. No device needed. + let mut w = LinuxTunBatchWriter::new(); + assert!(w.is_empty()); + let mut pkt = vec![0u8; 100]; + pkt[0] = 0x45; // IPv4-shaped, like a real packet + w.push(&pkt); + assert!(!w.is_empty()); + assert_eq!(w.staging.len(), 1); + let staged = &w.staging[0]; + assert_eq!(staged.len(), tun_rs::VIRTIO_NET_HDR_LEN + pkt.len()); + assert!(staged[..tun_rs::VIRTIO_NET_HDR_LEN].iter().all(|b| *b == 0)); + assert_eq!(&staged[tun_rs::VIRTIO_NET_HDR_LEN..], pkt.as_slice()); + assert_eq!(staged[tun_rs::VIRTIO_NET_HDR_LEN], 0x45); + } + + /// Real privileged Linux TUN round trip: writer → actual kernel TUN → + /// kernel ICMP reply → engine receive. Exercises BOTH real contracts + /// (device write framing AND `recv_multiple` slot semantics) that unit + /// tests cannot reach. + /// + /// Ignored by default: needs `CAP_NET_ADMIN` (real TUN device). Run on + /// a Linux dev machine with privileges: + /// `sudo -E cargo test -p tunnet-agent --lib tun_kernel_round_trip -- --ignored --nocapture` + #[test] + #[cfg(target_os = "linux")] + #[ignore = "needs CAP_NET_ADMIN + real TUN device"] + fn tun_kernel_round_trip() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let ip = std::net::Ipv4Addr::new(10, 99, 0, 1); + let ifname = format!("tnt{:x}", std::process::id() & 0xffff); + let dev = std::sync::Arc::new( + crate::tun_io::build_tun(&ifname, ip, 30, 2800) + .expect("TUN device (need CAP_NET_ADMIN)"), + ); + // Echo request to self: the kernel answers locally. + let echo_id = 0xbeefu16; + let b = etherparse::PacketBuilder::ipv4(ip.octets(), ip.octets(), 64) + .icmpv4_echo_request(echo_id, 1); + let mut req = Vec::new(); + b.write(&mut req, &[0xCCu8; 32]).unwrap(); + let mut writer = LinuxTunBatchWriter::new(); + writer.push(&req); + writer + .flush(&dev) + .await + .expect("flush echo request into the kernel"); + // Read back through the REAL batch engine (pooled slots). + let pool = PacketPool::new(8); + let mut engine = LinuxBatchEngine::new(pool, 2800); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + assert!( + std::time::Instant::now() < deadline, + "no kernel echo reply within 5s" + ); + let packets = tokio::time::timeout( + std::time::Duration::from_secs(5), + engine.recv_batch(&dev), + ) + .await + .expect("recv_batch must not hang") + .expect("recv_batch must succeed on a live device"); + for p in packets { + let meta = p.meta; + let is_reply = matches!( + meta.transport, + tunnet_common::packet::Transport::Icmpv4 { + type_u8: 0, + echo_id: Some(id), + .. + } if id == echo_id + ); + if !is_reply { + continue; + } + assert_eq!(meta.src_v4, Some(ip)); + assert_eq!(meta.dst_v4, Some(ip)); + assert_eq!( + &p.owner.as_bytes()[p.owner.as_bytes().len() - 32..], + &[0xCCu8; 32] + ); + return; + } + } + }); + } + + #[test] + #[cfg(not(target_os = "linux"))] + fn tun_write_batch_retains_tail() { + // Pure state-machine coverage (no device): pending ownership is + // explicit; nothing is silently discarded. + let mut b = TunWriteBatch::new(); + assert!(b.is_empty()); + b.push(Bytes::from_static(&[1, 2, 3])); + b.push(Bytes::from_static(&[4, 5])); + assert!(!b.is_empty()); + } + + #[test] + fn batch_slot_satisfies_tun_rs_contract() { + // Regression test for total Linux outbound loss (recv_batch failed + // every batch with "overflows bufs element len"): tun-rs validates + // writes against `as_ref().len()` (capacity) and writes into + // `as_mut()[offset..]`. The AsRef view must therefore cover the + // slot even when the slot holds NO packet yet (fresh buffers) or a + // smaller stale one — the packet view (`len`) is the wrong view. + // No TUN device needed: replicates tun-rs's exact checks. + use tunnet_common::packet::FRAME_HEADROOM; + let pool = PacketPool::new(8); + for mtu in [1280usize, 2800, 9000] { + let cap = slot_cap_for_mtu(mtu); + let mut slot = BatchSlot::new(&pool, cap); + slot.prepare(cap); + // Fresh slot: packet view empty, capacity view full. + assert_eq!(slot.0.len(), 0); + assert!( + slot.as_ref().len() >= cap, + "mtu={mtu}: AsRef must report receive capacity" + ); + assert!( + slot.as_mut().len() >= cap, + "mtu={mtu}: AsMut must expose the receive area" + ); + // tun-rs handle_virtio_read / gso_split checks, offset 0. + let offset = 0usize; + for packet_len in [60usize, 1400, cap] { + assert!( + !(offset > slot.as_ref().len()), + "mtu={mtu}: invalid offset must not trigger" + ); + assert!( + !(slot.as_ref().len() - offset < packet_len), + "mtu={mtu} len={packet_len}: output buffer too small must not trigger" + ); + } + // Simulate a kernel write at as_mut()[0..len], then take + // ownership exactly like recv_batch does. + let pkt = { + let b = etherparse::PacketBuilder::ipv4([10, 7, 0, 1], [10, 7, 0, 2], 64) + .udp(40000, 443); + let mut o = Vec::new(); + b.write(&mut o, &[0xABu8; 200]).unwrap(); + o + }; + slot.as_mut()[..pkt.len()].copy_from_slice(&pkt); + let mut pooled = slot.into_pooled(); + let parsed = LogicalPacket::from_pooled(pooled, pkt.len()).expect("must parse"); + assert_eq!(parsed.owner.as_bytes(), pkt.as_slice()); + // Headroom intact for the single-frame prepend (zero-copy path). + pooled = match parsed.owner { + tunnet_common::packet::PacketOwner::Pooled(b) => b, + _ => panic!("must stay pooled"), + }; + assert!(pooled.header_slot(FRAME_HEADROOM).is_some()); + // Recycled slot with a SMALLER stale packet length still + // reports full capacity (the stale-len trap). + let mut slot2 = BatchSlot::new(&pool, cap); + slot2.prepare(cap); + let mut reused = slot2.into_pooled(); + reused.set_len(60); + drop(reused); + let mut slot3 = BatchSlot::new(&pool, cap); + slot3.prepare(cap); + assert!( + slot3.as_ref().len() >= cap, + "recycled slot must still report full capacity" + ); + } + } +} diff --git a/crates/tunnet-agent/src/tun_io.rs b/crates/tunnet-agent/src/tun_io.rs index f3c081d8..ac791378 100644 --- a/crates/tunnet-agent/src/tun_io.rs +++ b/crates/tunnet-agent/src/tun_io.rs @@ -1,22 +1,30 @@ use std::collections::HashMap; use std::sync::Arc; +use std::sync::atomic::Ordering; use anyhow::Context; use bytes::Bytes; +use futures_util::FutureExt as _; use iroh::endpoint::Connection; use tun_rs::{AsyncDevice, DeviceBuilder}; -use tunnet_common::packet::{self, Packet}; +use tunnet_common::packet::{self, LogicalPacket}; use tunnet_common::policy::Direction; -use tunnet_core::direct::{ - AuthCache, EvalResult, FirewallEngine, PacketDirection, SpoofTracker, source_matches_peer, -}; -use tunnet_core::{AclEngine, ConnPool, RoutingTable, iroh_pool::send_datagram}; +use tunnet_core::direct::{AuthCache, SpoofTracker, source_matches_peer}; +use tunnet_core::peers::{PeerIdentity, PeerMembershipState, PeerRegistry}; +use tunnet_core::policy_runtime::{PolicyRuntime, PolicyVerdict}; +use tunnet_core::routing::{RouteDecision, RoutingTable}; +use tunnet_core::{AclEngine, ConnPool, iroh_pool::send_datagram}; use uuid::Uuid; use crate::actors::dataplane::PublishedPlane; use crate::metrics::AgentMetrics; -use crate::qos::{self, OutboundScheduler}; +use crate::pump::ensure_pump; use crate::ssh_nat; +use crate::tun_fast; + +/// Opportunistic inbound drain budget (§10): after each awaited datagram, +/// drain already-ready datagrams without busy-polling. +pub const INBOUND_DRAIN_BUDGET: usize = 32; pub fn build_tun( ifname: &str, @@ -24,10 +32,28 @@ pub fn build_tun( prefix: u8, mtu: u16, ) -> anyhow::Result { + // Fast-path builder: Linux enables offload; Windows uses the Wintun ring. + // Diagnostic override for A/B runs: TUNNET_TUN_OFFLOAD=0 disables + // tun-rs offload+GSO (plain single-packet TUN I/O instead). + #[cfg(target_os = "linux")] + let offload = std::env::var("TUNNET_TUN_OFFLOAD") + .map(|v| { + !matches!( + v.to_ascii_lowercase().as_str(), + "0" | "false" | "no" | "off" + ) + }) + .unwrap_or(true); let builder = DeviceBuilder::new() .name(ifname) .ipv4(ipv4, prefix, None) .mtu(mtu); + #[cfg(target_os = "linux")] + let builder = if offload { + builder.offload(true) + } else { + builder + }; #[cfg(windows)] let builder = { let path = crate::wintun::materialize()?; @@ -36,7 +62,7 @@ pub fn build_tun( .wintun_log(true) }; let dev = builder.build_async().context("build_async TUN device")?; - tracing::info!(%ipv4, prefix, mtu, "TUN device up"); + tracing::info!(%ipv4, prefix, mtu, "TUN device up (fast path)"); Ok(dev) } @@ -44,26 +70,285 @@ pub struct OutboundDeps { pub tun: Arc, pub routes: RoutingTable, pub pool: ConnPool, - pub acl: AclEngine, - pub firewalls: HashMap, + pub runtime: PolicyRuntime, pub metrics: AgentMetrics, + pub bufs: Arc, + pub meter: tunnet_core::CloudRelayMeter, pub mtu: u16, } -fn drop_parse(metrics: &AgentMetrics, err: packet::ParseError) { - metrics.dropped_inc(err.drop_reason()); +/// Handle one owned logical packet through the outbound pipeline. +/// Parse-once: `packet` already carries metadata; NAT refreshes it only when +/// a rewrite actually mutated the bytes. Policy uses the shared runtime with +/// the peer's stable network slot — no per-packet map lookups. +fn handle_outbound_one( + mut packet: LogicalPacket, + fast_ctx: &OutboundCtx<'_>, +) -> Option> { + let ctx = *fast_ctx; + let OutboundCtx { + routes, + runtime, + metrics, + pool, + bufs, + meter, + self_ip, + .. + } = ctx; + // SSH NAT consumes existing metadata (no second parse) — and ONLY takes + // the mutable/materializing path when metadata proves a rewrite is + // required (§2.1-7). Common packets stay immutable: zero copy. + let meta = packet.meta; + if ssh_nat::needs_outbound_rewrite_with_meta(&meta, self_ip) { + let Some(bytes) = packet_owner_bytes_mut(&mut packet, bufs) else { + metrics.dropped_inc("nat_materialize"); + return None; + }; + if ssh_nat::rewrite_outbound_with_meta(bytes, &meta, self_ip) && !packet.refresh() { + // Rewrite applied but re-parse failed: fail closed. + metrics.dropped_inc("nat_reparse"); + return None; + } + } + let Some(dst) = packet.meta.dst_v4 else { + metrics.dropped_inc("ipv6_unsupported"); + return None; + }; + + // Single immutable-snapshot route decision; the handle carries the + // stable fast state (no peer map lookup after routing). + let fast = match routes.route_once(&dst) { + RouteDecision::LocalMagic => { + metrics.dropped_inc("magic_dns_local"); + return None; + } + RouteDecision::LocalAdvertised => { + metrics.dropped_inc("local_subnet"); + return None; + } + RouteDecision::NoRoute => { + metrics.dropped_inc("no_route"); + return None; + } + RouteDecision::Peer(h) => h.peer.fast.clone(), + }; + + if fast.identity.read().ip == self_ip { + metrics.dropped_inc("self"); + return None; + } + + // One compiled verdict against the shared runtime. The firewall + // snapshot loads from the peer's STABLE network slot inside check() + // (ACL-then-firewall order) — publication swaps it in place, so no + // relink is ever needed (§2.1-3, §2.2-2). + let ident: Arc = fast.identity.read().clone(); + let slot = fast.policy.load(); + let verdict = runtime.check( + &packet.meta, + Direction::Outbound, + &ident.endpoint_hex, + &ident.tags, + Some(ident.hostname.as_str()), + Some(ident.network_id), + &slot, + &slot.counters, + ); + match verdict { + PolicyVerdict::Allow => {} + PolicyVerdict::Deny => { + metrics.dropped_inc("policy_deny"); + return None; + } + PolicyVerdict::Reject => { + metrics.dropped_inc("fw_reject_out"); + send_reject_reply(fast_ctx, &packet); + return None; + } + } + + let len = packet.len() as i64; + // Every shed/evicted packet is reported (gauges + telemetry) — the + // scheduler never drops silently (see EnqueueOutcome). + let outcome = { + let mut sched = fast.scheduler.lock(); + let outcome = sched.enqueue(packet, std::time::Instant::now()); + let deltas = sched.drain_drops(); + (outcome, deltas) + }; + match outcome.0 { + tunnet_core::scheduler::EnqueueOutcome::Accepted => {} + tunnet_core::scheduler::EnqueueOutcome::AcceptedEvicted { + reason, + evicted_len, + } => { + metrics.queue_add(-1, -(evicted_len as i64), 0); + metrics.dropped_inc(reason.as_str()); + metrics.sched_drop_inc(reason.as_str()); + } + tunnet_core::scheduler::EnqueueOutcome::Rejected { reason } => { + metrics.dropped_inc(reason.as_str()); + metrics.sched_drop_inc(reason.as_str()); + report_sched_deltas(metrics, outcome.1); + return None; + } + } + report_sched_deltas(metrics, outcome.1); + metrics.queue_add(1, len, 0); + ensure_pump( + &fast, + pool.clone(), + metrics.clone(), + bufs.clone(), + meter.clone(), + ); + Some(fast) } -fn require_ipv4<'a>(metrics: &AgentMetrics, pkt: Packet<'a>, inbound: bool) -> Option> { - if pkt.ip.v4_src().is_none() { - metrics.dropped_inc(if inbound { - "ipv6_unsupported_in" - } else { - "ipv6_unsupported" - }); +/// Report drained CoDel/emergency deltas (dequeue-side drops with no +/// enqueue decision site). +fn report_sched_deltas(metrics: &AgentMetrics, deltas: tunnet_core::scheduler::SchedDropDeltas) { + metrics.sched_drops_add(deltas.codel, deltas.emergency); +} + +/// Mutable packet bytes for NAT, materializing pooled/shared storage. +/// Returns None only when materialization fails (counts as drop). +fn packet_owner_bytes_mut<'a>( + packet: &'a mut LogicalPacket, + pool: &Arc, +) -> Option<&'a mut [u8]> { + if matches!(packet.owner, tunnet_common::packet::PacketOwner::Shared(_)) + && !packet.materialize(pool) + { return None; } - Some(pkt) + match &mut packet.owner { + tunnet_common::packet::PacketOwner::Pooled(b) => { + let len = b.len(); + Some(&mut b.recv_region(len)[..len]) + } + tunnet_common::packet::PacketOwner::Shared(_) => None, + } +} + +struct OutboundCtx<'a> { + tun: &'a Arc, + routes: &'a RoutingTable, + runtime: &'a PolicyRuntime, + metrics: &'a AgentMetrics, + pool: &'a ConnPool, + bufs: &'a Arc, + meter: &'a tunnet_core::CloudRelayMeter, + self_ip: std::net::Ipv4Addr, +} + +impl Copy for OutboundCtx<'_> {} +impl Clone for OutboundCtx<'_> { + fn clone(&self) -> Self { + *self + } +} + +/// Reject replies are rare, but they must be protocol-correct: the peer +/// expects every tunnel DATAGRAM to begin with 0x30/0x31 with its bound +/// network. Route the reply through the normal tunnel framing/transmit +/// path (scheduler and pump, which segments large replies) instead of a +/// second encoder path. Without a pool (no pump possible) fall back to a +/// single framed best-effort send (still net-bound). +async fn send_reject_framed( + reply: Bytes, + net: Uuid, + fast: &Arc, + pool: Option<&ConnPool>, + conn: &Connection, + bufs: &Arc, + metrics: &AgentMetrics, +) { + let Some(pool) = pool else { + // No pump available: single framed best-effort send, with the + // full v3 header ([kind][net][reply]). + let mut frame = Vec::with_capacity(reply.len() + tunnet_common::packet::SINGLE_OVERHEAD); + frame.push(tunnet_common::packet::KIND_SINGLE); + frame.extend_from_slice(net.as_bytes()); + frame.extend_from_slice(&reply); + if conn + .max_datagram_size() + .is_some_and(|max| frame.len() > max) + { + metrics.dropped_inc("datagram_too_large"); + return; + } + let _ = send_datagram(conn, Bytes::from(frame)).await; + return; + }; + // Zero-copy: the synthesized reply bytes ride straight into the + // scheduler; the pump frames/segments them like any other packet. + let Some(packet) = LogicalPacket::from_shared(reply) else { + metrics.dropped_inc("malformed_transport"); + return; + }; + let len = packet.len() as i64; + let (outcome, deltas) = { + let mut sched = fast.scheduler.lock(); + let outcome = sched.enqueue(packet, std::time::Instant::now()); + let deltas = sched.drain_drops(); + (outcome, deltas) + }; + match outcome { + tunnet_core::scheduler::EnqueueOutcome::Accepted => {} + tunnet_core::scheduler::EnqueueOutcome::AcceptedEvicted { + reason, + evicted_len, + } => { + metrics.queue_add(-1, -(evicted_len as i64), 0); + metrics.dropped_inc(reason.as_str()); + metrics.sched_drop_inc(reason.as_str()); + } + tunnet_core::scheduler::EnqueueOutcome::Rejected { reason } => { + metrics.dropped_inc(reason.as_str()); + metrics.sched_drop_inc(reason.as_str()); + report_sched_deltas(metrics, deltas); + return; + } + } + report_sched_deltas(metrics, deltas); + metrics.queue_add(1, len, 0); + ensure_pump( + fast, + pool.clone(), + metrics.clone(), + bufs.clone(), + pool.cloud_relay_meter(), + ); +} + +/// Outbound reject replies go to the LOCAL TUN device (raw IP framing — +/// correct there: TUN is not the tunnel wire). Rare: synthesize and send off +/// the hot path with correct platform framing. +fn send_reject_reply(ctx: &OutboundCtx<'_>, packet: &LogicalPacket) { + let reply = packet::parse(packet.owner.as_bytes()) + .ok() + .and_then(|p| packet::synthesize_reject(&p)); + let Some(reply) = reply.filter(|r| !r.is_empty()) else { + return; + }; + #[cfg(target_os = "linux")] + { + let tun = ctx.tun.clone(); + tokio::spawn(async move { + let mut w = tun_fast::LinuxTunBatchWriter::new(); + w.push(&reply); + let _ = w.flush(&tun).await; + }); + } + #[cfg(not(target_os = "linux"))] + { + let tun = ctx.tun.clone(); + tokio::spawn(async move { + let _ = tun.send(&reply).await; + }); + } } pub async fn run_outbound(deps: OutboundDeps) -> anyhow::Result<()> { @@ -71,87 +356,87 @@ pub async fn run_outbound(deps: OutboundDeps) -> anyhow::Result<()> { tun, routes, pool, - acl, - firewalls, + runtime, metrics, + bufs, + meter, mtu, } = deps; + // Cache pool hit/miss telemetry periodically (cheap atomics). + let metrics_pool = metrics.clone(); + let bufs_pool = bufs.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); + loop { + interval.tick().await; + let (h, m) = bufs_pool.hit_miss(); + metrics_pool.pool_hit_miss(h, m); + } + }); + + let self_ip = runtime.self_ip(); + metrics.mtu_set(mtu as u64); - let scheduler = OutboundScheduler::new(pool.clone(), metrics.clone(), mtu); + #[cfg(target_os = "linux")] + let mut batch = tun_fast::LinuxBatchEngine::new(bufs.clone(), mtu as usize); - let mut buf = vec![0u8; 65_536]; - tracing::info!("outbound TUN→iroh Byte-DRR loop started"); + tracing::info!("outbound TUN→iroh tunnel loop started"); loop { - let n = tun.recv(&mut buf).await?; - if n == 0 { - continue; - } - let self_ip = acl.self_id.load().ip; - let _ = ssh_nat::rewrite_outbound(&mut buf[..n], self_ip); - let packet = &buf[..n]; - let pkt = match packet::parse(packet) { - Ok(p) => p, - Err(e) => { - drop_parse(&metrics, e); + #[cfg(target_os = "linux")] + { + let packets = batch.recv_batch(&tun).await?; + metrics.tun_syscall_inc("recv_batch"); + if packets.is_empty() { continue; } - }; - let Some(pkt) = require_ipv4(&metrics, pkt, false) else { - continue; - }; - let dst = pkt.ip.v4_dst().unwrap(); - - if routes.is_magic_dns_destination(&dst) { - metrics.dropped_inc("magic_dns_local"); - continue; - } - - if routes.is_advertised_destination(&dst) { - metrics.dropped_inc("local_subnet"); - continue; - } - - let Some(peer) = routes.lookup_ip(&dst) else { - metrics.dropped_inc("no_route"); - continue; - }; - - if peer.ip == self_ip { - metrics.dropped_inc("self"); - continue; - } - - if !acl.allow_packet(&peer.endpoint_hex, Direction::Outbound, &pkt) { - metrics.dropped_inc("policy_deny"); + let ctx = OutboundCtx { + tun: &tun, + routes: &routes, + runtime: &runtime, + metrics: &metrics, + pool: &pool, + bufs: &bufs, + meter: &meter, + self_ip, + }; + for packet in packets { + if packet.len() > mtu as usize { + metrics.dropped_inc("oversize_mtu"); + continue; + } + handle_outbound_one(packet, &ctx); + } continue; } - if let Some(fw) = firewalls.get(&peer.network_id) { - match fw.evaluate( - PacketDirection::Outbound, - &pkt, - Some(&peer.endpoint_hex), - Some(&peer.hostname), - Some(peer.network_id), - ) { - EvalResult::Allow => {} - EvalResult::Deny => { - metrics.dropped_inc("fw_deny_out"); - continue; - } - EvalResult::Reject { reply } => { - metrics.dropped_inc("fw_reject_out"); - if !reply.is_empty() { - let _ = tun.send(&reply).await; - } + #[allow(unreachable_code)] + { + // Windows + fallback: burst-drain the ring into pooled buffers. + let burst = + tun_fast::windows_recv_burst(&tun, &bufs, mtu as usize, tun_fast::BURST_BUDGET) + .await?; + metrics.tun_syscall_inc("recv_burst"); + if burst.is_empty() { + continue; + } + let ctx = OutboundCtx { + tun: &tun, + routes: &routes, + runtime: &runtime, + metrics: &metrics, + pool: &pool, + bufs: &bufs, + meter: &meter, + self_ip, + }; + for packet in burst { + if packet.len() > mtu as usize { + metrics.dropped_inc("oversize_mtu"); continue; } + handle_outbound_one(packet, &ctx); } } - - let class = qos::classify(&pkt, mtu); - let payload = Bytes::copy_from_slice(packet); - scheduler.enqueue(peer.endpoint, class, payload); } } @@ -159,12 +444,16 @@ pub struct InboundDeps { pub conn: Connection, pub tun: PublishedPlane, pub routes: RoutingTable, + pub runtime: PolicyRuntime, pub acl: AclEngine, - pub firewalls: HashMap, pub spoofs: HashMap, pub pool: Option, + pub bufs: Arc, pub metrics: AgentMetrics, - pub direct_auth: Option, + /// Per-network auth bindings for inbound packet authorization + /// (§2.2-1). None in managed mode (ACL admission governs); enforced + /// per frame network when present. + pub auth: Option, } pub async fn serve_tunnel_connection(deps: InboundDeps) { @@ -172,12 +461,13 @@ pub async fn serve_tunnel_connection(deps: InboundDeps) { conn, tun, routes, + runtime, acl, - firewalls, spoofs, pool, + bufs, metrics, - direct_auth, + auth, } = deps; let remote_id = conn.remote_id(); let remote_hex = format!("{remote_id}"); @@ -199,28 +489,45 @@ pub async fn serve_tunnel_connection(deps: InboundDeps) { tracing::debug!(%remote_id, max_datagram_size = max, "quic datagram limit"); } } - let inbound_network = direct_auth - .as_ref() - .and_then(|a| a.networks_for(&remote_hex).into_iter().next()) - .or_else(|| routes.lookup_endpoint(&remote_hex).map(|p| p.network_id)); + // Membership resolution is lazy and per frame network (§2.2-1): the + // first datagram's bound network selects the (endpoint, network) + // membership; the cached Arc is reused while frames carry the same + // network. Never infer network identity from insertion order. + let registry = routes.peer_registry().clone(); + // Truly unknown endpoints still close at admission (no membership in + // any network); known endpoints resolve per frame network below. + if registry.get(remote_id).is_none() && routes.lookup_endpoint(&remote_hex).is_none() { + tracing::debug!(%remote_id, "unknown peer at admission; closing"); + conn.close(1u32.into(), b"no_route"); + metrics.active_conns_dec(); + return; + } + let mut fast_state: Option> = None; + let mut fast_net = Uuid::nil(); + let mut fast_epoch = 0u64; + let mut route_gen = routes.version(); - // Load the published generation once. Retain the device + its exact - // cancellation token; never reacquire a global lock per packet and never - // observe a newer generation. + // Load the published generation once (device + cancel token pinned). let Some(plane) = tun.load_full() else { + metrics.active_conns_dec(); return; }; let device = plane.device.clone(); let generation_cancel = plane.cancel.clone(); - // Pinned at reader start: this task never observes a newer generation. tracing::debug!(generation = plane.generation, %remote_id, "ingress reader pinned"); + #[cfg(target_os = "linux")] + let mut tun_batch = tun_fast::LinuxTunBatchWriter::new(); + #[cfg(not(target_os = "linux"))] + let mut tun_batch = tun_fast::TunWriteBatch::new(); + loop { if generation_cancel.is_cancelled() { break; } - // Cancellation first so BringDown promptly stops old readers. - let dg = tokio::select! { + // Await one datagram (cancellation-first), then opportunistically + // drain already-ready datagrams up to a bounded budget (§10). + let first = tokio::select! { biased; _ = generation_cancel.cancelled() => break, res = conn.read_datagram() => match res { @@ -234,106 +541,765 @@ pub async fn serve_tunnel_connection(deps: InboundDeps) { if generation_cancel.is_cancelled() { break; } - { - #[allow(clippy::collapsible_if)] - if let Some(p) = &pool { - p.touch_peer(remote_id); + if let Some(p) = &pool { + p.touch_peer(remote_id); + } + metrics.datagram_inc("in"); + let mut batch: Vec = vec![first]; + // Opportunistic drain: ReadDatagram::poll serves buffered datagrams + // synchronously first, so polling a fresh future once is a safe + // non-waiting drain probe (dropping a Pending future only drops its + // waker registration; no shared state is disturbed). + for _ in 0..INBOUND_DRAIN_BUDGET { + match conn.read_datagram().now_or_never() { + Some(Ok(dg)) => { + metrics.datagram_inc("in"); + batch.push(dg); + } + _ => break, } - - let pkt = match packet::parse(&dg) { - Ok(p) => p, - Err(e) => { - drop_parse(&metrics, e); + } + if generation_cancel.is_cancelled() { + break; + } + // Routing generation check (one atomic load per batch): when + // membership changed, drop the cached membership (per-packet + // resolve below re-resolves or drops). If the endpoint holds NO + // membership in any network anymore, the connection is dead: + // close and exit instead of forwarding through stale state. + let route_version = routes.version(); + if route_version != route_gen { + route_gen = route_version; + fast_state = None; + if !registry.has_any_membership(remote_id) { + tracing::info!(%remote_id, "peer removed from all networks; closing ingress reader"); + conn.close(1u32.into(), b"membership_removed"); + break; + } + } + // Deactivation without a generation change (e.g. pool drop_peer): + // drop the cached membership; per-packet resolve re-resolves or, + // when nothing remains, the generation check above exits. + if let Some(fast) = &fast_state + && fast.epoch.load(Ordering::Relaxed) != fast_epoch + { + tracing::info!(%remote_id, "membership deactivated; re-resolving"); + fast_state = None; + } + let self_ip = runtime.self_ip(); + let mut tun_pending: u32 = 0; + for dg in batch { + // Decode the frame header first (no allocation): it binds the + // network this packet belongs to. + let frame = match tunnet_common::packet::decode_frame(&dg) { + Ok(f) => f, + Err(_) => { + metrics.dropped_inc("malformed_frame"); + metrics.reassembly_inc("malformed"); continue; } }; - let Some(pkt) = require_ipv4(&metrics, pkt, true) else { - continue; + let net = match &frame { + tunnet_common::packet::Frame::Single { net, .. } => *net, + tunnet_common::packet::Frame::Segment { net, .. } => *net, }; - let src = pkt.ip.v4_src().unwrap(); - - let peer_info = inbound_network - .and_then(|nid| routes.lookup_network_ip(nid, &src)) - .or_else(|| routes.lookup_endpoint(&remote_hex)); - - if let Some(peer_info) = &peer_info - && !source_matches_peer(src, peer_info.ip) - { - metrics.dropped_inc("antispoof"); - if let Some(nid) = inbound_network.or(Some(peer_info.network_id)) - && let Some(tracker) = spoofs.get(&nid) - && tracker.record(&remote_hex) - { - let counts = tracker.drain_window_counts(); - for (peer, n) in counts { - tracing::warn!( - peer = %peer, - spoofed_packets = n, - "ingress anti-spoof drops in last window" - ); + // Resolve/switch the (endpoint, network) membership. A frame + // claiming a network with no membership — or a network the + // endpoint is not authenticated for — is dropped, never + // evaluated under another network's state. + if fast_state.is_none() || net != fast_net { + match resolve_membership( + ®istry, + &routes, + &remote_id, + &remote_hex, + net, + auth.as_ref(), + ) { + Some(next) => { + fast_epoch = next.epoch.load(Ordering::Relaxed); + fast_net = net; + fast_state = Some(next); + } + None => { + metrics.dropped_inc("unknown_network"); + continue; } } - continue; } - - if !acl.allow_packet(&remote_hex, Direction::Inbound, &pkt) { - metrics.dropped_inc("policy_deny_in"); + let fast = fast_state.as_ref().expect("resolved"); + // Membership revoked mid-batch: drop the cache; the next + // packet re-resolves (or drops when nothing remains). + if fast.epoch.load(Ordering::Relaxed) != fast_epoch { + fast_state = None; + metrics.dropped_inc("membership_revoked"); continue; } - - let peer_net = peer_info.as_ref().map(|p| p.network_id).or(inbound_network); - if let Some(nid) = peer_net - && let Some(fw) = firewalls.get(&nid) + if handle_inbound_one( + &dg, + frame, + fast, + &runtime, + &spoofs, + &conn, + pool.as_ref(), + &bufs, + &metrics, + &mut tun_batch, + self_ip, + ) + .await { - match fw.evaluate( - PacketDirection::Inbound, - &pkt, - Some(&remote_hex), - peer_info.as_ref().map(|p| p.hostname.as_str()), - Some(nid), - ) { - EvalResult::Allow => {} - EvalResult::Deny => { - metrics.dropped_inc("fw_deny_in"); - continue; - } - EvalResult::Reject { reply } => { - metrics.dropped_inc("fw_reject_in"); - if !reply.is_empty() { - let _ = send_datagram(&conn, reply).await; + tun_pending += 1; + } + // Flush mid-iteration so bursts larger than the batch still + // complete without loss (§9); the tail stays staged on failure. + if tun_pending >= tun_fast::TUN_WRITE_BATCH as u32 { + if !flush_tun_batch(&mut tun_batch, &device, &metrics).await { + break; + } + tun_pending = 0; + } + } + // Flush the TUN batch once per drain iteration (§9). + if tun_pending > 0 && !flush_tun_batch(&mut tun_batch, &device, &metrics).await { + break; + } + } + metrics.active_conns_dec(); + tracing::info!(%remote_id, "peer disconnected"); +} + +/// Slow-path resolve of the exact (endpoint, network) membership (§2.2-1): +/// registry first, else build from route info. Assigns the network's +/// stable firewall slot on created states, like routing rebuilds do. +/// Returns None when the endpoint has no such membership OR is not +/// authenticated for that network — the packet is then dropped, never +/// evaluated under another network's identity/policy. +fn resolve_membership( + registry: &PeerRegistry, + routes: &RoutingTable, + remote: &iroh::EndpointId, + remote_hex: &str, + net: Uuid, + auth: Option<&AuthCache>, +) -> Option> { + // Authenticated membership binding: the endpoint must be authenticated + // FOR THIS NETWORK, not merely known for any network. + if let Some(auth) = auth + && !auth.contains_network(remote_hex, net) + { + return None; + } + if let Some(fast) = registry.get_membership(*remote, net) { + return Some(fast); + } + // First packet after a rebuild race: construct from route info. + let info = routes.lookup_membership(remote_hex, net)?; + // Recheck auth for race-constructed states (membership data and auth + // cache update on different paths). + if let Some(auth) = auth + && !auth.contains_network(remote_hex, info.network_id) + { + return None; + } + let fast = registry.ensure_membership(Arc::new(tunnet_core::peers::PeerIdentity { + endpoint: info.endpoint, + endpoint_hex: info.endpoint_hex.clone(), + hostname: info.hostname.clone(), + ip: info.ip, + tags: info.tags.clone(), + network_id: info.network_id, + network_name: info.network_name.clone(), + })); + if let Some(slot) = routes.policy_slot_for(info.network_id) { + fast.policy.store(slot); + } + Some(fast) +} + +/// Handle one inbound DATAGRAM: reassemble → parse → antispoof → policy → +/// NAT → stage for the TUN batch. The frame (already decoded by the caller, +/// which used its bound network to resolve `fast`) and the membership are +/// passed in. Returns true when a TUN packet was staged. +#[allow(clippy::too_many_arguments)] +async fn handle_inbound_one( + dg: &Bytes, + frame: tunnet_common::packet::Frame<'_>, + fast: &Arc, + runtime: &PolicyRuntime, + spoofs: &HashMap, + conn: &Connection, + pool: Option<&ConnPool>, + pool_bufs: &Arc, + metrics: &AgentMetrics, + tun_batch: &mut TunBatchForPlatform, + self_ip: std::net::Ipv4Addr, +) -> bool { + use tunnet_core::reassembly::InsertOut; + let now = std::time::Instant::now(); + let logical: LogicalPacket = match frame { + tunnet_common::packet::Frame::Single { payload: p, .. } => { + // Zero-copy: retain the DATAGRAM's storage. + let off = p.as_ptr() as usize - dg.as_ptr() as usize; + let owned = dg.slice(off..off + p.len()); + match LogicalPacket::from_shared(owned) { + Some(pkt) => { + metrics.reassembly_inc("single"); + pkt + } + None => { + metrics.dropped_inc("malformed_transport"); + return false; + } + } + } + tunnet_common::packet::Frame::Segment { + header: h, payload, .. + } => { + let off = payload.as_ptr() as usize - dg.as_ptr() as usize; + let owned = dg.slice(off..off + payload.len()); + let mut table = fast.reassembly.lock(); + match table.insert(h, owned, now) { + InsertOut::Complete(logical) => { + metrics.reassembly_inc("complete"); + match LogicalPacket::from_vec(logical) { + Some(pkt) => pkt, + None => { + metrics.dropped_inc("malformed_transport"); + return false; } - continue; } } + InsertOut::Pending => { + metrics.reassembly_inc("pending"); + return false; + } + InsertOut::Duplicate => { + metrics.reassembly_inc("duplicate"); + return false; + } + InsertOut::Dropped(reason) => { + metrics.reassembly_inc("dropped"); + metrics.dropped_inc(match reason { + tunnet_core::reassembly::ReassemblyDrop::Conflict => "reasm_conflict", + tunnet_core::reassembly::ReassemblyDrop::OverBytes => "reasm_bytes", + tunnet_core::reassembly::ReassemblyDrop::TooManyEntries => "reasm_entries", + _ => "reasm_malformed", + }); + return false; + } + } + } + }; + // Anti-spoof against the connection's stable identity (exact match). + let Some(src) = logical.meta.src_v4 else { + metrics.dropped_inc("ipv6_unsupported_in"); + return false; + }; + let ident: Arc = fast.identity.read().clone(); + if !source_matches_peer(src, ident.ip) { + metrics.dropped_inc("antispoof"); + if let Some(tracker) = spoofs.get(&ident.network_id) + && tracker.record(&ident.endpoint_hex) + { + for (peer, n) in tracker.drain_window_counts() { + tracing::warn!( + peer = %peer, + spoofed_packets = n, + "ingress anti-spoof drops in last window" + ); } + } + return false; + } + // Snapshot the policy slot (guards are not Send; Arcs are). check() + // loads the firewall snapshot after the ACL snapshot inside, matching + // publish order — always current, no relink, no tear (§2.2-2). + let slot = fast.policy.load(); + let verdict = runtime.check( + &logical.meta, + Direction::Inbound, + &ident.endpoint_hex, + &ident.tags, + Some(ident.hostname.as_str()), + Some(ident.network_id), + &slot, + &slot.counters, + ); + match verdict { + PolicyVerdict::Allow => {} + PolicyVerdict::Deny => { + metrics.dropped_inc("policy_deny_in"); + return false; + } + PolicyVerdict::Reject => { + metrics.dropped_inc("fw_reject_in"); + let reply = packet::parse(logical.owner.as_bytes()) + .ok() + .and_then(|p| packet::synthesize_reject(&p)); + if let Some(reply) = reply.filter(|r| !r.is_empty()) { + // The frame already told us the bound network (used for + // resolve above); the reply carries the same binding. + let net = match &frame { + tunnet_common::packet::Frame::Single { net, .. } => *net, + tunnet_common::packet::Frame::Segment { net, .. } => *net, + }; + send_reject_framed(reply, net, fast, pool, conn, pool_bufs, metrics).await; + } + return false; + } + } + // Inbound SSH-NAT consumes parsed metadata (no second parse); shared + // storage materializes only when a rewrite actually applies. + let mut logical = logical; + if ssh_nat::needs_inbound_rewrite_with_meta(&logical.meta, self_ip) { + if !logical.materialize(pool_bufs) { + metrics.dropped_inc("nat_materialize"); + return false; + } + // PacketMeta is Copy: snapshot before the mutable borrow. + let meta = logical.meta; + let Some(region) = packet_owner_bytes_mut(&mut logical, pool_bufs) else { + metrics.dropped_inc("nat_materialize"); + return false; + }; + ssh_nat::rewrite_inbound_with_meta(region, &meta, self_ip); + } + let n = logical.len() as u64; + stage_tun_packet(tun_batch, logical, metrics); + fast.transport.record_rx(n); + metrics.packets_inc("in"); + metrics.bytes_add("in", n); + true +} - let n = dg.len() as u64; - let self_ip = acl.self_id.load().ip; - // Generation already verified: device + token belong to the - // generation loaded at reader start. Recheck cancellation - // (not a lock) before the send so BringDown wins races. - if generation_cancel.is_cancelled() { - break; +#[cfg(target_os = "linux")] +type TunBatchForPlatform = tun_fast::LinuxTunBatchWriter; +#[cfg(not(target_os = "linux"))] +type TunBatchForPlatform = tun_fast::TunWriteBatch; + +/// Flush the staged TUN batch. Returns false when the reader should stop +/// (device error); on temporary backpressure the tail stays staged for the +/// next iteration — never silently dropped. +async fn flush_tun_batch( + batch: &mut TunBatchForPlatform, + device: &Arc, + metrics: &AgentMetrics, +) -> bool { + #[cfg(target_os = "linux")] + { + if batch.is_empty() { + return true; + } + metrics.tun_syscall_inc("send_batch"); + match batch.flush(device).await { + Ok(_) => true, + Err(e) => { + tracing::warn!(?e, "tun batch send failed"); + metrics.dropped_inc("tun_send_failed"); + false } - let send_result = if ssh_nat::needs_inbound_rewrite(&dg, self_ip) { - let mut packet = dg.to_vec(); - let _ = ssh_nat::rewrite_inbound(&mut packet, self_ip); - device.send(&packet).await - } else { - device.send(dg.as_ref()).await - }; - if let Err(e) = send_result { - tracing::warn!(?e, "tun send failed"); + } + } + #[cfg(not(target_os = "linux"))] + { + if batch.is_empty() { + return true; + } + metrics.tun_syscall_inc("send_burst"); + match batch.drain_or_wait(device).await { + Ok(_) => true, + Err(e) => { + tracing::warn!(?e, "tun burst send failed"); metrics.dropped_inc("tun_send_failed"); - break; + false } - metrics.packets_inc("in"); - metrics.bytes_add("in", n); - if let Some(p) = &pool { - p.record_bytes_in(remote_id, n); + } + } +} + +#[cfg(not(target_os = "linux"))] +fn stage_tun_packet( + batch: &mut TunBatchForPlatform, + packet: LogicalPacket, + _metrics: &AgentMetrics, +) { + let bytes = match packet.owner { + tunnet_common::packet::PacketOwner::Shared(b) => b, + tunnet_common::packet::PacketOwner::Pooled(b) => Bytes::from_owner(b), + }; + batch.push(bytes); +} + +#[cfg(target_os = "linux")] +fn stage_tun_packet( + batch: &mut TunBatchForPlatform, + packet: LogicalPacket, + _metrics: &AgentMetrics, +) { + batch.push(packet.owner.as_bytes()); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::actors::test_support::test_metrics; + use crate::pump::ensure_pump; + + const NET_A: Uuid = Uuid::from_u128(0x0a0a); + const NET_B: Uuid = Uuid::from_u128(0x0b0b); + + fn two_net_routes() -> (RoutingTable, iroh::EndpointId, String) { + use tunnet_common::{DnsConfig, PeerEntry}; + let table = RoutingTable::new(); + let self_id = "a".repeat(64); + let ep_hex = "b".repeat(64); + let mk = |ip: &str| PeerEntry { + ip: ip.parse().unwrap(), + endpoint_id: ep_hex.clone(), + hostname: "gw".into(), + tags: vec![], + ssh_host_key: None, + }; + table.replace_network( + NET_A, + 0, + std::slice::from_ref(&mk("10.7.0.5")), + &DnsConfig::default(), + "neta", + &self_id, + 1, + ); + table.replace_network( + NET_B, + 1, + std::slice::from_ref(&mk("10.7.1.5")), + &DnsConfig::default(), + "netb", + &self_id, + 2, + ); + let ep: iroh::EndpointId = ep_hex.parse().unwrap(); + (table, ep, ep_hex) + } + + #[test] + fn resolve_binds_exact_membership_and_auth() { + // §2.2-1 (tests 5, 6): frames resolve to the EXACT (endpoint, + // network) membership — never the other network's state — and an + // endpoint authenticated only for A cannot claim B. + let (table, ep, ep_hex) = two_net_routes(); + let registry = table.peer_registry().clone(); + let auth = AuthCache::new(); + auth.insert(ep_hex.clone(), NET_A); + // Bound to A: A's membership (its own IP/identity). + let a = resolve_membership(®istry, &table, &ep, &ep_hex, NET_A, Some(&auth)) + .expect("A resolves"); + assert_eq!(a.identity.read().network_id, NET_A); + assert_eq!(a.identity.read().ip, std::net::Ipv4Addr::new(10, 7, 0, 5)); + // Bound to B without B-auth: rejected (no cross evaluation). + assert!( + resolve_membership(®istry, &table, &ep, &ep_hex, NET_B, Some(&auth)).is_none(), + "endpoint authed only for A must not claim B" + ); + // With B-auth: B's own membership, distinct object and IP. + auth.insert(ep_hex.clone(), NET_B); + let b = resolve_membership(®istry, &table, &ep, &ep_hex, NET_B, Some(&auth)) + .expect("B resolves once authed"); + assert!(!Arc::ptr_eq(&a, &b)); + assert_eq!(b.identity.read().ip, std::net::Ipv4Addr::new(10, 7, 1, 5)); + // Unknown network: rejected even without an auth cache + // (membership existence gates). + assert!( + resolve_membership(®istry, &table, &ep, &ep_hex, Uuid::from_u128(0xcc), None) + .is_none() + ); + } + + /// End-to-end loopback ping (total-loss diagnosis loop): machine A + /// (10.7.0.1) sends a real ICMP echo to machine B (10.7.0.2) and back + /// over loopback QUIC through the REAL outbound policy + scheduler + + /// pump, REAL datagram transport, and the REAL inbound handler + /// (decode → membership → antispoof → policy → stage, no TUN device). + /// Times out (RED) exactly when user ping would: no frames arrive, or + /// the inbound path drops everything. + struct Loopback { + conn_a: iroh::endpoint::Connection, + conn_b: iroh::endpoint::Connection, + reg_a: PeerRegistry, + reg_b: PeerRegistry, + rt_a: PolicyRuntime, + rt_b: PolicyRuntime, + pool_a: ConnPool, + pool_b: ConnPool, + id_a: iroh::EndpointId, + id_b: iroh::EndpointId, + hex_a: String, + hex_b: String, + net: Uuid, + } + + async fn loopback_fixture() -> Loopback { + use iroh::endpoint::presets::N0; + use tunnet_common::TUNNEL_ALPN; + let alpn = TUNNEL_ALPN; + let ep_a = iroh::Endpoint::builder(N0) + .alpns(vec![alpn.to_vec()]) + .relay_mode(iroh::RelayMode::Disabled) + .bind() + .await + .unwrap(); + let ep_b = iroh::Endpoint::builder(N0) + .alpns(vec![alpn.to_vec()]) + .relay_mode(iroh::RelayMode::Disabled) + .bind() + .await + .unwrap(); + let id_a = ep_a.id(); + let id_b = ep_b.id(); + let addr_b = ep_b.addr(); + let ep_b2 = ep_b.clone(); + let accept_b = tokio::spawn(async move { ep_b2.accept().await.unwrap().await.unwrap() }); + let conn_a = tokio::time::timeout( + std::time::Duration::from_secs(10), + ep_a.connect(addr_b, alpn), + ) + .await + .expect("dial A->B must succeed") + .unwrap(); + let conn_b = tokio::time::timeout(std::time::Duration::from_secs(10), accept_b) + .await + .expect("accept on B") + .unwrap(); + + let mk_self = |hex: String, ip: std::net::Ipv4Addr| tunnet_core::acl::SelfIdentity { + endpoint_hex: hex, + ip, + tags: vec![], + network: "net".into(), + }; + let rt_a = PolicyRuntime::bootstrap( + &Default::default(), + &Default::default(), + &mk_self(format!("{id_a}"), std::net::Ipv4Addr::new(10, 7, 0, 1)), + true, + false, + ); + let rt_b = PolicyRuntime::bootstrap( + &Default::default(), + &Default::default(), + &mk_self(format!("{id_b}"), std::net::Ipv4Addr::new(10, 7, 0, 2)), + true, + false, + ); + let net = Uuid::from_u128(0xE2E); + let reg_a = PeerRegistry::new(); + let reg_b = PeerRegistry::new(); + // A knows B (10.7.0.2), B knows A (10.7.0.1), same network. + reg_a.ensure_membership(Arc::new(PeerIdentity { + endpoint: id_b, + endpoint_hex: format!("{id_b}"), + hostname: "b".into(), + ip: std::net::Ipv4Addr::new(10, 7, 0, 2), + tags: vec![], + network_id: net, + network_name: "net".into(), + })); + reg_b.ensure_membership(Arc::new(PeerIdentity { + endpoint: id_a, + endpoint_hex: format!("{id_a}"), + hostname: "a".into(), + ip: std::net::Ipv4Addr::new(10, 7, 0, 1), + tags: vec![], + network_id: net, + network_name: "net".into(), + })); + reg_a.relink_policy(&rt_a); + reg_b.relink_policy(&rt_b); + // Mirror live conns into transports (slow path, as the pool does). + reg_a.set_transport_conn(id_b, Some(conn_a.clone())); + reg_b.set_transport_conn(id_a, Some(conn_b.clone())); + let pool_a = ConnPool::new(ep_a, alpn); + let pool_b = ConnPool::new(ep_b, alpn); + Loopback { + conn_a, + conn_b, + reg_a, + reg_b, + rt_a, + rt_b, + pool_a, + pool_b, + id_a, + id_b, + hex_a: format!("{id_a}"), + hex_b: format!("{id_b}"), + net, + } + } + + fn icmp_echo(src: [u8; 4], dst: [u8; 4]) -> Vec { + let b = etherparse::PacketBuilder::ipv4(src, dst, 64).icmpv4_echo_request(7, 1); + let mut o = Vec::new(); + b.write(&mut o, &[0xABu8; 32]).unwrap(); + o + } + + /// One directed leg: outbound policy + scheduler + real pump on the + /// sender; real QUIC datagrams; decode + membership + full inbound + /// handler on the receiver. Returns the staged TUN payload. + struct Leg<'a> { + tx_reg: &'a PeerRegistry, + tx_rt: &'a PolicyRuntime, + tx_pool: &'a ConnPool, + tx_peer: iroh::EndpointId, + tx_hex: &'a str, + tx_host: &'a str, + rx_conn: &'a iroh::endpoint::Connection, + rx_reg: &'a PeerRegistry, + rx_rt: &'a PolicyRuntime, + rx_self_ip: std::net::Ipv4Addr, + net: Uuid, + raw: Vec, + } + + async fn directed_leg(leg: Leg<'_>) -> Vec { + use tunnet_common::packet::PacketPool; + let Leg { + tx_reg, + tx_rt, + tx_pool, + tx_peer, + tx_hex, + tx_host, + rx_conn, + rx_reg, + rx_rt, + rx_self_ip, + net, + raw, + } = leg; + // Outbound policy through the sender's membership slot. + let member = tx_reg + .get_membership(tx_peer, net) + .expect("sender membership"); + let pkt = LogicalPacket::from_vec(raw.clone()).expect("valid test packet"); + let slot = member.policy.load(); + assert_eq!( + tx_rt.check( + &pkt.meta, + Direction::Outbound, + tx_hex, + &[], + Some(tx_host), + Some(net), + &slot, + &slot.counters + ), + PolicyVerdict::Allow, + "outbound policy must allow the echo" + ); + // Scheduler + REAL pump (exits idle on its own). + assert!( + member + .scheduler + .lock() + .enqueue(pkt, std::time::Instant::now()) + .is_accepted() + ); + ensure_pump( + &member, + tx_pool.clone(), + test_metrics(), + PacketPool::new(8), + tunnet_core::CloudRelayMeter::new(), + ); + // Receiver: real datagrams until the full logical packet stages. + let bufs = PacketPool::new(8); + let metrics = test_metrics(); + #[cfg(target_os = "linux")] + let mut batch = tun_fast::LinuxTunBatchWriter::new(); + #[cfg(not(target_os = "linux"))] + let mut batch = tun_fast::TunWriteBatch::new(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + assert!( + std::time::Instant::now() < deadline, + "receiver got nothing within 10s (ping would time out)" + ); + let dg = tokio::time::timeout_at(deadline.into(), rx_conn.read_datagram()) + .await + .expect("frame must arrive") + .unwrap(); + let frame = tunnet_common::packet::decode_frame(&dg).expect("must decode"); + let got_net = match &frame { + tunnet_common::packet::Frame::Single { net, .. } => *net, + tunnet_common::packet::Frame::Segment { net, .. } => *net, + }; + assert_eq!(got_net, net, "frame bound to the membership network"); + let rx_member = rx_reg + .get_membership(rx_conn.remote_id(), got_net) + .expect("receiver must resolve the membership"); + let staged = handle_inbound_one( + &dg, + frame, + &rx_member, + rx_rt, + &std::collections::HashMap::new(), + rx_conn, + None, + &bufs, + &metrics, + &mut batch, + rx_self_ip, + ) + .await; + if staged { + assert!(!batch.is_empty()); + return raw; } + // Segmented packet: keep draining until completion. } } - metrics.active_conns_dec(); - tracing::info!(%remote_id, "peer disconnected"); + + #[tokio::test] + async fn loopback_ping_round_trip() { + let fx = loopback_fixture().await; + let raw_ab = icmp_echo([10, 7, 0, 1], [10, 7, 0, 2]); + // Verify the staged payload equals what was sent (single-frame + // path stages the exact bytes). + let _ = directed_leg(Leg { + tx_reg: &fx.reg_a, + tx_rt: &fx.rt_a, + tx_pool: &fx.pool_a, + tx_peer: fx.id_b, + tx_hex: &fx.hex_b, + tx_host: "b", + rx_conn: &fx.conn_b, + rx_reg: &fx.reg_b, + rx_rt: &fx.rt_b, + rx_self_ip: std::net::Ipv4Addr::new(10, 7, 0, 2), + net: fx.net, + raw: raw_ab, + }) + .await; + // Reply direction (ping needs both ways). + let raw_ba = icmp_echo([10, 7, 0, 2], [10, 7, 0, 1]); + let _ = directed_leg(Leg { + tx_reg: &fx.reg_b, + tx_rt: &fx.rt_b, + tx_pool: &fx.pool_b, + tx_peer: fx.id_a, + tx_hex: &fx.hex_a, + tx_host: "a", + rx_conn: &fx.conn_a, + rx_reg: &fx.reg_a, + rx_rt: &fx.rt_a, + rx_self_ip: std::net::Ipv4Addr::new(10, 7, 0, 1), + net: fx.net, + raw: raw_ba, + }) + .await; + } } diff --git a/crates/tunnet-cli/src/cmds.rs b/crates/tunnet-cli/src/cmds.rs index ae165cb3..5ed53870 100644 --- a/crates/tunnet-cli/src/cmds.rs +++ b/crates/tunnet-cli/src/cmds.rs @@ -472,14 +472,42 @@ fn print_status( out.dim(&format!("· {primary_name}")) )); out.writeln(format!(" mode {mode}")); - out.writeln(format!( - " data plane {}", - if node.data_plane_up { - out.green("up") - } else { - out.dim("down") - } - )); + // Dataplane health: never a bare "up" when the packet worker is dead. + // Prefer the detailed state; fall back to the legacy boolean for old + // daemons that do not report it yet. + let dp_line = match node.data_plane.as_ref() { + Some(dp) => match dp.state.as_str() { + "up" => format!(" data plane {}", out.green("up")), + "degraded" => format!( + " data plane {} (outbound worker dead, restarts: {}){}", + out.yellow("degraded"), + dp.restart_count, + dp.last_error + .as_ref() + .map(|e| format!(" · {}", out.dim(e))) + .unwrap_or_default() + ), + "restarting" => format!( + " data plane {} (restarts: {}){}", + out.yellow("restarting"), + dp.restart_count, + dp.last_error + .as_ref() + .map(|e| format!(" · {}", out.dim(e))) + .unwrap_or_default() + ), + other => format!(" data plane {}", out.dim(other)), + }, + None => format!( + " data plane {}", + if node.data_plane_up { + out.green("up") + } else { + out.dim("down") + } + ), + }; + out.writeln(dp_line); out.writeln(format!( " endpoint {}", out.dim(&output::short_endpoint(&node.endpoint_id)) @@ -501,6 +529,25 @@ fn print_status( " uptime {} · daemon v{} · snap {}", uptime, node.daemon_version, node.snapshot_version )); + // Build identity: a bare version (v0.9.1) cannot distinguish + // protocol-breaking pre-1.0 commits. Show both sides' git hashes and + // warn loudly on mismatch (the classic stale-daemon trap: fresh CLI + // talking to an old service binary). + let cli_git = tunnet_common::git_hash(); + let mut build_line = format!(" build cli {cli_git}"); + if let Some(dg) = node.daemon_git.as_deref() { + build_line.push_str(&format!(" · daemon {dg}")); + if dg != "unknown" && cli_git != "unknown" && dg != cli_git { + build_line.push_str(&format!( + " · {}", + out.red("MISMATCH: daemon and CLI built from different commits — restart/redeploy the daemon") + )); + } + } + if let Some(alpn) = node.tunnel_alpn.as_deref() { + build_line.push_str(&format!(" · {alpn}")); + } + out.writeln(build_line); if let Some(od) = &node.on_demand { out.writeln(format!( diff --git a/crates/tunnet-common/build.rs b/crates/tunnet-common/build.rs new file mode 100644 index 00000000..c7da1e24 --- /dev/null +++ b/crates/tunnet-common/build.rs @@ -0,0 +1,16 @@ +//! Bake the git commit hash into the build for `tunnet status` reporting. +//! Lets operators (and the CLI's own mismatch warning) see exactly which +//! commit each binary was built from — version strings alone (`v0.9.1`) +//! cannot distinguish protocol-breaking changes during pre-1.0 development. + +fn main() { + let hash = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".to_string()); + println!("cargo:rustc-env=GIT_HASH={hash}"); +} diff --git a/crates/tunnet-common/proptest-regressions/packet/frame.txt b/crates/tunnet-common/proptest-regressions/packet/frame.txt new file mode 100644 index 00000000..05434b0d --- /dev/null +++ b/crates/tunnet-common/proptest-regressions/packet/frame.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc b97fffc66d68d5511cbe773665cc1a00ce3484c19f2125ed2ea0b2bb49474673 # shrinks to id = 0, index = 0, count = 2, total = 1 diff --git a/crates/tunnet-common/src/agent_policy.rs b/crates/tunnet-common/src/agent_policy.rs index 045ceadf..abb04c3c 100644 --- a/crates/tunnet-common/src/agent_policy.rs +++ b/crates/tunnet-common/src/agent_policy.rs @@ -230,7 +230,11 @@ pub struct EffectiveAgentConfig { const DEFAULT_MDNS: bool = true; const DEFAULT_LAN_DISCOVERY: bool = true; -const DEFAULT_TUNNEL_MTU: u16 = 1280; +/// Default logical/virtual MTU for the dataplane. 2800 matches the +/// ZeroTier-style comparison point: far fewer packets per GiB than 1280 while +/// staying cheap to segment onto typical QUIC path MTUs. See the benchmark +/// matrix before changing this; 9000 is not an automatic default. +const DEFAULT_TUNNEL_MTU: u16 = 2800; const DEFAULT_AUTO_UPDATE_ENABLED: bool = false; const DEFAULT_AUTO_UPDATE_INTERVAL: u64 = 6; const DEFAULT_POSTURE_INTERVAL: u64 = 300; diff --git a/crates/tunnet-common/src/lib.rs b/crates/tunnet-common/src/lib.rs index cff12a7c..37f89659 100644 --- a/crates/tunnet-common/src/lib.rs +++ b/crates/tunnet-common/src/lib.rs @@ -24,8 +24,22 @@ use uuid::Uuid; pub type EndpointIdHex = String; -/// ALPN identifier for our tunnel protocol (mesh datagrams). -pub const TUNNEL_ALPN: &[u8] = b"tunnet/tunnel/1"; +/// Short git hash baked at compile time (build.rs), `"unknown"` when git +/// was unavailable. Rendered by `tunnet status` for both the CLI and the +/// daemon so stale-binary mismatches are visible instead of debuggable +/// for hours. +pub fn git_hash() -> &'static str { + option_env!("GIT_HASH").unwrap_or("unknown") +} + +/// ALPN identifier for the tunnel protocol (mesh datagrams). +/// Tunnel framing is the only wire format (segmented logical packets, every +/// frame bound to one network). The `/3` is only the negotiated +/// wire-protocol version: it keeps older binaries (raw-IP `/1`, undisclosed +/// `/2` framing) from accidentally speaking an incompatible format. It does +/// not imply any older implementation remains — there is none, and there is +/// no compatibility decoder. +pub const TUNNEL_ALPN: &[u8] = b"tunnet/tunnel/3"; /// ALPN for agent ↔ public edge reverse tunnels. pub const EDGE_ALPN: &[u8] = b"tunnet/edge/1"; diff --git a/crates/tunnet-common/src/local_api.rs b/crates/tunnet-common/src/local_api.rs index 0b4a38ed..75fcfdba 100644 --- a/crates/tunnet-common/src/local_api.rs +++ b/crates/tunnet-common/src/local_api.rs @@ -117,7 +117,7 @@ impl Default for LocalUiPolicy { } // --------------------------------------------------------------------------- -// Node / network summary (v2 status model) +// Node / network summary (status model) // --------------------------------------------------------------------------- #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -211,6 +211,30 @@ pub struct NodeSummary { pub on_demand: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub control: Option, + /// Short git hash the DAEMON binary was built from ("unknown" when + /// unavailable). Compared against the CLI's own hash to catch + /// stale-daemon traps (fresh CLI, old service binary). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub daemon_git: Option, + /// Tunnel ALPN the daemon speaks (e.g. `tunnet/tunnel/3`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tunnel_alpn: Option, + /// Dataplane health detail (never "up" with a dead packet worker). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_plane: Option, +} + +/// Dataplane health detail for `tunnet status`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct DataPlaneInfo { + /// `up` | `degraded` | `restarting` | `down`. + pub state: String, + pub outbound_alive: bool, + pub restart_count: u64, + pub generation: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/tunnet-common/src/packet/frag.rs b/crates/tunnet-common/src/packet/frag.rs index c924d61c..47ba8de8 100644 --- a/crates/tunnet-common/src/packet/frag.rs +++ b/crates/tunnet-common/src/packet/frag.rs @@ -237,6 +237,35 @@ impl FragmentTable { self.map.clear(); } + /// Metadata-based lookup for the unified policy fast path (no reparse). + pub fn lookup_cached(&mut self, key: &FragKey) -> Option { + self.evict_expired(); + let entry = self.map.get(key)?; + if Instant::now() >= entry.expires { + self.map.remove(key); + return None; + } + Some(entry.transport.to_resolved()) + } + + /// Metadata-based insert for first fragments (no reparse). + pub fn insert_cached(&mut self, key: FragKey, transport: CachedTransport) { + self.evict_expired(); + if self.map.len() >= self.cap && !self.map.contains_key(&key) { + self.evict_oldest(); + } + if self.map.len() >= self.cap && !self.map.contains_key(&key) { + return; + } + self.map.insert( + key, + Entry { + transport, + expires: Instant::now() + self.ttl, + }, + ); + } + pub fn len(&self) -> usize { self.map.len() } diff --git a/crates/tunnet-common/src/packet/frame.rs b/crates/tunnet-common/src/packet/frame.rs new file mode 100644 index 00000000..26e02077 --- /dev/null +++ b/crates/tunnet-common/src/packet/frame.rs @@ -0,0 +1,519 @@ +//! Tunnel framing: the only tunnel wire format. +//! +//! A logical (inner IP) packet that fits the current QUIC DATAGRAM payload +//! limit travels as one `Single` frame. Larger logical packets are split into +//! `Segment` frames and reassembled by the peer. Endpoints negotiate +//! `tunnet/tunnel/3` and reject anything else; there is no legacy decoder. +//! +//! Every frame carries the full `NetworkId` it belongs to (§2.2-1): the +//! receiver binds each logical packet to exactly one authenticated network +//! membership, even for out-of-order segments, with no negotiation +//! handshake and no per-connection channel tables. 16 bytes per frame is +//! the deliberate price for unambiguous binding. +//! +//! Layout (all integers little-endian, minimal overhead): +//! +//! ```text +//! Single: [0x30][net_id 16B][logical packet bytes...] (17 bytes overhead) +//! Segment: [0x31][net_id 16B][id u32][index u16][count u16][total u16][payload] +//! (27 bytes overhead) +//! ``` +//! +//! Kinds `0x32..=0x3F` are reserved for future extensions (e.g. GSO-aware +//! frames); the version nibble `0x3_` leaves `0x4_`.. for future wire versions. +//! The old `0x20`/`0x21` kinds decode as `UnknownKind` (fail fast on wire +//! mismatch, never misparse). Decoder properties: fixed/cheap header parse, +//! no allocation to decode a header, malformed frames rejected before +//! allocation, no integer overflow (checked arithmetic throughout), no +//! ambiguous encodings, deterministic encoding, fuzzable decoder. + +use super::owned::MAX_LOGICAL_LEN; +use uuid::Uuid; + +/// Maximum segments per logical packet (9000 B / ~1200 B MPS ≈ 8; headroom 2×). +pub const MAX_SEGMENTS: usize = 16; +/// Minimum useful segment payload (pathological tiny segments rejected). +pub const MIN_SEGMENT_PAYLOAD: usize = 64; +/// Full NetworkId discriminator in every frame (§2.2-1). +pub const NET_ID_LEN: usize = 16; + +pub const KIND_SINGLE: u8 = 0x30; +pub const KIND_SEGMENT: u8 = 0x31; +const KIND_RESERVED_MAX: u8 = 0x3F; +const KIND_NIBBLE: u8 = 0x30; +const SEG_HEADER_LEN: usize = 1 + NET_ID_LEN + 4 + 2 + 2 + 2; +const SINGLE_HEADER_LEN: usize = 1 + NET_ID_LEN; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SegmentHeader { + pub id: u32, + pub index: u16, + pub count: u16, + pub total: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Frame<'a> { + Single { + net: Uuid, + payload: &'a [u8], + }, + Segment { + net: Uuid, + header: SegmentHeader, + payload: &'a [u8], + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecodeError { + Empty, + UnknownKind(u8), + ReservedKind(u8), + TruncatedHeader, + TruncatedPayload, + EmptyPayload, + BadCount, + BadIndex, + BadTotal, + SingleSegment, + OversizeSegment, +} + +impl std::fmt::Display for DecodeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for DecodeError {} + +/// Decode a tunnel frame header + payload borrows. No allocation. +/// Returns the bound network alongside the frame: the caller MUST resolve +/// the (endpoint, network) membership and authenticate it — never deliver +/// a frame to whichever membership was resolved last. +pub fn decode_frame(data: &[u8]) -> Result, DecodeError> { + let Some((&kind, rest)) = data.split_first() else { + return Err(DecodeError::Empty); + }; + match kind { + KIND_SINGLE => { + if rest.len() < NET_ID_LEN { + return Err(DecodeError::TruncatedHeader); + } + let net = Uuid::from_bytes(rest[..NET_ID_LEN].try_into().expect("len")); + let payload = &rest[NET_ID_LEN..]; + if payload.is_empty() { + return Err(DecodeError::EmptyPayload); + } + if payload.len() > MAX_LOGICAL_LEN { + return Err(DecodeError::BadTotal); + } + Ok(Frame::Single { net, payload }) + } + KIND_SEGMENT => { + if rest.len() < SEG_HEADER_LEN - 1 { + return Err(DecodeError::TruncatedHeader); + } + let net = Uuid::from_bytes(rest[..NET_ID_LEN].try_into().expect("len")); + let h = &rest[NET_ID_LEN..]; + let id = u32::from_le_bytes(h[0..4].try_into().expect("len")); + let index = u16::from_le_bytes(h[4..6].try_into().expect("len")); + let count = u16::from_le_bytes(h[6..8].try_into().expect("len")); + let total = u16::from_le_bytes(h[8..10].try_into().expect("len")); + let payload = &h[10..]; + if count < 2 || (count as usize) > MAX_SEGMENTS { + return Err(DecodeError::BadCount); + } + if index >= count { + return Err(DecodeError::BadIndex); + } + if total == 0 || (total as usize) > MAX_LOGICAL_LEN { + return Err(DecodeError::BadTotal); + } + if payload.is_empty() { + return Err(DecodeError::EmptyPayload); + } + // Last segment may be short; non-last segments must carry a + // meaningful payload (prevents index/count smuggling games). + if index + 1 < count && payload.len() < MIN_SEGMENT_PAYLOAD { + return Err(DecodeError::TruncatedPayload); + } + // A segment can never carry more than the whole logical packet. + // (Non-last segments are sized by the sender's path MPS, which + // the decoder cannot know; the reassembly layer additionally + // caps count × total per peer, so no allocation amplification.) + if payload.len() > total as usize { + return Err(DecodeError::OversizeSegment); + } + if payload.len() > MAX_LOGICAL_LEN { + return Err(DecodeError::OversizeSegment); + } + Ok(Frame::Segment { + net, + header: SegmentHeader { + id, + index, + count, + total, + }, + payload, + }) + } + k if (k & 0xF0) == KIND_NIBBLE && k <= KIND_RESERVED_MAX => { + Err(DecodeError::ReservedKind(k)) + } + k => Err(DecodeError::UnknownKind(k)), + } +} + +/// Encode a single frame header ([kind][net]) into `out[..17]`. Returns 17. +pub fn encode_single_prefix(out: &mut [u8], net: Uuid) -> usize { + out[0] = KIND_SINGLE; + out[1..SINGLE_HEADER_LEN].copy_from_slice(net.as_bytes()); + SINGLE_HEADER_LEN +} + +/// Encode a 27-byte segment header into `out[..27]`. Returns 27. +pub fn encode_segment_prefix(out: &mut [u8], net: Uuid, h: SegmentHeader) -> usize { + out[0] = KIND_SEGMENT; + out[1..1 + NET_ID_LEN].copy_from_slice(net.as_bytes()); + let b = 1 + NET_ID_LEN; + out[b..b + 4].copy_from_slice(&h.id.to_le_bytes()); + out[b + 4..b + 6].copy_from_slice(&h.index.to_le_bytes()); + out[b + 6..b + 8].copy_from_slice(&h.count.to_le_bytes()); + out[b + 8..b + 10].copy_from_slice(&h.total.to_le_bytes()); + SEG_HEADER_LEN +} + +pub const SINGLE_OVERHEAD: usize = SINGLE_HEADER_LEN; +pub const SEGMENT_OVERHEAD: usize = SEG_HEADER_LEN; + +/// Number of segments needed for `logical_len` bytes at `mps` payload bytes +/// per DATAGRAM (accounting framing overhead). None when impossible. +pub fn segment_count(logical_len: usize, mps: usize) -> Option { + if logical_len == 0 || logical_len > MAX_LOGICAL_LEN { + return None; + } + let single_cap = mps.checked_sub(SINGLE_OVERHEAD)?; + if logical_len <= single_cap { + return Some(1); + } + let seg_cap = mps.checked_sub(SEGMENT_OVERHEAD)?; + if seg_cap < MIN_SEGMENT_PAYLOAD { + return None; + } + Some(logical_len.div_ceil(seg_cap)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NET_A: Uuid = Uuid::from_u128(0x0a0a); + const NET_B: Uuid = Uuid::from_u128(0x0b0b); + + #[test] + fn single_round_trip() { + let mut buf = [0u8; 64]; + assert_eq!(encode_single_prefix(&mut buf, NET_A), SINGLE_HEADER_LEN); + buf[SINGLE_HEADER_LEN..SINGLE_HEADER_LEN + 5].copy_from_slice(b"hello"); + match decode_frame(&buf[..SINGLE_HEADER_LEN + 5]).unwrap() { + Frame::Single { net, payload } => { + assert_eq!(net, NET_A); + assert_eq!(payload, b"hello"); + } + _ => panic!("expected single"), + } + } + + #[test] + fn segment_round_trip() { + let mut buf = [0u8; 1024]; + let h = SegmentHeader { + id: 0xdead_beef, + index: 2, + count: 5, + total: 4000, + }; + assert_eq!(encode_segment_prefix(&mut buf, NET_A, h), SEG_HEADER_LEN); + // Non-last segments carry full payloads. + buf[SEG_HEADER_LEN..SEG_HEADER_LEN + 800].fill(0xAB); + match decode_frame(&buf[..SEG_HEADER_LEN + 800]).unwrap() { + Frame::Segment { + net, + header: got, + payload: p, + } => { + assert_eq!(net, NET_A); + assert_eq!(got, h); + assert_eq!(p.len(), 800); + } + _ => panic!("expected segment"), + } + // Last segment may be short. + let last = SegmentHeader { index: 4, ..h }; + assert_eq!(encode_segment_prefix(&mut buf, NET_B, last), SEG_HEADER_LEN); + buf[SEG_HEADER_LEN..SEG_HEADER_LEN + 5].copy_from_slice(b"world"); + match decode_frame(&buf[..SEG_HEADER_LEN + 5]).unwrap() { + Frame::Segment { + net, + header: got, + payload: p, + } => { + assert_eq!(net, NET_B); + assert_eq!(got, last); + assert_eq!(p, b"world"); + } + _ => panic!("expected segment"), + } + } + + #[test] + fn old_wire_kinds_rejected() { + // Pre-multinetwork kinds fail fast as unknown (never misparsed). + assert_eq!( + decode_frame(&[0x20, 1, 2, 3]), + Err(DecodeError::UnknownKind(0x20)) + ); + assert_eq!( + decode_frame(&[0x21, 1, 2, 3]), + Err(DecodeError::UnknownKind(0x21)) + ); + } + + #[test] + fn rejects_garbage_before_allocation() { + assert_eq!(decode_frame(&[]), Err(DecodeError::Empty)); + assert_eq!(decode_frame(&[0x99]), Err(DecodeError::UnknownKind(0x99))); + assert_eq!(decode_frame(&[0x32]), Err(DecodeError::ReservedKind(0x32))); + assert_eq!( + decode_frame(&[KIND_SINGLE]), + Err(DecodeError::TruncatedHeader) + ); + // Kind + full net but no payload. + let mut single = vec![KIND_SINGLE]; + single.extend_from_slice(NET_A.as_bytes()); + assert_eq!(decode_frame(&single), Err(DecodeError::EmptyPayload)); + assert_eq!( + decode_frame(&[KIND_SEGMENT]), + Err(DecodeError::TruncatedHeader) + ); + assert_eq!( + decode_frame(&[KIND_SEGMENT, 1, 2, 3]), + Err(DecodeError::TruncatedHeader) + ); + // count < 2 + let mut bad = [0u8; 32]; + encode_segment_prefix( + &mut bad, + NET_A, + SegmentHeader { + id: 1, + index: 0, + count: 1, + total: 100, + }, + ); + assert_eq!(decode_frame(&bad[..32]), Err(DecodeError::BadCount)); + // count > MAX + encode_segment_prefix( + &mut bad, + NET_A, + SegmentHeader { + id: 1, + index: 0, + count: 99, + total: 100, + }, + ); + assert_eq!(decode_frame(&bad[..32]), Err(DecodeError::BadCount)); + // index >= count + encode_segment_prefix( + &mut bad, + NET_A, + SegmentHeader { + id: 1, + index: 3, + count: 3, + total: 300, + }, + ); + bad[SEG_HEADER_LEN] = 7; + assert_eq!( + decode_frame(&bad[..SEG_HEADER_LEN + 1]), + Err(DecodeError::BadIndex) + ); + } + + #[test] + fn rejects_bad_totals_and_tiny_segments() { + let mut buf = [0u8; 96]; + // total 0 + encode_segment_prefix( + &mut buf, + NET_A, + SegmentHeader { + id: 1, + index: 0, + count: 2, + total: 0, + }, + ); + buf[SEG_HEADER_LEN] = 1; + assert_eq!( + decode_frame(&buf[..SEG_HEADER_LEN + 1]), + Err(DecodeError::BadTotal) + ); + // total > max + encode_segment_prefix( + &mut buf, + NET_A, + SegmentHeader { + id: 1, + index: 0, + count: 2, + total: 9001, + }, + ); + assert_eq!( + decode_frame(&buf[..SEG_HEADER_LEN + 1]), + Err(DecodeError::BadTotal) + ); + // non-last tiny payload + encode_segment_prefix( + &mut buf, + NET_A, + SegmentHeader { + id: 1, + index: 0, + count: 3, + total: 3000, + }, + ); + buf[SEG_HEADER_LEN] = 1; + assert_eq!( + decode_frame(&buf[..SEG_HEADER_LEN + 1]), + Err(DecodeError::TruncatedPayload) + ); + // empty payload + encode_segment_prefix( + &mut buf, + NET_A, + SegmentHeader { + id: 1, + index: 2, + count: 3, + total: 3000, + }, + ); + assert_eq!( + decode_frame(&buf[..SEG_HEADER_LEN]), + Err(DecodeError::EmptyPayload) + ); + } + + #[test] + fn segment_count_boundaries() { + // exact fit → single (single overhead is now 17) + assert_eq!(segment_count(1184, 1201), Some(1)); + // one byte over → segmented + assert_eq!(segment_count(1185, 1201), Some(2)); + assert_eq!(segment_count(0, 1200), None); + assert_eq!(segment_count(9001, 1500), None); + // 2800 logical at 1350 MPS: seg cap 1323 → 3 segments + assert_eq!(segment_count(2800, 1350), Some(3)); + // degenerate MPS + assert_eq!(segment_count(100, 10), None); + } + + #[test] + fn deterministic_encoding() { + let h = SegmentHeader { + id: 7, + index: 1, + count: 4, + total: 5000, + }; + let mut a = [0u8; SEG_HEADER_LEN]; + let mut b = [0u8; SEG_HEADER_LEN]; + encode_segment_prefix(&mut a, NET_A, h); + encode_segment_prefix(&mut b, NET_A, h); + assert_eq!(a, b); + assert_eq!(a[0], KIND_SEGMENT); + // Network binding participates in the encoding. + encode_segment_prefix(&mut b, NET_B, h); + assert_ne!(a, b); + } +} + +#[cfg(test)] +mod proptests { + use super::*; + use proptest::prelude::*; + + proptest! { + /// Any byte string either decodes deterministically or fails with a + /// stable error; decoding never panics and never allocates. + #[test] + fn decode_never_panics(data in prop::collection::vec(any::(), 0..64)) { + let a = decode_frame(&data).map_err(|e| format!("{e:?}")); + let b = decode_frame(&data).map_err(|e| format!("{e:?}")); + prop_assert_eq!(a.as_ref().map(|_| ()), b.as_ref().map(|_| ())); + prop_assert_eq!(a.is_ok(), b.is_ok()); + if let Ok(frame) = a { + match frame { + Frame::Single { payload: p, .. } => { + prop_assert!(!p.is_empty() && p.len() <= MAX_LOGICAL_LEN); + } + Frame::Segment { header: h, payload: p, .. } => { + prop_assert!((h.count as usize) >= 2 && (h.count as usize) <= MAX_SEGMENTS); + prop_assert!(h.index < h.count); + prop_assert!(!p.is_empty() && p.len() <= h.total as usize); + } + } + } + } + + /// Encoded segment headers always decode to themselves. + #[test] + fn segment_header_round_trip( + id in any::(), + index in 0..16u16, + count in 2..16u16, + total in 1..9000u16, + ) { + // Non-last segments must carry full payloads, so only test + // totals that admit them (small totals are covered by unit tests). + prop_assume!((total as usize) >= (count as usize) * MIN_SEGMENT_PAYLOAD); + let index = index % count; + let net = Uuid::from_u128(id as u128); + let h = SegmentHeader { id, index, count, total }; + let mut buf = [0u8; SEG_HEADER_LEN]; + prop_assert_eq!(encode_segment_prefix(&mut buf, net, h), SEG_HEADER_LEN); + let payload_len = if index + 1 < count { + MIN_SEGMENT_PAYLOAD + } else { + 1usize + }; + let mut full = vec![0u8; SEG_HEADER_LEN + payload_len]; + full[..SEG_HEADER_LEN].copy_from_slice(&buf); + for (i, b) in full[SEG_HEADER_LEN..].iter_mut().enumerate() { + *b = (i & 0xff) as u8; + } + match decode_frame(&full) { + Ok(Frame::Segment { + net: got_net, + header: got, + payload: p, + }) => { + prop_assert_eq!(got_net, net); + prop_assert_eq!(got, h); + prop_assert_eq!(p.len(), payload_len); + } + Ok(Frame::Single { .. }) => prop_assert!(false, "segment must not decode as single"), + Err(e) => prop_assert!(false, "unexpected decode error: {e:?}"), + } + } + } +} diff --git a/crates/tunnet-common/src/packet/meta.rs b/crates/tunnet-common/src/packet/meta.rs new file mode 100644 index 00000000..46472abc --- /dev/null +++ b/crates/tunnet-common/src/packet/meta.rs @@ -0,0 +1,154 @@ +//! Parse-once packet metadata: flow keys and compact IP/transport summary. +//! +//! These types are `Copy` and allocation-free; the data plane parses each +//! logical packet once and carries the metadata alongside the bytes. + +use std::net::{IpAddr, Ipv4Addr}; + +use super::{Fragmentation, IpMeta, Packet, Transport}; + +/// Stable per-flow scheduling key. +/// +/// TCP/UDP: IP 5-tuple. ICMP: src/dst/proto + echo id (cheap isolation). +/// Other protocols without ports: src/dst/proto. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FlowKey { + pub src: IpAddr, + pub dst: IpAddr, + pub proto: u8, + pub sport: u16, + pub dport: u16, +} + +impl FlowKey { + pub fn for_packet(pkt: &Packet<'_>) -> Self { + let (src, dst) = match pkt.ip { + IpMeta::V4 { src, dst, .. } => (IpAddr::V4(src), IpAddr::V4(dst)), + IpMeta::V6 { src, dst, .. } => (IpAddr::V6(src), IpAddr::V6(dst)), + }; + let proto = pkt.ip.ip_protocol(); + let (sport, dport) = match pkt.transport { + Transport::Tcp { + src_port, dst_port, .. + } + | Transport::Udp { + src_port, dst_port, .. + } => (src_port, dst_port), + Transport::Icmpv4 { echo_id, .. } => (echo_id.unwrap_or(0), 0), + Transport::Icmpv6 { .. } => (0, 0), + Transport::Other { .. } => (0, 0), + Transport::LaterFragment { + protocol, + identification, + .. + } => ( + (identification & 0xffff) as u16, + (protocol as u16).wrapping_mul(31), + ), + }; + Self { + src, + dst, + proto, + sport, + dport, + } + } + + /// Canonical bidirectional identity for conntrack fast-path hits. + pub fn canonical(self) -> (Self, bool) { + let rev = Self { + src: self.dst, + dst: self.src, + proto: self.proto, + sport: self.dport, + dport: self.sport, + }; + if (self.src, self.sport) <= (self.dst, self.dport) { + (self, false) + } else { + (rev, true) + } + } +} + +/// Compact parsed metadata stored alongside owned bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PacketMeta { + pub src_v4: Option, + pub dst_v4: Option, + pub src: IpAddr, + pub dst: IpAddr, + pub proto: u8, + pub wire_len: usize, + pub ip_header_len: usize, + pub transport: Transport, + pub fragmentation: Fragmentation, + pub tcp_flags: u8, +} + +impl PacketMeta { + pub fn from_packet(pkt: &Packet<'_>) -> Self { + let (src, dst) = match pkt.ip { + IpMeta::V4 { src, dst, .. } => (IpAddr::V4(src), IpAddr::V4(dst)), + IpMeta::V6 { src, dst, .. } => (IpAddr::V6(src), IpAddr::V6(dst)), + }; + let tcp_flags = match pkt.transport { + Transport::Tcp { flags, .. } => flags.0, + _ => 0, + }; + Self { + src_v4: pkt.ip.v4_src(), + dst_v4: pkt.ip.v4_dst(), + src, + dst, + proto: pkt.ip.ip_protocol(), + wire_len: pkt.wire_len, + ip_header_len: pkt.ip.header_len(), + transport: pkt.transport, + fragmentation: pkt.fragmentation, + tcp_flags, + } + } + + pub fn is_fragment(&self) -> bool { + !matches!(self.fragmentation, Fragmentation::None) + } + + pub fn is_later_fragment(&self) -> bool { + matches!(self.fragmentation, Fragmentation::Later { .. }) + } + + /// Cheap SSH-NAT precondition using stored metadata only (no reparse). + pub fn ssh_nat_class(&self, self_ip: Ipv4Addr) -> SshNatClass { + if self.is_later_fragment() { + return SshNatClass::None; + } + let Transport::Tcp { + src_port, + dst_port, + header_len, + .. + } = self.transport + else { + return SshNatClass::None; + }; + if header_len < 18 { + return SshNatClass::None; + } + if self.dst_v4 == Some(self_ip) && dst_port == 22 { + return SshNatClass::InboundToInternal; + } + if self.src_v4 == Some(self_ip) && src_port == 30022 { + return SshNatClass::OutboundToExternal; + } + SshNatClass::None + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SshNatClass { + None, + InboundToInternal, + OutboundToExternal, +} diff --git a/crates/tunnet-common/src/packet/mod.rs b/crates/tunnet-common/src/packet/mod.rs index 4b0f98a2..38b082ee 100644 --- a/crates/tunnet-common/src/packet/mod.rs +++ b/crates/tunnet-common/src/packet/mod.rs @@ -17,10 +17,25 @@ mod build; mod frag; +mod frame; +mod meta; +mod owned; mod parse; pub use build::{set_tcp_ipv4_checksum, synthesize_reject, tcp_ipv4_checksum_of}; -pub use frag::{CachedTransport, FRAGMENT_TTL, FragmentTable, MAX_FRAGMENT_ENTRIES, ResolvedL4}; +pub use frag::{ + CachedTransport, FRAGMENT_TTL, FragKey, FragmentTable, MAX_FRAGMENT_ENTRIES, ResolvedL4, +}; +pub use frame::{ + DecodeError, Frame, KIND_SEGMENT, KIND_SINGLE, MAX_SEGMENTS, MIN_SEGMENT_PAYLOAD, + SEGMENT_OVERHEAD, SINGLE_OVERHEAD, SegmentHeader, decode_frame, encode_segment_prefix, + encode_single_prefix, segment_count, +}; +pub use meta::{FlowKey, PacketMeta, SshNatClass}; +pub use owned::{ + DEFAULT_VIRTUAL_MTU, FRAME_HEADROOM, LogicalPacket, MAX_LOGICAL_LEN, MIN_VIRTUAL_MTU, + PacketOwner, PacketPool, PooledBuffer, +}; pub use parse::{ParseError, parse}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; diff --git a/crates/tunnet-common/src/packet/owned.rs b/crates/tunnet-common/src/packet/owned.rs new file mode 100644 index 00000000..5c7d4f05 --- /dev/null +++ b/crates/tunnet-common/src/packet/owned.rs @@ -0,0 +1,439 @@ +//! Dataplane packet ownership: zero/minimal-copy logical packets. +//! +//! One packet owner moves through TUN receive → parse → scheduler → +//! segmentation → Iroh without repeated allocation: +//! +//! ```text +//! PooledBuffer (Vec storage + pool handle, AsRef<[u8]>) +//! -- transmit path --> Bytes::from_owner(owner) (no copy; pool recycle on Drop) +//! -- shared path --> Bytes kept directly (inbound, no mutation) +//! ``` +//! +//! Safety rules: no lifetime tricks (owners are `'static`), no OS-ring +//! pinning (pool buffers are plain heap memory), bounded pools with MTU +//! capacity classes (no 64 KiB retention for normal packets). + +use std::sync::{Arc, Mutex, Weak}; + +use bytes::Bytes; + +use super::{FlowKey, PacketMeta, parse}; + +/// Headroom reserved at the front of every pooled buffer for the tunnel +/// frame header, so single-frame encoding never copies the payload. +pub const FRAME_HEADROOM: usize = 32; + +/// Logical/virtual MTU default for the dataplane. +pub const DEFAULT_VIRTUAL_MTU: usize = 2800; +/// Hard ceiling for a logical packet (framing `total_len` is u16-compatible). +pub const MAX_LOGICAL_LEN: usize = 9000; +/// Smallest usable logical MTU. +pub const MIN_VIRTUAL_MTU: usize = 576; + +/// Pool capacity classes (bytes). A buffer is always drawn from the smallest +/// class that fits, so normal packets never retain huge allocations. +const CLASSES: [usize; 5] = [512, 1536, 2816, 4096, 9216]; + +#[derive(Debug)] +struct ClassPool { + free: Mutex>>, + hits: std::sync::atomic::AtomicU64, + misses: std::sync::atomic::AtomicU64, +} + +/// Bounded packet buffer pool with MTU capacity classes. +#[derive(Debug)] +pub struct PacketPool { + classes: [ClassPool; 5], + per_class_cap: usize, +} + +impl Default for PacketPool { + fn default() -> Self { + Self::for_new(64) + } +} + +impl PacketPool { + fn for_new(per_class_cap: usize) -> Self { + let mk = || ClassPool { + free: Mutex::new(Vec::new()), + hits: Default::default(), + misses: Default::default(), + }; + Self { + classes: [mk(), mk(), mk(), mk(), mk()], + per_class_cap: per_class_cap.max(4), + } + } + + pub fn new(per_class_cap: usize) -> Arc { + Arc::new(Self::for_new(per_class_cap)) + } + + fn class_for(need: usize) -> usize { + CLASSES + .iter() + .position(|c| *c >= need) + .unwrap_or(CLASSES.len() - 1) + } + + /// Acquire storage for `need` bytes of packet payload plus frame headroom. + pub fn acquire(self: &Arc, need: usize) -> PooledBuffer { + let total = need.saturating_add(FRAME_HEADROOM).min(CLASSES[4]); + let class = Self::class_for(total); + let pool = &self.classes[class]; + let mut storage = pool.free.lock().expect("pool").pop().unwrap_or_default(); + if storage.capacity() < total { + storage.reserve(total - storage.capacity()); + pool.misses + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else { + pool.hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + storage.clear(); + let start = FRAME_HEADROOM.min(storage.capacity()); + PooledBuffer { + storage, + start, + len: 0, + pool: Arc::downgrade(self), + class: class as u8, + } + } + + fn release(&self, mut storage: Vec, class: usize) { + // Never retain absurd buffers: drop storage far above its class. + if storage.capacity() > CLASSES[class] * 2 { + return; + } + storage.clear(); + let pool = &self.classes[class]; + let mut free = pool.free.lock().expect("pool"); + if free.len() < self.per_class_cap { + free.push(storage); + } + } + + /// (hits, misses) across all classes for telemetry. + pub fn hit_miss(&self) -> (u64, u64) { + use std::sync::atomic::Ordering::Relaxed; + let mut h = 0; + let mut m = 0; + for c in &self.classes { + h += c.hits.load(Relaxed); + m += c.misses.load(Relaxed); + } + (h, m) + } +} + +/// Owned packet storage with frame headroom. `AsRef<[u8]>` exposes exactly +/// the live packet bytes, so `Bytes::from_owner` views the frame with no +/// copy and the pool recycles the storage on final drop. +pub struct PooledBuffer { + storage: Vec, + start: usize, + len: usize, + pool: Weak, + class: u8, +} + +impl std::fmt::Debug for PooledBuffer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PooledBuffer") + .field("len", &self.len) + .field("class", &self.class) + .finish() + } +} + +impl PooledBuffer { + /// Region with capacity for receiving up to `cap` bytes. + pub fn recv_region(&mut self, cap: usize) -> &mut [u8] { + let need = self.start + cap; + if self.storage.len() < need { + self.storage.resize(need, 0); + } + &mut self.storage[self.start..self.start + cap] + } + + pub fn set_len(&mut self, len: usize) { + self.len = len; + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Mutable headroom for in-place single-frame header encoding. + /// Returns `None` when the header does not fit (caller falls back to a + /// staged encode, never corrupting the payload). + pub fn header_slot(&mut self, hdr_len: usize) -> Option<&mut [u8]> { + if hdr_len > self.start { + return None; + } + self.start -= hdr_len; + self.len += hdr_len; + Some(&mut self.storage[self.start..self.start + hdr_len]) + } + + /// Mutable receive area from the headroom start to the end of storage, + /// for TUN batch slots used with `recv_multiple` at offset 0. Size it + /// first with [`recv_region`]; the received length is then set via + /// [`set_len`] (or `from_pooled`). Headroom stays intact, so a later + /// single-frame encode prepends its header with no copy. + pub fn recv_area_mut(&mut self) -> &mut [u8] { + debug_assert!(self.start <= self.storage.len()); + &mut self.storage[self.start..] + } + + /// Immutable receive area: exactly the same region and length as + /// [`recv_area_mut`](Self::recv_area_mut). Batch-slot `AsRef`/`AsMut` + /// impls must return the same region — tun-rs validates capacity + /// against `AsRef::len()` and writes into `AsMut`, so divergent views + /// fail or misframe batches. + pub fn recv_area(&self) -> &[u8] { + debug_assert!(self.start <= self.storage.len()); + &self.storage[self.start..] + } + + pub fn packet_bytes(&self) -> &[u8] { + debug_assert!(self.start + self.len <= self.storage.len()); + let end = (self.start + self.len).min(self.storage.len()); + &self.storage[self.start..end] + } +} + +impl AsRef<[u8]> for PooledBuffer { + fn as_ref(&self) -> &[u8] { + self.packet_bytes() + } +} + +impl Drop for PooledBuffer { + fn drop(&mut self) { + if let Some(pool) = self.pool.upgrade() { + let storage = std::mem::take(&mut self.storage); + pool.release(storage, self.class as usize); + } + } +} + +/// Ownership of logical packet bytes through the dataplane pipeline. +#[derive(Debug)] +pub enum PacketOwner { + /// Pooled heap storage; converts to `Bytes` via `from_owner` (no copy). + Pooled(PooledBuffer), + /// Already-owned bytes (inbound DATAGRAM, no mutation needed). + Shared(Bytes), +} + +impl PacketOwner { + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::Pooled(b) => b.as_ref(), + Self::Shared(b) => b.as_ref(), + } + } + + pub fn len(&self) -> usize { + self.as_bytes().len() + } + + pub fn is_empty(&self) -> bool { + self.as_bytes().is_empty() + } + + /// Consume into a QUIC DATAGRAM payload without copying the payload. + pub fn into_datagram(self) -> Bytes { + match self { + Self::Pooled(b) => Bytes::from_owner(b), + Self::Shared(b) => b, + } + } +} + +/// Owned logical (inner IP) packet: bytes + parse-once metadata + flow key. +#[derive(Debug)] +pub struct LogicalPacket { + pub owner: PacketOwner, + pub meta: PacketMeta, + pub flow: FlowKey, + pub enqueued_at: std::time::Instant, +} + +impl LogicalPacket { + /// Parse-and-own from a slice (copies once; prefer the pooled constructors + /// on hot paths). + pub fn from_slice(data: &[u8]) -> Option { + let (meta, flow) = { + let pkt = parse(data).ok()?; + (PacketMeta::from_packet(&pkt), FlowKey::for_packet(&pkt)) + }; + Some(Self { + owner: PacketOwner::Shared(Bytes::copy_from_slice(data)), + meta, + flow, + enqueued_at: std::time::Instant::now(), + }) + } + + /// Take ownership of pooled storage filled with exactly `len` bytes. + pub fn from_pooled(mut buf: PooledBuffer, len: usize) -> Option { + buf.set_len(len); + let (meta, flow) = { + let pkt = parse(buf.as_ref()).ok()?; + (PacketMeta::from_packet(&pkt), FlowKey::for_packet(&pkt)) + }; + Some(Self { + owner: PacketOwner::Pooled(buf), + meta, + flow, + enqueued_at: std::time::Instant::now(), + }) + } + + /// Zero-copy inbound: retain the DATAGRAM's bytes, parse directly. + pub fn from_shared(bytes: Bytes) -> Option { + let (meta, flow) = { + let pkt = parse(&bytes).ok()?; + (PacketMeta::from_packet(&pkt), FlowKey::for_packet(&pkt)) + }; + Some(Self { + owner: PacketOwner::Shared(bytes), + meta, + flow, + enqueued_at: std::time::Instant::now(), + }) + } + + /// Take ownership of a `Vec` without copying (`Bytes::from` moves + /// the allocation). Used for batch-slot transfers and reassembly output. + pub fn from_vec(data: Vec) -> Option { + let (meta, flow) = { + let pkt = parse(&data).ok()?; + (PacketMeta::from_packet(&pkt), FlowKey::for_packet(&pkt)) + }; + Some(Self { + owner: PacketOwner::Shared(Bytes::from(data)), + meta, + flow, + enqueued_at: std::time::Instant::now(), + }) + } + + pub fn len(&self) -> usize { + self.owner.len() + } + + pub fn is_empty(&self) -> bool { + self.owner.is_empty() + } + + pub fn sojourn(&self) -> std::time::Duration { + self.enqueued_at.elapsed() + } + + /// Materialize mutable pooled storage (NAT rewrite and other rare + /// mutations only). Returns false when the packet cannot be materialized. + pub fn materialize(&mut self, pool: &Arc) -> bool { + if matches!(self.owner, PacketOwner::Pooled(_)) { + return true; + } + let bytes = self.owner.as_bytes(); + let mut buf = pool.acquire(bytes.len()); + let region = buf.recv_region(bytes.len()); + region.copy_from_slice(bytes); + buf.set_len(bytes.len()); + // Re-derive metadata only if a later mutation needs it; the caller + // refreshes after mutating. + self.owner = PacketOwner::Pooled(buf); + true + } + + /// Refresh metadata/flow after an in-place mutation (rare path). + pub fn refresh(&mut self) -> bool { + let Ok(pkt) = parse(self.owner.as_bytes()) else { + return false; + }; + self.meta = PacketMeta::from_packet(&pkt); + self.flow = FlowKey::for_packet(&pkt); + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn udp_packet() -> Vec { + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 443); + let mut o = Vec::new(); + b.write(&mut o, &[0; 200]).unwrap(); + o + } + + #[test] + fn pooled_round_trip_no_copy_view() { + let pool = PacketPool::new(8); + let raw = udp_packet(); + let mut buf = pool.acquire(raw.len()); + buf.recv_region(raw.len()).copy_from_slice(&raw); + let p = LogicalPacket::from_pooled(buf, raw.len()).unwrap(); + assert_eq!(p.len(), raw.len()); + assert_eq!(p.owner.as_bytes(), raw.as_slice()); + } + + #[test] + fn header_slot_prepends_without_copy() { + let pool = PacketPool::new(8); + let raw = udp_packet(); + let mut buf = pool.acquire(raw.len()); + buf.recv_region(raw.len()).copy_from_slice(&raw); + buf.set_len(raw.len()); + let slot = buf.header_slot(4).unwrap(); + slot.copy_from_slice(&[9, 9, 9, 9]); + assert_eq!(&buf.as_ref()[..4], &[9, 9, 9, 9]); + assert_eq!(&buf.as_ref()[4..], raw.as_slice()); + } + + #[test] + fn from_owner_keeps_storage_alive() { + let pool = PacketPool::new(8); + let raw = udp_packet(); + let mut buf = pool.acquire(raw.len()); + buf.recv_region(raw.len()).copy_from_slice(&raw); + buf.set_len(raw.len()); + let b = Bytes::from_owner(buf); + assert_eq!(&b[..], raw.as_slice()); + let c = b.clone(); + drop(b); + assert_eq!(&c[..], raw.as_slice()); + } + + #[test] + fn pool_recycles_and_reports() { + let pool = PacketPool::new(8); + { + let _ = pool.acquire(100); + } + let (h, m) = pool.hit_miss(); + assert_eq!(h + m, 1); + let _ = pool.acquire(100); + let (h2, _) = pool.hit_miss(); + assert_eq!(h2, h + 1, "second acquire should hit the pool"); + } + + #[test] + fn shared_inbound_no_copy() { + let raw = udp_packet(); + let bytes = Bytes::from(raw.clone()); + let p = LogicalPacket::from_shared(bytes).unwrap(); + assert_eq!(p.owner.as_bytes(), raw.as_slice()); + } +} diff --git a/crates/tunnet-core/Cargo.toml b/crates/tunnet-core/Cargo.toml index df5e3277..25c604f2 100644 --- a/crates/tunnet-core/Cargo.toml +++ b/crates/tunnet-core/Cargo.toml @@ -63,6 +63,8 @@ hickory-proto = { workspace = true, optional = true } hickory-resolver = { workspace = true, optional = true } iroh.workspace = true +noq.workspace = true +noq-proto.workspace = true iroh-gossip.workspace = true iroh-blobs = { workspace = true, optional = true } iroh-docs = { workspace = true, optional = true } @@ -92,6 +94,7 @@ http-body-util = { workspace = true, optional = true } [dev-dependencies] tempfile = "3" rstest.workspace = true +proptest = "1" iroh = { workspace = true, features = ["test-utils"] } [target.'cfg(unix)'.dependencies] diff --git a/crates/tunnet-core/src/acl.rs b/crates/tunnet-core/src/acl.rs index 6ce8df2d..b1c3799b 100644 --- a/crates/tunnet-core/src/acl.rs +++ b/crates/tunnet-core/src/acl.rs @@ -1,32 +1,14 @@ -use std::collections::VecDeque; use std::net::Ipv4Addr; use std::sync::Arc; -use std::time::{Duration, Instant}; use arc_swap::ArcSwap; -use dashmap::DashMap; -use parking_lot::Mutex; -use serde::Serialize; -use tunnet_common::packet::{FragmentTable, Packet, ResolvedL4, TcpFlags}; +use parking_lot::RwLock; use tunnet_common::policy::{ - Action, Direction, EvalCtx, EvalReason, EvalVerdict, PolicyBundle, Protocol, evaluate_detailed, + Action, Direction, EvalCtx, PolicyBundle, Protocol, evaluate_detailed, }; -use crate::routing::{PeerInfo, RoutingTable}; - -const DENY_LOG_CAP: usize = 64; - -// Match `direct/firewall.rs` conntrack TTLs. -const TCP_ACTIVE_TTL: Duration = Duration::from_secs(300); -const TCP_TIME_WAIT_TTL: Duration = Duration::from_secs(10); -const UDP_TTL: Duration = Duration::from_secs(30); -const ICMP_TTL: Duration = Duration::from_secs(10); -const GC_INTERVAL: Duration = Duration::from_secs(10); - -const TCP_FIN: u8 = TcpFlags::FIN; -const TCP_SYN: u8 = TcpFlags::SYN; -const TCP_RST: u8 = TcpFlags::RST; -const TCP_ACK: u8 = TcpFlags::ACK; +use crate::policy_runtime::{AclDenyRecord, PolicyRuntime}; +use crate::routing::RoutingTable; #[derive(Debug, Clone)] pub struct SelfIdentity { @@ -36,58 +18,6 @@ pub struct SelfIdentity { pub network: String, } -#[derive(Debug, Clone, Serialize)] -pub struct AclDenyRecord { - pub peer_endpoint: String, - pub dst_port: Option, - pub protocol: String, - pub reason: String, - pub rule_slug: Option, - pub scope: Option, - pub at_unix: i64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -struct FlowKey { - proto: u8, - src: Ipv4Addr, - sport: u16, - dst: Ipv4Addr, - dport: u16, -} - -impl FlowKey { - fn reverse(self) -> Self { - Self { - proto: self.proto, - src: self.dst, - sport: self.dport, - dst: self.src, - dport: self.sport, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TcpPhase { - SynSent, - Established, - TimeWait, -} - -#[derive(Debug, Clone, Copy)] -enum FlowPhase { - Tcp(TcpPhase), - Udp, - Icmp, -} - -#[derive(Debug, Clone)] -struct FlowState { - phase: FlowPhase, - last_seen: Instant, -} - #[derive(Clone)] pub struct AclEngine { pub self_id: Arc>, @@ -96,9 +26,9 @@ pub struct AclEngine { pub stale: Arc>, /// When false, ACL rules that require source posture do not match. pub src_posture_ok: Arc>, - deny_log: Arc>>, - conntrack: Arc>, - fragments: Arc>, + /// Attached shared runtime. Every mutation publishes a fresh compiled + /// snapshot + generation bump (§0.3); packet state lives there, never here. + runtime: Arc>>, } impl AclEngine { @@ -117,48 +47,51 @@ impl AclEngine { bundle: PolicyBundle, src_posture_ok: Arc>, ) -> Self { - let engine = Self { + Self { self_id: Arc::new(ArcSwap::from_pointee(self_id)), routes, bundle: Arc::new(ArcSwap::from_pointee(bundle)), stale: Arc::new(ArcSwap::from_pointee(false)), src_posture_ok, - deny_log: Arc::new(Mutex::new(VecDeque::with_capacity(DENY_LOG_CAP))), - conntrack: Arc::new(DashMap::new()), - fragments: Arc::new(Mutex::new(FragmentTable::default())), - }; - engine.spawn_gc(); - engine + runtime: Arc::new(RwLock::new(None)), + } + } + + /// Attach the shared runtime (node build / dataplane bring-up). All + /// subsequent mutations publish to it. + pub fn attach_runtime(&self, runtime: PolicyRuntime) { + *self.runtime.write() = Some(runtime); + self.publish(); } - fn spawn_gc(&self) { - let Ok(handle) = tokio::runtime::Handle::try_current() else { + /// Compile current state and publish to the shared runtime (§0.3). + fn publish(&self) { + let Some(rt) = self.runtime.read().clone() else { return; }; - let conntrack = self.conntrack.clone(); - handle.spawn(async move { - let mut tick = tokio::time::interval(GC_INTERVAL); - loop { - tick.tick().await; - let now = Instant::now(); - conntrack.retain(|_, st| !is_expired(st, now)); - } - }); + rt.publish_acl( + &self.bundle.load(), + &self.self_id.load(), + **self.src_posture_ok.load(), + **self.stale.load(), + ); } pub fn set_src_posture_ok(&self, ok: bool) { self.src_posture_ok.store(Arc::new(ok)); + self.publish(); } pub fn replace_bundle(&self, b: PolicyBundle) { self.bundle.store(Arc::new(b)); self.stale.store(Arc::new(false)); - self.conntrack.clear(); - self.fragments.lock().clear(); + self.publish(); } pub fn flush_conntrack(&self) { - self.conntrack.clear(); + if let Some(rt) = self.runtime.read().clone() { + rt.invalidate(); + } } pub fn replace_self_tags(&self, tags: Vec) { @@ -172,14 +105,20 @@ impl AclEngine { tags, network: current.network.clone(), })); + self.publish(); } pub fn mark_stale(&self) { self.stale.store(Arc::new(true)); + self.publish(); } pub fn recent_denies(&self) -> Vec { - self.deny_log.lock().iter().cloned().collect() + self.runtime + .read() + .clone() + .map(|rt| rt.recent_denies()) + .unwrap_or_default() } pub fn allow_inbound_peer(&self, peer_endpoint_hex: &str) -> bool { @@ -221,325 +160,14 @@ impl AclEngine { evaluate_detailed(&bundle, &ctx, direction).action == Action::Allow } - pub fn allow_packet( - &self, - peer_endpoint_hex: &str, - direction: Direction, - packet: &Packet<'_>, - ) -> bool { - self.evaluate_packet(peer_endpoint_hex, direction, packet) - .action - == Action::Allow - } - - pub fn evaluate_packet( - &self, - peer_endpoint_hex: &str, - direction: Direction, - packet: &Packet<'_>, - ) -> EvalVerdict { - let Some(src) = packet.ip.v4_src() else { - return EvalVerdict { - action: Action::Deny, - reason: EvalReason::DefaultDeny, - rule_slug: None, - scope: None, - }; - }; - let Some(dst) = packet.ip.v4_dst() else { - return EvalVerdict { - action: Action::Deny, - reason: EvalReason::DefaultDeny, - rule_slug: None, - scope: None, - }; - }; - let Some(l4) = self.fragments.lock().resolve(packet) else { - return EvalVerdict { - action: Action::Deny, - reason: EvalReason::DefaultDeny, - rule_slug: None, - scope: None, - }; - }; - let peer = self.routes.lookup_endpoint(peer_endpoint_hex); - self.check(peer.as_deref(), peer_endpoint_hex, src, dst, direction, l4) - } - - fn check( - &self, - peer: Option<&PeerInfo>, - peer_hex: &str, - src: Ipv4Addr, - dst: Ipv4Addr, - direction: Direction, - l4: ResolvedL4, - ) -> EvalVerdict { - let empty_tags: Vec = Vec::new(); - let self_id = self.self_id.load(); - let bundle = self.bundle.load(); - - let proto = l4.protocol; - let src_port = l4.src_port; - let dst_port = l4.dst_port; - let tcp_flags = l4.tcp_flags.map(|f| f.0).unwrap_or(0); - let peer_ip = match direction { - Direction::Outbound => Some(dst), - Direction::Inbound => Some(src), - }; - - // 1) Established / return traffic via conntrack. - if let Some(key) = flow_key(proto, src, dst, src_port, dst_port) - && self.conntrack_allows(direction, key, tcp_flags) - { - return EvalVerdict { - action: Action::Allow, - reason: EvalReason::DefaultAllow, - rule_slug: None, - scope: None, - }; - } - - let posture_required = !bundle.default_src_posture.is_empty() - || bundle.rules.iter().any(|r| !r.src_posture.is_empty()); - let src_posture_ok = if posture_required { - **self.src_posture_ok.load() - } else { - true - }; - let ctx = EvalCtx { - self_endpoint_hex: &self_id.endpoint_hex, - self_ip: self_id.ip, - self_tags: &self_id.tags, - self_network: &self_id.network, - peer_endpoint_hex: peer_hex, - peer_ip, - peer_tags: peer.map(|p| p.tags.as_slice()).unwrap_or(&empty_tags), - peer_network: &self_id.network, - dst_port, - protocol: proto, - src_posture_ok, - }; - let verdict = evaluate_detailed(&bundle, &ctx, direction); - if verdict.action == Action::Deny { - // Fail-open only for open networks with no rules during poll outage. - if **self.stale.load() - && bundle.rules.is_empty() - && bundle.default_action == tunnet_common::policy::DefaultAction::Allow - { - return EvalVerdict { - action: Action::Allow, - reason: EvalReason::DefaultAllow, - rule_slug: None, - scope: None, - }; - } - self.record_deny(peer_hex, dst_port, proto, &verdict); - tracing::debug!( - peer = %peer_hex, - ?dst_port, - ?proto, - reason = ?verdict.reason, - slug = ?verdict.rule_slug, - "ACL deny" - ); - return verdict; - } - - // 2) Policy allowed → open / refresh flow for return traffic. - if let Some(key) = flow_key(proto, src, dst, src_port, dst_port) { - self.open_or_refresh_flow(key, proto, tcp_flags); - } - verdict - } - - fn conntrack_allows(&self, direction: Direction, fwd: FlowKey, tcp_flags: u8) -> bool { - let now = Instant::now(); - let rev = fwd.reverse(); - let key = if self.conntrack.contains_key(&fwd) { - fwd - } else if self.conntrack.contains_key(&rev) { - rev - } else { - return false; - }; - - let mut entry = match self.conntrack.get_mut(&key) { - Some(e) => e, - None => return false, - }; - if is_expired(&entry, now) { - drop(entry); - self.conntrack.remove(&key); - return false; - } - - match entry.phase { - FlowPhase::Tcp(phase) => match phase { - TcpPhase::SynSent => { - if matches!(direction, Direction::Inbound) - || (tcp_flags & TCP_ACK) != 0 - || (tcp_flags & TCP_RST) != 0 - { - if (tcp_flags & TCP_RST) != 0 || (tcp_flags & TCP_FIN) != 0 { - entry.phase = FlowPhase::Tcp(TcpPhase::TimeWait); - } else { - entry.phase = FlowPhase::Tcp(TcpPhase::Established); - } - entry.last_seen = now; - return true; - } - if matches!(direction, Direction::Outbound) { - entry.last_seen = now; - return true; - } - false - } - TcpPhase::Established => { - if (tcp_flags & TCP_RST) != 0 || (tcp_flags & TCP_FIN) != 0 { - entry.phase = FlowPhase::Tcp(TcpPhase::TimeWait); - } - entry.last_seen = now; - true - } - TcpPhase::TimeWait => { - entry.last_seen = now; - true - } - }, - FlowPhase::Udp | FlowPhase::Icmp => { - entry.last_seen = now; - true - } - } - } - - fn open_or_refresh_flow(&self, key: FlowKey, proto: Protocol, tcp_flags: u8) { - let now = Instant::now(); - let phase = match proto { - Protocol::Tcp => { - if (tcp_flags & TCP_SYN) != 0 && (tcp_flags & TCP_ACK) == 0 { - FlowPhase::Tcp(TcpPhase::SynSent) - } else if (tcp_flags & TCP_FIN) != 0 || (tcp_flags & TCP_RST) != 0 { - FlowPhase::Tcp(TcpPhase::TimeWait) - } else { - FlowPhase::Tcp(TcpPhase::Established) - } - } - Protocol::Udp => FlowPhase::Udp, - Protocol::Icmp | Protocol::Icmpv6 => FlowPhase::Icmp, - Protocol::Any | Protocol::Other(_) => return, - }; - - self.conntrack - .entry(key) - .and_modify(|st| { - st.last_seen = now; - if matches!(st.phase, FlowPhase::Tcp(TcpPhase::SynSent)) - && matches!(phase, FlowPhase::Tcp(TcpPhase::Established)) - { - st.phase = phase; - } - if matches!(phase, FlowPhase::Tcp(TcpPhase::TimeWait)) { - st.phase = phase; - } - }) - .or_insert(FlowState { - phase, - last_seen: now, - }); - } - - fn record_deny( - &self, - peer_hex: &str, - dst_port: Option, - proto: Protocol, - verdict: &EvalVerdict, - ) { - let reason = match verdict.reason { - EvalReason::OrgDeny => "org_deny", - EvalReason::NetworkDeny => "network_deny", - EvalReason::NetworkAllow => "network_allow", - EvalReason::DefaultAllow => "default_allow", - EvalReason::DefaultDeny => "default_deny", - EvalReason::IcmpPolicy => "icmp_policy", - EvalReason::PostureSkip => "posture_skip", - }; - let scope = verdict.scope.map(|s| match s { - tunnet_common::policy::RuleScope::Organization => "organization".to_string(), - tunnet_common::policy::RuleScope::Network => "network".to_string(), - }); - let record = AclDenyRecord { - peer_endpoint: peer_hex.to_string(), - dst_port, - protocol: format!("{proto:?}").to_lowercase(), - reason: reason.to_string(), - rule_slug: verdict.rule_slug.clone(), - scope, - at_unix: jiff::Timestamp::now().as_second(), - }; - let mut log = self.deny_log.lock(); - if log.len() >= DENY_LOG_CAP { - log.pop_front(); - } - log.push_back(record); - } -} - -fn proto_num(proto: Protocol) -> Option { - match proto { - Protocol::Tcp => Some(6), - Protocol::Udp => Some(17), - Protocol::Icmp => Some(1), - Protocol::Icmpv6 => Some(58), - Protocol::Other(n) => Some(n), - Protocol::Any => None, - } -} - -fn flow_key( - proto: Protocol, - src: Ipv4Addr, - dst: Ipv4Addr, - src_port: Option, - dst_port: Option, -) -> Option { - let proto = proto_num(proto)?; - if proto == 1 { - return Some(FlowKey { - proto, - src: src.min(dst), - sport: src_port.unwrap_or(0), - dst: src.max(dst), - dport: 0, - }); - } - Some(FlowKey { - proto, - src, - sport: src_port.unwrap_or(0), - dst, - dport: dst_port.unwrap_or(0), - }) -} - -fn is_expired(st: &FlowState, now: Instant) -> bool { - let ttl = match st.phase { - FlowPhase::Tcp(TcpPhase::TimeWait) => TCP_TIME_WAIT_TTL, - FlowPhase::Tcp(_) => TCP_ACTIVE_TTL, - FlowPhase::Udp => UDP_TTL, - FlowPhase::Icmp => ICMP_TTL, - }; - now.duration_since(st.last_seen) > ttl + // Packet-level evaluation lives in PolicyRuntime (§13); this engine owns + // connection admission (allow_peer above) and publishes configuration. } #[cfg(test)] mod tests { use super::*; - use tunnet_common::policy::{ - Action, DefaultAction, IcmpPolicy, PolicyRule, PortRange, RuleScope, Selector, - }; + use tunnet_common::policy::PolicyBundle; fn test_engine(bundle: PolicyBundle) -> AclEngine { let self_id = SelfIdentity { @@ -551,91 +179,73 @@ mod tests { AclEngine::new(self_id, RoutingTable::new(), bundle) } - fn allow_tcp_80_bundle() -> PolicyBundle { - PolicyBundle { - rules: vec![PolicyRule { - src: Selector::Any, - dst: Selector::Any, - action: Action::Allow, - ports: vec![PortRange { start: 80, end: 80 }], - protocol: Some(Protocol::Tcp), - priority: 100, - order_index: 0, - scope: RuleScope::Network, - enabled: true, - slug: Some("allow-http".into()), - src_posture: vec![], - }], - ssh_rules: vec![], - version: 1, - signature: String::new(), - default_action: DefaultAction::Deny, - icmp_policy: IcmpPolicy::Deny, - postures: Default::default(), - default_src_posture: vec![], - posture_enforcement: None, - } - } - - fn tcp_pkt(src: Ipv4Addr, dst: Ipv4Addr, sport: u16, dport: u16, flags: u8) -> Vec { - let mut b = etherparse::PacketBuilder::ipv4(src.octets(), dst.octets(), 64) - .tcp(sport, dport, 1, 1000); - if flags & TCP_SYN != 0 { - b = b.syn(); - } - if flags & TCP_ACK != 0 { - b = b.ack(1); - } - let mut out = Vec::new(); - b.write(&mut out, &[]).unwrap(); - out - } - #[test] - fn outbound_allow_opens_flow_for_inbound_return() { - let acl = test_engine(allow_tcp_80_bundle()); - let peer = "bb".repeat(32); - let self_ip = Ipv4Addr::new(100, 64, 0, 1); - let peer_ip = Ipv4Addr::new(100, 64, 0, 2); - let ephemeral = 52_000u16; - - let out = tcp_pkt(self_ip, peer_ip, ephemeral, 80, TCP_SYN); - let pkt = tunnet_common::packet::parse(&out).unwrap(); - assert!(acl.allow_packet(&peer, Direction::Outbound, &pkt)); - - let ret = tcp_pkt(peer_ip, self_ip, 80, ephemeral, TCP_ACK | TCP_SYN); - let pkt = tunnet_common::packet::parse(&ret).unwrap(); - assert!(acl.allow_packet(&peer, Direction::Inbound, &pkt)); + fn admission_follows_bundle_default() { + // Connection admission (not packet policy) still lives here. + let open = test_engine(PolicyBundle::default()); + assert!(open.allow_inbound_peer(&"bb".repeat(32))); + let restricted = test_engine(PolicyBundle { + default_action: tunnet_common::policy::DefaultAction::Deny, + ..PolicyBundle::default() + }); + assert!(!restricted.allow_inbound_peer(&"bb".repeat(32))); } #[test] - fn inbound_ephemeral_denied_without_prior_outbound() { - let acl = test_engine(allow_tcp_80_bundle()); - let peer = "bb".repeat(32); - let self_ip = Ipv4Addr::new(100, 64, 0, 1); - let peer_ip = Ipv4Addr::new(100, 64, 0, 2); - - let p = tcp_pkt(peer_ip, self_ip, 80, 52_000, TCP_ACK | TCP_SYN); - let pkt = tunnet_common::packet::parse(&p).unwrap(); - assert!(!acl.allow_packet(&peer, Direction::Inbound, &pkt)); + fn mutations_publish_to_attached_runtime() { + use crate::policy_runtime::PolicyRuntime; + use std::collections::HashMap; + let acl = test_engine(PolicyBundle::default()); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::new(), + &SelfIdentity { + endpoint_hex: "aa".repeat(32), + ip: Ipv4Addr::new(100, 64, 0, 1), + tags: vec![], + network: "net".into(), + }, + true, + false, + ); + let policy_gen = rt.generation(); + acl.attach_runtime(rt.clone()); + // attach publishes: generation bumps. + assert!(rt.generation() > policy_gen); + let gen2 = rt.generation(); + acl.replace_bundle(PolicyBundle::default()); + assert!(rt.generation() > gen2); + acl.mark_stale(); + // Stale flag propagates to the runtime snapshot. + assert!(rt.generation() > gen2); } #[test] - fn replace_bundle_flushes_conntrack() { - let acl = test_engine(allow_tcp_80_bundle()); - let peer = "bb".repeat(32); - let self_ip = Ipv4Addr::new(100, 64, 0, 1); - let peer_ip = Ipv4Addr::new(100, 64, 0, 2); - let ephemeral = 52_000u16; - - let out = tcp_pkt(self_ip, peer_ip, ephemeral, 80, TCP_SYN); - let pkt = tunnet_common::packet::parse(&out).unwrap(); - assert!(acl.allow_packet(&peer, Direction::Outbound, &pkt)); - - acl.replace_bundle(allow_tcp_80_bundle()); - - let ret = tcp_pkt(peer_ip, self_ip, 80, ephemeral, TCP_ACK); - let pkt = tunnet_common::packet::parse(&ret).unwrap(); - assert!(!acl.allow_packet(&peer, Direction::Inbound, &pkt)); + fn replace_self_tags_noop_skips_publish() { + use crate::policy_runtime::PolicyRuntime; + use std::collections::HashMap; + let acl = test_engine(PolicyBundle::default()); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::new(), + &SelfIdentity { + endpoint_hex: "aa".repeat(32), + ip: Ipv4Addr::new(100, 64, 0, 1), + tags: vec![], + network: "net".into(), + }, + true, + false, + ); + acl.attach_runtime(rt.clone()); + let policy_gen = rt.generation(); + acl.replace_self_tags(vec![]); + assert_eq!( + rt.generation(), + policy_gen, + "unchanged tags must not republish" + ); + acl.replace_self_tags(vec!["x".into()]); + assert!(rt.generation() > policy_gen); } } diff --git a/crates/tunnet-core/src/direct/connect.rs b/crates/tunnet-core/src/direct/connect.rs index abf1aad6..474aa604 100644 --- a/crates/tunnet-core/src/direct/connect.rs +++ b/crates/tunnet-core/src/direct/connect.rs @@ -13,7 +13,6 @@ use crate::identity::AgentIdentity; use tunnet_common::local_api::DirectConnectPendingInfo; use crate::local_api::LocalApiState; -use crate::routing::PeerInfo; use crate::state::PersistedState; const CONNECT_PENDING_FILE: &str = "connect_pending.json"; @@ -24,6 +23,13 @@ pub struct ConnectPending { pub endpoint_id: String, pub hostname: String, pub received_at: Timestamp, + /// Requester's mesh address, as it reported at request time. + /// + /// Absent for records written before the request carried it, and for peers + /// running an older agent. Falls back to derivation, which is only correct + /// when the peer's `collision_index` is 0. + #[serde(default)] + pub ipv4: Option, } fn load_allowlist(state: &LocalApiState) -> anyhow::Result> { @@ -71,16 +77,6 @@ fn install_peer_route( .unwrap_or(0) as u64; let hex = format!("{endpoint}"); - let _info = std::sync::Arc::new(PeerInfo { - endpoint, - endpoint_hex: hex.clone(), - hostname: hostname.to_string(), - ip, - tags: vec!["connect".into()], - network_id, - network_name: network_name.clone(), - ssh_host_key: None, - }); // Merge into routing table via replace with existing peers + this one. let mut peers: Vec = state .node @@ -155,6 +151,9 @@ pub async fn request_connect(state: &LocalApiState, contact_id: &str) -> anyhow: "contact_id": contact_id_from_endpoint(&state.node.endpoint.id()), "endpoint_id": self_hex, "hostname": hostname, + // Additive: lets the responder install a route to our real address + // instead of re-deriving it and guessing `collision_index` 0. + "ipv4": state.node.self_ipv4.to_string(), }); let bytes = serde_json::to_vec(&req)?; send.write_all(&(bytes.len() as u32).to_be_bytes()).await?; @@ -229,7 +228,17 @@ pub async fn accept_pending(state: &LocalApiState, contact_id: &str) -> anyhow:: .endpoint_id .parse() .context("parse pending endpoint")?; - let peer_ip = derive_ipv4(&pending.endpoint_id, 0); + // Prefer the address the peer reported. Deriving it here assumes a + // `collision_index` of 0, so a peer the coordinator had to deconflict + // would get a route to the wrong address. + let peer_ip = pending.ipv4.unwrap_or_else(|| { + tracing::warn!( + endpoint = %pending.endpoint_id, + "pending connect has no reported mesh address; deriving with \ + collision_index 0, which is wrong if the peer was deconflicted" + ); + derive_ipv4(&pending.endpoint_id, 0) + }); install_peer_route(state, peer, &pending.hostname, peer_ip)?; // Best-effort: dial back to notify acceptance. @@ -328,6 +337,10 @@ pub async fn handle_inbound_connect( .and_then(|v| v.as_str()) .unwrap_or(remote_hex) .to_string(); + let peer_ipv4 = req + .get("ipv4") + .and_then(|v| v.as_str()) + .and_then(|s| s.parse::().ok()); if allowlist.contains(&contact_id) { let resp = serde_json::json!({ @@ -355,6 +368,7 @@ pub async fn handle_inbound_connect( endpoint_id, hostname, received_at: Timestamp::now(), + ipv4: peer_ipv4, }); let _ = std::fs::write(&pending_path, serde_json::to_vec_pretty(&list)?); @@ -377,3 +391,67 @@ pub fn load_allowlist_from_dir(state_dir: &std::path::Path) -> HashSet { fn _persist_mode_check(p: &PersistedState) { let _ = p.is_direct(); } + +#[cfg(test)] +mod tests { + use super::*; + + /// Records written before the request carried an address must still load. + #[test] + fn pending_without_ipv4_still_deserializes() { + let old = r#"{ + "contact_id": "abc", + "endpoint_id": "aa", + "hostname": "peer", + "received_at": "2026-01-01T00:00:00Z" + }"#; + let p: ConnectPending = serde_json::from_str(old).expect("old record must load"); + assert_eq!(p.ipv4, None); + assert_eq!(p.contact_id, "abc"); + } + + #[test] + fn pending_with_ipv4_round_trips() { + let p = ConnectPending { + contact_id: "abc".into(), + endpoint_id: "aa".repeat(32), + hostname: "peer".into(), + received_at: Timestamp::now(), + ipv4: Some("100.72.1.2".parse().unwrap()), + }; + let back: ConnectPending = + serde_json::from_slice(&serde_json::to_vec(&p).unwrap()).unwrap(); + assert_eq!(back.ipv4, p.ipv4); + } + + /// Why the reported address matters: deriving assumes `collision_index` 0, + /// so a peer the coordinator had to deconflict resolves to a different + /// address entirely and the installed route would point at the wrong node. + #[test] + fn derivation_is_wrong_for_a_deconflicted_peer() { + let id = "fe".repeat(32); + let assumed = derive_ipv4(&id, 0); + let actual = derive_ipv4(&id, 1); + assert_ne!( + assumed, actual, + "derivation with a hardcoded 0 cannot represent a deconflicted peer" + ); + } + + /// The reported address is used verbatim, whatever its collision index. + #[test] + fn reported_address_is_preferred_over_derivation() { + let id = "fe".repeat(32); + let reported: std::net::Ipv4Addr = derive_ipv4(&id, 3); + let pending = ConnectPending { + contact_id: "abc".into(), + endpoint_id: id.clone(), + hostname: "peer".into(), + received_at: Timestamp::now(), + ipv4: Some(reported), + }; + let chosen = pending.ipv4.unwrap_or_else(|| derive_ipv4(&id, 0)); + assert_eq!(chosen, reported); + assert_ne!(chosen, derive_ipv4(&id, 0)); + } +} diff --git a/crates/tunnet-core/src/direct/connectivity.rs b/crates/tunnet-core/src/direct/connectivity.rs index 11e4039a..7160eac7 100644 --- a/crates/tunnet-core/src/direct/connectivity.rs +++ b/crates/tunnet-core/src/direct/connectivity.rs @@ -137,7 +137,23 @@ fn apply_relay_mode(builder: Builder, opts: &ConnectivityOptions) -> Builder { } /// Start an endpoint builder with the relay preset for this profile. +/// +/// The explicit [`crate::transport_profile::TunnetTransportProfile`] is applied +/// here: every mesh endpoint shares one controlled QUIC transport instead of +/// inheriting generic Iroh/noq defaults. pub fn endpoint_builder(opts: &ConnectivityOptions) -> Builder { + endpoint_builder_with_transport( + opts, + &crate::transport_profile::TunnetTransportProfile::default(), + ) +} + +/// Same as [`endpoint_builder`] with an explicit transport profile +/// (e.g. BBRv3 benchmark experiments). +pub fn endpoint_builder_with_transport( + opts: &ConnectivityOptions, + profile: &crate::transport_profile::TunnetTransportProfile, +) -> Builder { let builder = match opts.profile { ConnectivityProfile::LanOnly => Endpoint::builder(presets::Minimal), ConnectivityProfile::TunnetManaged if !opts.custom_relays.is_empty() => { @@ -153,7 +169,7 @@ pub fn endpoint_builder(opts: &ConnectivityOptions) -> Builder { | ConnectivityProfile::TunnetManaged | ConnectivityProfile::ServerlessDht => Endpoint::builder(presets::N0), }; - apply_relay_mode(builder, opts) + apply_relay_mode(profile.apply(builder), opts) } /// Attach address-lookup services to an endpoint builder. diff --git a/crates/tunnet-core/src/direct/firewall.rs b/crates/tunnet-core/src/direct/firewall.rs index bdcdee66..91cdaa9c 100644 --- a/crates/tunnet-core/src/direct/firewall.rs +++ b/crates/tunnet-core/src/direct/firewall.rs @@ -1,9 +1,11 @@ -//! Userspace stateful firewall for Direct mode. +//! Userspace stateful firewall configuration for Direct mode. //! -//! Defaults (authenticated mesh peers): -//! - Outbound: allow all (opens flow) -//! - Inbound from a known mesh peer: allow all (QUIC already gated by PSK/AuthCache) -//! - Inbound without a peer identity: ICMP echo only; TCP/UDP deny +//! This engine is a control-plane object: it owns firewall configuration +//! (local rules, suggested rules, enabled flag) and publishes compiled +//! snapshots to the shared [`crate::policy_runtime::PolicyRuntime`], which +//! owns all packet evaluation, conntrack, and expiry. Packet verdicts, +//! defaults (outbound allow; inbound from known peers allow; otherwise ICMP +//! echo only) and reject synthesis live in the runtime. //! //! Restrict further with local ACL rules (`tunnet firewall`). @@ -11,26 +13,17 @@ use std::collections::HashMap; use std::net::Ipv4Addr; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; use anyhow::Context; use arc_swap::ArcSwap; -use bytes::Bytes; -use dashmap::DashMap; -use parking_lot::Mutex; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; -use tunnet_common::packet::{FragmentTable, Packet, ResolvedL4, TcpFlags, synthesize_reject}; use tunnet_common::policy::{Action, PolicyBundle, PolicyRule, PortRange, Protocol, Selector}; use uuid::Uuid; +use crate::policy_runtime::PolicyRuntime; use crate::state::StatePaths; -const TCP_ACTIVE_TTL: Duration = Duration::from_secs(300); -const TCP_TIME_WAIT_TTL: Duration = Duration::from_secs(10); -const UDP_TTL: Duration = Duration::from_secs(30); -const ICMP_TTL: Duration = Duration::from_secs(10); -const GC_INTERVAL: Duration = Duration::from_secs(10); - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum FirewallDirection { @@ -191,96 +184,7 @@ pub fn direction_display(d: FirewallDirection) -> &'static str { } } -pub const TCP_FIN: u8 = TcpFlags::FIN; -pub const TCP_SYN: u8 = TcpFlags::SYN; -pub const TCP_RST: u8 = TcpFlags::RST; -pub const TCP_ACK: u8 = TcpFlags::ACK; - -// ── Conntrack ───────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -struct FlowKey { - proto: u8, - src: Ipv4Addr, - sport: u16, - dst: Ipv4Addr, - dport: u16, -} - -impl FlowKey { - fn forward(src: Ipv4Addr, dst: Ipv4Addr, l4: ResolvedL4) -> Option { - let proto = l4.protocol.ip_number()?; - if l4.protocol.is_icmp() { - Some(Self { - proto, - src: src.min(dst), - sport: l4.icmp_id.unwrap_or(0), - dst: src.max(dst), - dport: 0, - }) - } else { - Some(Self { - proto, - src, - sport: l4.src_port.unwrap_or(0), - dst, - dport: l4.dst_port.unwrap_or(0), - }) - } - } - - fn reverse(src: Ipv4Addr, dst: Ipv4Addr, l4: ResolvedL4) -> Option { - if l4.protocol.is_icmp() { - Self::forward(src, dst, l4) - } else { - Some(Self { - proto: l4.protocol.ip_number()?, - src: dst, - sport: l4.dst_port.unwrap_or(0), - dst: src, - dport: l4.src_port.unwrap_or(0), - }) - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TcpPhase { - SynSent, - Established, - TimeWait, -} - -#[derive(Debug, Clone, Copy)] -enum FlowPhase { - Tcp(TcpPhase), - Udp, - Icmp, -} - -#[derive(Debug, Clone)] -struct FlowState { - phase: FlowPhase, - last_seen: Instant, -} - -// ── Evaluation ──────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PacketDirection { - Inbound, - Outbound, -} - -#[derive(Debug)] -pub enum EvalResult { - Allow, - Deny, - /// Synthesized RST / ICMP unreachable for the local TUN. - Reject { - reply: Bytes, - }, -} +// ── Engine (control-plane configuration + runtime publishing) ───────────── pub struct FirewallStats { pub conntrack_entries: usize, @@ -305,13 +209,13 @@ struct EngineInner { local_rules: ArcSwap>, suggested_rules: ArcSwap>, version: AtomicU64, - conntrack: DashMap, - fragments: Mutex, - allowed: AtomicU64, - denied: AtomicU64, - rejected: AtomicU64, - /// Self mesh IP for default policy and reject synthesis. + /// Self mesh IP (kept for diagnostics/compat; policy uses runtime state). self_ip: ArcSwap, + /// Owning Direct network: published snapshots stay network-scoped (§0.2). + network_id: Uuid, + /// Attached shared runtime. Every mutation publishes a fresh compiled + /// snapshot + generation bump (§0.3); packet state lives there, never here. + runtime: RwLock>, } impl FirewallEngine { @@ -319,54 +223,72 @@ impl FirewallEngine { cfg: &FirewallConfig, self_ip: Ipv4Addr, _self_endpoint_hex: String, + network_id: Uuid, ) -> Self { - let engine = Self { + Self { inner: Arc::new(EngineInner { enabled: ArcSwap::from_pointee(cfg.enabled), local_rules: ArcSwap::from_pointee(cfg.rules.clone()), suggested_rules: ArcSwap::from_pointee(Vec::new()), version: AtomicU64::new(cfg.version), - conntrack: DashMap::new(), - fragments: Mutex::new(FragmentTable::default()), - allowed: AtomicU64::new(0), - denied: AtomicU64::new(0), - rejected: AtomicU64::new(0), self_ip: ArcSwap::from_pointee(self_ip), + network_id, + runtime: RwLock::new(None), }), + } + } + + /// Attach the shared runtime (node build / dataplane bring-up). All + /// subsequent mutations publish to it. + pub fn attach_runtime(&self, runtime: PolicyRuntime) { + *self.inner.runtime.write() = Some(runtime); + self.publish(); + } + + /// Compile this network's rules and publish to the shared runtime. + fn publish(&self) { + let Some(rt) = self.inner.runtime.read().clone() else { + return; }; - engine.spawn_gc(); - engine + rt.publish_firewall( + self.inner.network_id, + self.local_rules_snapshot(), + self.suggested_rules_snapshot(), + **self.inner.enabled.load(), + ); } - fn spawn_gc(&self) { - let inner = self.inner.clone(); - tokio::spawn(async move { - let mut tick = tokio::time::interval(GC_INTERVAL); - loop { - tick.tick().await; - let now = Instant::now(); - inner.conntrack.retain(|_, st| !is_expired(st, now)); - } - }); + fn bump_version(&self) -> u64 { + self.inner.version.fetch_add(1, Ordering::Relaxed) + 1 } pub fn reload_local(&self, cfg: &FirewallConfig) { self.inner.enabled.store(Arc::new(cfg.enabled)); self.inner.local_rules.store(Arc::new(cfg.rules.clone())); self.inner.version.store(cfg.version, Ordering::Relaxed); + self.bump_version(); + self.publish(); } pub fn set_suggested(&self, rules: Vec) { self.inner.suggested_rules.store(Arc::new(rules)); + // Version bump is the reliable generation signal (§0.3): the legacy + // code never bumped here, so suggested-rule edits were invisible. + self.bump_version(); + self.publish(); } pub fn clear_suggested(&self) { self.inner.suggested_rules.store(Arc::new(Vec::new())); + self.bump_version(); + self.publish(); } + /// Invalidate shared conntrack via the runtime (CLI flush, teardown). pub fn flush_conntrack(&self) { - self.inner.conntrack.clear(); - self.inner.fragments.lock().clear(); + if let Some(rt) = self.inner.runtime.read().clone() { + rt.invalidate(); + } } pub fn set_self_ip(&self, ip: Ipv4Addr) { @@ -374,15 +296,30 @@ impl FirewallEngine { } pub fn stats(&self) -> FirewallStats { + // Packet counters live in the shared runtime now (this engine no + // longer sees packets); conntrack_entries is dataplane-wide. + let (allowed, denied, rejected, conntrack_entries) = match self.inner.runtime.read().clone() + { + Some(rt) => { + let c = rt.fw_counters_for(self.inner.network_id); + ( + c.allowed.load(Ordering::Relaxed), + c.denied.load(Ordering::Relaxed), + c.rejected.load(Ordering::Relaxed), + rt.conntrack_len(), + ) + } + None => (0, 0, 0, 0), + }; FirewallStats { - conntrack_entries: self.inner.conntrack.len(), + conntrack_entries, local_rules: self.inner.local_rules.load().len(), suggested_rules: self.inner.suggested_rules.load().len(), enabled: **self.inner.enabled.load(), version: self.inner.version.load(Ordering::Relaxed), - packets_allowed: self.inner.allowed.load(Ordering::Relaxed), - packets_denied: self.inner.denied.load(Ordering::Relaxed), - packets_rejected: self.inner.rejected.load(Ordering::Relaxed), + packets_allowed: allowed, + packets_denied: denied, + packets_rejected: rejected, } } @@ -422,301 +359,10 @@ impl FirewallEngine { self.inner.local_rules.store(Arc::new(rules)); self.inner.version.store(version, Ordering::Relaxed); tracing::info!(port, "firewall: ensured inbound TCP allow for SSH"); - } - - /// Evaluate a packet. `peer_endpoint_hex` is the remote mesh peer (if known). - /// `network_id` is the peer's Direct network (for `PeerFilter::NetworkId`). - pub fn evaluate( - &self, - direction: PacketDirection, - packet: &Packet<'_>, - peer_endpoint_hex: Option<&str>, - peer_hostname: Option<&str>, - network_id: Option, - ) -> EvalResult { - if !**self.inner.enabled.load() { - self.inner.allowed.fetch_add(1, Ordering::Relaxed); - return EvalResult::Allow; - } - - let Some(src) = packet.ip.v4_src() else { - self.inner.denied.fetch_add(1, Ordering::Relaxed); - return EvalResult::Deny; - }; - let Some(dst) = packet.ip.v4_dst() else { - self.inner.denied.fetch_add(1, Ordering::Relaxed); - return EvalResult::Deny; - }; - let Some(l4) = self.inner.fragments.lock().resolve(packet) else { - self.inner.denied.fetch_add(1, Ordering::Relaxed); - return EvalResult::Deny; - }; - - if self.conntrack_allows(direction, src, dst, l4) { - self.inner.allowed.fetch_add(1, Ordering::Relaxed); - return EvalResult::Allow; - } - - if let Some(action) = self.match_rules( - &self.inner.local_rules.load(), - direction, - l4, - peer_endpoint_hex, - peer_hostname, - network_id, - ) { - return self.apply_action(action, direction, packet, src, dst, l4); - } - if let Some(action) = self.match_rules( - &self.inner.suggested_rules.load(), - direction, - l4, - peer_endpoint_hex, - peer_hostname, - network_id, - ) { - return self.apply_action(action, direction, packet, src, dst, l4); - } - - let default = default_policy(direction, l4, peer_endpoint_hex); - self.apply_action(default, direction, packet, src, dst, l4) - } - - fn apply_action( - &self, - action: FirewallAction, - direction: PacketDirection, - packet: &Packet<'_>, - src: Ipv4Addr, - dst: Ipv4Addr, - l4: ResolvedL4, - ) -> EvalResult { - match action { - FirewallAction::Allow => { - self.open_or_refresh_flow(direction, src, dst, l4); - self.inner.allowed.fetch_add(1, Ordering::Relaxed); - EvalResult::Allow - } - FirewallAction::Deny => { - self.inner.denied.fetch_add(1, Ordering::Relaxed); - EvalResult::Deny - } - FirewallAction::Reject => { - self.inner.rejected.fetch_add(1, Ordering::Relaxed); - let reply = synthesize_reject(packet).unwrap_or_default(); - EvalResult::Reject { reply } - } - } - } - - fn match_rules( - &self, - rules: &[FirewallRule], - direction: PacketDirection, - l4: ResolvedL4, - peer_hex: Option<&str>, - peer_hostname: Option<&str>, - network_id: Option, - ) -> Option { - let want_dir = match direction { - PacketDirection::Inbound => FirewallDirection::In, - PacketDirection::Outbound => FirewallDirection::Out, - }; - for rule in rules { - if rule.direction != want_dir { - continue; - } - if !l4.protocol.matches_rule(Some(rule.protocol)) { - continue; - } - if !rule.ports.is_empty() && !l4.protocol.is_icmp() { - let Some(port) = l4.dst_port else { - continue; - }; - if !rule.ports.iter().any(|p| p.contains(port)) { - continue; - } - } - if !peer_matches(&rule.peer, peer_hex, peer_hostname, network_id) { - continue; - } - return Some(rule.action); - } - None - } - - fn conntrack_allows( - &self, - direction: PacketDirection, - src: Ipv4Addr, - dst: Ipv4Addr, - l4: ResolvedL4, - ) -> bool { - let now = Instant::now(); - let Some(fwd) = FlowKey::forward(src, dst, l4) else { - return false; - }; - let Some(rev) = FlowKey::reverse(src, dst, l4) else { - return false; - }; - let tcp_flags = l4.tcp_flags.map(|f| f.0).unwrap_or(0); - - let key = if self.inner.conntrack.contains_key(&fwd) { - fwd - } else if self.inner.conntrack.contains_key(&rev) { - rev - } else { - return false; - }; - - let mut entry = match self.inner.conntrack.get_mut(&key) { - Some(e) => e, - None => return false, - }; - if is_expired(&entry, now) { - drop(entry); - self.inner.conntrack.remove(&key); - return false; - } - - match entry.phase { - FlowPhase::Tcp(phase) => match phase { - TcpPhase::SynSent => { - if direction == PacketDirection::Inbound - || (tcp_flags & TCP_ACK) != 0 - || (tcp_flags & TCP_RST) != 0 - { - if (tcp_flags & TCP_RST) != 0 || (tcp_flags & TCP_FIN) != 0 { - entry.phase = FlowPhase::Tcp(TcpPhase::TimeWait); - } else { - entry.phase = FlowPhase::Tcp(TcpPhase::Established); - } - entry.last_seen = now; - return true; - } - if direction == PacketDirection::Outbound { - entry.last_seen = now; - return true; - } - false - } - TcpPhase::Established => { - if (tcp_flags & TCP_RST) != 0 || (tcp_flags & TCP_FIN) != 0 { - entry.phase = FlowPhase::Tcp(TcpPhase::TimeWait); - } - entry.last_seen = now; - true - } - TcpPhase::TimeWait => { - entry.last_seen = now; - true - } - }, - FlowPhase::Udp | FlowPhase::Icmp => { - entry.last_seen = now; - true - } - } - } - - fn open_or_refresh_flow( - &self, - _direction: PacketDirection, - src: Ipv4Addr, - dst: Ipv4Addr, - l4: ResolvedL4, - ) { - let now = Instant::now(); - let Some(key) = FlowKey::forward(src, dst, l4) else { - return; - }; - let tcp_flags = l4.tcp_flags.map(|f| f.0).unwrap_or(0); - - let phase = match l4.protocol { - Protocol::Tcp => { - if (tcp_flags & TCP_SYN) != 0 && (tcp_flags & TCP_ACK) == 0 { - FlowPhase::Tcp(TcpPhase::SynSent) - } else if (tcp_flags & TCP_FIN) != 0 || (tcp_flags & TCP_RST) != 0 { - FlowPhase::Tcp(TcpPhase::TimeWait) - } else { - FlowPhase::Tcp(TcpPhase::Established) - } - } - Protocol::Udp => FlowPhase::Udp, - Protocol::Icmp | Protocol::Icmpv6 => FlowPhase::Icmp, - Protocol::Any | Protocol::Other(_) => return, - }; - - self.inner - .conntrack - .entry(key) - .and_modify(|st| { - st.last_seen = now; - if matches!(st.phase, FlowPhase::Tcp(TcpPhase::SynSent)) - && matches!(phase, FlowPhase::Tcp(TcpPhase::Established)) - { - st.phase = phase; - } - if matches!(phase, FlowPhase::Tcp(TcpPhase::TimeWait)) { - st.phase = phase; - } - }) - .or_insert(FlowState { - phase, - last_seen: now, - }); - } -} -fn peer_matches( - filter: &PeerFilter, - peer_hex: Option<&str>, - peer_hostname: Option<&str>, - network_id: Option, -) -> bool { - match filter { - PeerFilter::Any => true, - PeerFilter::Endpoint(id) => peer_hex.is_some_and(|h| h.eq_ignore_ascii_case(id)), - PeerFilter::Hostname(h) => peer_hostname.is_some_and(|n| n.eq_ignore_ascii_case(h)), - PeerFilter::NetworkId(n) => { - let Some(id) = network_id else { - return false; - }; - id.to_string().eq_ignore_ascii_case(n) - || n.parse::().ok().is_some_and(|parsed| parsed == id) - } - } -} - -fn default_policy( - direction: PacketDirection, - l4: ResolvedL4, - peer_endpoint_hex: Option<&str>, -) -> FirewallAction { - match direction { - PacketDirection::Outbound => FirewallAction::Allow, - PacketDirection::Inbound => { - if peer_endpoint_hex.is_some() { - return FirewallAction::Allow; - } - if l4.protocol == Protocol::Icmp && matches!(l4.icmp_type, Some(0) | Some(8)) { - FirewallAction::Allow - } else { - FirewallAction::Deny - } - } + self.publish(); } } -fn is_expired(st: &FlowState, now: Instant) -> bool { - let ttl = match st.phase { - FlowPhase::Tcp(TcpPhase::TimeWait) => TCP_TIME_WAIT_TTL, - FlowPhase::Tcp(_) => TCP_ACTIVE_TTL, - FlowPhase::Udp => UDP_TTL, - FlowPhase::Icmp => ICMP_TTL, - }; - now.duration_since(st.last_seen) > ttl -} - pub fn firewall_to_policy( cfg: &FirewallConfig, self_endpoint_hex: &str, @@ -829,17 +475,14 @@ pub fn firewall_to_policy( #[cfg(test)] mod tests { use super::*; + use tunnet_common::packet::synthesize_reject; fn engine() -> FirewallEngine { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let _guard = rt.enter(); FirewallEngine::from_config( &default_firewall(), Ipv4Addr::new(100, 64, 0, 1), "aa".repeat(32), + Uuid::nil(), ) } @@ -861,16 +504,6 @@ mod tests { out } - fn eval( - e: &FirewallEngine, - dir: PacketDirection, - raw: &[u8], - peer: Option<&str>, - ) -> EvalResult { - let pkt = tunnet_common::packet::parse(raw).unwrap(); - e.evaluate(dir, &pkt, peer, None, None) - } - #[test] fn parse_tcp() { let src = Ipv4Addr::new(100, 64, 0, 1); @@ -881,136 +514,41 @@ mod tests { assert_eq!(v.transport.src_port(), Some(12345)); assert_eq!(v.transport.dst_port(), Some(80)); assert!(v.transport.tcp_flags().unwrap().syn()); + let _ = tcp_ack(src, dst, 80, 12345); } #[test] - fn outbound_allowed_by_default() { - let e = engine(); - let p = tcp_syn( - Ipv4Addr::new(100, 64, 0, 1), - Ipv4Addr::new(100, 64, 0, 2), - 12345, - 443, - ); - assert!(matches!( - eval(&e, PacketDirection::Outbound, &p, Some("peer")), - EvalResult::Allow - )); - } - - #[test] - fn inbound_tcp_allowed_from_authenticated_peer() { - let e = engine(); - let p = tcp_syn( - Ipv4Addr::new(100, 64, 0, 2), - Ipv4Addr::new(100, 64, 0, 1), - 443, - 12345, - ); - assert!(matches!( - eval(&e, PacketDirection::Inbound, &p, Some("peer")), - EvalResult::Allow - )); - } - - #[test] - fn inbound_tcp_denied_without_peer_identity() { - let e = engine(); - let p = tcp_syn( - Ipv4Addr::new(100, 64, 0, 2), - Ipv4Addr::new(100, 64, 0, 1), - 443, - 12345, - ); - assert!(matches!( - eval(&e, PacketDirection::Inbound, &p, None), - EvalResult::Deny - )); - } - - #[test] - fn return_traffic_allowed_via_conntrack() { - let e = engine(); - let out = tcp_syn( - Ipv4Addr::new(100, 64, 0, 1), - Ipv4Addr::new(100, 64, 0, 2), - 12345, - 443, - ); - assert!(matches!( - eval(&e, PacketDirection::Outbound, &out, Some("peer")), - EvalResult::Allow - )); - let ret = tcp_ack( - Ipv4Addr::new(100, 64, 0, 2), - Ipv4Addr::new(100, 64, 0, 1), - 443, - 12345, - ); - assert!(matches!( - eval(&e, PacketDirection::Inbound, &ret, Some("peer")), - EvalResult::Allow - )); - } - - #[test] - fn local_deny_outbound() { + fn suggested_rules_bump_version() { + // Regression: legacy set_suggested never bumped the version, so + // suggested-rule edits were invisible to generation signals. let e = engine(); - e.reload_local(&FirewallConfig { - enabled: true, - version: 2, - rules: vec![FirewallRule { - direction: FirewallDirection::Out, - action: FirewallAction::Deny, - protocol: Protocol::Tcp, - ports: vec![PortRange { - start: 443, - end: 443, - }], - peer: PeerFilter::Any, - }], - }); - let p = tcp_syn( - Ipv4Addr::new(100, 64, 0, 1), - Ipv4Addr::new(100, 64, 0, 2), - 12345, - 443, - ); - assert!(matches!( - eval(&e, PacketDirection::Outbound, &p, Some("peer")), - EvalResult::Deny - )); + let v0 = e.stats().version; + e.set_suggested(vec![FirewallRule { + direction: FirewallDirection::Out, + action: FirewallAction::Deny, + protocol: Protocol::Tcp, + ports: vec![], + peer: PeerFilter::Any, + }]); + assert!(e.stats().version > v0); + assert_eq!(e.suggested_rules_snapshot().len(), 1); + e.clear_suggested(); + assert!(e.suggested_rules_snapshot().is_empty()); } #[test] - fn later_fragment_cannot_bypass_port_deny() { + fn ensure_inbound_tcp_allow_idempotent() { let e = engine(); - e.reload_local(&FirewallConfig { - enabled: true, - version: 2, - rules: vec![FirewallRule { - direction: FirewallDirection::Out, - action: FirewallAction::Deny, - protocol: Protocol::Tcp, - ports: vec![PortRange { - start: 443, - end: 443, - }], - peer: PeerFilter::Any, - }], - }); - let mut later = tcp_syn( - Ipv4Addr::new(100, 64, 0, 1), - Ipv4Addr::new(100, 64, 0, 2), - 12345, - 443, + e.ensure_inbound_tcp_allow(22); + e.ensure_inbound_tcp_allow(22); + let rules = e.local_rules_snapshot(); + assert_eq!( + rules + .iter() + .filter(|r| r.direction == FirewallDirection::In) + .count(), + 1 ); - later[6] = 0; - later[7] = 8; - assert!(matches!( - eval(&e, PacketDirection::Outbound, &later, Some("peer")), - EvalResult::Deny - )); } #[test] diff --git a/crates/tunnet-core/src/direct/mod.rs b/crates/tunnet-core/src/direct/mod.rs index c4f96f8e..6f2e96e1 100644 --- a/crates/tunnet-core/src/direct/mod.rs +++ b/crates/tunnet-core/src/direct/mod.rs @@ -54,8 +54,8 @@ pub use contact::{contact_id_from_endpoint, contact_id_from_hex, is_contact_id, #[cfg(feature = "direct")] pub use discovery::{DiscoveryHandle, spawn_discovery, spawn_seed_auth, topic_from_name_secret}; pub use firewall::{ - EvalResult, FirewallConfig, FirewallEngine, FirewallRule, FirewallStats, PacketDirection, - default_firewall, firewall_to_policy, + FirewallConfig, FirewallEngine, FirewallRule, FirewallStats, default_firewall, + firewall_to_policy, }; #[cfg(feature = "direct")] pub use grants::{ diff --git a/crates/tunnet-core/src/iroh_pool.rs b/crates/tunnet-core/src/iroh_pool.rs index fe159085..223edccd 100644 --- a/crates/tunnet-core/src/iroh_pool.rs +++ b/crates/tunnet-core/src/iroh_pool.rs @@ -174,15 +174,16 @@ pub struct ConnPool { /// Keyed by endpoint only for the pool's default ALPN (on-demand state). /// Secondary ALPNs use `extra` without idle management. entries: Arc>>>, + /// Established fast states (owned by routing; slow paths only: adopt, + /// dial, close, drop, heartbeats). The established packet path never + /// touches this — it uses the `Arc` from routing. + peer_registry: Arc>>>, extra: Arc, policy: Arc, metrics: Arc, - bytes_in: Arc>, - bytes_out: Arc>, tunnel_hook: Arc>>, cloud_relay_meter: CloudRelayMeter, cloud_relay_urls: Arc>>, - peer_cloud_relay: Arc>, } struct PoolPolicy { @@ -198,6 +199,7 @@ impl ConnPool { endpoint, alpn, entries: Arc::new(DashMap::new()), + peer_registry: Arc::new(Mutex::new(None)), extra: Arc::new(DashMap::new()), policy: Arc::new(PoolPolicy { keep_alive: AtomicBool::new(true), @@ -206,12 +208,9 @@ impl ConnPool { keep_alive_peers: DashMap::new(), }), metrics: Arc::new(PoolMetrics::default()), - bytes_in: Arc::new(DashMap::new()), - bytes_out: Arc::new(DashMap::new()), tunnel_hook: Arc::new(Mutex::new(None)), cloud_relay_meter: CloudRelayMeter::new(), cloud_relay_urls: Arc::new(RwLock::new(HashSet::new())), - peer_cloud_relay: Arc::new(DashMap::new()), }; pool.spawn_idle_sweeper(); pool @@ -226,23 +225,37 @@ impl ConnPool { endpoint, alpn, entries: Arc::new(DashMap::new()), + peer_registry: other.peer_registry.clone(), extra: Arc::new(DashMap::new()), policy: other.policy.clone(), metrics: other.metrics.clone(), - bytes_in: other.bytes_in.clone(), - bytes_out: other.bytes_out.clone(), tunnel_hook: Arc::new(Mutex::new(None)), cloud_relay_meter: other.cloud_relay_meter.clone(), cloud_relay_urls: other.cloud_relay_urls.clone(), - peer_cloud_relay: other.peer_cloud_relay.clone(), } } /// Register a hook invoked whenever this pool dials a tunnel connection. + /// Dialed connections are read ONLY by this hook (single ownership); + /// accepted connections are read ONLY by the accept path (`adopt` + /// never fires the hook). pub fn set_tunnel_hook(&self, hook: TunnelConnHook) { *self.tunnel_hook.lock() = Some(hook); } + /// Attach the shared established-peer registry (slow-path mirror for + /// adopt/dial/close/drop; the packet path never touches it). + pub fn set_peer_registry(&self, registry: Arc) { + *self.peer_registry.lock() = Some(registry); + } + + /// Mirror a live connection into its endpoint transport (slow paths only). + fn sync_fast_conn(&self, peer: EndpointId, conn: Option) { + if let Some(reg) = self.peer_registry.lock().clone() { + reg.set_transport_conn(peer, conn); + } + } + pub fn cloud_relay_meter(&self) -> CloudRelayMeter { self.cloud_relay_meter.clone() } @@ -252,20 +265,17 @@ impl ConnPool { let normalized: HashSet = urls.into_iter().map(|u| normalize_relay_url(&u)).collect(); *self.cloud_relay_urls.write() = normalized; - // Clear stale peer flags; path watchers will recompute on next event. - self.peer_cloud_relay.clear(); } fn spawn_cloud_relay_path_watch(&self, peer: EndpointId, conn: Connection) { let urls = self.cloud_relay_urls.clone(); - let flags = self.peer_cloud_relay.clone(); + let registry = self.peer_registry.lock().clone(); tokio::spawn(async move { let refresh = |conn: &Connection| { let metered = selected_path_is_cloud_relay(conn, &urls.read()); - flags - .entry(peer) - .or_insert_with(|| AtomicBool::new(false)) - .store(metered, Ordering::Relaxed); + if let Some(reg) = ®istry { + reg.refresh_transport_path(peer, Some(metered)); + } }; refresh(&conn); let mut events = conn.path_events(); @@ -280,7 +290,6 @@ impl ConnPool { _ => {} } } - flags.remove(&peer); }); } @@ -307,6 +316,13 @@ impl ConnPool { } /// Install an accepted connection. Returns false if tie-break keeps the existing conn. + /// + /// Ownership rule: whoever calls `adopt` owns reading this connection + /// (accept path) or takes over an existing reader explicitly. `adopt` + /// deliberately does NOT fire the dialer tunnel hook — that hook belongs + /// to connections this pool dialed itself (`get`), so each connection + /// has exactly one reader and datagrams are never split across two + /// tasks (the loser would silently eat packets before being aborted). pub async fn adopt(&self, peer: EndpointId, conn: Connection) -> bool { let local = self.endpoint.id(); let slot = self.slot(peer); @@ -314,6 +330,8 @@ impl ConnPool { if let Some(existing) = guard.live_conn() { if existing.stable_id() == conn.stable_id() { guard.touch(); + drop(guard); + self.sync_fast_conn(peer, Some(conn)); return true; } if !Self::prefer_incoming(local, peer, guard.opened_by_us, false) { @@ -328,7 +346,7 @@ impl ConnPool { guard.state = PeerConnState::Connected; guard.touch(); drop(guard); - self.fire_tunnel_hook(peer, conn); + self.sync_fast_conn(peer, Some(conn)); true } @@ -348,6 +366,7 @@ impl ConnPool { g.state = PeerConnState::Suspended; g.drop_buf(); tracing::debug!(%peer, "closed tunnel pool connection"); + self.sync_fast_conn(peer, None); } for entry in self.extra.iter() { let mut g = entry.value().lock().await; @@ -594,6 +613,11 @@ impl ConnPool { let buffered = guard.take_buf(); (conn, buffered, true) } else { + // Tie-break loss: keep the existing connection + // (already owned+read by whoever installed it) + // and close ours. Do NOT fire the hook: firing + // would spawn a second reader on a connection + // that already has one, splitting datagrams. let existing = existing.clone(); if let Some(tx) = guard.dial_waiters.take() { let _ = tx.send(Ok(existing.clone())); @@ -601,7 +625,7 @@ impl ConnPool { let buffered = guard.take_buf(); drop(guard); conn.close(0u32.into(), b"tie_break"); - (existing, buffered, true) + (existing, buffered, false) } } else { guard.conn = Some(conn.clone()); @@ -618,9 +642,10 @@ impl ConnPool { for pkt in buffered { if let Err(e) = send_datagram(&canonical, pkt).await { - tracing::warn!(%peer, ?e, "flush buffered datagram failed"); + tracing::debug!(%peer, ?e, "flush buffered datagram dropped"); } } + self.sync_fast_conn(peer, Some(canonical.clone())); if fire_hook { self.fire_tunnel_hook(peer, canonical.clone()); } @@ -665,6 +690,9 @@ impl ConnPool { } /// Send a packet, buffering + reconnecting when the peer is suspended (on-demand). + /// + /// Slow path only: connection setup, reconnect buffering, tie-breaking. + /// Established forwarding goes through `PeerMembershipState::try_send_frame`. pub async fn send_or_buffer(&self, peer: EndpointId, packet: Bytes) -> anyhow::Result<()> { let slot = self.slot(peer); { @@ -709,8 +737,19 @@ impl ConnPool { } } + /// Hard-drop a peer (§2.1-9, §2.2-1): close the live tunnel connection, + /// deactivate ALL of its memberships (epoch bumps close readers/pumps + /// holding Arcs), and forget the slot. Idempotent. pub async fn drop_peer(&self, peer: EndpointId) { - self.entries.remove(&peer); + if let Some((_, slot)) = self.entries.remove(&peer) { + let mut g = slot.lock().await; + if let Some(c) = g.conn.take() { + c.close(0u32.into(), b"membership_removed"); + } + } + if let Some(reg) = self.peer_registry.lock().clone() { + reg.remove_transport(peer); + } self.extra.retain(|(p, _), _| *p != peer); } @@ -734,6 +773,7 @@ impl ConnPool { } /// Counts live on-demand slots plus aggregated byte counters for heartbeats. + /// Byte totals come from the shared fast-state registry (slow path only). pub fn heartbeat_counters(&self) -> (u32, u64, u64) { let active_conns = self .entries @@ -743,56 +783,25 @@ impl ConnPool { Err(_) => true, }) .count() as u32; - let bytes_rx: u64 = self - .bytes_in - .iter() - .map(|e| e.value().load(Ordering::Relaxed)) - .sum(); - let bytes_tx: u64 = self - .bytes_out - .iter() - .map(|e| e.value().load(Ordering::Relaxed)) - .sum(); - (active_conns, bytes_tx, bytes_rx) + let (extra_conns, bytes_tx, bytes_rx) = self + .peer_registry + .lock() + .clone() + .map(|r| r.heartbeat_counters()) + .unwrap_or((0, 0, 0)); + (active_conns.max(extra_conns), bytes_tx, bytes_rx) } pub fn keep_alive_global(&self) -> bool { self.policy.keep_alive.load(Ordering::Relaxed) } - pub fn record_bytes_out(&self, peer: EndpointId, n: u64) { - self.bytes_out - .entry(peer) - .or_insert_with(|| AtomicU64::new(0)) - .fetch_add(n, Ordering::Relaxed); - if self - .peer_cloud_relay - .get(&peer) - .is_some_and(|f| f.load(Ordering::Relaxed)) - { - self.cloud_relay_meter.record(n); - } - } - - pub fn record_bytes_in(&self, peer: EndpointId, n: u64) { - self.bytes_in - .entry(peer) - .or_insert_with(|| AtomicU64::new(0)) - .fetch_add(n, Ordering::Relaxed); - } - pub fn peer_bytes(&self, peer: EndpointId) -> (u64, u64) { - let inn = self - .bytes_in - .get(&peer) - .map(|v| v.load(Ordering::Relaxed)) - .unwrap_or(0); - let out = self - .bytes_out - .get(&peer) - .map(|v| v.load(Ordering::Relaxed)) - .unwrap_or(0); - (inn, out) + self.peer_registry + .lock() + .clone() + .map(|r| r.peer_bytes(peer)) + .unwrap_or((0, 0)) } /// Best-effort snapshot of a peer's on-demand connection state. @@ -832,28 +841,59 @@ impl ConnPool { } } -/// Send a datagram, waiting for buffer space when congested instead of dropping. +/// Non-blocking DATAGRAM error. The scheduler owns drop/retry decisions; +/// the transport is never awaited while holding a stale packet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrySendError { + Full, + TooLarge, + Closed, +} + +impl std::fmt::Display for TrySendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Full => write!(f, "transport buffer full"), + Self::TooLarge => write!(f, "datagram_too_large"), + Self::Closed => write!(f, "connection closed"), + } + } +} + +impl std::error::Error for TrySendError {} + +/// Non-blocking DATAGRAM submit with Model A ownership (§0.6): never awaits +/// transport capacity, and never submits unless the reported free space fits +/// the ENTIRE frame. /// -/// Drops packets larger than the connection's current `max_datagram_size`. -pub async fn send_datagram(conn: &Connection, packet: Bytes) -> anyhow::Result<()> { +/// Iroh guarantees no older buffered datagram is displaced only when the new +/// datagram is `<= datagram_send_buffer_space()`; plain `send_datagram` +/// otherwise discards oldest-first to make room. Tunnet therefore treats +/// insufficient space as [`TrySendError::Full`] and lets its flow-aware +/// scheduler own the drop/retry decision (QUIC has no flow information). +pub fn try_send_datagram(conn: &Connection, packet: Bytes) -> Result<(), TrySendError> { + if conn.close_reason().is_some() { + return Err(TrySendError::Closed); + } if let Some(max) = conn.max_datagram_size() && packet.len() > max { - anyhow::bail!( - "datagram_too_large: packet {} > max_datagram_size {}", - packet.len(), - max - ); + return Err(TrySendError::TooLarge); } - if conn.datagram_send_buffer_space() == 0 { - conn.send_datagram_wait(packet) - .await - .context("send_datagram_wait (datagram buffer full or connection closed)")?; - return Ok(()); + if conn.datagram_send_buffer_space() < packet.len() { + return Err(TrySendError::Full); } - conn.send_datagram(packet) - .context("send_datagram (packet too big or unsupported)")?; - Ok(()) + conn.send_datagram(packet).map_err(|_| TrySendError::Closed) +} + +/// Send a datagram without ever awaiting transport capacity. +/// +/// Replaces the old `send_datagram_wait` semantics: when the QUIC DATAGRAM +/// buffer is full the packet is dropped (caller records `transport_full`) +/// instead of converting one stale packet into an arbitrarily long awaited +/// future that blocks the whole peer scheduler. +pub async fn send_datagram(conn: &Connection, packet: Bytes) -> anyhow::Result<()> { + try_send_datagram(conn, packet).map_err(|e| anyhow::anyhow!("{e}")) } #[cfg(test)] diff --git a/crates/tunnet-core/src/lib.rs b/crates/tunnet-core/src/lib.rs index 2c9e8348..2c1a0720 100644 --- a/crates/tunnet-core/src/lib.rs +++ b/crates/tunnet-core/src/lib.rs @@ -19,10 +19,14 @@ pub mod local_api; #[cfg(feature = "direct")] pub mod mdns_relay; pub mod node; +pub mod peers; pub mod ping; +pub mod policy_runtime; +pub mod reassembly; #[cfg(feature = "recording")] pub mod recording; pub mod routing; +pub mod scheduler; pub mod secret_store; #[cfg(feature = "send")] pub mod send; @@ -33,6 +37,7 @@ pub mod stream; pub mod stream_proxy; #[cfg(feature = "managed")] pub mod sync; +pub mod transport_profile; #[cfg(feature = "tunnel")] pub mod tunnel; #[cfg(feature = "managed")] @@ -52,12 +57,17 @@ pub use control::{ManagementClient, SignedClient, UnauthedClient}; pub use identity::AgentIdentity; #[cfg(feature = "direct")] pub use iroh_docs::protocol::Docs; -pub use iroh_pool::ConnPool; +pub use iroh_pool::{ConnPool, TrySendError, try_send_datagram}; pub use leave::leave_direct_network; #[cfg(feature = "direct")] pub use node::DirectNetworkRuntime; pub use node::{CoreNode, CoreNodeConfig}; -pub use routing::{PeerInfo, RoutingTable}; +pub use peers::{ + FastSendError, PeerIdentity, PeerMembershipState, PeerRegistry, PeerTransportState, +}; +pub use policy_runtime::AclDenyRecord; +pub use policy_runtime::{FwCounters, FwSet, FwSlot, FwSnapshot, PolicyRuntime, PolicyVerdict}; +pub use routing::{FastPeerHandle, PeerInfo, RouteDecision, RoutingTable}; #[cfg(feature = "send")] pub use send::{SendConfig, SendManager, TransferDirection, TransferRecord, TransferStatus}; #[cfg(feature = "serve")] diff --git a/crates/tunnet-core/src/local_api/dataplane.rs b/crates/tunnet-core/src/local_api/dataplane.rs index 7399cf24..f5a1baa6 100644 --- a/crates/tunnet-core/src/local_api/dataplane.rs +++ b/crates/tunnet-core/src/local_api/dataplane.rs @@ -5,9 +5,10 @@ //! from a cheap atomic snapshot, never via an actor round-trip. use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use async_trait::async_trait; +use parking_lot::Mutex; /// Narrow control capability used by the Local Management API. /// @@ -18,20 +19,65 @@ pub trait DataPlaneControl: Send + Sync { fn is_up(&self) -> bool; async fn bring_up(&self) -> Result<(), String>; async fn bring_down(&self) -> Result<(), String>; + /// Health detail for status rendering. Defaults to unknown/down; + /// the actor-backed implementation reads the shared snapshot. + fn data_plane_info(&self) -> tunnet_common::local_api::DataPlaneInfo { + tunnet_common::local_api::DataPlaneInfo { + state: DataPlaneState::Down.to_string(), + outbound_alive: false, + restart_count: 0, + generation: 0, + last_error: None, + } + } } /// Cheap shared read model for dataplane status. /// /// The `DataPlaneActor` is the only writer; HTTP GETs read this directly. -#[derive(Clone, Default)] +/// A dataplane with a dead packet worker must never report healthy: the +/// `state()` below distinguishes Up / Degraded / Restarting / Down from +/// the worker-liveness and restart flags (see the crash-loop incident +/// where `data plane up` masked a dead outbound loop). +#[derive(Clone, Default, Debug)] pub struct DataPlaneStatusSnapshot { up: Arc, + outbound_alive: Arc, + restarting: Arc, + restart_count: Arc, + generation: Arc, + last_error: Arc>>, +} + +/// Dataplane health state (rendered by `tunnet status`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataPlaneState { + Up, + Degraded, + Restarting, + Down, +} + +impl std::fmt::Display for DataPlaneState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Up => write!(f, "up"), + Self::Degraded => write!(f, "degraded"), + Self::Restarting => write!(f, "restarting"), + Self::Down => write!(f, "down"), + } + } } impl DataPlaneStatusSnapshot { pub fn new(up: bool) -> Self { Self { up: Arc::new(AtomicBool::new(up)), + outbound_alive: Arc::new(AtomicBool::new(up)), + restarting: Arc::new(AtomicBool::new(false)), + restart_count: Arc::new(AtomicU64::new(0)), + generation: Arc::new(AtomicU64::new(0)), + last_error: Arc::new(Mutex::new(None)), } } @@ -39,7 +85,99 @@ impl DataPlaneStatusSnapshot { self.up.load(Ordering::SeqCst) } + pub fn outbound_alive(&self) -> bool { + self.outbound_alive.load(Ordering::SeqCst) + } + pub fn set_up(&self, v: bool) { self.up.store(v, Ordering::SeqCst); } + + pub fn set_outbound_alive(&self, v: bool) { + self.outbound_alive.store(v, Ordering::SeqCst); + } + + pub fn set_restarting(&self, v: bool) { + self.restarting.store(v, Ordering::SeqCst); + } + + pub fn set_generation(&self, v: u64) { + self.generation.store(v, Ordering::SeqCst); + } + + pub fn note_restart(&self, error: String) { + self.restart_count.fetch_add(1, Ordering::SeqCst); + *self.last_error.lock() = Some(error); + } + + pub fn set_last_error(&self, error: String) { + *self.last_error.lock() = Some(error); + } + + pub fn restart_count(&self) -> u64 { + self.restart_count.load(Ordering::SeqCst) + } + + pub fn generation(&self) -> u64 { + self.generation.load(Ordering::SeqCst) + } + + pub fn last_error(&self) -> Option { + self.last_error.lock().clone() + } + + pub fn state(&self) -> DataPlaneState { + if self.restarting.load(Ordering::SeqCst) { + DataPlaneState::Restarting + } else if self.up.load(Ordering::SeqCst) { + if self.outbound_alive.load(Ordering::SeqCst) { + DataPlaneState::Up + } else { + DataPlaneState::Degraded + } + } else { + DataPlaneState::Down + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn health_state_transitions() { + // A dead packet worker must never read as healthy: crash sets + // restarting (not up), bring-up clears it, bring-down clears all. + let s = DataPlaneStatusSnapshot::new(false); + assert_eq!(s.state(), DataPlaneState::Down); + s.set_up(true); + s.set_outbound_alive(true); + assert_eq!(s.state(), DataPlaneState::Up); + // Worker death with the device half up: degraded, not up. + s.set_outbound_alive(false); + assert_eq!(s.state(), DataPlaneState::Degraded); + // Unexpected end: restarting + error + count, published before + // supervision restarts the actor. + s.note_restart("outbound TUN loop unexpectedly terminated".into()); + s.set_restarting(true); + assert_eq!(s.state(), DataPlaneState::Restarting); + assert_eq!(s.restart_count(), 1); + assert_eq!( + s.last_error().as_deref(), + Some("outbound TUN loop unexpectedly terminated") + ); + // Successful bring-up recovers fully. + s.set_up(true); + s.set_restarting(false); + s.set_outbound_alive(true); + s.set_generation(7); + assert_eq!(s.state(), DataPlaneState::Up); + assert_eq!(s.generation(), 7); + // Intentional shutdown: plain down. + s.set_up(false); + s.set_restarting(false); + s.set_outbound_alive(false); + assert_eq!(s.state(), DataPlaneState::Down); + } } diff --git a/crates/tunnet-core/src/local_api/handlers.rs b/crates/tunnet-core/src/local_api/handlers.rs index 09a2cc0a..1abff862 100644 --- a/crates/tunnet-core/src/local_api/handlers.rs +++ b/crates/tunnet-core/src/local_api/handlers.rs @@ -590,6 +590,13 @@ pub(crate) fn build_node_summary(state: &LocalApiState) -> NodeSummary { packets_dropped_timeout: od.packets_dropped_timeout, }), control, + daemon_git: Some(tunnet_common::git_hash().to_string()), + tunnel_alpn: Some( + std::str::from_utf8(tunnet_common::TUNNEL_ALPN) + .unwrap_or("?") + .to_string(), + ), + data_plane: Some(state.data_plane.data_plane_info()), } } @@ -646,6 +653,13 @@ pub(crate) fn idle_node_summary(daemon_version: &str) -> NodeSummary { networks: vec![], on_demand: None, control: None, + daemon_git: Some(tunnet_common::git_hash().to_string()), + tunnel_alpn: Some( + std::str::from_utf8(tunnet_common::TUNNEL_ALPN) + .unwrap_or("?") + .to_string(), + ), + data_plane: None, } } diff --git a/crates/tunnet-core/src/node.rs b/crates/tunnet-core/src/node.rs index 9b314383..4e17a132 100644 --- a/crates/tunnet-core/src/node.rs +++ b/crates/tunnet-core/src/node.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "direct")] +#[cfg(any(feature = "managed", feature = "direct"))] use std::collections::HashMap; use std::sync::Arc; #[cfg(feature = "direct")] @@ -36,6 +36,7 @@ use crate::direct::{ use crate::direct::{ConnectivityOptions, apply_connectivity, endpoint_builder}; use crate::identity::AgentIdentity; use crate::iroh_pool::ConnPool; +use crate::policy_runtime::PolicyRuntime; use crate::routing::RoutingTable; #[cfg(feature = "send")] use crate::send::SendManager; @@ -121,6 +122,8 @@ pub struct CoreNode { pub effective_config: crate::EffectiveConfigStore, pub routes: RoutingTable, pub acl: AclEngine, + /// Shared packet-policy runtime (dataplane authority, §0.1). + pub policy: PolicyRuntime, pub version: Arc>, pub self_ipv4: std::net::Ipv4Addr, pub paths: StatePaths, @@ -453,39 +456,48 @@ impl CoreNode { None }; - Ok(( - Self { - identity, - persisted: PersistedState::Managed(managed), - endpoint, - pool, - tunnel_pool, - effective_config, - routes, - acl, - version, - self_ipv4: membership.assigned_ipv4, - paths, - #[cfg(feature = "serve")] - serves, - #[cfg(feature = "tunnel")] - tunnels, - #[cfg(feature = "send")] - send, - signed: Some(signed), - control_link, - #[cfg(feature = "direct")] - direct_auth: None, - #[cfg(feature = "direct")] - direct: HashMap::new(), - gossip, - #[cfg(feature = "direct")] - docs_engine: None, - #[cfg(feature = "direct")] - presence_tables: Arc::new(Mutex::new(HashMap::new())), - }, - Some(pending), - )) + // Shared packet-policy runtime: bootstrapped from live control state, + // then wired (engine attach, peer relink, pool registries) below. + let policy = PolicyRuntime::bootstrap( + &acl.bundle.load(), + &HashMap::new(), + &acl.self_id.load(), + **acl.src_posture_ok.load(), + **acl.stale.load(), + ); + let mut node = Self { + identity, + persisted: PersistedState::Managed(managed), + endpoint, + pool, + tunnel_pool, + effective_config, + routes, + acl, + version, + self_ipv4: membership.assigned_ipv4, + paths, + #[cfg(feature = "serve")] + serves, + #[cfg(feature = "tunnel")] + tunnels, + #[cfg(feature = "send")] + send, + signed: Some(signed), + control_link, + #[cfg(feature = "direct")] + direct_auth: None, + #[cfg(feature = "direct")] + direct: HashMap::new(), + gossip, + #[cfg(feature = "direct")] + docs_engine: None, + #[cfg(feature = "direct")] + presence_tables: Arc::new(Mutex::new(HashMap::new())), + policy, + }; + node.install_policy_runtime(); + Ok((node, Some(pending))) } #[cfg(feature = "direct")] @@ -681,7 +693,28 @@ impl CoreNode { tracing::info!(%contact, networks = direct_runtimes.len(), "direct contact id"); let _ = cfg.agent_version; - Ok(Self { + // Shared packet-policy runtime from live control state (per-network + // firewall sets included), then wired below. + let mut fw_source = HashMap::new(); + for (network_id, runtime) in &direct_runtimes { + let fw = &runtime.firewall; + fw_source.insert( + *network_id, + ( + fw.local_rules_snapshot(), + fw.suggested_rules_snapshot(), + fw.stats().enabled, + ), + ); + } + let policy = PolicyRuntime::bootstrap( + &acl.bundle.load(), + &fw_source, + &acl.self_id.load(), + **acl.src_posture_ok.load(), + **acl.stale.load(), + ); + let mut node = Self { identity, persisted: PersistedState::Direct { networks: persisted_networks, @@ -710,7 +743,10 @@ impl CoreNode { gossip: Some(gossip), docs_engine: Some(docs_engine), presence_tables: Arc::new(Mutex::new(HashMap::new())), - }) + policy, + }; + node.install_policy_runtime(); + Ok(node) } /// Shared Gossip for presence / service-relay topics. @@ -718,6 +754,25 @@ impl CoreNode { self.gossip.clone() } + /// Wire the shared policy runtime after construction (both modes): + /// attach engines (future mutations publish automatically), install it + /// on the routing table (slot assignment at every (re)resolution), + /// relink existing peer fast states once, and hand pool slow paths the + /// peer registry. Publication after this point needs no relink (§2.1-3). + pub fn install_policy_runtime(&mut self) { + self.acl.attach_runtime(self.policy.clone()); + #[cfg(feature = "direct")] + for runtime in self.direct.values() { + runtime.firewall.attach_runtime(self.policy.clone()); + } + self.routes.set_policy_runtime(self.policy.clone()); + self.routes.peer_registry().relink_policy(&self.policy); + self.pool + .set_peer_registry(self.routes.peer_registry().clone()); + self.tunnel_pool + .set_peer_registry(self.routes.peer_registry().clone()); + } + pub fn endpoint_id_hex(&self) -> String { self.identity.endpoint_id_hex() } @@ -830,8 +885,12 @@ async fn bootstrap_one_direct_network( let fw_cfg = crate::agent_config::load_firewall_for(args.paths, &direct.network_name); let policy = firewall_to_policy(&fw_cfg, args.my_id_hex, net_ipv4); - let firewall = - crate::direct::FirewallEngine::from_config(&fw_cfg, net_ipv4, args.my_id_hex.to_string()); + let firewall = crate::direct::FirewallEngine::from_config( + &fw_cfg, + net_ipv4, + args.my_id_hex.to_string(), + direct.network_id, + ); let spoof_tracker = crate::direct::SpoofTracker::new(); let self_entry = MembershipEntry { diff --git a/crates/tunnet-core/src/peers.rs b/crates/tunnet-core/src/peers.rs new file mode 100644 index 00000000..df5d34d4 --- /dev/null +++ b/crates/tunnet-core/src/peers.rs @@ -0,0 +1,801 @@ +//! Established-peer state (§0.5, §2.2-1). +//! +//! Transport identity and network membership are SEPARATE objects because +//! one endpoint may belong to many networks (Direct mode): +//! +//! ```text +//! PeerTransportState (key: EndpointId) +//! live QUIC connection, MPS/RTT/path state, transport counters, +//! frame-ID counter (unique across the endpoint's memberships) +//! +//! PeerMembershipState (key: (EndpointId, NetworkId)) +//! network_id, mesh IP, hostname/tags, network firewall slot, +//! per-membership scheduler + reassembly, pump task + epoch +//! ``` +//! +//! There is no mutable network identity inside endpoint-global transport +//! state, and no endpoint-global scheduler shared across networks. Routing +//! hands out `Arc` clones embedded in peer handles; +//! inbound readers resolve (endpoint, frame network) per connection and +//! switch membership when the frame network changes. +//! +//! The registries (transport + membership DashMaps) are touched only on +//! slow paths: creation, reconnect, teardown, policy relink, heartbeats. + +use std::net::Ipv4Addr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use std::time::Duration; + +use arc_swap::{ArcSwap, ArcSwapOption}; +use dashmap::DashMap; +use iroh::EndpointId; +use iroh::endpoint::Connection; +use parking_lot::{Mutex, RwLock}; +use tokio::sync::Notify; +use uuid::Uuid; + +use crate::policy_runtime::{FwSlot, PolicyRuntime}; +use crate::reassembly::ReassemblyTable; +use crate::scheduler::PeerScheduler; + +/// Per-network peer identity: who this endpoint IS in one network. +/// The same endpoint has one of these per network it belongs to — never a +/// single mutable network context shared across networks. +#[derive(Debug, Clone)] +pub struct PeerIdentity { + pub endpoint: EndpointId, + pub endpoint_hex: String, + pub hostname: String, + pub ip: Ipv4Addr, + pub tags: Vec, + pub network_id: Uuid, + pub network_name: String, +} + +/// Default DRR quantum: one logical MTU-ish chunk (retuned with MPS). +pub const DEFAULT_QUANTUM: usize = 1536; +/// Default effective DATAGRAM payload before the first measurement. +pub const DEFAULT_MPS: usize = 1280; + +/// Endpoint-global transport state: the live QUIC connection and path +/// measurements shared by all of the endpoint's network memberships. +/// Carries NO network identity, NO firewall state, NO scheduler. +pub struct PeerTransportState { + pub endpoint: EndpointId, + pub conn: ArcSwapOption, + pub tx_packets: AtomicU64, + pub tx_bytes: AtomicU64, + pub rx_packets: AtomicU64, + pub rx_bytes: AtomicU64, + pub last_activity_ms: AtomicU64, + pub relay: AtomicBool, + /// Effective DATAGRAM payload size (frame bytes), adapted to path MTU. + pub mps: AtomicUsize, + /// Cached RTT millis for adaptive backoff (updated by path watcher). + pub rtt_ms: AtomicU64, + /// Frame-ID counter shared across memberships (unique per endpoint). + pub next_frame_id: AtomicU32, + /// Sends since the last MPS refresh (periodic re-measurement). + pub sends_since_mps_check: AtomicU64, +} + +impl PeerTransportState { + fn new(endpoint: EndpointId) -> Arc { + Arc::new(Self { + endpoint, + conn: ArcSwapOption::empty(), + tx_packets: AtomicU64::new(0), + tx_bytes: AtomicU64::new(0), + rx_packets: AtomicU64::new(0), + rx_bytes: AtomicU64::new(0), + last_activity_ms: AtomicU64::new(now_millis()), + relay: AtomicBool::new(false), + mps: AtomicUsize::new(DEFAULT_MPS), + rtt_ms: AtomicU64::new(90), + next_frame_id: AtomicU32::new(rand::random()), + sends_since_mps_check: AtomicU64::new(0), + }) + } + + /// Non-blocking DATAGRAM submit with Model A ownership (§0.6): submit + /// only when the reported free space fits the ENTIRE frame, so QUIC never + /// silently displaces an older buffered datagram behind our back. + /// + /// The frame is returned on EVERY error path (§2.1-8) — including a + /// failed `send_datagram` after the prechecks passed (via a cheap + /// refcount clone handed to QUIC) — so the pump can requeue or resume + /// losslessly. A stall never consumes bytes. + pub fn try_send_frame(&self, frame: bytes::Bytes) -> Result<(), (FastSendError, bytes::Bytes)> { + let frame_len = frame.len(); + let Some(conn) = self.conn.load_full() else { + return Err((FastSendError::NoConnection, frame)); + }; + if conn.close_reason().is_some() { + self.conn.store(None); + return Err((FastSendError::NoConnection, frame)); + } + if let Some(max) = conn.max_datagram_size() + && frame_len > max + { + return Err((FastSendError::TooLarge, frame)); + } + if conn.datagram_send_buffer_space() < frame_len { + return Err((FastSendError::TransportFull, frame)); + } + // Clone before handing to QUIC: `send_datagram` consumes its + // argument without returning it on error, so without this clone a + // late failure would silently eat the frame and break the + // ownership/requeue invariant. `Bytes::clone` is a refcount bump. + match conn.send_datagram(frame.clone()) { + Ok(()) => { + self.tx_packets.fetch_add(1, Ordering::Relaxed); + self.tx_bytes.fetch_add(frame_len as u64, Ordering::Relaxed); + self.touch(); + Ok(()) + } + Err(_) => Err((FastSendError::Closed, frame)), + } + } + + /// Refresh the cached MPS from the live connection (slow-ish: locks the + /// QUIC connection state; called periodically, not per packet). + pub fn refresh_mps(&self) -> Option { + let conn = self.conn.load_full()?; + let mps = conn.max_datagram_size()?; + self.mps.store(mps, Ordering::Relaxed); + // Sample RTT from the selected path for adaptive backoff. + if let Some(rtt) = conn + .paths() + .iter() + .find(|p| p.is_selected()) + .map(|p| p.stats().rtt) + { + self.rtt_ms.store( + rtt.as_millis().min(u128::from(u64::MAX)) as u64, + Ordering::Relaxed, + ); + } + Some(mps) + } + + pub fn live_conn(&self) -> Option { + let conn = self.conn.load_full()?; + if conn.close_reason().is_some() { + return None; + } + Some(conn.as_ref().clone()) + } + + pub fn touch(&self) { + let now = now_millis(); + let last = self.last_activity_ms.load(Ordering::Relaxed); + if now.wrapping_sub(last) >= 1000 { + self.last_activity_ms.store(now, Ordering::Relaxed); + } + } + + pub fn record_rx(&self, n: u64) { + self.rx_packets.fetch_add(1, Ordering::Relaxed); + self.rx_bytes.fetch_add(n, Ordering::Relaxed); + self.touch(); + } +} + +/// Per-(endpoint, network) membership state: everything the established +/// packet path needs for ONE network, so after routing there are no map +/// lookups, no async mutexes, and no string conversions. +pub struct PeerMembershipState { + /// Shared endpoint transport (connection, MPS, counters). + pub transport: Arc, + pub identity: RwLock>, + /// Stable network firewall slot (§2.1-3): assigned once per network + /// (re)resolution, swapped in place by firewall publication. The hot + /// path loads set + counters with two atomic loads — no map lookup, + /// no relink. + pub policy: ArcSwap, + pub scheduler: Mutex, + pub reassembly: Mutex, + pub notify: Notify, + pub pump_running: AtomicBool, + /// Membership epoch: bumped when THIS membership is revoked. Its pump + /// drains and exits; readers holding this Arc observe the change. Other + /// memberships of the same endpoint are unaffected. + pub epoch: AtomicU64, +} + +impl PeerMembershipState { + pub fn new( + transport: Arc, + identity: Arc, + reassembly_budget: Arc, + ) -> Arc { + Arc::new(Self { + transport, + identity: RwLock::new(identity), + policy: ArcSwap::from_pointee(FwSlot::default()), + scheduler: Mutex::new(PeerScheduler::new(DEFAULT_QUANTUM)), + reassembly: Mutex::new(ReassemblyTable::new(reassembly_budget)), + notify: Notify::new(), + pump_running: AtomicBool::new(false), + epoch: AtomicU64::new(0), + }) + } + + /// Hard-deactivate THIS membership (§2.1-9, §2.2-1): epoch bump (its + /// pump drains and exits; readers holding this Arc observe it) plus a + /// pump wakeup for prompt exit. Never touches the shared transport + /// connection — sibling memberships keep working. Idempotent. + pub fn deactivate(&self) { + self.epoch.fetch_add(1, Ordering::Relaxed); + self.notify.notify_one(); + } + + /// Refresh path measurements from the shared transport and retune this + /// membership's DRR quantum to the effective payload. + pub fn refresh_mps(&self) -> Option { + let mps = self.transport.refresh_mps()?; + // Scale the DRR quantum with the effective payload: one logical + // MTU-ish chunk keeps DRR fair as paths change. + self.scheduler.lock().set_quantum(mps.max(512)); + Some(mps) + } +} + +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FastSendError { + /// No live connection: caller must take the slow reconnect path. + NoConnection, + /// QUIC DATAGRAM buffer full: scheduler owns the drop/retry decision. + TransportFull, + TooLarge, + Closed, +} + +impl std::fmt::Display for FastSendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoConnection => write!(f, "no live connection"), + Self::TransportFull => write!(f, "transport buffer full"), + Self::TooLarge => write!(f, "datagram_too_large"), + Self::Closed => write!(f, "connection closed"), + } + } +} + +impl std::error::Error for FastSendError {} + +/// Slow-path-only registries. Packet paths never touch these maps: routing +/// embeds `Arc` in peer handles and inbound readers +/// cache one `Arc` per (endpoint, network). +#[derive(Clone, Default)] +pub struct PeerRegistry { + transports: Arc>>, + memberships: Arc>>, + reassembly_budget: Arc, +} + +impl PeerRegistry { + pub fn new() -> Self { + Self { + transports: Arc::new(DashMap::new()), + memberships: Arc::new(DashMap::new()), + reassembly_budget: Arc::new(AtomicU64::new(0)), + } + } + + /// Shared reassembly byte counter (global cap denominator). + pub fn reassembly_budget(&self) -> &Arc { + &self.reassembly_budget + } + + /// Get-or-create the endpoint transport (slow path). + pub fn ensure_transport(&self, endpoint: EndpointId) -> Arc { + self.transports + .entry(endpoint) + .or_insert_with(|| PeerTransportState::new(endpoint)) + .clone() + } + + /// Get-or-create the (endpoint, network) membership (slow path: + /// routing rebuild, adopt, dial). The transport is shared across the + /// endpoint's memberships; identity refreshes in place. Refreshing with + /// a DIFFERENT network id is a caller bug — memberships are keyed by + /// network, so assert instead of silently mutating (no last-writer-wins). + pub fn ensure_membership(&self, identity: Arc) -> Arc { + let key = (identity.endpoint, identity.network_id); + if let Some(existing) = self.memberships.get(&key) { + let current = existing.identity.read().clone(); + debug_assert_eq!( + current.network_id, identity.network_id, + "membership key/network mismatch" + ); + // Refresh mutable context in place; the object stays stable. + *existing.identity.write() = identity; + return existing.value().clone(); + } + let transport = self.ensure_transport(identity.endpoint); + let state = + PeerMembershipState::new(transport, identity.clone(), self.reassembly_budget.clone()); + self.memberships.entry(key).or_insert(state).clone() + } + + /// Backwards-compatible single-network ensure (tests, legacy callers): + /// exactly `ensure_membership`. + pub fn ensure(&self, identity: Arc) -> Arc { + self.ensure_membership(identity) + } + + pub fn get_transport(&self, peer: EndpointId) -> Option> { + self.transports.get(&peer).map(|e| e.value().clone()) + } + + /// True when the endpoint holds any network membership (reader-exit + /// check: a connection with zero memberships left is dead). + pub fn has_any_membership(&self, peer: EndpointId) -> bool { + self.memberships.iter().any(|e| e.key().0 == peer) + } + + pub fn get_membership( + &self, + peer: EndpointId, + network: Uuid, + ) -> Option> { + self.memberships + .get(&(peer, network)) + .map(|e| e.value().clone()) + } + + /// Backwards-compatible get (legacy callers that only know the + /// endpoint): returns a membership only when the endpoint has EXACTLY + /// ONE — ambiguous endpoints must resolve with a network. Never guesses. + pub fn get(&self, peer: EndpointId) -> Option> { + let mut found = None; + for entry in self.memberships.iter() { + if entry.key().0 == peer { + if found.is_some() { + return None; + } + found = Some(entry.value().clone()); + } + } + found + } + + /// Mirror a live connection into the endpoint transport (slow paths + /// only). `Some` stores + re-measures + resets frame pacing + retunes + /// member schedulers; `None` clears the connection and deactivates all + /// of the endpoint's memberships (teardown without replacement). + pub fn set_transport_conn(&self, peer: EndpointId, conn: Option) { + let transport = self.ensure_transport(peer); + match conn { + Some(c) => { + transport.conn.store(Some(Arc::new(c.clone()))); + // Fresh connection: reset pacing state to measured values. + transport + .next_frame_id + .store(rand::random(), Ordering::Relaxed); + transport.sends_since_mps_check.store(0, Ordering::Relaxed); + drop(transport); + self.refresh_transport_path(peer, None); + } + None => { + transport.conn.store(None); + drop(transport); + for entry in self.memberships.iter() { + if entry.key().0 == peer { + entry.value().deactivate(); + } + } + } + } + } + + /// Path-event refresh (slow path): re-measure transport MPS/RTT, + /// optionally update the relay flag, and retune member schedulers. + pub fn refresh_transport_path(&self, peer: EndpointId, metered: Option) { + let Some(t) = self.transports.get(&peer).map(|e| e.value().clone()) else { + return; + }; + if let Some(m) = metered { + t.relay.store(m, Ordering::Relaxed); + } + if let Some(mps) = t.refresh_mps() { + for entry in self.memberships.iter() { + if entry.key().0 == peer { + entry.value().scheduler.lock().set_quantum(mps.max(512)); + } + } + } + } + + /// Legacy single-peer set_conn (pool slow path): delegates to + /// `set_transport_conn`. + pub fn set_conn(&self, peer: EndpointId, conn: Option) { + self.set_transport_conn(peer, conn); + } + + /// Remove ONE membership (network revoked, endpoint stays for others): + /// deactivate it, forget it. The shared transport connection is + /// untouched — sibling memberships keep working. + pub fn remove_membership(&self, peer: EndpointId, network: Uuid) { + if let Some((_, state)) = self.memberships.remove(&(peer, network)) { + // Hard revoke (§2.2-1): readers holding the Arc observe the + // epoch bump and exit; its pump drains and stops. + state.deactivate(); + } + self.prune_empty_transport(peer); + } + + /// Remove the endpoint entirely (all memberships + transport): + /// deactivate every membership, close the live tunnel connection, + /// forget everything. + pub fn remove_transport(&self, peer: EndpointId) { + let mut members = Vec::new(); + self.memberships.retain(|k, v| { + if k.0 == peer { + members.push(v.clone()); + false + } else { + true + } + }); + for m in members { + m.deactivate(); + } + if let Some((_, t)) = self.transports.remove(&peer) + && let Some(conn) = t.conn.swap(None) + { + conn.close(0u32.into(), b"membership_removed"); + } + } + + /// Legacy remove: full endpoint removal. + pub fn remove(&self, peer: EndpointId) { + self.remove_transport(peer); + } + + /// Retain only live (endpoint, network) memberships (slow path: routing + /// rebuild prunes departed memberships). Removed memberships are + /// hard-deactivated first, so no stale Arc keeps forwarding through + /// dead identity/policy state. Transports left with no memberships are + /// closed and forgotten. + pub fn retain(&self, live: &std::collections::HashSet<(EndpointId, Uuid)>) { + let mut departed = Vec::new(); + self.memberships.retain(|k, v| { + let keep = live.contains(k); + if !keep { + departed.push(v.clone()); + } + keep + }); + for state in departed { + state.deactivate(); + } + let live_eps: std::collections::HashSet = + live.iter().map(|(ep, _)| *ep).collect(); + let mut closed = Vec::new(); + self.transports.retain(|ep, t| { + let keep = live_eps.contains(ep); + if !keep { + closed.push(t.clone()); + } + keep + }); + for t in closed { + if let Some(conn) = t.conn.swap(None) { + conn.close(0u32.into(), b"membership_removed"); + } + } + } + + /// Legacy retain by endpoint set (pool-era callers): keeps every + /// membership of a live endpoint. Prefer the (endpoint, network) form. + pub fn retain_endpoints(&self, live: &std::collections::HashSet) { + let mut departed = Vec::new(); + self.memberships.retain(|k, v| { + let keep = live.contains(&k.0); + if !keep { + departed.push(v.clone()); + } + keep + }); + for state in departed { + state.deactivate(); + } + let mut closed = Vec::new(); + self.transports.retain(|ep, t| { + let keep = live.contains(ep); + if !keep { + closed.push(t.clone()); + } + keep + }); + for t in closed { + if let Some(conn) = t.conn.swap(None) { + conn.close(0u32.into(), b"membership_removed"); + } + } + } + + /// Drop a transport left with no memberships (after single-membership + /// removal): close its connection so no orphan conn lingers. + fn prune_empty_transport(&self, peer: EndpointId) { + if self.memberships.iter().any(|e| e.key().0 == peer) { + return; + } + if let Some((_, t)) = self.transports.remove(&peer) + && let Some(conn) = t.conn.swap(None) + { + conn.close(0u32.into(), b"membership_removed"); + } + } + + pub fn clear(&self) { + let all: Vec<_> = self.memberships.iter().map(|e| e.value().clone()).collect(); + self.memberships.clear(); + for state in all { + state.deactivate(); + } + let conns: Vec<_> = self.transports.iter().map(|e| e.value().clone()).collect(); + self.transports.clear(); + for t in conns { + if let Some(conn) = t.conn.swap(None) { + conn.close(0u32.into(), b"membership_removed"); + } + } + } + + /// Install-time policy slot assignment (slow/control path): every + /// membership points at its network's stable slot. Firewall publication + /// NEVER needs this — slots swap in place (§2.1-3). + pub fn relink_policy(&self, runtime: &PolicyRuntime) { + for entry in self.memberships.iter() { + let state = entry.value(); + let network = state.identity.read().network_id; + state.policy.store(runtime.slot_for_network(network)); + } + } + + /// Heartbeat aggregates (slow path only). + pub fn heartbeat_counters(&self) -> (u32, u64, u64) { + let mut conns = 0u32; + let mut tx = 0u64; + let mut rx = 0u64; + for entry in self.transports.iter() { + let s = entry.value(); + if s.live_conn().is_some() { + conns += 1; + } + tx += s.tx_bytes.load(Ordering::Relaxed); + rx += s.rx_bytes.load(Ordering::Relaxed); + } + (conns, tx, rx) + } + + pub fn peer_bytes(&self, peer: EndpointId) -> (u64, u64) { + match self.transports.get(&peer) { + Some(s) => ( + s.rx_bytes.load(Ordering::Relaxed), + s.tx_bytes.load(Ordering::Relaxed), + ), + None => (0, 0), + } + } + + /// Adaptive transport-full backoff (§0.7): RTT/4 clamped to + /// [100µs, max]. The ceiling defaults to 2 ms and can be raised for + /// A/B runs via `TUNNET_PUMP_BACKOFF_MAX_US` (diagnostic only). No + /// fixed 5 ms stall, no spin, no send_datagram_wait. New enqueues + /// notify immediately, so this timeout is only the no-new-work + /// fallback. (A public `datagrams_unblocked` waiter in Iroh/noq would + /// be the cleaner upstream primitive; investigated, not available — + /// the internal Notify stays private.) + pub fn backoff_for(transport: &PeerTransportState) -> Duration { + static MAX_MICROS: std::sync::OnceLock = std::sync::OnceLock::new(); + let max = *MAX_MICROS.get_or_init(|| { + std::env::var("TUNNET_PUMP_BACKOFF_MAX_US") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 100) + .unwrap_or(2000) + }); + let rtt_ms = transport.rtt_ms.load(Ordering::Relaxed); + let micros = rtt_ms.saturating_mul(250).clamp(100, max); + Duration::from_micros(micros) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iroh::SecretKey; + + fn test_endpoint() -> EndpointId { + SecretKey::generate().public() + } + + fn identity(endpoint: EndpointId) -> Arc { + identity_in(endpoint, Uuid::nil(), [10, 0, 0, 2]) + } + + fn identity_in(endpoint: EndpointId, network: Uuid, ip: [u8; 4]) -> Arc { + Arc::new(PeerIdentity { + endpoint, + endpoint_hex: format!("{endpoint}"), + hostname: "peer".into(), + ip: std::net::Ipv4Addr::from(ip), + tags: vec![], + network_id: network, + network_name: "net".into(), + }) + } + + const NET_A: Uuid = Uuid::from_u128(0x0a0a); + const NET_B: Uuid = Uuid::from_u128(0x0b0b); + + #[test] + fn registry_reuses_stable_state() { + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let a = reg.ensure(identity(ep)); + let b = reg.ensure(identity(ep)); + assert!(Arc::ptr_eq(&a, &b), "same stable object"); + // Identity refresh keeps the object. + let mut id = identity(ep); + let idm = Arc::get_mut(&mut id).unwrap(); + idm.hostname = "renamed".into(); + let c = reg.ensure(id); + assert!(Arc::ptr_eq(&a, &c)); + assert_eq!(c.identity.read().hostname, "renamed"); + } + + #[test] + fn try_send_without_conn_returns_frame() { + // §2.1-8: every error path returns the frame for lossless requeue. + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let s = reg.ensure(identity(ep)); + let frame = bytes::Bytes::from_static(b"frame-bytes"); + let (err, back) = s.transport.try_send_frame(frame.clone()).unwrap_err(); + assert_eq!(err, FastSendError::NoConnection); + assert_eq!(back, frame, "frame must come back byte-identical"); + } + + #[test] + fn removal_deactivates_fast_state() { + // §2.1-9: removing a peer hard-revokes its fast state — epoch + // bumped (pumps/readers holding the Arc observe it and exit), + // connection cleared. A subsequent resolve finds nothing. + use std::collections::HashSet; + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let s = reg.ensure(identity(ep)); + let epoch0 = s.epoch.load(Ordering::Relaxed); + // Simulate a live peer: epoch + scheduler contents (pump-owned). + s.epoch.fetch_add(0, Ordering::Relaxed); + assert!(reg.get(ep).is_some()); + reg.remove(ep); + assert!(reg.get(ep).is_none(), "removed peer must not resolve"); + assert!(reg.get_transport(ep).is_none(), "transport forgotten too"); + assert_eq!( + s.epoch.load(Ordering::Relaxed), + epoch0 + 1, + "reader/pump exit signal" + ); + assert!(s.transport.conn.load_full().is_none()); + // Retain with an empty live set deactivates too. + let ep2 = test_endpoint(); + let s2 = reg.ensure(identity(ep2)); + let epoch2 = s2.epoch.load(Ordering::Relaxed); + reg.retain(&HashSet::new()); + assert!(reg.get(ep2).is_none()); + assert_eq!(s2.epoch.load(Ordering::Relaxed), epoch2 + 1); + } + + #[test] + fn same_endpoint_two_networks_isolated() { + // §2.2-1 (tests 1, 2, 10): one EndpointId in networks A and B gets + // two independent membership states sharing one transport. Ensuring + // B never mutates A's identity (no last-writer-wins), in either + // insertion order (reverse order covered by the next test). + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let id_a = identity_in(ep, NET_A, [10, 0, 0, 2]); + let id_b = identity_in(ep, NET_B, [10, 0, 1, 2]); + let a = reg.ensure_membership(id_a); + let b = reg.ensure_membership(id_b); + assert!(!Arc::ptr_eq(&a, &b), "distinct membership objects"); + // Shared transport, distinct memberships. + assert!(Arc::ptr_eq(&a.transport, &b.transport)); + assert_eq!(a.identity.read().network_id, NET_A); + assert_eq!(b.identity.read().network_id, NET_B); + assert_eq!(a.identity.read().ip, std::net::Ipv4Addr::new(10, 0, 0, 2)); + assert_eq!(b.identity.read().ip, std::net::Ipv4Addr::new(10, 0, 1, 2)); + // Exact resolution per (endpoint, network). + assert!( + reg.get_membership(ep, NET_A) + .is_some_and(|m| Arc::ptr_eq(&m, &a)) + ); + assert!( + reg.get_membership(ep, NET_B) + .is_some_and(|m| Arc::ptr_eq(&m, &b)) + ); + // Bare endpoint resolve refuses to guess across networks. + assert!(reg.get(ep).is_none(), "ambiguous endpoint must not resolve"); + } + + #[test] + fn same_endpoint_two_networks_reverse_order() { + // Insert B before A: A must still resolve exactly, with its own IP. + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let b = reg.ensure_membership(identity_in(ep, NET_B, [10, 0, 1, 2])); + let a = reg.ensure_membership(identity_in(ep, NET_A, [10, 0, 0, 2])); + assert_eq!(a.identity.read().ip, std::net::Ipv4Addr::new(10, 0, 0, 2)); + assert_eq!(b.identity.read().ip, std::net::Ipv4Addr::new(10, 0, 1, 2)); + assert!( + reg.get_membership(ep, NET_A) + .is_some_and(|m| Arc::ptr_eq(&m, &a)) + ); + } + + #[test] + fn removing_one_membership_leaves_sibling() { + // §2.2-1 (tests 8, 9): revoking A deactivates only A; B keeps its + // epoch, transport, and resolvability. + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let a = reg.ensure_membership(identity_in(ep, NET_A, [10, 0, 0, 2])); + let b = reg.ensure_membership(identity_in(ep, NET_B, [10, 0, 1, 2])); + let epoch_b = b.epoch.load(Ordering::Relaxed); + reg.remove_membership(ep, NET_A); + assert!(reg.get_membership(ep, NET_A).is_none()); + assert_eq!(a.epoch.load(Ordering::Relaxed), 1); + // Sibling untouched: same epoch, still resolvable, transport alive. + assert_eq!(b.epoch.load(Ordering::Relaxed), epoch_b); + assert!( + reg.get_membership(ep, NET_B) + .is_some_and(|m| Arc::ptr_eq(&m, &b)) + ); + assert!(reg.get_transport(ep).is_some()); + // Endpoint-wide get() now unambiguous again. + assert!(reg.get(ep).is_some_and(|m| Arc::ptr_eq(&m, &b))); + } + + #[test] + fn backoff_bounds() { + let reg = PeerRegistry::new(); + let ep = test_endpoint(); + let s = reg.ensure(identity(ep)); + s.transport.rtt_ms.store(0, Ordering::Relaxed); + assert_eq!( + PeerRegistry::backoff_for(&s.transport), + Duration::from_micros(100) + ); + s.transport.rtt_ms.store(10_000, Ordering::Relaxed); + assert_eq!( + PeerRegistry::backoff_for(&s.transport), + Duration::from_micros(2000) + ); + s.transport.rtt_ms.store(90, Ordering::Relaxed); + // 90 ms → 22.5 ms raw, clamped to the 2 ms ceiling. + assert_eq!( + PeerRegistry::backoff_for(&s.transport), + Duration::from_micros(2000) + ); + s.transport.rtt_ms.store(4, Ordering::Relaxed); + // 4 ms → 1 ms raw, inside the band. + assert_eq!( + PeerRegistry::backoff_for(&s.transport), + Duration::from_micros(1000) + ); + } +} diff --git a/crates/tunnet-core/src/policy_runtime.rs b/crates/tunnet-core/src/policy_runtime.rs new file mode 100644 index 00000000..d7bcb93a --- /dev/null +++ b/crates/tunnet-core/src/policy_runtime.rs @@ -0,0 +1,2623 @@ +//! Unified compiled packet policy: one flow key, one conntrack, one verdict. +//! +//! Consolidates the overlapping ACL + Direct-firewall packet work into a +//! single hot path: +//! +//! ```text +//! not fragmented → L4 from PacketMeta (no fragment lock) +//! fragmented → fragment slow path (fail-closed without first-fragment state) +//! established → single canonical conntrack lookup → Allow +//! new flow → compiled ACL phases + compiled firewall rules → verdict +//! ``` +//! +//! Policy is compiled at configuration time (pre-sorted phases, merged port +//! intervals, lowercased selector keys, integer endpoint ids where possible). +//! The hot path allocates nothing, sorts nothing, and formats no strings +//! (notably no `format!("user:{id}")` per packet). + +use std::collections::HashMap; +use std::net::Ipv4Addr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use arc_swap::ArcSwap; +use dashmap::DashMap; +use parking_lot::Mutex; +use tunnet_common::packet::{ + CachedTransport, FragKey, FragmentTable, PacketMeta, ResolvedL4, TcpFlags, Transport, +}; +use tunnet_common::policy::{ + Action, DefaultAction, Direction, IcmpPolicy, PolicyBundle, Protocol, RuleScope, Selector, +}; +use uuid::Uuid; + +use crate::direct::firewall::FirewallRule; + +// Reuse TTLs from the established engines. +const TCP_ACTIVE_TTL: Duration = Duration::from_secs(300); +const TCP_TIME_WAIT_TTL: Duration = Duration::from_secs(10); +const UDP_TTL: Duration = Duration::from_secs(30); +const ICMP_TTL: Duration = Duration::from_secs(10); + +/// Canonical bidirectional conntrack key: one lookup in the common case. +/// Network-scoped (§2.2-1): the same 5-tuple in two networks is two flows. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct CanonKey { + net: Uuid, + proto: u8, + a: Ipv4Addr, + aport: u16, + b: Ipv4Addr, + bport: u16, +} + +fn proto_num(p: Protocol) -> Option { + match p { + Protocol::Tcp => Some(6), + Protocol::Udp => Some(17), + Protocol::Icmp => Some(1), + Protocol::Icmpv6 => Some(58), + Protocol::Other(n) => Some(n), + Protocol::Any => None, + } +} + +fn canon_key( + net: Uuid, + proto: Protocol, + src: Ipv4Addr, + dst: Ipv4Addr, + sport: Option, + dport: Option, +) -> Option { + let num = proto_num(proto)?; + if num == 1 { + // ICMP: direction-independent, keyed by sorted endpoints + echo id. + let id = sport.or(dport).unwrap_or(0); + let (a, b) = if src <= dst { (src, dst) } else { (dst, src) }; + return Some(CanonKey { + net, + proto: num, + a, + aport: id, + b, + bport: 0, + }); + } + let (a, aport, b, bport) = if (src, sport.unwrap_or(0)) <= (dst, dport.unwrap_or(0)) { + (src, sport.unwrap_or(0), dst, dport.unwrap_or(0)) + } else { + (dst, dport.unwrap_or(0), src, sport.unwrap_or(0)) + }; + Some(CanonKey { + net, + proto: num, + a, + aport, + b, + bport, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TcpPhase { + SynSent, + Established, + TimeWait, +} + +#[derive(Debug, Clone, Copy)] +enum Phase { + Tcp(TcpPhase), + Udp, + Icmp, +} + +#[derive(Debug, Clone, Copy)] +struct FlowState { + phase: Phase, + last_seen: Instant, + /// Generations that admitted this flow (§0.4, §2.2-2): the ACL snapshot + /// generation AND the firewall snapshot generation that decided. ANY + /// mismatch revalidates — an old firewall can never be trusted under a + /// new generation. + admitted_acl_gen: u64, + admitted_fw_gen: u64, + /// Expiry-wheel token: bumped whenever a new heap node is pushed. + seq: u64, + /// Millis timestamp of the last heap node push (throttles refresh churn). + heap_ms: u64, +} + +fn ttl_of(s: &FlowState) -> Duration { + match s.phase { + Phase::Tcp(TcpPhase::TimeWait) => TCP_TIME_WAIT_TTL, + Phase::Tcp(_) => TCP_ACTIVE_TTL, + Phase::Udp => UDP_TTL, + Phase::Icmp => ICMP_TTL, + } +} + +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Amortized expiry wheel (§14): 16 independent shards, no global lock. +/// Each packet-path call pops at most [`REAP_BUDGET`] expired nodes from the +/// single shard selected by its flow key — O(1) amortized, never a full-map +/// retain on the hot path. Stale nodes (superseded `seq`) are skipped; a +/// background task performs the rare heap rebuilds. +const EXPIRY_SHARDS: usize = 16; +const REAP_BUDGET: usize = 4; + +#[derive(Debug, Default)] +struct ExpiryWheel { + shards: [Mutex>; EXPIRY_SHARDS], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ExpiryNode { + expires_ms: u64, + seq: u64, + key: CanonKey, +} + +impl PartialOrd for ExpiryNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ExpiryNode { + // Reversed: BinaryHeap is a max-heap; earliest expiry pops first. + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other + .expires_ms + .cmp(&self.expires_ms) + .then_with(|| other.seq.cmp(&self.seq)) + } +} + +impl ExpiryWheel { + fn shard(key: &CanonKey) -> usize { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + key.hash(&mut h); + (h.finish() as usize) % EXPIRY_SHARDS + } + + fn push(&self, key: CanonKey, expires_ms: u64, seq: u64) { + self.shards[Self::shard(&key)].lock().push(ExpiryNode { + expires_ms, + seq, + key, + }); + } + + /// Pop up to `REAP_BUDGET` expired nodes for one shard; the caller + /// validates each against live state (missing / seq-mismatch / refreshed + /// entries are simply skipped). + fn reap_shard(&self, shard: usize, now_ms: u64, mut consume: impl FnMut(CanonKey, u64)) { + let mut heap = self.shards[shard].lock(); + for _ in 0..REAP_BUDGET { + let Some(top) = heap.peek() else { break }; + if top.expires_ms > now_ms { + break; + } + let node = *top; + heap.pop(); + consume(node.key, node.seq); + } + } + + fn clear(&self) { + for s in &self.shards { + s.lock().clear(); + } + } +} + +/// Precompiled selector: no per-packet allocation or case folding. +#[derive(Debug, Clone)] +enum Sel { + Any, + Endpoint(Box), + Tag(Box), + Network(Box), + Cidr(ipnet::IpNet), + User { id: Box, marker: Box }, +} + +impl Sel { + fn compile(s: &Selector) -> Self { + match s { + Selector::Any => Self::Any, + Selector::Endpoint(id) => Self::Endpoint(id.to_ascii_lowercase().into()), + Selector::Tag(t) => Self::Tag(t.clone().into()), + Selector::Network(n) => Self::Network(n.clone().into()), + Selector::Cidr(net) => Self::Cidr(*net), + Selector::User(id) => { + let lower = id.to_ascii_lowercase(); + Self::User { + marker: format!("user:{id}").into(), + id: lower.into(), + } + } + } + } + + fn matches( + &self, + endpoint_hex: &str, + tags: &[String], + network: &str, + ip: Option, + ) -> bool { + match self { + Self::Any => true, + Self::Endpoint(id) => id.as_ref().eq_ignore_ascii_case(endpoint_hex), + Self::Tag(t) => tags.iter().any(|x| x.as_str() == t.as_ref()), + Self::Network(n) => n.as_ref() == network, + Self::Cidr(net) => ip.is_some_and(|ip| net.contains(&std::net::IpAddr::V4(ip))), + Self::User { id, marker } => tags + .iter() + .any(|x| x.as_str() == marker.as_ref() || x.eq_ignore_ascii_case(id)), + } + } +} + +#[derive(Debug, Clone)] +struct CompiledRule { + src: Sel, + dst: Sel, + action: Action, + order_index: i32, + priority: i32, + protocol: Option, + /// Merged, sorted, non-overlapping port intervals. Empty = any. + ports: Vec<(u16, u16)>, + has_posture: bool, +} + +impl CompiledRule { + fn port_hit(&self, port: Option) -> bool { + if self.ports.is_empty() { + return true; + } + let Some(p) = port else { return false }; + self.ports.iter().any(|(a, b)| p >= *a && p <= *b) + } +} + +fn compile_ports(r: &tunnet_common::policy::PolicyRule) -> Vec<(u16, u16)> { + let mut v: Vec<(u16, u16)> = r.ports.iter().map(|p| (p.start, p.end)).collect(); + if v.is_empty() { + return v; + } + v.sort(); + let mut out = Vec::with_capacity(v.len()); + let mut cur = v[0]; + for (a, b) in v.into_iter().skip(1) { + if a <= cur.1.saturating_add(1) { + cur.1 = cur.1.max(b); + } else { + out.push(cur); + cur = (a, b); + } + } + out.push(cur); + out +} + +/// Allocation-free compiled ACL snapshot. +#[derive(Debug)] +pub struct CompiledAcl { + org_deny: Vec, + net_deny: Vec, + net_allow: Vec, + default_action: DefaultAction, + icmp_policy: IcmpPolicy, +} + +impl CompiledAcl { + pub fn compile(bundle: &PolicyBundle) -> Self { + let mut org_deny = Vec::new(); + let mut net_deny = Vec::new(); + let mut net_allow = Vec::new(); + for r in &bundle.rules { + if !r.enabled { + continue; + } + let c = CompiledRule { + src: Sel::compile(&r.src), + dst: Sel::compile(&r.dst), + action: r.action, + order_index: r.order_index, + priority: r.priority, + protocol: r.protocol, + ports: compile_ports(r), + has_posture: !r.src_posture.is_empty(), + }; + match (r.scope, r.action) { + (RuleScope::Organization, Action::Deny) => org_deny.push(c), + (RuleScope::Network, Action::Deny) => net_deny.push(c), + (RuleScope::Network, Action::Allow) => net_allow.push(c), + _ => {} + } + } + for v in [&mut org_deny, &mut net_deny, &mut net_allow] { + v.sort_by(|a, b| { + a.order_index + .cmp(&b.order_index) + .then_with(|| a.priority.cmp(&b.priority)) + }); + } + Self { + org_deny, + net_deny, + net_allow, + default_action: bundle.default_action, + icmp_policy: bundle.icmp_policy, + } + } + + #[allow(clippy::too_many_arguments)] + fn verdict( + &self, + protocol: Protocol, + self_hex: &str, + self_ip: Ipv4Addr, + self_tags: &[String], + self_net: &str, + peer_hex: &str, + peer_ip: Option, + peer_tags: &[String], + dst_port: Option, + direction: Direction, + src_posture_ok: bool, + ) -> Action { + if protocol == Protocol::Icmp { + match self.icmp_policy { + IcmpPolicy::Allow => return Action::Allow, + IcmpPolicy::Deny => return Action::Deny, + IcmpPolicy::Acl => {} + } + } + // Three ordered phases: org deny, network deny, network allow. + // First hit in a phase wins; deny phases precede the allow phase. + let mut posture_skip = false; + for phase_rules in [&self.org_deny, &self.net_deny, &self.net_allow] { + for rule in phase_rules.iter() { + if !rule_hit( + rule, protocol, self_hex, self_ip, self_tags, self_net, peer_hex, peer_ip, + peer_tags, dst_port, direction, + ) { + continue; + } + if rule.has_posture && !src_posture_ok { + posture_skip = true; + continue; + } + return rule.action; + } + } + let _ = posture_skip; + self.default_action.into() + } + + /// True when the bundle carries no rules (fail-open outage analysis). + pub fn is_empty_open(&self) -> bool { + self.org_deny.is_empty() && self.net_deny.is_empty() && self.net_allow.is_empty() + } + + pub fn default_is_allow(&self) -> bool { + matches!(self.default_action, DefaultAction::Allow) + } +} + +#[allow(clippy::too_many_arguments)] +fn rule_hit( + r: &CompiledRule, + protocol: Protocol, + self_hex: &str, + self_ip: Ipv4Addr, + self_tags: &[String], + self_net: &str, + peer_hex: &str, + peer_ip: Option, + peer_tags: &[String], + dst_port: Option, + direction: Direction, +) -> bool { + if !protocol.matches_rule(r.protocol) { + return false; + } + if protocol.is_icmp() { + // port-restricted rules still match ICMP (matches legacy semantics) + } else if matches!(protocol, Protocol::Other(_)) { + if !r.ports.is_empty() { + return false; + } + } else if !r.port_hit(dst_port) { + return false; + } + let (src_ok, dst_ok) = match direction { + Direction::Inbound => ( + r.src.matches(peer_hex, peer_tags, self_net, peer_ip), + r.dst.matches(self_hex, self_tags, self_net, Some(self_ip)), + ), + Direction::Outbound => ( + r.src.matches(self_hex, self_tags, self_net, Some(self_ip)), + r.dst.matches(peer_hex, peer_tags, self_net, peer_ip), + ), + }; + // Note: peer_network uses self_net, matching legacy AclEngine behavior + // (peer network context was the local network name). + src_ok && dst_ok +} + +/// Compiled local-firewall rule (direction + action + proto + ports + peer). +#[derive(Debug, Clone)] +pub struct CompiledFwRule { + pub inbound: bool, + pub allow: bool, + pub reject: bool, + pub protocol: Protocol, + pub ports: Vec<(u16, u16)>, + pub peer_endpoint: Option>, + pub peer_hostname: Option>, + pub peer_network: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PolicyVerdict { + Allow, + Deny, + Reject, +} + +/// Compiled firewall rule set for exactly one network (§0.2). +/// Rules from network A can never affect network B: the fast path uses the +/// set resolved into the peer's fast state, never a per-packet UUID lookup. +#[derive(Debug, Clone, Default)] +pub struct FwSet { + pub enabled: bool, + pub rules: Vec, +} + +/// Per-network firewall verdict counters, fed by the runtime and read by +/// control-plane stats (legacy engines no longer see packets). +#[derive(Debug, Default)] +pub struct FwCounters { + pub allowed: AtomicU64, + pub denied: AtomicU64, + pub rejected: AtomicU64, +} + +/// Immutable firewall publication: ruleset + the publication generation +/// that installed it (§2.2-2). Packets always read rules and generation +/// from the same snapshot — never a torn mix. +#[derive(Debug, Clone)] +pub struct FwSnapshot { + pub generation: u64, + pub set: FwSet, +} + +/// Stable per-network policy slot (§2.1-3, §2.2-2): +/// +/// ```text +/// network → stable Arc → ArcSwap → Arc +/// ``` +/// +/// Fast states hold the stable slot forever. Firewall publication swaps the +/// slot's snapshot atomically — existing peers observe new rules +/// immediately with no per-packet network map lookup and no registry-wide +/// peer relink. The counters object is equally stable, so control-plane +/// stats survive republishes. +pub struct FwSlot { + pub snapshot: ArcSwap, + pub counters: Arc, +} + +impl Default for FwSlot { + fn default() -> Self { + Self { + snapshot: ArcSwap::from_pointee(FwSnapshot { + generation: 0, + set: FwSet::default(), + }), + counters: Arc::new(FwCounters::default()), + } + } +} + +/// Deny-log capacity (control-plane diagnostics, not the hot path). +pub const DENY_LOG_CAP: usize = 64; + +/// Deny record for control-plane diagnostics (not the hot path). +#[derive(Debug, Clone, serde::Serialize)] +pub struct AclDenyRecord { + pub peer_endpoint: String, + pub dst_port: Option, + pub protocol: String, + pub reason: String, + pub rule_slug: Option, + pub scope: Option, + pub at_unix: i64, +} + +/// One dataplane-generation-owned packet-policy runtime (§0.1). +/// +/// A single `PolicyRuntime` is shared by outbound processing and every +/// inbound connection: one canonical conntrack, one fragment table, one +/// verdict. Snapshots compile off the packet path and publish with ONE +/// atomic store (§2.1-4): the generation lives INSIDE the immutable +/// snapshot, so a packet can never observe new policy with an old +/// generation (no torn publication). Conntrack entries carry the admitting +/// generations (taken from the same snapshots) and revalidate on mismatch. +/// +/// Publication model (§2.2-2): one unified `publication` token is bumped +/// per publish; the ACL snapshot AND every touched firewall snapshot carry +/// the new token. Publish order is always firewall-slot-swap first, ACL +/// snapshot second; packets load ACL first, firewall second. By the SeqCst +/// total order, observing the new ACL generation implies observing the new +/// firewall snapshot — the (new ACL, old firewall) poison pair is +/// impossible. Conntrack admission stamps exactly the pair that decided. +#[derive(Clone)] +pub struct PolicyRuntime { + inner: Arc>, + /// Stable per-network firewall slots, shared across ALL generations: + /// publication swaps slot contents, so every holder observes updates + /// without relink (§2.1-3). + slots: Arc>>, + /// Unified publication token (§2.2-2): bumped once per publish and + /// stamped on the ACL snapshot and every touched firewall snapshot. + publication: Arc, + /// Serializes publishers (ACL, firewall, invalidate) into one atomic + /// transaction each: load → allocate generation → compile → swap slots + /// → store snapshot. Committed generations are strictly monotonic and + /// no publish can clobber a concurrent one (§2.2 blocker). Control + /// path only — the packet hot path never takes this lock. + publish_lock: Arc>, +} + +struct RuntimeInner { + /// Policy generation, published atomically WITH this snapshot. + generation: u64, + acl: CompiledAcl, + acl_source: PolicyBundle, + fw_source: HashMap, Vec, bool)>, + self_source: crate::acl::SelfIdentity, + self_hex: String, + self_ip: Ipv4Addr, + self_tags: Vec, + self_net: String, + src_posture_ok: bool, + stale: bool, + conntrack: DashMap, + expiry: ExpiryWheel, + fragments: Mutex, + deny_log: Arc>>, +} + +impl PolicyRuntime { + /// Bootstrap a runtime from control-plane state (dataplane bring-up). + /// `fw` maps network → (local rules, suggested rules, enabled). + pub fn bootstrap( + bundle: &PolicyBundle, + fw: &HashMap, Vec, bool)>, + self_id: &crate::acl::SelfIdentity, + src_posture_ok: bool, + stale: bool, + ) -> Self { + let this = Self { + inner: Arc::new(ArcSwap::from_pointee(RuntimeInner::empty())), + slots: Arc::new(DashMap::new()), + publication: Arc::new(AtomicU64::new(1)), + publish_lock: Arc::new(parking_lot::Mutex::new(())), + }; + let inner = this.compile_new(bundle, fw, self_id, src_posture_ok, stale, None, 1); + // No sweeper here: the dataplane actor starts exactly one per + // generation via spawn_sweeper (tests stay task-free). + this.inner.store(Arc::new(inner)); + this + } + + /// Background expiry sweeper (§14): rate-limited, shard-local, never + /// blocking packets. Each 250 ms tick reaps at most a few dozen expired + /// nodes per shard and rebuilds a heap only when stale nodes dominate. + /// Tied to the dataplane generation token: BringDown cancels it, so no + /// sweeper task leaks across bring-up cycles. + pub fn spawn_sweeper(&self, cancel: tokio_util::sync::CancellationToken) { + let Ok(handle) = tokio::runtime::Handle::try_current() else { + return; + }; + let inner = self.inner.clone(); + handle.spawn(async move { + let mut tick = tokio::time::interval(std::time::Duration::from_millis(250)); + loop { + tokio::select! { + biased; + _ = cancel.cancelled() => break, + _ = tick.tick() => {} + } + // Reload every tick: republishes replace the table and the + // sweeper must follow, never pinning a dead generation. + let snapshot = inner.load(); + let now_ms = now_millis(); + let now = Instant::now(); + for shard in 0..EXPIRY_SHARDS { + let mut budget = 32; + while budget > 0 { + let node = snapshot.expiry.shards[shard].lock().pop(); + let Some(node) = node else { break }; + if node.expires_ms > now_ms { + snapshot.expiry.shards[shard].lock().push(node); + break; + } + budget -= 1; + let remove = match snapshot.conntrack.get(&node.key) { + Some(e) => { + e.seq == node.seq && now.duration_since(e.last_seen) > ttl_of(&e) + } + None => false, + }; + if remove { + snapshot.conntrack.remove(&node.key); + } + } + } + // Heap hygiene: rebuild a shard when stale nodes dominate, so + // refresh churn cannot grow memory without bound. Rare, + // background-only, shard by shard. + for shard in 0..EXPIRY_SHARDS { + let live = snapshot.conntrack.len() / EXPIRY_SHARDS + 8; + let mut heap = snapshot.expiry.shards[shard].lock(); + if heap.len() > live.saturating_mul(4).max(256) { + let nodes: Vec<_> = heap.drain().collect(); + drop(heap); + for node in nodes { + let keep = match snapshot.conntrack.get(&node.key) { + Some(e) => e.seq == node.seq, + None => false, + }; + if keep { + snapshot.expiry.shards[shard].lock().push(node); + } + } + // One shard per tick is enough; remaining shards wait. + break; + } + } + } + }); + } + + /// Current policy generation, read from the live snapshot (single + /// atomic load — always consistent with the policy it describes). + pub fn generation(&self) -> u64 { + self.inner.load().generation + } + + /// Stable firewall slot for a network (slow paths only: fast-state + /// creation and install-time relink, never established packets). + /// Creates a disabled default slot for unknown networks. + pub fn slot_for_network(&self, network: Uuid) -> Arc { + self.slots + .entry(network) + .or_insert_with(|| Arc::new(FwSlot::default())) + .clone() + } + + /// Resolve the compiled firewall set for a network (slow paths only). + pub fn fw_for_network(&self, network: Uuid) -> Arc { + Arc::new(self.slot_for_network(network).snapshot.load().set.clone()) + } + + /// Counters for a network's firewall (control-plane stats surface). + pub fn fw_counters_for(&self, network: Uuid) -> Arc { + self.slot_for_network(network).counters.clone() + } + + pub fn recent_denies(&self) -> Vec { + self.inner.load().deny_log.lock().iter().cloned().collect() + } + + /// This node's mesh IP (for self-traffic drops and NAT). + pub fn self_ip(&self) -> Ipv4Addr { + self.inner.load().self_ip + } + + pub fn conntrack_len(&self) -> usize { + self.inner.load().conntrack.len() + } + + pub fn publish_acl( + &self, + bundle: &PolicyBundle, + self_id: &crate::acl::SelfIdentity, + src_posture_ok: bool, + stale: bool, + ) -> u64 { + let _guard = self.publish_lock.lock(); + let generation = self.publication.fetch_add(1, Ordering::SeqCst) + 1; + let prev = self.inner.load(); + let inner = self.compile_new( + bundle, + &prev.fw_source, + self_id, + src_posture_ok, + stale, + Some(&prev), + generation, + ); + self.inner.store(Arc::new(inner)); + generation + } + + pub fn publish_firewall( + &self, + network: Uuid, + local: Vec, + suggested: Vec, + enabled: bool, + ) -> u64 { + let _guard = self.publish_lock.lock(); + let generation = self.publication.fetch_add(1, Ordering::SeqCst) + 1; + let prev = self.inner.load(); + let mut fw_source = prev.fw_source.clone(); + fw_source.insert(network, (local, suggested, enabled)); + let inner = self.compile_new( + &prev.acl_source, + &fw_source, + &prev.self_source, + prev.src_posture_ok, + prev.stale, + Some(&prev), + generation, + ); + self.inner.store(Arc::new(inner)); + generation + } + + pub fn invalidate(&self) -> u64 { + let _guard = self.publish_lock.lock(); + let generation = self.publication.fetch_add(1, Ordering::SeqCst) + 1; + let inner = self.inner.load(); + inner.conntrack.clear(); + inner.expiry.clear(); + let next = self.compile_new( + &inner.acl_source, + &inner.fw_source, + &inner.self_source, + inner.src_posture_ok, + inner.stale, + Some(&inner), + generation, + ); + self.inner.store(Arc::new(next)); + generation + } +} + +impl RuntimeInner { + /// Placeholder before the first real compile (bootstrap only). + fn empty() -> Self { + Self { + generation: 0, + acl: CompiledAcl::compile(&PolicyBundle::default()), + acl_source: PolicyBundle::default(), + fw_source: HashMap::new(), + self_source: crate::acl::SelfIdentity { + endpoint_hex: String::new(), + ip: Ipv4Addr::UNSPECIFIED, + tags: vec![], + network: String::new(), + }, + self_hex: String::new(), + self_ip: Ipv4Addr::UNSPECIFIED, + self_tags: vec![], + self_net: String::new(), + src_posture_ok: false, + stale: false, + conntrack: DashMap::new(), + expiry: ExpiryWheel::default(), + fragments: Mutex::new(FragmentTable::default()), + deny_log: Arc::new(Mutex::new(std::collections::VecDeque::with_capacity( + DENY_LOG_CAP, + ))), + } + } + + /// Carry the live conntrack table across a republish (entries revalidate + /// by generation instead of being dropped). + fn rebuild_conntrack(&self) -> DashMap { + // Move entries without revalidating here: stale generations are + // rechecked lazily on next hit, which spreads the cost. + let next = DashMap::with_capacity(self.conntrack.len()); + for entry in self.conntrack.iter() { + next.insert(*entry.key(), *entry.value()); + } + next + } +} + +impl PolicyRuntime { + #[allow(clippy::too_many_arguments)] + fn compile_new( + &self, + bundle: &PolicyBundle, + fw: &HashMap, Vec, bool)>, + self_id: &crate::acl::SelfIdentity, + src_posture_ok: bool, + stale: bool, + prev: Option<&RuntimeInner>, + generation: u64, + ) -> RuntimeInner { + for (net, (local, suggested, enabled)) in fw { + // Swap the stable slot's snapshot stamped with this publish's + // token: every live fast state holding this slot observes the + // new rules atomically, with no relink. Counters objects are + // never replaced, so stats survive. This store precedes the + // ACL snapshot store below — the packet path's load order + // (ACL first, firewall second) depends on it (§2.2-2). + let slot = self.slot_for_network(*net); + slot.snapshot.store(Arc::new(FwSnapshot { + generation, + set: FwSet { + enabled: *enabled, + rules: compile_fw_rules(local, suggested), + }, + })); + } + // Preserve shared state across republishes: conntrack entries carry + // their admitting generation and revalidate (§0.4); the deny log + // survives so diagnostics are not wiped by updates. Fragment state + // is short-TTL (2 s) and starts fresh — at most a few fail-closed + // drops of in-flight fragments. + let (conntrack, deny_log) = match prev { + Some(p) => (p.rebuild_conntrack(), p.deny_log.clone()), + None => ( + DashMap::new(), + Arc::new(Mutex::new(std::collections::VecDeque::with_capacity( + DENY_LOG_CAP, + ))), + ), + }; + RuntimeInner { + generation, + acl: CompiledAcl::compile(bundle), + acl_source: bundle.clone(), + fw_source: fw.clone(), + self_source: self_id.clone(), + self_hex: self_id.endpoint_hex.clone(), + self_ip: self_id.ip, + self_tags: self_id.tags.clone(), + self_net: self_id.network.clone(), + src_posture_ok, + stale, + conntrack, + expiry: ExpiryWheel::default(), + fragments: Mutex::new(FragmentTable::default()), + deny_log, + } + } +} + +impl PolicyRuntime { + /// Hot-path check. `fw_slot` is the peer's stable network slot — the + /// snapshot is loaded INSIDE, after the ACL snapshot, matching publish + /// order (§2.2-2). `peer_*` are cheap slices from the same fast state. + /// No allocation, no sorting, no string formatting; unfragmented + /// traffic never touches the fragment lock. Both generations used come + /// from the snapshots actually evaluated — publication can never tear. + #[allow(clippy::too_many_arguments)] + pub fn check( + &self, + meta: &PacketMeta, + direction: Direction, + peer_hex: &str, + peer_tags: &[String], + peer_hostname: Option<&str>, + peer_network: Option, + fw_slot: &FwSlot, + fw_counters: &FwCounters, + ) -> PolicyVerdict { + let inner = self.inner.load(); + self.check_inner( + &inner, + inner.generation, + meta, + direction, + peer_hex, + peer_tags, + peer_hostname, + peer_network, + fw_slot, + fw_counters, + ) + .0 + } + + /// Check returning the snapshot generations actually used (concurrency + /// tests pair verdicts with generations to prove atomic publication). + #[allow(clippy::too_many_arguments)] + pub fn check_with_generation( + &self, + meta: &PacketMeta, + direction: Direction, + peer_hex: &str, + peer_tags: &[String], + peer_hostname: Option<&str>, + peer_network: Option, + fw_slot: &FwSlot, + fw_counters: &FwCounters, + ) -> (PolicyVerdict, u64, u64) { + let inner = self.inner.load(); + let policy_gen = inner.generation; + let (verdict, _, fw_gen) = self.check_inner( + &inner, + policy_gen, + meta, + direction, + peer_hex, + peer_tags, + peer_hostname, + peer_network, + fw_slot, + fw_counters, + ); + (verdict, policy_gen, fw_gen) + } + + #[allow(clippy::too_many_arguments)] + fn check_inner( + &self, + inner: &RuntimeInner, + policy_gen: u64, + meta: &PacketMeta, + direction: Direction, + peer_hex: &str, + peer_tags: &[String], + peer_hostname: Option<&str>, + peer_network: Option, + fw_slot: &FwSlot, + fw_counters: &FwCounters, + ) -> (PolicyVerdict, u64, u64) { + // Load order is the protocol: ACL snapshot first, firewall snapshot + // second — the reverse of publish order (slot swap, then ACL + // store). Observing the new ACL generation therefore implies + // observing the new firewall snapshot; the (new ACL, old firewall) + // poison pair cannot occur (§2.2-2). + let fw_snap = fw_slot.snapshot.load(); + let fw_gen = fw_snap.generation; + let fw = &fw_snap.set; + // Fast path: unfragmented traffic never touches the fragment lock. + let l4: ResolvedL4 = if meta.is_later_fragment() { + let Some(hit) = inner.fragments.lock().lookup_meta(meta) else { + return (PolicyVerdict::Deny, policy_gen, fw_gen); + }; + hit + } else { + if meta.is_fragment() { + inner.fragments.lock().remember_meta(meta); + } + match ResolvedL4::from_transport(meta.transport) { + Some(l4) => l4, + None => return (PolicyVerdict::Deny, policy_gen, fw_gen), + } + }; + + let (Some(src), Some(dst)) = (meta.src_v4, meta.dst_v4) else { + return (PolicyVerdict::Deny, policy_gen, fw_gen); + }; + let tcp_flags = l4.tcp_flags.map(|f| f.0).unwrap_or(0); + // Conntrack is network-scoped: the membership network joins the key. + let net = peer_network.unwrap_or(Uuid::nil()); + + // Single canonical established lookup, shared both directions (§0.1). + // Entries admitted under older generations revalidate once (§0.4, + // §2.2-2: ANY generation mismatch — ACL or firewall — revalidates). + if let Some(key) = canon_key(net, l4.protocol, src, dst, l4.src_port, l4.dst_port) + && self.conntrack_allows(inner, policy_gen, fw_gen, key, direction, tcp_flags) + { + self.reap_for_key(inner, &key); + return (PolicyVerdict::Allow, policy_gen, fw_gen); + } + + let peer_ip = match direction { + Direction::Outbound => Some(dst), + Direction::Inbound => Some(src), + }; + let action = inner.acl.verdict( + l4.protocol, + &inner.self_hex, + inner.self_ip, + &inner.self_tags, + &inner.self_net, + peer_hex, + peer_ip, + peer_tags, + l4.dst_port, + direction, + inner.src_posture_ok, + ); + if action == Action::Deny { + // Fail-open only for open networks with no rules during control + // outage (preserved legacy semantics); otherwise deny + log. + let open_failover = + inner.stale && inner.acl.is_empty_open() && inner.acl.default_is_allow(); + if !open_failover { + self.record_deny(inner, peer_hex, l4.dst_port, l4.protocol); + return (PolicyVerdict::Deny, policy_gen, fw_gen); + } + } + + // Network-scoped firewall second (pre-resolved set, then defaults). + // The set already belongs to the peer's network; the NetworkId + // filter inside rules keeps its legacy meaning against peer_network. + if fw.enabled { + match fw_verdict( + &fw.rules, + direction, + l4, + peer_hex, + peer_hostname, + peer_network, + ) { + Some(PolicyVerdict::Allow) => { + fw_counters.allowed.fetch_add(1, Ordering::Relaxed); + } + Some(v) => { + if v == PolicyVerdict::Reject { + fw_counters.rejected.fetch_add(1, Ordering::Relaxed); + } else { + fw_counters.denied.fetch_add(1, Ordering::Relaxed); + } + return (v, policy_gen, fw_gen); + } + None => { + // Built-in defaults: outbound allow; inbound from a known + // peer allow; inbound without peer identity: ICMP echo. + let allowed = match direction { + Direction::Outbound => true, + Direction::Inbound => { + if !peer_hex.is_empty() { + true + } else { + matches!(l4.protocol, Protocol::Icmp) && l4.icmp_type == Some(8) + } + } + }; + if !allowed { + fw_counters.denied.fetch_add(1, Ordering::Relaxed); + return (PolicyVerdict::Deny, policy_gen, fw_gen); + } + fw_counters.allowed.fetch_add(1, Ordering::Relaxed); + } + } + } + + if let Some(key) = canon_key(net, l4.protocol, src, dst, l4.src_port, l4.dst_port) { + self.open_flow(inner, policy_gen, fw_gen, key, l4.protocol, tcp_flags); + } + (PolicyVerdict::Allow, policy_gen, fw_gen) + } + + /// Established-flow fast path with generation revalidation (§0.4, + /// §2.2-2). Entries admitted under older generations are re-evaluated + /// once against current policy instead of being trusted blindly; a + /// revocation therefore takes effect on the next packet of the flow, + /// not after its TTL. ANY mismatch — ACL or firewall — revalidates, so + /// an old firewall snapshot can never be trusted under a new ACL + /// generation or vice versa. + fn conntrack_allows( + &self, + inner: &RuntimeInner, + policy_gen: u64, + fw_gen: u64, + key: CanonKey, + direction: Direction, + tcp_flags: u8, + ) -> bool { + let now = Instant::now(); + let now_ms = now_millis(); + let mut e = match inner.conntrack.get_mut(&key) { + Some(e) => e, + None => return false, + }; + if now.duration_since(e.last_seen) > ttl_of(&e) { + drop(e); + inner.conntrack.remove(&key); + return false; + } + if e.admitted_acl_gen != policy_gen || e.admitted_fw_gen != fw_gen { + // Security-relevant publish happened since admission: revalidate + // fully below (the caller falls through to policy evaluation). + return false; + } + let allowed = match e.phase { + Phase::Tcp(TcpPhase::SynSent) => { + if matches!(direction, Direction::Inbound) + || (tcp_flags & TcpFlags::ACK) != 0 + || (tcp_flags & TcpFlags::RST) != 0 + { + if (tcp_flags & TcpFlags::RST) != 0 || (tcp_flags & TcpFlags::FIN) != 0 { + e.phase = Phase::Tcp(TcpPhase::TimeWait); + } else { + e.phase = Phase::Tcp(TcpPhase::Established); + } + e.last_seen = now; + true + } else if matches!(direction, Direction::Outbound) { + e.last_seen = now; + true + } else { + false + } + } + Phase::Tcp(TcpPhase::Established) => { + if (tcp_flags & TcpFlags::RST) != 0 || (tcp_flags & TcpFlags::FIN) != 0 { + e.phase = Phase::Tcp(TcpPhase::TimeWait); + } + e.last_seen = now; + true + } + Phase::Tcp(TcpPhase::TimeWait) => { + e.last_seen = now; + true + } + Phase::Udp | Phase::Icmp => { + e.last_seen = now; + true + } + }; + if allowed { + // Throttled wheel maintenance: at most ~1 push per TTL/4 per flow. + let ttl = ttl_of(&e); + if now_ms.wrapping_sub(e.heap_ms) > (ttl.as_millis() as u64 / 4).max(1000) { + e.seq = e.seq.wrapping_add(1); + e.heap_ms = now_ms; + let seq = e.seq; + drop(e); + inner + .expiry + .push(key, now_ms.saturating_add(ttl.as_millis() as u64), seq); + } + } + allowed + } + + /// Bounded amortized expiry for one flow's shard (§14): pop at most + /// REAP_BUDGET expired nodes, removing only entries whose token still + /// matches (missing / refreshed entries are skipped, never trusted). + fn reap_for_key(&self, inner: &RuntimeInner, key: &CanonKey) { + let now_ms = now_millis(); + let now = Instant::now(); + inner + .expiry + .reap_shard(ExpiryWheel::shard(key), now_ms, |k, seq| { + let remove = match inner.conntrack.get(&k) { + Some(e) => e.seq == seq && now.duration_since(e.last_seen) > ttl_of(&e), + None => false, + }; + if remove { + inner.conntrack.remove(&k); + } + }); + } + + fn open_flow( + &self, + inner: &RuntimeInner, + policy_gen: u64, + fw_gen: u64, + key: CanonKey, + proto: Protocol, + tcp_flags: u8, + ) { + let now = Instant::now(); + let phase = match proto { + Protocol::Tcp => { + if (tcp_flags & TcpFlags::SYN) != 0 && (tcp_flags & TcpFlags::ACK) == 0 { + Phase::Tcp(TcpPhase::SynSent) + } else if (tcp_flags & TcpFlags::FIN) != 0 || (tcp_flags & TcpFlags::RST) != 0 { + Phase::Tcp(TcpPhase::TimeWait) + } else { + Phase::Tcp(TcpPhase::Established) + } + } + Protocol::Udp => Phase::Udp, + Protocol::Icmp | Protocol::Icmpv6 => Phase::Icmp, + Protocol::Any | Protocol::Other(_) => return, + }; + let now_ms = now_millis(); + let ttl = match phase { + Phase::Tcp(TcpPhase::TimeWait) => TCP_TIME_WAIT_TTL, + Phase::Tcp(_) => TCP_ACTIVE_TTL, + Phase::Udp => UDP_TTL, + Phase::Icmp => ICMP_TTL, + }; + inner + .conntrack + .entry(key) + .and_modify(|st| { + st.last_seen = now; + // Stamp exactly the pair that decided (load-ordered above). + st.admitted_acl_gen = policy_gen; + st.admitted_fw_gen = fw_gen; + if matches!(st.phase, Phase::Tcp(TcpPhase::SynSent)) + && matches!(phase, Phase::Tcp(TcpPhase::Established)) + { + st.phase = phase; + } + if matches!(phase, Phase::Tcp(TcpPhase::TimeWait)) { + st.phase = phase; + } + }) + .or_insert_with(|| { + inner + .expiry + .push(key, now_ms.saturating_add(ttl.as_millis() as u64), 1); + FlowState { + phase, + last_seen: now, + admitted_acl_gen: policy_gen, + admitted_fw_gen: fw_gen, + seq: 1, + heap_ms: now_ms, + } + }); + } + + fn record_deny( + &self, + inner: &RuntimeInner, + peer_hex: &str, + dst_port: Option, + proto: Protocol, + ) { + let record = AclDenyRecord { + peer_endpoint: peer_hex.to_string(), + dst_port, + protocol: format!("{proto:?}").to_lowercase(), + reason: "policy_deny".to_string(), + rule_slug: None, + scope: None, + at_unix: jiff::Timestamp::now().as_second(), + }; + let mut log = inner.deny_log.lock(); + if log.len() >= DENY_LOG_CAP { + log.pop_front(); + } + log.push_back(record); + } +} + +fn fw_verdict( + rules: &[CompiledFwRule], + direction: Direction, + l4: ResolvedL4, + peer_hex: &str, + peer_hostname: Option<&str>, + peer_network: Option, +) -> Option { + let inbound = matches!(direction, Direction::Inbound); + for r in rules { + if r.inbound != inbound { + continue; + } + if !l4.protocol.matches_rule(Some(r.protocol)) { + continue; + } + if !r.ports.is_empty() && !l4.protocol.is_icmp() { + let Some(p) = l4.dst_port else { continue }; + if !r.ports.iter().any(|(a, b)| p >= *a && p <= *b) { + continue; + } + } + if let Some(ep) = r.peer_endpoint.as_ref() + && !ep.as_ref().eq_ignore_ascii_case(peer_hex) + { + continue; + } + if let Some(h) = r.peer_hostname.as_ref() + && peer_hostname.is_none_or(|ph| !ph.eq_ignore_ascii_case(h)) + { + continue; + } + if let Some(n) = r.peer_network + && peer_network != Some(n) + { + continue; + } + if r.allow { + return Some(PolicyVerdict::Allow); + } + if r.reject { + return Some(PolicyVerdict::Reject); + } + return Some(PolicyVerdict::Deny); + } + None +} + +trait FragMetaExt { + fn lookup_meta(&mut self, meta: &PacketMeta) -> Option; + fn remember_meta(&mut self, meta: &PacketMeta); +} + +impl FragMetaExt for FragmentTable { + fn lookup_meta(&mut self, meta: &PacketMeta) -> Option { + let key = FragKey { + src: meta.src, + dst: meta.dst, + protocol: meta.proto, + identification: meta.fragmentation.identification()?, + }; + self.lookup_cached(&key) + } + + fn remember_meta(&mut self, meta: &PacketMeta) { + use tunnet_common::packet::Fragmentation; + if !matches!(meta.fragmentation, Fragmentation::First { .. }) { + return; + } + let Some(id) = meta.fragmentation.identification() else { + return; + }; + let key = FragKey { + src: meta.src, + dst: meta.dst, + protocol: meta.proto, + identification: id, + }; + let cached = match meta.transport { + Transport::Tcp { + src_port, + dst_port, + flags, + .. + } => CachedTransport::Tcp { + src_port, + dst_port, + flags, + }, + Transport::Udp { + src_port, dst_port, .. + } => CachedTransport::Udp { src_port, dst_port }, + Transport::Icmpv4 { + type_u8, + code, + echo_id, + echo_seq, + .. + } => CachedTransport::Icmpv4 { + type_u8, + code, + echo_id, + echo_seq, + }, + Transport::Icmpv6 { type_u8, code, .. } => CachedTransport::Icmpv6 { type_u8, code }, + Transport::Other { protocol, .. } => CachedTransport::Other { protocol }, + Transport::LaterFragment { .. } => return, + }; + self.insert_cached(key, cached); + } +} + +/// Compile a firewall rule list once (local + suggested concatenated, local first). +pub fn compile_fw_rules( + local: &[tunnet_core_firewall_types::FirewallRule], + suggested: &[tunnet_core_firewall_types::FirewallRule], +) -> Vec { + local + .iter() + .chain(suggested.iter()) + .map(|r| { + let mut ports: Vec<(u16, u16)> = r.ports.iter().map(|p| (p.start, p.end)).collect(); + ports.sort(); + let mut merged: Vec<(u16, u16)> = Vec::with_capacity(ports.len()); + for (a, b) in ports { + if let Some(last) = merged.last_mut() + && a <= last.1.saturating_add(1) + { + last.1 = last.1.max(b); + continue; + } + merged.push((a, b)); + } + let (peer_endpoint, peer_hostname, peer_network) = match &r.peer { + tunnet_core_firewall_types::PeerFilter::Any => (None, None, None), + tunnet_core_firewall_types::PeerFilter::Endpoint(e) => { + (Some(e.clone().into_boxed_str()), None, None) + } + tunnet_core_firewall_types::PeerFilter::Hostname(h) => { + (None, Some(h.clone().into_boxed_str()), None) + } + tunnet_core_firewall_types::PeerFilter::NetworkId(n) => { + (None, None, n.parse().ok()) + } + }; + CompiledFwRule { + inbound: matches!( + r.direction, + tunnet_core_firewall_types::FirewallDirection::In + ), + allow: matches!(r.action, tunnet_core_firewall_types::FirewallAction::Allow), + reject: matches!(r.action, tunnet_core_firewall_types::FirewallAction::Reject), + protocol: r.protocol, + ports: merged, + peer_endpoint, + peer_hostname, + peer_network, + } + }) + .collect() +} + +// Re-export firewall types without a hard module dependency cycle. +pub mod tunnet_core_firewall_types { + pub use crate::direct::firewall::{ + FirewallAction, FirewallDirection, FirewallRule, PeerFilter, + }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::acl::SelfIdentity; + use std::sync::Arc; + use tunnet_common::policy::{PolicyRule, RuleScope, Selector}; + use tunnet_core_firewall_types::FirewallRule; + + fn self_id() -> SelfIdentity { + SelfIdentity { + endpoint_hex: "aa".into(), + ip: Ipv4Addr::new(10, 0, 0, 1), + tags: vec![], + network: "net".into(), + } + } + + /// Test runtime with an explicit firewall slot (no engine polling). + /// Returns (runtime, stable slot). The slot is standalone (not the + /// runtime's): tests that need publication use `slot_for_network`. + fn harness(bundle: PolicyBundle, fw_enabled: bool) -> (PolicyRuntime, Arc) { + let rt = PolicyRuntime::bootstrap(&bundle, &HashMap::new(), &self_id(), true, false); + let slot = Arc::new(FwSlot { + snapshot: ArcSwap::from_pointee(FwSnapshot { + generation: rt.generation(), + set: FwSet { + enabled: fw_enabled, + rules: vec![], + }, + }), + counters: Arc::new(FwCounters::default()), + }); + (rt, slot) + } + + fn check_out(rt: &PolicyRuntime, m: &PacketMeta, slot: &FwSlot) -> PolicyVerdict { + rt.check( + m, + Direction::Outbound, + "bb", + &[], + None, + None, + slot, + &slot.counters, + ) + } + + fn check_in(rt: &PolicyRuntime, m: &PacketMeta, slot: &FwSlot) -> PolicyVerdict { + rt.check( + m, + Direction::Inbound, + "bb", + &[], + None, + None, + slot, + &slot.counters, + ) + } + + fn meta_tcp(dst_port: u16) -> PacketMeta { + meta_tcp_ports(40000, dst_port) + } + + fn meta_tcp_ports(sport: u16, dst_port: u16) -> PacketMeta { + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) + .tcp(sport, dst_port, 1, 1000); + let mut o = Vec::new(); + b.write(&mut o, b"hello").unwrap(); + let pkt = tunnet_common::packet::parse(&o).unwrap(); + PacketMeta::from_packet(&pkt) + } + + fn open_bundle() -> PolicyBundle { + PolicyBundle::default() + } + + #[test] + fn open_bundle_allows_and_establishes() { + let (p, slot) = harness(open_bundle(), false); + let m = meta_tcp(80); + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Allow); + // Second packet of the same flow: single conntrack hit. + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Allow); + assert_eq!(p.conntrack_len(), 1); + } + + #[test] + fn deny_rule_matches_legacy_semantics() { + let bundle = PolicyBundle { + rules: vec![PolicyRule { + src: Selector::Any, + dst: Selector::Any, + action: Action::Deny, + ports: vec![tunnet_common::policy::PortRange { start: 22, end: 22 }], + protocol: Some(Protocol::Tcp), + priority: 0, + order_index: 0, + scope: RuleScope::Network, + enabled: true, + slug: None, + src_posture: vec![], + }], + default_action: DefaultAction::Allow, + ..PolicyBundle::default() + }; + let (p, slot) = harness(bundle.clone(), false); + let m22 = meta_tcp(22); + let m80 = meta_tcp(80); + assert_eq!(check_out(&p, &m22, &slot), PolicyVerdict::Deny); + assert_eq!(check_out(&p, &m80, &slot), PolicyVerdict::Allow); + // Legacy evaluator agrees (differential equivalence probe). + let legacy = { + use tunnet_common::policy::{EvalCtx, evaluate_detailed}; + let ctx = EvalCtx { + self_endpoint_hex: "aa", + self_ip: Ipv4Addr::new(10, 0, 0, 1), + self_tags: &[], + self_network: "net", + peer_endpoint_hex: "bb", + peer_ip: Some(Ipv4Addr::new(10, 0, 0, 2)), + peer_tags: &[], + peer_network: "net", + dst_port: Some(22), + protocol: Protocol::Tcp, + src_posture_ok: true, + }; + evaluate_detailed(&bundle, &ctx, Direction::Outbound).action + }; + assert_eq!(legacy, Action::Deny); + } + + #[test] + fn later_fragment_without_state_denied() { + let (p, slot) = harness(open_bundle(), false); + // Craft a later fragment manually. + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 443); + let mut o = Vec::new(); + b.write(&mut o, &[0; 100]).unwrap(); + o[6] = 0x20; // MF + offset bit pattern => fragment offset nonzero + o[7] = 0x08; + let pkt = tunnet_common::packet::parse(&o).unwrap(); + let meta = PacketMeta::from_packet(&pkt); + assert!(meta.is_later_fragment()); + assert_eq!(check_out(&p, &meta, &slot), PolicyVerdict::Deny); + } + + fn meta_udp(sport: u16, dport: u16) -> PacketMeta { + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(sport, dport); + let mut o = Vec::new(); + b.write(&mut o, &[0; 40]).unwrap(); + let pkt = tunnet_common::packet::parse(&o).unwrap(); + PacketMeta::from_packet(&pkt) + } + + fn legacy_action(bundle: &PolicyBundle, port: Option, proto: Protocol) -> Action { + use tunnet_common::policy::{EvalCtx, evaluate_detailed}; + let ctx = EvalCtx { + self_endpoint_hex: "aa", + self_ip: Ipv4Addr::new(10, 0, 0, 1), + self_tags: &[], + self_network: "net", + peer_endpoint_hex: "bb", + peer_ip: Some(Ipv4Addr::new(10, 0, 0, 2)), + peer_tags: &[], + peer_network: "net", + dst_port: port, + protocol: proto, + src_posture_ok: true, + }; + evaluate_detailed(bundle, &ctx, Direction::Outbound).action + } + + fn new_policy(bundle: PolicyBundle) -> (PolicyRuntime, Arc) { + harness(bundle, false) + } + + #[test] + fn differential_matrix_matches_legacy() { + // order_index ascending first-match, port ranges, org-deny priority, + // disabled rules, protocol scoping — new engine must equal legacy. + let bundle = PolicyBundle { + rules: vec![ + PolicyRule { + src: Selector::Tag("admin".into()), + dst: Selector::Any, + action: Action::Allow, + ports: vec![], + protocol: None, + priority: 0, + order_index: 5, + scope: RuleScope::Network, + enabled: false, + slug: Some("disabled".into()), + src_posture: vec![], + }, + PolicyRule { + src: Selector::Any, + dst: Selector::Any, + action: Action::Deny, + ports: vec![ + tunnet_common::policy::PortRange { + start: 8000, + end: 8010, + }, + tunnet_common::policy::PortRange { + start: 8005, + end: 8020, + }, + ], + protocol: Some(Protocol::Tcp), + priority: 0, + order_index: 1, + scope: RuleScope::Organization, + enabled: true, + slug: Some("org-deny-range".into()), + src_posture: vec![], + }, + PolicyRule { + src: Selector::Any, + dst: Selector::Any, + action: Action::Allow, + ports: vec![tunnet_common::policy::PortRange { + start: 8000, + end: 9000, + }], + protocol: Some(Protocol::Tcp), + priority: 0, + order_index: 0, + scope: RuleScope::Network, + enabled: true, + slug: Some("net-allow-wide".into()), + src_posture: vec![], + }, + ], + default_action: DefaultAction::Deny, + ..PolicyBundle::default() + }; + let (p, slot) = new_policy(bundle.clone()); + // Org deny (merged 8000-8020) beats network allow despite higher order. + for port in [8000, 8015, 8020] { + let m = meta_tcp(port); + let got = check_out(&p, &m, &slot); + assert_eq!(got, PolicyVerdict::Deny, "port {port}"); + assert_eq!( + legacy_action(&bundle, Some(port), Protocol::Tcp), + Action::Deny + ); + } + // Outside the org-deny range but inside the network allow range, + // the network allow wins. + for port in [8021, 8500] { + let m = meta_tcp(port); + let got = check_out(&p, &m, &slot); + assert_eq!(got, PolicyVerdict::Allow, "port {port}"); + assert_eq!( + legacy_action(&bundle, Some(port), Protocol::Tcp), + Action::Allow + ); + } + // Outside every range the restrictive default applies in both engines. + let m = meta_tcp(7999); + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Deny); + assert_eq!( + legacy_action(&bundle, Some(7999), Protocol::Tcp), + Action::Deny + ); + // UDP to the same port is not matched by TCP-only rules → default deny. + let u = meta_udp(40000, 8010); + assert_eq!(check_out(&p, &u, &slot), PolicyVerdict::Deny); + assert_eq!( + legacy_action(&bundle, Some(8010), Protocol::Udp), + Action::Deny + ); + } + + #[test] + fn first_fragment_allows_later_fragment() { + let (p, slot) = new_policy(open_bundle()); + // First fragment (offset 0 + MF) is policy-evaluated and remembered. + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 443); + let mut first = Vec::new(); + b.write(&mut first, &[0; 100]).unwrap(); + first[6] = 0x20; // MF set, offset 0 + first[7] = 0x00; + let pkt = tunnet_common::packet::parse(&first).unwrap(); + let meta = PacketMeta::from_packet(&pkt); + assert!(matches!( + meta.fragmentation, + tunnet_common::packet::Fragmentation::First { .. } + )); + assert_eq!(check_out(&p, &meta, &slot), PolicyVerdict::Allow); + // Later fragment of the same datagram now resolves via cached state. + let mut later = first.clone(); + later[6] = 0x20; + later[7] = 0x08; + let pkt = tunnet_common::packet::parse(&later).unwrap(); + let meta = PacketMeta::from_packet(&pkt); + assert!(meta.is_later_fragment()); + assert_eq!(check_out(&p, &meta, &slot), PolicyVerdict::Allow); + } + + #[test] + fn malformed_packets_denied() { + let (p, _slot) = new_policy(open_bundle()); + // Truncated garbage must never reach the transport. + assert!(tunnet_common::packet::parse(&[0x45, 0x00]).is_err()); + assert!(tunnet_common::packet::parse(&[]).is_err()); + let _ = p; + } + + #[test] + fn conntrack_is_shared_bidirectionally() { + // One canonical flow resolves to the same entry in either direction: + // outbound SYN opens it, the inbound reply hits it (no second eval). + let (p, slot) = harness(open_bundle(), true); + let out = meta_tcp(443); + assert_eq!(check_out(&p, &out, &slot), PolicyVerdict::Allow); + assert_eq!(p.conntrack_len(), 1); + // Reply direction: src/dst swapped, same 5-tuple. + let b = + etherparse::PacketBuilder::ipv4([10, 0, 0, 2], [10, 0, 0, 1], 64).tcp(443, 40000, 1, 2); + let mut o = Vec::new(); + b.write(&mut o, b"hi").unwrap(); + let pkt = tunnet_common::packet::parse(&o).unwrap(); + let reply = PacketMeta::from_packet(&pkt); + assert_eq!(check_in(&p, &reply, &slot), PolicyVerdict::Allow); + assert_eq!(p.conntrack_len(), 1, "same canonical entry both ways"); + } + + fn deny_80_bundle() -> PolicyBundle { + PolicyBundle { + rules: vec![PolicyRule { + src: Selector::Any, + dst: Selector::Any, + action: Action::Deny, + ports: vec![tunnet_common::policy::PortRange { start: 80, end: 80 }], + protocol: Some(Protocol::Tcp), + priority: 0, + order_index: 0, + scope: RuleScope::Network, + enabled: true, + slug: None, + src_posture: vec![], + }], + default_action: DefaultAction::Allow, + ..PolicyBundle::default() + } + } + + #[test] + fn revocation_tcp_allow_to_deny() { + // Established TCP must not survive an allow→deny publish. + let (p, slot) = harness(PolicyBundle::default(), false); + let m = meta_tcp(80); + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Allow); + assert_eq!(p.conntrack_len(), 1); + p.publish_acl(&deny_80_bundle(), &self_id(), true, false); + // Next packet of the SAME flow revalidates and is denied. + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Deny); + // And a fresh flow is denied too. + let m2 = meta_tcp(80); + assert_eq!(check_out(&p, &m2, &slot), PolicyVerdict::Deny); + } + + #[test] + fn revocation_udp_allow_to_deny() { + let (p, slot) = harness(PolicyBundle::default(), false); + let m = meta_udp(40000, 53); + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Allow); + let deny_udp = PolicyBundle { + rules: vec![PolicyRule { + src: Selector::Any, + dst: Selector::Any, + action: Action::Deny, + ports: vec![], + protocol: Some(Protocol::Udp), + priority: 0, + order_index: 0, + scope: RuleScope::Network, + enabled: true, + slug: None, + src_posture: vec![], + }], + default_action: DefaultAction::Allow, + ..PolicyBundle::default() + }; + p.publish_acl(&deny_udp, &self_id(), true, false); + assert_eq!(check_out(&p, &m, &slot), PolicyVerdict::Deny); + } + + fn fw_deny_all_inbound() -> (Vec, Vec) { + use tunnet_core_firewall_types::{FirewallAction, FirewallDirection, PeerFilter}; + ( + vec![FirewallRule { + direction: FirewallDirection::In, + action: FirewallAction::Deny, + protocol: Protocol::Any, + ports: vec![], + peer: PeerFilter::Any, + }], + vec![], + ) + } + + /// Check helper reading firewall state the way the hot path does: from + /// the network's stable slot, with no post-publish re-resolution. + fn check_in_slot( + rt: &PolicyRuntime, + m: &PacketMeta, + net: Uuid, + direction: tunnet_common::policy::Direction, + ) -> PolicyVerdict { + let slot = rt.slot_for_network(net); + rt.check( + m, + direction, + "bb", + &[], + None, + Some(net), + &slot, + &slot.counters, + ) + } + + #[test] + fn revocation_suggested_rule_change() { + // Suggested rules arrive via publish_firewall and must revoke too — + // observed through the STABLE slot, with no manual re-resolution + // after publication (§2.1-3). + let net = Uuid::nil(); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([(net, (vec![], vec![], true))]), + &self_id(), + true, + false, + ); + // Pin the slot the way a fast state would (before any publish). + let slot = rt.slot_for_network(net); + let m = meta_tcp(443); + assert_eq!( + rt.check( + &m, + Direction::Outbound, + "bb", + &[], + None, + Some(net), + &slot, + &slot.counters + ), + PolicyVerdict::Allow + ); + let (local, suggested) = fw_deny_all_inbound(); + rt.publish_firewall(net, local, suggested, true); + // Same slot object, no re-fetch: the new rules are visible. + assert_eq!( + check_in_slot(&rt, &m, net, Direction::Inbound), + PolicyVerdict::Deny + ); + } + + #[test] + fn revocation_firewall_disabled_then_deny() { + // Disabled firewall admits; enabling with a deny rule revokes; all + // observed through the stable slot with no re-resolution (§2.1-3). + let net = Uuid::nil(); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([(net, (vec![], vec![], false))]), + &self_id(), + true, + false, + ); + let m = meta_tcp(443); + assert_eq!( + check_in_slot(&rt, &m, net, Direction::Inbound), + PolicyVerdict::Allow + ); + let (local, suggested) = fw_deny_all_inbound(); + rt.publish_firewall(net, local, suggested, true); + assert_eq!( + check_in_slot(&rt, &m, net, Direction::Inbound), + PolicyVerdict::Deny + ); + // And disabling again admits without clearing unrelated state. + rt.publish_firewall(net, vec![], vec![], false); + assert_eq!( + check_in_slot(&rt, &m, net, Direction::Inbound), + PolicyVerdict::Allow + ); + } + + #[test] + fn live_fast_state_observes_publication() { + // §2.1-3: a fast state holding its network slot (like an active + // peer) immediately observes local rule changes, suggested rule + // changes, enable/disable flips, and allow→deny revocation of an + // ESTABLISHED flow — with zero relink calls after install. + use crate::peers::{PeerIdentity, PeerRegistry}; + use iroh::SecretKey; + let net = Uuid::from_u128(0x21); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([(net, (vec![], vec![], true))]), + &self_id(), + true, + false, + ); + let reg = PeerRegistry::new(); + let ep = SecretKey::generate().public(); + let fast = reg.ensure(Arc::new(PeerIdentity { + endpoint: ep, + endpoint_hex: format!("{ep}"), + hostname: "peer".into(), + ip: Ipv4Addr::new(10, 0, 0, 2), + tags: vec![], + network_id: net, + network_name: "net".into(), + })); + // Install-time assignment only. + reg.relink_policy(&rt); + // The hot path, exactly as tun_io does it. + let check_live = |rt: &PolicyRuntime, m: &PacketMeta| { + let slot = fast.policy.load(); + rt.check( + m, + Direction::Inbound, + "bb", + &[], + None, + Some(net), + &slot, + &slot.counters, + ) + }; + // Establish a flow through the live state. + let m = meta_tcp(443); + assert_eq!(check_live(&rt, &m), PolicyVerdict::Allow); + // Local deny rule published: established flow revoked, no relink. + let (local, _) = fw_deny_all_inbound(); + rt.publish_firewall(net, local, vec![], true); + assert_eq!(check_live(&rt, &m), PolicyVerdict::Deny); + // Back to allow via empty rules. + rt.publish_firewall(net, vec![], vec![], true); + let m2 = meta_tcp_ports(40001, 443); + assert_eq!(check_live(&rt, &m2), PolicyVerdict::Allow); + // Suggested deny published: revoked again (suggested rules flow + // through the same slot swap as local ones). + let (suggested_deny, _) = fw_deny_all_inbound(); + rt.publish_firewall(net, vec![], suggested_deny, true); + assert_eq!(check_live(&rt, &m2), PolicyVerdict::Deny); + // Counters object stayed stable across all publishes (stats intact). + let slot = fast.policy.load(); + assert!(slot.counters.denied.load(Ordering::Relaxed) > 0); + assert!(slot.counters.allowed.load(Ordering::Relaxed) > 0); + } + + #[test] + fn publication_is_atomic_under_concurrency() { + // §2.1-4: publishers alternate allow-all/deny-all bundles while + // readers evaluate one fixed flow. Every verdict is paired with the + // generation the reader ACTUALLY used: a verdict at a deny + // generation must be Deny (a torn snapshot+generation pair would + // trust stale conntrack and wrongly Allow). + use std::sync::Mutex; + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::new(), + &self_id(), + true, + false, + ); + let deny_gens = Arc::new(Mutex::new(Vec::::new())); + let records = Arc::new(Mutex::new(Vec::<(PolicyVerdict, u64, u64)>::new())); + let slot = Arc::new(FwSlot { + snapshot: ArcSwap::from_pointee(FwSnapshot { + generation: rt.generation(), + set: FwSet { + enabled: false, + rules: vec![], + }, + }), + counters: Arc::new(FwCounters::default()), + }); + // Establish the flow once (admitted under the bootstrap generation). + let m = meta_tcp(80); + assert_eq!(check_out(&rt, &m, &slot), PolicyVerdict::Allow); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + std::thread::scope(|scope| { + // Publisher: alternate deny/allow, recording deny generations. + let rt_p = rt.clone(); + let deny_gens_p = deny_gens.clone(); + let stop_p = stop.clone(); + let publisher = scope.spawn(move || { + let mut deny = true; + while !stop_p.load(std::sync::atomic::Ordering::Relaxed) { + if deny { + rt_p.publish_acl(&deny_80_bundle(), &self_id(), true, false); + deny_gens_p.lock().unwrap().push(rt_p.generation()); + } else { + rt_p.publish_acl(&PolicyBundle::default(), &self_id(), true, false); + } + deny = !deny; + } + }); + // Readers: hammer the SAME flow, pairing verdict + used gens. + let mut readers = Vec::new(); + for _ in 0..4 { + let rt_r = rt.clone(); + let records_r = records.clone(); + let stop_r = stop.clone(); + let slot_r = slot.clone(); + let m_r = meta_tcp(80); + readers.push(scope.spawn(move || { + while !stop_r.load(std::sync::atomic::Ordering::Relaxed) { + let (v, g, fg) = rt_r.check_with_generation( + &m_r, + Direction::Outbound, + "bb", + &[], + None, + None, + &slot_r, + &slot_r.counters, + ); + records_r.lock().unwrap().push((v, g, fg)); + } + })); + } + std::thread::sleep(std::time::Duration::from_millis(200)); + stop.store(true, std::sync::atomic::Ordering::Relaxed); + let _ = publisher.join(); + for r in readers { + let _ = r.join(); + } + }); + let deny_gens = deny_gens.lock().unwrap(); + let records = records.lock().unwrap(); + assert!(!deny_gens.is_empty(), "publisher must have run"); + assert!(!records.is_empty(), "readers must have run"); + // Verdicts at deny generations are Deny — no torn allow-through. + // (Verdicts at allow generations may be Allow or Deny: a flow + // denied under a deny gen stays denied until... conntrack only + // opens on Allow, so re-allow re-admits; either is consistent.) + for (v, g, _fg) in records.iter() { + if deny_gens.contains(g) { + assert_eq!( + *v, + PolicyVerdict::Deny, + "torn publication: Allow verdict at deny generation {g}" + ); + } + } + // Generations observed are monotonic per snapshot (sanity: the + // sequence of publishes is a total order). + let mut gens: Vec = records.iter().map(|(_, g, _)| *g).collect(); + gens.sort(); + gens.dedup(); + for w in gens.windows(2) { + assert!(w[1] > w[0], "generations must be distinct per publish"); + } + } + + #[test] + fn firewall_publication_is_atomic_under_concurrency() { + // §2.2-2: publisher alternates firewall allow/deny (local rules, + // then suggested, then enabled flips) while readers hammer one + // ESTABLISHED flow. Every verdict pairs with the (acl, fw) + // generations actually used. Invariants: a verdict stamped with a + // deny fw_gen is Deny (the old-allow-firewall-under-new-generation + // poison pair is impossible); a verdict at an allow fw_gen Allows + // (re-allow re-admits, no stuck deny). + use std::sync::Mutex; + use tunnet_core_firewall_types::{FirewallAction, FirewallDirection, PeerFilter}; + let net = Uuid::from_u128(0x22); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([(net, (vec![], vec![], true))]), + &self_id(), + true, + false, + ); + let deny_rule = FirewallRule { + direction: FirewallDirection::Out, + action: FirewallAction::Deny, + protocol: Protocol::Tcp, + ports: vec![], + peer: PeerFilter::Any, + }; + let deny_fw_gens = Arc::new(Mutex::new(Vec::::new())); + let allow_fw_gens = Arc::new(Mutex::new(Vec::::new())); + let records = Arc::new(Mutex::new(Vec::<(PolicyVerdict, u64, u64)>::new())); + let m = meta_tcp(80); + let slot0 = rt.slot_for_network(net); + assert_eq!( + rt.check( + &m, + Direction::Outbound, + "bb", + &[], + None, + Some(net), + &slot0, + &slot0.counters + ), + PolicyVerdict::Allow + ); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + std::thread::scope(|scope| { + let rt_p = rt.clone(); + let deny_p = deny_fw_gens.clone(); + let allow_p = allow_fw_gens.clone(); + let stop_p = stop.clone(); + let deny_rule_p = deny_rule.clone(); + let publisher = scope.spawn(move || { + let mut deny = true; + let mut use_suggested = false; + while !stop_p.load(std::sync::atomic::Ordering::Relaxed) { + if deny { + // Alternate local / suggested / disable-flip denies + // so every mutation kind participates. + if use_suggested { + rt_p.publish_firewall(net, vec![], vec![deny_rule_p.clone()], true); + } else { + rt_p.publish_firewall(net, vec![deny_rule_p.clone()], vec![], true); + } + use_suggested = !use_suggested; + deny_p + .lock() + .unwrap() + .push(rt_p.slot_for_network(net).snapshot.load().generation); + } else { + rt_p.publish_firewall(net, vec![], vec![], true); + allow_p + .lock() + .unwrap() + .push(rt_p.slot_for_network(net).snapshot.load().generation); + } + deny = !deny; + } + }); + let mut readers = Vec::new(); + for _ in 0..4 { + let rt_r = rt.clone(); + let records_r = records.clone(); + let stop_r = stop.clone(); + let m_r = meta_tcp(80); + readers.push(scope.spawn(move || { + let slot_r = rt_r.slot_for_network(net); + while !stop_r.load(std::sync::atomic::Ordering::Relaxed) { + let (v, ag, fg) = rt_r.check_with_generation( + &m_r, + Direction::Outbound, + "bb", + &[], + None, + Some(net), + &slot_r, + &slot_r.counters, + ); + records_r.lock().unwrap().push((v, ag, fg)); + } + })); + } + std::thread::sleep(std::time::Duration::from_millis(200)); + stop.store(true, std::sync::atomic::Ordering::Relaxed); + let _ = publisher.join(); + for r in readers { + let _ = r.join(); + } + }); + let deny_fw_gens = deny_fw_gens.lock().unwrap(); + let allow_fw_gens = allow_fw_gens.lock().unwrap(); + let records = records.lock().unwrap(); + assert!(!deny_fw_gens.is_empty() && !allow_fw_gens.is_empty()); + assert!(!records.is_empty()); + let mut saw_deny = 0u32; + let mut saw_allow = 0u32; + for (v, _ag, fg) in records.iter() { + if deny_fw_gens.contains(fg) { + saw_deny += 1; + assert_eq!( + *v, + PolicyVerdict::Deny, + "old firewall admitted under new generation (fw_gen={fg})" + ); + } else if allow_fw_gens.contains(fg) { + saw_allow += 1; + assert_eq!( + *v, + PolicyVerdict::Allow, + "re-allow failed to re-admit (fw_gen={fg})" + ); + } + } + assert!(saw_deny > 0 && saw_allow > 0, "must observe both phases"); + } + + #[test] + fn concurrent_publishers_lose_no_updates() { + use std::sync::Mutex; + use tunnet_common::policy::PortRange; + use tunnet_core_firewall_types::{FirewallAction, FirewallDirection, PeerFilter}; + let net = Uuid::from_u128(0x23); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([(net, (vec![], vec![], true))]), + &self_id(), + true, + false, + ); + let m = meta_tcp(80); + let slot0 = rt.slot_for_network(net); + assert_eq!( + rt.check( + &m, + Direction::Outbound, + "bb", + &[], + None, + Some(net), + &slot0, + &slot0.counters + ), + PolicyVerdict::Allow + ); + const PUBLISHERS_PER_KIND: u64 = 4; + const ITERS: u64 = 200; + const PUBLISHERS: u64 = 2 * PUBLISHERS_PER_KIND; + let total = PUBLISHERS * ITERS; + let seq = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let log = Arc::new(Mutex::new(Vec::<(u64, u8, u16)>::new())); + let gen_samples = Arc::new(Mutex::new(Vec::::new())); + let acl_bundle = |port: u16| PolicyBundle { + rules: vec![PolicyRule { + src: Selector::Any, + dst: Selector::Any, + action: Action::Deny, + ports: vec![PortRange { + start: port, + end: port, + }], + protocol: Some(Protocol::Tcp), + priority: 0, + order_index: 0, + scope: RuleScope::Network, + enabled: true, + slug: None, + src_posture: vec![], + }], + default_action: DefaultAction::Allow, + ..PolicyBundle::default() + }; + let fw_rule = |port: u16| FirewallRule { + direction: FirewallDirection::Out, + action: FirewallAction::Deny, + protocol: Protocol::Tcp, + ports: vec![PortRange { + start: port, + end: port, + }], + peer: PeerFilter::Any, + }; + std::thread::scope(|scope| { + for kind in [0u8, 0, 0, 0, 1, 1, 1, 1] { + let rt_p = rt.clone(); + let seq_p = seq.clone(); + let log_p = log.clone(); + scope.spawn(move || { + for _ in 0..ITERS { + let s = seq_p.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let port = 9000 + (s % 500) as u16; + let committed = if kind == 0 { + rt_p.publish_acl(&acl_bundle(port), &self_id(), true, false) + } else { + rt_p.publish_firewall(net, vec![fw_rule(port)], vec![], true) + }; + log_p.lock().unwrap().push((committed, kind, port)); + } + }); + } + for _ in 0..2 { + let rt_r = rt.clone(); + let samples_r = gen_samples.clone(); + scope.spawn(move || { + let mut last = 0u64; + for _ in 0..ITERS * 4 { + let g = rt_r.generation(); + assert!(g >= last, "generation regressed {last} -> {g}"); + last = g; + samples_r.lock().unwrap().push(g); + } + }); + } + }); + assert_eq!(rt.generation(), 1 + total); + let log = log.lock().unwrap(); + assert_eq!(log.len() as u64, total); + let mut gens: Vec = log.iter().map(|(g, _, _)| *g).collect(); + gens.sort(); + gens.dedup(); + assert_eq!( + gens.len() as u64, + total, + "every publish committed distinctly" + ); + assert_eq!(gens[0], 2); + assert_eq!(gens[gens.len() - 1], 1 + total); + let (last_gen, last_kind, last_port) = *log.iter().max_by_key(|(g, _, _)| g).unwrap(); + assert_eq!(last_gen, 1 + total); + let inner = rt.inner.load(); + if last_kind == 0 { + assert!( + inner.acl_source.rules.iter().any(|r| r + .ports + .iter() + .any(|p| p.start == last_port && p.end == last_port)), + "final ACL must be the last-committed bundle (port {last_port})" + ); + } else { + let snap = rt.slot_for_network(net).snapshot.load(); + assert_eq!(snap.generation, last_gen); + assert!( + snap.set + .rules + .iter() + .any(|r| r.ports.contains(&(last_port, last_port))), + "final firewall must be the last-committed set (port {last_port})" + ); + } + assert!(!gen_samples.lock().unwrap().is_empty()); + rt.publish_acl(&PolicyBundle::default(), &self_id(), true, false); + rt.publish_firewall(net, vec![], vec![], true); + let slot = rt.slot_for_network(net); + assert_eq!( + rt.check( + &m, + Direction::Outbound, + "bb", + &[], + None, + Some(net), + &slot, + &slot.counters + ), + PolicyVerdict::Allow + ); + } + + #[test] + fn conntrack_is_network_scoped() { + // §2.2-1 (test 7): identical 5-tuples in A and B create independent + // conntrack entries; a flow admitted in A never satisfies B, and a + // deny published in B revokes B while A keeps working. + use tunnet_core_firewall_types::{FirewallAction, FirewallDirection, PeerFilter}; + let net_a = Uuid::from_u128(0x0a); + let net_b = Uuid::from_u128(0x0b); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([ + (net_a, (vec![], vec![], true)), + (net_b, (vec![], vec![], true)), + ]), + &self_id(), + true, + false, + ); + let slot_a = rt.slot_for_network(net_a); + let slot_b = rt.slot_for_network(net_b); + let m = meta_tcp_ports(40000, 443); + let check_net = |rt: &PolicyRuntime, net: Uuid, slot: &FwSlot| { + rt.check( + &m, + Direction::Outbound, + "bb", + &[], + None, + Some(net), + slot, + &slot.counters, + ) + }; + assert_eq!(check_net(&rt, net_a, &slot_a), PolicyVerdict::Allow); + assert_eq!(rt.conntrack_len(), 1); + // Same 5-tuple under B: full evaluation, second entry. + assert_eq!(check_net(&rt, net_b, &slot_b), PolicyVerdict::Allow); + assert_eq!(rt.conntrack_len(), 2, "one entry per network"); + // Deny in B only: B revokes, A keeps working. + let deny_out = FirewallRule { + direction: FirewallDirection::Out, + action: FirewallAction::Deny, + protocol: Protocol::Tcp, + ports: vec![], + peer: PeerFilter::Any, + }; + rt.publish_firewall(net_b, vec![deny_out], vec![], true); + assert_eq!(check_net(&rt, net_b, &slot_b), PolicyVerdict::Deny); + assert_eq!(check_net(&rt, net_a, &slot_a), PolicyVerdict::Allow); + } + + #[test] + fn cross_network_firewall_isolation() { + // Rules from network A must never affect network B, including + // endpoint/hostname-scoped rules and disabled-network handling. + // Uses the SAME endpoint and 5-tuple in both networks: only the + // network-scoped conntrack key + per-network slot keeps them apart. + use tunnet_core_firewall_types::{ + FirewallAction, FirewallDirection, FirewallRule, PeerFilter, + }; + let net_a = Uuid::from_u128(0x0a); + let net_b = Uuid::from_u128(0x0b); + let a_rule = FirewallRule { + direction: FirewallDirection::In, + action: FirewallAction::Deny, + protocol: Protocol::Tcp, + ports: vec![], + peer: PeerFilter::Any, + }; + let a_ep_rule = FirewallRule { + direction: FirewallDirection::Out, + action: FirewallAction::Deny, + protocol: Protocol::Tcp, + ports: vec![], + peer: PeerFilter::Endpoint("cc".into()), + }; + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([ + (net_a, (vec![a_rule, a_ep_rule], vec![], true)), + (net_b, (vec![], vec![], true)), + ]), + &self_id(), + true, + false, + ); + let slot_a = rt.slot_for_network(net_a); + let slot_b = rt.slot_for_network(net_b); + // Identical 5-tuple + endpoint in both networks: conntrack MUST NOT + // leak across (network is in the key), and firewall verdicts come + // from each network's own slot. + let m_in = meta_tcp_ports(40000, 443); + let m_in_b = meta_tcp_ports(40000, 443); + let m_out_cc = meta_tcp_ports(40002, 443); + let m_out_cc_b = meta_tcp_ports(40002, 443); + let m_out_bb = meta_tcp_ports(40004, 443); + // A denies inbound; B allows the identical packet. + assert_eq!( + rt.check( + &m_in, + Direction::Inbound, + "bb", + &[], + None, + Some(net_a), + &slot_a, + &slot_a.counters + ), + PolicyVerdict::Deny + ); + assert_eq!( + rt.check( + &m_in_b, + Direction::Inbound, + "bb", + &[], + None, + Some(net_b), + &slot_b, + &slot_b.counters + ), + PolicyVerdict::Allow + ); + // Endpoint-scoped rule in A denies peer "cc" outbound under A... + assert_eq!( + rt.check( + &m_out_cc, + Direction::Outbound, + "cc", + &[], + None, + Some(net_a), + &slot_a, + &slot_a.counters + ), + PolicyVerdict::Deny + ); + // ...but not under B, and not for other peers under A. + assert_eq!( + rt.check( + &m_out_cc_b, + Direction::Outbound, + "cc", + &[], + None, + Some(net_b), + &slot_b, + &slot_b.counters + ), + PolicyVerdict::Allow + ); + assert_eq!( + rt.check( + &m_out_bb, + Direction::Outbound, + "bb", + &[], + None, + Some(net_a), + &slot_a, + &slot_a.counters + ), + PolicyVerdict::Allow + ); + } + + #[test] + fn disabled_network_does_not_disable_others() { + // The old global `enabled &&` flattening bug, proven gone: one + // disabled network firewall must not affect another network. + let net_a = Uuid::from_u128(0x0a); + let net_b = Uuid::from_u128(0x0b); + let rt = PolicyRuntime::bootstrap( + &PolicyBundle::default(), + &HashMap::from([ + (net_a, (vec![], vec![], false)), + (net_b, (vec![], vec![], true)), + ]), + &self_id(), + true, + false, + ); + assert!(!rt.fw_for_network(net_a).enabled); + assert!(rt.fw_for_network(net_b).enabled); + let m = meta_tcp(443); + let slot_b = rt.slot_for_network(net_b); + assert_eq!( + rt.check( + &m, + Direction::Inbound, + "bb", + &[], + None, + Some(net_b), + &slot_b, + &slot_b.counters + ), + PolicyVerdict::Allow + ); + } +} diff --git a/crates/tunnet-core/src/reassembly.rs b/crates/tunnet-core/src/reassembly.rs new file mode 100644 index 00000000..50851508 --- /dev/null +++ b/crates/tunnet-core/src/reassembly.rs @@ -0,0 +1,771 @@ +//! Bounded per-peer overlay reassembly (§4). +//! +//! Handles out-of-order/duplicate/missing segments, timeouts, path changes, +//! ID wrap and collisions, malformed indexes, inconsistent lengths, and +//! conflicting duplicates. Never allocates from an untrusted claimed length +//! without validating against hard limits; authentication of the QUIC peer +//! does not grant unlimited memory amplification. +//! +//! A lost segment means the logical packet is eventually discarded — no +//! overlay retransmission (QUIC DATAGRAM semantics preserved; inner TCP +//! recovers). + +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; +use uuid::Uuid; + +use bytes::Bytes; +use tunnet_common::packet::{Frame, MAX_LOGICAL_LEN, MAX_SEGMENTS, SegmentHeader, decode_frame}; + +/// Maximum concurrent reassemblies per peer. +pub const MAX_ENTRIES_PER_PEER: usize = 32; +/// Maximum bytes held in reassembly per peer. +pub const MAX_BYTES_PER_PEER: usize = 256 * 1024; +/// Hard GLOBAL reassembly budget across all peers (§2.1-5): enforced by +/// atomic reservation, impossible to exceed even with concurrent peers. +/// 4 MiB = 16 fully-loaded peers; beyond that, senders back off via QUIC +/// DATAGRAM drops (inner TCP recovers). +pub const MAX_BYTES_GLOBAL: usize = 4 * 1024 * 1024; +/// Reassembly lifetime: a missing segment kills the packet after this. +pub const REASSEMBLY_TIMEOUT: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReassemblyDrop { + Malformed, + TooManyEntries, + OverBytes, + Timeout, + Conflict, + Incomplete, +} + +#[derive(Debug)] +pub enum InsertOut { + /// Logical packet complete (single assembled buffer). + Complete(Vec), + /// Segment stored, waiting for more. + Pending, + /// Exact duplicate segment (same bytes): ignored. + Duplicate, + /// Dropped with reason (conflict/timeout/caps). Entry removed. + Dropped(ReassemblyDrop), +} + +struct Entry { + total: u16, + count: u16, + segments: Vec>, + have: usize, + bytes: usize, + deadline: Instant, +} + +pub struct ReassemblyTable { + entries: HashMap, + order: VecDeque, + bytes: usize, + global_bytes: Arc, + max_bytes_global: u64, +} + +impl ReassemblyTable { + pub fn new(global_bytes: Arc) -> Self { + Self::with_global_cap(global_bytes, MAX_BYTES_GLOBAL as u64) + } + + pub fn with_global_cap(global_bytes: Arc, max_bytes_global: u64) -> Self { + Self { + entries: HashMap::new(), + order: VecDeque::new(), + bytes: 0, + global_bytes, + max_bytes_global, + } + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn bytes(&self) -> usize { + self.bytes + } + + /// Insert a validated segment (decoder already bounds-checked it). + /// `now` is the observation time (allows deterministic tests). + pub fn insert(&mut self, h: SegmentHeader, payload: Bytes, now: Instant) -> InsertOut { + self.expire(now); + let total = h.total as usize; + let count = h.count as usize; + // Defensive re-validation (decoder guarantees, but tables outlive frames). + if total == 0 + || total > MAX_LOGICAL_LEN + || !(2..=MAX_SEGMENTS).contains(&count) + || (h.index as usize) >= count + || payload.is_empty() + || payload.len() > total + { + return InsertOut::Dropped(ReassemblyDrop::Malformed); + } + // ID collision with incompatible shape: drop the old generation and + // start over (bounded loss, never mixed bytes). + if let Some(e) = self.entries.get(&h.id) + && (e.total != h.total || e.count != h.count) + { + self.remove(h.id); + return self.insert(h, payload, now); + } + // Admit or fetch the entry, enforcing caps first. + if !self.entries.contains_key(&h.id) { + if self.entries.len() >= MAX_ENTRIES_PER_PEER { + // Evict the oldest to make room (bounded eviction). + if let Some(old) = self.order.pop_front() { + self.remove(old); + } + } + if self.entries.len() >= MAX_ENTRIES_PER_PEER { + return InsertOut::Dropped(ReassemblyDrop::TooManyEntries); + } + let mut segments = Vec::with_capacity(count); + segments.resize_with(count, || None); + self.entries.insert( + h.id, + Entry { + total: h.total, + count: h.count, + segments, + have: 0, + bytes: 0, + deadline: now + REASSEMBLY_TIMEOUT, + }, + ); + self.order.push_back(h.id); + } + let idx = h.index as usize; + // Duplicate handling needs no mutation: check first. + if let Some(existing) = self + .entries + .get(&h.id) + .and_then(|e| e.segments[idx].as_ref()) + { + // Identical bytes are harmless; conflicting bytes mean someone + // is rewriting history → drop the whole reassembly. + if existing.as_ref() == payload.as_ref() { + return InsertOut::Duplicate; + } + self.remove(h.id); + return InsertOut::Dropped(ReassemblyDrop::Conflict); + } + // Byte caps (per-peer + hard global) before retaining. + // Per-peer first (no shared state touched), then the global atomic + // reservation (CAS: impossible to exceed, even concurrently). + if self.bytes + payload.len() > MAX_BYTES_PER_PEER { + // Try one bounded eviction of the oldest entry, then give up. + if let Some(old) = self.order.front().cloned() + && old != h.id + { + self.remove(old); + } + if self.bytes + payload.len() > MAX_BYTES_PER_PEER { + return InsertOut::Dropped(ReassemblyDrop::OverBytes); + } + } + if !self.reserve_global(payload.len() as u64) { + // Global pressure: evict the oldest entry (releases global + // bytes), then re-check BOTH caps — the eviction may have been + // a same-peer entry (per-peer changed too) or another shape. + if let Some(old) = self.order.front().cloned() + && old != h.id + { + self.remove(old); + } + if self.bytes + payload.len() > MAX_BYTES_PER_PEER { + return InsertOut::Dropped(ReassemblyDrop::OverBytes); + } + if !self.reserve_global(payload.len() as u64) { + return InsertOut::Dropped(ReassemblyDrop::OverBytes); + } + } + let entry = self.entries.get_mut(&h.id).expect("admitted"); + entry.bytes += payload.len(); + self.bytes += payload.len(); + // Global bytes were already reserved (CAS) above; removal paths + // release exactly entry.bytes, so the counter stays exact. + entry.segments[idx] = Some(payload); + entry.have += 1; + if entry.have < count { + return InsertOut::Pending; + } + // Complete: assemble in index order into one bounded buffer. + let mut out = Vec::with_capacity(total); + for seg in entry.segments.iter().flatten() { + out.extend_from_slice(seg); + } + // `entry` is dead here; removal below is safe. + if out.len() != total { + // Overlapping/gapped indexes or short writes: fail closed. + self.remove(h.id); + return InsertOut::Dropped(ReassemblyDrop::Incomplete); + } + self.remove(h.id); + InsertOut::Complete(out) + } + + /// Feed a raw DATAGRAM through frame decode + insert (convenience for the + /// inbound path and tests). Singles are returned directly, with their + /// bound network. + pub fn feed_datagram(&mut self, data: Bytes, now: Instant) -> FeedOut { + match decode_frame(&data) { + Ok(Frame::Single { net, payload: p }) => FeedOut::Single(net, p.to_vec()), + Ok(Frame::Segment { + header: h, payload, .. + }) => { + // Retain the payload without copying: slice the DATAGRAM. + let start = data.len() - payload.len(); + let owned = data.slice(start..); + match self.insert(h, owned, now) { + InsertOut::Complete(logical) => FeedOut::Complete(logical), + InsertOut::Pending => FeedOut::Pending, + InsertOut::Duplicate => FeedOut::Duplicate, + InsertOut::Dropped(reason) => FeedOut::Dropped(reason), + } + } + Err(_) => FeedOut::Dropped(ReassemblyDrop::Malformed), + } + } + + fn remove(&mut self, id: u32) { + if let Some(e) = self.entries.remove(&id) { + self.bytes = self.bytes.saturating_sub(e.bytes); + self.global_bytes + .fetch_sub(e.bytes as u64, Ordering::Relaxed); + self.order.retain(|k| *k != id); + } + } + + /// Atomically reserve `n` global bytes (CAS loop via fetch_update). + /// Returns false without changing anything when the hard cap would be + /// exceeded. Every successful reservation is paired with exactly one + /// release in `remove` (which subtracts entry.bytes), so the counter + /// is exact and the cap is impossible to exceed — even with concurrent + /// peers racing on the shared counter. + fn reserve_global(&self, n: u64) -> bool { + self.global_bytes + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |cur| { + cur.checked_add(n) + .filter(|next| *next <= self.max_bytes_global) + .map(|_| cur.saturating_add(n)) + }) + .is_ok() + } + + fn expire(&mut self, now: Instant) { + // Bounded sweep: entries are capped (32), so a full pass is O(1). + let mut timed_out = Vec::new(); + for (id, e) in self.entries.iter() { + if now >= e.deadline { + timed_out.push(*id); + } + } + for id in timed_out { + self.remove(id); + } + } + + #[cfg(test)] + fn force_expire_all(&mut self) { + let ids: Vec = self.entries.keys().cloned().collect(); + for id in ids { + self.remove(id); + } + } +} + +impl Drop for ReassemblyTable { + /// Release outstanding global reservations (§2.2-4): dropping a table + /// with pending reassemblies (peer churn) must return its bytes to the + /// shared budget, or repeated churn would permanently exhaust the cap + /// with no live reassemblies. Invariant: after all operations, + /// `global_bytes == sum(bytes of live tables)`. + fn drop(&mut self) { + let held = self.bytes; + self.bytes = 0; + self.entries.clear(); + self.order.clear(); + self.global_bytes.fetch_sub(held as u64, Ordering::Relaxed); + } +} + +#[derive(Debug)] +pub enum FeedOut { + Single(Uuid, Vec), + Complete(Vec), + Pending, + Duplicate, + Dropped(ReassemblyDrop), +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + use tunnet_common::packet::{SegmentHeader, encode_segment_prefix}; + + fn table() -> ReassemblyTable { + ReassemblyTable::new(Arc::new(AtomicU64::new(0))) + } + + fn seg(h: SegmentHeader, payload: &[u8]) -> (SegmentHeader, Bytes) { + (h, Bytes::copy_from_slice(payload)) + } + + fn logical_bytes(n: usize) -> Vec { + // Minimal valid IPv4/UDP packet of exactly n bytes. + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(40000, 443); + let mut o = Vec::new(); + b.write(&mut o, &vec![0xABu8; n.saturating_sub(28)]) + .unwrap(); + o + } + + #[test] + fn out_of_order_round_trip() { + let mut t = table(); + let total = 3000u16; + let now = Instant::now(); + // 3 segments of 1000 (last may be short). + let s0 = vec![1u8; 1000]; + let s1 = vec![2u8; 1000]; + let s2 = vec![3u8; 1000]; + // Arrive 2, 0, 1. + let (h, p) = seg( + SegmentHeader { + id: 9, + index: 2, + count: 3, + total, + }, + &s2, + ); + assert!(matches!(t.insert(h, p, now), InsertOut::Pending)); + let (h, p) = seg( + SegmentHeader { + id: 9, + index: 0, + count: 3, + total, + }, + &s0, + ); + assert!(matches!(t.insert(h, p, now), InsertOut::Pending)); + let (h, p) = seg( + SegmentHeader { + id: 9, + index: 1, + count: 3, + total, + }, + &s1, + ); + match t.insert(h, p, now) { + InsertOut::Complete(logical) => { + assert_eq!(logical.len(), 3000); + assert_eq!(&logical[..1000], &s0[..]); + assert_eq!(&logical[1000..2000], &s1[..]); + assert_eq!(&logical[2000..], &s2[..]); + } + other => panic!("expected complete, got {other:?}"), + } + assert!(t.is_empty()); + } + + #[test] + fn duplicates_and_conflicts() { + let mut t = table(); + let now = Instant::now(); + let (h, p) = seg( + SegmentHeader { + id: 1, + index: 0, + count: 2, + total: 200, + }, + &[5u8; 100], + ); + assert!(matches!(t.insert(h, p.clone(), now), InsertOut::Pending)); + // Identical duplicate: ignored. + assert!(matches!(t.insert(h, p, now), InsertOut::Duplicate)); + // Conflicting duplicate: whole reassembly dropped. + let (h2, p2) = seg( + SegmentHeader { + id: 1, + index: 0, + count: 2, + total: 200, + }, + &[6u8; 100], + ); + assert!(matches!( + t.insert(h2, p2, now), + InsertOut::Dropped(ReassemblyDrop::Conflict) + )); + assert!(t.is_empty()); + } + + #[test] + fn timeout_discards() { + let mut t = table(); + let now = Instant::now(); + let (h, p) = seg( + SegmentHeader { + id: 2, + index: 0, + count: 2, + total: 200, + }, + &[1u8; 100], + ); + assert!(matches!(t.insert(h, p, now), InsertOut::Pending)); + // Late second segment past the deadline starts over (old entry gone). + let late = now + REASSEMBLY_TIMEOUT + Duration::from_millis(10); + let (h2, p2) = seg( + SegmentHeader { + id: 2, + index: 1, + count: 2, + total: 200, + }, + &[2u8; 100], + ); + // Old entry expired on insert sweep; index-1 alone cannot complete. + assert!(matches!(t.insert(h2, p2, late), InsertOut::Pending)); + assert_eq!(t.len(), 1); + t.force_expire_all(); + assert!(t.is_empty()); + } + + #[test] + fn id_collision_with_different_shape_restarts() { + let mut t = table(); + let now = Instant::now(); + let (h, p) = seg( + SegmentHeader { + id: 3, + index: 0, + count: 2, + total: 200, + }, + &[1u8; 100], + ); + assert!(matches!(t.insert(h, p, now), InsertOut::Pending)); + // Same ID, different shape (wrap/collision): old dropped, new pending. + let (h2, p2) = seg( + SegmentHeader { + id: 3, + index: 0, + count: 3, + total: 300, + }, + &[1u8; 100], + ); + assert!(matches!(t.insert(h2, p2, now), InsertOut::Pending)); + assert_eq!(t.len(), 1); + } + + #[test] + fn caps_bound_memory() { + let global = Arc::new(AtomicU64::new(0)); + let mut t = ReassemblyTable::with_global_cap(global.clone(), 500); + let now = Instant::now(); + // Fill per-peer entries to the cap. + for id in 0..40u32 { + let (h, p) = seg( + SegmentHeader { + id, + index: 0, + count: 2, + total: 200, + }, + &[1u8; 100], + ); + let _ = t.insert(h, p, now); + } + assert!(t.len() <= MAX_ENTRIES_PER_PEER); + // Hard global cap: NEVER exceeded (no intentional overshoot). + assert!( + global.load(Ordering::Relaxed) <= 500, + "global cap is a limit, not telemetry" + ); + } + + #[test] + fn global_cap_holds_under_concurrent_peers() { + // §2.1-5: many peers sharing one counter hammer inserts from + // threads; the shared counter must never exceed the cap, and every + // table must respect the per-peer cap. + use std::sync::Mutex; + let global = Arc::new(AtomicU64::new(0)); + let tables: Vec> = (0..8) + .map(|_| Mutex::new(ReassemblyTable::with_global_cap(global.clone(), 4096))) + .collect(); + let peak = Arc::new(AtomicU64::new(0)); + std::thread::scope(|s| { + for (pi, table) in tables.iter().enumerate() { + let peak = peak.clone(); + let global = global.clone(); + s.spawn(move || { + let now = Instant::now(); + for i in 0..200u32 { + let id = (pi as u32) * 1000 + (i % 40); + let h = SegmentHeader { + id, + index: (i % 2) as u16, + count: 2, + total: 200, + }; + let mut payload = vec![0u8; 100]; + payload[0] = (i & 0xff) as u8; + let _ = table.lock().unwrap().insert(h, Bytes::from(payload), now); + // Sample the shared counter mid-race. + let cur = global.load(Ordering::Relaxed); + peak.fetch_max(cur, Ordering::Relaxed); + assert!(cur <= 4096, "global cap exceeded under concurrency"); + } + }); + } + }); + assert!(global.load(Ordering::Relaxed) <= 4096); + assert!(peak.load(Ordering::Relaxed) <= 4096); + for t in &tables { + assert!((t.lock().unwrap().bytes() as u64) <= MAX_BYTES_PER_PEER as u64); + } + } + + #[test] + fn malicious_huge_total_rejected() { + let mut t = table(); + let now = Instant::now(); + // Decoder rejects total > MAX; insert double-checks too. + let (h, p) = seg( + SegmentHeader { + id: 4, + index: 0, + count: 2, + total: 9001, + }, + &[1u8; 100], + ); + assert!(matches!( + t.insert(h, p, now), + InsertOut::Dropped(ReassemblyDrop::Malformed) + )); + assert!(t.is_empty()); + } + + #[test] + fn feed_single_passthrough() { + use tunnet_common::packet::{KIND_SINGLE, SINGLE_OVERHEAD}; + let mut t = table(); + let now = Instant::now(); + let net = Uuid::from_u128(0x0e); + let mut raw = vec![KIND_SINGLE]; + raw.extend_from_slice(net.as_bytes()); + raw.extend_from_slice(&logical_bytes(200)); + match t.feed_datagram(Bytes::from(raw.clone()), now) { + FeedOut::Single(got_net, v) => { + assert_eq!(got_net, net); + assert_eq!(v, raw[SINGLE_OVERHEAD..]); + } + other => panic!("expected single, got {other:?}"), + } + } + + #[test] + fn drop_releases_global_reservation() { + // §2.2-4: dropping a table with pending reassemblies returns its + // bytes; repeated churn cannot exhaust the cap. + let global = Arc::new(AtomicU64::new(0)); + { + let mut t = ReassemblyTable::with_global_cap(global.clone(), 4096); + let now = Instant::now(); + let (h, p) = seg( + SegmentHeader { + id: 11, + index: 0, + count: 2, + total: 200, + }, + &[9u8; 100], + ); + assert!(matches!(t.insert(h, p, now), InsertOut::Pending)); + assert_eq!(global.load(Ordering::Relaxed), 100); + } + assert_eq!(global.load(Ordering::Relaxed), 0, "drop must release"); + // Churn stress: create/fill/drop repeatedly, counter returns to 0. + for round in 0..25u32 { + let mut t = ReassemblyTable::with_global_cap(global.clone(), 4096); + let now = Instant::now(); + for id in 0..8u32 { + let (h, p) = seg( + SegmentHeader { + id: round * 100 + id, + index: 0, + count: 2, + total: 200, + }, + &[7u8; 100], + ); + let _ = t.insert(h, p, now); + } + drop(t); + assert_eq!( + global.load(Ordering::Relaxed), + 0, + "round {round}: counter must return to zero" + ); + } + } + + #[test] + fn multiple_peers_same_id_isolated() { + // Tables are per-peer: identical IDs never interact. + let mut a = table(); + let mut b = table(); + let now = Instant::now(); + let (h, p) = seg( + SegmentHeader { + id: 7, + index: 0, + count: 2, + total: 200, + }, + &[1u8; 100], + ); + assert!(matches!(a.insert(h, p, now), InsertOut::Pending)); + let (h2, p2) = seg( + SegmentHeader { + id: 7, + index: 1, + count: 2, + total: 200, + }, + &[2u8; 100], + ); + assert!(matches!(b.insert(h2, p2, now), InsertOut::Pending)); + assert_eq!(a.len(), 1); + assert_eq!(b.len(), 1); + } + + proptest! { + /// Random segment streams never panic, never exceed caps, and either + /// complete with exactly `total` bytes or stay bounded. + #[test] + fn segment_streams_bounded( + id in 0..4u32, + count in 2..5u16, + total in 128..3000u16, + idxs in prop::collection::vec(0..5u16, 0..12), + seed in any::(), + ) { + let mut t = table(); + let now = Instant::now(); + // Deterministic payload per (seed, index) so duplicates are exact. + for idx in idxs { + let index = idx % count; + let h = SegmentHeader { id, index, count, total }; + let plen = if index + 1 < count { 64 } else { 1 }; + let mut pb = vec![0u8; plen.min(total as usize)]; + for (i, b) in pb.iter_mut().enumerate() { + *b = (seed.wrapping_add(index as u64).wrapping_add(i as u64) & 0xff) as u8; + } + match t.insert(h, Bytes::from(pb), now) { + InsertOut::Complete(v) => { + prop_assert_eq!(v.len(), total as usize); + } + InsertOut::Pending | InsertOut::Duplicate | InsertOut::Dropped(_) => {} + } + prop_assert!(t.len() <= MAX_ENTRIES_PER_PEER); + prop_assert!((t.bytes() as u64) <= MAX_BYTES_PER_PEER as u64); + } + } + } + + #[test] + fn large_mtu_full_pipeline() { + // §22: a 2800-byte logical packet through planning → segmentation → + // out-of-order delivery → reassembly → IP parse, with identity intact. + use tunnet_common::packet::{PacketMeta, parse, segment_count}; + let logical = logical_bytes(2800); + assert_eq!(logical.len(), 2800); + let mps = 1350; + let count = segment_count(2800, mps).expect("segmentable"); + assert!(count > 1); + let seg_cap = mps - tunnet_common::packet::SEGMENT_OVERHEAD; + let mut t = table(); + let now = Instant::now(); + let id = 0xABCDu32; + // Encode segments like the pump does. + let mut frames: Vec<(SegmentHeader, Bytes)> = Vec::new(); + for i in 0..count { + let off = i * seg_cap; + let end = (off + seg_cap).min(2800); + let h = SegmentHeader { + id, + index: i as u16, + count: count as u16, + total: 2800, + }; + frames.push((h, Bytes::copy_from_slice(&logical[off..end]))); + } + // Deliver in reverse order. + let mut completed = None; + for (h, p) in frames.into_iter().rev() { + match t.insert(h, p, now) { + InsertOut::Complete(v) => completed = Some(v), + InsertOut::Pending => {} + other => panic!("unexpected {other:?}"), + } + } + let out = completed.expect("must complete"); + assert_eq!(out, logical); + // The reassembled bytes parse as the original IP packet. + let pkt = parse(&out).unwrap(); + let meta = PacketMeta::from_packet(&pkt); + assert_eq!(meta.dst_v4, Some(std::net::Ipv4Addr::new(10, 0, 0, 2))); + assert_eq!(meta.transport.dst_port(), Some(443)); + } + + #[test] + fn encode_prefix_shapes_match_decoder() { + // encode_segment_prefix output must always decode (property bridge). + use tunnet_common::packet::SEGMENT_OVERHEAD; + let net = Uuid::from_u128(0x0f); + let h = SegmentHeader { + id: 42, + index: 1, + count: 4, + total: 5000, + }; + let mut buf = [0u8; 128]; + let n = encode_segment_prefix(&mut buf, net, h); + assert_eq!(n, SEGMENT_OVERHEAD); + buf[SEGMENT_OVERHEAD..SEGMENT_OVERHEAD + 64].fill(0xCC); + match tunnet_common::packet::decode_frame(&buf[..SEGMENT_OVERHEAD + 64]) { + Ok(tunnet_common::packet::Frame::Segment { + net: got_net, + header: got, + .. + }) => { + assert_eq!(got_net, net); + assert_eq!(got, h); + } + other => panic!("unexpected {other:?}"), + } + } +} diff --git a/crates/tunnet-core/src/routing.rs b/crates/tunnet-core/src/routing.rs index ac7679f3..1c921cdb 100644 --- a/crates/tunnet-core/src/routing.rs +++ b/crates/tunnet-core/src/routing.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::net::Ipv4Addr; use std::sync::Arc; -use arc_swap::ArcSwap; +use arc_swap::{ArcSwap, ArcSwapOption}; use dashmap::DashMap; use ipnet::Ipv4Net; use iroh::EndpointId; @@ -13,6 +13,9 @@ use tunnet_common::{ }; use uuid::Uuid; +use crate::peers::{PeerIdentity, PeerMembershipState, PeerRegistry}; +use crate::policy_runtime::{FwSlot, PolicyRuntime}; + pub struct PeerInfo { pub endpoint: EndpointId, pub endpoint_hex: String, @@ -22,6 +25,56 @@ pub struct PeerInfo { pub network_id: Uuid, pub network_name: String, pub ssh_host_key: Option, + /// Stable fast state shared with the established packet path (§0.5). + /// Reused across routing snapshot rebuilds; cloned (not looked up) per + /// packet via the route decision. + pub fast: Arc, +} + +// Manual Debug: EndpointId may not implement it in all feature sets; keep a +// compact representation for logs/tests. +impl std::fmt::Debug for PeerInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PeerInfo") + .field("endpoint_hex", &self.endpoint_hex) + .field("hostname", &self.hostname) + .field("ip", &self.ip) + .field("network_id", &self.network_id) + .finish() + } +} + +/// Stable per-peer fast-path handle returned directly by routing. +/// +/// Carries the binary endpoint ID plus the resolved peer, so established +/// packet forwarding needs no DashMap lookup, no endpoint-hex conversion, +/// and no second route-table load. +#[derive(Debug, Clone)] +pub struct FastPeerHandle { + pub endpoint: EndpointId, + pub peer: Arc, +} + +impl FastPeerHandle { + pub fn endpoint_hex(&self) -> &str { + &self.peer.endpoint_hex + } +} + +/// Single routing verdict from one immutable snapshot load. +#[derive(Debug, Clone)] +pub enum RouteDecision { + LocalMagic, + LocalAdvertised, + Peer(FastPeerHandle), + NoRoute, +} + +impl RouteDecision { + pub fn peer(peer: Arc) -> Self { + let endpoint = peer.endpoint; + Self::Peer(FastPeerHandle { endpoint, peer }) + } } /// Resolved hostname route (exact or wildcard). @@ -55,8 +108,9 @@ pub struct Tables { pub by_hostname: std::collections::HashMap>, /// Longest-prefix-match subnet routes (via PrefixMap). pub subnets: PrefixMap>, - /// CIDRs this node itself advertises (local LAN forwarding). - pub advertised: Vec, + /// CIDRs this node itself advertises, as a prefix structure (§12: no + /// linear scan on the hot path). + pub advertised: PrefixMap, /// Exact hostname → gateway. pub hostname_exact: std::collections::HashMap>, /// Wildcard suffixes, longest first. @@ -73,6 +127,8 @@ pub struct Tables { pub exit_node: Option>, /// When true, RFC1918 destinations are not sent via the exit node. pub allow_local_lan: bool, + /// This node advertises a default route (computed at rebuild). + pub is_exit: bool, pub version: u64, } @@ -84,6 +140,13 @@ pub struct RoutingTable { slices: Arc>>, /// Manual IP overrides: (network_id, peer_key) → ip. peer_key is hostname or endpoint hex. overrides: Arc>, + /// Stable per-peer fast states, shared with ConnPool slow paths (§0.5). + /// Rebuilds reuse these objects; the packet path never touches this map + /// (states ride inside PeerInfo/route decisions instead). + fast_registry: Arc, + /// Shared policy runtime for firewall slot assignment at (re)resolution + /// (§2.1-3). Set once at dataplane install; slow path only. + policy_runtime: Arc>, } impl Default for RoutingTable { @@ -101,7 +164,7 @@ impl RoutingTable { by_endpoint: Default::default(), by_hostname: Default::default(), subnets: PrefixMap::new(), - advertised: Default::default(), + advertised: PrefixMap::new(), hostname_exact: Default::default(), hostname_wildcards: Default::default(), advertised_hostnames: Default::default(), @@ -111,14 +174,66 @@ impl RoutingTable { magic_ip: Ipv4Addr::new(100, 100, 100, 53), exit_node: None, allow_local_lan: true, + is_exit: false, version: 0, })), dynamic_synth: Arc::new(DashMap::new()), slices: Arc::new(Mutex::new(BTreeMap::new())), overrides: Arc::new(DashMap::new()), + fast_registry: Arc::new(PeerRegistry::new()), + policy_runtime: Arc::new(ArcSwapOption::empty()), } } + /// Install the shared policy runtime (dataplane bring-up, slow path). + /// Every (re)resolution from then on assigns the peer's stable network + /// firewall slot (§2.1-3). + pub fn set_policy_runtime(&self, runtime: PolicyRuntime) { + self.policy_runtime.store(Some(Arc::new(runtime))); + } + + /// Stable firewall slot for a network, if the runtime is installed. + pub fn policy_slot_for(&self, network: Uuid) -> Option> { + self.policy_runtime + .load_full() + .map(|rt| rt.slot_for_network(network)) + } + + /// Stable fast-state registry shared with connection slow paths. + pub fn peer_registry(&self) -> &Arc { + &self.fast_registry + } + + /// Ensure (slow path: rebuild only) the stable fast state for a peer. + /// (Re)assigns the network's stable firewall slot, so peers joining + /// after install and peers changing networks observe publication with + /// no relink (§2.1-3). + #[allow(clippy::too_many_arguments)] + fn ensure_fast( + &self, + endpoint: EndpointId, + endpoint_hex: &str, + hostname: &str, + ip: Ipv4Addr, + tags: &[String], + network_id: Uuid, + network_name: &str, + ) -> Arc { + let state = self.fast_registry.ensure(Arc::new(PeerIdentity { + endpoint, + endpoint_hex: endpoint_hex.to_string(), + hostname: hostname.to_string(), + ip, + tags: tags.to_vec(), + network_id, + network_name: network_name.to_string(), + })); + if let Some(slot) = self.policy_slot_for(network_id) { + state.policy.store(slot); + } + state + } + /// Look up peer by (network, ip) for inbound / firewall context. pub fn lookup_network_ip(&self, network_id: Uuid, ip: &Ipv4Addr) -> Option> { self.inner @@ -141,30 +256,47 @@ impl RoutingTable { self.rebuild(None); } - /// Direct peer IP, subnet LPM, then selected exit node for internet. - /// On birthday collisions across networks, first-joined network wins. - pub fn lookup_ip(&self, ip: &Ipv4Addr) -> Option> { + /// Single immutable-snapshot route decision for the hot path. + /// + /// Loads route state exactly once and returns a [`RouteDecision`] carrying + /// a stable peer handle. Replaces the old multi-call sequence of + /// `is_magic_dns_destination` + `is_advertised_destination` + `lookup_ip`. + pub fn route_once(&self, dst: &Ipv4Addr) -> RouteDecision { let tables = self.inner.load(); - if let Some(peer) = tables.by_ip.get(ip).cloned() { - return Some(peer); + if *dst == tables.magic_ip { + return RouteDecision::LocalMagic; } - if let Some(peer) = self.dynamic_synth.get(ip) { - return Some(peer.clone()); + if tables.advertised.get_lpm(&Ipv4Net::from(*dst)).is_some() { + return RouteDecision::LocalAdvertised; } - if let Some((prefix, peer)) = tables.subnets.get_lpm(&Ipv4Net::from(*ip)) { - // Full-tunnel exit CIDR must not steal LAN when allow_local_lan is on. - if !(tables.allow_local_lan && is_rfc1918(ip) && prefix.prefix_len() == 0) { - return Some(peer.clone()); - } + if let Some(peer) = tables.by_ip.get(dst).cloned() { + return RouteDecision::peer(peer); + } + if let Some(peer) = self.dynamic_synth.get(dst) { + return RouteDecision::peer(peer.clone()); + } + if let Some((prefix, peer)) = tables.subnets.get_lpm(&Ipv4Net::from(*dst)) + && !(tables.allow_local_lan && is_rfc1918(dst) && prefix.prefix_len() == 0) + { + return RouteDecision::peer(peer.clone()); } - // Exit node catches remaining (non-mesh, non-LAN when allowed) destinations. - if !is_mesh_or_link_local(ip) - && !(tables.allow_local_lan && is_rfc1918(ip)) + if !is_mesh_or_link_local(dst) + && !(tables.allow_local_lan && is_rfc1918(dst)) && let Some(exit) = &tables.exit_node { - return Some(exit.clone()); + return RouteDecision::peer(exit.clone()); + } + RouteDecision::NoRoute + } + + /// Direct peer IP, subnet LPM, then selected exit node for internet. + /// On birthday collisions across networks, first-joined network wins. + /// Delegates to [`Self::route_once`]; retained for non-hot-path callers. + pub fn lookup_ip(&self, ip: &Ipv4Addr) -> Option> { + match self.route_once(ip) { + RouteDecision::Peer(h) => Some(h.peer), + _ => None, } - None } pub fn exit_node(&self) -> Option> { @@ -172,18 +304,28 @@ impl RoutingTable { } pub fn is_exit_node(&self) -> bool { - // Advertised default route means we are an exit. - self.inner - .load() - .advertised - .iter() - .any(|n| n.prefix_len() == 0) + // Computed at rebuild: we advertise a default route. + self.inner.load().is_exit } pub fn lookup_endpoint(&self, hex: &str) -> Option> { self.inner.load().by_endpoint.get(hex).cloned() } + /// Exact (endpoint, network) membership lookup (§2.2-1): the only + /// correct way to resolve an inbound frame, which binds its network. + /// Never infer network identity from whichever membership was inserted + /// last — `by_endpoint` (first-joined) is for legacy single-membership + /// consumers only. Slow path (rebuild races); linear scan is fine. + pub fn lookup_membership(&self, hex: &str, network: Uuid) -> Option> { + self.inner + .load() + .by_network_ip + .values() + .find(|p| p.endpoint_hex == hex && p.network_id == network) + .cloned() + } + /// Peer hostname (mesh member), then hostname-route exact/wildcard. pub fn lookup_hostname(&self, host: &str) -> Option> { let host = host.to_ascii_lowercase(); @@ -214,8 +356,8 @@ impl RoutingTable { self.inner .load() .advertised - .iter() - .any(|net| net.contains(ip)) + .get_lpm(&Ipv4Net::from(*ip)) + .is_some() } /// True when this node is the gateway for a hostname route matching `host`. @@ -232,7 +374,12 @@ impl RoutingTable { } pub fn advertised_subnets(&self) -> Vec { - self.inner.load().advertised.clone() + self.inner + .load() + .advertised + .iter() + .map(|(p, _)| p) + .collect() } pub fn peers(&self) -> Vec> { @@ -369,7 +516,8 @@ impl RoutingTable { } /// Cache a synthetic IP → peer mapping for a hostname (wildcard routes). - /// Survives [`Self::rebuild`] because `dynamic_synth` lives outside the tables Arc. + /// Cleared on every membership rebuild (§2.2-5): the cache never owns + /// membership lifetime. DNS regenerates entries on demand. pub fn remember_dns_synth(&self, name: &str, ip: Ipv4Addr) { let tables = self.inner.load(); let suffix = format!(".{}", tables.dns_suffix); @@ -554,7 +702,7 @@ impl RoutingTable { let mut by_hostname: std::collections::HashMap> = std::collections::HashMap::new(); let mut subnets: PrefixMap> = PrefixMap::new(); - let mut advertised = Vec::new(); + let mut advertised: PrefixMap = PrefixMap::new(); let mut hostname_exact: std::collections::HashMap> = std::collections::HashMap::new(); let mut hostname_wildcards = Vec::new(); @@ -595,6 +743,15 @@ impl RoutingTable { break; } } + let fast = self.ensure_fast( + ep, + &p.endpoint_id, + &p.hostname, + ip, + &p.tags, + *network_id, + &slice.network_name, + ); let info = Arc::new(PeerInfo { endpoint: ep, endpoint_hex: p.endpoint_id.clone(), @@ -604,6 +761,7 @@ impl RoutingTable { network_id: *network_id, network_name: slice.network_name.clone(), ssh_host_key: p.ssh_host_key.clone(), + fast, }); by_network_ip.insert((*network_id, ip), info.clone()); // First-joined wins on outbound by_ip. @@ -641,10 +799,10 @@ impl RoutingTable { for route in &slice.subnet_routes { if route.via_endpoint_id == slice.self_endpoint_id { - advertised.push(route.cidr); + advertised.insert(route.cidr, ()); continue; } - let peer = peer_for_via( + let peer = self.peer_for_via( &local_by_endpoint, &route.via_endpoint_id, route.via_ip, @@ -661,7 +819,7 @@ impl RoutingTable { for exit in &slice.exit_nodes { if exit.endpoint_id == slice.self_endpoint_id { for cidr in &exit.allowed_cidrs { - advertised.push(*cidr); + advertised.insert(*cidr, ()); } } } @@ -669,7 +827,7 @@ impl RoutingTable { if let Some(exit_id) = &slice.profile.exit_node_endpoint_id && let Some(exit) = slice.exit_nodes.iter().find(|e| &e.endpoint_id == exit_id) { - let peer = peer_for_via( + let peer = self.peer_for_via( &local_by_endpoint, &exit.endpoint_id, exit.via_ip, @@ -691,7 +849,7 @@ impl RoutingTable { for route in &slice.hostname_routes { let hostname = route.hostname.to_ascii_lowercase(); - let peer = peer_for_via( + let peer = self.peer_for_via( &local_by_endpoint, &route.via_endpoint_id, route.via_ip, @@ -722,8 +880,31 @@ impl RoutingTable { hostname_wildcards.sort_by_key(|route| std::cmp::Reverse(route.hostname.len())); - // Keep dynamic_synth across rebuild - it lives outside the tables Arc so - // wildcard DNS answers survive membership/policy refreshes. + // Synthetic DNS cache must never own peer membership lifetime + // (§2.2-5): a removed peer's synthetic IP must stop routing and its + // membership must deactivate. Clear the cache on every membership + // rebuild — DNS regenerates entries on demand. + self.dynamic_synth.clear(); + + // Prune departed MEMBERSHIPS from fast states (slow path only): keep + // every (endpoint, network) reachable via direct, subnet, or exit + // routes. Same endpoint in two networks yields two live entries; + // removal of one never touches the other. + let mut live = std::collections::HashSet::new(); + // by_network_ip is the superset (same IP may lose by_ip on clash + // but stays reachable for inbound/policy context). + for peer in by_network_ip.values() { + live.insert((peer.endpoint, peer.network_id)); + } + for (_, peer) in subnets.iter() { + live.insert((peer.endpoint, peer.network_id)); + } + if let Some(exit) = &exit_node { + live.insert((exit.endpoint, exit.network_id)); + } + self.fast_registry.retain(&live); + + let is_exit = advertised.iter().any(|(p, _)| p.prefix_len() == 0); self.inner.store(Arc::new(Tables { by_ip, by_network_ip, @@ -740,9 +921,47 @@ impl RoutingTable { magic_ip, exit_node, allow_local_lan, + is_exit, version, })); } + + fn peer_for_via( + &self, + by_endpoint: &std::collections::HashMap>, + via_endpoint_id: &str, + via_ip: Ipv4Addr, + network_id: Uuid, + network_name: &str, + ) -> Option> { + if let Some(existing) = by_endpoint.get(via_endpoint_id) { + return Some(existing.clone()); + } + let Ok(ep) = via_endpoint_id.parse::() else { + tracing::warn!(id = %via_endpoint_id, "skip route with bad via endpoint id"); + return None; + }; + let fast = self.ensure_fast( + ep, + via_endpoint_id, + "", + via_ip, + &[], + network_id, + network_name, + ); + Some(Arc::new(PeerInfo { + endpoint: ep, + endpoint_hex: via_endpoint_id.to_string(), + hostname: String::new(), + ip: via_ip, + tags: Vec::new(), + network_id, + network_name: network_name.to_string(), + ssh_host_key: None, + fast, + })) + } } fn is_mesh_or_link_local(ip: &Ipv4Addr) -> bool { @@ -766,32 +985,6 @@ fn synthetic_ip_for(host: &str) -> Ipv4Addr { Ipv4Addr::new(100, 100, hi, low) } -fn peer_for_via( - by_endpoint: &std::collections::HashMap>, - via_endpoint_id: &str, - via_ip: Ipv4Addr, - network_id: Uuid, - network_name: &str, -) -> Option> { - if let Some(existing) = by_endpoint.get(via_endpoint_id) { - return Some(existing.clone()); - } - let Ok(ep) = via_endpoint_id.parse::() else { - tracing::warn!(id = %via_endpoint_id, "skip route with bad via endpoint id"); - return None; - }; - Some(Arc::new(PeerInfo { - endpoint: ep, - endpoint_hex: via_endpoint_id.to_string(), - hostname: String::new(), - ip: via_ip, - tags: Vec::new(), - network_id, - network_name: network_name.to_string(), - ssh_host_key: None, - })) -} - fn hostname_matches_wildcard(host: &str, suffix: &str) -> bool { host == suffix || host @@ -950,6 +1143,133 @@ mod tests { assert!(table.lookup_hostname_route("other.com").is_none()); } + #[test] + fn removed_peer_is_unroutable_and_deactivated() { + // §2.1-9: peer connected → membership removed → routing generation + // changes → outbound packets rejected (NoRoute), registry resolve + // fails (inbound readers must exit), old fast state deactivated. + let table = RoutingTable::new(); + let self_id = "a".repeat(64); + let peer_id = "b".repeat(64); + let net = Uuid::nil(); + table.replace( + &[peer(&peer_id, "10.7.0.5", "gw")], + &[], + &[], + &[], + &profile(), + &dns(), + "office", + net, + &self_id, + 1, + ); + let ip: Ipv4Addr = "10.7.0.5".parse().unwrap(); + let info = table.lookup_ip(&ip).expect("peer connected"); + assert_eq!(info.endpoint_hex, peer_id); + let fast = info.fast.clone(); + let epoch0 = fast.epoch.load(std::sync::atomic::Ordering::Relaxed); + let v0 = table.version(); + // Membership removal (delta path, as live sync would send it). + table.apply_peer_delta( + net, + &[], + std::slice::from_ref(&peer_id), + 2, + &self_id, + "office", + ); + assert!(table.version() > v0, "routing generation must change"); + assert!( + matches!(table.route_once(&ip), RouteDecision::NoRoute), + "next outbound packet is rejected" + ); + let ep: EndpointId = peer_id.parse().unwrap(); + assert!( + table.peer_registry().get(ep).is_none(), + "inbound resolve finds nothing → reader exits" + ); + assert_eq!( + fast.epoch.load(std::sync::atomic::Ordering::Relaxed), + epoch0 + 1, + "old fast state hard-revoked" + ); + } + + #[test] + fn same_endpoint_two_networks_route_to_distinct_memberships() { + // §2.2-1 (tests 3, 4): one endpoint in networks A and B with + // different mesh IPs. Outbound routes resolve each network's OWN + // membership state — never a shared endpoint-global context. + let table = RoutingTable::new(); + let self_id = "a".repeat(64); + let ep_hex = "b".repeat(64); + let net_a = Uuid::from_u128(0x0a); + let net_b = Uuid::from_u128(0x0b); + table.replace_network( + net_a, + 0, + &[peer(&ep_hex, "10.7.0.5", "gw")], + &dns(), + "neta", + &self_id, + 1, + ); + table.replace_network( + net_b, + 1, + &[peer(&ep_hex, "10.7.1.5", "gw")], + &dns(), + "netb", + &self_id, + 2, + ); + let a = table + .lookup_ip(&"10.7.0.5".parse().unwrap()) + .expect("route A"); + let b = table + .lookup_ip(&"10.7.1.5".parse().unwrap()) + .expect("route B"); + assert_eq!(a.network_id, net_a); + assert_eq!(b.network_id, net_b); + assert!(!Arc::ptr_eq(&a.fast, &b.fast), "distinct membership states"); + assert_eq!( + a.fast.identity.read().ip, + std::net::Ipv4Addr::new(10, 7, 0, 5) + ); + assert_eq!( + b.fast.identity.read().ip, + std::net::Ipv4Addr::new(10, 7, 1, 5) + ); + // Exact registry + route resolution per (endpoint, network). + let ep: EndpointId = ep_hex.parse().unwrap(); + assert!( + table + .peer_registry() + .get_membership(ep, net_a) + .is_some_and(|m| Arc::ptr_eq(&m, &a.fast)) + ); + assert!( + table + .peer_registry() + .get_membership(ep, net_b) + .is_some_and(|m| Arc::ptr_eq(&m, &b.fast)) + ); + assert_eq!( + table.lookup_membership(&ep_hex, net_a).unwrap().network_id, + net_a + ); + assert_eq!( + table.lookup_membership(&ep_hex, net_b).unwrap().network_id, + net_b + ); + assert!( + table + .lookup_membership(&ep_hex, Uuid::from_u128(0xcc)) + .is_none() + ); + } + #[test] fn peer_dns_resolves_self() { let table = RoutingTable::new(); @@ -1063,8 +1383,7 @@ mod tests { assert!(table.lookup_endpoint(&peer_b).is_some()); } - #[test] - fn dynamic_synth_survives_rebuild() { + fn wildcard_table() -> (RoutingTable, String, String, Uuid) { let table = RoutingTable::new(); let self_id = "a".repeat(64); let gw = "b".repeat(64); @@ -1087,11 +1406,18 @@ mod tests { &self_id, 1, ); + (table, self_id, gw, nid) + } + + #[test] + fn dynamic_synth_cleared_on_rebuild() { + // §2.2-5: the synthetic DNS cache never owns membership lifetime. + // Any membership rebuild clears it; DNS regenerates on demand. + let (table, self_id, gw, nid) = wildcard_table(); let synth = table.resolve_dns_a("api.internal.tunnet").unwrap(); table.remember_dns_synth("api.internal.tunnet", synth); assert_eq!(table.lookup_ip(&synth).unwrap().endpoint_hex, gw); - - // Rebuild via peer delta must keep dynamic_synth. + // Unrelated rebuild clears the cache: synth stops routing... table.apply_peer_delta( nid, &[peer(&"d".repeat(64), "10.7.0.7", "dave")], @@ -1100,7 +1426,51 @@ mod tests { &self_id, "office", ); - assert_eq!(table.lookup_ip(&synth).unwrap().endpoint_hex, gw); + assert!( + matches!(table.route_once(&synth), RouteDecision::NoRoute), + "stale synthetic route must not survive rebuild" + ); + // ...but DNS regenerates it immediately (route still exists). + let synth2 = table.resolve_dns_a("api.internal.tunnet").unwrap(); + assert_eq!(synth2, synth); + table.remember_dns_synth("api.internal.tunnet", synth2); + assert_eq!(table.lookup_ip(&synth2).unwrap().endpoint_hex, gw); + } + + #[test] + fn synth_peer_removal_revokes_and_deactivates() { + // §2.2-5: removing the peer behind a synthetic IP stops the route + // AND deactivates its membership (no stale Arc keeps forwarding). + use std::sync::atomic::Ordering; + let (table, self_id, gw, nid) = wildcard_table(); + let synth = table.resolve_dns_a("api.internal.tunnet").unwrap(); + table.remember_dns_synth("api.internal.tunnet", synth); + let ep: EndpointId = gw.parse().unwrap(); + let fast = table + .peer_registry() + .get_membership(ep, nid) + .expect("membership ensured at rebuild"); + let epoch0 = fast.epoch.load(Ordering::Relaxed); + table.apply_peer_delta(nid, &[], std::slice::from_ref(&gw), 2, &self_id, "office"); + assert!(matches!(table.route_once(&synth), RouteDecision::NoRoute)); + assert!(table.peer_registry().get_membership(ep, nid).is_none()); + assert_eq!( + fast.epoch.load(Ordering::Relaxed), + epoch0 + 1, + "removed peer's membership must deactivate" + ); + } + + #[test] + fn synth_network_removal_revokes() { + // Removing the whole network clears synthetic routes too. + let (table, _self_id, _gw, nid) = wildcard_table(); + let synth = table.resolve_dns_a("api.internal.tunnet").unwrap(); + table.remember_dns_synth("api.internal.tunnet", synth); + assert!(table.lookup_ip(&synth).is_some()); + table.remove_network(nid); + assert!(matches!(table.route_once(&synth), RouteDecision::NoRoute)); + assert!(table.resolve_dns_a("api.internal.tunnet").is_none()); } #[test] diff --git a/crates/tunnet-core/src/scheduler.rs b/crates/tunnet-core/src/scheduler.rs new file mode 100644 index 00000000..f2395d8e --- /dev/null +++ b/crates/tunnet-core/src/scheduler.rs @@ -0,0 +1,1392 @@ +//! Per-peer FQ-CoDel packet scheduler state (RFC 8290, Tunnet-sized). +//! +//! Pure state machine: no I/O, no transport calls, no metrics registry. The +//! agent pump drives it (`next` → transmit or drop) and reports counters. +//! +//! ```text +//! PeerScheduler +//! ├─ new flows (sparse/interactive, bounded epoch budget) +//! ├─ old flows (backlogged, byte-DRR across flows) +//! ├─ per-flow FIFO (ordering preserved within a flow) +//! ├─ per-flow CoDel state (first_above_time/dropping/drop_next/count) +//! └─ byte caps + emergency sojourn ceiling (safety bound only) +//! ``` +//! +//! Complexity: dequeue performs rotation rounds over old flows (one +//! quantum per flow per round); rounds repeat immediately only while no +//! flow could send, bounded by [`MAX_DRR_ROUNDS`]. No linear scans for +//! drops, no per-packet allocation on the dequeue path. +//! +//! The scheduler queues LOGICAL packets (§7): one inner packet is one +//! scheduling object. Segmentation happens after dequeue; the pump reports +//! each logical packet ONCE at completion via +//! [`PeerScheduler::account_sent`] with `(logical_len, total_wire_len)` so +//! fairness reflects transmitted bytes including framing overhead (and +//! segmented traffic is never double-charged). +//! +//! `Empty` means genuinely no schedulable work: rounds repeat immediately +//! inside the call, so a flow whose head exceeds one quantum is served +//! after enough rounds — never deferred to a 50 ms pump sleep. Service is +//! proper byte-DRR: each visit grants one quantum and serves every +//! affordable head as a burst, so a 9000-byte flow and a 100-byte flow +//! receive equal byte shares over time, not equal packet counts. + +use std::collections::{HashMap, VecDeque}; +use std::time::{Duration, Instant}; + +use tunnet_common::packet::{FlowKey, LogicalPacket}; + +/// CoDel target: minimum sojourn indicating a standing queue (~5 ms baseline; +/// consider serialization time on slow links per RFC 8290 §4.2). +pub const CODEL_TARGET: Duration = Duration::from_millis(5); +/// CoDel interval: standing-queue observation window. +pub const CODEL_INTERVAL: Duration = Duration::from_millis(100); +/// Emergency maximum queue lifetime: hard safety bound only, not the AQM. +pub const EMERGENCY_CEILING: Duration = Duration::from_millis(1000); +/// Total queued bytes per peer (queueing budget shared with transport). +pub const PEER_BYTE_CAP: usize = 256 * 1024; +/// Hard packet cap per peer (memory bound for tiny packets). +pub const PEER_PACKET_CAP: usize = 512; +/// Per-flow packet cap default (diagnostic override via +/// `TUNNET_FLOW_PACKET_CAP`, e.g. 64 vs 256 A/B runs). One flow cannot +/// dominate peer memory. +pub const FLOW_PACKET_CAP: usize = 64; + +/// Default per-flow packet cap, honoring the diagnostic override. +fn flow_packet_cap_default() -> usize { + std::env::var("TUNNET_FLOW_PACKET_CAP") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 8) + .unwrap_or(FLOW_PACKET_CAP) +} +/// New flows stay "sparse" until they send this many bytes. +pub const NEW_FLOW_BYTE_BUDGET: usize = 16 * 1024; +/// Sparse flow sojourn bar: heads older than this are not "interactive". +pub const SPARSE_SOJOURN_BAR: Duration = Duration::from_millis(25); +/// Cap-pressure probe bound (flows inspected per enqueue tdrops). +pub const CAP_PROBE_BOUND: usize = 4; +/// Upper bound on immediate DRR rounds inside one `next()` call (§2.2-3). +/// One quantum (≥512 B) per round per flow; worst-case deficit gap is one +/// max head plus accumulated wire overshoot (~19 KB ⇒ ≤37 rounds). 64 is a +/// safe deterministic margin; typical calls send on the first round. +pub const MAX_DRR_ROUNDS: u8 = 64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DropReason { + PeerByteCap, + PeerPacketCap, + FlowCap, + Codel, + EmergencyCeiling, + TooLarge, + NoConnection, +} + +impl DropReason { + pub fn as_str(self) -> &'static str { + match self { + Self::PeerByteCap => "sched_peer_bytes", + Self::PeerPacketCap => "sched_peer_packets", + Self::FlowCap => "sched_flow_cap", + Self::Codel => "sched_codel", + Self::EmergencyCeiling => "sched_emergency", + Self::TooLarge => "datagram_too_large", + Self::NoConnection => "no_connection", + } + } +} + +/// Per-flow CoDel state (RFC 8290 §5.2, adapted to logical packets). +#[derive(Debug, Clone, Copy)] +struct CodelState { + /// When the current above-target episode started (None = below target). + first_above_time: Option, + /// In dropping state (sustained standing queue). + dropping: bool, + /// Next scheduled drop time while dropping. + drop_next: Instant, + /// Drops in the current episode (controls drop frequency). + count: u32, +} + +impl CodelState { + fn new() -> Self { + Self { + first_above_time: None, + dropping: false, + drop_next: Instant::now(), + count: 0, + } + } +} + +struct QueuedPacket { + packet: LogicalPacket, + len: usize, +} + +struct FlowQueue { + packets: VecDeque, + bytes: usize, + deficit: isize, + epoch_bytes: usize, + is_new: bool, + codel: CodelState, +} + +impl FlowQueue { + fn new(now: Instant) -> Self { + let _ = now; + Self { + packets: VecDeque::new(), + bytes: 0, + deficit: 0, + epoch_bytes: 0, + is_new: true, + codel: CodelState::new(), + } + } +} + +/// Scheduler counters reported to telemetry (deltas applied by the pump). +#[derive(Debug, Default, Clone, Copy)] +pub struct SchedCounters { + pub enqueued: u64, + pub sent_packets: u64, + pub sent_bytes: u64, + pub wire_bytes: u64, + pub drops_codel: u64, + pub drops_cap: u64, + pub drops_emergency: u64, + pub transport_full: u64, +} + +/// Sojourn observation for histogram telemetry (filled by dequeue). +#[derive(Debug, Clone, Copy)] +pub struct SojournSample { + pub sojourn: Duration, +} + +/// One DRR service opportunity: every queued packet the visited flow could +/// afford with its deficit (§2.2-3). Non-empty by construction. Classic DRR +/// serves a flow's eligible packets together — this is what makes byte +/// shares fair when packet sizes differ wildly (a 9000 B flow and a 100 B +/// flow each receive ~one quantum of service per visit, not one packet). +pub struct DequeueBurst { + pub packets: Vec<(Box, SojournSample)>, +} + +/// Dequeue decision returned to the pump. +pub enum Dequeue { + /// Transmit this burst (all packets, in order). Boxed: a burst is + /// heap-sized and Empty is unit-sized. + Send(Box), + /// Scheduler empty. + Empty, +} + +/// Per-peer FQ-CoDel scheduler. Not thread-safe; owned by the peer's pump +/// (either behind the fast-state lock or by a single pump task). +pub struct PeerScheduler { + flows: HashMap, + /// New (sparse) flows first, oldest-first. + new_list: VecDeque, + /// Backlogged flows in DRR order. + old_list: VecDeque, + bytes: usize, + packets: usize, + quantum: usize, + /// Per-flow packet cap (diagnostic override via TUNNET_FLOW_PACKET_CAP). + flow_packet_cap: usize, + target: Duration, + interval: Duration, + counters: SchedCounters, + /// Baselines for [`Self::drain_drops`]: cumulative counters already + /// reported to telemetry (codel, emergency). Split across however many + /// threads drain — the sum stays exact, never double-counted. + reported_codel: u64, + reported_emergency: u64, +} + +/// Enqueue decision. EVERY shed or evicted packet is reported here — there +/// are no invisible drops: the caller reconciles gauges and telemetry for +/// both the admitted packet and any evicted one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnqueueOutcome { + /// Admitted (gauges: +1 packet, +len bytes). + Accepted, + /// Admitted, but an older packet was evicted to make room (gauges: +1 + /// packet/+len for the newcomer AND -1/-evicted_len for the victim; + /// report `reason` to telemetry). + AcceptedEvicted { + reason: DropReason, + evicted_len: usize, + }, + /// Shed (report `reason` to telemetry; gauges untouched). + Rejected { reason: DropReason }, +} + +impl EnqueueOutcome { + pub fn is_accepted(self) -> bool { + matches!(self, Self::Accepted | Self::AcceptedEvicted { .. }) + } +} + +/// CoDel/emergency drops since the last drain (see [`Self::drain_drops`]). +/// Cap-pressure sheds are reported at the enqueue decision site instead +/// (see [`EnqueueOutcome`]), so the two paths never double-count. +#[derive(Debug, Default, Clone, Copy)] +pub struct SchedDropDeltas { + pub codel: u64, + pub emergency: u64, +} + +impl SchedDropDeltas { + pub fn is_empty(self) -> bool { + self.codel == 0 && self.emergency == 0 + } +} + +impl PeerScheduler { + pub fn new(quantum: usize) -> Self { + Self::with_params(quantum, CODEL_TARGET, CODEL_INTERVAL) + } + + /// Custom CoDel timing (tests, link-specific tuning). Production uses + /// [`CODEL_TARGET`]/[`CODEL_INTERVAL`]. + pub fn with_params(quantum: usize, target: Duration, interval: Duration) -> Self { + Self { + flows: HashMap::new(), + new_list: VecDeque::new(), + old_list: VecDeque::new(), + bytes: 0, + packets: 0, + quantum: quantum.max(512), + flow_packet_cap: flow_packet_cap_default(), + target, + interval: interval.max(Duration::from_millis(1)), + counters: SchedCounters::default(), + reported_codel: 0, + reported_emergency: 0, + } + } + + pub fn set_quantum(&mut self, quantum: usize) { + self.quantum = quantum.max(512); + } + + /// Override the per-flow packet cap (diagnostic A/B only). + pub fn set_flow_packet_cap(&mut self, cap: usize) { + self.flow_packet_cap = cap.max(8); + } + + pub fn levels(&self) -> (u64, u64, u64) { + ( + self.packets as u64, + self.bytes as u64, + self.flows.len() as u64, + ) + } + + pub fn counters(&self) -> SchedCounters { + self.counters + } + + /// Take unreported CoDel/emergency drops since the last drain (for + /// telemetry; call after `next()` batches and after enqueues). Safe to + /// call from multiple threads sharing the scheduler lock: deltas are + /// partitioned, the sum stays exact. + pub fn drain_drops(&mut self) -> SchedDropDeltas { + let out = SchedDropDeltas { + codel: self.counters.drops_codel - self.reported_codel, + emergency: self.counters.drops_emergency - self.reported_emergency, + }; + self.reported_codel = self.counters.drops_codel; + self.reported_emergency = self.counters.drops_emergency; + out + } + + pub fn is_empty(&self) -> bool { + self.packets == 0 + } + + /// Drop all queued packets (teardown ownership change). Returns the + /// dropped (packets, bytes, flows) for gauge reconciliation. + pub fn clear(&mut self) -> (u64, u64, u64) { + let out = ( + self.packets as u64, + self.bytes as u64, + self.flows.len() as u64, + ); + self.flows.clear(); + self.new_list.clear(); + self.old_list.clear(); + self.bytes = 0; + self.packets = 0; + out + } + + /// Enqueue a logical packet. The outcome reports EVERYTHING: admission, + /// eviction (with the victim's length for gauge reconciliation), or + /// rejection with its reason. Callers must reconcile gauges and report + /// telemetry for evictions/rejections — no invisible drops. + /// `now` should be the packet's observation time (usually Instant::now()). + pub fn enqueue(&mut self, packet: LogicalPacket, now: Instant) -> EnqueueOutcome { + let flow = packet.flow; + let len = packet.len(); + // Memory bounds first: probe a bounded number of flows for an + // over-ceiling head to evict; otherwise shed the newcomer (tail drop + // keeps the work O(1) instead of scanning all flows). + if self.packets >= PEER_PACKET_CAP || self.bytes + len > PEER_BYTE_CAP { + if !self.evict_one(now) && self.packets >= PEER_PACKET_CAP { + self.counters.drops_cap += 1; + return EnqueueOutcome::Rejected { + reason: DropReason::PeerPacketCap, + }; + } + if self.bytes + len > PEER_BYTE_CAP { + self.counters.drops_cap += 1; + return EnqueueOutcome::Rejected { + reason: if self.packets >= PEER_PACKET_CAP { + DropReason::PeerPacketCap + } else { + DropReason::PeerByteCap + }, + }; + } + } + let is_new_flow = !self.flows.contains_key(&flow); + let flow_cap = self.flow_packet_cap; + let q = self + .flows + .entry(flow) + .or_insert_with(|| FlowQueue::new(now)); + if q.packets.len() >= flow_cap { + // Per-flow cap: drop the flow's own stalest head (tail stays + // fresh: retransmits and sparse signals survive). REPORTED via + // the outcome (with victim length) — never silent. + if let Some(old) = q.packets.pop_front() { + let evicted_len = old.len; + q.bytes -= evicted_len; + self.bytes -= evicted_len; + self.packets -= 1; + self.counters.drops_cap += 1; + q.bytes += len; + q.packets.push_back(QueuedPacket { packet, len }); + self.bytes += len; + self.packets += 1; + self.counters.enqueued += 1; + if is_new_flow && !self.new_list.contains(&flow) && !self.old_list.contains(&flow) { + self.new_list.push_back(flow); + } + return EnqueueOutcome::AcceptedEvicted { + reason: DropReason::FlowCap, + evicted_len, + }; + } + self.counters.drops_cap += 1; + return EnqueueOutcome::Rejected { + reason: DropReason::FlowCap, + }; + } + q.bytes += len; + q.packets.push_back(QueuedPacket { packet, len }); + self.bytes += len; + self.packets += 1; + self.counters.enqueued += 1; + if is_new_flow && !self.new_list.contains(&flow) && !self.old_list.contains(&flow) { + self.new_list.push_back(flow); + } + EnqueueOutcome::Accepted + } + + /// Bounded cap-pressure eviction: inspect at most CAP_PROBE_BOUND flows + /// (round-robin from the old list, then new list) for an emergency head. + /// Returns true when something was evicted. + fn evict_one(&mut self, now: Instant) -> bool { + for _ in 0..CAP_PROBE_BOUND { + let key = if let Some(k) = self.old_list.pop_front() { + k + } else if let Some(k) = self.new_list.pop_front() { + k + } else { + return false; + }; + let evicted = match self.flows.get_mut(&key) { + Some(q) => match q.packets.front() { + Some(h) + if now.saturating_duration_since(h.packet.enqueued_at) + > EMERGENCY_CEILING => + { + let old = q.packets.pop_front().expect("head"); + q.bytes -= old.len; + self.bytes -= old.len; + self.packets -= 1; + self.counters.drops_emergency += 1; + true + } + Some(_) => { + // Not evictable: rotate to the back and keep probing. + if q.is_new { + self.new_list.push_back(key); + } else { + self.old_list.push_back(key); + } + false + } + None => false, + }, + None => false, + }; + if evicted { + // Keep a non-empty flow scheduled. + if let Some(q) = self.flows.get(&key) { + if !q.packets.is_empty() { + if q.is_new { + self.new_list.push_front(key); + } else { + self.old_list.push_front(key); + } + } else { + self.flows.remove(&key); + } + } + return true; + } + } + false + } + + /// Dequeue the next service opportunity: sparse flows first (one + /// packet: bounded epoch budget, young head), else proper byte-DRR + /// across old flows with per-flow CoDel standing-queue control. + /// + /// DRR discipline (§2.2-3): each visit grants the flow exactly ONE + /// quantum; the flow serves every head packet it can afford (burst); + /// unaffordable heads rotate for a later round. Rounds repeat + /// immediately — no `Empty`, no sleep — until some flow sends, drops, + /// or retires. `Empty` is returned ONLY when no schedulable work + /// remains. Each round strictly grows deficits or shrinks the queue, + /// and rounds are hard-bounded, so the loop terminates. + pub fn next(&mut self, now: Instant) -> Dequeue { + // 1) Sparse/new flows: single packet (interactive latency first). + // Drain Gone heads; one Demote breaks to DRR. + while let Some(key) = self.new_list.pop_front() { + match self.serve_sparse(key, now) { + SparseOut::Send(packet, sample) => { + return Dequeue::Send(Box::new(DequeueBurst { + packets: vec![(packet, sample)], + })); + } + SparseOut::Gone => continue, + SparseOut::Demoted => break, + } + } + // 2) Byte-DRR across old flows with CoDel. List rotation lives + // here: serve_old only signals, never pushes (one owner, no dupes, + // no zombie keys for emptied flows). + for _ in 0..MAX_DRR_ROUNDS { + let n = self.old_list.len(); + if n == 0 { + return Dequeue::Empty; + } + let mut burst: Option = None; + for _ in 0..n { + let Some(key) = self.old_list.pop_front() else { + break; + }; + match self.serve_old(key, now) { + OldOut::Send(b) => { + if self.flows.get(&key).is_some_and(|q| !q.packets.is_empty()) { + self.old_list.push_back(key); + } + burst = Some(b); + break; + } + OldOut::Rotate => { + if self.flows.get(&key).is_some_and(|q| !q.packets.is_empty()) { + self.old_list.push_back(key); + } + } + OldOut::Gone => {} + } + } + if let Some(b) = burst { + return Dequeue::Send(Box::new(b)); + } + // No send this round: every visit either dropped a packet + // (CoDel/emergency, strictly reducing queued packets) or + // retired a flow/stale key (strictly shrinking the list) or + // rotated an unaffordable head (deficit grew by one quantum). + // Loop for another immediate round — never Empty with work. + } + // Unreachable safety: the deficit gap is bounded (~19 KB worst + // case ⇒ ≤37 rounds at minimum quantum), so 64 rounds always + // serve or drain. Never hang the pump. + debug_assert!(false, "scheduler DRR round bound exhausted"); + Dequeue::Empty + } + + /// Requeue a packet at its flow head (transport-full: retry later without + /// losing order). Restores both flow and global accounting so a + /// dequeue→requeue cycle is a no-op on the books. Counts a + /// transport-full event for backoff telemetry. + pub fn requeue_head(&mut self, flow: FlowKey, packet: LogicalPacket) { + let len = packet.len(); + match self.flows.get_mut(&flow) { + Some(q) => { + q.bytes += len; + q.packets.push_front(QueuedPacket { packet, len }); + } + None => { + let mut q = FlowQueue::new(Instant::now()); + q.bytes = len; + q.packets.push_front(QueuedPacket { packet, len }); + self.flows.insert(flow, q); + self.old_list.push_front(flow); + } + } + self.bytes += len; + self.packets += 1; + self.counters.transport_full += 1; + } + + /// Account one COMPLETED logical packet (logical + total wire bytes). + /// Called exactly once per logical packet when transmission finishes — + /// never per segment — so segmented traffic is charged once: the + /// dequeue already debited the logical length from DRR deficit, and + /// only the wire overhead beyond it leans future rounds here. + pub fn account_sent(&mut self, flow: FlowKey, logical_len: usize, wire_len: usize) { + self.counters.sent_packets += 1; + self.counters.sent_bytes += logical_len as u64; + self.counters.wire_bytes += wire_len as u64; + if let Some(q) = self.flows.get_mut(&flow) { + // Extra wire overhead beyond the DRR deficit debit leans future + // rounds slightly against overhead-heavy flows. + let overhead = wire_len.saturating_sub(logical_len); + q.deficit -= overhead as isize; + } + } + + fn remove_flow(&mut self, key: &FlowKey) { + self.flows.remove(key); + self.new_list.retain(|k| k != key); + self.old_list.retain(|k| k != key); + } + + fn serve_sparse(&mut self, key: FlowKey, now: Instant) -> SparseOut { + enum Prep { + Send, + Demote, + Gone, + } + let prep = { + let Some(q) = self.flows.get_mut(&key) else { + return SparseOut::Gone; + }; + // Emergency ceiling applies everywhere (safety bound only). + while let Some(h) = q.packets.front() { + if now.saturating_duration_since(h.packet.enqueued_at) > EMERGENCY_CEILING { + let old = q.packets.pop_front().expect("head"); + q.bytes -= old.len; + self.bytes -= old.len; + self.packets -= 1; + self.counters.drops_emergency += 1; + } else { + break; + } + } + if q.packets.is_empty() { + Prep::Gone + } else { + let young = q.packets.front().is_some_and(|h| { + now.saturating_duration_since(h.packet.enqueued_at) <= SPARSE_SOJOURN_BAR + }); + if !q.is_new || q.epoch_bytes >= NEW_FLOW_BYTE_BUDGET || !young { + q.is_new = false; + Prep::Demote + } else { + Prep::Send + } + } + }; + match prep { + Prep::Gone => { + self.remove_flow(&key); + SparseOut::Gone + } + Prep::Demote => { + self.old_list.push_back(key); + SparseOut::Demoted + } + Prep::Send => { + let q = self.flows.get_mut(&key).expect("present"); + let qp = q.packets.pop_front().expect("head"); + let sample = SojournSample { + sojourn: now.saturating_duration_since(qp.packet.enqueued_at), + }; + q.bytes -= qp.len; + self.bytes -= qp.len; + self.packets -= 1; + q.epoch_bytes += qp.len; + q.deficit += self.quantum as isize; + q.deficit -= qp.len as isize; + if q.packets.is_empty() { + self.remove_flow(&key); + } else if q.epoch_bytes >= NEW_FLOW_BYTE_BUDGET { + q.is_new = false; + self.old_list.push_back(key); + } else { + self.new_list.push_front(key); + } + SparseOut::Send(Box::new(qp.packet), sample) + } + } + } + + fn serve_old(&mut self, key: FlowKey, now: Instant) -> OldOut { + // Emergency ceiling + CoDel observe the head first. + enum Head { + Ready, + Dropped, + Gone, + } + let head = { + let Some(q) = self.flows.get_mut(&key) else { + return OldOut::Gone; + }; + // Emergency safety bound. + while let Some(h) = q.packets.front() { + if now.saturating_duration_since(h.packet.enqueued_at) > EMERGENCY_CEILING { + let old = q.packets.pop_front().expect("head"); + q.bytes -= old.len; + self.bytes -= old.len; + self.packets -= 1; + self.counters.drops_emergency += 1; + } else { + break; + } + } + let sojourn = match q.packets.front() { + Some(h) => now.saturating_duration_since(h.packet.enqueued_at), + None => { + self.remove_flow(&key); + return OldOut::Gone; + } + }; + // CoDel control law (RFC 8290 §5.2) on head sojourn. + let target = self.target; + let interval = self.interval; + let c = &mut q.codel; + if sojourn < target { + c.first_above_time = None; + Head::Ready + } else if c.first_above_time.is_none() { + c.first_above_time = Some(now + interval); + Head::Ready + } else if now < c.first_above_time.expect("set") { + Head::Ready + } else { + // Standing queue: enter/continue dropping state. + if !c.dropping { + c.dropping = true; + // First drop is immediate on entering dropping state. + c.drop_next = now; + c.count = 0; + } + if now < c.drop_next { + Head::Ready + } else { + c.count += 1; + // Next drop scheduled per control law: interval/sqrt(count). + let div = (c.count as f64).sqrt().max(1.0); + let step = interval.div_f64(div); + c.drop_next = now + step; + // Drop the head now. + let old = q.packets.pop_front().expect("head"); + q.bytes -= old.len; + self.bytes -= old.len; + self.packets -= 1; + self.counters.drops_codel += 1; + // Re-observe the new head next visit. + if q.packets.is_empty() { + Head::Gone + } else { + Head::Dropped + } + } + } + }; + match head { + Head::Gone => { + self.remove_flow(&key); + OldOut::Gone + } + Head::Dropped => { + // Dropped above; the caller requeues this flow for its next + // packet (list ownership lives in `next`). An emptied flow + // is removed outright so no zombie key lingers. + if self.flows.get(&key).is_some_and(|q| q.packets.is_empty()) { + self.remove_flow(&key); + OldOut::Gone + } else { + OldOut::Rotate + } + } + Head::Ready => { + let q = self.flows.get_mut(&key).expect("present"); + // Proper DRR (§2.2-3): exactly ONE quantum per visit. Serve + // every head the flow can afford as one burst; an + // unaffordable head rotates for a later round (the caller's + // round loop retries immediately — no Empty, no sleep). + // Burst bytes are naturally bounded: at most one quantum + // plus one max head per visit. + q.deficit += self.quantum as isize; + let mut burst = DequeueBurst { + packets: Vec::new(), + }; + while let Some(h) = q.packets.front() { + let sojourn = now.saturating_duration_since(h.packet.enqueued_at); + let head_len = h.len; + // Emergency ceiling applies per packet inside bursts too. + if sojourn > EMERGENCY_CEILING { + let old = q.packets.pop_front().expect("head"); + q.bytes -= old.len; + self.bytes -= old.len; + self.packets -= 1; + self.counters.drops_emergency += 1; + continue; + } + if (head_len as isize) > q.deficit { + break; + } + let qp = q.packets.pop_front().expect("head"); + let sample = SojournSample { sojourn }; + q.bytes -= qp.len; + self.bytes -= qp.len; + self.packets -= 1; + q.deficit -= qp.len as isize; + q.epoch_bytes += qp.len; + burst.packets.push((Box::new(qp.packet), sample)); + } + // Leaving dropping state: sojourn fell below target on a + // previous observation (first_above_time cleared above). + if q.codel.first_above_time.is_none() { + q.codel.dropping = false; + q.codel.count = 0; + } + if q.packets.is_empty() { + self.remove_flow(&key); + } + // List rotation belongs to the caller (`next` requeues on + // Send/Rotate); serve_old never pushes. + if burst.packets.is_empty() { + OldOut::Rotate + } else { + OldOut::Send(burst) + } + } + } + } +} + +enum SparseOut { + Send(Box, SojournSample), + Gone, + Demoted, +} + +enum OldOut { + Send(DequeueBurst), + Rotate, + Gone, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use tunnet_common::packet::PacketPool; + + fn pool() -> Arc { + PacketPool::new(64) + } + + fn logical(pool: &Arc, sport: u16, size: usize) -> LogicalPacket { + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64).udp(sport, 443); + let mut raw = Vec::new(); + b.write(&mut raw, &vec![0u8; size]).unwrap(); + let mut buf = pool.acquire(raw.len()); + buf.recv_region(raw.len()).copy_from_slice(&raw); + LogicalPacket::from_pooled(buf, raw.len()).unwrap() + } + + fn drain_all(s: &mut PeerScheduler) -> Vec<(FlowKey, usize)> { + let mut out = Vec::new(); + let mut guard = 4096; + while guard > 0 { + guard -= 1; + match s.next(Instant::now()) { + Dequeue::Send(burst) => { + for (p, _) in burst.packets { + let (f, l) = (p.flow, p.len()); + s.account_sent(f, l, l); + out.push((f, l)); + } + } + Dequeue::Empty => break, + } + } + out + } + + /// Demote every flow to the old/DRR list, as the pump would. + fn demote_all(s: &mut PeerScheduler) { + for k in s.flows.keys().cloned().collect::>() { + let q = s.flows.get_mut(&k).unwrap(); + q.is_new = false; + q.epoch_bytes = NEW_FLOW_BYTE_BUDGET; + } + s.new_list.clear(); + for k in s.flows.keys().cloned().collect::>() { + s.old_list.push_back(k); + } + } + + #[test] + fn large_heads_serve_without_stall() { + // §2.2-3: logical packets far larger than the DRR quantum (2800 and + // 9000 byte packets against ~1200-byte MPS-scaled quanta) must be + // served via immediate internal rounds — Empty means empty, never + // "needs more rounds", and never a 50 ms pump sleep with work. + for (size, quantum) in [(2800usize, 1200usize), (9000, 1400), (9000, 512)] { + let p = pool(); + let mut s = PeerScheduler::new(quantum); + assert!( + s.enqueue(logical(&p, 1111, size - 28), Instant::now()) + .is_accepted() + ); + demote_all(&mut s); + let mut empties = 0u32; + let mut sent = 0u32; + while !s.is_empty() { + match s.next(Instant::now()) { + Dequeue::Send(burst) => { + for (pkt, _) in burst.packets { + let (f, l) = (pkt.flow, pkt.len()); + assert_eq!(l, logical(&p, 1111, size - 28).len()); + s.account_sent(f, l, l + 12); + sent += 1; + } + } + Dequeue::Empty => empties += 1, + } + assert!(sent + empties < 10, "stall: Empty with work queued"); + } + assert_eq!(sent, 1); + assert_eq!( + empties, 0, + "size={size} quantum={quantum}: no Empty while queued" + ); + } + } + + #[test] + fn wire_accounted_once_per_logical() { + // §2.1-2: one logical packet => exactly one account_sent at + // completion with (logical, total_wire); segmented traffic must not + // be double-charged through per-segment calls. + let p = pool(); + let mut s = PeerScheduler::new(1200); + let want = logical(&p, 1111, 2800 - 28); + let (flow, len) = (want.flow, want.len()); + assert!(s.enqueue(want, Instant::now()).is_accepted()); + let (f, l) = match s.next(Instant::now()) { + Dequeue::Send(burst) => { + assert_eq!(burst.packets.len(), 1); + let (pkt, _) = &burst.packets[0]; + (pkt.flow, pkt.len()) + } + Dequeue::Empty => panic!("expected packet"), + }; + assert_eq!((f, l), (flow, len)); + // Pump transmits 3 segments then accounts once with total wire. + let total_wire = l + 3 * 11; + s.account_sent(f, l, total_wire); + let c = s.counters(); + assert_eq!(c.sent_packets, 1); + assert_eq!(c.sent_bytes, len as u64); + assert_eq!(c.wire_bytes, total_wire as u64); + } + + /// Queued length of one flow (by sport) for backlog maintenance. + fn qlen(s: &PeerScheduler, sport: u16) -> usize { + s.flows + .iter() + .filter(|(k, _)| k.sport == sport) + .map(|(_, q)| q.packets.len()) + .sum() + } + + /// Continuously-backlogged byte shares for two flows. + /// Returns (big_bytes, small_bytes) served over `calls` dequeue calls. + /// Backlogs are TOPPED UP to bounded depths that fit the peer byte cap + /// together (a blind refill lets the jumbo flow hog the cap and sheds + /// the small flow's packets, which measures cap hogging, not DRR). + /// Depths stay >0 so flows never empty/recreate as "new" (which would + /// measure sparse priority, not DRR). + fn fairness_ratio( + big_payload: usize, + small_payload: usize, + quantum: usize, + calls: usize, + ) -> (u64, u64) { + let p = pool(); + let mut s = PeerScheduler::new(quantum); + for _ in 0..12 { + let _ = s.enqueue(logical(&p, 1111, big_payload), Instant::now()); + } + for _ in 0..64 { + let _ = s.enqueue(logical(&p, 2222, small_payload), Instant::now()); + } + demote_all(&mut s); + let mut bytes = [0u64; 2]; + for _ in 0..calls { + while qlen(&s, 1111) < 12 { + if !s + .enqueue(logical(&p, 1111, big_payload), Instant::now()) + .is_accepted() + { + break; + } + } + while qlen(&s, 2222) < 64 { + if !s + .enqueue(logical(&p, 2222, small_payload), Instant::now()) + .is_accepted() + { + break; + } + } + match s.next(Instant::now()) { + Dequeue::Send(burst) => { + for (pkt, _) in burst.packets { + let (f, l) = (pkt.flow, pkt.len()); + s.account_sent(f, l, l); + if f.sport == 1111 { + bytes[0] += l as u64; + } else { + bytes[1] += l as u64; + } + } + } + Dequeue::Empty => panic!("stall with work queued"), + } + } + (bytes[0], bytes[1]) + } + + #[test] + fn drr_byte_fairness_jumbo_vs_small() { + // §2.2-3: 9000 B vs 100 B backlogged flows with a 1200 quantum must + // split BYTES ~evenly (real DRR), not 90:1 by packet count. Packet- + // count fairness would give ratio ≈ 90; byte fairness gives ≈ 1. + let (big, small) = fairness_ratio(9000 - 28, 100 - 28, 1200, 400); + let ratio = big as f64 / small as f64; + assert!( + (0.65..=1.5).contains(&ratio), + "byte shares must be ~equal, got big={big} small={small} ratio={ratio:.2}" + ); + } + + #[test] + fn drr_byte_fairness_matrix() { + // Same property across size pairs: byte shares, not packet shares. + for (big_total, small_total, quantum) in [ + (2800usize, 100usize, 1200usize), + (9000, 1200, 1400), + (2800, 1200, 512), + ] { + let (big, small) = fairness_ratio(big_total - 28, small_total - 28, quantum, 300); + let ratio = big as f64 / small as f64; + assert!( + (0.6..=1.7).contains(&ratio), + "big={big_total} small={small_total} q={quantum}: ratio={ratio:.2}" + ); + } + } + + #[test] + fn drr_byte_fairness_three_flows() { + // Three mixed-size backlogged flows split bytes ~three ways. + let p = pool(); + let mut s = PeerScheduler::new(1200); + let sizes = [ + (1111u16, 9000usize - 28), + (2222, 1200 - 28), + (3333, 100 - 28), + ]; + let depths = [12usize, 32, 64]; + for (i, (sport, size)) in sizes.iter().enumerate() { + for _ in 0..depths[i] { + let _ = s.enqueue(logical(&p, *sport, *size), Instant::now()); + } + } + demote_all(&mut s); + let mut bytes = [0u64; 3]; + for _ in 0..600 { + for (i, (sport, size)) in sizes.iter().enumerate() { + while qlen(&s, *sport) < depths[i] { + if !s + .enqueue(logical(&p, *sport, *size), Instant::now()) + .is_accepted() + { + break; + } + } + } + match s.next(Instant::now()) { + Dequeue::Send(burst) => { + for (pkt, _) in burst.packets { + let (f, l) = (pkt.flow, pkt.len()); + s.account_sent(f, l, l); + let i = sizes.iter().position(|(sp, _)| *sp == f.sport).unwrap(); + bytes[i] += l as u64; + } + } + Dequeue::Empty => panic!("stall with work queued"), + } + } + let total: u64 = bytes.iter().sum(); + for (i, b) in bytes.iter().enumerate() { + let share = *b as f64 / total as f64; + assert!( + (0.22..=0.45).contains(&share), + "flow {i} share must be ~1/3, got {share:.2} (bytes={bytes:?})" + ); + } + } + + #[test] + fn eviction_reports_victim_for_gauge_reconcile() { + // The silent-drop bug: flow-cap eviction used to return None. + // Now the victim length rides the outcome so gauges stay exact. + let p = pool(); + let mut s = PeerScheduler::new(1536); + let one = logical(&p, 1111, 100).len(); + for _ in 0..64 { + assert!( + s.enqueue(logical(&p, 1111, 100), Instant::now()) + .is_accepted() + ); + } + match s.enqueue(logical(&p, 1111, 100), Instant::now()) { + EnqueueOutcome::AcceptedEvicted { + reason, + evicted_len, + } => { + assert_eq!(reason, DropReason::FlowCap); + assert_eq!(evicted_len, one); + } + other => panic!("expected eviction report, got {other:?}"), + } + // Still exactly at cap (evict-one-accept-one is net-zero). + assert_eq!(s.levels().0 as usize, 64); + assert_eq!(s.counters().drops_cap, 1); + // No codel/emergency involved. + assert!(s.drain_drops().is_empty()); + } + + #[test] + fn drain_drops_reports_codel_then_quiet() { + // CoDel drops surface through drain_drops exactly once. + let target = Duration::from_millis(2); + let interval = Duration::from_millis(10); + let p = pool(); + let mut s = PeerScheduler::with_params(1536, target, interval); + let t0 = Instant::now() - interval - Duration::from_millis(5); + for _ in 0..10 { + let mut pkt = logical(&p, 1111, 1200); + pkt.enqueued_at = t0; + assert!(s.enqueue(pkt, t0).is_accepted()); + } + demote_all(&mut s); + let deadline = Instant::now() + Duration::from_millis(500); + let mut reported = 0u64; + while Instant::now() < deadline { + match s.next(Instant::now()) { + Dequeue::Send(burst) => { + for (pkt, _) in burst.packets.into_iter().rev() { + s.requeue_head(pkt.flow, *pkt); + } + } + Dequeue::Empty => {} + } + reported += s.drain_drops().codel; + if reported > 0 { + break; + } + } + assert!(reported > 0, "CoDel drops must surface via drain"); + // Second drain is quiet (deltas partition, never double-count). + assert!(s.drain_drops().is_empty()); + } + + #[test] + fn per_flow_order_preserved() { + let p = pool(); + let mut s = PeerScheduler::new(1536); + for _ in 0..5 { + assert!( + s.enqueue(logical(&p, 1111, 100), Instant::now()) + .is_accepted() + ); + } + let out = drain_all(&mut s); + assert_eq!(out.len(), 5); + assert!(out.windows(2).all(|w| w[0].0 == w[1].0)); + assert!(s.is_empty()); + } + + #[test] + fn sparse_flow_jumps_bulk_backlog() { + let p = pool(); + let mut s = PeerScheduler::new(1536); + for _ in 0..20 { + assert!( + s.enqueue(logical(&p, 1111, 1200), Instant::now()) + .is_accepted() + ); + } + // Age the bulk flow out of sparsity, as the pump would via demotion. + let bulk_key = { + let k = *s.new_list.iter().next().unwrap(); + let q = s.flows.get_mut(&k).unwrap(); + q.is_new = false; + q.epoch_bytes = NEW_FLOW_BYTE_BUDGET; + s.new_list.clear(); + s.old_list.push_back(k); + k + }; + assert!( + s.enqueue(logical(&p, 2222, 100), Instant::now()) + .is_accepted() + ); + match s.next(Instant::now()) { + Dequeue::Send(burst) => assert_ne!(burst.packets[0].0.flow, bulk_key), + Dequeue::Empty => panic!("expected sparse packet"), + } + } + + #[test] + fn byte_drr_no_starvation() { + let p = pool(); + let mut s = PeerScheduler::new(1536); + for sport in [1111u16, 2222] { + for _ in 0..4 { + assert!( + s.enqueue(logical(&p, sport, 1200), Instant::now()) + .is_accepted() + ); + } + } + for k in s.flows.keys().cloned().collect::>() { + let q = s.flows.get_mut(&k).unwrap(); + q.is_new = false; + q.epoch_bytes = NEW_FLOW_BYTE_BUDGET; + } + // Move both to the old list like the pump would via demotion. + s.new_list.clear(); + for k in s.flows.keys().cloned().collect::>() { + s.old_list.push_back(k); + } + let out = drain_all(&mut s); + assert_eq!(out.len(), 8); + assert!(out.iter().any(|(f, _)| f.sport == 1111)); + assert!(out.iter().any(|(f, _)| f.sport == 2222)); + } + + #[test] + fn codel_drops_standing_queue() { + // A persistently backlogged flow with old arrivals must see CoDel + // drops (not just emergency-ceiling drops) once its sojourn exceeds + // target for longer than the interval. Custom short timing keeps the + // test fast while exercising the real control law. + let target = Duration::from_millis(2); + let interval = Duration::from_millis(10); + let p = pool(); + let mut s = PeerScheduler::with_params(1536, target, interval); + let t0 = Instant::now() - interval - Duration::from_millis(5); + for _ in 0..10 { + let mut pkt = logical(&p, 1111, 1200); + pkt.enqueued_at = t0; + assert!(s.enqueue(pkt, t0).is_accepted()); + } + // Demote to old so CoDel (not sparse preference) governs. + demote_all(&mut s); + // Keep the queue standing past the interval: serve + requeue. + // Note: a CoDel drop ends the current drain round (the pump then + // waits briefly and continues), so Empty does not end the test — + // only the deadline or an observed drop does. + let deadline = Instant::now() + Duration::from_millis(500); + let mut codel_drops = 0u64; + while Instant::now() < deadline { + match s.next(Instant::now()) { + Dequeue::Send(burst) => { + // Requeue the whole burst to keep the queue standing. + for (pkt, _) in burst.packets.into_iter().rev() { + s.requeue_head(pkt.flow, *pkt); + } + } + Dequeue::Empty => {} + } + codel_drops = s.counters().drops_codel; + if codel_drops > 0 { + break; + } + } + assert!(codel_drops > 0, "CoDel must drop a standing queue"); + assert_eq!(s.counters().drops_emergency, 0); + } + + #[test] + fn emergency_ceiling_is_safety_only() { + // Fresh traffic never hits the emergency path. + let p = pool(); + let mut s = PeerScheduler::new(1536); + for _ in 0..8 { + assert!( + s.enqueue(logical(&p, 1111, 200), Instant::now()) + .is_accepted() + ); + } + let out = drain_all(&mut s); + assert_eq!(out.len(), 8); + assert_eq!(s.counters().drops_emergency, 0); + assert_eq!(s.counters().drops_codel, 0); + } + + #[test] + fn memory_bounds_enforced_without_scan() { + let p = pool(); + let mut s = PeerScheduler::new(1536); + let offered = PEER_PACKET_CAP + 64; + let mut admitted = 0usize; + let mut evicted_victims = 0usize; + let mut rejected = 0usize; + for i in 0..offered { + match s.enqueue(logical(&p, 1000 + (i % 8) as u16, 1400), Instant::now()) { + EnqueueOutcome::Accepted => admitted += 1, + EnqueueOutcome::AcceptedEvicted { .. } => { + admitted += 1; + evicted_victims += 1; + } + EnqueueOutcome::Rejected { .. } => rejected += 1, + } + } + let (packets, bytes, _) = s.levels(); + assert!((packets as usize) <= PEER_PACKET_CAP); + assert!((bytes as usize) <= PEER_BYTE_CAP + 1500); + // Packet conservation under the reporting model: every admission + // is either still retained or was later evicted; every offered + // packet is retained, rejected, or an evicted victim — each + // reported exactly once. + assert_eq!(admitted, (packets as usize) + evicted_victims); + assert_eq!((packets as usize) + rejected + evicted_victims, offered); + let c = s.counters(); + assert_eq!(c.drops_cap as usize, rejected + evicted_victims); + assert_eq!(c.drops_emergency, 0); + // Fresh traffic: no CoDel/emergency drops to drain. + assert!(s.drain_drops().is_empty()); + } + + #[test] + fn icmp_isolates_from_tcp_bulk() { + use tunnet_common::packet::LogicalPacket as LP; + let icmp_raw = { + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) + .icmpv4_echo_request(7, 1); + let mut o = Vec::new(); + b.write(&mut o, &[0; 32]).unwrap(); + o + }; + let tcp_raw = { + let b = etherparse::PacketBuilder::ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) + .tcp(40000, 443, 1, 1); + let mut o = Vec::new(); + b.write(&mut o, &[0; 1200]).unwrap(); + o + }; + let icmp = LP::from_slice(&icmp_raw).unwrap(); + let tcp = LP::from_slice(&tcp_raw).unwrap(); + assert_ne!(icmp.flow, tcp.flow); + let mut s = PeerScheduler::new(1536); + for _ in 0..20 { + let mut t = LP::from_slice(&tcp_raw).unwrap(); + t.enqueued_at = Instant::now(); + assert!(s.enqueue(t, Instant::now()).is_accepted()); + } + // Bulk is backlogged (demoted as the pump would), so the fresh ICMP + // flow must jump it. + for k in s.flows.keys().cloned().collect::>() { + let q = s.flows.get_mut(&k).unwrap(); + q.is_new = false; + q.epoch_bytes = NEW_FLOW_BYTE_BUDGET; + } + s.new_list.clear(); + for k in s.flows.keys().cloned().collect::>() { + s.old_list.push_back(k); + } + assert!(s.enqueue(icmp, Instant::now()).is_accepted()); + match s.next(Instant::now()) { + Dequeue::Send(burst) => assert_ne!(burst.packets[0].0.flow, tcp.flow), + Dequeue::Empty => panic!("expected icmp"), + } + } + + #[test] + fn requeue_preserves_order_per_peer() { + // Transport-full requeue restores the head packet in order, and peer + // schedulers are fully independent objects (no global HOL). + let p = pool(); + let mut a = PeerScheduler::new(1536); + let mut b = PeerScheduler::new(1536); + for _ in 0..3 { + assert!( + a.enqueue(logical(&p, 1111, 100), Instant::now()) + .is_accepted() + ); + } + assert!( + b.enqueue(logical(&p, 9999, 100), Instant::now()) + .is_accepted() + ); + // Simulate transport-full on A: dequeue (sparse serves one packet) + // then requeue the burst. + let burst = match a.next(Instant::now()) { + Dequeue::Send(burst) => burst, + Dequeue::Empty => panic!("expected packet"), + }; + assert_eq!(burst.packets.len(), 1, "sparse serves one packet"); + for (pkt, _) in burst.packets.into_iter().rev() { + a.requeue_head(pkt.flow, *pkt); + } + // B drains independently (same packet shape, same length). + let expect_len = logical(&p, 9999, 100).len(); + match b.next(Instant::now()) { + Dequeue::Send(burst) => assert_eq!(burst.packets[0].0.len(), expect_len), + Dequeue::Empty => panic!("peer B must be independent of A"), + } + // A still holds all 3 packets in order. + let mut n = 0; + while !a.is_empty() { + match a.next(Instant::now()) { + Dequeue::Send(burst) => n += burst.packets.len() as u32, + Dequeue::Empty => break, + } + } + assert_eq!(n, 3); + assert!(a.is_empty()); + } +} diff --git a/crates/tunnet-core/src/transport_profile.rs b/crates/tunnet-core/src/transport_profile.rs new file mode 100644 index 00000000..9c31ba1a --- /dev/null +++ b/crates/tunnet-core/src/transport_profile.rs @@ -0,0 +1,137 @@ +//! Centrally owned Tunnet QUIC transport profile for the mesh dataplane. +//! +//! Replaces implicit Iroh/noq defaults with one explicit configuration. +//! Baseline: CUBIC, small DATAGRAM send buffer (64 KiB), GSO on, DPLPMTUD on. +//! BBRv3 is an opt-in benchmark experiment, never the silent default. + +use std::sync::Arc; +use std::time::Duration; + +use iroh::endpoint::{AckFrequencyConfig, QuicTransportConfig, VarInt}; + +/// Tunnet DATAGRAM send buffer: 64 KiB. Large enough for pacing/bursts, +/// small enough that ~80 Mbps serializes it in ~6 ms instead of ~100 ms. +/// Diagnostic override via `TUNNET_QUIC_DATAGRAM_BUFFER_KB` (e.g. 64 vs +/// 128 vs 256 A/B runs); clamped to [4 KiB, 1 MiB] like `with_send_buffer`. +pub const DATAGRAM_SEND_BUFFER: usize = 64 * 1024; +/// Receive buffer: 256 KiB (generous inbound headroom, still bounded). +pub const DATAGRAM_RECV_BUFFER: usize = 256 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CongestionControl { + /// Safe production baseline. + Cubic, + /// Benchmark experiment only. + Bbr3, + /// Control for experiments. + NewReno, +} + +#[derive(Debug, Clone)] +pub struct TunnetTransportProfile { + pub congestion: CongestionControl, + pub datagram_send_buffer: usize, + pub datagram_recv_buffer: usize, + pub initial_rtt: Duration, + pub initial_mtu: u16, + pub ack_frequency_packets: Option, +} + +impl Default for TunnetTransportProfile { + fn default() -> Self { + let send_buffer = std::env::var("TUNNET_QUIC_DATAGRAM_BUFFER_KB") + .ok() + .and_then(|v| v.parse::().ok()) + .map(|kb| kb.saturating_mul(1024).clamp(4096, 1024 * 1024)) + .unwrap_or(DATAGRAM_SEND_BUFFER); + Self { + congestion: CongestionControl::Cubic, + datagram_send_buffer: send_buffer, + datagram_recv_buffer: DATAGRAM_RECV_BUFFER, + initial_rtt: Duration::from_millis(90), + initial_mtu: 1200, + ack_frequency_packets: None, + } + } +} + +impl TunnetTransportProfile { + pub fn bbr3_experiment() -> Self { + Self { + congestion: CongestionControl::Bbr3, + ..Self::default() + } + } + + /// Override the DATAGRAM send buffer for the §19 experiment matrix + /// (16/32/64/128 KiB, 64 KiB control). The application queue and the QUIC + /// staging queue are one queueing budget: at ~80 Mbps even 64 KiB is + /// several milliseconds, so smaller is not automatically worse. + pub fn with_send_buffer(mut self, bytes: usize) -> Self { + self.datagram_send_buffer = bytes.clamp(4096, 1024 * 1024); + self + } + + pub fn build(&self) -> QuicTransportConfig { + let b = QuicTransportConfig::builder(); + let b = b + .datagram_send_buffer_size(self.datagram_send_buffer) + .datagram_receive_buffer_size(Some(self.datagram_recv_buffer)) + .initial_rtt(self.initial_rtt) + .initial_mtu(self.initial_mtu) + // Never raise min MTU aggressively on arbitrary internet paths. + .min_mtu(1200) + // GSO stays enabled: valuable pacing/offload, on by default. + .enable_segmentation_offload(true); + // Multipath intentionally left at Iroh default (disabled unless both + // ends negotiate) so NAT traversal behavior is unchanged. + let b = match self.congestion { + CongestionControl::Cubic => b.congestion_controller_factory(Arc::new( + noq_proto::congestion::CubicConfig::default(), + )), + CongestionControl::NewReno => b.congestion_controller_factory(Arc::new( + noq_proto::congestion::NewRenoConfig::default(), + )), + CongestionControl::Bbr3 => b.congestion_controller_factory(Arc::new( + noq_proto::congestion::Bbr3Config::default(), + )), + }; + let b = if let Some(n) = self.ack_frequency_packets + && let Ok(v) = VarInt::from_u64(n) + { + let mut cfg = AckFrequencyConfig::default(); + cfg.ack_eliciting_threshold(v); + b.ack_frequency_config(Some(cfg)) + } else { + b + }; + b.build() + } + + /// Apply to an Iroh endpoint builder (single choke point for mesh endpoints). + pub fn apply(&self, builder: iroh::endpoint::Builder) -> iroh::endpoint::Builder { + builder.transport_config(self.build()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_profile_builds() { + let p = TunnetTransportProfile::default(); + assert_eq!(p.datagram_send_buffer, 64 * 1024); + let _cfg = p.build(); + } + + #[test] + fn bbr3_is_explicit_experiment() { + let p = TunnetTransportProfile::bbr3_experiment(); + assert_eq!(p.congestion, CongestionControl::Bbr3); + assert_eq!( + TunnetTransportProfile::default().congestion, + CongestionControl::Cubic + ); + } +} diff --git a/scripts/bench.ps1 b/scripts/bench.ps1 index a0813907..d6b6402d 100644 --- a/scripts/bench.ps1 +++ b/scripts/bench.ps1 @@ -1,129 +1,387 @@ param( [string]$Peer = "10.7.0.2", - [int]$Duration = 30 + [int]$Duration = 10, + [string]$Product = "tunnet", + [string]$TunnetApi = "http://127.0.0.1:8899", + [int]$Repeats = 2, + [int]$Mtu = 0, + # Independent iperf3 server ports per direction: two simultaneous + # clients against one default port conflict (single active test per + # listener). The server side must listen on both ports. + [int]$ServerPortUp = 5201, + [int]$ServerPortDown = 5202 ) -$iperf3 = "$env:USERPROFILE\bin\iperf3\iperf3.exe" +# Tunnet Benchmark v3 — structured, repeatable, hard to lie with. +# Schema (shared with bench.sh): every scenario appends one JSON object per +# line to results.jsonl with fields: +# {ts, product, scenario, direction, fraction, offered_mbps, actual_mbps, +# loss_pct, retransmits, latency:{n,p50,p95,p99,p999,max}, path:{...}, +# meta:{...}, note, valid} +# Throughput matrix (TCP 1/4, up/down/bidir with explicit JSON parse), +# loaded-latency sweeps per direction plus full-duplex bidir at fractions +# of independently measured directional capacity (download load uses -R), +# UDP rate x size sweep, warmup + repeats, path-state capture before/after +# every scenario (results flagged on migration). p99.9 only with >=1000 +# samples, else null. Loaded scenarios use 200 Test-Connection samples: +# p50/p95/p99 are meaningful, p999 is null BY DESIGN (1000+ ICMP echoes per +# fraction via Test-Connection would take minutes; Bash uses 1000 fast +# pings for real p99.9 — see bench.sh). Failed loads mark valid=false. + +$ErrorActionPreference = "Continue" +$iperf3 = "$env:USERPROFILE\bin\iperf3\iperf3.exe" if (-not (Test-Path $iperf3)) { - $iperf3 = Get-Command iperf3.exe -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Source + $iperf3 = (Get-Command iperf3.exe -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Source) } if (-not $iperf3) { Write-Host "iperf3.exe not found. Download from: https://github.com/ar51an/iperf3-win-builds/releases" -ForegroundColor Red exit 1 } -$ResultsDir = ".\bench-results\$(Get-Date -Format 'yyyyMMdd-HHmmss')" +$ResultsDir = ".\bench-results\$(Get-Date -Format 'yyyyMMdd-HHmmss')-$Product" New-Item -ItemType Directory -Path $ResultsDir -Force | Out-Null +$Jsonl = "$ResultsDir\results.jsonl" + +function Get-Meta { + $sha = "" + try { $sha = (git rev-parse --short HEAD 2>$null).Trim() } catch {} + $cpu = (Get-CimInstance Win32_Processor | Select-Object -First 1).Name + return [ordered]@{ + commit = $sha; mtu = $Mtu; os = (Get-CimInstance Win32_OperatingSystem).Caption + cpu = $cpu; peer = $Peer; duration_s = $Duration + } +} +$META = Get-Meta +$META_JSON = $META | ConvertTo-Json -Compress + +function Get-PathState { + $state = [ordered]@{ product = $Product; mode = "unknown"; detail = "" } + if ($Product -eq "tunnet") { + try { + $status = Invoke-RestMethod -Uri "$TunnetApi/api/status" -TimeoutSec 5 + $state.mode = "$($status.path_state)"; $state.detail = "$($status.selected_path)" + } catch { $state.detail = "tunnet api unreachable" } + } else { + try { + $peers = zerotier-cli peers 2>$null + $state.detail = (($peers | Select-Object -First 6) -join " | ") + if ($peers -match "DIRECT") { $state.mode = "direct" } elseif ($peers -match "RELAY") { $state.mode = "relay" } + } catch { $state.detail = "zerotier-cli unavailable" } + } + return $state +} + +function Write-Row([hashtable]$row) { + $row["ts"] = (Get-Date -Format o); $row["product"] = $Product; $row["meta"] = $META + ($row | ConvertTo-Json -Depth 6 -Compress) | Out-File $Jsonl -Append -Encoding utf8 +} + +function Get-Percentiles([double[]]$Samples) { + if ($Samples.Count -eq 0) { return $null } + $s = $Samples | Sort-Object + function pct([double]$p) { $s[[math]::Min($s.Count - 1, [math]::Floor($p * $s.Count))] } + $o = [ordered]@{ count = $s.Count; min = [math]::Round($s[0], 2); p50 = [math]::Round((pct 0.50), 2); p95 = [math]::Round((pct 0.95), 2); p99 = [math]::Round((pct 0.99), 2); max = [math]::Round($s[-1], 2) } + # p99.9 needs >=1000 samples to mean anything. + if ($s.Count -ge 1000) { $o["p999"] = [math]::Round((pct 0.999), 2) } else { $o["p999"] = $null } + return $o +} + +function Invoke-IperfJson([string]$Args, [string]$OutFile) { + # Every invocation captures command + exit code + stdout + stderr + + # JSON parse status. No generic ERROR: failures carry their cause. + $errFile = "$OutFile.stderr.txt" + $out = & $iperf3 ($Args.Split(" ") + "--json") 2> $errFile + $exitCode = $LASTEXITCODE + $stdout = ($out | Out-String) + $stdout | Out-File $OutFile -Encoding utf8 + $stderr = "" + try { $stderr = Get-Content $errFile -Raw -ErrorAction SilentlyContinue } catch {} + if ([string]::IsNullOrWhiteSpace($stdout)) { + return [ordered]@{ ok = $false; json = $null; exitCode = $exitCode; error = "empty stdout (exit=$exitCode) stderr=$($stderr.Trim())" } + } + try { + $j = $stdout | ConvertFrom-Json + if ($j.error) { + return [ordered]@{ ok = $false; json = $j; exitCode = $exitCode; error = "iperf error: $($j.error)" } + } + return [ordered]@{ ok = $true; json = $j; exitCode = $exitCode; error = "" } + } catch { + return [ordered]@{ ok = $false; json = $null; exitCode = $exitCode; error = "JSON parse failed (exit=$exitCode): $($_.Exception.Message) stderr=$($stderr.Trim())" } + } +} -Write-Host "=== Tunnet Mesh Benchmark ===" -ForegroundColor Cyan -Write-Host "Peer: $Peer | Duration: ${Duration}s | iperf3: $iperf3" -Write-Host "Results: $ResultsDir" -Write-Host "" +# High-frequency latency probe: rapid ping for p99.9-grade sample counts. +function Measure-Latency([int]$Count, [int]$GapMs) { + $samples = @() + for ($i = 0; $i -lt $Count; $i++) { + $r = Test-Connection -ComputerName $Peer -Count 1 -ErrorAction SilentlyContinue + if ($r) { $samples += [double]$r.Latency } + if ($GapMs -gt 0) { Start-Sleep -Milliseconds $GapMs } + } + return Get-Percentiles $samples +} + +Write-Host "=== Tunnet Benchmark v3 ($Product) ===" -ForegroundColor Cyan +Write-Host "Peer: $Peer | Duration: ${Duration}s | Repeats: $Repeats | Results: $ResultsDir" -# ------------------------------------------------------- -# 0. Connectivity -# ------------------------------------------------------- -Write-Host "[0/5] Connectivity..." -ForegroundColor Yellow +# --- connectivity + warmup --- +Write-Host "[0] Connectivity + warmup..." -ForegroundColor Yellow $ping = Test-Connection -ComputerName $Peer -Count 4 -ErrorAction SilentlyContinue -if (-not $ping) { - Write-Host " FAIL: $Peer unreachable" -ForegroundColor Red +if (-not $ping) { Write-Host " FAIL: $Peer unreachable" -ForegroundColor Red; exit 1 } +& $iperf3 -c $Peer -p $ServerPortUp -t 5 -P 2 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host " FAIL: warmup iperf3 exited $LASTEXITCODE (is the server listening on port $ServerPortUp?)" -ForegroundColor Red exit 1 } -Write-Host " OK" -ForegroundColor Green -Write-Host "" - -# ------------------------------------------------------- -# 1. ICMP Ping - 100 packets -# ------------------------------------------------------- -Write-Host "[1/5] ICMP Ping (100 packets)..." -ForegroundColor Yellow -$pingAll = Test-Connection -ComputerName $Peer -Count 100 -ErrorAction SilentlyContinue -$s = $pingAll | ForEach-Object { $_.Latency } | Measure-Object -Minimum -Maximum -Average -StandardDeviation -Write-Host " Min: $($s.Minimum)ms | Avg: $([math]::Round($s.Average,2))ms | Max: $($s.Maximum)ms | StdDev: $([math]::Round($s.StandardDeviation,2))ms" -$pingAll | Export-Csv "$ResultsDir\ping.csv" -NoTypeInformation -Write-Host "" - -# ------------------------------------------------------- -# 2. TCP Upload - single stream -# ------------------------------------------------------- -Write-Host "[2/5] TCP Upload (single stream, ${Duration}s)..." -ForegroundColor Yellow -& $iperf3 -c $Peer -t $Duration --json | Out-File "$ResultsDir\tcp-upload.json" -$up = Get-Content "$ResultsDir\tcp-upload.json" | ConvertFrom-Json -if ($up.error) { - Write-Host " ERROR: $($up.error)" -ForegroundColor Red - Write-Host " Is iperf3 -s running on the peer?" -ForegroundColor Red -} else { - $sent = $up.end.sum_sent - $recv = $up.end.sum_received - Write-Host " Send: $([math]::Round($sent.bits_per_second / 1e6, 1)) Mbps | Recv: $([math]::Round($recv.bits_per_second / 1e6, 1)) Mbps" -} -Write-Host "" - -# ------------------------------------------------------- -# 3. TCP Download - reverse -# ------------------------------------------------------- -Write-Host "[3/5] TCP Download (reverse, ${Duration}s)..." -ForegroundColor Yellow -& $iperf3 -c $Peer -t $Duration -R --json | Out-File "$ResultsDir\tcp-download.json" -$dl = Get-Content "$ResultsDir\tcp-download.json" | ConvertFrom-Json -if ($dl.error) { - Write-Host " ERROR: $($dl.error)" -ForegroundColor Red -} else { - $dlRecv = $dl.end.sum_received - Write-Host " Download: $([math]::Round($dlRecv.bits_per_second / 1e6, 1)) Mbps" +Write-Host " OK, warmed up" + +# --- idle latency: 1200 samples for real p99.9 --- +Write-Host "[1] Idle latency (1200 samples)..." -ForegroundColor Yellow +$idle = Measure-Latency 1200 5 +Write-Host " idle: p50=$($idle.p50)ms p95=$($idle.p95)ms p99=$($idle.p99)ms p999=$($idle.p999)ms max=$($idle.max)ms" +Write-Row @{ scenario = "idle"; direction = "none"; path = (Get-PathState); latency = $idle } + +# --- throughput matrix with repeats; explicit bidir parse --- +Write-Host "[2] Throughput matrix..." -ForegroundColor Yellow +$tpCases = @( + @{ name = "tcp-up-1"; args = "-c $Peer -p $ServerPortUp -t $Duration -P 1"; dir = "up" }, + @{ name = "tcp-up-4"; args = "-c $Peer -p $ServerPortUp -t $Duration -P 4"; dir = "up" }, + @{ name = "tcp-down-1"; args = "-c $Peer -p $ServerPortDown -t $Duration -P 1 -R"; dir = "down" }, + @{ name = "tcp-down-4"; args = "-c $Peer -p $ServerPortDown -t $Duration -P 4 -R"; dir = "down" } +) +$cap = @{ up = 0.0; down = 0.0 } +foreach ($rep in 1..$Repeats) { + foreach ($c in $tpCases) { + $pathBefore = Get-PathState + $r = Invoke-IperfJson $c.args "$ResultsDir\$($c.name)-r$rep.json" + if ($r.ok) { + $j = $r.json + $mbps = [math]::Round($j.end.sum_received.bits_per_second / 1e6, 1) + $retr = 0; try { $retr = [int]$j.end.sum_sent.retransmits } catch {} + $sentMbps = 0; try { $sentMbps = [math]::Round($j.end.sum_sent.bits_per_second / 1e6, 1) } catch {} + Write-Host " $($c.name) r$rep : $mbps Mbps (retr=$retr)" + if ($c.name -like "*-4") { + if ($mbps -gt $cap[$c.dir]) { $cap[$c.dir] = $mbps } + } + Write-Row @{ scenario = $c.name; direction = $c.dir; repeat = $rep; offered_mbps = $null + actual_mbps = $mbps; sent_mbps = $sentMbps; retransmits = $retr; path = $pathBefore; valid = $true } + } else { + Write-Host " $($c.name) r$rep : FAILED: $($r.error)" -ForegroundColor Red + Write-Row @{ scenario = $c.name; direction = $c.dir; repeat = $rep; offered_mbps = $null + actual_mbps = -1; sent_mbps = -1; retransmits = -1; path = $pathBefore + note = "TCP FAILED: $($r.error)"; valid = $false } + } + } + # Bidirectional: parse both directions explicitly (v2 bug: bidir was unread). + $pathBefore = Get-PathState + $r = Invoke-IperfJson "-c $Peer -p $ServerPortUp -t $Duration -P 4 --bidir" "$ResultsDir\tcp-bidir-r$rep.json" + if ($r.ok) { + $j = $r.json + # iperf3 --bidir JSON: sum_sent/sum_received cover the client direction; + # server-side streams appear under server_output_text; parse both. + $upMbps = 0; $downMbps = 0; $retr = 0 + try { $upMbps = [math]::Round($j.end.sum_sent.bits_per_second / 1e6, 1) } catch {} + try { $downMbps = [math]::Round($j.end.sum_received.bits_per_second / 1e6, 1) } catch {} + try { $retr = [int]$j.end.sum_sent.retransmits } catch {} + try { + foreach ($s in $j.server_output_text -split "`n") { + if ($s -match "receiver" -and $s -match "([0-9.]+)\s+Mbits/sec") { + $downMbps = [math]::Round([double]$Matches[1], 1) + } + } + } catch {} + Write-Host " tcp-bidir r$rep : up=${upMbps}Mbps down=${downMbps}Mbps (retr=$retr)" + Write-Row @{ scenario = "tcp-bidir"; direction = "bidir"; repeat = $rep + actual_mbps = $upMbps; down_mbps = $downMbps; retransmits = $retr; path = $pathBefore; valid = $true } + } else { + Write-Host " tcp-bidir r$rep : FAILED: $($r.error)" -ForegroundColor Red + Write-Row @{ scenario = "tcp-bidir"; direction = "bidir"; repeat = $rep + actual_mbps = -1; down_mbps = -1; retransmits = -1; path = $pathBefore + note = "TCP FAILED: $($r.error)"; valid = $false } + } } -Write-Host "" - -# ------------------------------------------------------- -# 4. UDP - jitter + packet loss -# ------------------------------------------------------- -Write-Host "[4/5] UDP (500M target, ${Duration}s)..." -ForegroundColor Yellow -& $iperf3 -c $Peer -u -b 500M -t $Duration --json | Out-File "$ResultsDir\udp.json" -$udp = Get-Content "$ResultsDir\udp.json" | ConvertFrom-Json -if ($udp.error) { - Write-Host " ERROR: $($udp.error)" -ForegroundColor Red -} else { - $us = $udp.end.sum - Write-Host " Bitrate: $([math]::Round($us.bits_per_second / 1e6, 1)) Mbps" - Write-Host " Jitter: $([math]::Round($us.jitter_ms, 3)) ms" - Write-Host " Lost: $([math]::Round($us.lost_percent, 2))%" +# No invented capacity: if the TCP matrix failed, every capacity-dependent +# sweep would be built on fiction. Stop loudly instead. +if ($cap.up -eq 0 -or $cap.down -eq 0) { + Write-Host " FATAL: TCP capacity measurement failed (up=$($cap.up) down=$($cap.down)). Refusing to invent 50 Mbps; fix the TCP path first." -ForegroundColor Red + Write-Host "Results so far: $Jsonl" -ForegroundColor Yellow + exit 1 } -Write-Host "" - -# ------------------------------------------------------- -# 5. Latency under load -# ------------------------------------------------------- -Write-Host "[5/5] Latency under load (${Duration}s)..." -ForegroundColor Yellow -$job = Start-Job -ScriptBlock { - param($exe, $p, $d) - & $exe -c $p -t $d -P 4 2>&1 -} -ArgumentList $iperf3, $Peer, $Duration - -Start-Sleep 3 - -$loadPing = Test-Connection -ComputerName $Peer -Count ([math]::Min($Duration - 6, 50)) -ErrorAction SilentlyContinue -$ls = $loadPing | ForEach-Object { $_.Latency } | Measure-Object -Minimum -Maximum -Average -StandardDeviation - -Wait-Job $job | Out-Null -Remove-Job $job - -Write-Host " Min: $($ls.Minimum)ms | Avg: $([math]::Round($ls.Average,2))ms | Max: $($ls.Maximum)ms | StdDev: $([math]::Round($ls.StandardDeviation,2))ms" -$loadPing | Export-Csv "$ResultsDir\ping-under-load.csv" -NoTypeInformation -Write-Host "" - -# ------------------------------------------------------- -# Summary -# ------------------------------------------------------- -Write-Host "========================================" -ForegroundColor Cyan -Write-Host " ICMP (idle): $([math]::Round($s.Average,2)) ms avg" -Write-Host " ICMP (load): $([math]::Round($ls.Average,2)) ms avg" -if (-not $udp.error) { - Write-Host " UDP Jitter: $([math]::Round($us.jitter_ms,3)) ms" +Write-Host " measured capacity: up=$($cap.up)Mbps down=$($cap.down)Mbps" + +# --- loaded latency per direction at fractions of directional capacity --- +# NOTE on samples: Measure-Latency 200 gives meaningful p50/p95/p99; +# p999 stays null by design (see header). Bash uses 1000 fast pings. +Write-Host "[3] Loaded-latency sweeps (200 samples/dir: p99 max, p999 null)..." -ForegroundColor Yellow +$fractions = @(0.25, 0.50, 0.75, 0.90, 1.00, 1.10) +$dirs = @( + @{ name = "upload"; cap = $cap.up; port = $ServerPortUp }, + @{ name = "download"; cap = $cap.down; port = $ServerPortDown } +) +foreach ($d in $dirs) { + foreach ($f in $fractions) { + $rate = [math]::Round($d.cap * $f, 1) + $pct = [int]($f * 100) + $pathBefore = Get-PathState + # Direction-specific load: download MUST use -R (server sends), or + # the "download" test silently measures upload load. + $isDown = ($d.name -eq "download") + $port = $d.port + $loadFile = "$ResultsDir\load-$($d.name)-$F.json" + $job = Start-Job -ScriptBlock { + param($exe, $p, $dd, $r, $rev, $pp, $out) + if ($rev) { & $exe -c $p -p $pp -t $dd -u -b "${r}M" -R --json 2>&1 | Out-File $out -Encoding utf8 } + else { & $exe -c $p -p $pp -t $dd -u -b "${r}M" --json 2>&1 | Out-File $out -Encoding utf8 } + if ($LASTEXITCODE -ne 0) { "EXIT:$LASTEXITCODE" | Out-File "$out.exit" -Encoding utf8 } + } -ArgumentList $iperf3, $Peer, $Duration, $rate, $isDown, $port, $loadFile + Start-Sleep 2 + $lat = Measure-Latency 200 5 + $null = Receive-Job $job -Wait -AutoRemoveJob + $loadJson = "" + $jobExit = $null + if (Test-Path $loadFile) { $loadJson = Get-Content $loadFile -Raw } + if (Test-Path "$loadFile.exit") { $jobExit = (Get-Content "$loadFile.exit" -Raw).Trim() } + $valid = $true + $loadErr = "" + if ([string]::IsNullOrWhiteSpace($loadJson)) { $valid = $false; $loadErr = "no load output (job crashed?)" } + if ($jobExit) { $valid = $false; $loadErr = "load client $jobExit" } + try { + if ([string]::IsNullOrWhiteSpace($loadJson)) { throw "empty load output" } + $lj = $loadJson | ConvertFrom-Json + if ($lj.error) { throw "iperf error: $($lj.error)" } + $actual = [math]::Round($lj.end.sum.bits_per_second / 1e6, 1) + $loss = [math]::Round($lj.end.sum.lost_percent, 2) + } catch { $actual = -1; $loss = -1; $valid = $false; if (-not $loadErr) { $loadErr = $_.Exception.Message } } + $pathAfter = Get-PathState + $note = "" + if ($pathBefore.mode -ne $pathAfter.mode) { $note = "PATH CHANGED mid-run; result flagged" } + if ($actual -gt 0 -and $actual -lt $rate * 0.7 -and $f -le 1.0) { $note += " under-delivered load" } + if (-not $valid) { $note += " LOAD FAILED ($loadErr): row invalid, values are placeholders" } + Write-Host " $($d.name) ${pct}%: actual=${actual}Mbps loss=${loss}% p50=$($lat.p50) p95=$($lat.p95) p99=$($lat.p99) max=$($lat.max) valid=$valid $note" + Write-Row @{ scenario = "loaded-latency"; direction = $d.name; fraction = $f + offered_mbps = $rate; actual_mbps = $actual; loss_pct = $loss + latency = $lat; path = $pathBefore; path_after = $pathAfter; note = $note.Trim(); valid = $valid } + } } -if (-not $up.error) { - Write-Host " Upload: $([math]::Round($sent.bits_per_second / 1e6, 1)) Mbps" + +# --- bidirectional loaded latency: full-duplex UDP at fractions --- +# Up and down loads run on SEPARATE server ports: two clients against one +# listener conflict on a normal iperf3 server. A bidir row is only valid +# when BOTH directions ran; failures mark valid=false explicitly. +Write-Host " bidir (full duplex, 200 samples: p99 max, p999 null)..." -ForegroundColor Yellow +foreach ($f in @(0.25, 0.50, 0.75, 0.90, 1.00)) { + $rateUp = [math]::Round($cap.up * $f, 1) + $rateDown = [math]::Round($cap.down * $f, 1) + $pct = [int]($f * 100) + $pathBefore = Get-PathState + $upFile = "$ResultsDir\load-bidi-$f-up.json" + $downFile = "$ResultsDir\load-bidi-$f-down.json" + $jobUp = Start-Job -ScriptBlock { + param($exe, $p, $dd, $r, $pp, $out) & $exe -c $p -p $pp -t $dd -u -b "${r}M" --json 2>&1 | Out-File $out -Encoding utf8 + if ($LASTEXITCODE -ne 0) { "EXIT:$LASTEXITCODE" | Out-File "$out.exit" -Encoding utf8 } + } -ArgumentList $iperf3, $Peer, $Duration, $rateUp, $ServerPortUp, $upFile + $jobDown = Start-Job -ScriptBlock { + param($exe, $p, $dd, $r, $pp, $out) & $exe -c $p -p $pp -t $dd -u -b "${r}M" -R --json 2>&1 | Out-File $out -Encoding utf8 + if ($LASTEXITCODE -ne 0) { "EXIT:$LASTEXITCODE" | Out-File "$out.exit" -Encoding utf8 } + } -ArgumentList $iperf3, $Peer, $Duration, $rateDown, $ServerPortDown, $downFile + Start-Sleep 2 + $lat = Measure-Latency 200 5 + $null = Receive-Job $jobUp -Wait -AutoRemoveJob + $null = Receive-Job $jobDown -Wait -AutoRemoveJob + $upJson = ""; $downJson = "" + if (Test-Path $upFile) { $upJson = Get-Content $upFile -Raw } + if (Test-Path $downFile) { $downJson = Get-Content $downFile -Raw } + $exitUp = $null; $exitDown = $null + if (Test-Path "$upFile.exit") { $exitUp = (Get-Content "$upFile.exit" -Raw).Trim() } + if (Test-Path "$downFile.exit") { $exitDown = (Get-Content "$downFile.exit" -Raw).Trim() } + $actualUp = -1; $lossUp = -1; $actualDown = -1; $lossDown = -1 + $errUp = $null; $errDown = $null + if ([string]::IsNullOrWhiteSpace($upJson)) { $errUp = "no up-load output (job crashed?)" } + if ($exitUp) { $errUp = "up-load client $exitUp" } + if ([string]::IsNullOrWhiteSpace($downJson)) { $errDown = "no down-load output (job crashed?)" } + if ($exitDown) { $errDown = "down-load client $exitDown" } + if (-not $errUp) { + try { + $uj = $upJson | ConvertFrom-Json + if ($uj.error) { throw "iperf error: $($uj.error)" } + $actualUp = [math]::Round($uj.end.sum.bits_per_second / 1e6, 1) + $lossUp = [math]::Round($uj.end.sum.lost_percent, 2) + } catch { $errUp = $_.Exception.Message } + } + if (-not $errDown) { + try { + $dj = $downJson | ConvertFrom-Json + if ($dj.error) { throw "iperf error: $($dj.error)" } + $actualDown = [math]::Round($dj.end.sum.bits_per_second / 1e6, 1) + $lossDown = [math]::Round($dj.end.sum.lost_percent, 2) + } catch { $errDown = $_.Exception.Message } + } + $pathAfter = Get-PathState + $note = "" + if ($pathBefore.mode -ne $pathAfter.mode) { $note = "PATH CHANGED mid-run; result flagged" } + if ($actualUp -gt 0 -and $actualUp -lt $rateUp * 0.7 -and $f -le 1.0) { $note += " under-delivered up load" } + if ($actualDown -gt 0 -and $actualDown -lt $rateDown * 0.7 -and $f -le 1.0) { $note += " under-delivered down load" } + $valid = ($null -eq $errUp) -and ($null -eq $errDown) + if ($errUp) { $note += " BIDIR INVALID: up load failed ($errUp)" } + if ($errDown) { $note += " BIDIR INVALID: down load failed ($errDown)" } + Write-Host " bidir ${pct}%: up=${actualUp}Mbps loss=${lossUp}% down=${actualDown}Mbps loss=${lossDown}% p50=$($lat.p50) p95=$($lat.p95) p99=$($lat.p99) valid=$valid $note" + Write-Row @{ scenario = "loaded-latency"; direction = "bidir"; fraction = $f + offered_up_mbps = $rateUp; offered_down_mbps = $rateDown + actual_up_mbps = $actualUp; actual_down_mbps = $actualDown + loss_up_pct = $lossUp; loss_down_pct = $lossDown + latency = $lat; path = $pathBefore; path_after = $pathAfter; note = $note.Trim(); valid = $valid } } -if (-not $dl.error) { - Write-Host " Download: $([math]::Round($dlRecv.bits_per_second / 1e6, 1)) Mbps" + +# --- UDP sweep: rates x sizes x directions, sender vs receiver split --- +# Sizes span single-frame (512, 900) and segmented (1200, 1460, 2700) +# dataplane behavior; both directions run so upload/download asymmetries +# (like the Linux→Windows collapse) show up with numbers, not vibes. +# `actual_mbps` is RECEIVER-side delivered throughput (sum_received), never +# the sender offer: iperf3 UDP `sum` can be sender-side, and misreading it +# once reported "actual=50Mbps loss=92%" for ~3.8 Mbps delivered. +Write-Host "[4] UDP sweep (sizes x directions)..." -ForegroundColor Yellow +$udpDirs = @( + @{ name = "up"; port = $ServerPortUp; extra = "" }, + @{ name = "down"; port = $ServerPortDown; extra = "-R" } +) +foreach ($ud in $udpDirs) { + foreach ($f in @(0.25, 0.50, 1.00)) { + $rate = [math]::Round($cap.up * $f, 1) + foreach ($len in @(512, 900, 1200, 1460, 2700)) { + $args = "-c $Peer -p $($ud.port) -u -b ${rate}M -l $len -t $Duration $($ud.extra)".Trim() + $r = Invoke-IperfJson $args "$ResultsDir\udp-$($ud.name)-${rate}M-${len}B.json" + if ($r.ok) { + $j = $r.json + # Sender side (offer) vs receiver side (delivered): read + # both explicitly; a missing receiver summary invalidates + # the row instead of masquerading the offer as delivered. + $send = $null; $recv = $null + try { $send = $j.end.sum } catch {} + try { $recv = $j.end.sum_received } catch {} + if ($null -eq $recv) { throw "no sum_received in iperf JSON" } + $sentMbps = [math]::Round($send.bits_per_second / 1e6, 1) + $del = [math]::Round($recv.bits_per_second / 1e6, 1) + $ppsSent = $null; $ppsRecv = $null + try { $ppsSent = [math]::Round($send.packets / $Duration, 0) } catch {} + try { $ppsRecv = [math]::Round($recv.packets_received / $Duration, 0) } catch {} + if ($null -eq $ppsRecv) { try { $ppsRecv = [math]::Round($recv.packets / $Duration, 0) } catch {} } + $loss = -1; $jitter = $null + try { $loss = [math]::Round($recv.lost_percent, 2) } catch {} + try { $jitter = [math]::Round($recv.jitter_ms, 3) } catch {} + Write-Host (" $($ud.name) offered={0}Mbps sent={1}Mbps delivered={2}Mbps pps_sent={3} pps_recv={4} loss={5}% jitter={6}ms len={7}B" -f $rate, $sentMbps, $del, $ppsSent, $ppsRecv, $loss, $jitter, $len) + Write-Row @{ scenario = "udp"; direction = $ud.name; offered_mbps = $rate; packet_len = $len + sent_mbps = $sentMbps; actual_mbps = $del; pps_sent = $ppsSent; pps_received = $ppsRecv + loss_pct = $loss; jitter_ms = $jitter; path = (Get-PathState); valid = $true } + } else { + Write-Host " $($ud.name) offered=${rate}Mbps len=${len}B : FAILED: $($r.error)" -ForegroundColor Red + Write-Row @{ scenario = "udp"; direction = $ud.name; offered_mbps = $rate; packet_len = $len + sent_mbps = -1; actual_mbps = -1; pps_sent = $null; pps_received = $null + loss_pct = -1; jitter_ms = $null; path = (Get-PathState) + note = "UDP FAILED: $($r.error)"; valid = $false } + } + } + } } -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "Results saved: $ResultsDir" -ForegroundColor Green + +Write-Host "`nResults: $ResultsDir\results.jsonl (shared schema)" -ForegroundColor Green diff --git a/scripts/bench.sh b/scripts/bench.sh index 6119d156..eb081f7d 100644 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -1,76 +1,370 @@ #!/bin/bash -PEER="10.7.0.2" -DURATION=30 -RESULTS_DIR="./bench-results/$(date +%Y%m%d-%H%M%S)" +# Tunnet Benchmark v3 — structured, repeatable, hard to lie with. +# Shared schema with bench.ps1: one JSON object per line in results.jsonl: +# {ts, product, scenario, direction, fraction, offered_mbps, actual_mbps, +# loss_pct, retransmits, latency:{n,p50,p95,p99,p999,max}, path:{...}, +# meta:{...}, note} +# Throughput matrix with explicit JSON fields (no regexed human output), +# loaded-latency sweeps per direction plus full-duplex bidir at fractions +# of independently measured directional capacity, UDP rate x size sweep, +# warmup + repeats, path-state capture around every scenario (flagged on +# migration), p99.9 from >=1000 high-frequency samples (gated to null when +# short). Up/down loads use separate server ports ($6/$7, default +# 5201/5202 — the server must listen on both); failed loads mark their row +# valid=false instead of hiding behind -1 placeholders. +set -u +PEER="${1:-10.7.0.2}" +DURATION="${2:-10}" +PRODUCT="${3:-tunnet}" +REPEATS="${4:-2}" +MTU="${5:-0}" +# Independent iperf3 server ports per direction: two simultaneous clients +# against one default port conflict (single active test per listener). +# The server side must listen on both: `iperf3 -s -p 5201` + `-p 5202`. +PORT_UP="${6:-5201}" +PORT_DOWN="${7:-5202}" +RESULTS_DIR="./bench-results/$(date +%Y%m%d-%H%M%S)-$PRODUCT" mkdir -p "$RESULTS_DIR" +JSONL="$RESULTS_DIR/results.jsonl" -echo "=== Tunnet Mesh Benchmark ===" -echo "Peer: $PEER | Duration: ${DURATION}s per test" -echo "Results: $RESULTS_DIR" -echo "" +# Exported BEFORE the first consumer (every python block below reads them). +export BENCH_PRODUCT="$PRODUCT" BENCH_DURATION="$DURATION" +export BENCH_JSONL="$JSONL" -# 1. Baseline ping -echo "[1/5] Baseline ping..." -ping -c 100 -i 0.01 "$PEER" > "$RESULTS_DIR/ping.txt" 2>&1 -tail -3 "$RESULTS_DIR/ping.txt" -echo "" +meta_json() { + python3 -c " +import json,platform,subprocess +try: sha=subprocess.check_output(['git','rev-parse','--short','HEAD'],text=True).strip() +except Exception: sha='' +print(json.dumps({'commit':sha,'mtu':$MTU,'os':platform.platform(),'cpu':platform.processor() or platform.machine(),'peer':'$PEER','duration_s':$DURATION}))" +} +META=$(meta_json) +export BENCH_META="$META" -# 2. Latency - TCP RR -echo "[2/5] Latency (TCP_RR)..." -netperf -H "$PEER" -t TCP_RR -l "$DURATION" -- \ - -o min_latency,mean_latency,max_latency,p99_latency,stddev_latency \ - > "$RESULTS_DIR/latency-tcp.txt" 2>&1 -cat "$RESULTS_DIR/latency-tcp.txt" -echo "" +# Product-aware path collection: the Tunnet API only exists for tunnet runs. +path_json() { + if [ "$PRODUCT" = "tunnet" ]; then + python3 -c " +import json,subprocess +mode='unknown'; detail='' +try: + import urllib.request + st=json.load(urllib.request.urlopen('http://127.0.0.1:8899/api/status',timeout=5)) + mode=str(st.get('path_state','unknown')); detail=str(st.get('selected_path','')) +except Exception as e: + detail='tunnet api unreachable' +print(json.dumps({'product':'$PRODUCT','mode':mode,'detail':detail[:400]}))" + else + # e.g. zerotier: summarize peer path states instead of querying Tunnet. + zerotier-cli peers 2>/dev/null | head -6 | tr '\n' '|' | cut -c1-400 > "$RESULTS_DIR/.pathdetail" 2>/dev/null || echo "path collection n/a for $PRODUCT" > "$RESULTS_DIR/.pathdetail" + python3 -c " +import json +detail=open('$RESULTS_DIR/.pathdetail').read().strip() +mode='unknown' +if 'DIRECT' in detail: mode='direct' +elif 'RELAY' in detail: mode='relay' +print(json.dumps({'product':'$PRODUCT','mode':mode,'detail':detail[:400]}))" + fi +} + +# High-frequency latency probe: COUNT samples back to back (p99.9 needs n>=1000). +ping_samples_json() { # COUNT OUTFILE + ping -c "$1" -i 0.005 "$PEER" 2>/dev/null | grep -oE 'time=[0-9.]+' | cut -d= -f2 > "$2" + python3 -c " +import json +xs=sorted(float(x) for x in open('$2') if x.strip()) +n=len(xs) +def q(p): return xs[min(n-1,int(p*n))] if n else -1 +r={'n':n} +if n: + r.update({'min':round(xs[0],2),'p50':round(q(.5),2),'p95':round(q(.95),2),'p99':round(q(.99),2),'max':round(xs[-1],2)}) + r['p999']=round(q(.999),2) if n>=1000 else None +print(json.dumps(r))" +} -# 3. Latency - UDP RR -echo "[3/5] Latency (UDP_RR)..." -netperf -H "$PEER" -t UDP_RR -l "$DURATION" -- \ - -o min_latency,mean_latency,max_latency,p99_latency,stddev_latency \ - > "$RESULTS_DIR/latency-udp.txt" 2>&1 -cat "$RESULTS_DIR/latency-udp.txt" -echo "" +# Structured iperf runner: prints nothing human; writes JSON file; echoes path. +run_iperf() { # NAME EXTRA_ARGS... + local name="$1"; shift + iperf3 -c "$PEER" -t "$DURATION" "$@" --json > "$RESULTS_DIR/$name.json" 2>&1 +} -# 4. Throughput - TCP -echo "[4/5] Throughput (TCP, 4 streams)..." -iperf3 -c "$PEER" -t "$DURATION" -P 4 --json \ - > "$RESULTS_DIR/throughput-tcp.json" 2>&1 -# Extract summary -python3 -c " -import json, sys -d = json.load(open('$RESULTS_DIR/throughput-tcp.json')) -e = d['end']['sum_sent'] -print(f\" Sent: {e['bits_per_second']/1e6:.1f} Mbps\") -e = d['end']['sum_received'] -print(f\" Recv: {e['bits_per_second']/1e6:.1f} Mbps\") -" 2>/dev/null || grep -A2 "sender" "$RESULTS_DIR/throughput-tcp.json" -echo "" +echo "=== Tunnet Benchmark v3 ($PRODUCT) ===" +echo "Peer: $PEER | Duration: ${DURATION}s | Repeats: $REPEATS | Results: $RESULTS_DIR" -# 5. Throughput - UDP -echo "[5/5] Throughput (UDP, 500M target)..." -iperf3 -c "$PEER" -u -b 500M -t "$DURATION" --json \ - > "$RESULTS_DIR/throughput-udp.json" 2>&1 -python3 -c " -import json -d = json.load(open('$RESULTS_DIR/throughput-udp.json')) -e = d['end']['sum'] -print(f\" Bitrate: {e['bits_per_second']/1e6:.1f} Mbps\") -print(f\" Jitter: {e['jitter_ms']:.3f} ms\") -print(f\" Lost: {e['lost_percent']:.2f}%\") -" 2>/dev/null -echo "" +# --- connectivity + warmup --- +echo "[0] Connectivity + warmup..." +ping -c 4 "$PEER" > /dev/null 2>&1 || { echo " FAIL: $PEER unreachable"; exit 1; } +iperf3 -c "$PEER" -p "$PORT_UP" -t 5 -P 2 > /dev/null 2>&1 || { echo " FAIL: warmup iperf3 failed (is the server listening on port $PORT_UP?)"; exit 1; } +echo " OK, warmed up" + +# --- idle latency: 1200 samples for real p99.9 --- +echo "[1] Idle latency (1200 samples)..." +IDLE_JSON=$(ping_samples_json 1200 "$RESULTS_DIR/ping-idle.txt") +echo " idle: $IDLE_JSON" +path_json > "$RESULTS_DIR/idle.path" +echo "$IDLE_JSON" > "$RESULTS_DIR/idle.lat" +python3 - "$RESULTS_DIR/idle.lat" "$RESULTS_DIR/idle.path" <<'EOF' +import json,sys,datetime,os +lat = json.load(open(sys.argv[1])) +path = json.load(open(sys.argv[2])) +row = {'scenario': 'idle', 'direction': 'none', 'latency': lat, 'path': path} +row['ts'] = datetime.datetime.now(datetime.timezone.utc).isoformat() +row['product'] = os.environ['BENCH_PRODUCT'] +row['meta'] = json.loads(os.environ['BENCH_META']) +open(os.environ['BENCH_JSONL'], 'a').write(json.dumps(row) + chr(10)) +EOF + +# --- throughput matrix with repeats; explicit bidir parse --- +echo "[2] Throughput matrix..." +CAP_UP=0; CAP_DOWN=0 +tp_case() { # NAME DIR REPEAT EXTRA... + local name="$1" dir="$2" rep="$3"; shift 3 + path_json > "$RESULTS_DIR/$name-r$rep.path" + run_iperf "$name-r$rep" "$@" + MBPS=$(python3 - "$RESULTS_DIR/$name-r$rep.json" "$RESULTS_DIR/$name-r$rep.path" "$name" "$dir" "$rep" <<'EOF' +import json,sys,datetime,os +ipath, ppath, name, direction, rep = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] +d = json.load(open(ipath)) +path = json.load(open(ppath)) +if isinstance(d.get('error'), str): + print(f" {name} r{rep}: ERROR {d['error']}", flush=True) + print(0) +else: + mbps = round(d['end']['sum_received']['bits_per_second']/1e6, 1) + try: sent = round(d['end']['sum_sent']['bits_per_second']/1e6, 1) + except Exception: sent = -1 + try: retr = int(d['end']['sum_sent'].get('retransmits', 0)) + except Exception: retr = -1 + print(f" {name} r{rep}: {mbps} Mbps (retr={retr})", flush=True) + row = {'scenario': name, 'direction': direction, 'repeat': int(rep), + 'actual_mbps': mbps, 'sent_mbps': sent, 'retransmits': retr, + 'path': path} + row['ts'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + row['product'] = os.environ['BENCH_PRODUCT'] + row['meta'] = json.loads(os.environ['BENCH_META']) + open(os.environ['BENCH_JSONL'], 'a').write(json.dumps(row) + chr(10)) + print(mbps) +EOF +) + echo "$MBPS" | tail -1 +} +for rep in $(seq 1 "$REPEATS"); do + tp_case tcp-up-1 up "$rep" -p "$PORT_UP" -P 1 > /dev/null + MB=$(tp_case tcp-up-4 up "$rep" -p "$PORT_UP" -P 4 | tail -1) + CAP_UP=$(python3 -c "print(max(float('$CAP_UP'), float('$MB')))") + tp_case tcp-down-1 down "$rep" -p "$PORT_DOWN" -P 1 -R > /dev/null + MB=$(tp_case tcp-down-4 down "$rep" -p "$PORT_DOWN" -P 4 -R | tail -1) + CAP_DOWN=$(python3 -c "print(max(float('$CAP_DOWN'), float('$MB')))") + # Bidirectional: parse both directions explicitly (v2 never did). + path_json > "$RESULTS_DIR/tcp-bidir-r$rep.path" + run_iperf "tcp-bidir-r$rep" -p "$PORT_UP" -P 4 --bidir + python3 - "$RESULTS_DIR/tcp-bidir-r$rep.json" "$RESULTS_DIR/tcp-bidir-r$rep.path" "$rep" <<'EOF' +import json,sys,datetime,os +ipath, ppath, rep = sys.argv[1], sys.argv[2], sys.argv[3] +d = json.load(open(ipath)) +path = json.load(open(ppath)) +up = round(d['end']['sum_sent']['bits_per_second']/1e6, 1) +down = round(d['end']['sum_received']['bits_per_second']/1e6, 1) +try: retr = int(d['end']['sum_sent'].get('retransmits', 0)) +except Exception: retr = -1 +print(f" tcp-bidir r{rep}: up={up}Mbps down={down}Mbps (retr={retr})") +row = {'scenario': 'tcp-bidir', 'direction': 'bidir', 'repeat': int(rep), + 'actual_mbps': up, 'down_mbps': down, 'retransmits': retr, 'path': path} +row['ts'] = datetime.datetime.now(datetime.timezone.utc).isoformat() +row['product'] = os.environ['BENCH_PRODUCT'] +row['meta'] = json.loads(os.environ['BENCH_META']) +open(os.environ['BENCH_JSONL'], 'a').write(json.dumps(row) + chr(10)) +EOF +done +# No invented capacity: if the TCP matrix failed, every capacity-dependent +# sweep would be built on fiction. Stop loudly instead. +if ! python3 -c "import sys; sys.exit(0 if float('$CAP_UP')>0 and float('$CAP_DOWN')>0 else 1)"; then + echo " FATAL: TCP capacity measurement failed (up=${CAP_UP} down=${CAP_DOWN}). Refusing to invent 50 Mbps; fix the TCP path first." + echo "Results so far: $JSONL" + exit 1 +fi +echo " measured capacity: up=${CAP_UP}Mbps down=${CAP_DOWN}Mbps" + +# --- loaded latency per direction at fractions of directional capacity --- +echo "[3] Loaded-latency sweeps..." +for dir in "upload:$CAP_UP:$PORT_UP:" "download:$CAP_DOWN:$PORT_DOWN:-R"; do + name="${dir%%:*}"; rest="${dir#*:}"; cap="${rest%%:*}"; rest2="${rest#*:}" + port="${rest2%%:*}"; extra="${rest2#*:}" + for F in 0.25 0.50 0.75 0.90 1.00 1.10; do + RATE=$(python3 -c "print(round(float('$cap')*float('$F'),1))") + PCT=$(python3 -c "print(int(float('$F')*100))") + echo " $name ${PCT}% (${RATE}Mbps)..." + path_json > "$RESULTS_DIR/load-$name-$F.path0" + # shellcheck disable=SC2086 + iperf3 -c "$PEER" -p "$port" -t "$DURATION" -u -b "${RATE}M" $extra --json > "$RESULTS_DIR/load-$name-$F.json" 2>&1 & + LOAD_PID=$! + sleep 2 + # 1000 samples: p99.9 under load is real (gated to null when short). + ping_samples_json 1000 "$RESULTS_DIR/ping-load-$name-$F.txt" > "$RESULTS_DIR/ping-load-$name-$F.lat" + wait $LOAD_PID + path_json > "$RESULTS_DIR/load-$name-$F.path1" + python3 - "$RESULTS_DIR/load-$name-$F.json" "$RESULTS_DIR/load-$name-$F.path0" "$RESULTS_DIR/load-$name-$F.path1" "$RESULTS_DIR/ping-load-$name-$F.lat" "$name" "$F" "$RATE" <<'EOF' +import json,sys,datetime,os +ipath, p0path, p1path, latpath, name, frac, rate = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], float(sys.argv[6]), float(sys.argv[7])) +valid = True +try: + d = json.load(open(ipath)) + if isinstance(d.get('error'), str): + raise ValueError(d['error']) + s = d['end']['sum'] + actual = round(s['bits_per_second']/1e6, 1); loss = round(s.get('lost_percent', -1), 2) +except Exception as e: + actual, loss = -1, -1 + valid = False + print(f" {name} {frac}: LOAD ERROR {e}", flush=True) +lat = json.load(open(latpath)) +b = json.load(open(p0path)); a = json.load(open(p1path)) +notes = [] +if b.get('mode') != a.get('mode'): + notes.append('PATH CHANGED mid-run; result flagged') +if actual > 0 and actual < rate*0.7 and frac <= 1.0: + notes.append('under-delivered load') +if not valid: + notes.append('LOAD FAILED: row invalid, values are placeholders') +note = '; '.join(notes) +print(f" actual={actual}Mbps loss={loss}% p50={lat.get('p50')} p95={lat.get('p95')} p99={lat.get('p99')} max={lat.get('max')} {note}") +row = {'scenario': 'loaded-latency', 'direction': name, 'fraction': frac, + 'offered_mbps': rate, 'actual_mbps': actual, 'loss_pct': loss, + 'latency': lat, 'path': b, 'path_after': a, 'note': note, + 'valid': valid} +row['ts'] = datetime.datetime.now(datetime.timezone.utc).isoformat() +row['product'] = os.environ['BENCH_PRODUCT'] +row['meta'] = json.loads(os.environ['BENCH_META']) +open(os.environ['BENCH_JSONL'], 'a').write(json.dumps(row) + chr(10)) +EOF + done +done + +# --- bidirectional loaded latency: full-duplex UDP at fractions --- +# Up and down loads run on SEPARATE server ports (see PORT_UP/PORT_DOWN): +# two clients against one listener conflict on a normal iperf3 server. +echo " bidir (full duplex)..." +for F in 0.25 0.50 0.75 0.90 1.00; do + RATE_UP=$(python3 -c "print(round(float('$CAP_UP')*float('$F'),1))") + RATE_DOWN=$(python3 -c "print(round(float('$CAP_DOWN')*float('$F'),1))") + PCT=$(python3 -c "print(int(float('$F')*100))") + echo " bidir ${PCT}% (up=${RATE_UP}Mbps down=${RATE_DOWN}Mbps)..." + path_json > "$RESULTS_DIR/load-bidi-$F.path0" + iperf3 -c "$PEER" -p "$PORT_UP" -t "$DURATION" -u -b "${RATE_UP}M" --json > "$RESULTS_DIR/load-bidi-$F-up.json" 2>&1 & + UP_PID=$! + iperf3 -c "$PEER" -p "$PORT_DOWN" -t "$DURATION" -u -b "${RATE_DOWN}M" -R --json > "$RESULTS_DIR/load-bidi-$F-down.json" 2>&1 & + DOWN_PID=$! + sleep 2 + ping_samples_json 1000 "$RESULTS_DIR/ping-load-bidi-$F.txt" > "$RESULTS_DIR/ping-load-bidi-$F.lat" + UP_OK=0; DOWN_OK=0 + wait $UP_PID || UP_OK=$? + wait $DOWN_PID || DOWN_OK=$? + path_json > "$RESULTS_DIR/load-bidi-$F.path1" + python3 - "$RESULTS_DIR/load-bidi-$F-up.json" "$RESULTS_DIR/load-bidi-$F-down.json" "$RESULTS_DIR/load-bidi-$F.path0" "$RESULTS_DIR/load-bidi-$F.path1" "$RESULTS_DIR/ping-load-bidi-$F.lat" "$F" "$RATE_UP" "$RATE_DOWN" "$UP_OK" "$DOWN_OK" <<'EOF' +import json,sys,datetime,os +uppath, downpath, p0path, p1path, latpath, frac, rate_up, rate_down, up_ok, down_ok = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], float(sys.argv[6]), float(sys.argv[7]), float(sys.argv[8]), int(sys.argv[9]), int(sys.argv[10])) +def loadsum(p): + try: + d = json.load(open(p)) + if isinstance(d.get('error'), str): + raise ValueError(d['error']) + s = d['end']['sum'] + return round(s['bits_per_second']/1e6, 1), round(s.get('lost_percent', -1), 2), None + except Exception as e: + return -1, -1, str(e)[:200] +actual_up, loss_up, err_up = loadsum(uppath) +actual_down, loss_down, err_down = loadsum(downpath) +if up_ok != 0: + err_up = (err_up + '; ' if err_up else '') + f'iperf up client exited {up_ok}' +if down_ok != 0: + err_down = (err_down + '; ' if err_down else '') + f'iperf down client exited {down_ok}' +lat = json.load(open(latpath)) +b = json.load(open(p0path)); a = json.load(open(p1path)) +notes = [] +if b.get('mode') != a.get('mode'): + notes.append('PATH CHANGED mid-run; result flagged') +if actual_up > 0 and actual_up < rate_up*0.7 and frac <= 1.0: + notes.append('under-delivered up load') +if actual_down > 0 and actual_down < rate_down*0.7 and frac <= 1.0: + notes.append('under-delivered down load') +# A bidirectional row is only produced when BOTH directions ran: errors +# mark the row invalid explicitly instead of hiding behind -1 values. +valid = err_up is None and err_down is None +if err_up: + notes.append(f'BIDIR INVALID: up load failed ({err_up})') +if err_down: + notes.append(f'BIDIR INVALID: down load failed ({err_down})') +note = '; '.join(notes) +print(f" up={actual_up}Mbps loss={loss_up}% down={actual_down}Mbps loss={loss_down}% p50={lat.get('p50')} p95={lat.get('p95')} p99={lat.get('p99')} p999={lat.get('p999')} valid={valid} {note}") +row = {'scenario': 'loaded-latency', 'direction': 'bidir', 'fraction': frac, + 'offered_up_mbps': rate_up, 'offered_down_mbps': rate_down, + 'actual_up_mbps': actual_up, 'actual_down_mbps': actual_down, + 'loss_up_pct': loss_up, 'loss_down_pct': loss_down, + 'latency': lat, 'path': b, 'path_after': a, 'note': note, + 'valid': valid} +row['ts'] = datetime.datetime.now(datetime.timezone.utc).isoformat() +row['product'] = os.environ['BENCH_PRODUCT'] +row['meta'] = json.loads(os.environ['BENCH_META']) +open(os.environ['BENCH_JSONL'], 'a').write(json.dumps(row) + chr(10)) +EOF +done -# 6. Latency under load (bonus) -echo "[BONUS] Latency under load (30s)..." -iperf3 -c "$PEER" -t "$DURATION" -P 4 > /dev/null 2>&1 & -IPERF_PID=$! -sleep 2 # let throughput stabilize -netperf -H "$PEER" -t TCP_RR -l $((DURATION - 4)) -- \ - -o min_latency,mean_latency,max_latency,p99_latency,stddev_latency \ - > "$RESULTS_DIR/latency-under-load.txt" 2>&1 -wait $IPERF_PID 2>/dev/null -echo " Latency under load:" -cat "$RESULTS_DIR/latency-under-load.txt" -echo "" +# --- UDP sweep: rates x sizes x directions, sender vs receiver split --- +# Sizes span single-frame (512, 900) and segmented (1200, 1460, 2700) +# dataplane behavior; both directions run. `actual_mbps` is RECEIVER-side +# delivered throughput (sum_received), never the sender offer. +echo "[4] UDP sweep (sizes x directions)..." +for dir in "up:$PORT_UP:" "down:$PORT_DOWN:-R"; do + name="${dir%%:*}"; rest="${dir#*:}"; port="${rest%%:*}"; extra="${rest#*:}" + for F in 0.25 0.50 1.00; do + RATE=$(python3 -c "print(round(float('$CAP_UP')*float('$F'),1))") + for LEN in 512 900 1200 1460 2700; do + path_json > "$RESULTS_DIR/udp.path" + # shellcheck disable=SC2086 + run_iperf "udp-$name-${RATE}M-${LEN}B" -p "$port" -u -b "${RATE}M" -l "$LEN" $extra + python3 - "$RESULTS_DIR/udp-$name-${RATE}M-${LEN}B.json" "$RESULTS_DIR/udp.path" "$RATE" "$LEN" "$name" <<'EOF' +import json,sys,datetime,os +ipath, ppath, rate, length, direction = sys.argv[1], sys.argv[2], float(sys.argv[3]), int(sys.argv[4]), sys.argv[5] +valid = True +err = '' +try: + d = json.load(open(ipath)) + if isinstance(d.get('error'), str): + raise ValueError(d['error']) + send = d['end']['sum'] + recv = d['end']['sum_received'] +except Exception as e: + valid = False + err = f'UDP FAILED: {e}'[:200] + send, recv = {}, {} +path = json.load(open(ppath)) +def num(o, *keys, default=-1): + for k in keys: + try: + v = o[k] + if v is not None: + return v + except Exception: + pass + return default +sent_mbps = round(num(send, 'bits_per_second')/1e6, 1) if valid else -1 +actual = round(num(recv, 'bits_per_second')/1e6, 1) if valid else -1 +pps_sent = round(num(send, 'packets', default=0)/float(os.environ.get('BENCH_DURATION', '10'))) if valid else -1 +pps_recv = round(num(recv, 'packets_received', 'packets', default=0)/float(os.environ.get('BENCH_DURATION', '10'))) if valid else -1 +loss = round(num(recv, 'lost_percent'), 2) if valid else -1 +jitter = round(num(recv, 'jitter_ms'), 3) if valid else -1 +note = '' if valid else err +print(f" {direction} offered={rate}Mbps sent={sent_mbps}Mbps delivered={actual}Mbps pps_sent={pps_sent} pps_recv={pps_recv} len={length}B loss={loss}% jitter={jitter}ms valid={valid} {note}") +row = {'scenario': 'udp', 'direction': direction, 'offered_mbps': rate, 'packet_len': length, + 'sent_mbps': sent_mbps, 'actual_mbps': actual, 'pps_sent': pps_sent, 'pps_received': pps_recv, + 'loss_pct': loss, 'jitter_ms': jitter, 'path': path, 'note': note, 'valid': valid} +row['ts'] = datetime.datetime.now(datetime.timezone.utc).isoformat() +row['product'] = os.environ['BENCH_PRODUCT'] +row['meta'] = json.loads(os.environ['BENCH_META']) +open(os.environ['BENCH_JSONL'], 'a').write(json.dumps(row) + chr(10)) +EOF + done + done +done -echo "=== Done. Results in $RESULTS_DIR ===" +echo "=== Done: $JSONL (shared schema) ===" diff --git a/tunnet-performance-audit.md b/tunnet-performance-audit.md new file mode 100644 index 00000000..648a959a --- /dev/null +++ b/tunnet-performance-audit.md @@ -0,0 +1,1781 @@ +# Tunnet Extreme Performance Audit + +**Repository:** `tunnetio/Tunnet` +**Audit baseline commit:** `2d20b0c1790634212df2e8ee738ad11474c2ca9e` +**Initial benchmark date:** 2026-09-03 +**Primary comparison:** Tunnet vs ZeroTier, Windows client → Linux peer +**Objective:** materially outperform ZeroTier in throughput, loaded latency, packet rate, CPU/byte, jitter, recovery, and reliability without weakening security or correctness. + +> This is a living audit. Findings are classified as **Confirmed**, **Strong hypothesis**, or **Experiment**. Do not turn experiments into defaults without measurement. + +--- + +## 1. Baseline + +### Tunnet +- Idle ICMP: 93.76 ms avg +- Loaded ICMP: 221.67 ms avg +- Loaded max: 476 ms +- TCP upload: 80.4 Mbps sender / 79.4 Mbps receiver +- TCP download: 99 Mbps +- UDP 500 Mbps offered: 0.714 ms jitter, 83.11% loss + +### ZeroTier +- Idle ICMP: 83.54 ms avg +- Loaded ICMP: 85.71 ms avg +- Loaded max: 111 ms +- TCP upload: 85.6 Mbps sender / 84.5 Mbps receiver +- TCP download: 136 Mbps +- UDP 500 Mbps offered: 0.556 ms jitter, 83.82% loss + +### Immediate interpretation +- Tunnet's major defect is not raw throughput. It is **queueing latency under saturation**. +- Tunnet adds roughly 128 ms of latency under this load while ZeroTier adds only ~2 ms. +- The 500 Mbps UDP loss test is not a valid reliability comparison. Both products are offered several times more traffic than the measured path can sustain. +- Idle RTT is also ~10 ms worse on Tunnet, but the current benchmark does not prove whether Tunnet and ZeroTier used equivalent physical/direct paths. Path state must be captured. + +--- + +# 2. Current data-plane map + +## Outbound, Windows/Linux host → peer + +`TUN` +→ `tun_io::run_outbound` +→ SSH-NAT pre-pass +→ packet parse +→ route classification / lookup +→ peer lookup +→ ACL +→ Direct firewall +→ QoS classification +→ `Bytes::copy_from_slice` +→ `OutboundScheduler` +→ per-peer class queues +→ per-peer sender task +→ `ConnPool::send_or_buffer` +→ peer-slot lookup / async mutex +→ Iroh `Connection` +→ QUIC DATAGRAM buffer +→ noq congestion control / pacing / packetization / crypto +→ noq-udp +→ OS UDP socket +→ physical network + +## Inbound, peer → local host + +physical network +→ OS UDP socket +→ noq-udp +→ noq QUIC +→ Iroh `Connection::read_datagram` +→ one per-connection inbound task +→ packet parse +→ route / endpoint checks +→ anti-spoof +→ ACL +→ Direct firewall +→ SSH-NAT check/rewrite +→ one `AsyncDevice::send().await` +→ TUN +→ host stack + +--- + +# 3. Highest-priority findings + +## F-001 — Stacked multi-megabyte queues create artificial bufferbloat +**Status:** Confirmed +**Priority:** P0 +**Impact:** Extreme + +Tunnet's application QoS scheduler can hold: +- latency: 64 packets +- normal: 256 packets +- bulk: 512 packets +- total: 832 packets + +At the default 1280-byte virtual MTU this is approximately 1.016 MiB. + +Below that, current `noq` defaults to: +- `datagram_send_buffer_size = 1 MiB` + +On Windows, tun-rs/Wintun defaults to: +- ring capacity = 2 MiB + +Not every buffer is necessarily full simultaneously, but the scale is enough to explain the benchmark. At ~80 Mbps, 1 MiB is about 105 ms of serialization time. Two 1 MiB queues are about 210 ms before considering other buffering. + +**Evidence** +- `crates/tunnet-agent/src/qos.rs` +- `crates/tunnet-core/src/iroh_pool.rs` +- `noq-proto/src/config/transport.rs` +- tun-rs Windows device/session defaults + +**Required change** +- Treat queueing budget in **time/bytes**, not packet count. +- Keep one intentionally controlled queueing point. +- Shrink Iroh/noq DATAGRAM buffering substantially for VPN traffic. +- Avoid using Wintun ring size as a latency buffer. +- Instrument sojourn time at every queue. + +--- + +## F-002 — `send_datagram_wait` reverses low-latency DATAGRAM semantics +**Status:** Confirmed +**Priority:** P0 +**Impact:** Extreme + +Current Tunnet behavior: +1. check `datagram_send_buffer_space()` +2. when zero, call `send_datagram_wait(packet).await` + +Iroh explicitly documents that `send_datagram_wait` waits during congestion and **effectively prioritizes old datagrams over new datagrams**. + +That is a poor default for an IP tunnel: +- stale TCP packets amplify inner TCP queueing +- interactive packets arriving later cannot replace stale bulk +- one sender can await a bulk packet while latency traffic is already queued + +This produces application-level HOL blocking over an intentionally unreliable/unordered QUIC DATAGRAM primitive. + +**Required change** +- Never block the class scheduler on one stale packet. +- Make freshness/drop policy explicit in Tunnet. +- Prefer bounded `send_datagram()` semantics with Tunnet-owned scheduling. +- Use current send-buffer-space information to decide whether to dequeue, not to turn a packet into a long-lived awaited future. +- Preserve per-flow ordering in Tunnet's queue design. + +--- + +## F-003 — Per-peer sender has priority inversion under transport pressure +**Status:** Confirmed +**Priority:** P0 +**Impact:** Extreme + +Three priority classes ultimately feed **one sender future per peer**. DRR priority helps only before the sender starts awaiting transport capacity. + +If a bulk packet reaches `send_datagram_wait`, later latency packets cannot preempt it. + +**Required change** +Build a non-blocking scheduler/pump: +- queue packet +- classify +- dequeue only while transport can accept useful work +- on congestion, drop/replace stale packets according to sojourn/AQM policy +- wake/retry without holding a particular bulk packet as the scheduler's awaited future + +Do not immediately solve this with separate QUIC connections per class; independent congestion controllers can create fairness and path behavior problems. + +--- + +## F-004 — Tunnet parses normal outbound packets twice +**Status:** Confirmed +**Priority:** P1 +**Impact:** High at high PPS + +`run_outbound`: +1. calls `ssh_nat::rewrite_outbound` +2. that function parses the packet +3. `run_outbound` then parses the packet again + +Inbound normal packets are also parsed for the main path and again for SSH-NAT detection; rewritten SSH packets can parse yet again. + +**Required change** +- Parse exactly once. +- Pass parsed metadata to SSH-NAT. +- Use a very cheap precondition before any SSH rewrite work. +- Reparse only if a rewrite changes fields required by later stages. + +--- + +## F-005 — Every outbound packet is copied into a new `Bytes` +**Status:** Confirmed +**Priority:** P1 +**Impact:** High + +Current path reads into a reusable `Vec`, then creates: +`Bytes::copy_from_slice(packet)`. + +That performs an allocation/copy before every outgoing QUIC DATAGRAM. + +**Phase 1 solution** +- pooled owned packet buffers +- receive TUN data directly into an owned buffer +- turn ownership into `Bytes` without the second copy +- recycle on Drop + +`Bytes::from_owner` is particularly interesting for a custom pooled owner. + +**Phase 3 experiment** +Raw Wintun receive-ring ownership via RAII + `Bytes::from_owner`, releasing the Wintun packet on final Drop. This can be true zero-copy from Wintun shared memory into the QUIC application buffer, but it must not pin receive-ring entries for long under congestion. + +--- + +## F-006 — Linux ignores tun-rs's high-performance API +**Status:** Confirmed +**Priority:** P0 +**Impact:** Potentially extreme + +tun-rs currently supports on Linux: +- `offload(true)` +- async `recv_multiple` +- async `send_multiple` +- reusable `GROTable` +- `IDEAL_BATCH_SIZE` = 128 +- multi-queue +- additional queue creation + +Tunnet currently uses one-packet `recv()` / `send()` and does not enable those features. + +**Required Linux fast path** +- enable TUN offload +- use `recv_multiple` and `send_multiple` +- preallocate batch storage +- reuse `GROTable` +- evaluate multi-queue after single-queue batching is correct +- shard flows deterministically if multiple queues are used +- preserve ordering per flow + +--- + +## F-007 — Windows uses Wintun as one-packet async I/O instead of a burst ring +**Status:** Confirmed +**Priority:** P0 +**Impact:** High + +tun-rs exposes `try_recv` and `try_send` on Windows. +Wintun itself is a shared-memory ring and recommends spinning briefly / draining under heavy load before falling back to the read-wait event. + +tun-rs's Windows async fallback can involve a blocking worker; its send fallback copies the source to a `Vec`. + +**Required Windows fast path** +- after readiness/wakeup, repeatedly `try_recv` until `WouldBlock` or batch budget +- process a bounded burst +- repeatedly `try_send` inbound packets before falling back to waiting +- benchmark spin budget vs CPU consumption +- tune Wintun ring capacity based on measured BDP/queueing, not “bigger is faster” +- do not allow ring buffering to mask queue management + +--- + +## F-008 — Fragment mutex is paid on every packet, even non-fragments +**Status:** Confirmed +**Priority:** P1 +**Impact:** High under PPS/concurrency + +Both ACL and Direct firewall have fragment tracking guarded by a `parking_lot::Mutex`. +The normal packet path calls fragment resolution unconditionally. + +**Required change** +- branch on packet fragmentation first +- non-fragment path must not acquire fragment state lock +- keep fragment tracking as a slow path +- optionally shard fragment state by flow hash + +--- + +## F-009 — Conntrack does redundant hash-table operations +**Status:** Confirmed +**Priority:** P1 + +Established-flow lookup can do: +- `contains_key(fwd)` +- `contains_key(reverse)` +- `get_mut(key)` + +in both ACL and Direct firewall state. + +**Required change** +- one `get_mut(fwd)`, then one reverse lookup only on miss +- consider canonical bidirectional flow keys +- separate hot “established” state from policy metadata +- replace global periodic retain behavior if it damages p99; evaluate expiry wheel/sharded GC + +--- + +## F-010 — Direct packet policy is structurally duplicated +**Status:** Confirmed semantic overlap; consolidation design pending +**Priority:** P1 + +Direct boot converts local firewall config into a `PolicyBundle` consumed by `AclEngine`, while `FirewallEngine` is also built from the firewall config and evaluated for every packet. +Suggested policies are later synchronized into `FirewallEngine`. + +It is not safe to simply delete one path because the engines also serve different lifecycle/admission roles, but packet-level work can be unified. + +**Required architecture** +- connection admission/auth policy at Iroh hook +- one compiled packet-policy engine for org/network/local/suggested rules +- one conntrack +- one fragment table +- one verdict +- differential tests proving exact old/new semantics + +--- + +## F-011 — Policy rules are filtered, allocated, and sorted during evaluation +**Status:** Confirmed +**Priority:** P1 +**Impact:** Very high for new/stateless flows + +The policy engine builds a `Vec<&PolicyRule>`, filters candidates and sorts by order/priority in `first_matching_in_phase`. This can happen for multiple phases. + +Some selectors also construct strings such as `format!("user:{id}")`. + +**Required change** +Compile policy at install/update time: +- pre-sort phases +- normalize selectors +- compile port ranges +- use integer IDs / enums, not formatted strings +- optional protocol/port/peer indices +- immutable `ArcSwap` snapshot +- allocation-free evaluation + +--- + +## F-012 — Routing performs multiple snapshots/lookups per packet +**Status:** Confirmed +**Priority:** P1 + +Outbound route work calls separate methods for: +- magic DNS destination +- advertised destination +- peer lookup + +These can load route state more than once. Advertised destination checking includes a linear `iter().any(...)`. + +**Required change** +One immutable snapshot + one `RouteDecision`: +- LocalMagic +- LocalAdvertised +- Peer(FastPeer) +- NoRoute + +Compile advertised prefixes into a prefix index. +Return a fast peer handle keyed by binary endpoint ID / compact PeerIndex. + +--- + +## F-013 — Established send path repeatedly hits DashMap + async Mutex +**Status:** Confirmed +**Priority:** P1 + +`ConnPool::send_or_buffer` resolves the peer slot through DashMap, clones an Arc, locks a `tokio::sync::Mutex`, checks connection state, updates activity, clones the connection, unlocks, and then sends. + +Per-packet counters and activity/path accounting add more map/atomic work. + +**Required change** +Introduce `FastPeerTx`/`PeerFastState`: +- stable per-peer object returned by routing +- live connection via atomic/ArcSwap-style pointer +- direct atomics for counters +- coarse activity update, not every packet +- reconnect/control path may still use locked state +- no DashMap lookup on established hot send + +--- + +## F-014 — Metrics registry lookup occurs per packet +**Status:** Confirmed from current metric call pattern +**Priority:** P1/P2 + +The hot path calls metric macros/counters per packet with labels. + +**Required change** +- pre-register/store metric handles +- for the hottest counters, use per-task local counters and flush periodically +- retain exact drop/error counters where needed +- never allocate labels on the data path + +--- + +## F-015 — Virtual MTU 1280 imposes high per-packet CPU cost +**Status:** Confirmed +**Priority:** P2 +**Impact:** Potentially large + +Tunnet defaults to 1280. +ZeroTier uses a much larger virtual MTU (2800) and handles fragmentation in its overlay. + +At the same useful byte rate, Tunnet's small virtual MTU forces more: +- TUN packets +- packet parses +- route lookups +- ACL/conntrack hits +- queue operations +- application DATAGRAMs +- metric increments + +Tunnet cannot simply set 2800 today because one inner IP packet maps to one QUIC DATAGRAM and is rejected when larger than Iroh's current max datagram size. + +**Required investigation** +- safe adaptive virtual MTU +- overlay segmentation/reassembly +- GSO/GRO-aware framing +- loss amplification of fragmented unreliable DATAGRAMs +- bounded reassembly memory/time +- path MTU changes +- inner TCP MSS behavior + +Do not put generic inner TCP data on reliable QUIC streams as a simple workaround; transport-over-transport HOL is unacceptable. + +--- + +## F-016 — Iroh/noq transport configuration is completely implicit +**Status:** Confirmed +**Priority:** P0/P1 + +Tunnet builds Iroh endpoints from presets but does not provide a custom `QuicTransportConfig`. + +Current noq defaults include: +- CUBIC +- 1 MiB application DATAGRAM send buffer +- datagram receive buffer tied to the default stream receive window +- initial RTT 333 ms +- initial MTU 1200 +- DPLPMTUD enabled +- GSO enabled +- ACK-frequency control disabled +- multipath disabled in base transport config + +Iroh 1.1 exposes knobs including: +- datagram send/receive buffers +- CUBIC / NewReno / experimental BBRv3 +- ACK frequency +- initial RTT +- MTU discovery +- initial/min MTU +- GSO +- multipath +- path idle/keepalive +- connection stats/congestion state + +**Required plan** +Create an explicit `TunnetTransportProfile`, versioned and benchmarked. No “magic defaults.” + +Initial experiment matrix: +- DATAGRAM send buffer: 16 / 32 / 64 / 128 / 256 KiB +- CUBIC vs experimental BBRv3; NewReno as control +- ACK-frequency threshold variants +- initial RTT based on known path estimate vs default +- initial MTU 1200 vs 1300/1400 only with robust DPLPMTUD +- direct and relay paths separately + +Do not disable GSO; it is already enabled by default. + +--- + +## F-017 — noq already optimizes UDP I/O; do not duplicate it above Iroh +**Status:** Confirmed + +Current noq/noq-udp already has: +- Linux UDP GSO +- Linux batched receive (`recvmmsg`) +- Windows GSO/GRO support +- `WSASendMsg` +- `WSARecvMsg` +- bounded endpoint receive/send iterations + +Therefore the major optimization opportunity is not “manual UDP batching” around Iroh. It is: +- feed Iroh efficiently +- avoid application queue stacks +- configure transport explicitly +- remove Tunnet hot-path overhead + +--- + +## F-018 — ZeroTier CoDel exists but is currently disabled +**Status:** Confirmed correction + +ZeroTier contains a CoDel/DRR AQM implementation with a 5 ms target and 100 ms interval, but current `Network::qosEnabled()` returns `false`. + +Therefore it is **not valid** to attribute this benchmark's excellent ZeroTier loaded latency to active CoDel. + +This strengthens the conclusion that Tunnet's load-latency defect is mostly self-inflicted queueing. + +AQM/FQ-CoDel remains a useful Tunnet option after unnecessary buffering and `send_datagram_wait` are removed. + +--- + +# 4. Benchmark defects that must be fixed + +## Current PowerShell benchmark weaknesses +- 500 Mbps UDP offered rate massively exceeds path capacity. +- under-load iperf result is discarded +- only upload load is tested +- no bidirectional load +- no load-rate sweep +- no path-state capture +- no CPU/alloc/context-switch measurements +- averages dominate; no p95/p99/p99.9 +- no repeated randomized runs + +## Benchmark v2 requirements + +### Path validation +For every run record: +- Tunnet direct / relay / selected address +- Iroh path state +- ZeroTier direct / relay +- native underlay RTT/path to the same physical peer when possible + +### Throughput +- TCP 1 / 4 / 8 streams +- upload +- download +- bidirectional +- JSON output +- actual achieved throughput +- retransmits and congestion stats + +### Loaded latency +For upload, download and bidirectional: +- 25% +- 50% +- 75% +- 90% +- 100% +- 110% of measured capacity + +Record: +- p50 +- p95 +- p99 +- p99.9 +- max +- actual throughput during latency sampling + +A product must not “win latency” by starving throughput. + +### UDP +Sweep useful offered rates instead of only 500 Mbps: +- 25/50/75/90/100/110% of path capacity +- packet sizes 64/128/256/512/~1200/max-safe +- loss +- jitter +- delivered Mbps +- delivered PPS + +### Resource efficiency +Both peers: +- total CPU +- hottest core CPU +- cycles/byte if tooling allows +- cycles/packet +- context switches +- allocations/sec +- bytes allocated/sec +- working set +- scheduler wakeups + +### Reliability / adverse network +- 0.1%, 1%, 3%, 5% loss +- reordering +- 10/50/100 ms jitter +- sudden bandwidth reduction +- NAT rebinding +- direct→relay transition +- relay→direct upgrade +- temporary network outage +- MTU black hole scenario + +--- + +# 5. Instrumentation required before aggressive tuning + +Add low-overhead histograms/counters for: + +1. TUN receive timestamp +2. parse/policy completion +3. scheduler enqueue +4. scheduler dequeue +5. scheduler sojourn time +6. transport submit +7. transport blocked/full event +8. Iroh datagram send buffer free bytes +9. remote datagram receive +10. remote policy completion +11. remote TUN write +12. queue lengths in bytes and packets +13. drops by reason/class +14. reconnect-buffer depth +15. per-peer active path +16. QUIC RTT +17. congestion window / bytes in flight if exposed +18. path MTU +19. direct vs relay status + +Use sampled or per-task accumulation where necessary; instrumentation itself must not become the bottleneck. + +--- + +# 6. Proposed architecture + +## 6.1 Packet object + +Create a `PacketBuf` / `OwnedPacket` abstraction: +- pooled storage +- immutable after classification except explicit NAT rewrite +- parsed metadata stored alongside bytes +- `Bytes` view without copying +- reusable on drop if practical + +Conceptually: + +``` +PacketBuf { + bytes, + parsed: PacketMeta, + flow_key, + class, + enqueue_ts, + peer_fast_handle, +} +``` + +Do not keep a borrowed `etherparse::SlicedPacket` across mutations. Store compact metadata. + +--- + +## 6.2 One-pass outbound pipeline + +``` +TUN batch/burst + -> minimal parse once + -> optional NAT rewrite using parsed metadata + -> one route snapshot / RouteDecision + -> one unified policy verdict + -> classify flow/class + -> enqueue into bounded freshness-aware peer scheduler + -> nonblocking transport pump +``` + +No strings, formatting, map lookups, allocations or blocking locks in the established hot path unless unavoidable. + +--- + +## 6.3 Scheduler design + +First implementation: +- per-peer +- per-flow sparse queues inside three service classes +- byte/time bounds +- DRR across active flows +- latency class strict low sojourn budget +- normal/bulk AQM (CoDel or equivalent) +- preserve order within each flow +- never await a bulk packet while a higher-priority packet exists + +Alternative simpler P0: +- retain 3 classes +- reduce queues drastically +- add enqueue timestamps +- drop stale bulk/normal +- remove `send_datagram_wait` +- pump only while transport has room + +Ship simple P0 first to prove the bufferbloat diagnosis, then FQ/AQM. + +--- + +## 6.4 OS-specific TUN engines + +### Linux +Dedicated optimized engine: +- offload +- async recv_multiple/send_multiple +- GROTable reuse +- preallocated 128-packet batch +- optional multi-queue +- flow-preserving sharding +- benchmark queue count against CPU topology + +### Windows +Dedicated Wintun engine: +- `try_recv` burst drain +- bounded spin under load +- event wait when idle +- `try_send` burst fill +- pooled packet storage +- tuned ring capacity +- later: raw-ring zero-copy experiment + +Avoid forcing a single generic abstraction to hide every performance capability if it costs throughput. Share policy/scheduler semantics, not necessarily I/O mechanics. + +--- + +# 7. Optimization phases + +## Phase 0 — prove and remove the latency catastrophe +1. Benchmark v2 minimum: capture loaded throughput and path state. +2. Add queue-sojourn and transport-buffer telemetry. +3. Replace `send_datagram_wait` in packet tunnel path. +4. Reduce app queue limits from ~1 MiB to a time/byte budget. +5. Explicitly reduce noq DATAGRAM send buffer. +6. Windows burst `try_recv` / `try_send`. +7. Linux offload + async batching. +8. Compare CUBIC vs BBRv3 experimentally. +9. Repeat exact Windows→Linux baseline. + +**Exit target** +- loaded RTT inflation < 10 ms at 90% capacity +- < 25 ms at saturation, ideally much lower +- no material throughput regression +- loss/drop reasons understood + +## Phase 1 — remove per-packet taxes +1. Parse once. +2. Remove unconditional fragment locks. +3. Consolidate packet policy/conntrack. +4. Compile policy rules at update time. +5. Single routing snapshot/decision. +6. Fast peer handle; eliminate established-path DashMap + async mutex. +7. Pre-register/cache metrics handles or batch counters. +8. pooled packet buffers; remove `Bytes::copy_from_slice`. +9. coarse activity accounting. + +**Exit target** +- substantial PPS increase +- lower CPU/byte +- download catches/exceeds ZeroTier +- p99 remains bounded + +## Phase 2 — packetization/MTU +1. Measure path MTU distribution. +2. Evaluate virtual MTU > 1280. +3. Design safe segmentation/reassembly if needed. +4. Integrate GSO/GRO semantics with overlay framing. +5. tune MSS/PMTUD behavior. +6. measure loss amplification. + +**Exit target** +- reduce inner packets per GiB materially +- throughput/CPU gains without reliability regression + +## Phase 3 — extreme platform-specific work +1. raw Wintun packet ownership / true zero-copy experiment +2. custom tun-rs upstream API if required +3. multi-core Linux queue sharding +4. CPU affinity only if profiling proves scheduler migration cost +5. specialized packet parser only if etherparse remains hot +6. SIMD/checksum specialization only if profiles justify it +7. allocator/pool tuning based on allocation profiles + +--- + +# 8. Transport experiment matrix + +Do not select by intuition. + +## Congestion control +- CUBIC baseline +- BBRv3 experimental +- NewReno control + +Measure on: +- clean 90 ms path +- 1% loss +- varying bandwidth +- relay path +- asymmetric upload/download +- cross traffic + +## DATAGRAM buffering +Test 16/32/64/128/256 KiB. +The correct value is likely related to pacing/BDP and Tunnet queue strategy, not a generic 1 MiB. + +## ACK frequency +noq/Iroh supports the extension and Iroh's own benchmark code uses a threshold of 10 in benchmark configuration. +Treat this as an experiment: +- default +- 2 +- 5 +- 10 +- loss-heavy cases + +## Initial RTT +Default is 333 ms. +Useful mainly during startup/path changes. +Experiment with a more realistic initial RTT only if it improves first-second behavior without hurting unknown networks. + +## MTU +- default initial 1200 + DPLPMTUD +- 1280/1350/1400 initial where safe +- never raise min MTU aggressively on arbitrary internet paths + +--- + +# 9. Reliability invariants + +Every optimization must preserve: + +- authenticated encrypted transport +- anti-spoof semantics +- ACL/firewall semantics +- bounded memory +- bounded reconnect buffering +- fragment security +- per-flow packet order at Tunnet's own queue +- safe NAT rebinding +- direct/relay path migration +- no blocking shutdown regressions +- no packet lifetime that can pin an OS ring indefinitely + +Testing: +- differential old/new policy evaluator +- property/fuzz packet parser and NAT rewrite +- scheduler model tests +- queue bound tests +- simulated transport backpressure +- packet-loss/reordering integration tests + +--- + +# 10. Findings to investigate next + +- Exact direct/relay path used by the supplied benchmark. +- Whether Iroh's current preset modifies multipath beyond base noq defaults. +- End-to-end connection/path stats available cheaply enough for adaptive scheduling. +- Incoming Iroh DATAGRAM drain batching opportunities. +- Cost of `pool.touch_peer` per packet and replacement with coarse activity sampling. +- Exact metrics cost under 100k/500k/1M PPS. +- Actual CPU profile Windows and Linux after P0. +- TUN queue/ring occupancy telemetry options. +- impact of Wintun ring capacity reductions. +- whether packet-policy consolidation can share state cleanly between Managed and Direct modes. +- relay performance path, including `tunnet-relay` configuration. +- asymmetric bottleneck explaining 99 Mbps download vs 136 Mbps ZeroTier. +- endpoint worker contention inside noq under many peers. +- multipath QUIC: enablement, scheduling, fairness, and loss behavior. +- path migration and multiple active physical interfaces. +- RSS/CPU-affinity interactions on Windows. +- Linux `busy_poll`, socket buffers, and UDP offload only if noq profiling says they matter. +- PGO/BOLT after architectural work; current `opt-level=3`, ThinLTO, codegen-units=1 are already reasonable. +- allocator choice only after allocations remain material. + +--- + +# 11. Things explicitly not to do yet + +- Do not blindly enlarge buffers. +- Do not use `send_datagram_wait` as a reliability mechanism. +- Do not move inner TCP to QUIC streams. +- Do not enable experimental BBRv3 as default without adverse-network testing. +- Do not raise virtual MTU above QUIC DATAGRAM limits without segmentation design. +- Do not optimize crypto before profiling proves it hot. +- Do not fork tun-rs before exhausting the current public high-performance API. +- Do not add CPU affinity before demonstrating scheduler migration/cache problems. +- Do not trust single-run throughput averages. +- Do not call a 500 Mbps offered UDP test “packet-loss reliability” on a ~100 Mbps path. + +--- + +# 12. Current top ten implementation order + +1. Make benchmark/path telemetry trustworthy. +2. Delete tunnel-path `send_datagram_wait` semantics. +3. Bound queueing by time/bytes and shrink stacked buffers. +4. Windows burst Wintun pump. +5. Linux offload + recv/send batching. +6. Explicit Tunnet Iroh/noq transport profile. +7. Parse once + pooled packet buffers. +8. Unified compiled policy/conntrack + fragment slow path. +9. Fast route/peer handle without per-packet maps/async mutex. +10. Larger-MTU/GSO-aware overlay design. + +The first six are expected to change the supplied benchmark materially. The rest are how Tunnet moves from “competitive” to “architected for extreme PPS/throughput.” + +--- + +# 13. Phase 1 implementation record (2026-09-04) + +Phase 1 executed the full data-plane rewrite: the P0 queueing/backpressure work +**and** the P1 hot-path cleanup together. The old architecture no longer +controls the data plane. All findings below were verified against the exact +resolved dependency sources (`iroh 1.1.0`, `noq/noq-proto 1.2.0`, +`tun-rs 2.8.9`, `bytes 1.12.1`). + +## 13.1 Dependency facts confirmed in source (invalidate/confirm audit assumptions) + +- **noq defaults confirmed** (`noq-proto/src/config/transport.rs`): `initial_rtt` + 333 ms, `initial_mtu` 1200, `datagram_send_buffer_size` **1 MiB**, + `datagram_receive_buffer_size` = STREAM_RWND, `ack_frequency_config: None`, + GSO enabled. F-016's premise holds exactly. +- **tun-rs 2.8.9 Linux API is NOT `recv_multiple(bufs, sizes, gro)`**. + Real signatures (`async_device/unix/mod.rs`): + `recv_multiple(&mut original_buffer, &mut bufs, &mut sizes, offset)` and + `send_multiple(&mut gro_table, &mut bufs, offset)`, Linux-only + (`#[cfg(target_os = "linux")]`), with `VIRTIO_NET_HDR_LEN + 65535` original + buffer convention from `platform/linux/offload.rs`. `Vec` implements the + (`pub`, re-exported) `ExpandBuffer` trait, so no wrapper type is needed. + `GROTable: Default`, `IDEAL_BATCH_SIZE = 128`, `offload(true)` on + `DeviceBuilder` — all as the audit assumed. +- **Windows `AsyncDevice`** exposes `try_recv` / `try_send` (plus async + `recv`/`send`); there is no `recv_multiple` on Windows. Burst drain/fill + around one async wait is the correct shape — implemented. +- **Iroh 1.1 transport knobs** (`endpoint/quic.rs`): the builder is + constructed via `QuicTransportConfig::builder()` (there is **no** + `QuicTransportConfigBuilder::default()`); all setters consume `self` + (`mut self -> Self`, must reassign). `AckFrequencyConfig` is re-exported at + `iroh::endpoint::AckFrequencyConfig` (the `quic` module itself is private) + and `ack_eliciting_threshold` takes a `VarInt`, not `u64`. + `max_concurrent_multipath_paths(0)` is **ignored with a warning** (minimum + enforced), so multipath is left at the Iroh default rather than set to 0. +- **Congestion controllers** live in `noq_proto::congestion` as + `CubicConfig` / `NewRenoConfig` / `Bbr3Config` (iroh re-exports only the + `ControllerFactory` trait, so `noq-proto` is now a direct dependency). +- **Linux `send_multiple` framing requirement (new fact, correctness-critical).** + With offload enabled the kernel expects a virtio-net header in front of + every written packet: each send buffer must carry `VIRTIO_NET_HDR_LEN` + headroom and `offset` must equal it (tun-rs's own framed writer does + exactly this; `offset = 0` underflows `offset -= VIRTIO_NET_HDR_LEN` and + fails). Plain `send()` of a raw IP packet misframes under offload, so + **all** Linux TUN writes — including inbound — go through a reused + `GROTable` + staged headroom (`LinuxTunWriter`), not just the batch + receive path. Verified natively on Linux (WSL): `cargo check`, + `cargo clippy --all-targets --all-features -- -D warnings`, and the + `tunnet-common` / `tunnet-core` / `tunnet-agent` nextest suites are green + (262 passed; the single exclusion is the loopback-dependent DNS test in + §13.6, failed only by sandbox networking). +- **F-018 stands**: ZeroTier's CoDel is disabled upstream; Tunnet's loaded + latency is self-inflicted. The new scheduler therefore targets Tunnet's own + queueing first. + +## 13.2 Architecture deleted (not preserved, no shims) + +- `qos.rs` three-class (`latency/normal/bulk`, `Class`, `classify`, DRR + quanta, `drr_round_drain` test helper) — **deleted**, replaced by + flow-aware FQ-CoDel-style scheduler in the same module path. +- `iroh_pool::send_datagram` wait branch (`send_datagram_wait` when + `datagram_send_buffer_space() == 0`) — **deleted**. The function is now + non-blocking; `try_send_datagram`/`TrySendError::{Full,TooLarge,Closed}` + is the only primitive. No `send_datagram_wait` remains on any IP-tunnel + path (verified by search). +- SSH-NAT double parse (`needs_inbound_rewrite(&[u8])`, + `rewrite_{in,out}bound(&mut [u8])`, internal `eligible`/`rewrite`) — + **deleted**. Only `*_with_meta` entry points remain; tests migrated. +- Outbound `Bytes::copy_from_slice` per packet — **deleted**; + `PacketBuf::into_bytes` (`Bytes::from(Vec)`) without a second copy. +- Outbound multi-lookup sequence (`is_magic_dns_destination` + + `is_advertised_destination` + `lookup_ip` + second endpoint lookup) — + replaced by `RoutingTable::route_once` (old methods retained only as + non-hot-path delegators, not duplicated datapaths). +- Per-packet `AclEngine::allow_packet` + `FirewallEngine::evaluate` pair in + `tun_io` — replaced by one `PacketPolicy::check`. The engines themselves + remain as configuration/admission owners (connection-level auth stays + separate by design); their fragment tables and conntracks are no longer + touched by the established hot path. +- `tun_fast::build_fast_tun` duplicate builder and `is_would_block` helper + removed during implementation; `tun_io::build_tun` is the single TUN + constructor (Linux `offload(true)` inside). + +## 13.3 New data plane (established-packet path) + +```text +PlatformTunRx (LinuxBatchEngine recv_multiple / Windows try_recv burst) + → PacketBuf { data: Vec, meta, flow, enqueued_at } # parse once + → ssh_nat::*_with_meta # metadata only + → RoutingTable::route_once → RouteDecision::Peer(handle) # 1 snapshot + → PacketPolicy::check → Allow # 1 verdict + → PeerScheduler::enqueue (flow FIFO, byte caps) # bounded + → run_peer_pump → ConnPool::try_send_fast # non-blocking + → Iroh QUIC DATAGRAM (64 KiB buffer, CUBIC, GSO on) +``` + +Inbound mirrors: `read_datagram → PacketBuf::from_slice → anti-spoof → +PacketPolicy::check → NAT-with-meta → try_send/send burst → TUN`. + +Scheduler algorithm (selected): per-peer FQ-CoDel concept — +sparse/new-flow priority (16 KiB epoch budget, 25 ms sojourn target), +byte-DRR (`FLOW_QUANTUM` 1536) across backlogged flows, per-flow FIFO order, +absolute sojourn ceiling (250 ms) with head-drop AQM, peer caps +(256 KiB / 512 packets), per-flow cap (64). Chosen because it isolates +interactive/ICMP flows from bulk without any strict-priority pipe, keeps +fairness byte-based, bounds memory/time, and — critically — never awaits a +packet: the pump stops at `TransportFull` and requeues the head. + +Transport profile (`tunnet-core/src/transport_profile.rs`, applied in +`direct/connectivity.rs::endpoint_builder` for every mesh endpoint): +DATAGRAM send 64 KiB (was 1 MiB: ~105 ms → ~6 ms serialization at 80 Mbps), +receive 256 KiB, initial RTT 90 ms (was 333 ms), initial/min MTU 1200, +GSO on, multipath untouched, CUBIC default, BBRv3/NewReno as explicit +experiments (`TunnetTransportProfile::bbr3_experiment`, +`endpoint_builder_with_transport`). + +## 13.4 Remaining per-packet costs (honest accounting) + +Outbound established: 1 TUN batch amortized syscall share, 1 etherparse +parse, 1 ArcSwap snapshot load (routing) + 1 HashMap/DashMap-free peer-handle +use (no DashMap, no endpoint-hex alloc), 1 compiled policy verdict +(0 alloc / 0 sort / 0 format; 1 DashMap `get_mut` only on conntrack miss… +actually one `get_mut` per packet for the established lookup — single +sharded read-lock, no async mutex), scheduler enqueue (2 mutexes: +flows + occasionally order; pump side same), 1 non-blocking +`send_datagram`, coarse atomics (bytes, activity ≤1/s). +Remaining copies: TUN buffer → owned `Vec` (1 copy; raw Wintun-ring +zero-copy deferred to Phase 3 as planned). Remaining allocs: owned packet +buffer (poolable — `PacketPool` exists, pump-wide adoption deferred), +scheduler `Bytes` handle (refcount, no payload copy). +Inbound established: 1 DATAGRAM read, 1 parse, anti-spoof compares, +1 policy verdict, 1 TUN send (Windows: `try_send` fast path). + +## 13.5 Benchmark v2 (`scripts/bench.ps1`, `scripts/bench.sh`) + +Rewritten per §13 requirements: throughput matrix (TCP 1/4 streams, +up/down/bidir, JSON, retransmits), loaded-latency sweep at 25/50/75/90/100/ +110% of *measured* capacity with actual throughput + loss next to +p50/p95/p99/max (under-delivery >30% flagged invalid), UDP sweep over +rates × {64,256,1200}B with delivered Mbps/pps/loss/jitter, path-state +capture before/after (Tunnet API + `zerotier-cli peers` / `ip route get`). + +## 13.6 Tests added (all passing) + +- Scheduler: sparse-jumps-bulk, per-flow order, byte-DRR no-starvation, + stale-ceiling drops, memory bounds, **ICMP-vs-TCP-bulk isolation with real + packets**, transport-full requeue order + per-peer (no global HOL) isolation. +- Policy: differential legacy-equivalence matrix (org-deny range merge, + order_index, disabled rules, TCP/UDP scoping, default-deny), first-fragment + remembers → later-fragment allowed, later-without-state denied, malformed + denied. +- Transport: profile builds, BBRv3 explicit-not-default. +- TUN: Linux batch constants, Windows burst budget relations. +- NAT: parse-once rewrite tests (migrated, old double-parse tests deleted). +- Packet: flow-key stability, zero-copy `into_bytes`. +- Validation: `cargo fmt --check` clean; `cargo check --workspace + --all-targets` clean; `cargo clippy --workspace --all-targets + --all-features -- -D warnings` clean — on Windows **and** natively on + Linux (WSL; only exclusion is the Tauri desktop system-dependency + package). `cargo test` and `cargo nextest run` green: common 67, core lib + 150, agent 77 on Windows (294 total); 218 + 76 on Linux excluding the + single sandbox-loopback DNS test proven environmental in Phase 1 + (identical signature reproduced in WSL localhost; raw-socket proof + stands). Pump tasks are epoch-owned (teardown advances the epoch; + schedulers drain and tasks exit — no cross-generation writes, no leaked + tasks); the runtime sweeper is tied to the generation token. + +## 13.7 Known bottlenecks / intentionally deferred + +- Single TUN reader + per-peer pump tasks (no multi-queue sharding yet; + flow-preserving sharding needs measurements first). +- Inbound DATAGRAM drain is one packet per `read_datagram` per connection + (Iroh API shape); cross-connection batching not yet implemented. +- `PacketPool` exists but the outbound path still allocates one `Vec` + per packet (pool acquire/release wired in the Linux engine sketch, full + pump-wide recycling pending). +- No virtual-MTU/GSO-overlay work (Phase 2), no raw Wintun zero-copy + (Phase 3), no CPU affinity, no PGO/allocator tuning. +- Policy `sync_from_engines` polls bundle pointer + firewall versions every + 256 packets (amortized); a push-based generation counter would be cleaner. +- No live-network measurements in this environment (Windows dev host, no + Linux peer pair); benchmark v2 awaits the Windows→Linux baseline rerun. +- The old `AclEngine`/`FirewallEngine` conntrack/fragment/GC code is now + off the hot path but not yet deleted; removal awaits a longer soak period + proving the unified policy in production-like traffic (candidate for the + next cleanup pass, since only the dataplane used them per-packet). + +--- + +# 14. Phase 2 implementation record (2026-09-04) — new dataplane + +Phase 2 rebuilt the packet plane around logical packets, tunnel framing, and a +shared policy runtime. All §0 verified issues were fixed first; the old +architecture was deleted, not shimmed. + +## 14.1 Verified §0 fixes + +- **§0.1 shared runtime**: `PolicyRuntime` (tunnet-core/src/policy_runtime.rs) + is owned by the dataplane generation (`CoreNode::policy`, installed at + node build) and shared by outbound + every inbound connection. Conntrack + is one canonical bidirectional table; `sync_from_engines` polling is gone. +- **§0.2 network scoping**: firewall compiles per-`NetworkId` (`FwSet`); + `from_engines` flattening and the global `enabled &&` are deleted. Fast + states carry the pre-resolved set — no per-packet UUID lookup. Proven by + `cross_network_firewall_isolation` and + `disabled_network_does_not_disable_others` tests. +- **§0.3 event-driven**: engines hold the runtime and publish on every + mutation (`replace_bundle`, `reload_local`, `set_suggested` — which now + bumps the previously-missing version, `ensure_inbound_tcp_allow`, + posture/stale changes). No packet-count polling remains. +- **§0.4 revocation**: conntrack entries carry `admitted_gen`; generation + mismatch revalidates once against current policy. Tests: TCP allow→deny, + UDP allow→deny, suggested-rule change, enabled/disabled flips. +- **§0.5 fast path**: `PeerFastState` (identity, ArcSwap connection, + FQ-CoDel state, policy link, counters, relay/MPS/RTT cache, reassembly, + pump wakeup) rides inside `PeerInfo`; `route_once` hands out the Arc with + zero map lookups. Removed from the hot path: `fast_conns`, `fast_touch_ms`, + `bytes_in/out`, `peer_cloud_relay` maps, scheduler peer map, + `try_send_fast`, `record_bytes_*`. +- **§0.6 Model A**: `try_send_frame` submits only when + `datagram_send_buffer_space() >= frame.len()` (the exact Iroh guarantee; + plain `send_datagram` otherwise displaces oldest-first). The frame is + returned on every error so stalls never consume bytes. +- **§0.7 adaptive backoff**: the fixed 5 ms sleep is replaced by + notify-or-`clamp(RTT/4, 100µs, 2ms)`. Investigated upstream: the + `datagrams_unblocked` Notify is private to noq with no public waiter; + documented in code as the desired extension. No spin, no + `send_datagram_wait` anywhere. + +## 14.2 Iroh/noq source facts (exact versions) + +- `send_datagram` → `datagrams().send(data, true)`: `true` = displace + oldest-first when full (Model B is the default — Model A must check + space first). `datagram_send_buffer_space()` guarantee is exactly + "no displacement iff new datagram <= reported space". +- `ReadDatagram::poll` drains buffered datagrams synchronously before + waiting on `datagram_received.notified()`; each `read_datagram()` mints a + fresh notify, so single-poll `now_or_never` drain probes are safe (dropping + a Pending probe only drops its waker). Used for the §10 bounded + opportunistic drain (32) with no busy polling. +- `max_datagram_size()` changes with path MTU (documented); cached per + fast state as MPS and refreshed periodically, on path events, and on + `TooLarge`. +- `Path::id()`/`stats()` (+ `PathStats.rtt`) back the RTT cache for + adaptive backoff; `AckFrequencyConfig`/`VarInt` quirks from Phase 1 stand. +- `Bytes::from_owner + Send + 'static>` confirmed; the pool + owner exposes exactly the frame bytes. + +## 14.3 Tunnel framing, segmentation, reassembly + +- Wire: `tunnet/tunnel/2` ALPN. The `/2` is only the negotiated + wire-protocol version (it keeps old `/1` raw-IP binaries from speaking + the incompatible framing protocol); no v1 implementation remains and + there is no compatibility decoder. `Single [0x20][packet]` + (1 B overhead); `Segment [0x21][id u32][index u16][count u16][total u16]` + (11 B). Kinds `0x22..=0x2F` reserved (GSO extension space); version nibble + `0x2_`. Decoder: no allocation, checked arithmetic, fail-closed bounds. +- Segmentation is incremental from the retained owner with a cursor + (id/count/total); path shrink restarts with a fresh id (never mixed + shapes); `TooLarge` refreshes MPS and restarts (bounded retries); + ≤16 DATAGRAMs per packet by construction. No QUIC streams (§23). +- Reassembly bounds: total ≤9000, count ≤16, 32 entries/peer, 256 KiB/peer, + shared global counter, 500 ms timeout, LRU eviction, conflicting-duplicate + drop, ID-collision restart. Loss of a segment loses the packet (no overlay + retransmit). Policy sees only reassembled logical packets (§5). +- Virtual MTU default 2800 (was 1280; direct-mode hardcode removed, clamp + 576–9000); all TUN slots MTU-derived (`slot_cap_for_mtu`), no fixed 2048. + 2800 chosen as the ZeroTier-parity point pending the §17 matrix (no live + network in this environment; do not treat as measured optimum). + +## 14.4 Scheduler, ownership, batching + +- FQ-CoDel (`tunnet-core/src/scheduler.rs`, pure state machine): new/old + lists (no scans), byte DRR with MPS-scaled quantum, per-flow CoDel + (`first_above_time`/`dropping`/`drop_next`/`count`, target 5 ms / + interval 100 ms, tunable via `with_params`), byte caps, 1 s emergency + ceiling as safety only, bounded cap probes, wire-byte fairness feedback. + Test-driven debugging during implementation caught and fixed three real + bugs (requeue accounting, list double-ownership, spurious Empty). +- Ownership: `PooledBuffer` (headroom + classes 512–9216 B, `from_owner` + transmit, pool recycle on Drop), `LogicalPacket::{from_pooled, + from_shared (zero-copy inbound), from_vec}`; scheduler queues logical + packets; segments encode from borrows into fresh pooled buffers. +- Linux: `recv_multiple` into pool-owned slots (swapped into packets), + genuine multi-packet `send_multiple` batches (GSO coalescing) for both the + reject path and a real inbound TUN batch; virtio headroom staging is + mandatory with offload (plain `send` misframes — §13.1 fact retained). +- Windows: pooled burst receives; `TunWriteBatch` (shared with non-Linux + fallback) retains its tail across waits — no silent loss. +- Inbound Iroh drain is burst-oriented (§10 probe above); TUN writes are + batched per drain iteration (§9). +- Routing: advertised routes in a `PrefixMap` (no linear scan), + `is_exit` computed at rebuild, fast states embedded in `PeerInfo` and + pruned on rebuild; inbound resolves fast state once per connection and + re-resolves on routing-generation change only. +- Telemetry (§15): aggregate (summed, never overwritten) queue gauges, + sojourn bucket histogram with p50/p95/p99/avg export, cached drop + handles, frame/segment/reassembly/TUN-syscall/datagram/pool counters; + the dead `hot.drops` counter is deleted. + +## 14.5 Deleted in Phase 2 (§13, §26) + +`policy_fast.rs`, `qos.rs`, `PacketBuf`, `AclEngine` packet eval + +conntrack/fragments/GC + its tests, `FirewallEngine` packet eval + +conntrack/fragments/GC + `EvalResult`/`PacketDirection`/`peer_matches`/ +`default_policy` + eval tests, pool fast maps + `try_send_fast` + +`record_bytes_*`, `send_datagram_wait` remnants (none remained), +`sync_from_engines`, `from_engines` flattening, fixed `SLOT_CAP`, fake +one-element batch wrappers, `NoopTunBatch`, dead metrics. `EvalResult` and +`PacketDirection` removals rippled only to `direct/mod.rs` re-exports. + +## 14.6 Benchmark v3 (§16) + +Shared JSONL schema (`results.jsonl`) for ps1/sh: throughput matrix (TCP +1/4, up/down/bidir with explicit bidir JSON parse — the v2 gap), per-run +path state, warmup, repeats, idle 1200-sample p99.9, loaded sweeps per +direction at 25–110% of independently measured directional capacity with +actual-Mbps/loss beside p50/p95/p99/max, migration/under-delivery flags, +UDP rate×size sweep with delivered/pps/loss/jitter, commit/MTU/OS/CPU +metadata. Fixes: ps1 bidir parse, sh capacity-regex bug (structured +functions, no reparsed human output). Not yet: adverse-network impairment +runs (§18 — no impairment harness in this environment; matrix specified in +§17/§19 with `with_send_buffer` and profile hooks ready), live results +(§17 matrix and §19 transport matrix await hardware). + +## 14.7 Validation status + +- `cargo fmt --check`, workspace check + clippy + (`--all-targets --all-features -D warnings`): clean on Windows AND native + Linux (WSL; only exclusion remains the Tauri desktop system-dep package). +- Tests: 294 green on Windows (common 67, core lib 149+24 integration?, + agent 77 — see report), 218/218 green on Linux excluding the one + sandbox-loopback DNS test proven environmental in Phase 1 (identical + signature reproduced in WSL; raw-socket proof stands). +- Fuzzing: no cargo-fuzz harness in repo; decoder/reassembly covered by + proptest suites (byte-string never-panics, header round-trip, bounded + segment streams) instead. +- GSO preservation (§11): tun-rs `recv_multiple` splits super-packets + (no public API exposes coalesced segments + `VirtioNetHdr`), so receive- + side GSO metadata cannot be preserved without a tun-rs fork — documented + blocker; the framing reserves `0x22..=0x2F` frame kinds for a future GSO-aware + extension instead of a second wire rewrite. Send-side GSO coalescing via + `send_multiple` is fully used. +- Multi-queue (§20): investigated, not enabled — needs flow-affinity + measurements first (documented, unchanged from Phase 1). + +# 15. Phase 2.1 implementation record (2026-09-04) — correctness/hardening + +Ten correctness/security/performance defects in the Phase 2 architecture, +fixed at the architectural level. No compat shims; `PeerPolicyLink` deleted, +`PeerRegistry::relink_policy` narrowed to install-time slot assignment, +`ReassemblyTable::new` no longer takes an effectively-infinite global cap. + +## 15.1 Segmentation restart tracks full geometry (pump.rs) + +- `PartialPacket` now stores the complete `SegmentPlan` (count + seg_cap + + shape), the packet id, the next index, and a wire-byte accumulator — + never just a count. `transmit_cursor` adopts the current geometry + wholesale for fresh cursors; resumed cursors continue their stored + geometry even if the path changed. +- `TooLarge` refreshes MPS and calls `replan()`, which compares the FULL + geometry: any change (count, seg_cap, or single↔segmented shape) + restarts from byte 0 with a fresh id (`Replan::Restarted`); identical + geometry retries the segment in place (`Replan::Retry`, transient); + degenerate paths drop (`Replan::Impossible`). Old offsets are never + reused with a new MPS. Restart budget (≤2) is threaded through + `transmit_segmented_budgeted` so flapping paths terminate (boxed + recursion for the restart cycle; the first implementation reset the + budget and was fixed before merge). +- Tests: `replan_restarts_on_segcap_change_with_same_count` (2800 B needs + 3 segments at both MPS 1350 and 1400 — same count, different cap — + still restarts with fresh id/offset), `replan_retries_in_place_...`, + `replan_handles_shape_transitions`. + +## 15.2 Scheduler: Empty means empty; account once (scheduler.rs, pump.rs) + +- DRR deficits accumulate immediately inside `serve_old`: a head larger + than one quantum (2800–9000 B logical vs ~1200 B MPS-scaled quantum) is + served in the same `next()` call. Long-term byte fairness is unchanged + (the full head length is still debited, so oversized heads borrow + against future rounds and wait afterwards — proven by + `oversized_head_borrows_future_rounds`). +- `next()` repeats rotation passes only after a pass dropped packets + (each pass strictly reduces queued packets or retires list entries), so + `Empty` is returned only with genuinely no schedulable work; a + `packets + flows + 1` pass bound guards the pump against hangs. The + 50 ms idle sleep can no longer fire with packets queued. +- Wire accounting: the pump accumulates wire bytes in the cursor + (preserved across TransportFull resume, reset on restart) and calls + `account_sent(flow, logical_len, total_wire_len)` exactly once at + completion. The old per-segment `account_sent(flow, 0, wire)` charged + `len` at dequeue plus full wire per segment (double charge); the + `account_sent` contract is now documented as once-per-logical. +- Tests: `large_heads_serve_without_stall` (2800/9000 B vs 512–1400 + quanta, zero Empty while queued), `wire_accounted_once_per_logical`. + +## 15.3 Firewall publication via stable slots (policy_runtime.rs, peers.rs, routing.rs, node.rs) + +- New architecture: `network → stable Arc → ArcSwap → + Arc`. Slots live on `PolicyRuntime` (shared across all + generations); `PeerFastState.policy` is now `ArcSwap`. + Publication swaps slot contents in place — live peers observe new rules + with two atomic loads, no per-packet map lookup, no relink. Counters + objects are never replaced, so stats survive republishes (the old + identical-ruleset reuse hack is deleted). +- Slot assignment: install-time `relink_policy` (narrowed to slot + assignment), every routing rebuild (`ensure_fast`), and the inbound + resolve race (`resolve_fast`), via `RoutingTable::{set_policy_runtime, + policy_slot_for}` wired in `install_policy_runtime`. Peers joining after + install and peers changing networks are covered — both were gaps before. +- Tests: `live_fast_state_observes_publication` (local rule change, + suggested rule change, enable/disable flips, established-flow allow→deny + revocation through a live fast state with zero post-install relink); + old tests that manually re-fetched sets after publish were rewritten to + slot-style observation. + +## 15.4 Atomic policy generation (policy_runtime.rs) + +- The generation now lives INSIDE the immutable `RuntimeInner` snapshot; + every publish compiles `prev.generation + 1` and performs ONE `ArcSwap` + store. `check()`/`check_with_generation()` read generation and policy + from the same snapshot load — the torn window (new policy + old + generation trusting stale conntrack) is structurally impossible. The + separate `version: Arc` is deleted. `invalidate()` publishes + a fresh generation with cleared conntrack through the same path. +- Test: `publication_is_atomic_under_concurrency` (publisher alternates + allow-all/deny-all bundles while 4 readers hammer one flow, pairing each + verdict with the generation actually used; any Allow at a deny + generation fails — would have caught the old race). + +## 15.5 Hard global reassembly cap (reassembly.rs) + +- `MAX_BYTES_GLOBAL = 4 MiB` (16 fully-loaded peers); `new()` enforces it + (the old `u64::MAX` default made the global counter telemetry-only). +- Reservation model: per-peer check first (table lock), then a global + `fetch_update` CAS reservation — impossible to exceed even with + concurrent peers racing the shared counter. After a global-pressure + eviction BOTH caps are re-checked (the old code re-checked per-peer + only). Every reservation pairs with exactly one release in `remove()`, + so the counter stays exact across complete/conflict/timeout/collision + paths. +- Tests: `caps_bound_memory` rewritten to assert hard `global <= 500` + (the old `<= 500 + 200` overshoot allowance is deleted); + `global_cap_holds_under_concurrent_peers` (8 tables × threads on one + counter, mid-race sampling, never exceeds). + +## 15.6 Reject path through normal tunnel framing (tun_io.rs) + +- Inbound `Reject` no longer sends a raw IP reply via the generic sender + (malformed tunnel traffic — peers require `0x20`/`0x21`). `send_reject_framed` + routes the reply through the normal machinery: `from_shared` (zero + copy) → scheduler enqueue (+ gauges) → `ensure_pump` (segments large + replies). Pool-less fallback (no pump possible) still frames a single + correctly (`KIND_SINGLE` prefix, MPS check). Outbound (TUN-side) + `send_reject_reply` is unchanged — raw IP is correct for the device. + +## 15.7 Linux outbound zero-copy restored (tun_fast.rs, owned.rs, ssh_nat.rs, tun_io.rs) + +- `needs_outbound_rewrite_with_meta` gates materialization: + `handle_outbound_one` only takes the mutable path when metadata proves + an SSH-NAT rewrite is required. Common packets stay immutable. +- `LinuxBatchEngine` slots are now `BatchSlot(PooledBuffer)`: + `recv_multiple` writes at offset 0 of the headroomed receive area (new + `PooledBuffer::recv_area_mut`; tun-rs `B: AsMut` bound verified against + tun-rs 2.8.9 source), and receipt moves the slot wholesale into + `LogicalPacket::from_pooled` — pool ownership AND 32 B headroom intact, + pool recycling on drop (the old `into_vec` detached without recycling). +- Remaining copies, Linux outbound, common packet: ZERO (pooled TUN slot + → headroom single-frame prepend → `from_owner` → QUIC). Copies remain + only where inherent or rare: one copy per segment into pooled staging + (QUIC needs contiguous frames), SSH-NAT rewrites (in-place on already- + pooled storage, no extra copy), pool-miss allocation (amortized), + shared-owner staging (reject replies synthesized off-pool). + +## 15.8 Frame ownership on QUIC errors (peers.rs) + +- `try_send_frame` clones the frame (one `Bytes` refcount bump) before + `send_datagram`, which consumes its argument without returning it on + error. The late-`Closed` path now returns the original frame — the + return-on-every-path contract is real. Test asserts byte-identical + return on `NoConnection`. + +## 15.9 Membership-removal revocation (peers.rs, routing.rs, iroh_pool.rs, tun_io.rs) + +- `PeerFastState::deactivate()`: epoch bump (pumps drain+exit, readers + observe), live QUIC conn dropped AND closed (`membership_removed`), + pump woken for prompt exit. Idempotent. `PeerRegistry::{remove, retain, + clear}` deactivate before forgetting (rebuild pruning included). +- `ConnPool::drop_peer` now closes the slot conn explicitly (was leaked + to idle timeout) and deactivates before removing. +- Inbound reader: tracks the fast-state epoch; a generation change that + no longer resolves closes the connection and exits (was: kept + forwarding through stale state); deactivation without a generation + change also exits. +- Tests: `removal_deactivates_fast_state` (registry-level epoch/conn/ + resolve), `removed_peer_is_unroutable_and_deactivated` (routing-level: + replace → delta-remove → version bump → `NoRoute` → resolve None → + epoch+1). + +## 15.10 Benchmark v3 fixes (bench.sh, bench.ps1) + +- sh: `BENCH_*` exports moved before the first consumer (were after the + idle block); `path_json` is product-aware (Tunnet API only for tunnet + runs; zerotier-cli peer summary otherwise); loaded ping 200→1000 + samples for real p99.9 (null-gate retained); NEW full-duplex bidir + loaded-latency scenario (simultaneous up + down UDP loads, both + parsed, per-direction under-delivery flags). +- ps1: download load job now honors `-R` (was silently sending upload + load); NEW bidir loaded-latency scenario (two jobs, both parsed); + p99.9 null-gate retained (200 Test-Connection samples stay honest). +- Both scripts syntax-verified (`bash -n`, PowerShell parser, zero + errors). No live runs yet — still awaiting hardware (§17–§19). + +## 15.11 Validation status (Phase 2.1) + +- `cargo fmt --check`, `cargo check --workspace --all-targets`, + `cargo clippy --workspace --all-targets --all-features -- -D warnings`: + clean on Windows AND native Linux (WSL; desktop excluded for the + system `glib` dep). Feature-combo checks also clean + (`tunnet --no-default-features --features managed`, + `tunnet-core` no-default/direct-only) — the combo that caught the + Phase-2 `HashMap` import gap. +- `cargo test --workspace` (minus desktop): all suites ok, zero + failures. `cargo nextest run --workspace`: 391 passed, 1 skipped on + Windows. Linux: 305 green (common 67 + core 158 + agent 79/80) minus + the one sandbox-loopback DNS test proven environmental in Phase 1. +- Counts: 305 focused tests (was 294; +11 new, none weakened). + +# 16. Phase 2.2 implementation record — multi-network isolation, atomic publication, remaining hardening + +Pre-v1 breaking pass. ALPN bumped to `tunnet/tunnel/3` with new frame +kinds (`0x30`/`0x31`); no compatibility with the undisclosed `/2` +framing is maintained. `PeerFastState` is gone (renamed/split into +`PeerMembershipState` + `PeerTransportState`); `PeerPolicyLink` stays +deleted; `frame::decode` is now `decode_frame`. + +## 16.1 P0 — transport vs membership state model (peers.rs, routing.rs, pump.rs, tun_io.rs, iroh_pool.rs) + +- `PeerTransportState` (key `EndpointId`): live QUIC connection, MPS/RTT, + transport counters, relay flag, endpoint-shared frame-ID counter. No + network identity, no firewall state, no scheduler. +- `PeerMembershipState` (key `(EndpointId, NetworkId)`): network identity + (mesh IP, hostname/tags), stable firewall slot, per-membership FQ-CoDel + scheduler + reassembly table, pump task + wakeup, membership epoch. +- `PeerRegistry` holds both maps. `ensure_membership` refreshes identity + in place and debug-asserts key/network agreement (no last-writer-wins); + bare `get(endpoint)` returns a membership only when exactly one exists + and otherwise refuses to guess. `remove_membership` deactivates one + membership without touching the shared transport; `remove_transport` / + `retain` (now keyed by `(EndpointId, NetworkId)` pairs) deactivate and — + for fully departed endpoints — close the QUIC connection. +- Routing `ensure_fast` assigns slots per rebuild; `PeerInfo.fast` is the + membership Arc, so outbound routes resolve each network's own state. + `lookup_membership(hex, net)` is the exact inbound resolve path; + `by_endpoint` (first-joined) remains for legacy single-membership + consumers only. Connection pool mirrors into transports + (`set_transport_conn`, `refresh_transport_path`); `drop_peer` closes and + removes the whole endpoint. +- Pumps are per membership (`ensure_pump` unchanged shape); the pump + processes DRR bursts packet by packet with a `pending` cursor queue, + requeues unstarted remainders in order on stall (with gauge re-credit), + requeues intact cursor packets on `Wait` (closes a pre-existing rare + loss), and requeues in-flight packets before epoch-exit clearing so + gauge reconciliation stays exact. + +## 16.2 P0 — framing network discriminator + ALPN/3 (frame.rs, lib.rs) + +- Every frame carries the full 16-byte `NetworkId`: `Single + [0x30][net][packet]` (17 B overhead), `Segment + [0x31][net][id][index][count][total][payload]` (27 B). Full-ID-over- + channel-ID was deliberate: no negotiation handshake, no per-connection + channel tables, unambiguous binding for out-of-order segments; 16 + B/frame is the documented price. Old `0x20`/`0x21` kinds decode as + `UnknownKind` (fail fast, never misparse). `SegmentHeader` itself is + unchanged, so reassembly keys are untouched. +- Outbound frames bind the route's membership network captured at dequeue + (`PartialPacket.net`); `strip_single_prefix` accounts for the 17-byte + header on requeue paths. All frame tests updated to the layout plus new + `old_wire_kinds_rejected` and net-participates-in-encoding assertions. + +## 16.3 P0 — authenticated network binding on inbound (tun_io.rs, accept.rs, dgram_pump.rs) + +- The reader decodes each frame header first and resolves/switches the + cached `(endpoint, network)` membership per packet; a frame claiming a + network with no membership is dropped (`unknown_network`), never + evaluated under another network's state. Mid-batch revocation drops the + cache and re-resolves; a generation change with zero memberships left + closes the connection and exits. +- `resolve_membership` enforces `auth.contains_network(endpoint, net)` + whenever an `AuthCache` is present (wired through `InboundDeps.auth` + from the accept path; the dialer-side pump passes `None`, where + membership existence still gates). `DirectAuthHook` deliberately stays + any-network: it admits CONNECTIONS (which may legitimately carry many + networks), while packet authorization is per frame network in the + reader. Admission still closes truly unknown endpoints at handshake. +- Antispoof, policy slot, and reassembly all run against the resolved + per-packet membership (membership IP/slot/table). + +## 16.4 P0 — network-scoped conntrack (policy_runtime.rs) + +- `CanonKey` now starts with `NetworkId` (from the membership network + passed to `check`; `None` maps to nil, isolating tests). Identical + 5-tuples in A and B are independent entries — proven by + `conntrack_is_network_scoped` (len 1→2, B-deny revokes B while A keeps + working) and by the strengthened `cross_network_firewall_isolation`, + which now reuses the SAME endpoint and 5-tuple in both networks. + +## 16.5 P0 — atomic firewall publication (policy_runtime.rs) + +- `FwSlot → ArcSwap`. One unified + `publication` token is bumped per publish and stamped on the ACL + snapshot and every touched firewall snapshot. Publish order is + slot-swap-then-ACL-store; packets load ACL-then-firewall. By SeqCst + ordering, observing the new ACL generation implies observing the new + firewall snapshot — the (new ACL, old firewall) poison pair from the + brief is structurally impossible (reversing either order would + reintroduce it; both orders are documented at the code sites). +- `check()` takes the slot (not a bare set) so the ordered pair of loads + lives in one place. Conntrack admission stamps exactly the deciding + pair (`admitted_acl_gen`, `admitted_fw_gen`); ANY mismatch revalidates. +- Publisher-vs-publisher serialization: `publish_acl`, + `publish_firewall`, and `invalidate` each run as one transaction under a + shared `publish_lock` (load → allocate generation → compile → swap + slots → store). Generation allocation happens inside the lock, so + committed generations are strictly monotonic and no publish can clobber + a concurrent one (previously a newer generation could be overwritten by + an older one, and an ACL publish could lose a concurrent firewall + update compiled from stale `fw_source`). The lock is control-path only; + the packet hot path never takes it. All three publishers return the + committed generation. +- Tests: `firewall_publication_is_atomic_under_concurrency` hammers one + established flow while alternating local/suggested/disabled-deny + publishes — Deny stamped at a deny fw_gen, Allow at an allow fw_gen, + both asserted deterministically. `concurrent_publishers_lose_no_updates` + runs 4+4 simultaneous publishers (unique content per publish): final + generation is exactly 1+P, committed generations are exactly 2..=1+P + with no overwrite, final snapshots contain each dimension's + last-committed content, reader samples never regress, and post-churn + revalidation re-admits. Mutation-checked: with the lock removed the + test fails (observed generation 7→5 regression). + +## 16.6 Multi-network test suite (§2.2-1, tests 1–10) + +1+2+10 `same_endpoint_two_networks_isolated` (+ reverse-order variant): +one endpoint, two networks/IPs → distinct membership objects, shared +transport, exact per-network resolution, ambiguous bare resolve refuses, +insertion-order independent. 3+4 +`same_endpoint_two_networks_route_to_distinct_memberships` (routing): +outbound routes resolve each network's own membership/IP/slot path. +5+6 `resolve_binds_exact_membership_and_auth` (agent): cross-network +frames resolve only their own membership; A-only auth rejects B claims; +unknown networks reject without any cache. 7 `conntrack_is_network_scoped` +(see 16.4). 8+9 `removing_one_membership_leaves_sibling` (epoch, +transport, resolvability of B untouched) and +`removed_peer_is_unroutable_and_deactivated` (routing-level NoRoute + +deactivate). All use the SAME EndpointId across networks. + +## 16.7 HIGH — real byte-DRR fairness (scheduler.rs, pump.rs) + +- Replaced the 2.1 within-visit multi-quantum grant (which degenerated to + packet fairness: every call served every affordable-after-k-rounds head) + with proper DRR: one quantum per flow per visit; each visit serves every + affordable head as a `DequeueBurst`; unaffordable heads rotate; rounds + repeat immediately (bounded by `MAX_DRR_ROUNDS = 64`, derived from the + ~19 KB worst deficit gap at minimum quantum) with no `Empty` and no + sleep. Burst bytes are naturally bounded (≤ quantum + one max head). + Sparse flows still serve one packet (interactive latency). +- The pump transmits bursts packet by packet (§16.1). `Empty` still means + genuinely empty. +- Measured ratios (continuously backlogged, 1200 quantum unless noted): + 9000-vs-100 ≈ 0.94 (packet-count fairness would give ≈ 90); + 2800-vs-100, 9000-vs-1200 (q1400), 2800-vs-1200 (q512) all within + 0.6–1.7; three mixed flows split ≈ 1/3 each (asserted 0.22–0.45). +- Debugging this caught two test-design traps (documented in test + comments): blind refills overflow the peer byte cap so the jumbo flow + hogs queue space, and emptied-then-recreated flows get stuck in sparse + priority — fairness tests now top up to bounded depths that fit the + caps together. Production sparse starvation is bounded by the 16 KB + new-flow budget (transient by design). +- `oversized_head_borrows_future_rounds` replaced by the ratio tests + above (it only proved eventual progress, per review item 7). + +## 16.8 HIGH — reassembly Drop guard (reassembly.rs) + +- `impl Drop for ReassemblyTable` releases `self.bytes` from the shared + counter (entries cleared with it). Invariant restored: `global_bytes == + sum(bytes of live tables)` after operations complete. Tests: + `drop_releases_global_reservation` (partial insert → drop → 0) plus a + 25-round create/fill/drop churn stress asserting zero every round. + +## 16.9 HIGH — synthetic-route invalidation (routing.rs) + +- `rebuild` now clears `dynamic_synth` first (before computing the live + set, so stale entries can't keep peers alive through `retain`). DNS + regenerates on demand. The old `dynamic_synth_survives_rebuild` test + asserted the removed behavior and was replaced by: + `dynamic_synth_cleared_on_rebuild` (unrelated rebuild clears; DNS + re-resolves), `synth_peer_removal_revokes_and_deactivates` (NoRoute + + epoch advance), `synth_network_removal_revokes` (route + DNS gone). + +## 16.10 Benchmark v3 cleanup (bench.sh, bench.ps1) + +- Separate configurable server ports (`$6`/`$7`, `-ServerPortUp`/ + `-ServerPortDown`, defaults 5201/5202) used by every directional + invocation; bidir runs up/down loads on different ports (the + same-listener conflict is gone). +- Failed iperf runs (JSON `error`, unparsable output, nonzero client + exit) mark rows `valid=false` with explicit `BIDIR INVALID` / `LOAD + FAILED` notes instead of hiding behind -1 placeholders; bidir rows + additionally print validity. +- PowerShell loaded latency stays at 200 Test-Connection samples with + `p999=null` BY DESIGN — documented in the header and the sweep banner + (1000+ samples would take minutes per fraction; Bash uses 1000 fast + pings for real p99.9). Both scripts syntax-verified; no live runs yet + (awaiting hardware, unchanged). + +## 16.11 Validation status (Phase 2.2) + +- `cargo fmt --check`, `cargo check --workspace --all-targets`, + `cargo clippy --workspace --all-targets --all-features -- -D warnings`: + clean on Windows AND native Linux (WSL; desktop excluded for the + system `glib` dep). Feature-combo checks clean + (`tunnet --no-default-features --features managed`, + `tunnet-core` direct-only). +- `cargo test --workspace` (minus desktop): all suites ok, zero + failures. `cargo nextest run --workspace`: 406 passed, 1 skipped on + Windows (was 405; +1 publisher-serialization test, none weakened). + Linux: 320 green (common 68 + core 170 + agent 82, minus the + proven-environmental DNS test). +- DRR fairness measured (see 16.7); scheduler/frame/policy proptests all + green under the new layout. + +# 17. Incident record (2026-09-04) — total connectivity loss, two Linux TUN boundary bugs + +Symptom: 100% ping loss between a Windows and a Linux machine (even +32-byte ICMP, both directions), builds since `be4ffc7`. Both daemons +reported `data plane up`, control connected, peers online — the health +readout masked a dead packet plane (see 17.5). + +## 17.1 Bug A — Linux TUN receive capacity contract (fixed in 3031604) + +`LinuxBatchEngine` slots (`BatchSlot(PooledBuffer)`) implemented +`AsRef<[u8]>` as the live packet view (length 0 for fresh slots), but +tun-rs `recv_multiple` uses `as_ref().len()` as RECEIVE CAPACITY +(`device.rs` non-GSO overflow check, `gso_split` output-buffer check) +and writes into `as_mut()[offset..]`. Every `recv_batch` therefore +failed deterministically from boot +(`read len … overflows bufs element len 0`), `run_outbound` propagated +with `?`, `spawn_outbound` logged `outbound TUN loop exited`, and Linux +never transmitted a single packet. Diagnosis signature on the Linux box: +`outbound TUN loop exited` with the overflow error. + +## 17.2 Bug B — Linux TUN write misframing (fixed here, was still in main) + +`LinuxTunBatchWriter::push()` staged packets in `PooledBuffer` (32-byte +tunnel-frame headroom), producing `[32B headroom][12B virtio zeros][IP]`, +but `flush()` passed `VIRTIO_NET_HDR_LEN` (12) as the packet offset. +tun-rs reads the IP packet at `buf[offset..]` and encodes virtio into +`buf[offset-12..offset]` (`offload.rs handle_gro`, verified in source): +the kernel received 32 zero bytes instead of IPv4 (`0x45`) and dropped +every packet silently (`flush` still returned Ok). Windows→Linux +requests never reached the Linux stack; Linux→Windows replies died the +same way. Either bug alone explains 100% bidirectional loss; both were +present. + +Fix: the writer now owns dedicated reusable `Vec>` staging with +exactly `[12B zeros][IP packet]` (`push` resizes + extends; `flush` +passes offset 12; buffers cleared and retained, no per-packet alloc +after warmup). Tunnel-frame buffers and TUN-offload buffers are +separated by construction — no abstraction can mix the two headrooms +again. Dead fallout removed: `release_raw`, `PooledBuffer::into_vec`, +`storage_mut` (all unused after the split). + +## 17.3 Receive abstraction cleanup + +`PooledBuffer::{recv_area, recv_area_mut}` now expose exactly the same +region and length; `BatchSlot::{AsRef, AsMut}` both return it (tun-rs +validates capacity against `AsRef::len()` and writes into `AsMut` — +divergent views fail or misframe). `prepare()` debug-asserts capacity +coverage. Same-round-trip coverage as before (pool ownership + headroom +intact for zero-copy single transmit). + +## 17.4 Reader ownership + ingress generation (hardening) + +- `ConnPool::adopt` no longer fires the dialer tunnel hook: whoever + adopts a connection already owns reading it (accept path spawns its + own reader; inbound readers adopt their own conn). Previously every + accepted connection briefly had TWO readers splitting datagrams (the + hook-spawned one ate packets before being aborted). +- Dial tie-break loss no longer refires the hook on the kept connection + (it already has a reader — refiring spawned a persistent duplicate). + Hook fires only for genuinely new dialed connections. +- `IngressRegistry` entries are now `(generation, handle)`; exit cleanup + removes only its own generation (`remove_if`). Previously a normally- + exiting old reader could unregister a live replacement, leaving the + peer readerless in the registry and breaking the next abort. Test + `stale_reader_exit_keeps_new_registration` fails deterministically on + the old code. +- Known follow-up (documented, not fixed here): a membership re-added + while its transport connection is live but readerless gets outbound + (pump) but no inbound reader until the connection flaps. Rare, + pre-existing, needs a reader-respawn trigger on re-add. + +## 17.5 Honest health reporting + +`DataPlaneStatusSnapshot` now tracks `outbound_alive`, `restarting`, +`restart_count`, `generation`, `last_error` with a `state()` of +Up/Degraded/Restarting/Down. `OutboundExited` publishes +restarting+error+count BEFORE supervision restarts (previously the crash +loop reported `data plane up` between restarts); bring-up success clears +to Up; intentional shutdown reports Down. Unit test covers all +transitions. + +## 17.6 Build identity in status + +`tunnet-common/build.rs` bakes `GIT_HASH`; `tunnet status` shows +`build cli · daemon · ` plus dataplane detail, and +warns loudly on CLI/daemon hash mismatch (the stale-daemon trap: fresh +`cargo run --bin tunnet` CLI against an old service binary, with `v0.9.1` +on both). `NodeSummary` carries `daemon_git`, `tunnel_alpn`, +`data_plane` (all optional for old-daemon compat). + +## 17.7 v3 reject framing leftover + +`send_reject_framed`'s pool-less fallback built `[0x30][reply]` without +the mandatory 16-byte network — now `[0x30][net][reply]` (stale `0x20` +doc reference fixed too). + +## 17.8 Tests added + +- `batch_slot_satisfies_tun_rs_contract` (all platforms): replicates + tun-rs's exact checks + simulated kernel write + pool/headroom + round-trip, fresh and recycled slots. Mutation-checked. +- `tun_batch_writer_stages_virtio_layout` (Linux): locks + `[12B zeros][IP]` staging layout without a device. +- `tun_kernel_round_trip` (Linux, `#[ignore]`, needs `CAP_NET_ADMIN`): + real writer→kernel→reply→engine round trip through a temporary + offload TUN. Run with `sudo -E` on a Linux dev machine; the missing + coverage class that let both boundary bugs ship green. +- `loopback_ping_round_trip` (existing, still green): same-binary core + pipeline guard. +- Ingress generation + snapshot health unit tests (both + mutation-checked where applicable). + +Validation after fix: `fmt --check`, workspace `check`, workspace +`clippy --all-targets --all-features -D warnings` clean Windows + +Linux; `nextest --workspace`: 410 passed, 1 skipped (Windows); +Linux 325 green (minus the proven-environmental DNS test). + + +# 18. Under-load diagnosis pass (2026-09-04) — scheduler visibility, bench honesty, knobs + +Context: after §17 the black hole was gone (ping 4/4), but benchmarks showed Linux→Windows collapsing under sustained load (86–93% UDP loss, ~2–4 Mbps delivered ceiling) while Windows→Linux did 55 Mbps at 0% loss, plus an all-ERROR TCP matrix. This pass makes every drop visible, stops the benchmark from lying, and adds diagnostic knobs — before any tuning guesses. + +## 18.1 Silent scheduler eviction now reported (scheduler.rs, tun_io.rs, pump.rs, metrics.rs) + +- `enqueue` returned `None` after evicting the flow's stalest head at `FLOW_PACKET_CAP=64`: a real loss mechanism invisible to gauges and telemetry (only `drops_cap` moved, consumed by tests alone). +- New `EnqueueOutcome::{Accepted, AcceptedEvicted{reason, evicted_len}, Rejected{reason}}`: every shed/evicted packet is reported with its reason, and evictions carry the victim length so gauges reconcile exactly (previously the victim's +1/+len leaked). Both agent enqueue sites reconcile gauges and report `dropped_inc` + `sched_drop_inc` for evictions/rejections. +- `drain_drops()` surfaces CoDel/emergency drops (which happen inside `next()` with no enqueue decision site): drained by the pump every iteration and after every agent enqueue; deltas partition across lock holders so the sum stays exact, never double-counted. New `sched_drops_add(codel, emergency)` counted sink. +- Tests: `eviction_reports_victim_for_gauge_reconcile`, `drain_drops_reports_codel_then_quiet` (plus exact-once semantics), `memory_bounds_enforced_without_scan` rewritten to the reporting model (packet conservation: retained + rejected + evicted == offered). + +## 18.2 Diagnostic A/B knobs (env, documented, CI must leave them unset) + +- TUNNET_FLOW_PACKET_CAP (default 64): per-flow cap override (64 vs 256 runs). Read in PeerScheduler::new; set_flow_packet_cap for programmatic use. +- TUNNET_TUN_OFFLOAD=0: disables tun-rs offload+GSO (plain TUN I/O). Safe both ways: the writer layout works with and without vnet, and +ecv_multiple degrades to the single-packet path. +- TUNNET_QUIC_DATAGRAM_BUFFER_KB (default 64): QUIC DATAGRAM staging (64/128/256 runs), clamped to [4 KiB, 1 MiB]. +- TUNNET_PUMP_BACKOFF_MAX_US (default 2000): transport-full backoff ceiling (OnceLock, stall path only). +- With these + the §18.1 counters (sched_flow_cap, sched_peer_bytes/packets, sched_codel, sched_emergency, ransport_full), a loaded run attributes every missing packet to exactly one cause instead of guessing. + +## 18.3 Eager preconnect (actors/dataplane.rs) + +- On bring-up with keep-alive, dial every known routed peer concurrently (semaphore 8, skip-when-full, errors ignored — the pump still dials on demand). Kills the classic first-ping-timeout: the first real packet no longer pays connection setup. Skipped entirely without keep-alive. + +## 18.4 bench.ps1 honesty rework + +- Invoke-IperfJson returns {ok, json, exitCode, error}: command, exit code, stdout, stderr file, and JSON parse status travel with every invocation. No more `2>$null` + bare `ERROR`. +- Warmup checks the exit code (fails loudly if no server listens). +- The 50.0 capacity fallback is deleted: TCP matrix failure now STOPS the benchmark (`exit 1`) instead of building all sweeps on an invented number. +- UDP reports offered / sent_mbps / actual_mbps(=receiver sum_received) / pps_sent / pps_received / loss / jitter with defensive field access (iperf JSON field names vary by version; a missing receiver summary invalidates the row rather than promoting the sender offer — the exact `actual=50Mbps loss=92%` misread). +- UDP sweep is now sizes (512/900/1200/1460/2700: single vs segmented separation) x directions (up + -R down). +- Loaded-latency jobs write result files + exit markers; crashes, bad exits, and parse failures mark `valid=false` with the cause in the note. Bidir rows require BOTH directions; 200-sample p999=null stays documented-by-design. Parser-verified. + +## 18.5 bench.sh parity + +- Warmup exit-checked; TCP capacity failure stops (same rule, no invented 50); UDP uses sum_received with sender/receiver split, pps both sides, `valid` flag, and error capture; UDP sweep is sizes x directions like ps1. `bash -n clean. + +## 18.6 What the numbers say so far (unverified hypotheses, for the A/B) + +- Plateau shape (~2–4 Mbps delivered regardless of offer) fits a sender-side ceiling: 64-packet flow cap with silent head-eviction + 64 KiB QUIC staging + backoff sleeps. The §18.1 counters will confirm or refute on the next loaded run — no tuning applied yet. +- TCP matrix all-ERROR is still unexplained (likely server-side: no iperf3 listener on the peer, or control-channel blocking); the new error capture will name it on the next run. + +Validation: fmt/check/clippy clean both platforms; `nextest --workspace` 412 passed Windows; Linux 327 green; both bench scripts syntax-verified.