diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 4cb10a2d..9fa1e65e 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -34,6 +34,7 @@ interning = [] wasm = ["dep:wasmtime"] z3 = [] leapfrog = [] # experimental: route flat conjunctive exec bodies to the WCO leapfrog join +weighted_select = [] no_search = [] grounding = [] specialize_io = [] diff --git a/kernel/resources/weighted_select.mm2 b/kernel/resources/weighted_select.mm2 new file mode 100644 index 00000000..8b98f94e --- /dev/null +++ b/kernel/resources/weighted_select.mm2 @@ -0,0 +1,9 @@ +; Weighted selection as a grounded sink (build with --features weighted_select). +; Each (w ) fact contributes an item and a signed weight; the +; (wselect ) sink accumulates them and writes the item at +; cumulative-weight (deterministic inverse-CDF) back as (wselected ). +; Here a:3 b:5 c:2 give blocks a[0,3) b[3,8) c[8,10); offset 6 selects b. +(w a 3) +(w b 5) +(w c 2) +(exec 0 (, (w $item $weight)) (O (wselect 6 $item $weight))) diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index f02de111..dc0b48d6 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -15,3 +15,4 @@ mod pure; pub use sinks::WriteResourceRequest; pub use sources::ResourceRequest; +pub mod weighted_paths; diff --git a/kernel/src/sinks.rs b/kernel/src/sinks.rs index 4c6bacbf..879087f6 100644 --- a/kernel/src/sinks.rs +++ b/kernel/src/sinks.rs @@ -1243,7 +1243,66 @@ impl Sink for Z3Sink { } +// (wselect ): a grounded weighted-selection sink. Each +// matched fact contributes (item, weight); the weights accumulate into a signed +// WeightedPathIndex (PR #101), and at finalize the item at cumulative-weight +// (deterministic inverse-CDF selection) is written back as a +// (wselected ) atom, which ordinary queries then read. This wires #101's +// select_by_offset primitive instead of leaving it as an un-wired island, modeled +// on the write-back sinks; nothing is reimplemented. and are +// decimal-digit symbols, parsed the way SumSink parses its numbers. Soundness of +// the selection partition is modeled in Alloy (fac24_weighted_select). +#[cfg(feature = "weighted_select")] +static WSELECTED_PREFIX: [u8; 11] = [ + item_byte(Tag::Arity(2)), item_byte(Tag::SymbolSize(9)), + b'w', b's', b'e', b'l', b'e', b'c', b't', b'e', b'd', +]; + +#[cfg(feature = "weighted_select")] +pub struct WeightedSelectSink { + e: Expr, + offset: u64, + index: crate::weighted_paths::WeightedPathIndex, +} + +#[cfg(feature = "weighted_select")] +impl Sink for WeightedSelectSink { + fn new(e: Expr) -> Self { + // (wselect ): the offset is a fixed decimal symbol. + let offset = destruct!(e, ("wselect" {offset_s: &str} {_item: Expr} {_weight: Expr}), { + offset_s.parse::().unwrap_or(0) + }, _err => { 0 }); + WeightedSelectSink { e, offset, index: crate::weighted_paths::WeightedPathIndex::new() } + } + fn request(&self) -> impl Iterator { + std::iter::once(WriteResourceRequest::BTM(&WSELECTED_PREFIX[..])) + } + fn sink<'w, 'a, 'k, It : Iterator>>(&mut self, _it: It, path: &[u8]) where 'a : 'w, 'k : 'w { + let e = unsafe { Expr { ptr: path.as_ptr().cast_mut() } }; + destruct!(e, ("wselect" {_offset_s: &str} {item: Expr} {weight_s: &str}), { + if let Ok(weight) = weight_s.parse::() { + let item_bytes = unsafe { item.span().as_ref().unwrap() }; + // accumulate: an item matched multiple times sums its weight (set semantics on the key) + self.index.apply_delta(item_bytes, weight); + } + }, _err => {}); + } + fn finalize<'w, 'a, 'k, It : Iterator>>(&mut self, mut it: It) -> bool where 'a : 'w, 'k : 'w { + let WriteResource::BTM(wz) = it.next().unwrap() else { unreachable!() }; + wz.reset(); + let mut changed = false; + if let Some(item) = self.index.select_by_offset(self.offset) { + wz.move_to_path(&item); + changed |= wz.set_val(()).is_none(); + } + wz.reset(); + changed + } +} + pub enum ASink { AddSink(AddSink), RemoveSink(RemoveSink), HeadSink(HeadTailSink), TailSink(HeadTailSink), CountSink(CountSink), HashSink(HashSink), SumSink(SumSink), AndSink(AndSink), ACTSink(ACTSink), + #[cfg(feature = "weighted_select")] + WeightedSelectSink(WeightedSelectSink), #[cfg(feature = "wasm")] WASMSink(WASMSink), #[cfg(feature = "grounding")] @@ -1326,6 +1385,12 @@ impl Sink for ASink { return ASink::Z3Sink(Z3Sink::new(e)); #[cfg(not(feature = "z3"))] panic!("MORK was not built with the z3 feature, yet trying to call {:?}", e); + } else if unsafe { *e.ptr == item_byte(Tag::Arity(4)) && *e.ptr.offset(1) == item_byte(Tag::SymbolSize(7)) && + *e.ptr.offset(2) == b'w' && *e.ptr.offset(3) == b's' && *e.ptr.offset(4) == b'e' && *e.ptr.offset(5) == b'l' && *e.ptr.offset(6) == b'e' && *e.ptr.offset(7) == b'c' && *e.ptr.offset(8) == b't' } { + #[cfg(feature = "weighted_select")] + return ASink::WeightedSelectSink(WeightedSelectSink::new(e)); + #[cfg(not(feature = "weighted_select"))] + panic!("MORK was not built with the weighted_select feature, yet trying to call {:?}", e); } else { panic!("unrecognized sink") } @@ -1356,6 +1421,8 @@ impl Sink for ASink { ASink::FMinSink(s) => { for i in s.request().into_iter() { yield i } } ASink::FMaxSink(s) => { for i in s.request().into_iter() { yield i } } ASink::FProdSink(s) => { for i in s.request().into_iter() { yield i } } + #[cfg(feature = "weighted_select")] + ASink::WeightedSelectSink(s) => { for i in s.request().into_iter() { yield i } } } } } @@ -1383,6 +1450,8 @@ impl Sink for ASink { ASink::FMinSink(s) => { s.sink(it, path) } ASink::FMaxSink(s) => { s.sink(it, path) } ASink::FProdSink(s) => { s.sink(it, path) } + #[cfg(feature = "weighted_select")] + ASink::WeightedSelectSink(s) => { s.sink(it, path) } } } @@ -1410,6 +1479,37 @@ impl Sink for ASink { ASink::FMinSink(s) => { s.finalize(it) } ASink::FMaxSink(s) => { s.finalize(it) } ASink::FProdSink(s) => { s.finalize(it) } + #[cfg(feature = "weighted_select")] + ASink::WeightedSelectSink(s) => { s.finalize(it) } } } } + +#[cfg(all(test, feature = "weighted_select"))] +mod weighted_select_tests { + use crate::space::Space; + fn run(program: &[u8]) -> String { + let mut s = Space::new(); + s.add_all_sexpr(program).unwrap(); + s.metta_calculus(100); + let mut out = Vec::new(); + s.dump_all_sexpr(&mut out).unwrap(); + String::from_utf8_lossy(&out).into_owned() + } + // weights a:3, b:5, c:2 in PathMap order a cumulative blocks a[0,3) b[3,8) c[8,10). + // The (O ...) template functor routes (wselect ...) through the sink dispatch. + #[test] + fn wselect_picks_item_at_cumulative_offset() { + assert!(run(b"(w a 3)\n(w b 5)\n(w c 2)\n(exec 0 (, (w $i $wt)) (O (wselect 4 $i $wt)))\n") + .contains("(wselected b)"), "offset 4 in [3,8) -> b"); + assert!(run(b"(w a 3)\n(w b 5)\n(w c 2)\n(exec 0 (, (w $i $wt)) (O (wselect 0 $i $wt)))\n") + .contains("(wselected a)"), "offset 0 in [0,3) -> a"); + assert!(run(b"(w a 3)\n(w b 5)\n(w c 2)\n(exec 0 (, (w $i $wt)) (O (wselect 9 $i $wt)))\n") + .contains("(wselected c)"), "offset 9 in [8,10) -> c"); + } + #[test] + fn wselect_out_of_range_selects_nothing() { + assert!(!run(b"(w a 3)\n(w b 5)\n(exec 0 (, (w $i $wt)) (O (wselect 99 $i $wt)))\n") + .contains("(wselected"), "offset 99 >= total 8 -> None"); + } +} diff --git a/kernel/src/weighted_paths.rs b/kernel/src/weighted_paths.rs new file mode 100644 index 00000000..1a7aa1b1 --- /dev/null +++ b/kernel/src/weighted_paths.rs @@ -0,0 +1,374 @@ +use std::collections::BTreeMap; + +use pathmap::morphisms::Catamorphism; +use pathmap::zipper::{Zipper, ZipperAbsolutePath, ZipperIteration, ZipperValues}; +use pathmap::PathMap; + +/// Derived weighted index over encoded MORK paths. +/// +/// This keeps weights outside the authoritative `PathMap<()>` atom store. It is +/// intended as the safe version of the `ws` experiment from the iCog fork: a +/// future sink can maintain this sidecar without changing byte-path semantics. +#[derive(Clone, Debug, Default)] +pub struct WeightedPathIndex { + weights: PathMap, + total_positive_weight: u64, + updates: usize, +} + +/// Read-only counters for a [`WeightedPathIndex`]. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct WeightedPathStats { + /// Number of retained non-zero weighted paths. + pub entries: usize, + /// Number of retained paths with positive sampling weight. + pub positive_entries: usize, + /// Number of retained paths with zero-or-negative signed weight. + pub non_positive_entries: usize, + /// Sum of positive weights visible to weighted selection. + pub total_positive_weight: u64, + /// Number of explicit set/delta operations applied to this sidecar. + pub updates: usize, +} + +/// Aggregate positive-weight snapshot for structural descent. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct WeightedSelectionTree { + total_positive_weight: u64, + nodes: BTreeMap, WeightedSelectionNode>, +} + +/// Read-only counters for a [`WeightedSelectionTree`]. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct WeightedSelectionTreeStats { + /// Structural trie positions retained in the aggregate snapshot. + pub nodes: usize, + /// Child edges retained across all aggregate nodes. + pub child_edges: usize, + /// Nodes with a positive value at the exact node path. + pub positive_value_nodes: usize, + /// Sum of positive weights visible to weighted selection. + pub total_positive_weight: u64, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct WeightedSelectionNode { + self_weight: u64, + children: Box<[(u8, u64)]>, +} + +impl WeightedPathIndex { + /// Creates an empty weighted sidecar. + pub fn new() -> Self { + Self::default() + } + + /// Returns the signed weight stored for `path`, or zero when absent. + pub fn weight(&self, path: &[u8]) -> i64 { + self.weights.get_val_at(path).copied().unwrap_or(0) + } + + /// Returns the total positive weight used by [`select_by_offset`](Self::select_by_offset). + pub fn total_positive_weight(&self) -> u64 { + self.total_positive_weight + } + + /// Sets the signed weight for `path`. + /// + /// Zero removes the sidecar entry. Negative values are retained as signed + /// maintenance state, but are ignored by weighted selection. + pub fn set_weight(&mut self, path: &[u8], weight: i64) { + let previous = self.weight(path); + self.update_total(previous, weight); + + if weight == 0 { + self.weights.remove(path); + } else { + self.weights.insert(path, weight); + } + + self.updates += 1; + } + + /// Adds `delta` to the signed weight for `path`. + /// + /// The addition saturates at the `i64` bounds so malformed or adversarial + /// updates cannot overflow debug builds. + pub fn apply_delta(&mut self, path: &[u8], delta: i64) { + let previous = self.weight(path); + self.set_weight(path, previous.saturating_add(delta)); + } + + /// Selects the path containing `offset` in cumulative positive-weight order. + /// + /// `offset` is zero-based and must be smaller than + /// [`total_positive_weight`](Self::total_positive_weight). Paths are visited + /// in the `PathMap` value iteration order, which is deterministic for a + /// fixed set of encoded paths. + pub fn select_by_offset(&self, offset: u64) -> Option> { + if offset >= self.total_positive_weight { + return None; + } + + let mut remaining = offset; + let mut zipper = self.weights.read_zipper(); + + if let Some(path) = select_here(&zipper, &mut remaining) { + return Some(path); + } + + while zipper.to_next_val() { + if let Some(path) = select_here(&zipper, &mut remaining) { + return Some(path); + } + } + + None + } + + /// Builds a subtree-aggregate snapshot for repeated weighted selections. + /// + /// This is the sidecar-safe version of the iCog `btm_i32_ws_test` branch's + /// weighted traversal idea: aggregate weights live outside the authoritative + /// atom `PathMap<()>`, and selection can descend by child totals rather than + /// scanning every weighted value for every sample. + pub fn selection_tree(&self) -> WeightedSelectionTree { + WeightedSelectionTree::from_weights(&self.weights) + } + + /// Selects through a freshly built aggregate snapshot. + /// + /// Prefer [`selection_tree`](Self::selection_tree) when drawing several + /// samples from the same weights. + pub fn select_by_offset_tree(&self, offset: u64) -> Option> { + self.selection_tree().select_by_offset(offset) + } + + /// Returns sidecar counters without exposing the retained path data. + pub fn stats(&self) -> WeightedPathStats { + let mut stats = WeightedPathStats { + total_positive_weight: self.total_positive_weight, + updates: self.updates, + ..WeightedPathStats::default() + }; + + // PathMap dropped `for_each_value`; `iter` is the surviving whole-map walk and yields + // the same values. + for (_, &weight) in self.weights.iter() { + stats.entries += 1; + if weight > 0 { + stats.positive_entries += 1; + } else { + stats.non_positive_entries += 1; + } + } + + stats + } + + fn update_total(&mut self, previous: i64, next: i64) { + let previous_positive = positive_weight(previous); + let next_positive = positive_weight(next); + + if next_positive >= previous_positive { + self.total_positive_weight = self + .total_positive_weight + .saturating_add(next_positive - previous_positive); + } else { + self.total_positive_weight -= previous_positive - next_positive; + } + } +} + +impl WeightedSelectionTree { + fn from_weights(weights: &PathMap) -> Self { + let mut nodes = BTreeMap::new(); + let total_positive_weight = weights.read_zipper().into_cata_side_effect( + |mask, children: &mut [u64], value, path| { + let self_weight = value.copied().map(positive_weight).unwrap_or(0); + let mut total_weight = self_weight; + let children = mask + .iter() + .zip(children.iter().copied()) + .map(|(byte, child_total)| { + total_weight = total_weight.saturating_add(child_total); + (byte, child_total) + }) + .collect::>() + .into_boxed_slice(); + + nodes.insert( + path.to_vec(), + WeightedSelectionNode { + self_weight, + children, + }, + ); + + total_weight + }, + ); + + Self { + total_positive_weight, + nodes, + } + } + + /// Returns the total positive weight represented by this snapshot. + pub fn total_positive_weight(&self) -> u64 { + self.total_positive_weight + } + + /// Selects the path containing `offset` in cumulative positive-weight order + /// by descending subtree aggregates. + pub fn select_by_offset(&self, offset: u64) -> Option> { + if offset >= self.total_positive_weight { + return None; + } + + let mut remaining = offset; + let mut path = Vec::new(); + + loop { + let node = self.nodes.get(path.as_slice())?; + if remaining < node.self_weight { + return Some(path); + } + remaining -= node.self_weight; + + let mut descended = false; + for &(byte, child_total) in node.children.iter() { + if child_total == 0 { + continue; + } + if remaining < child_total { + path.push(byte); + descended = true; + break; + } + remaining -= child_total; + } + + if !descended { + return None; + } + } + } + + /// Returns aggregate snapshot counters. + pub fn stats(&self) -> WeightedSelectionTreeStats { + WeightedSelectionTreeStats { + nodes: self.nodes.len(), + child_edges: self.nodes.values().map(|node| node.children.len()).sum(), + positive_value_nodes: self + .nodes + .values() + .filter(|node| node.self_weight > 0) + .count(), + total_positive_weight: self.total_positive_weight, + } + } +} + +fn positive_weight(weight: i64) -> u64 { + if weight > 0 { + weight as u64 + } else { + 0 + } +} + +fn select_here(zipper: &Z, remaining: &mut u64) -> Option> +where + Z: Zipper + ZipperAbsolutePath + ZipperValues, +{ + let weight = positive_weight(*zipper.val()?); + if weight == 0 { + return None; + } + + if *remaining < weight { + return Some(zipper.path().to_vec()); + } + + *remaining -= weight; + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn select_by_offset_returns_paths_by_positive_weight_ranges() { + let mut index = WeightedPathIndex::new(); + index.set_weight(b"foo", 2); + index.set_weight(b"bar", 1); + index.set_weight(b"zap", 3); + + assert_eq!(index.total_positive_weight(), 6); + assert_eq!(index.select_by_offset(0).as_deref(), Some(&b"bar"[..])); + assert_eq!(index.select_by_offset(1).as_deref(), Some(&b"foo"[..])); + assert_eq!(index.select_by_offset(2).as_deref(), Some(&b"foo"[..])); + assert_eq!(index.select_by_offset(3).as_deref(), Some(&b"zap"[..])); + assert_eq!(index.select_by_offset(5).as_deref(), Some(&b"zap"[..])); + assert_eq!(index.select_by_offset(6), None); + } + + #[test] + fn apply_delta_removes_zero_weight_entries_and_updates_total() { + let mut index = WeightedPathIndex::new(); + index.apply_delta(b"foo", 5); + index.apply_delta(b"foo", -2); + index.apply_delta(b"foo", -3); + + assert_eq!(index.weight(b"foo"), 0); + assert_eq!(index.total_positive_weight(), 0); + assert_eq!(index.stats().entries, 0); + } + + #[test] + fn negative_weights_are_retained_but_not_selected() { + let mut index = WeightedPathIndex::new(); + index.set_weight(b"cold", -4); + index.set_weight(b"hot", 2); + + assert_eq!(index.weight(b"cold"), -4); + assert_eq!(index.select_by_offset(0).as_deref(), Some(&b"hot"[..])); + + let stats = index.stats(); + assert_eq!(stats.entries, 2); + assert_eq!(stats.positive_entries, 1); + assert_eq!(stats.non_positive_entries, 1); + assert_eq!(stats.total_positive_weight, 2); + } + + #[test] + fn selection_tree_matches_linear_selection_with_prefix_values() { + let mut index = WeightedPathIndex::new(); + index.set_weight(b"a", 2); + index.set_weight(b"ab", 3); + index.set_weight(b"ac", 1); + index.set_weight(b"b", -10); + index.set_weight(b"bd", 4); + + let tree = index.selection_tree(); + + assert_eq!(tree.total_positive_weight(), index.total_positive_weight()); + for offset in 0..index.total_positive_weight() { + assert_eq!( + tree.select_by_offset(offset), + index.select_by_offset(offset), + "offset {offset}", + ); + } + assert_eq!(tree.select_by_offset(index.total_positive_weight()), None); + + let stats = tree.stats(); + assert_eq!(stats.positive_value_nodes, 4); + assert_eq!(stats.total_positive_weight, 10); + assert!(stats.nodes >= stats.positive_value_nodes); + assert!(stats.child_edges >= 4); + } +} diff --git a/kernel/tests/public_api.rs b/kernel/tests/public_api.rs new file mode 100644 index 00000000..c8621c58 --- /dev/null +++ b/kernel/tests/public_api.rs @@ -0,0 +1,41 @@ +use mork::weighted_paths::{WeightedPathIndex, WeightedPathStats, WeightedSelectionTreeStats}; + +#[test] +fn weighted_path_api_exposes_sidecar_and_stats() { + let mut index = WeightedPathIndex::new(); + + index.set_weight(b"public", 2); + + let stats: WeightedPathStats = index.stats(); + assert_eq!(index.weight(b"missing"), 0); + assert_eq!(index.total_positive_weight(), 2); + assert_eq!(index.select_by_offset(0).as_deref(), Some(&b"public"[..])); + assert_eq!( + stats, + WeightedPathStats { + entries: 1, + positive_entries: 1, + non_positive_entries: 0, + total_positive_weight: 2, + updates: 1, + } + ); +} + +#[test] +fn weighted_path_api_exposes_selection_tree() { + let mut index = WeightedPathIndex::new(); + + index.set_weight(b"a", 2); + index.set_weight(b"ab", 3); + + let tree = index.selection_tree(); + let stats: WeightedSelectionTreeStats = tree.stats(); + + assert_eq!(tree.total_positive_weight(), 5); + assert_eq!(tree.select_by_offset(0).as_deref(), Some(&b"a"[..])); + assert_eq!(tree.select_by_offset(2).as_deref(), Some(&b"ab"[..])); + assert_eq!(index.select_by_offset_tree(4).as_deref(), Some(&b"ab"[..])); + assert_eq!(stats.total_positive_weight, 5); + assert_eq!(stats.positive_value_nodes, 2); +} diff --git a/kernel/tests/weighted_paths.rs b/kernel/tests/weighted_paths.rs new file mode 100644 index 00000000..8b838bc3 --- /dev/null +++ b/kernel/tests/weighted_paths.rs @@ -0,0 +1,26 @@ +use mork::weighted_paths::{WeightedPathIndex, WeightedPathStats}; + +#[test] +fn weighted_path_index_keeps_weights_outside_authoritative_atom_store() { + let mut index = WeightedPathIndex::new(); + + index.apply_delta(b"(foo 1)", 4); + index.apply_delta(b"(bar 1)", 1); + index.apply_delta(b"(foo 1)", -1); + + assert_eq!(index.weight(b"(foo 1)"), 3); + assert_eq!(index.total_positive_weight(), 4); + assert_eq!(index.select_by_offset(0).as_deref(), Some(&b"(bar 1)"[..])); + assert_eq!(index.select_by_offset(1).as_deref(), Some(&b"(foo 1)"[..])); + + assert_eq!( + index.stats(), + WeightedPathStats { + entries: 2, + positive_entries: 2, + non_positive_entries: 0, + total_positive_weight: 4, + updates: 3, + } + ); +}