fix(search): keep bound preparation and LUCK refinement within the deadline

crates/ournotes-search/src/handler.rs

@@ -164,7 +164,15 @@ pub(crate) fn compile_execution(
             Err(reason) => (None, Some(reason.to_string())),
         }
     } else if matches!(strategy, Strategy::BranchAndBound) {
-        match crate::search::joint::JointBounds::compile(pool, request, &domain, metric, event_input, simulation) {
+        match crate::search::joint::JointBounds::compile(
+            pool,
+            request,
+            &domain,
+            metric,
+            event_input,
+            simulation,
+            crate::search::budget::SearchBudget::unlimited(),
+        ) {
             Ok(bound) => {
                 // A played Live whose payoff steps with the local score is first ranked by deck under its score cap:
                 // a combined metric when every term does.

crates/ournotes-search/src/search/budget.rs

@@ -20,6 +20,11 @@ impl SearchBudget {
         Ok(Self { deadline })
     }
 
+    /// No deadline: preparation without a request time limit (diagnostics, fixed evaluation).
+    pub(crate) fn unlimited() -> Self {
+        Self { deadline: None }
+    }
+
     pub(crate) fn deadline(self) -> Option<Instant> {
         self.deadline
     }

crates/ournotes-search/src/search/certified_engine.rs

@@ -197,8 +197,29 @@ pub(super) struct CertifiedState<'m> {
     /// The lowest point tier no performance order of a team of the domain reaches in expected score, when proved:
     /// the bounds compiled from then on drop the tiers from it on (`JointBounds::cap_scores`).
     pub(super) tier_cap: Option<i64>,
+    /// The longest unit of each kind of exact refinement this request has run, in [`RefinementUnit`] kinds. A unit
+    /// that cannot be interrupted is not started with less time left than its kind's longest one, so a refinement the
+    /// deadline cuts short runs past it by at most one such unit.
+    unit_ms: [f64; RefinementUnit::COUNT],
 }
 
+/// The exact refinement work that runs to its end once started.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(in crate::search) enum RefinementUnit {
+    /// The residue law of every order's rank bonuses ([`ournotes_sim::live::full::luck_exact_score_refined`]).
+    Residues,
+    /// The power-twin comparison of two candidates ([`ournotes_sim::live::full::luck_exact_orders_power_twin`]).
+    Twin,
+}
+
+impl RefinementUnit {
+    const COUNT: usize = 2;
+}
+
+/// Milliseconds a unit of exact refinement is assumed to take before this request has run one of its kind: a
+/// conservative prior for a chart of ordinary length. A unit observed to take longer raises its kind's estimate.
+const REFINEMENT_UNIT_PRIOR_MS: f64 = 100.0;
+
 /// Key bytes the request's lottery-curve cache may hold.
 pub(super) const LUCK_CURVE_CACHE_BYTES: usize = 32 * 1024 * 1024;
 
@@ -227,6 +248,7 @@ impl CertifiedState<'_> {
             floor: None,
             order_score_seen: f64::NEG_INFINITY,
             tier_cap: None,
+            unit_ms: [0.0; RefinementUnit::COUNT],
         })
     }
     /// The denominator of the payoff numerators of the frontier's grid: a mean over the performance orders is a sum
@@ -464,11 +486,37 @@ impl Engine<'_, '_> {
         self.refine_joint_orders()
     }
 
+    /// The longest unit of this kind so far, or the conservative prior before the first one.
+    fn unit_estimate_ms(&self, unit: RefinementUnit) -> f64 {
+        let observed = self.certified.as_ref().map_or(0.0, |state| state.unit_ms[unit as usize]);
+        if observed > 0.0 { observed } else { REFINEMENT_UNIT_PRIOR_MS }
+    }
+
+    /// Whether a unit of `unit` that runs to its end cannot pass the deadline: false once the request has stopped,
+    /// and false when less time is left than the longest such unit took so far. A stopped request starts nothing.
+    fn unit_fits(&self, unit: RefinementUnit) -> bool {
+        if self.stop.is_some() {
+            return false;
+        }
+        let Some(deadline) = self.budget.deadline() else { return true };
+        let left = deadline.saturating_duration_since(crate::search::budget::now()).as_secs_f64() * 1000.0;
+        left >= self.unit_estimate_ms(unit)
+    }
+
+    /// Records how long a unit of `unit` took, for [`Self::unit_fits`].
+    fn observe_unit(&mut self, unit: RefinementUnit, ms: f64) {
+        if let Some(state) = self.certified.as_mut() {
+            state.unit_ms[unit as usize] = state.unit_ms[unit as usize].max(ms);
+        }
+    }
+
     /// Every installed order is a complete nominal law; declined/partial trees keep their old bounds.
     fn refine_certified_orders(&mut self) -> Result<(), Error> {
         use ournotes_sim::live::full::{LuckExactBudget, LuckExactDecline, LuckExactSession};
         let state = self.certified.as_mut().expect("certified request");
         state.domain_exhausted = true;
+        // The provider stops an order's law at the deadline; a declined attempt is no certificate, so the frontier
+        // keeps its enclosures.
         let mut work = LuckExactBudget::default();
         let mut attempted = std::collections::BTreeSet::<(u64, usize)>::new();
         let mut materialized = std::collections::BTreeSet::new();
@@ -778,6 +826,10 @@ impl Engine<'_, '_> {
                 if self.expired() {
                     return Ok(());
                 }
+                // The comparison of two candidates is not interrupted once begun.
+                if !self.unit_fits(RefinementUnit::Twin) {
+                    return Ok(());
+                }
                 let Some(setup) = a.4.gekisou.as_ref() else { return Ok(()) };
                 self.tel.lottery_refinement.power_pairs += 1;
                 let mut curves = std::mem::take(&mut self.certified.as_mut().expect("certified request").luck_curves);
@@ -800,6 +852,7 @@ impl Engine<'_, '_> {
                     b.1,
                 );
                 self.tel.lottery_refinement.twin_ms += elapsed_ms(started);
+                self.observe_unit(RefinementUnit::Twin, elapsed_ms(started));
                 self.rec.clock.lap(resume);
                 self.tel.caches.luck_curves.record(curves.stats());
                 let state = self.certified.as_mut().expect("certified request");
@@ -836,6 +889,10 @@ impl Engine<'_, '_> {
             if self.expired() {
                 return Ok(());
             }
+            // One order's residue law runs to its end once begun.
+            if !self.unit_fits(RefinementUnit::Residues) {
+                return Ok(());
+            }
             let performers = order.map(|slot| input.performers[slot].clone());
             let (_, resume) = self.rec.clock.lap(slot::SIMULATION);
             let started = crate::search::budget::now();
@@ -852,6 +909,7 @@ impl Engine<'_, '_> {
                 input.rank_confirmations.as_deref(),
             );
             self.tel.lottery_refinement.residue_ms += elapsed_ms(started);
+            self.observe_unit(RefinementUnit::Residues, elapsed_ms(started));
             self.rec.clock.lap(resume);
             let score = match score {
                 Ok(score) if score.wide_notes == 0 => score,

crates/ournotes-search/src/search/diagnostics.rs

@@ -463,6 +463,7 @@ pub fn luck_fine_order_sums(
         &spec.simulation,
         &laws,
         None,
+        crate::search::budget::SearchBudget::unlimited(),
     )?;
     let positions: Vec<_> = uniform::all_orders().iter().map(uniform::positions_of).collect();
     let mut scratch = super::snaps::JointScratch::default();

crates/ournotes-search/src/search/domain_laws.rs

@@ -608,6 +608,7 @@ impl Engine<'_, '_> {
             || !self.metric.weighs_laws()
             || self.simulation.music_length_ms.is_some()
             || self.simulation.score_music_length_ms.is_some()
+            || self.expired()
         {
             return None;
         }
@@ -652,7 +653,11 @@ impl Engine<'_, '_> {
         domain: &CandidateDomain,
         clusters: bool,
     ) -> Option<JointBounds> {
+        if self.expired() {
+            return None;
+        }
         let pool = self.pool;
+        let budget = self.budget;
         let mut kinds = self.take_luck_kinds();
         let mut curves = std::mem::take(&mut self.certified.as_mut().expect("certified request").luck_curves);
         let mut stats = self.tel.environment.bounds.domain_laws.take().unwrap_or_default();
@@ -672,7 +677,7 @@ impl Engine<'_, '_> {
         let mut bounds = laws.as_ref().map_err(Clone::clone).and_then(|laws| {
             let (request, metric, event_input, simulation) =
                 (self.request, self.metric, self.event_input, self.simulation);
-            JointBounds::compile_with_laws(pool, request, domain, metric, event_input, simulation, laws, memo)
+            JointBounds::compile_with_laws(pool, request, domain, metric, event_input, simulation, laws, memo, budget)
                 .map_err(|error| format!("the weighed bounds do not compile: {error}"))
         });
         stats.compile_ms += started.elapsed().as_secs_f64() * 1000.0;
@@ -708,15 +713,26 @@ impl Engine<'_, '_> {
         let (prefixes, teams) = multiset_places(multisets, &cluster)?;
         let (pool, request, metric, event_input, simulation) =
             (self.pool, self.request, self.metric, self.event_input, self.simulation);
+        let budget = self.budget;
         let mut bounds = Vec::with_capacity(sets.len());
         for set in &sets {
             // each cluster's bounds compile as one step of preparation; none begins after the deadline
             if self.expired() {
                 return None;
             }
             bounds.push(
-                JointBounds::compile_with_laws(pool, request, domain, metric, event_input, simulation, set, None)
-                    .ok()?,
+                JointBounds::compile_with_laws(
+                    pool,
+                    request,
+                    domain,
+                    metric,
+                    event_input,
+                    simulation,
+                    set,
+                    None,
+                    budget,
+                )
+                .ok()?,
             );
         }
         let mut kinds = vec![Box::<[u32]>::default(); pool.members.len()];

crates/ournotes-search/src/search/joint.rs

@@ -14,7 +14,6 @@ use super::{
     tables::Tables,
 };
 use crate::{
-    clock::Instant,
     domain::CandidateDomain,
     types::{Metric, SimulationInput},
 };

crates/ournotes-search/src/search/joint/compile.rs

@@ -55,8 +55,9 @@ impl JointBounds {
         metric: &Metric,
         input: Option<&EventPayoffInput>,
         simulation: &SimulationInput,
+        budget: SearchBudget,
     ) -> Result<Self, Error> {
-        Self::compile_with_laws(pool, request, domain, metric, input, simulation, &[], None)
+        Self::compile_with_laws(pool, request, domain, metric, input, simulation, &[], None, budget)
     }
 
     /// [`JointBounds::compile`] with the certified lottery laws of every team of the domain in every order: the
@@ -65,6 +66,8 @@ impl JointBounds {
     ///
     /// A weighted bound caps the expected score of an order, which a certified LUCK live ranks its orders by
     /// ([`PointBound::ranked`]). The luck weights are read from `memo` when it holds those of the same laws and notes.
+    /// `budget` is the request's: preparation that would run past its deadline fails as unavailable instead of
+    /// starting another step, and the caller then searches without these bounds.
     #[allow(clippy::too_many_arguments)]
     pub(crate) fn compile_with_laws(
         pool: &Pool,
@@ -75,7 +78,11 @@ impl JointBounds {
         simulation: &SimulationInput,
         laws: &[std::sync::Arc<ournotes_sim::live::full::LuckDpCertifiedResult>],
         memo: Option<&crate::search::snaps::WeightsMemo>,
+        budget: SearchBudget,
     ) -> Result<Self, Error> {
+        if budget.expired() {
+            return Err(unavailable("the request deadline was reached before the bounds compiled"));
+        }
         if !domain.is_feasible() {
             return Err(unavailable("empty legal domain"));
         }
@@ -108,8 +115,8 @@ impl JointBounds {
             return Err(unavailable("joint preparation exceeds bounded table capacity"));
         }
         let (song, event, _) = crate::search::objective_song(pool, &request.objective)?;
-        let budget = SearchBudget::new(Instant::now(), None)?;
-        let t = Tables::new(pool, song, event, domain.snaps(), budget)?.expect("unlimited preparation");
+        let t = Tables::new(pool, song, event, domain.snaps(), budget)?
+            .ok_or_else(|| unavailable("the request deadline was reached while the power tables built"))?;
         let mut allowed = vec![false; n];
         for &m in domain.members() {
             allowed[m] = true;
@@ -133,6 +140,9 @@ impl JointBounds {
         }
         let setup =
             crate::search::full_setup(pool, &request.objective)?.ok_or_else(|| unavailable("missing Live setup"))?;
+        if budget.expired() {
+            return Err(unavailable("the request deadline was reached while the bounds compiled"));
+        }
         let envelope = SnapLive::new(pool, &t, &allowed, &setup, laws, memo)?;
         let nonnegative_scores = envelope.nonnegative_scores();
         let (a0, global, eps, order_gains) = envelope.joint_envelope();
@@ -302,12 +312,18 @@ impl JointBounds {
             root_keys: None,
         };
         compiled.compile_tables(pool, domain);
+        if budget.expired() {
+            return Err(unavailable("the request deadline was reached while the bounds compiled"));
+        }
         if !compiled.gekisou {
             compiled.composition = composition::CompositionTables::compile(&compiled, pool, domain);
             if let Some(tables) = lambda::LambdaTables::compile(&compiled, pool, domain) {
                 compiled.modules.push(Box::new(tables));
             }
         }
+        if budget.expired() {
+            return Err(unavailable("the request deadline was reached while the bounds compiled"));
+        }
         if let Some((levels, carrier)) = levels {
             let at = levels
                 .into_iter()

crates/ournotes-search/src/search/leaf.rs

@@ -115,11 +115,21 @@ impl Engine<'_, '_> {
         let (_, resume) = self.rec.clock.lap(slot::FINE);
         let state = self.certified.as_mut().expect("certified request");
         let tier_cap = state.tier_cap;
+        let bounded = self.budget;
         let bounds = state.law_bounds.get(domain, laws, |laws| {
             *compiled += 1;
-            let mut bounds =
-                JointBounds::compile_with_laws(pool, request, domain, metric, event_input, simulation, laws, None)
-                    .ok()?;
+            let mut bounds = JointBounds::compile_with_laws(
+                pool,
+                request,
+                domain,
+                metric,
+                event_input,
+                simulation,
+                laws,
+                None,
+                bounded,
+            )
+            .ok()?;
             if let Some(threshold) = tier_cap {
                 bounds.drop_tiers(pool, domain, threshold);
             }

crates/ournotes-search/src/search/mod.rs

@@ -21,7 +21,7 @@ pub use physical::{
     StepBudget, evaluate_declared_context, score_summary, solve_physical, solve_physical_with_aggregation,
 };
 
-mod budget;
+pub(crate) mod budget;
 pub(crate) mod deck_payoff;
 #[cfg(feature = "search-diagnostics")]
 pub mod diagnostics;

crates/ournotes-search/src/search/physical/engine.rs

@@ -63,6 +63,10 @@ pub(super) struct Offered {
     pub(super) score: Option<i32>,
 }
 
+/// What `bound_start` interrupted: the phase to reopen once the `bound` phase ends, if any, and the activity the
+/// clock continues at.
+pub(super) type BoundMark = crate::search::telemetry::PhaseMark;
+
 impl Engine<'_, '_> {
     /// Formal deterministic objectives also retain the leader and four unordered member/Snap pairs.
     /// The explicit v1 session cursor and fixed deterministic evaluation keep their physical-deck contracts.
@@ -366,10 +370,13 @@ impl Engine<'_, '_> {
         Ok(())
     }
 
-    /// Start bounding what the stop leaves unexplored. Its time goes to `proof.boundMs` and `otherMs`, not to the
-    /// search activity the stop interrupted; pass the returned mark to `fold_unexplored`.
-    pub(super) fn bound_start(&mut self) -> (Instant, usize) {
-        self.rec.clock.lap(slot::OTHER)
+    /// Start bounding what the stop leaves unexplored. Its time goes to `proof.boundMs`, `otherMs` and its own `bound`
+    /// phase; the interrupted phase is reopened by `fold_unexplored`. Pass the returned mark to it.
+    pub(super) fn bound_start(&mut self) -> (Instant, BoundMark) {
+        // `pause` charges the interrupted work to its own activity and continues the clock at `otherMs`.
+        let paused = self.rec.pause(&mut self.tel);
+        self.rec.begin(&mut self.tel, "bound", None);
+        (crate::search::budget::now(), paused)
     }
 
     /// The branches still open are bounded by `open` (None: no branch open): offer the larger of it and the best
@@ -385,11 +392,19 @@ impl Engine<'_, '_> {
         }
     }
 
-    pub(super) fn fold_unexplored(&mut self, upper: Option<i128>, (started, resume): (Instant, usize)) {
+    pub(super) fn fold_unexplored(&mut self, upper: Option<i128>, (started, paused): (Instant, BoundMark)) {
         if let Some(upper) = upper {
             self.rec.unexplored(upper);
         }
-        let (now, _) = self.rec.clock.lap(resume);
+        let (now, _) = self.rec.clock.lap(slot::OTHER);
+        if self.rec.is_open() {
+            self.rec.end(&mut self.tel);
+        }
+        let (phase, activity) = paused;
+        if let Some(phase) = phase {
+            self.rec.resume(&mut self.tel, phase);
+        }
+        self.rec.clock.switch(activity);
         self.tel.proof.bound_ms += now.saturating_duration_since(started).as_secs_f64() * 1000.0;
     }