Add ereports for host panic/boot failure

app/cosmo/base.toml

@@ -258,7 +258,7 @@ features = ["stm32h753", "usart6", "baud_rate_3M", "hardware_flow_control", "vla
 uses = ["usart6", "dbgmcu"]
 interrupts = {"usart6.irq" = "usart-irq"}
 priority = 9
-max-sizes = {flash = 70000, ram = 65536}
+max-sizes = {flash = 74000, ram = 65536}
 stacksize = 5400
 start = true
 task-slots = ["sys", { cpu_seq = "cosmo_seq" }, "hf", "control_plane_agent", "net", "packrat", "i2c_driver", { spi_driver = "spi2_driver" }, "sprot", "auxflash"]

app/gimlet/base.toml

@@ -243,7 +243,7 @@ features = ["stm32h753", "uart7", "baud_rate_3M", "hardware_flow_control", "vlan
 uses = ["uart7", "dbgmcu"]
 interrupts = {"uart7.irq" = "usart-irq"}
 priority = 8
-max-sizes = {flash = 70000, ram = 65536}
+max-sizes = {flash = 74000, ram = 65536}
 stacksize = 5376
 start = true
 task-slots = ["sys", { cpu_seq = "gimlet_seq" }, "hf", "control_plane_agent", "net", "packrat", "i2c_driver", { spi_driver = "spi2_driver" }, "sprot"]

task/host-sp-comms/Cargo.toml

@@ -48,6 +48,10 @@ task-sensor-api = { path = "../../task/sensor-api", optional = true, features =
 ksz8463 = { path = "../../drv/ksz8463", optional = true }
 drv-sprot-api = { path = "../../drv/sprot-api"}
 
+# ereports deps
+ereports = { path = "../../lib/ereports", features = ["ereporter-macro"] }
+microcbor = { path = "../../lib/microcbor" }
+
 [build-dependencies]
 build-util.path = "../../build/util"
 build-i2c = { path = "../../build/i2c", optional = true }

task/host-sp-comms/src/main.rs

@@ -28,6 +28,7 @@ use host_sp_messages::{
 };
 use hubpack::SerializedSize;
 use idol_runtime::{NotificationHandler, RequestError};
+use microcbor::Encode;
 use multitimer::{Multitimer, Repeat};
 use ringbuf::{counted_ringbuf, ringbuf_entry};
 use static_assertions::const_assert;
@@ -275,6 +276,15 @@ impl HostKeyValueStorage {
     }
 }
 
+/// Metadata about panics observed from the host
+struct HostPanicMetadata {
+    /// Length in bytes of the currently stored panic message
+    /// (not currently used, will be used in https://github.com/oxidecomputer/hubris/issues/2504)
+    _total_length: usize,
+    /// (hopefully not) Rolling counter of panic messages observed this power cycle
+    total_count: u32,
+}
+
 struct ServerImpl {
     uart: Usart,
     sys: sys_api::Sys,
@@ -292,6 +302,10 @@ struct ServerImpl {
     host_kv_storage: HostKeyValueStorage,
     hf_mux_state: Option<HfMuxState>,
 
+    ereporter: Ereporter,
+    host_panic_state: Option<HostPanicMetadata>,
+    host_boot_fail_state: Option<HostPanicMetadata>,
+
     /// Temporary space for inventory data, which is a large `enum`
     scratch: &'static mut host_sp_messages::InventoryData,
 
@@ -380,6 +394,9 @@ impl ServerImpl {
             });
             BUFS.claim()
         };
+
+        let packrat = Packrat::from(PACKRAT.get_task_id());
+
         Self {
             uart,
             sys,
@@ -399,7 +416,7 @@ impl ServerImpl {
             cp_agent: ControlPlaneAgent::from(
                 CONTROL_PLANE_AGENT.get_task_id(),
             ),
-            packrat: Packrat::from(PACKRAT.get_task_id()),
+            packrat: packrat.clone(),
             sprot: SpRot::from(SPROT.get_task_id()),
             reboot_state: None,
             host_kv_storage: HostKeyValueStorage {
@@ -414,6 +431,9 @@ impl ServerImpl {
             hf_mux_state: None,
             last_power_off: None,
             scratch,
+            host_panic_state: None,
+            host_boot_fail_state: None,
+            ereporter: Ereporter::claim_static_resources(packrat),
         }
     }
 
@@ -986,12 +1006,40 @@ impl ServerImpl {
                     data.len(),
                     self.host_kv_storage.last_boot_fail.len(),
                 );
-                self.host_kv_storage.last_boot_fail[..n]
-                    .copy_from_slice(&data[..n]);
-                for b in &mut self.host_kv_storage.last_boot_fail[n..] {
-                    *b = 0;
-                }
+                let (now, later) =
+                    self.host_kv_storage.last_boot_fail.split_at_mut(n);
+                now.copy_from_slice(&data[..n]);
+                later.iter_mut().for_each(|b| *b = 0);
+
                 self.host_kv_storage.last_boot_fail_reason = reason;
+
+                // Take the old count, if any, and add one to it. If that count wrapped,
+                // or if we didn't have an old count, set it to 1, so we never return
+                // a count of zero if we've ever observed a boot failure.
+                let new_ct = self
+                    .host_boot_fail_state
+                    .take()
+                    .map(|s| s.total_count.wrapping_add(1))
+                    .unwrap_or(0)
+                    .max(1);
+                self.host_boot_fail_state = Some(HostPanicMetadata {
+                    _total_length: n,
+                    total_count: new_ct,
+                });
+
+                let flashidx = match self.hf.get_dev() {
+                    Ok(HfDevSelect::Flash0) => 0,
+                    Ok(HfDevSelect::Flash1) => 1,
+                    Err(_) => 0xFF,
+                };
+
+                // ereport!
+                _ = self.ereporter.deliver_ereport(&HostBootFail {
+                    n: new_ct,
+                    msglen: n as u32,
+                    reason,
+                    flashidx,
+                });
                 Some(SpToHost::Ack)
             }
             HostToSp::HostPanic => {
@@ -1011,11 +1059,38 @@ impl ServerImpl {
                     data.len(),
                     self.host_kv_storage.last_panic.len(),
                 );
-                self.host_kv_storage.last_panic[..n]
-                    .copy_from_slice(&data[..n]);
-                for b in &mut self.host_kv_storage.last_panic[n..] {
-                    *b = 0;
-                }
+                let (now, later) =
+                    self.host_kv_storage.last_panic.split_at_mut(n);
+                now.copy_from_slice(&data[..n]);
+                later.iter_mut().for_each(|b| *b = 0);
+
+                // Take the old count, if any, and add one to it. If that count wrapped,
+                // or if we didn't have an old count, set it to 1, so we never return
+                // a count of zero if we've ever observed a boot failure.
+                let new_ct = self
+                    .host_panic_state
+                    .take()
+                    .map(|s| s.total_count.wrapping_add(1))
+                    .unwrap_or(0)
+                    .max(1);
+                self.host_panic_state = Some(HostPanicMetadata {
+                    _total_length: n,
+                    total_count: new_ct,
+                });
+
+                let flashidx = match self.hf.get_dev() {
+                    Ok(HfDevSelect::Flash0) => 0,
+                    Ok(HfDevSelect::Flash1) => 1,
+                    Err(_) => 0xFF,
+                };
+
+                // ereport!
+                _ = self.ereporter.deliver_ereport(&HostPanic {
+                    n: new_ct,
+                    msglen: n as u32,
+                    flashidx,
+                });
+
                 Some(SpToHost::Ack)
             }
             HostToSp::GetStatus => {
@@ -2012,3 +2087,50 @@ mod idl {
 }
 
 include!(concat!(env!("OUT_DIR"), "/notifications.rs"));
+
+ereports::declare_ereporter! {
+    struct Ereporter<Ereport> {
+        HostPanic(HostPanic),
+        BootPanic(HostBootFail),
+    }
+}
+
+/// An ereport represent a host reported panic
+#[derive(Encode)]
+#[ereport(class = "host.panic", version = 0)]
+struct HostPanic {
+    /// The total number of host panics observed by this invocation of
+    /// host-sp-comms.
+    ///
+    /// This count will wrap, but is guaranteed to never be zero.
+    n: u32,
+    /// The length, in bytes, of the stored panic message.
+    ///
+    /// This quantity may be less than the amount received, as it is capped
+    /// by the available storage space allocated (`MAX_HOST_FAIL_MESSAGE_LEN`).
+    msglen: u32,
+    /// The flash boot index, directly correlated to which boot slot we are
+    /// operating from. Currently 0 (BSU: A), 1 (BSU: B), or 0xFF (unknown).
+    flashidx: u8,
+}
+
+/// An ereport represent a host reported boot failure
+#[derive(Encode)]
+#[ereport(class = "host.btfail", version = 0)]
+struct HostBootFail {
+    /// The total number of host boot failures observed by this invocation
+    /// of host-sp-comms.
+    ///
+    /// This count will wrap, but is guaranteed to never be zero.
+    n: u32,
+    /// The length, in bytes, of the stored panic message.
+    ///
+    /// This quantity may be less than the amount received, as it is capped
+    /// by the available storage space allocated (`MAX_HOST_FAIL_MESSAGE_LEN`).
+    msglen: u32,
+    /// The reported reason code for the host boot failure
+    reason: u8,
+    /// The flash boot index, directly correlated to which boot slot we are
+    /// operating from. Currently 0 (BSU: A), 1 (BSU: B), or 0xFF (unknown).
+    flashidx: u8,
+}