Add load-time bytecode validation pass

VM.md

@@ -216,10 +216,30 @@ Section 2 — global/define names:
 | Byte-string literal | 255 bytes (`u8` length of `BYTES`) | `CompileError::BytesLiteralTooLong` |
 | Heap | 65,535 words (`u16` addresses, `0xFFFF` = `NULL`) | a larger buffer is used only up to that size |
 
+## Load-time validation
+
+The interpreter reads the code stream without bounds checks, so `Vm::load` first runs `Program::validate`, a single `no_std`, allocation-free pass that returns `VmError::Invalid { pc, reason }` (or `VmError::InvalidOpcode`) for malformed code. `Program::parse` checks only the header, so the disassembler can still open broken files.
+
+Validation rejects a program when:
+
+- an opcode is unknown, or an instruction's operands run past `code_len`;
+- the last instruction can fall through (it must be `FIN`, `ENCORE`, `MATCH` or `BRANCH`);
+- a static jump target (a `MATCH` table entry, a `BRANCH` target, a `CLOSURE`/`FUNCTION` code pointer, or a global entry point) is outside the code or does not land on an instruction boundary;
+- a `PACK`/`UNPACK` tag is `>= n_arities`, or `UNPACK` would write past the register file (`rd + arity > 256`);
+- a `GLOBAL`/`GLOBAL_W` index is `>= n_globals`;
+- an `EXTERN` slot is `>= 32`;
+- the code is longer than `0xFFFF` bytes (code pointers are 16-bit, and `0xFFFF` is the `NULL` continuation).
+
+`MATCH`/`BRANCH` tags are only compared, never used as indices, so they are not checked against the arity table.
+
+At run time, `ENCORE` also checks that a dynamic call target lies inside the code. This catches a call to the `NULL` continuation. Together, the two checks keep the program counter on an instruction boundary of validated code. This invariant backs the unchecked reads in `Code`.
+
+**Remaining trust assumption:** values are not type-checked at run time. The compiler only emits `FIELD`/`UNPACK` after a `MATCH` on a constructor, `CAPTURE` inside a closure body with enough captures, and so on. A hand-crafted file can still apply `FIELD`, `CAPTURE`, `ENCORE` or a bytes operation to a value of the wrong type or shape (for example `FIELD` on an integer, a field index past the constructor's arity, or a capture index past the closure's environment). The resulting heap access is unchecked. Only load bytecode produced by the Encore compiler, or bytecode from a source you trust to the same degree.
+
 ## Entry points
 
 - **`Vm::init(mem)`** — create a VM instance with a heap arena.
-- **`vm.load(&prog)`** — parse a program binary, initialize globals by running each define's thunk.
+- **`vm.load(&prog)`** — validate the program (see above), then initialize globals by running each define's thunk.
 - **`vm.call(global_idx, arg)`** — call a global function with an argument, return the result.
 - **`vm.call_value(func, arg)`** — call an arbitrary function value with an argument.
 - **`vm.register_extern(slot, f)`** — register a host function at a given extern slot.

crates/encore_fleche/tests/validate_examples.rs

@@ -0,0 +1,29 @@
+//! Every example program must pass the VM's load-time validation, with and
+//! without the optimizer.
+
+use std::path::Path;
+
+use encore_compiler::pass::cps_optimize::OptimizeConfig;
+use encore_compiler::pipeline;
+use encore_vm::program::Program;
+
+#[test]
+fn test_examples_validate() {
+    let examples = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples");
+    let mut seen = 0;
+    for entry in std::fs::read_dir(&examples).unwrap() {
+        let dir = entry.unwrap().path();
+        let name = dir.file_name().unwrap().to_str().unwrap().to_string();
+        let src_file = dir.join(format!("{name}.fleche"));
+        let Ok(src) = std::fs::read_to_string(&src_file) else { continue };
+        for config in [None, Some(OptimizeConfig::default())] {
+            let binary = pipeline::compile_module(encore_fleche::parse(&src), config, None).unwrap();
+            let prog = Program::parse(&binary).unwrap();
+            if let Err(e) = prog.validate() {
+                panic!("{}: {e}", src_file.display());
+            }
+        }
+        seen += 1;
+    }
+    assert!(seen > 0, "no examples found in {}", examples.display());
+}

crates/encore_vm/src/code.rs

@@ -1,15 +1,38 @@
+//! Cursor over the code stream, read without bounds checks.
+//!
+//! SAFETY: a `Code` is only built over bytes that passed
+//! [`validate`](crate::validate), and the interpreter only moves `pc` by
+//! decoding one instruction at a time from a boundary or by jumping to a
+//! validated target (static targets are checked at load, dynamic `ENCORE`
+//! targets by `Vm::resolve_code_ptr`). Validation guarantees every operand of
+//! an instruction starting at a boundary lies inside the code and that no
+//! instruction falls through past its end, so each unchecked read below is
+//! in bounds.
+
 use crate::value::{CodeAddress, Reg};
 
-pub struct Code<'a> {
+pub(crate) struct Code<'a> {
     bytes: &'a [u8],
     pc: usize,
 }
 
 impl<'a> Code<'a> {
-    pub fn new(bytes: &'a [u8]) -> Self {
+    /// A cursor over no code. Nothing can execute: every call target is
+    /// out of range.
+    pub fn empty() -> Self {
+        Self { bytes: &[], pc: 0 }
+    }
+
+    /// # Safety
+    /// `bytes` must be the code of a program that passed validation.
+    pub unsafe fn new(bytes: &'a [u8]) -> Self {
         Self { bytes, pc: 0 }
     }
 
+    pub fn len(&self) -> usize {
+        self.bytes.len()
+    }
+
     pub fn read_u8(&mut self) -> u8 {
         let b = unsafe { *self.bytes.get_unchecked(self.pc) };
         self.pc += 1;

crates/encore_vm/src/error.rs

@@ -52,6 +52,9 @@ pub enum VmError {
     Extern { error: ExternError, slot: u16, pc: u16 },
     TypeError { expected: &'static str, got: &'static str },
     GlobalsOverflow { needed: usize, available: usize },
+    /// Malformed bytecode, rejected at load time (or a call to a target
+    /// outside the code). `pc` is the offending code offset.
+    Invalid { pc: u16, reason: &'static str },
 }
 
 impl VmError {
@@ -67,6 +70,7 @@ impl VmError {
             VmError::Extern { .. } => "extern call failed",
             VmError::TypeError { .. } => "type error",
             VmError::GlobalsOverflow { .. } => "globals do not fit in the heap",
+            VmError::Invalid { reason, .. } => *reason,
         }
     }
 }
@@ -91,6 +95,8 @@ impl fmt::Display for VmError {
                 write!(f, "type error: expected {expected}, got {got}"),
             VmError::GlobalsOverflow { needed, available } =>
                 write!(f, "globals overflow: program needs {needed} global slots, heap has {available} free words"),
+            VmError::Invalid { pc, reason } =>
+                write!(f, "invalid bytecode at pc=0x{pc:04x}: {reason}"),
         }
     }
 }

crates/encore_vm/src/lib.rs

@@ -12,7 +12,7 @@ macro_rules! stat { ($($t:tt)*) => {} }
 
 pub mod arena;
 pub mod builtins;
-pub mod code;
+mod code;
 pub mod error;
 pub mod ffi;
 pub mod gc;
@@ -22,6 +22,7 @@ pub mod program;
 mod registers;
 #[cfg(feature = "stats")]
 pub mod stats;
+pub mod validate;
 pub mod value;
 pub mod vm;
 

crates/encore_vm/src/program.rs

@@ -19,6 +19,10 @@ pub const MAGIC: [u8; 4] = *b"ENCR";
 ///   Section 2 - global/define names:
 ///     [n_globals: u16 LE]
 ///     For each: [idx: u16 LE] [name_len: u8] [name: name_len bytes, UTF-8]
+///
+/// `parse` checks the header only, so tools such as the disassembler can
+/// still open malformed code. [`Vm::load`](crate::vm::Vm::load) runs the full
+/// [`validate`](Self::validate) pass before executing anything.
 #[derive(Debug)]
 pub struct Program<'a> {
     pub arity_table: &'a [u8],
@@ -85,6 +89,11 @@ impl<'a> Program<'a> {
         }
     }
 
+    /// Check that the code is safe to execute; see [`crate::validate`].
+    pub fn validate(&self) -> Result<(), VmError> {
+        crate::validate::validate(self)
+    }
+
     pub fn has_metadata(&self) -> bool {
         self.metadata.len() >= 2
     }