From 9ee90215158f596a20cea45fbcb27767a794f1ed Mon Sep 17 00:00:00 2001 From: CheckPickerUpper Date: Tue, 25 Aug 2026 18:10:06 -0400 Subject: [PATCH] test: validate compiler against wasm core suite --- Cargo.lock | 52 +- Cargo.toml | 1 + tests/fixtures/wasm-spec/core/README.md | 24 + .../wasm-spec/core/i32-arithmetic.wast | 36 ++ tests/wasm_spec.rs | 474 ++++++++++++++++++ 5 files changed, 584 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/wasm-spec/core/README.md create mode 100644 tests/fixtures/wasm-spec/core/i32-arithmetic.wast create mode 100644 tests/wasm_spec.rs diff --git a/Cargo.lock b/Cargo.lock index feebe0a..fe972f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,6 +115,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "cfg-if" version = "1.0.4" @@ -453,6 +459,7 @@ dependencies = [ "tracing-subscriber", "walkdir", "walrus", + "wast", ] [[package]] @@ -971,6 +978,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1014,8 +1027,8 @@ dependencies = [ "leb128", "log", "walrus-macro", - "wasm-encoder", - "wasmparser", + "wasm-encoder 0.245.1", + "wasmparser 0.245.1", ] [[package]] @@ -1037,7 +1050,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9dca005e69bf015e45577e415b9af8c67e8ee3c0e38b5b0add5aa92581ed5c" dependencies = [ "leb128fmt", - "wasmparser", + "wasmparser 0.245.1", +] + +[[package]] +name = "wasm-encoder" +version = "0.258.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e974fe6821a8cf64575d51ea2194e2c8f77e7b66e9afe7419ce8a97f9ee0d251" +dependencies = [ + "leb128fmt", + "wasmparser 0.258.0", ] [[package]] @@ -1053,6 +1076,29 @@ dependencies = [ "serde", ] +[[package]] +name = "wasmparser" +version = "0.258.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a61719f93a87b16d325921e251800c4833f8fab50fa21c7de73aed50086313" +dependencies = [ + "bitflags", + "indexmap", +] + +[[package]] +name = "wast" +version = "258.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97f7defc7ecca8b19ac7f824598eadd0c53985ee00c74060d65051e9da5b58a1" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder 0.258.0", +] + [[package]] name = "winapi-util" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index f72e7f7..c4d65eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ insta = "1.48.0" predicates = "3.1.4" rstest = "0.26.1" tempfile = "3.27.0" +wast = { version = "258.0.0", default-features = false, features = ["wasm-module"] } [lints.rust] unsafe_code = "forbid" diff --git a/tests/fixtures/wasm-spec/core/README.md b/tests/fixtures/wasm-spec/core/README.md new file mode 100644 index 0000000..49bae04 --- /dev/null +++ b/tests/fixtures/wasm-spec/core/README.md @@ -0,0 +1,24 @@ +# Official WebAssembly core-suite fixture + +`i32-arithmetic.wast` is a bounded, checked-in slice of the official +WebAssembly core test suite. It is copied from +[`test/core/i32.wast`](https://github.com/WebAssembly/spec/blob/fc209c5ed8afc4dfeb9252024d217da3376c7a6f/test/core/i32.wast) +at source revision `fc209c5ed8afc4dfeb9252024d217da3376c7a6f`. + +The slice covers signed i32 add, subtract, multiply, and divide, including +wraparound values and the official divide-by-zero and signed-overflow trap +assertions. It is intentionally not presented as the whole specification +suite: the current compiler can only execute the subset represented by this +fixture through its existing decode, translation, and pinned Luau runtime path. + +`tests/wasm_spec.rs` parses this fixture with the `wast` parser, encodes each +module, sends it through `decode_module` and `translate_module`, and executes +the generated Luau with the pinned official Luau binary. The harness reports +passed, failed, and skipped-by-scope assertions separately. Return assertions +for unsupported shapes are skipped by scope; a trap assertion that cannot be +compiled or executed is a failure, never a pass. + +At the current compiler revision this slice reports 17 passed and 4 failed: +the four failures are the deliberately exercised `div_s` trap assertions. +That red result is the regression signal for missing trap semantics, not a +reason to widen the fixture or count unsupported behavior as success. diff --git a/tests/fixtures/wasm-spec/core/i32-arithmetic.wast b/tests/fixtures/wasm-spec/core/i32-arithmetic.wast new file mode 100644 index 0000000..9e0737d --- /dev/null +++ b/tests/fixtures/wasm-spec/core/i32-arithmetic.wast @@ -0,0 +1,36 @@ +;; This fixture is copied from the official WebAssembly core suite's i32.wast. +;; Keep the source revision and selected assertions documented alongside it. + +(module + (func (export "add") (param $x i32) (param $y i32) (result i32) (i32.add (local.get $x) (local.get $y))) + (func (export "sub") (param $x i32) (param $y i32) (result i32) (i32.sub (local.get $x) (local.get $y))) + (func (export "mul") (param $x i32) (param $y i32) (result i32) (i32.mul (local.get $x) (local.get $y))) + (func (export "div_s") (param $x i32) (param $y i32) (result i32) (i32.div_s (local.get $x) (local.get $y))) +) + +(assert_return (invoke "add" (i32.const 1) (i32.const 1)) (i32.const 2)) +(assert_return (invoke "add" (i32.const -1) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "add" (i32.const 0x7fffffff) (i32.const 1)) (i32.const 0x80000000)) +(assert_return (invoke "add" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x7fffffff)) + +(assert_return (invoke "sub" (i32.const 1) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "sub" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0x80000000)) +(assert_return (invoke "sub" (i32.const 0x80000000) (i32.const 1)) (i32.const 0x7fffffff)) +(assert_return (invoke "sub" (i32.const 0x80000000) (i32.const 0x80000000)) (i32.const 0)) + +(assert_return (invoke "mul" (i32.const 1) (i32.const 1)) (i32.const 1)) +(assert_return (invoke "mul" (i32.const -1) (i32.const -1)) (i32.const 1)) +(assert_return (invoke "mul" (i32.const 0x10000000) (i32.const 4096)) (i32.const 0)) +(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000)) +(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0x80000001)) +(assert_return (invoke "mul" (i32.const 0x01234567) (i32.const 0x76543210)) (i32.const 0x358e7470)) + +;; These trap assertions must fail until the generated helper models the +;; specification's divide-by-zero and signed-overflow traps. +(assert_trap (invoke "div_s" (i32.const 1) (i32.const 0)) "integer divide by zero") +(assert_trap (invoke "div_s" (i32.const 0) (i32.const 0)) "integer divide by zero") +(assert_trap (invoke "div_s" (i32.const 0x80000000) (i32.const -1)) "integer overflow") +(assert_trap (invoke "div_s" (i32.const 0x80000000) (i32.const 0)) "integer divide by zero") +(assert_return (invoke "div_s" (i32.const 1) (i32.const 1)) (i32.const 1)) +(assert_return (invoke "div_s" (i32.const -5) (i32.const 2)) (i32.const -2)) +(assert_return (invoke "div_s" (i32.const 5) (i32.const -2)) (i32.const -2)) diff --git a/tests/wasm_spec.rs b/tests/wasm_spec.rs new file mode 100644 index 0000000..3d99797 --- /dev/null +++ b/tests/wasm_spec.rs @@ -0,0 +1,474 @@ +//! Bounded official WebAssembly core-suite contracts for the compiler pipeline. + +mod support; + +use luau_rs::{ + decode_module, translate_module, DecodeOutcome, MainInvocation, TranslateOptions, + TranslateOutcome, +}; +use rstest::rstest; +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::io::Error; +use std::process::Command; +use support::official_luau_tool; +use wast::core::{WastArgCore, WastRetCore}; +use wast::parser::{self, ParseBuffer}; +use wast::{QuoteWat, Wast, WastArg, WastDirective, WastExecute, WastRet}; + +const SPEC_FIXTURE: &str = include_str!("fixtures/wasm-spec/core/i32-arithmetic.wast"); +const MINIMUM_PASSED_ASSERTIONS: usize = 21; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct AssertionId(usize); + +#[derive(Debug, Clone, Copy)] +enum AssertionExpectation { + Return(i32), + Trap, +} + +#[derive(Debug)] +struct SpecAssertion { + id: AssertionId, + line: usize, + export_name: String, + arguments: Vec, + expectation: AssertionExpectation, +} + +#[derive(Debug)] +enum ModuleCompilation { + Ready(String), + Rejected(String), +} + +#[derive(Debug)] +struct ModuleCase { + compilation: ModuleCompilation, + assertions: Vec, +} + +#[derive(Debug, Default)] +struct SuiteReport { + passed: usize, + failures: Vec, + skipped_by_scope: Vec, +} + +#[derive(Debug, thiserror::Error)] +enum HarnessError { + #[error("unsupported fixture construct: {0}")] + Unsupported(&'static str), +} + +impl SuiteReport { + const fn record_pass(&mut self) { + self.passed += 1; + } + + fn record_failure(&mut self, detail: impl Into) { + self.failures.push(detail.into()); + } + + fn record_skip(&mut self, detail: impl Into) { + self.skipped_by_scope.push(detail.into()); + } + + fn summary(&self) -> String { + format!( + "official wasm core subset: passed={}, failed={}, skipped-by-scope={}", + self.passed, + self.failures.len(), + self.skipped_by_scope.len() + ) + } + + fn rendered(&self) -> String { + let mut report = self.summary(); + for failure in &self.failures { + let _ = write!(report, "\nFAIL: {failure}"); + } + for skipped in &self.skipped_by_scope { + let _ = write!(report, "\nSKIP: {skipped}"); + } + report + } +} + +#[rstest] +fn given_official_i32_core_assertions_when_run_through_luau_then_they_pass() -> Result<(), Error> { + let report = run_spec_suite(); + tracing::info!( + passed = report.passed, + failed = report.failures.len(), + skipped_by_scope = report.skipped_by_scope.len(), + failures = ?report.failures, + skipped = ?report.skipped_by_scope, + "official WebAssembly core-suite result" + ); + if !report.failures.is_empty() { + return Err(Error::other(report.rendered())); + } + if report.passed == MINIMUM_PASSED_ASSERTIONS { + Ok(()) + } else { + Err(Error::other(format!( + "official wasm core pass-count regression: expected {}, observed {}", + MINIMUM_PASSED_ASSERTIONS, report.passed + ))) + } +} + +fn run_spec_suite() -> SuiteReport { + let mut report = SuiteReport::default(); + let fixture: &'static str = Box::leak(SPEC_FIXTURE.to_owned().into_boxed_str()); + let buffer: &'static ParseBuffer<'static> = match ParseBuffer::new(fixture) { + Ok(buffer) => Box::leak(Box::new(buffer)), + Err(error) => { + report.record_failure(format!("could not parse official fixture: {error}")); + return report; + } + }; + let mut script = match parser::parse::(buffer) { + Ok(script) => script, + Err(error) => { + report.record_failure(format!("could not parse official fixture: {error}")); + return report; + } + }; + let mut current = None; + let mut next_id = 0; + + for directive in &mut script.directives { + match directive { + WastDirective::Module(module) => { + finish_module(current.take(), &mut report); + current = Some(ModuleCase { + compilation: compile_module(module), + assertions: Vec::new(), + }); + } + WastDirective::AssertReturn { + span, + exec, + results, + } => { + let assertion = + match parse_return_assertion(*span, exec, results, AssertionId(next_id)) { + Ok(assertion) => { + next_id += 1; + assertion + } + Err(reason) => { + report.record_skip(format!("line {}: {reason}", line_number(*span))); + continue; + } + }; + match current.as_mut() { + Some(module) => module.assertions.push(assertion), + None => report.record_skip(format!( + "line {}: return assertion has no executable module", + line_number(*span) + )), + } + } + WastDirective::AssertTrap { span, exec, .. } => { + let assertion = match parse_trap_assertion(*span, exec, AssertionId(next_id)) { + Ok(assertion) => { + next_id += 1; + assertion + } + Err(reason) => { + report.record_failure(format!( + "line {}: could not exercise official trap assertion: {reason}", + line_number(*span) + )); + continue; + } + }; + match current.as_mut() { + Some(module) => module.assertions.push(assertion), + None => report.record_failure(format!( + "line {}: official trap assertion has no executable module", + line_number(*span) + )), + } + } + WastDirective::AssertExhaustion { span, .. } + | WastDirective::AssertUnlinkable { span, .. } + | WastDirective::AssertMalformed { span, .. } + | WastDirective::AssertMalformedCustom { span, .. } + | WastDirective::AssertInvalid { span, .. } + | WastDirective::AssertInvalidCustom { span, .. } + | WastDirective::AssertException { span, .. } + | WastDirective::AssertSuspension { span, .. } => { + report.record_skip(format!( + "line {}: assertion kind is outside the executable i32 subset", + line_number(*span) + )); + } + WastDirective::ModuleDefinition { .. } + | WastDirective::ModuleInstance { .. } + | WastDirective::Register { .. } + | WastDirective::Invoke(_) + | WastDirective::Thread(_) + | WastDirective::Wait { .. } => {} + } + } + finish_module(current, &mut report); + report +} + +fn compile_module(module: &mut QuoteWat<'static>) -> ModuleCompilation { + let wasm = match module.encode() { + Ok(wasm) => wasm, + Err(error) => return ModuleCompilation::Rejected(format!("WAST encoding failed: {error}")), + }; + let decoded = match decode_module(&wasm) { + DecodeOutcome::Decoded(decoded) => decoded, + DecodeOutcome::Rejected(rejection) => { + return ModuleCompilation::Rejected(format!("decoder rejected module: {rejection:?}")); + } + }; + let options = TranslateOptions::with_main_invocation(MainInvocation::LeaveIdle); + match translate_module(&decoded, options) { + TranslateOutcome::Translated(artifact) => ModuleCompilation::Ready(artifact.into_text()), + TranslateOutcome::Rejected(rejection) => { + ModuleCompilation::Rejected(format!("translator rejected module: {rejection:?}")) + } + } +} + +fn parse_return_assertion( + span: wast::token::Span, + exec: &mut WastExecute<'static>, + results: &[WastRet<'static>], + id: AssertionId, +) -> Result { + let invoke = parse_invoke(exec)?; + let arguments = parse_i32_arguments(&invoke.args)?; + let [WastRet::Core(WastRetCore::I32(expected))] = results else { + return Err(HarnessError::Unsupported("return values are not i32")); + }; + Ok(SpecAssertion { + id, + line: line_number(span), + export_name: invoke.name.to_owned(), + arguments, + expectation: AssertionExpectation::Return(*expected), + }) +} + +fn parse_trap_assertion( + span: wast::token::Span, + exec: &mut WastExecute<'static>, + id: AssertionId, +) -> Result { + let invoke = parse_invoke(exec)?; + let arguments = parse_i32_arguments(&invoke.args)?; + Ok(SpecAssertion { + id, + line: line_number(span), + export_name: invoke.name.to_owned(), + arguments, + expectation: AssertionExpectation::Trap, + }) +} + +const fn parse_invoke<'a>( + exec: &'a mut WastExecute<'static>, +) -> Result<&'a mut wast::WastInvoke<'static>, HarnessError> { + match exec { + WastExecute::Invoke(invoke) if invoke.module.is_none() => Ok(invoke), + WastExecute::Invoke(_) => Err(HarnessError::Unsupported("named module invocations")), + WastExecute::Wat(_) => Err(HarnessError::Unsupported("inline module execution")), + WastExecute::Get { .. } => Err(HarnessError::Unsupported("global reads")), + } +} + +fn parse_i32_arguments(arguments: &[WastArg<'static>]) -> Result, HarnessError> { + arguments + .iter() + .map(|argument| match argument { + WastArg::Core(WastArgCore::I32(value)) => Ok(*value), + WastArg::Core(_) => Err(HarnessError::Unsupported("non-i32 invocation arguments")), + _ => Err(HarnessError::Unsupported("component invocation arguments")), + }) + .collect() +} + +fn finish_module(current: Option, report: &mut SuiteReport) { + let Some(module) = current else { + return; + }; + match module.compilation { + ModuleCompilation::Ready(generated) => { + run_ready_module(&generated, &module.assertions, report); + } + ModuleCompilation::Rejected(reason) => { + for assertion in module.assertions { + match assertion.expectation { + AssertionExpectation::Return(_) => report.record_skip(format!( + "line {}: return assertion skipped because the module is outside compiler scope ({reason})", + assertion.line + )), + AssertionExpectation::Trap => report.record_failure(format!( + "line {}: trap assertion could not reach the backend ({reason})", + assertion.line + )), + } + } + } + } +} + +fn run_ready_module(generated: &str, assertions: &[SpecAssertion], report: &mut SuiteReport) { + let luau = match official_luau_tool(("LUAU_BIN", "luau")) { + Ok(path) => path, + Err(error) => { + for assertion in assertions { + report.record_failure(format!( + "line {}: pinned Luau runtime unavailable: {error}", + assertion.line + )); + } + return; + } + }; + let temp_dir = match tempfile::Builder::new() + .prefix("luau-rs-wasm-spec") + .tempdir() + { + Ok(temp_dir) => temp_dir, + Err(error) => { + for assertion in assertions { + report.record_failure(format!( + "line {}: could not create runner directory: {error}", + assertion.line + )); + } + return; + } + }; + let source_path = temp_dir.path().join("driver.luau"); + let driver = build_driver(generated, assertions); + if let Err(error) = fs_err::write(&source_path, driver) { + for assertion in assertions { + report.record_failure(format!( + "line {}: could not write runner: {error}", + assertion.line + )); + } + return; + } + let output = match Command::new(luau).arg(&source_path).output() { + Ok(output) => output, + Err(error) => { + for assertion in assertions { + report.record_failure(format!( + "line {}: could not run pinned Luau: {error}", + assertion.line + )); + } + return; + } + }; + let execution = parse_execution_output(&output.stdout); + for assertion in assertions { + match execution.get(&assertion.id) { + Some(ExecutionResult::Passed) => report.record_pass(), + Some(ExecutionResult::Failed(detail)) => report.record_failure(format!( + "line {}: {} ({detail})", + assertion.line, + assertion_label(assertion) + )), + None => report.record_failure(format!( + "line {}: {} produced no result; stderr={}", + assertion.line, + assertion_label(assertion), + String::from_utf8_lossy(&output.stderr) + )), + } + } +} + +fn build_driver(generated: &str, assertions: &[SpecAssertion]) -> String { + let mut driver = format!( + "local function load_module()\n{generated}\nend\nlocal module = load_module()({{}})\n" + ); + for assertion in assertions { + let call = format!( + "module[{export:?}]({arguments})", + export = assertion.export_name, + arguments = assertion + .arguments + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ); + let check = match assertion.expectation { + AssertionExpectation::Return(expected) => format!( + "local ok_{id}, actual_{id} = pcall(function() return {call} end)\nif not ok_{id} then\n print(\"FAIL|{id}|unexpected trap: \" .. tostring(actual_{id}))\nelseif actual_{id} ~= {expected} then\n print(\"FAIL|{id}|expected {expected} but got \" .. tostring(actual_{id}))\nelse\n print(\"PASS|{id}\")\nend\n", + id = assertion.id.0, + call = call, + expected = expected + ), + AssertionExpectation::Trap => format!( + "local ok_{id}, actual_{id} = pcall(function() return {call} end)\nif ok_{id} then\n print(\"FAIL|{id}|expected a trap\")\nelse\n print(\"PASS|{id}\")\nend\n", + id = assertion.id.0, + call = call + ), + }; + driver.push_str(&check); + } + driver +} + +#[derive(Debug)] +enum ExecutionResult { + Passed, + Failed(String), +} + +fn parse_execution_output(stdout: &[u8]) -> BTreeMap { + let mut results = BTreeMap::new(); + for line in String::from_utf8_lossy(stdout).lines() { + let mut fields = line.splitn(3, '|'); + let Some(kind) = fields.next() else { continue }; + let id = match fields.next() { + Some(field) => match field.parse::() { + Ok(id) => AssertionId(id), + Err(_) => continue, + }, + None => continue, + }; + match kind { + "PASS" => { + results.insert(id, ExecutionResult::Passed); + } + "FAIL" => { + let detail = fields + .next() + .map_or_else(|| "unspecified failure".to_owned(), str::to_owned); + results.insert(id, ExecutionResult::Failed(detail)); + } + _ => {} + } + } + results +} + +fn assertion_label(assertion: &SpecAssertion) -> String { + match assertion.expectation { + AssertionExpectation::Return(expected) => { + format!("invoke {} expected {expected}", assertion.export_name) + } + AssertionExpectation::Trap => format!("invoke {} expected trap", assertion.export_name), + } +} + +fn line_number(span: wast::token::Span) -> usize { + span.linecol_in(SPEC_FIXTURE).0 + 1 +}