PMAT-3733: apr inspect --json shows every GGUF header key (qwen35.* was dropped by a parse allowlist)

crates/apr-cli/src/commands/inspect.rs

@@ -215,6 +215,20 @@ pub(crate) fn run(
 /// GGUF/SafeTensors inspect via RosettaStone
 /// Print rosetta inspection report as JSON.
 fn output_rosetta_json(path: &Path, report: &aprender::format::rosetta::InspectionReport) {
+    if let Ok(json) = serde_json::to_string_pretty(&rosetta_json(path, report)) {
+        println!("{json}");
+    }
+}
+
+/// The `apr inspect --json` document for a GGUF/SafeTensors file.
+///
+/// `metadata` is every key in the report, unfiltered: #3733's gate reads a
+/// model's declared context and memory arithmetic from it, and a second GGUF
+/// reader is refused (`check_no_hand_rolled_parsers.sh`).
+fn rosetta_json(
+    path: &Path,
+    report: &aprender::format::rosetta::InspectionReport,
+) -> serde_json::Value {
     let mut json_map = serde_json::Map::new();
     json_map.insert(
         "file".to_string(),
@@ -262,10 +276,7 @@ fn output_rosetta_json(path: &Path, report: &aprender::format::rosetta::Inspecti
         .collect::<serde_json::Map<_, _>>()
         .into();
     json_map.insert("metadata".to_string(), metadata);
-
-    if let Ok(json) = serde_json::to_string_pretty(&json_map) {
-        println!("{json}");
-    }
+    serde_json::Value::Object(json_map)
 }
 
 /// Print rosetta inspection report as rich text.

crates/apr-cli/src/commands/inspect_tests.rs

@@ -5,6 +5,38 @@ mod inspect_tests {
     use super::*;
     use std::path::Path;
 
+    // ========================================================================
+    // rosetta_json (#3733): --json shows every metadata key, unfiltered
+    // ========================================================================
+
+    #[test]
+    fn rosetta_json_emits_every_metadata_key() {
+        use aprender::format::rosetta::{FormatType, InspectionReport};
+        let keys = [
+            "general.architecture",
+            "qwen35.context_length",
+            "qwen35.full_attention_interval",
+            "quantize.imatrix.file",
+            "zzz_never_heard_of.depth",
+            "tokenizer.ggml.model",
+        ];
+        let report = InspectionReport {
+            format: FormatType::Gguf,
+            file_size: 1,
+            metadata: keys.iter().map(|k| ((*k).to_string(), "v".to_string())).collect(),
+            tensors: vec![],
+            total_params: 0,
+            quantization: None,
+            architecture: Some("qwen35".to_string()),
+        };
+        let v = rosetta_json(Path::new("m.gguf"), &report);
+        let shown = v["metadata"].as_object().expect("metadata object");
+        assert_eq!(shown.len(), keys.len());
+        for k in keys {
+            assert!(shown.contains_key(k), "--json dropped {k}");
+        }
+    }
+
     // ========================================================================
     // validate_path
     // ========================================================================

crates/aprender-core/src/format/gguf/api.rs

@@ -339,9 +339,11 @@ pub fn load_gguf_raw<P: AsRef<Path>>(path: P) -> Result<GgufRawLoadResult> {
     // Contract: apr-inspect-metadata-propagation-v1 F-INSPECT-META-001 (paiml/aprender#622).
     // Propagate raw GGUF KV metadata to downstream consumers (inspect/rosetta) so they
     // can display authentic on-disk keys instead of fabricated ML-shorthand names.
+    // #3733: EVERY header key, including those outside the reader's parse allowlist.
     let raw_metadata: BTreeMap<String, String> = reader
         .metadata
         .iter()
+        .chain(&reader.display_only_metadata)
         .map(|(k, v)| (k.clone(), gguf_value_display(v)))
         .collect();
 

crates/aprender-core/src/format/gguf/api_tests.rs

@@ -449,3 +449,6 @@ fn test_gguf_model_config_clone() {
 
 #[path = "api_tests_raw_tensor_rope.rs"]
 mod api_tests_raw_tensor_rope;
+
+#[path = "api_tests_all_keys_3733.rs"]
+mod api_tests_all_keys_3733;

crates/aprender-core/src/format/gguf/api_tests_all_keys_3733.rs

@@ -0,0 +1,149 @@
+//! #3733: `apr inspect --json` must show EVERY key the GGUF header carries.
+//!
+//! The reader's parse allowlist (`tokenizer.`, `general.`, six arch prefixes)
+//! decided what inspect could show, so a Qwen3.5 file listed 24 of its 46 header
+//! keys and none of the 18 `qwen35.*` ones: `qwen35.` does not start with
+//! `qwen3.`. The fixture below uses that prefix, a prefix nothing has heard of,
+//! and the `quantize.*` group real quantized files carry.
+
+use super::*;
+use crate::format::gguf::GGUF_MAGIC;
+use crate::format::rosetta::RosettaStone;
+
+/// Every key the fixture's header carries. The count assertions compare against
+/// THIS list, which is exactly what `export_tensors_to_gguf` writes.
+fn header() -> Vec<(String, GgufValue)> {
+    vec![
+        (
+            "general.architecture".into(),
+            GgufValue::String("qwen35".into()),
+        ),
+        (
+            "general.name".into(),
+            GgufValue::String("pygmy-hybrid".into()),
+        ),
+        ("qwen35.context_length".into(), GgufValue::Uint32(262_144)),
+        ("qwen35.block_count".into(), GgufValue::Uint32(32)),
+        (
+            "qwen35.full_attention_interval".into(),
+            GgufValue::Uint32(4),
+        ),
+        (
+            "qwen35.attention.head_count_kv".into(),
+            GgufValue::Uint32(4),
+        ),
+        ("qwen35.attention.key_length".into(), GgufValue::Uint32(256)),
+        (
+            "qwen35.rope.dimension_sections".into(),
+            GgufValue::ArrayInt32(vec![11, 11, 10, 0]),
+        ),
+        ("qwen35.rope.freq_base".into(), GgufValue::Float32(1.0e7)),
+        (
+            "quantize.imatrix.file".into(),
+            GgufValue::String("imatrix.gguf".into()),
+        ),
+        ("zzz_never_heard_of.depth".into(), GgufValue::Uint64(7)),
+        (
+            "tokenizer.ggml.model".into(),
+            GgufValue::String("gpt2".into()),
+        ),
+    ]
+}
+
+fn fixture() -> NamedTempFile {
+    let file = NamedTempFile::with_suffix(".gguf").expect("temp gguf");
+    let tensors = vec![GgufTensor {
+        name: "token_embd.weight".to_string(),
+        shape: vec![4, 4],
+        dtype: GgmlType::F32,
+        data: vec![0u8; 64],
+    }];
+    let mut bytes = Vec::new();
+    export_tensors_to_gguf(&mut bytes, &tensors, &header()).expect("export gguf");
+    std::fs::write(file.path(), &bytes).expect("write gguf");
+    file
+}
+
+fn assert_every_header_key(shown: &BTreeMap<String, String>, layer: &str) {
+    let header = header();
+    let missing: Vec<&str> = header
+        .iter()
+        .map(|(k, _)| k.as_str())
+        .filter(|k| !shown.contains_key(*k))
+        .collect();
+    assert!(
+        missing.is_empty(),
+        "{layer} dropped header keys: {missing:?}"
+    );
+    assert_eq!(
+        shown.len(),
+        header.len(),
+        "{layer} must show exactly the header's KV count"
+    );
+}
+
+#[test]
+fn raw_metadata_carries_every_header_key() {
+    let file = fixture();
+    let raw = load_gguf_raw(file.path()).expect("load");
+    assert_every_header_key(&raw.raw_metadata, "load_gguf_raw");
+    assert_eq!(raw.raw_metadata["qwen35.context_length"], "262144");
+    assert_eq!(
+        raw.raw_metadata["qwen35.rope.dimension_sections"],
+        "[len=4]"
+    );
+    assert_eq!(raw.raw_metadata["zzz_never_heard_of.depth"], "7");
+}
+
+/// The path `apr inspect` takes for a GGUF (`run_rosetta_inspect`).
+#[test]
+fn rosetta_inspect_carries_every_header_key() {
+    let file = fixture();
+    let report = RosettaStone::new().inspect(file.path()).expect("inspect");
+    assert_every_header_key(&report.metadata, "RosettaStone::inspect");
+}
+
+/// The keys are shown, not consumed: the config accessors still read only the
+/// allowlisted map, so an architecture the loaders do not know gets no config it
+/// never had (a widened accessor would hand `apr import` a qwen35 hidden size).
+#[test]
+fn display_only_keys_do_not_reach_the_config_accessors() {
+    let file = fixture();
+    let reader = GgufReader::from_file(file.path()).expect("read");
+    assert!(reader
+        .display_only_metadata
+        .contains_key("qwen35.context_length"));
+    assert!(!reader.metadata.contains_key("qwen35.context_length"));
+    assert_eq!(reader.context_length(), None);
+    assert_eq!(reader.num_layers(), None);
+
+    // `keep_all` (the shard merge) still puts everything in `metadata`.
+    let full = GgufReader::from_file_full(file.path()).expect("read full");
+    assert!(full.display_only_metadata.is_empty());
+    assert_eq!(full.metadata.len(), header().len());
+}
+
+/// Parsing (not skipping) keys outside the allowlist must not turn a hostile
+/// array count into an allocation or an out-of-bounds index.
+#[test]
+fn an_array_count_past_the_end_of_the_file_is_an_error_not_a_panic() {
+    for elem_type in [4u32, 5, 6, 8] {
+        let key = "zzz.hostile";
+        let mut data = Vec::new();
+        data.extend_from_slice(&GGUF_MAGIC.to_le_bytes());
+        data.extend_from_slice(&3u32.to_le_bytes());
+        data.extend_from_slice(&0u64.to_le_bytes()); // tensors
+        data.extend_from_slice(&1u64.to_le_bytes()); // one KV
+        data.extend_from_slice(&(key.len() as u64).to_le_bytes());
+        data.extend_from_slice(key.as_bytes());
+        data.extend_from_slice(&9u32.to_le_bytes()); // array
+        data.extend_from_slice(&elem_type.to_le_bytes());
+        data.extend_from_slice(&(u64::MAX / 2).to_le_bytes());
+        data.extend_from_slice(&[0u8; 16]);
+        let err = GgufReader::from_bytes(data).expect_err("a count past EOF must be refused");
+        assert!(
+            err.to_string().contains("overruns"),
+            "type {elem_type}: {err}"
+        );
+    }
+}

crates/aprender-core/src/format/gguf/builder.rs

@@ -161,10 +161,13 @@ fn test_from_bytes_tensor_at_max_dims() {
 }
 
 // ========================================================================
-// skip_metadata_value Tests (via from_bytes with non-parsed keys)
+// Keys outside the parse allowlist (via from_bytes with non-parsed keys).
+// They were skipped by length arithmetic until #3733; they are now decoded into
+// `display_only_metadata`, so `apr inspect` shows them, and still kept out of
+// `metadata`, which the config accessors read.
 // ========================================================================
 
-/// Build a GGUF with metadata that will be skipped (key prefix not in parsed set)
+/// Build a GGUF with one metadata key outside the parse allowlist
 fn build_gguf_with_skipped_metadata(key: &str, value_type: u32, value_bytes: &[u8]) -> Vec<u8> {
     let mut data = Vec::new();
 
@@ -175,7 +178,7 @@ fn build_gguf_with_skipped_metadata(key: &str, value_type: u32, value_bytes: &[u
     data.extend_from_slice(&1u64.to_le_bytes()); // metadata_count = 1
 
     // Metadata KV: key with prefix that does NOT match tokenizer./general./llama./qwen2./phi./mistral.
-    // so it will be skipped via skip_metadata_value
+    // so it lands in display_only_metadata (#3733)
     data.extend_from_slice(&(key.len() as u64).to_le_bytes());
     data.extend_from_slice(key.as_bytes());
     data.extend_from_slice(&value_type.to_le_bytes());
@@ -189,34 +192,39 @@ fn test_skip_metadata_value_uint8() {
     let data = build_gguf_with_skipped_metadata("custom.u8", 0, &[42u8]);
     let reader = GgufReader::from_bytes(data).expect("skip uint8");
     assert!(!reader.metadata.contains_key("custom.u8"));
+    assert!(reader.display_only_metadata.contains_key("custom.u8"));
 }
 
 #[test]
 fn test_skip_metadata_value_int8() {
     let data = build_gguf_with_skipped_metadata("custom.i8", 1, &[0xFEu8]);
     let reader = GgufReader::from_bytes(data).expect("skip int8");
     assert!(!reader.metadata.contains_key("custom.i8"));
+    assert!(reader.display_only_metadata.contains_key("custom.i8"));
 }
 
 #[test]
 fn test_skip_metadata_value_uint16() {
     let data = build_gguf_with_skipped_metadata("custom.u16", 2, &1000u16.to_le_bytes());
     let reader = GgufReader::from_bytes(data).expect("skip uint16");
     assert!(!reader.metadata.contains_key("custom.u16"));
+    assert!(reader.display_only_metadata.contains_key("custom.u16"));
 }
 
 #[test]
 fn test_skip_metadata_value_int16() {
     let data = build_gguf_with_skipped_metadata("custom.i16", 3, &(-500i16).to_le_bytes());
     let reader = GgufReader::from_bytes(data).expect("skip int16");
     assert!(!reader.metadata.contains_key("custom.i16"));
+    assert!(reader.display_only_metadata.contains_key("custom.i16"));
 }
 
 #[test]
 fn test_skip_metadata_value_bool() {
     let data = build_gguf_with_skipped_metadata("custom.flag", 7, &[1u8]);
     let reader = GgufReader::from_bytes(data).expect("skip bool");
     assert!(!reader.metadata.contains_key("custom.flag"));
+    assert!(reader.display_only_metadata.contains_key("custom.flag"));
 }
 
 #[test]
@@ -229,20 +237,23 @@ fn test_skip_metadata_value_string() {
     let data = build_gguf_with_skipped_metadata("custom.str", 8, &value_bytes);
     let reader = GgufReader::from_bytes(data).expect("skip string");
     assert!(!reader.metadata.contains_key("custom.str"));
+    assert!(reader.display_only_metadata.contains_key("custom.str"));
 }
 
 #[test]
 fn test_skip_metadata_value_uint64() {
     let data = build_gguf_with_skipped_metadata("custom.u64", 10, &999u64.to_le_bytes());
     let reader = GgufReader::from_bytes(data).expect("skip uint64");
     assert!(!reader.metadata.contains_key("custom.u64"));
+    assert!(reader.display_only_metadata.contains_key("custom.u64"));
 }
 
 #[test]
 fn test_skip_metadata_value_int64() {
     let data = build_gguf_with_skipped_metadata("custom.i64", 11, &(-1i64).to_le_bytes());
     let reader = GgufReader::from_bytes(data).expect("skip int64");
     assert!(!reader.metadata.contains_key("custom.i64"));
+    assert!(reader.display_only_metadata.contains_key("custom.i64"));
 }
 
 #[test]
@@ -251,6 +262,7 @@ fn test_skip_metadata_value_float64() {
         build_gguf_with_skipped_metadata("custom.f64", 12, &std::f64::consts::E.to_le_bytes());
     let reader = GgufReader::from_bytes(data).expect("skip float64");
     assert!(!reader.metadata.contains_key("custom.f64"));
+    assert!(reader.display_only_metadata.contains_key("custom.f64"));
 }
 
 #[test]
@@ -259,6 +271,7 @@ fn test_skip_metadata_value_unknown_type() {
     let data = build_gguf_with_skipped_metadata("custom.unk", 99, &[0u8; 4]);
     let reader = GgufReader::from_bytes(data).expect("skip unknown");
     assert!(!reader.metadata.contains_key("custom.unk"));
+    assert!(reader.display_only_metadata.contains_key("custom.unk"));
 }
 
 #[test]