diff --git a/AGENTS.md b/AGENTS.md index f2a4bd4..75a5dd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,14 @@ Keep runtime-data and citation versions synchronized through `cargo-release` rep Release notes describe user-facing changes; the first regular release introduces the product and its migration path. Use `cargo-dist` for native GitHub platform builds and archive smoke tests; publishing is an explicit maintainer workflow dispatch. +## Firmware extraction + +`fat_extract::pipeline` owns the recursive container and embedded-region queue and per-artifact outcomes. +`output::OutputTree` owns staged publication, path validation, shared output limits, and deferred symlinks. +SquashFS header, inode, codec, and cached metadata readers remain separate; new formats reuse the output contract. +Carved images and directory names alone do not establish rootfs recovery; unresolved branches must leave fallback available. +Explicit native selection must not invoke external filesystem fallback. + ## MCU analysis and terminal reports **Raw MCU identification** (`fat_analyze::mcu`): Structural vector candidates, family evidence, and code mapping are independent. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4457155..344df6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,13 @@ ### Added +- Native recursive ZIP, TAR, and gzip extraction with SquashFS 2/3/4 decoding, including both byte orders and legacy LZMA block layouts. +- Extraction manifests include per-artifact lineage, decoder outcomes, limits, and recovery status. - An Arch Linux and Omarchy installation guide with tested prerequisites, optional tools, and extraction verification steps. ### Fixed +- Carved images and empty output directories no longer stop rootfs recovery attempts. - Healthy bootloader tmux sessions are reused on tmux versions that sanitize control characters in display formats. - Binwalk runs that exit without scanning any files are retried up to twice within the original timeout, preserving failed attempts and reporting a failure if scanning never starts. - Automatic native dependency installation recognizes Omarchy and other distributions declaring `ID_LIKE=arch`, while preserving explicitly supported distribution recipes. diff --git a/Cargo.lock b/Cargo.lock index eb1ee66..7277953 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -503,14 +503,18 @@ dependencies = [ name = "fat-toolkit-extract" version = "2.0.0" dependencies = [ + "aho-corasick", "crc32fast", "flate2", "libc", + "lzma-rs", "serde", "serde_json", "tar", "tempfile", "walkdir", + "xz2", + "zip", ] [[package]] diff --git a/crates/fat_cli/src/main.rs b/crates/fat_cli/src/main.rs index fb1ba75..b6592f4 100644 --- a/crates/fat_cli/src/main.rs +++ b/crates/fat_cli/src/main.rs @@ -3,7 +3,6 @@ use fat_analyze::bootloader::{ analyze_firmware_path as analyze_bootloader_firmware, analyze_text as analyze_bootloader_text, merge_snapshots as merge_bootloader_snapshots, }; -use fat_analyze::firmware_formats::scan as scan_firmware_formats; use fat_bootloader::{ assist_boot, discover_boot_artifacts, evaluate_true_boot_chain, launch_in_tmux, materialize_workspace, prepare_launch, stop_all_tmux_sessions, stop_tmux_session, @@ -25,9 +24,7 @@ use fat_extract::extractors::{ StrategyContext, Sufficiency, }; use fat_extract::manifest::{EngineReport, EngineStatus, ExtractionManifest}; -use fat_extract::native::{extract_gzip_member, GzipExtractionOptions}; use fat_extract::rootfs::{find_all_trees, find_rootfs}; -use fat_extract::{cramfs::extract_cramfs, cramfs::CramfsLimits}; use sha2::{Digest, Sha256}; use std::error::Error; use std::ffi::OsString; @@ -3675,7 +3672,10 @@ fn cmd_extract( engine_reports.extend(outcomes.iter().map(engine_report_for)); - if !attempted_tools.is_empty() && !tool_succeeded { + if !attempted_tools.is_empty() + && !tool_succeeded + && !outcomes.iter().any(|outcome| outcome.evidence.has_rootfs()) + { let native_dir = extraction_root.join("native"); if file_count(&native_dir) == 0 { remove_existing_path_if_present(&extraction_root)?; @@ -3727,12 +3727,15 @@ fn cmd_extract( } else { crate::style::PipelineProgress::none() }; - let mut manifest = recover_rootfs_if_needed( - build_extraction_manifest(&extraction_root)?, - &extraction_root, - &firmware_path, - &project_dir.join("work").join("rootfs-fallback.log"), - )?; + let mut manifest = build_extraction_manifest(&extraction_root)?; + if !matches!(&extractor_selection, ExtractorSelection::Only(engine) if engine == "native") { + manifest = recover_rootfs_if_needed( + manifest, + &extraction_root, + &firmware_path, + &project_dir.join("work").join("rootfs-fallback.log"), + )?; + } drop(recover_spinner); manifest.engine = attribute_engine(&manifest, &extraction_root, &engine_reports); if let Some(engine) = manifest.engine.clone() { @@ -3749,6 +3752,20 @@ fn cmd_extract( .unwrap_or_default(); } manifest.engine_reports = engine_reports; + manifest.artifacts = engine_outcomes + .iter() + .flat_map(|outcome| outcome.artifacts.clone()) + .collect(); + manifest.recovery_status = Some( + if engine_outcomes.iter().any(|outcome| outcome.incomplete) { + "partial" + } else if manifest.rootfs_path.is_some() { + "rootfs_recovered" + } else { + "files_only" + } + .to_string(), + ); if manifest.file_count == 0 { let mut message = "firmware extraction produced no recoverable files; refusing an empty success state" @@ -3932,6 +3949,8 @@ fn extract_json( "engine": manifest.engine, "engine_version": manifest.engine_version, "engine_args": manifest.engine_args, + "recovery_status": manifest.recovery_status, + "artifacts": manifest.artifacts, "engines": engines, "rootfs": rootfs, "kernel_paths": kernel_paths, @@ -4004,40 +4023,80 @@ impl Extractor for NativeExtractor { _on_tick: &mut dyn FnMut(Duration), ) -> ExtractionOutcome { let started = Instant::now(); - let outcome = try_extract_native_regions( + let result = fat_extract::pipeline::extract( &request.firmware, - request - .work_dir - .parent() - .unwrap_or(&request.work_dir) - .to_path_buf() - .as_path(), - &request.log_path, + &request.work_dir, + fat_extract::pipeline::PipelineOptions::default(), ); - let duration = started.elapsed(); - - match outcome { - Ok(NativeExtractionOutcome::RootfsRecovered) => { - ExtractionOutcome::new(self.id(), ExtractStatus::Succeeded) - .with_detail("recovered a rootfs") - .with_evidence(CarvedEvidence::survey(&request.work_dir)) - .with_duration(duration) - } - Ok(NativeExtractionOutcome::EvidenceOnly) => { - ExtractionOutcome::new(self.id(), ExtractStatus::Succeeded) - .with_detail("carved boot evidence without a rootfs") - .with_evidence(CarvedEvidence::survey(&request.work_dir)) - .with_duration(duration) - } - Ok(NativeExtractionOutcome::NoSupportedRegion) => { - ExtractionOutcome::new(self.id(), ExtractStatus::Skipped) - .with_detail("no supported regions in the image") - .with_duration(duration) + let mut outcome = match result { + Ok(report) => { + let evidence = CarvedEvidence::survey(&request.work_dir); + let recovered = report + .artifacts + .iter() + .filter(|artifact| { + artifact.status == fat_extract::pipeline::ArtifactStatus::Recovered + }) + .count(); + let status = if recovered > 0 { + ExtractStatus::Succeeded + } else if report.artifacts.is_empty() { + ExtractStatus::Skipped + } else { + ExtractStatus::Failed + }; + let mut outcome = ExtractionOutcome::new(self.id(), status) + .with_detail(format!( + "{recovered} artifacts recovered; rootfs {}", + if evidence.has_rootfs() { + "recovered" + } else { + "not recovered" + } + )) + .with_evidence(evidence); + if report.artifacts.is_empty() { + outcome.detail = Some("no supported regions in the image".into()); + } + if report.truncated { + outcome.detail = Some(format!( + "{recovered} artifacts recovered; artifact budget reached" + )); + } + outcome.incomplete = report.has_unresolved(); + outcome.artifacts = report.artifacts; + outcome } - Err(err) => ExtractionOutcome::new(self.id(), ExtractStatus::Failed) - .with_detail(err.to_string()) - .with_duration(duration), + Err(error) => ExtractionOutcome::new(self.id(), ExtractStatus::Failed) + .with_detail(error.to_string()), + }; + outcome.duration = started.elapsed(); + if let Some(parent) = request.log_path.parent() { + let _ = fs::create_dir_all(parent); } + let mut log = format!("{}\n", outcome.detail.as_deref().unwrap_or_default()); + for artifact in &outcome.artifacts { + let _ = writeln!( + log, + "{} @ 0x{:X}: {:?}{}", + artifact.format, + artifact.offset, + artifact.status, + artifact + .detail + .as_ref() + .map(|detail| format!(" ({detail})")) + .unwrap_or_default() + ); + } + let _ = fs::write(&request.log_path, log); + if let Ok(bytes) = serde_json::to_vec_pretty(&outcome.artifacts) { + let _ = fs::write( + request.log_path.with_file_name("native-artifacts.json"), + bytes, + ); + } + outcome } } @@ -4262,6 +4321,9 @@ const SUMMARY_LIST_CAP: usize = 10; /// rootfs is and how large, where each kernel landed, and what the tree looks /// like at its top level — instead of bare counts that need a follow-up `ls`. fn print_extraction_artifacts(manifest: &ExtractionManifest, project_dir: &Path) { + if let Some(status) = &manifest.recovery_status { + println!("recovery: {status}"); + } match manifest.rootfs_path.as_deref() { Some(rootfs) => println!("rootfs: {}", describe_tree(rootfs, project_dir)), None => println!("rootfs: not found"), @@ -4335,6 +4397,10 @@ fn extract_panel_lines( lines.push(format!("{} {}", palette.dot_warn(), palette.muted(note))); } + if let Some(status) = &manifest.recovery_status { + lines.push(palette.kv("recovery", status)); + } + if !manifest.engine_reports.is_empty() { lines.push(String::new()); lines.push(palette.heading("Engines")); @@ -4610,175 +4676,6 @@ fn engine_for_extraction_path( } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum NativeExtractionOutcome { - RootfsRecovered, - EvidenceOnly, - NoSupportedRegion, -} - -fn try_extract_native_regions( - firmware_path: &Path, - extraction_root: &Path, - log_path: &Path, -) -> DynResult { - let bytes = fs::read(firmware_path)?; - let scan = scan_firmware_formats(&bytes); - let native_root = extraction_root.join("native"); - let mut log = String::new(); - let mut recovered_rootfs = false; - let mut recovered_evidence = false; - - for (index, header) in scan - .filesystem_headers - .iter() - .filter(|header| header.format == "cramfs") - .enumerate() - { - let Some(image_size) = header.image_size else { - continue; - }; - let Some(start) = usize::try_from(header.offset).ok() else { - log.push_str("skipped CramFS region whose offset does not fit this platform\n"); - continue; - }; - let Some(end_u64) = header.offset.checked_add(image_size) else { - log.push_str("skipped CramFS region whose span overflows\n"); - continue; - }; - let Some(end) = usize::try_from(end_u64).ok() else { - log.push_str("skipped CramFS region whose end does not fit this platform\n"); - continue; - }; - let Some(image) = bytes.get(start..end) else { - log.push_str("skipped CramFS region whose declared span exceeds the firmware\n"); - continue; - }; - let directory_name = if index == 0 { - "cramfs-root".to_string() - } else { - format!("cramfs-root-{}", index + 1) - }; - let destination = native_root.join(directory_name); - match extract_cramfs( - image, - &destination, - CramfsLimits { - max_inodes: 262_144, - max_depth: 128, - max_file_size: 16 * 1024 * 1024, - max_total_output: 512 * 1024 * 1024, - }, - ) { - Ok(result) => { - recovered_rootfs = true; - recovered_evidence = true; - writeln!( - log, - "extracted CramFS @ 0x{:08X} to {} ({} files, {} directories, {} symlinks, {} special entries skipped)", - header.offset, - result.root.display(), - result.files, - result.directories, - result.symlinks, - result.skipped_special, - )?; - } - Err(error) => { - writeln!( - log, - "native CramFS extraction failed @ 0x{:08X}: {error}", - header.offset - )?; - } - } - } - - for (index, member) in scan - .compression_members - .iter() - .filter(|member| member.format == "gzip") - .filter(|member| { - !scan.filesystem_headers.iter().any(|header| { - header.image_size.is_some_and(|size| { - member.offset > header.offset - && member.offset < header.offset.saturating_add(size) - }) - }) - }) - .enumerate() - { - let Some(compressed_size) = member.compressed_size else { - continue; - }; - let Some(start) = usize::try_from(member.offset).ok() else { - log.push_str("skipped gzip member whose offset does not fit this platform\n"); - continue; - }; - let Some(end_u64) = member.offset.checked_add(compressed_size) else { - log.push_str("skipped gzip member whose span overflows\n"); - continue; - }; - let Some(end) = usize::try_from(end_u64).ok() else { - log.push_str("skipped gzip member whose end does not fit this platform\n"); - continue; - }; - let Some(member_bytes) = bytes.get(start..end) else { - log.push_str("skipped gzip member whose span exceeds the firmware\n"); - continue; - }; - let directory_name = if index == 0 { - "kernel".to_string() - } else { - format!("kernel-{}", index + 1) - }; - let destination = native_root.join(directory_name); - match extract_gzip_member( - member_bytes, - &destination, - GzipExtractionOptions { - original_name: member.original_name.clone(), - max_output: 128 * 1024 * 1024, - }, - ) { - Ok(result) => { - recovered_evidence = true; - writeln!( - log, - "extracted gzip @ 0x{:08X} to {} ({} bytes)", - member.offset, - result.decompressed_path.display(), - result.decompressed_size - )?; - } - Err(error) => { - writeln!( - log, - "native gzip extraction failed @ 0x{:08X}: {error}", - member.offset - )?; - } - } - } - - if !recovered_evidence { - remove_existing_path_if_present(&native_root)?; - log.push_str("no supported native region was materialized\n"); - } - if let Some(parent) = log_path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(log_path, log)?; - - Ok(if recovered_rootfs { - NativeExtractionOutcome::RootfsRecovered - } else if recovered_evidence { - NativeExtractionOutcome::EvidenceOnly - } else { - NativeExtractionOutcome::NoSupportedRegion - }) -} - fn fail_extract_with_error( db: &ProjectDb, project: &mut Project, diff --git a/crates/fat_extract/Cargo.toml b/crates/fat_extract/Cargo.toml index 3378ef8..c432543 100644 --- a/crates/fat_extract/Cargo.toml +++ b/crates/fat_extract/Cargo.toml @@ -9,7 +9,7 @@ readme.workspace = true repository.workspace = true keywords.workspace = true categories.workspace = true -description = "Firmware extraction wrapper around binwalk/unblob for FAT" +description = "Bounded firmware container and filesystem extraction for FAT" publish = false [lib] @@ -19,9 +19,14 @@ path = "src/lib.rs" [dependencies] crc32fast = "1" flate2 = "1" +lzma-rs = "0.3" +aho-corasick = "1" serde = { version = "1", features = ["derive"] } tar = "0.4" +tempfile = "3" walkdir = "2" +xz2 = "0.1" +zip = { version = "2", default-features = false, features = ["deflate"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/fat_extract/src/archive.rs b/crates/fat_extract/src/archive.rs new file mode 100644 index 0000000..1f7c034 --- /dev/null +++ b/crates/fat_extract/src/archive.rs @@ -0,0 +1,103 @@ +use std::io::{self, Cursor, Read}; +use std::path::{Path, PathBuf}; + +use crate::output::OutputTree; + +pub(crate) fn is_tar(bytes: &[u8]) -> bool { + let Some(block) = bytes.get(..512) else { + return false; + }; + let header = tar::Header::from_byte_slice(block); + let Ok(expected) = header.cksum() else { + return false; + }; + let sum = block + .iter() + .enumerate() + .map(|(i, b)| { + if (148..156).contains(&i) { + 32 + } else { + *b as u32 + } + }) + .sum::(); + expected == sum && expected != 0 +} + +pub(crate) fn unpack_tar(bytes: &[u8], output: &mut OutputTree) -> io::Result> { + let mut archive = tar::Archive::new(Cursor::new(bytes)); + let mut children = Vec::new(); + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let kind = entry.header().entry_type(); + let mode = entry.header().mode()?; + if kind.is_dir() { + output.create_dir(&path, mode)?; + } else if kind.is_file() { + output.write_reader(&path, &mut entry, mode)?; + children.push(path); + } else if kind.is_symlink() { + let target = entry + .link_name()? + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing link target"))?; + output.write_symlink(&path, &target)?; + } else if kind.is_block_special() || kind.is_character_special() || kind.is_fifo() { + output.skip_special()?; + } else { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "unsupported TAR entry type", + )); + } + } + Ok(children) +} + +pub(crate) fn unpack_zip(bytes: &[u8], output: &mut OutputTree) -> io::Result> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes))?; + if archive.len() as u64 > output.limits().max_entries { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "entry budget exceeded", + )); + } + let mut children = Vec::new(); + for i in 0..archive.len() { + let mut entry = archive.by_index(i)?; + let path = entry.enclosed_name().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "ZIP path leaves the extraction tree", + ) + })?; + let mode = entry.unix_mode().unwrap_or(0o644); + if entry.is_dir() { + output.create_dir(&path, mode | 0o700)?; + } else if mode & 0o170000 == 0o120000 { + let mut target = Vec::new(); + entry.by_ref().take(4097).read_to_end(&mut target)?; + if target.len() > 4096 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "link target exceeds limit", + )); + } + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + output.write_symlink(&path, Path::new(std::ffi::OsStr::from_bytes(&target)))?; + } + #[cfg(not(unix))] + output.write_symlink( + &path, + Path::new(std::str::from_utf8(&target).map_err(io::Error::other)?), + )?; + } else { + output.write_reader(&path, &mut entry, mode)?; + children.push(path); + } + } + Ok(children) +} diff --git a/crates/fat_extract/src/cramfs.rs b/crates/fat_extract/src/cramfs.rs index fbd6cee..99887d2 100644 --- a/crates/fat_extract/src/cramfs.rs +++ b/crates/fat_extract/src/cramfs.rs @@ -41,6 +41,7 @@ pub struct CramfsExtractionResult { pub directories: u64, pub symlinks: u64, pub skipped_special: u64, + pub bytes: u64, } #[derive(Debug)] @@ -120,6 +121,7 @@ pub fn extract_cramfs( directories: 1, symlinks: 0, skipped_special: 0, + bytes: 0, }, }; @@ -128,6 +130,7 @@ pub fn extract_cramfs( return Err(invalid("CramFS root inode is not a directory")); } state.extract_tree(root, &root_path)?; + state.result.bytes = state.total_output; if header .declared_inodes .is_some_and(|count| count != state.inode_count) diff --git a/crates/fat_extract/src/extractors.rs b/crates/fat_extract/src/extractors.rs index 5155ddb..fb4b15b 100644 --- a/crates/fat_extract/src/extractors.rs +++ b/crates/fat_extract/src/extractors.rs @@ -38,6 +38,8 @@ pub struct ExtractionOutcome { pub requires_external_tool: bool, /// Exact arguments the extractor was invoked with, for reproducibility. pub args: Vec, + pub artifacts: Vec, + pub incomplete: bool, } impl ExtractionOutcome { @@ -50,6 +52,8 @@ impl ExtractionOutcome { duration: Duration::ZERO, requires_external_tool: true, args: Vec::new(), + artifacts: Vec::new(), + incomplete: false, } } @@ -266,6 +270,7 @@ pub fn run_extraction_strategy( if selection.preempts() && outcome.status.succeeded() + && !outcome.incomplete && extractor.sufficiency().is_met(&outcome.evidence) { preempted_by = extractor.precedence_reason(&outcome.evidence); @@ -375,6 +380,10 @@ impl Extractor for ExternalExtractor { self.id } + fn sufficiency(&self) -> Sufficiency { + Sufficiency::RootfsOnly + } + fn extract( &self, request: &ExtractionRequest, diff --git a/crates/fat_extract/src/lib.rs b/crates/fat_extract/src/lib.rs index 23c4141..c883f10 100644 --- a/crates/fat_extract/src/lib.rs +++ b/crates/fat_extract/src/lib.rs @@ -1,10 +1,14 @@ +mod archive; pub mod command; pub mod cramfs; pub mod evidence; pub mod extractors; pub mod manifest; pub mod native; +pub mod output; +pub mod pipeline; pub mod rootfs; pub mod safe_tar; +pub mod squashfs; pub use manifest::ExtractionManifest; diff --git a/crates/fat_extract/src/manifest.rs b/crates/fat_extract/src/manifest.rs index d78c06b..98fd459 100644 --- a/crates/fat_extract/src/manifest.rs +++ b/crates/fat_extract/src/manifest.rs @@ -64,6 +64,10 @@ impl EngineReport { #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] pub struct ExtractionManifest { + #[serde(default)] + pub artifacts: Vec, + #[serde(default)] + pub recovery_status: Option, pub rootfs_path: Option, pub kernel_paths: Vec, pub file_count: usize, diff --git a/crates/fat_extract/src/output.rs b/crates/fat_extract/src/output.rs new file mode 100644 index 0000000..102957f --- /dev/null +++ b/crates/fat_extract/src/output.rs @@ -0,0 +1,296 @@ +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ExtractionLimits { + pub max_entries: u64, + pub max_file_bytes: u64, + pub max_output_bytes: u64, + pub max_depth: usize, +} + +impl Default for ExtractionLimits { + fn default() -> Self { + Self { + max_entries: 262_144, + max_file_bytes: 256 * 1024 * 1024, + max_output_bytes: 1024 * 1024 * 1024, + max_depth: 64, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExtractionStats { + pub files: u64, + pub directories: u64, + pub symlinks: u64, + pub skipped_special: u64, + pub bytes: u64, +} + +impl ExtractionStats { + pub fn entries(&self) -> u64 { + self.files + self.directories + self.symlinks + self.skipped_special + } +} + +enum Entry { + Directory(u32), + File, + Symlink(PathBuf), +} + +pub struct OutputTree { + destination: PathBuf, + staging: tempfile::TempDir, + entries: BTreeMap, + limits: ExtractionLimits, + stats: ExtractionStats, +} + +fn invalid(message: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +impl OutputTree { + pub fn new(destination: &Path, limits: ExtractionLimits) -> io::Result { + if destination.symlink_metadata().is_ok() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "output already exists", + )); + } + if limits.max_entries == 0 { + return Err(invalid("entry budget exceeded")); + } + let parent = destination + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::create_dir_all(parent)?; + let staging = tempfile::Builder::new() + .prefix(".extract-") + .tempdir_in(parent)?; + Ok(Self { + destination: destination.to_path_buf(), + staging, + entries: BTreeMap::from([(PathBuf::new(), Entry::Directory(0o755))]), + limits, + stats: ExtractionStats { + directories: 1, + ..Default::default() + }, + }) + } + + pub fn limits(&self) -> ExtractionLimits { + self.limits + } + + pub fn stats(&self) -> &ExtractionStats { + &self.stats + } + + fn normalize(&self, path: &Path) -> io::Result { + let mut result = PathBuf::new(); + for component in path.components() { + match component { + Component::Normal(name) => result.push(name), + Component::CurDir => {} + _ => return Err(invalid("output path leaves the extraction tree")), + } + } + if result.components().count() > self.limits.max_depth { + return Err(invalid("path depth budget exceeded")); + } + Ok(result) + } + + fn reserve_entry(&self) -> io::Result<()> { + if self.stats.entries() >= self.limits.max_entries { + return Err(invalid("entry budget exceeded")); + } + Ok(()) + } + + fn parents(&mut self, path: &Path) -> io::Result<()> { + if let Some(parent) = path.parent() { + match self.entries.get(parent) { + Some(Entry::Directory(_)) => {} + Some(_) => return Err(invalid("output parent is not a directory")), + None => self.create_dir(parent, 0o755)?, + } + } + Ok(()) + } + + pub fn create_dir(&mut self, relative: &Path, mode: u32) -> io::Result<()> { + let path = self.normalize(relative)?; + match self.entries.get_mut(&path) { + Some(Entry::Directory(existing)) => { + *existing = mode; + return Ok(()); + } + Some(_) => return Err(invalid("output directory collides with another entry")), + None => {} + } + self.parents(&path)?; + self.reserve_entry()?; + fs::create_dir(self.staging.path().join(&path)) + .map_err(|error| output_error(error, &path))?; + self.entries.insert(path, Entry::Directory(mode)); + self.stats.directories += 1; + Ok(()) + } + + pub fn write_file(&mut self, relative: &Path, bytes: &[u8], mode: u32) -> io::Result<()> { + if bytes.len() as u64 > self.limits.max_file_bytes + || bytes.len() as u64 + > self + .limits + .max_output_bytes + .saturating_sub(self.stats.bytes) + { + return Err(invalid("output byte budget exceeded")); + } + self.write_reader(relative, bytes, mode).map(|_| ()) + } + + pub fn write_reader( + &mut self, + relative: &Path, + mut reader: impl Read, + mode: u32, + ) -> io::Result { + let path = self.normalize(relative)?; + if self.entries.contains_key(&path) { + return Err(invalid("duplicate output entry")); + } + self.parents(&path)?; + self.reserve_entry()?; + let destination = self.staging.path().join(&path); + let mut file = + File::create_new(&destination).map_err(|error| output_error(error, &path))?; + let budget = self.limits.max_file_bytes.min( + self.limits + .max_output_bytes + .saturating_sub(self.stats.bytes), + ); + let result = (|| { + let mut buffer = [0u8; 64 * 1024]; + let mut count = 0; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + if read as u64 > budget.saturating_sub(count) { + return Err(invalid("output byte budget exceeded")); + } + file.write_all(&buffer[..read])?; + count += read as u64; + } + set_mode(&destination, mode)?; + Ok(count) + })(); + drop(file); + match result { + Ok(count) => { + self.entries.insert(path, Entry::File); + self.stats.files += 1; + self.stats.bytes += count; + Ok(count) + } + Err(error) => { + let _ = fs::remove_file(destination); + Err(error) + } + } + } + + pub fn write_symlink(&mut self, relative: &Path, target: &Path) -> io::Result<()> { + let length = target.as_os_str().len() as u64; + if length > 4096 + || length > self.limits.max_file_bytes + || length + > self + .limits + .max_output_bytes + .saturating_sub(self.stats.bytes) + { + return Err(invalid("symlink target exceeds output byte budget")); + } + let path = self.normalize(relative)?; + if self.entries.contains_key(&path) { + return Err(invalid("duplicate output entry")); + } + self.parents(&path)?; + self.reserve_entry()?; + self.entries + .insert(path, Entry::Symlink(target.to_path_buf())); + self.stats.symlinks += 1; + self.stats.bytes += length; + Ok(()) + } + + pub fn skip_special(&mut self) -> io::Result<()> { + self.reserve_entry()?; + self.stats.skipped_special += 1; + Ok(()) + } + + pub fn finish(self) -> io::Result { + for (path, entry) in &self.entries { + if let Entry::Symlink(target) = entry { + #[cfg(unix)] + std::os::unix::fs::symlink(target, self.staging.path().join(path)) + .map_err(|error| output_error(error, path))?; + #[cfg(not(unix))] + { + let _ = (target, path); + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "symlink output requires Unix", + )); + } + } + } + for (path, entry) in self.entries.iter().rev() { + if let Entry::Directory(mode) = entry { + set_mode(&self.staging.path().join(path), *mode)?; + } + } + if self.destination.symlink_metadata().is_ok() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "output already exists", + )); + } + fs::rename(self.staging.path(), &self.destination)?; + Ok(self.stats) + } +} + +fn output_error(error: io::Error, path: &Path) -> io::Error { + if error.kind() == io::ErrorKind::AlreadyExists { + io::Error::new(error.kind(), format!("host filesystem cannot represent distinct output names at {}; use a case-sensitive output volume", path.display())) + } else { + error + } +} + +fn set_mode(path: &Path, mode: u32) -> io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(mode & 0o777))?; + } + #[cfg(not(unix))] + let _ = (path, mode); + Ok(()) +} diff --git a/crates/fat_extract/src/pipeline.rs b/crates/fat_extract/src/pipeline.rs new file mode 100644 index 0000000..ce63120 --- /dev/null +++ b/crates/fat_extract/src/pipeline.rs @@ -0,0 +1,400 @@ +use std::collections::VecDeque; +use std::fs; +use std::io::{self, Cursor, Read}; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use serde::{Deserialize, Serialize}; + +use crate::archive; +use crate::cramfs::{extract_cramfs, CramfsLimits}; +use crate::output::{ExtractionLimits, ExtractionStats, OutputTree}; +use crate::squashfs; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactStatus { + Recovered, + Unsupported, + Failed, + Limited, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactRecord { + pub id: usize, + pub parent: Option, + pub source: PathBuf, + pub offset: u64, + pub size: u64, + pub format: String, + pub decoder: String, + pub status: ArtifactStatus, + pub output: Option, + pub stats: ExtractionStats, + pub detail: Option, + pub duration_ms: u64, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct PipelineReport { + pub artifacts: Vec, + pub truncated: bool, +} + +impl PipelineReport { + pub fn has_unresolved(&self) -> bool { + self.truncated + || self + .artifacts + .iter() + .any(|a| a.status != ArtifactStatus::Recovered) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct PipelineOptions { + pub limits: ExtractionLimits, + pub max_input_bytes: u64, + pub max_container_depth: usize, + pub max_artifacts: usize, +} + +impl Default for PipelineOptions { + fn default() -> Self { + Self { + limits: ExtractionLimits::default(), + max_input_bytes: 512 * 1024 * 1024, + max_container_depth: 8, + max_artifacts: 4096, + } + } +} + +struct WorkItem { + path: PathBuf, + parent: Option, + depth: usize, +} + +#[derive(Clone, Copy)] +enum Format { + Zip, + Tar, + Gzip, + Squashfs, + Cramfs, +} + +impl Format { + fn label(self) -> &'static str { + match self { + Self::Zip => "zip", + Self::Tar => "tar", + Self::Gzip => "gzip", + Self::Squashfs => "squashfs", + Self::Cramfs => "cramfs", + } + } +} + +pub fn extract( + input: &Path, + destination: &Path, + options: PipelineOptions, +) -> io::Result { + if options.max_artifacts == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "artifact budget must be positive", + )); + } + let scanner = + aho_corasick::AhoCorasick::new(SIGNATURES.iter().map(|(signature, _)| *signature)) + .map_err(io::Error::other)?; + let mut queue = VecDeque::from([WorkItem { + path: input.to_path_buf(), + parent: None, + depth: 0, + }]); + let mut report = PipelineReport::default(); + let mut remaining = options.limits; + let mut cramfs_count = 0; + let mut gzip_count = 0; + while let Some(item) = queue.pop_front() { + if report.artifacts.len() >= options.max_artifacts { + report.truncated = true; + return Ok(report); + } + let read_result = (|| -> io::Result> { + if item.depth > options.max_container_depth { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "container depth budget exceeded", + )); + } + if fs::metadata(&item.path)?.len() > options.max_input_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "input size budget exceeded", + )); + } + let mut bytes = Vec::new(); + fs::File::open(&item.path)? + .take(options.max_input_bytes.saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > options.max_input_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "input size budget exceeded", + )); + } + Ok(bytes) + })(); + let bytes = match read_result { + Ok(bytes) => bytes, + Err(error) => { + report.artifacts.push(ArtifactRecord { + id: report.artifacts.len(), + parent: item.parent, + source: item.path, + offset: 0, + size: 0, + format: "uninspected".into(), + decoder: "native".into(), + status: error_status(&error), + output: None, + stats: ExtractionStats::default(), + detail: Some(error.to_string()), + duration_ms: 0, + }); + continue; + } + }; + let whole = if bytes.starts_with(b"PK\x03\x04") || bytes.starts_with(b"PK\x05\x06") { + Some(Format::Zip) + } else if archive::is_tar(&bytes) { + Some(Format::Tar) + } else { + None + }; + let mut cursor = 0; + loop { + let next = if let Some(format) = whole { + (cursor == 0).then_some((0, format)) + } else { + next_region(&scanner, &bytes, cursor) + }; + let Some((offset, format)) = next else { + break; + }; + let id = report.artifacts.len(); + if id >= options.max_artifacts { + report.truncated = true; + return Ok(report); + } + let path = match format { + Format::Squashfs => destination + .join(format!("{id:04}-{offset:08x}")) + .join("squashfs-root"), + Format::Cramfs => { + cramfs_count += 1; + destination.join(if cramfs_count == 1 { + "cramfs-root".into() + } else { + format!("cramfs-root-{cramfs_count}") + }) + } + Format::Gzip => { + gzip_count += 1; + destination.join(if gzip_count == 1 { + "kernel".into() + } else { + format!("kernel-{gzip_count}") + }) + } + _ => destination.join(format!("{id:04}-{}", format.label())), + }; + let mut record = ArtifactRecord { + id, + parent: item.parent, + source: item.path.clone(), + offset: offset as u64, + size: (bytes.len() - offset) as u64, + format: format.label().into(), + decoder: format!("native-{}", format.label()), + status: ArtifactStatus::Failed, + output: None, + stats: ExtractionStats::default(), + detail: None, + duration_ms: 0, + }; + let started = Instant::now(); + let result = decode(format, &bytes[offset..], &path, remaining, &mut record); + record.duration_ms = started.elapsed().as_millis().min(u64::MAX as u128) as u64; + cursor = offset + 4; + match result { + Ok(children) => { + remaining.max_output_bytes = remaining + .max_output_bytes + .saturating_sub(record.stats.bytes); + remaining.max_entries = + remaining.max_entries.saturating_sub(record.stats.entries()); + record.status = ArtifactStatus::Recovered; + record.output = Some(path.clone()); + cursor = offset.saturating_add(record.size as usize).max(cursor); + let mut children = children; + children.sort(); + queue.extend(children.into_iter().map(|child| WorkItem { + path: path.join(child), + parent: Some(id), + depth: item.depth + 1, + })); + } + Err(error) => { + record.status = error_status(&error); + record.detail = Some(error.to_string()); + } + } + report.artifacts.push(record); + } + } + Ok(report) +} + +const SIGNATURES: &[(&[u8], Format)] = &[ + (b"hsqs", Format::Squashfs), + (b"sqsh", Format::Squashfs), + (b"qshs", Format::Squashfs), + (b"shsq", Format::Squashfs), + (b"sqlz", Format::Squashfs), + (b"hsqt", Format::Squashfs), + (b"tqsh", Format::Squashfs), + (b"\x45\x3d\xcd\x28", Format::Cramfs), + (b"\x28\xcd\x3d\x45", Format::Cramfs), + (b"\x1f\x8b\x08", Format::Gzip), +]; + +fn next_region( + scanner: &aho_corasick::AhoCorasick, + bytes: &[u8], + mut start: usize, +) -> Option<(usize, Format)> { + while let Some(tail) = bytes.get(start..) { + let matched = scanner.find(tail)?; + let offset = start + matched.start(); + let format = SIGNATURES[matched.pattern().as_usize()].1; + let candidate = &bytes[offset..]; + let plausible = match format { + Format::Squashfs => candidate.get(28..32).is_some_and(|version| { + let little = u16::from_le_bytes([version[0], version[1]]); + let big = u16::from_be_bytes([version[0], version[1]]); + ((1..=16).contains(&little) && u16::from_le_bytes([version[2], version[3]]) <= 16) + || ((1..=16).contains(&big) + && u16::from_be_bytes([version[2], version[3]]) <= 16) + }), + Format::Gzip => candidate.len() >= 10 && candidate[3] & 0xe0 == 0, + Format::Cramfs => candidate.get(16..32) == Some(b"Compressed ROMFS"), + _ => true, + }; + if offset == 0 || plausible { + return Some((offset, format)); + } + start += matched.end(); + } + None +} + +fn error_status(error: &io::Error) -> ArtifactStatus { + if error.kind() == io::ErrorKind::Unsupported { + ArtifactStatus::Unsupported + } else if error.to_string().contains("budget") || error.to_string().contains("limit") { + ArtifactStatus::Limited + } else { + ArtifactStatus::Failed + } +} + +fn decode( + format: Format, + bytes: &[u8], + path: &Path, + limits: ExtractionLimits, + record: &mut ArtifactRecord, +) -> io::Result> { + if matches!(format, Format::Cramfs) { + let size_bytes: [u8; 4] = bytes + .get(4..8) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "truncated CramFS header"))? + .try_into() + .unwrap(); + let size = if bytes.starts_with(b"\x45\x3d\xcd\x28") { + u32::from_le_bytes(size_bytes) + } else { + u32::from_be_bytes(size_bytes) + }; + let image = bytes + .get(..size as usize) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "truncated CramFS image"))?; + let result = extract_cramfs( + image, + path, + CramfsLimits { + max_inodes: limits.max_entries.min(u32::MAX as u64) as u32, + max_depth: limits.max_depth as u32, + max_file_size: limits.max_file_bytes, + max_total_output: limits.max_output_bytes, + }, + ) + .map_err(io::Error::other)?; + record.size = size as u64; + record.stats = ExtractionStats { + files: result.files, + directories: result.directories, + symlinks: result.symlinks, + skipped_special: result.skipped_special, + bytes: result.bytes, + }; + return Ok(Vec::new()); + } + let mut output = OutputTree::new(path, limits)?; + let children = match format { + Format::Zip => archive::unpack_zip(bytes, &mut output)?, + Format::Tar => archive::unpack_tar(bytes, &mut output)?, + Format::Gzip => { + let mut decoder = flate2::bufread::GzDecoder::new(Cursor::new(bytes)); + let name = decoder + .header() + .and_then(|header| header.filename()) + .and_then(|name| std::str::from_utf8(name).ok()) + .filter(|name| { + !name.is_empty() + && !name.contains(['/', '\\', '\0']) + && !matches!(*name, "." | "..") + }) + .unwrap_or("decompressed.bin") + .to_string(); + output.write_reader(Path::new(&name), &mut decoder, 0o644)?; + record.size = decoder.into_inner().position(); + output.write_file( + Path::new(&format!("{name}.gz")), + &bytes[..record.size as usize], + 0o644, + )?; + vec![PathBuf::from(name)] + } + Format::Squashfs => { + let info = squashfs::probe(bytes)?; + record.size = info.image_size; + let info = squashfs::extract(bytes, &mut output)?; + record.detail = Some(format!( + "version {}, {} endian, {}", + info.version, info.endianness, info.compression + )); + Vec::new() + } + Format::Cramfs => unreachable!(), + }; + record.stats = output.finish()?; + Ok(children) +} diff --git a/crates/fat_extract/src/rootfs.rs b/crates/fat_extract/src/rootfs.rs index b63798f..e2b1f38 100644 --- a/crates/fat_extract/src/rootfs.rs +++ b/crates/fat_extract/src/rootfs.rs @@ -59,7 +59,15 @@ fn is_unblob_filesystem_tree(path: &Path) -> bool { if unblob_filesystem_kind(&name.to_ascii_lowercase()).is_none() { return false; } - fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some()) + has_tree_content(path) +} + +fn has_tree_content(path: &Path) -> bool { + walkdir::WalkDir::new(path) + .follow_links(false) + .into_iter() + .filter_map(Result::ok) + .any(|entry| entry.file_type().is_file() || entry.file_type().is_symlink()) } pub fn find_rootfs(dir: impl AsRef) -> Option { @@ -137,7 +145,7 @@ pub fn find_all_trees(dir: impl AsRef) -> Vec<(String, PathBuf)> { .any(|pat| dir_name == *pat || dir_name.starts_with(&format!("{pat}-"))) || is_unblob_filesystem_tree(&path); - if is_known_fs { + if is_known_fs && has_tree_content(&path) { let label = infer_tree_label(&path, &dir_name); // Don't add duplicates (same canonical path) if !trees.iter().any(|(_, p)| p == &path) { @@ -180,13 +188,20 @@ fn is_rootfs_dir(path: &Path) -> bool { .and_then(|name| name.to_str()) .map(|name| ROOTFS_CANDIDATES.contains(&name.to_ascii_lowercase().as_str())) .unwrap_or(false); - name_matched || looks_like_rootfs_tree(path) || is_unblob_filesystem_tree(path) + looks_like_rootfs_tree(path) + || ((name_matched || is_unblob_filesystem_tree(path)) + && has_file_entry(&path.join("bin/busybox"))) +} + +fn has_file_entry(path: &Path) -> bool { + fs::symlink_metadata(path) + .is_ok_and(|metadata| metadata.is_file() || metadata.file_type().is_symlink()) } fn looks_like_rootfs_tree(path: &Path) -> bool { - let has_boot_anchor = path.join("sbin").join("init").is_file() - || path.join("etc").join("inittab").is_file() - || path.join("init").is_file(); + let has_boot_anchor = has_file_entry(&path.join("sbin/init")) + || has_file_entry(&path.join("etc/inittab")) + || has_file_entry(&path.join("init")); if !has_boot_anchor { return false; } @@ -213,6 +228,9 @@ fn infer_tree_label(path: &Path, dir_name: &str) -> String { if looks_like_rootfs_tree(path) { return "rootfs".to_string(); } + if dir_name == "rootfs" && !is_rootfs_dir(path) { + return "filesystem".to_string(); + } // If dir_name is a known FS pattern, use it directly for pat in FS_EXTRACTION_PATTERNS { diff --git a/crates/fat_extract/src/squashfs/codec.rs b/crates/fat_extract/src/squashfs/codec.rs new file mode 100644 index 0000000..b85a5af --- /dev/null +++ b/crates/fat_extract/src/squashfs/codec.rs @@ -0,0 +1,142 @@ +use std::io::{self, Cursor, Read, Write}; + +use flate2::read::ZlibDecoder; +use xz2::{read::XzDecoder, stream::Stream}; + +use super::{invalid, unsupported}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Codec { + Auto, + Zlib, + Lzma, + Xz, + Other(u16), +} + +impl Codec { + pub fn new(id: Option) -> Self { + match id { + None => Self::Auto, + Some(1) => Self::Zlib, + Some(2) => Self::Lzma, + Some(4) => Self::Xz, + Some(id) => Self::Other(id), + } + } + + pub fn name(self) -> &'static str { + match self { + Self::Auto => "undetermined", + Self::Zlib => "zlib", + Self::Lzma => "lzma", + Self::Xz => "xz", + Self::Other(3) => "lzo", + Self::Other(5) => "lz4", + Self::Other(6) => "zstd", + Self::Other(_) => "unknown", + } + } + + pub fn decode(&mut self, input: &[u8], limit: usize) -> io::Result> { + match self { + Self::Other(id) => Err(unsupported(format!("SquashFS compression {id}"))), + Self::Zlib => bounded_read(ZlibDecoder::new(input), limit), + Self::Xz => { + let stream = Stream::new_stream_decoder(64 * 1024 * 1024, 0)?; + bounded_read(XzDecoder::new_stream(input, stream), limit) + } + Self::Lzma => decode_lzma(input, limit), + Self::Auto => { + let (codec, decoded) = if input.starts_with(b"7zip") + || input.first().is_some_and(|v| *v == 0x5d || *v == 0x6d) + { + (Self::Lzma, decode_lzma(input, limit)?) + } else { + match bounded_read(ZlibDecoder::new(input), limit) { + Ok(data) => (Self::Zlib, data), + Err(_) => (Self::Lzma, decode_lzma(input, limit)?), + } + }; + *self = codec; + Ok(decoded) + } + } + } +} + +fn bounded_read(reader: impl Read, limit: usize) -> io::Result> { + let mut result = Vec::new(); + reader.take(limit as u64 + 1).read_to_end(&mut result)?; + if result.len() > limit { + return Err(invalid("SquashFS block exceeds decompression limit")); + } + Ok(result) +} + +fn decode_lzma(input: &[u8], limit: usize) -> io::Result> { + if input.starts_with(b"7zip") { + let mut header = [0u8; 13]; + header[0] = 0x5d; + header[1..5].copy_from_slice(&(1_048_576u32).to_le_bytes()); + header[5..].fill(0xff); + return lzma_reader(&header, &input[4..], limit); + } + if input.len() < 5 || input[0] >= 225 { + return Err(invalid("invalid SquashFS LZMA properties")); + } + if input.len() >= 13 { + let length = u64::from_le_bytes(input[5..13].try_into().unwrap()); + if length <= limit as u64 || length == u64::MAX { + if let Ok(decoded) = lzma_reader(&input[..13], &input[13..], limit) { + return Ok(decoded); + } + } + } + let mut header = [0xff; 13]; + header[..5].copy_from_slice(&input[..5]); + lzma_reader(&header, &input[5..], limit) +} + +fn lzma_reader(header: &[u8], payload: &[u8], limit: usize) -> io::Result> { + let properties = header[0]; + let lc = properties % 9; + let lp = (properties / 9) % 5; + let input = Cursor::new(header).chain(Cursor::new(payload)); + if lc + lp <= 4 { + let stream = Stream::new_lzma_decoder(64 * 1024 * 1024)?; + bounded_read(XzDecoder::new_stream(input, stream), limit) + } else { + let mut output = LimitedBuffer { + bytes: Vec::new(), + limit, + }; + let mut input = io::BufReader::new(input); + let options = lzma_rs::decompress::Options { + memlimit: Some(16 * 1024 * 1024), + ..Default::default() + }; + lzma_rs::lzma_decompress_with_options(&mut input, &mut output, &options) + .map_err(|error| invalid(format!("invalid SquashFS LZMA block: {error}")))?; + Ok(output.bytes) + } +} + +struct LimitedBuffer { + bytes: Vec, + limit: usize, +} + +impl Write for LimitedBuffer { + fn write(&mut self, data: &[u8]) -> io::Result { + if data.len() > self.limit.saturating_sub(self.bytes.len()) { + return Err(invalid("SquashFS block exceeds decompression limit")); + } + self.bytes.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/crates/fat_extract/src/squashfs/header.rs b/crates/fat_extract/src/squashfs/header.rs new file mode 100644 index 0000000..d978cbf --- /dev/null +++ b/crates/fat_extract/src/squashfs/header.rs @@ -0,0 +1,166 @@ +use std::io; + +use super::{invalid, unsupported, SquashfsInfo}; + +#[derive(Clone, Copy, Debug)] +pub(super) enum Endian { + Little, + Big, +} + +impl Endian { + pub fn bits(self, data: &[u8], bit: usize, width: usize) -> io::Result { + if width > 64 + || bit + .checked_add(width) + .is_none_or(|end| end > data.len() * 8) + { + return Err(invalid("truncated SquashFS field")); + } + let mut value = 0; + for i in 0..width { + let position = bit + i; + match self { + Self::Little => value |= (((data[position / 8] >> (position % 8)) & 1) as u64) << i, + Self::Big => { + value = (value << 1) | ((data[position / 8] >> (7 - position % 8)) & 1) as u64 + } + } + } + Ok(value) + } + + pub fn uint(self, data: &[u8], offset: usize, bytes: usize) -> io::Result { + self.bits(data, offset * 8, bytes * 8) + } +} + +pub(super) struct Header { + pub endian: Endian, + pub major: u16, + pub minor: u16, + pub inode_count: u64, + pub image_size: u64, + pub superblock_size: u64, + pub block_size: usize, + pub root: u64, + pub inode_table: u64, + pub directory_table: u64, + pub fragment_table: u64, + pub fragments: u64, + pub check_byte: bool, + pub compression: Option, +} + +impl Header { + pub fn parse(image: &[u8]) -> io::Result { + let endian = match image.get(..4) { + Some(b"hsqs" | b"shsq") => Endian::Little, + Some(b"sqsh" | b"qshs") => Endian::Big, + _ => return Err(invalid("SquashFS magic not found")), + }; + let get = |offset, bytes| endian.uint(image, offset, bytes); + let major = get(28, 2)? as u16; + let minor = get(30, 2)? as u16; + if !matches!((major, minor), (2, 0 | 1) | (3, 0 | 1) | (4, 0)) { + return Err(unsupported(format!("SquashFS version {major}.{minor}"))); + } + let ( + minimum, + image_size, + block_size, + root, + inode_table, + directory_table, + fragment_table, + fragments, + check_byte, + compression, + block_log, + ) = if major == 4 { + ( + 96, + get(40, 8)?, + get(12, 4)?, + get(32, 8)?, + get(64, 8)?, + get(72, 8)?, + get(80, 8)?, + get(16, 4)?, + false, + Some(get(20, 2)? as u16), + get(22, 2)?, + ) + } else { + ( + if major == 3 { 119 } else { 63 }, + if major == 3 { get(63, 8)? } else { get(8, 4)? }, + get(51, 4)?, + get(43, 8)?, + if major == 3 { get(87, 8)? } else { get(20, 4)? }, + if major == 3 { get(95, 8)? } else { get(24, 4)? }, + if major == 3 { + get(103, 8)? + } else { + get(59, 4)? + }, + get(55, 4)?, + get(36, 1)? & 4 != 0, + None, + get(34, 2)?, + ) + }; + if image_size < minimum || image_size > image.len() as u64 { + return Err(invalid("SquashFS declared size exceeds input")); + } + if !(4096..=1_048_576).contains(&block_size) + || !block_size.is_power_of_two() + || block_log != block_size.trailing_zeros() as u64 + { + return Err(invalid("invalid SquashFS block size")); + } + let inode_count = get(4, 4)?; + if inode_count == 0 + || inode_table < minimum + || inode_table >= directory_table + || directory_table >= image_size + { + return Err(invalid("invalid SquashFS table bounds")); + } + if fragments > inode_count + || (fragments > 0 && (fragment_table < directory_table || fragment_table >= image_size)) + { + return Err(invalid("invalid SquashFS fragment table")); + } + Ok(Self { + endian, + major, + minor, + inode_count, + image_size, + superblock_size: minimum, + block_size: block_size as usize, + root, + inode_table, + directory_table, + fragment_table, + fragments, + check_byte, + compression, + }) + } + + pub fn info(&self, compression: &str) -> SquashfsInfo { + SquashfsInfo { + version: format!("{}.{}", self.major, self.minor), + endianness: match self.endian { + Endian::Little => "little", + Endian::Big => "big", + } + .into(), + compression: compression.into(), + image_size: self.image_size, + inode_count: self.inode_count, + } + } +} diff --git a/crates/fat_extract/src/squashfs/inode.rs b/crates/fat_extract/src/squashfs/inode.rs new file mode 100644 index 0000000..3a7fd29 --- /dev/null +++ b/crates/fat_extract/src/squashfs/inode.rs @@ -0,0 +1,172 @@ +use std::io; + +use super::{ + invalid, + metadata::{Position, Reader}, + unsupported, +}; + +pub(super) struct Inode { + pub mode: u32, + pub kind: Kind, +} + +pub(super) enum Kind { + Directory { + block: u64, + offset: usize, + size: usize, + }, + File { + start: u64, + size: u64, + fragment: u32, + fragment_offset: usize, + blocks: Position, + }, + Symlink(Vec), + Special, +} + +pub(super) fn read(reader: &mut Reader<'_>, reference: u64, max_file: u64) -> io::Result { + let mut position = reader.inode_position(reference)?; + let endian = reader.header.endian; + let major = reader.header.major; + let base_size = match major { + 4 => 16, + 3 => 12, + _ => 4, + }; + let base = reader.read(&mut position, base_size)?; + let (kind, mode) = if major == 4 { + ( + endian.uint(&base, 0, 2)? as u16, + endian.uint(&base, 2, 2)? as u32, + ) + } else { + ( + endian.bits(&base, 0, 4)? as u16, + endian.bits(&base, 4, 12)? as u32, + ) + }; + let tail_length = match (major, kind) { + (4, 1 | 2) => 16, + (4, 8) => 24, + (4, 9) => 40, + (4, 3 | 10) => 8, + (3, 1) => 16, + (3, 8) => 19, + (3, 2) => 20, + (3, 9) => 28, + (3, 3) => 6, + (2, 1) => 11, + (2, 8) => 14, + (2, 2) => 20, + (2, 3) => 2, + (_, 4..=7) | (4, 11..=14) => { + return Ok(Inode { + mode, + kind: Kind::Special, + }) + } + _ => return Err(unsupported(format!("SquashFS {major} inode type {kind}"))), + }; + let tail = reader.read(&mut position, tail_length)?; + let get = |offset, width| endian.uint(&tail, offset, width); + let bits = |offset, width| endian.bits(&tail, offset, width); + let kind = match (major, kind) { + (4, 1) => directory(get(0, 4)?, get(10, 2)?, get(8, 2)?, true)?, + (4, 8) => directory(get(8, 4)?, get(18, 2)?, get(4, 4)?, true)?, + (3, 1) => directory(get(8, 4)?, bits(51, 13)?, bits(32, 19)?, true)?, + (3, 8) => directory(get(9, 4)?, bits(59, 13)?, bits(32, 27)?, true)?, + (2, 1) => directory(bits(64, 24)?, bits(19, 13)?, bits(0, 19)?, false)?, + (2, 8) => directory(bits(72, 24)?, bits(27, 13)?, bits(0, 27)?, false)?, + (4, 2) => file( + get(0, 4)?, + get(12, 4)?, + get(4, 4)?, + get(8, 4)?, + position, + max_file, + )?, + (4, 9) => file( + get(0, 8)?, + get(8, 8)?, + get(28, 4)?, + get(32, 4)?, + position, + max_file, + )?, + (3, 2) => file( + get(0, 8)?, + get(16, 4)?, + get(8, 4)?, + get(12, 4)?, + position, + max_file, + )?, + (3, 9) => file( + get(4, 8)?, + get(20, 8)?, + get(12, 4)?, + get(16, 4)?, + position, + max_file, + )?, + (2, 2) => file( + get(4, 4)?, + get(16, 4)?, + get(8, 4)?, + get(12, 4)?, + position, + max_file, + )?, + (_, 3) | (4, 10) => { + let size = match major { + 4 => get(4, 4)?, + 3 => get(4, 2)?, + _ => get(0, 2)?, + }; + if size == 0 || size > 65535 || size > max_file { + return Err(invalid("SquashFS symlink exceeds size limit")); + } + Kind::Symlink(reader.read(&mut position, size as usize)?) + } + _ => unreachable!(), + }; + Ok(Inode { mode, kind }) +} + +fn directory(block: u64, offset: u64, size: u64, bias: bool) -> io::Result { + let size = if bias { + size.checked_sub(3) + .ok_or_else(|| invalid("invalid SquashFS directory size"))? + } else { + size + }; + Ok(Kind::Directory { + block, + offset: offset as usize, + size: size as usize, + }) +} + +fn file( + start: u64, + size: u64, + fragment: u64, + fragment_offset: u64, + blocks: Position, + max_file: u64, +) -> io::Result { + if size > max_file || size > usize::MAX as u64 { + return Err(invalid("SquashFS file exceeds output size limit")); + } + Ok(Kind::File { + start, + size, + fragment: fragment as u32, + fragment_offset: fragment_offset as usize, + blocks, + }) +} diff --git a/crates/fat_extract/src/squashfs/metadata.rs b/crates/fat_extract/src/squashfs/metadata.rs new file mode 100644 index 0000000..b54a5b1 --- /dev/null +++ b/crates/fat_extract/src/squashfs/metadata.rs @@ -0,0 +1,212 @@ +use std::collections::HashMap; +use std::io; +use std::sync::Arc; + +use super::{codec::Codec, header::Header, invalid}; + +const METADATA_LIMIT: usize = 8192; +const CACHE_LIMIT: usize = 16 * 1024 * 1024; +const CACHE_ENTRIES: usize = 8192; + +#[derive(Clone, Copy)] +pub(super) struct Position { + pub block: u64, + pub offset: usize, + pub end: u64, +} + +struct MetadataBlock { + bytes: Vec, + next: u64, +} + +pub(super) struct Reader<'a> { + pub image: &'a [u8], + pub header: Header, + pub codec: Codec, + metadata: HashMap>, + fragments: HashMap>>, + cached: usize, +} + +impl<'a> Reader<'a> { + pub fn new(image: &'a [u8], header: Header) -> Self { + let codec = Codec::new(header.compression); + Self { + image: &image[..header.image_size as usize], + header, + codec, + metadata: HashMap::new(), + fragments: HashMap::new(), + cached: 0, + } + } + + pub fn inode_position(&self, reference: u64) -> io::Result { + let block = self + .header + .inode_table + .checked_add(reference >> 16) + .ok_or_else(|| invalid("SquashFS inode address overflow"))?; + let offset = (reference & 0xffff) as usize; + if offset >= METADATA_LIMIT || block >= self.header.directory_table { + return Err(invalid("SquashFS inode reference outside table")); + } + Ok(Position { + block, + offset, + end: self.header.directory_table, + }) + } + + pub fn directory_position(&self, block: u64, offset: usize) -> io::Result { + let block = self + .header + .directory_table + .checked_add(block) + .ok_or_else(|| invalid("SquashFS directory address overflow"))?; + if offset >= METADATA_LIMIT || block >= self.header.image_size { + return Err(invalid("SquashFS directory reference outside image")); + } + Ok(Position { + block, + offset, + end: self.header.image_size, + }) + } + + pub fn read(&mut self, position: &mut Position, length: usize) -> io::Result> { + let mut bytes = Vec::with_capacity(length); + while bytes.len() < length { + let block = self.metadata_block(position.block)?; + if block.next > position.end || position.offset > block.bytes.len() { + return Err(invalid("SquashFS metadata reference outside table")); + } + let amount = (length - bytes.len()).min(block.bytes.len() - position.offset); + bytes.extend_from_slice(&block.bytes[position.offset..position.offset + amount]); + position.offset += amount; + if position.offset == block.bytes.len() { + position.block = block.next; + position.offset = 0; + } + } + Ok(bytes) + } + + fn metadata_block(&mut self, offset: u64) -> io::Result> { + if let Some(block) = self.metadata.get(&offset) { + return Ok(Arc::clone(block)); + } + let encoded = self.header.endian.uint(self.slice(offset, 2)?, 0, 2)? as usize; + let size = encoded & 0x7fff; + if size == 0 || size > METADATA_LIMIT { + return Err(invalid("invalid SquashFS metadata block size")); + } + let skip = if self.header.check_byte { 3 } else { 2 }; + let payload = offset + .checked_add(skip) + .ok_or_else(|| invalid("SquashFS metadata offset overflow"))?; + if self.header.check_byte && self.slice(offset + 2, 1)?[0] != 0xff { + return Err(invalid("invalid SquashFS metadata check byte")); + } + let input = self.slice(payload, size)?; + let bytes = if encoded & 0x8000 != 0 { + input.to_vec() + } else { + self.codec.decode(input, METADATA_LIMIT)? + }; + if bytes.is_empty() { + return Err(invalid("empty SquashFS metadata block")); + } + let block = Arc::new(MetadataBlock { + bytes, + next: payload + size as u64, + }); + if self.cached + block.bytes.len() <= CACHE_LIMIT + && self.metadata.len() + self.fragments.len() < CACHE_ENTRIES + { + self.cached += block.bytes.len(); + self.metadata.insert(offset, Arc::clone(&block)); + } + Ok(block) + } + + pub fn data_block( + &mut self, + offset: u64, + encoded: u32, + expected: usize, + ) -> io::Result> { + if encoded & 0xfe00_0000 != 0 { + return Err(invalid("invalid SquashFS data block flags")); + } + let length = (encoded & 0x00ff_ffff) as usize; + if (encoded != 0 && offset < self.header.superblock_size) + || length > self.header.block_size + || offset + .checked_add(length as u64) + .is_none_or(|end| end > self.header.inode_table) + { + return Err(invalid("SquashFS file block outside data region")); + } + if encoded == 0 { + return Ok(vec![0; expected]); + } + let input = self.slice(offset, length)?; + if encoded & 0x0100_0000 != 0 { + if input.len() > expected { + return Err(invalid("SquashFS uncompressed block exceeds expected size")); + } + Ok(input.to_vec()) + } else { + self.codec.decode(input, expected) + } + } + + pub fn fragment(&mut self, index: u64) -> io::Result>> { + if index >= self.header.fragments { + return Err(invalid("SquashFS fragment index outside table")); + } + if let Some(bytes) = self.fragments.get(&index) { + return Ok(Arc::clone(bytes)); + } + let entry_size = if self.header.major == 2 { 8 } else { 16 }; + let pointer_size = if self.header.major == 2 { 4 } else { 8 }; + let table_offset = index * entry_size; + let pointer_offset = + self.header.fragment_table + (table_offset / METADATA_LIMIT as u64) * pointer_size; + let block = self.header.endian.uint( + self.slice(pointer_offset, pointer_size as usize)?, + 0, + pointer_size as usize, + )?; + let mut position = Position { + block, + offset: table_offset as usize % METADATA_LIMIT, + end: self.header.fragment_table, + }; + let entry = self.read(&mut position, entry_size as usize)?; + let start_size = if self.header.major == 2 { 4 } else { 8 }; + let start = self.header.endian.uint(&entry, 0, start_size)?; + let encoded = self.header.endian.uint(&entry, start_size, 4)? as u32; + let bytes = Arc::new(self.data_block(start, encoded, self.header.block_size)?); + if self.cached + bytes.len() <= CACHE_LIMIT + && self.metadata.len() + self.fragments.len() < CACHE_ENTRIES + { + self.cached += bytes.len(); + self.fragments.insert(index, Arc::clone(&bytes)); + } + Ok(bytes) + } + + fn slice(&self, offset: u64, length: usize) -> io::Result<&'a [u8]> { + let start = usize::try_from(offset) + .map_err(|_| invalid("SquashFS offset does not fit platform"))?; + let end = start + .checked_add(length) + .ok_or_else(|| invalid("SquashFS range overflow"))?; + self.image + .get(start..end) + .ok_or_else(|| invalid("SquashFS range exceeds input")) + } +} diff --git a/crates/fat_extract/src/squashfs/mod.rs b/crates/fat_extract/src/squashfs/mod.rs new file mode 100644 index 0000000..dba5586 --- /dev/null +++ b/crates/fat_extract/src/squashfs/mod.rs @@ -0,0 +1,229 @@ +mod codec; +mod header; +mod inode; +mod metadata; + +use std::collections::HashSet; +use std::ffi::OsString; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::output::OutputTree; +use codec::Codec; +use header::Header; +use inode::Kind; +use metadata::Reader; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SquashfsInfo { + pub version: String, + pub endianness: String, + pub compression: String, + pub image_size: u64, + pub inode_count: u64, +} + +pub fn probe(image: &[u8]) -> io::Result { + let header = Header::parse(image)?; + Ok(header.info(Codec::new(header.compression).name())) +} + +pub fn extract(image: &[u8], output: &mut OutputTree) -> io::Result { + let header = Header::parse(image)?; + if header.inode_count > output.limits().max_entries { + return Err(invalid("SquashFS inode count exceeds entry limit")); + } + if let Codec::Other(id) = Codec::new(header.compression) { + return Err(unsupported(format!("SquashFS compression {id}"))); + } + let root = header.root; + let mut state = State { + reader: Reader::new(image, header), + output, + directories: HashSet::new(), + }; + state.visit(root, Path::new(""), 0)?; + Ok(state.reader.header.info(state.reader.codec.name())) +} + +struct State<'a, 'b> { + reader: Reader<'a>, + output: &'b mut OutputTree, + directories: HashSet, +} + +impl State<'_, '_> { + fn visit(&mut self, reference: u64, path: &Path, depth: usize) -> io::Result<()> { + if depth > self.output.limits().max_depth { + return Err(invalid("SquashFS directory depth exceeds limit")); + } + let inode = inode::read( + &mut self.reader, + reference, + self.output.limits().max_file_bytes, + )?; + if path.as_os_str().is_empty() && !matches!(inode.kind, Kind::Directory { .. }) { + return Err(invalid("SquashFS root inode is not a directory")); + } + match inode.kind { + Kind::Directory { + block, + offset, + size, + } => { + if !self.directories.insert(reference) { + return Err(invalid("SquashFS directory cycle or duplicate reference")); + } + self.output.create_dir(path, inode.mode)?; + let mut position = self.reader.directory_position(block, offset)?; + let major = self.reader.header.major; + let endian = self.reader.header.endian; + let mut remaining = size; + let mut names = HashSet::new(); + while remaining > 0 { + let header_size = match major { + 4 => 12, + 3 => 9, + _ => 4, + }; + consume(&mut remaining, header_size)?; + let header = self.reader.read(&mut position, header_size)?; + let (count, start) = if major == 4 { + (endian.uint(&header, 0, 4)? + 1, endian.uint(&header, 4, 4)?) + } else { + ( + header[0] as u64 + 1, + endian.uint(&header, 1, if major == 2 { 3 } else { 4 })?, + ) + }; + if count > 256 || count > self.output.limits().max_entries { + return Err(invalid("invalid SquashFS directory entry count")); + } + for _ in 0..count { + let entry_size = match major { + 4 => 8, + 3 => 5, + _ => 3, + }; + consume(&mut remaining, entry_size)?; + let entry = self.reader.read(&mut position, entry_size)?; + let (offset, length) = if major == 4 { + (endian.uint(&entry, 0, 2)?, endian.uint(&entry, 6, 2)? + 1) + } else { + (endian.bits(&entry, 0, 13)?, entry[2] as u64 + 1) + }; + if length > 256 { + return Err(invalid("SquashFS filename exceeds format limit")); + } + consume(&mut remaining, length as usize)?; + let name = self.reader.read(&mut position, length as usize)?; + if name == b"." + || name == b".." + || name.contains(&0) + || name.contains(&b'/') + || !names.insert(name.clone()) + { + return Err(invalid("invalid or duplicate SquashFS filename")); + } + let child = path.join(path_bytes(name)?); + self.visit((start << 16) | offset, &child, depth + 1) + .map_err(|error| { + io::Error::new( + error.kind(), + format!("{}: {error}", child.display()), + ) + })?; + } + } + } + Kind::File { + mut start, + size, + fragment, + fragment_offset, + mut blocks, + } => { + if size + > self + .output + .limits() + .max_output_bytes + .saturating_sub(self.output.stats().bytes) + { + return Err(invalid("SquashFS file exceeds remaining output budget")); + } + let block_size = self.reader.header.block_size; + let full_blocks = size as usize / block_size; + let remainder = size as usize % block_size; + let block_count = full_blocks + usize::from(fragment == u32::MAX && remainder != 0); + let mut bytes = Vec::with_capacity(size as usize); + for _ in 0..block_count { + let descriptor = self.reader.read(&mut blocks, 4)?; + let encoded = self.reader.header.endian.uint(&descriptor, 0, 4)? as u32; + let expected = (size as usize - bytes.len()).min(block_size); + let decoded = self.reader.data_block(start, encoded, expected)?; + if decoded.len() != expected { + return Err(invalid("SquashFS file block size mismatch")); + } + bytes.extend_from_slice(&decoded); + start = start + .checked_add((encoded & 0x00ff_ffff) as u64) + .ok_or_else(|| invalid("SquashFS block offset overflow"))?; + } + if fragment != u32::MAX && remainder > 0 { + let decoded = self.reader.fragment(fragment as u64)?; + let end = fragment_offset + .checked_add(remainder) + .ok_or_else(|| invalid("SquashFS fragment offset overflow"))?; + let tail = decoded + .get(fragment_offset..end) + .ok_or_else(|| invalid("SquashFS file tail exceeds fragment"))?; + bytes.extend_from_slice(tail); + } + if bytes.len() != size as usize { + return Err(invalid("SquashFS recovered file size mismatch")); + } + self.output.write_file(path, &bytes, inode.mode)?; + } + Kind::Symlink(target) => { + if target.contains(&0) { + return Err(invalid("SquashFS symlink contains a NUL byte")); + } + self.output + .write_symlink(path, &PathBuf::from(path_bytes(target)?))?; + } + Kind::Special => self.output.skip_special()?, + } + Ok(()) + } +} + +fn consume(remaining: &mut usize, count: usize) -> io::Result<()> { + *remaining = remaining + .checked_sub(count) + .ok_or_else(|| invalid("SquashFS directory entry exceeds declared size"))?; + Ok(()) +} + +#[cfg(unix)] +fn path_bytes(bytes: Vec) -> io::Result { + use std::os::unix::ffi::OsStringExt; + Ok(OsString::from_vec(bytes)) +} + +#[cfg(not(unix))] +fn path_bytes(bytes: Vec) -> io::Result { + String::from_utf8(bytes) + .map(OsString::from) + .map_err(|_| unsupported("non-UTF-8 SquashFS path on this platform")) +} + +fn invalid(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message.into()) +} + +fn unsupported(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::Unsupported, message.into()) +} diff --git a/crates/fat_extract/tests/native_extract.rs b/crates/fat_extract/tests/native_extract.rs index c347cbe..83aa7f0 100644 --- a/crates/fat_extract/tests/native_extract.rs +++ b/crates/fat_extract/tests/native_extract.rs @@ -129,6 +129,13 @@ fn cramfs_extracts_little_and_big_endian_filesystems() { assert_eq!(result.directories, 3); assert_eq!(result.symlinks, 1); assert_eq!(result.skipped_special, 0); + assert_eq!( + result.bytes, + (b"#!/bin/sh\necho busybox\n".len() + + b"::sysinit:/etc/init.d/rcS\n".len() + + fixture.large_file.len() + + b"bin/busybox".len()) as u64 + ); } } diff --git a/crates/fat_extract/tests/output_tree.rs b/crates/fat_extract/tests/output_tree.rs new file mode 100644 index 0000000..17b9210 --- /dev/null +++ b/crates/fat_extract/tests/output_tree.rs @@ -0,0 +1,104 @@ +use std::path::Path; + +use fat_extract::output::{ExtractionLimits, OutputTree}; + +#[test] +fn output_is_published_only_after_finish() { + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("tree"); + let mut tree = OutputTree::new(&destination, ExtractionLimits::default()).unwrap(); + tree.create_dir(Path::new("etc"), 0o755).unwrap(); + tree.write_file(Path::new("etc/config"), b"ready\n", 0o644) + .unwrap(); + assert!(!destination.exists()); + let stats = tree.finish().unwrap(); + assert_eq!(stats.files, 1); + assert_eq!(stats.bytes, 6); + assert_eq!( + std::fs::read(destination.join("etc/config")).unwrap(), + b"ready\n" + ); +} + +#[test] +fn failed_or_abandoned_output_does_not_publish() { + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("tree"); + let limits = ExtractionLimits { + max_output_bytes: 3, + ..Default::default() + }; + let mut tree = OutputTree::new(&destination, limits).unwrap(); + assert!(tree.write_file(Path::new("large"), b"four", 0o644).is_err()); + assert!(tree + .write_file(Path::new("../outside"), b"x", 0o644) + .is_err()); + drop(tree); + assert!(!destination.exists()); + assert!(!temp.path().join("outside").exists()); +} + +#[cfg(unix)] +#[test] +fn links_are_preserved_without_following_them_during_writes() { + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("tree"); + let mut tree = OutputTree::new(&destination, ExtractionLimits::default()).unwrap(); + tree.write_symlink(Path::new("sbin/init"), Path::new("../bin/busybox")) + .unwrap(); + tree.write_file(Path::new("bin/busybox"), b"ELF", 0o755) + .unwrap(); + tree.finish().unwrap(); + assert_eq!( + std::fs::read_link(destination.join("sbin/init")).unwrap(), + Path::new("../bin/busybox") + ); + assert_eq!( + std::fs::read(destination.join("sbin/init")).unwrap(), + b"ELF" + ); +} + +#[test] +fn a_link_cannot_be_used_as_an_output_parent() { + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("tree"); + let mut tree = OutputTree::new(&destination, ExtractionLimits::default()).unwrap(); + tree.write_symlink(Path::new("outside"), temp.path()) + .unwrap(); + assert!(tree + .write_file(Path::new("outside/escape"), b"x", 0o644) + .is_err()); + assert!(!temp.path().join("escape").exists()); +} + +#[test] +fn entries_and_individual_files_are_bounded() { + let temp = tempfile::tempdir().unwrap(); + let limits = ExtractionLimits { + max_entries: 2, + max_file_bytes: 2, + ..Default::default() + }; + let mut tree = OutputTree::new(&temp.path().join("tree"), limits).unwrap(); + assert!(tree.write_file(Path::new("large"), b"123", 0o644).is_err()); + tree.write_file(Path::new("one"), b"12", 0o644).unwrap(); + assert!(tree.write_file(Path::new("two"), b"1", 0o644).is_err()); +} + +#[test] +fn symlink_targets_share_the_output_budget() { + let temp = tempfile::tempdir().unwrap(); + let limits = ExtractionLimits { + max_output_bytes: 5, + ..Default::default() + }; + let mut tree = OutputTree::new(&temp.path().join("tree"), limits).unwrap(); + tree.write_symlink(Path::new("link"), Path::new("12345")) + .unwrap(); + assert_eq!(tree.stats().bytes, 5); + assert!(tree + .write_symlink(Path::new("second"), Path::new("x")) + .is_err()); + assert!(tree.write_file(Path::new("file"), b"x", 0o644).is_err()); +} diff --git a/crates/fat_extract/tests/pipeline.rs b/crates/fat_extract/tests/pipeline.rs new file mode 100644 index 0000000..d59d556 --- /dev/null +++ b/crates/fat_extract/tests/pipeline.rs @@ -0,0 +1,178 @@ +use std::io::{Cursor, Write}; + +use fat_extract::pipeline::{extract, ArtifactStatus, PipelineOptions}; + +fn tar_file(name: &str, bytes: &[u8]) -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, name, bytes).unwrap(); + archive.into_inner().unwrap() +} + +#[test] +fn follows_zip_tar_gzip_without_filename_hints() { + let temp = tempfile::tempdir().unwrap(); + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gzip.write_all(&tar_file("etc/inittab", b"::sysinit:/etc/init.d/rcS\n")) + .unwrap(); + let nested = tar_file("opaque", &gzip.finish().unwrap()); + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + zip.start_file("payload", zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(&nested).unwrap(); + let input = temp.path().join("input"); + std::fs::write(&input, zip.finish().unwrap().into_inner()).unwrap(); + let report = extract( + &input, + &temp.path().join("output"), + PipelineOptions::default(), + ) + .unwrap(); + assert_eq!( + report + .artifacts + .iter() + .map(|a| a.format.as_str()) + .collect::>(), + ["zip", "tar", "gzip", "tar"] + ); + assert!(report + .artifacts + .iter() + .all(|a| a.status == ArtifactStatus::Recovered)); + for (id, artifact) in report.artifacts.iter().enumerate() { + assert_eq!(artifact.parent, id.checked_sub(1)); + } + assert!(report + .artifacts + .last() + .unwrap() + .output + .as_ref() + .unwrap() + .join("etc/inittab") + .is_file()); +} + +#[test] +fn malformed_child_does_not_erase_recovered_container() { + let temp = tempfile::tempdir().unwrap(); + let input = temp.path().join("input"); + std::fs::write(&input, tar_file("broken", b"hsqs")).unwrap(); + let report = extract( + &input, + &temp.path().join("output"), + PipelineOptions::default(), + ) + .unwrap(); + assert_eq!(report.artifacts.len(), 2); + assert_eq!(report.artifacts[0].status, ArtifactStatus::Recovered); + assert_eq!(report.artifacts[1].status, ArtifactStatus::Failed); + assert!(report.has_unresolved()); + assert!(report.artifacts[1].output.is_none()); +} + +#[test] +fn output_budget_is_shared_across_nested_containers() { + let temp = tempfile::tempdir().unwrap(); + let nested = tar_file("large", &[1; 512]); + let input = temp.path().join("input"); + std::fs::write(&input, tar_file("nested", &nested)).unwrap(); + let mut options = PipelineOptions::default(); + options.limits.max_output_bytes = nested.len() as u64 + 511; + let report = extract(&input, &temp.path().join("output"), options).unwrap(); + assert_eq!(report.artifacts[0].status, ArtifactStatus::Recovered); + assert_eq!(report.artifacts[1].status, ArtifactStatus::Limited); + assert!(report.artifacts[1].output.is_none()); +} + +#[test] +fn depth_limit_is_a_reported_outcome() { + let temp = tempfile::tempdir().unwrap(); + let input = temp.path().join("input"); + std::fs::write(&input, tar_file("nested", &tar_file("leaf", b"hello"))).unwrap(); + let options = PipelineOptions { + max_container_depth: 0, + ..Default::default() + }; + let report = extract(&input, &temp.path().join("output"), options).unwrap(); + assert_eq!(report.artifacts[1].status, ArtifactStatus::Limited); +} + +#[test] +fn embedded_magic_strings_without_plausible_headers_are_not_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let input = temp.path().join("input"); + let mut bytes = vec![0x41; 256]; + bytes[16..20].copy_from_slice(b"hsqs"); + bytes[64..68].copy_from_slice(b"sqlz"); + bytes[128..131].copy_from_slice(b"\x1f\x8b\x08"); + bytes[131] = 0xff; + std::fs::write(&input, bytes).unwrap(); + let report = extract( + &input, + &temp.path().join("output"), + PipelineOptions::default(), + ) + .unwrap(); + assert!(report.artifacts.is_empty()); +} + +#[test] +fn artifact_limit_also_bounds_uninspected_children() { + let temp = tempfile::tempdir().unwrap(); + let input = temp.path().join("input"); + std::fs::write(&input, tar_file("nested", &tar_file("leaf", b"hello"))).unwrap(); + let options = PipelineOptions { + max_container_depth: 0, + max_artifacts: 1, + ..Default::default() + }; + let report = extract(&input, &temp.path().join("output"), options).unwrap(); + assert_eq!(report.artifacts.len(), 1); + assert!(report.truncated); + assert!(report.has_unresolved()); +} + +#[cfg(unix)] +#[test] +fn unreadable_child_is_reported_and_readable_sibling_continues() { + let temp = tempfile::tempdir().unwrap(); + let mut archive = tar::Builder::new(Vec::new()); + for (name, mode) in [("a-unreadable", 0), ("b-readable", 0o644)] { + let bytes = tar_file("leaf", b"hello"); + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_cksum(); + archive + .append_data(&mut header, name, bytes.as_slice()) + .unwrap(); + } + let input = temp.path().join("input"); + std::fs::write(&input, archive.into_inner().unwrap()).unwrap(); + let report = extract( + &input, + &temp.path().join("output"), + PipelineOptions::default(), + ) + .unwrap(); + assert_eq!(report.artifacts.len(), 3); + assert_eq!(report.artifacts[0].status, ArtifactStatus::Recovered); + assert_eq!(report.artifacts[2].status, ArtifactStatus::Recovered); + if std::fs::File::open( + report.artifacts[0] + .output + .as_ref() + .unwrap() + .join("a-unreadable"), + ) + .is_err() + { + assert_eq!(report.artifacts[1].status, ArtifactStatus::Failed); + assert!(report.has_unresolved()); + } +} diff --git a/crates/fat_extract/tests/squashfs_extract.rs b/crates/fat_extract/tests/squashfs_extract.rs new file mode 100644 index 0000000..fdb9189 --- /dev/null +++ b/crates/fat_extract/tests/squashfs_extract.rs @@ -0,0 +1,559 @@ +use fat_extract::squashfs; +use std::io::ErrorKind; + +#[test] +fn rejects_truncated_superblock() { + assert_eq!( + squashfs::probe(b"hsqs").unwrap_err().kind(), + ErrorKind::InvalidData + ); +} + +#[test] +fn identifies_v4_header_without_executing_a_decoder() { + let image = image(4, false); + let info = squashfs::probe(&image).unwrap(); + assert_eq!(info.version, "4.0"); + assert_eq!(info.endianness, "little"); + assert_eq!(info.inode_count, 2); +} + +#[test] +fn identifies_legacy_headers_in_both_byte_orders() { + for version in [2, 3] { + for big in [false, true] { + let info = squashfs::probe(&image(version, big)).unwrap(); + assert_eq!(info.version, format!("{version}.0")); + assert_eq!(info.endianness, if big { "big" } else { "little" }); + } + } +} + +fn put(data: &mut [u8], bit: usize, width: usize, value: u64, big: bool) { + for i in 0..width { + let source = if big { width - i - 1 } else { i }; + let target = if big { + 7 - (bit + i) % 8 + } else { + (bit + i) % 8 + }; + data[(bit + i) / 8] |= (((value >> source) & 1) as u8) << target; + } +} + +fn image(version: u16, big: bool) -> Vec { + let mut data = vec![0u8; 256]; + data[..4].copy_from_slice(if big { b"sqsh" } else { b"hsqs" }); + put(&mut data, 32, 32, 2, big); + put(&mut data, 224, 16, version as u64, big); + if version == 4 { + put(&mut data, 96, 32, 4096, big); + put(&mut data, 160, 16, 1, big); + put(&mut data, 176, 16, 12, big); + put(&mut data, 320, 64, 256, big); + put(&mut data, 512, 64, 128, big); + put(&mut data, 576, 64, 200, big); + } else { + put(&mut data, 272, 16, 12, big); + put(&mut data, 408, 32, 4096, big); + if version == 3 { + put(&mut data, 504, 64, 256, big); + put(&mut data, 696, 64, 128, big); + put(&mut data, 760, 64, 200, big); + } else { + put(&mut data, 64, 32, 256, big); + put(&mut data, 160, 32, 128, big); + put(&mut data, 192, 32, 200, big); + } + } + data +} + +#[test] +fn recovers_files_and_symlinks_across_versions_and_byte_orders() { + for version in [2, 3, 4] { + for big in [false, true] { + let image = tree_image(version, big, Encoding::Plain); + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("root"); + let mut output = + fat_extract::output::OutputTree::new(&destination, Default::default()).unwrap(); + squashfs::extract(&image, &mut output).unwrap(); + let stats = output.finish().unwrap(); + assert_eq!( + std::fs::read(destination.join("hello.txt")).unwrap(), + b"hello firmware\n" + ); + assert_eq!( + std::fs::read_link(destination.join("link")).unwrap(), + std::path::Path::new("hello.txt") + ); + assert_eq!((stats.files, stats.symlinks), (1, 1)); + } + } +} + +#[test] +fn recovers_supported_block_encodings() { + for (encoding, codec) in [ + (Encoding::Zlib, "zlib"), + (Encoding::Lzma, "lzma"), + (Encoding::CompactLzma, "lzma"), + (Encoding::MarkedLzma, "lzma"), + ] { + let image = tree_image(3, true, encoding); + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("root"); + let mut output = + fat_extract::output::OutputTree::new(&destination, Default::default()).unwrap(); + assert_eq!( + squashfs::extract(&image, &mut output).unwrap().compression, + codec + ); + output.finish().unwrap(); + assert_eq!( + std::fs::read(destination.join("hello.txt")).unwrap(), + b"hello firmware\n" + ); + } +} + +#[test] +fn failed_extraction_does_not_publish_output() { + let mut image = tree_image(4, false, Encoding::Plain); + image[32..40].copy_from_slice(&0u64.to_le_bytes()); + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("root"); + let mut output = + fat_extract::output::OutputTree::new(&destination, Default::default()).unwrap(); + assert!(squashfs::extract(&image, &mut output).is_err()); + drop(output); + assert!(!destination.exists()); +} + +#[test] +fn rejects_files_over_the_configured_limit() { + let image = tree_image(3, true, Encoding::Plain); + let temp = tempfile::tempdir().unwrap(); + let limits = fat_extract::output::ExtractionLimits { + max_file_bytes: 4, + ..Default::default() + }; + let mut output = + fat_extract::output::OutputTree::new(&temp.path().join("root"), limits).unwrap(); + assert!(squashfs::extract(&image, &mut output) + .unwrap_err() + .to_string() + .contains("size limit")); +} + +#[test] +#[ignore = "requires a local filesystem image"] +fn local_image_recovery() { + let source = std::env::var_os("FAT_SQUASHFS_IMAGE").expect("FAT_SQUASHFS_IMAGE"); + let destination = std::env::var_os("FAT_SQUASHFS_OUTPUT").expect("FAT_SQUASHFS_OUTPUT"); + let image = std::fs::read(source).unwrap(); + let start = std::time::Instant::now(); + let mut output = fat_extract::output::OutputTree::new( + std::path::Path::new(&destination), + Default::default(), + ) + .unwrap(); + let info = squashfs::extract(&image, &mut output).unwrap(); + let stats = output.finish().unwrap(); + eprintln!( + "{}", + serde_json::json!({ "info": info, "stats": stats, "milliseconds": start.elapsed().as_millis() }) + ); +} + +#[derive(Clone, Copy)] +enum Encoding { + Plain, + Zlib, + Lzma, + CompactLzma, + MarkedLzma, + Xz, +} + +fn metadata(data: &[u8], big: bool, encoding: Encoding) -> Vec { + use std::io::Write; + let encoded = match encoding { + Encoding::Plain => data.to_vec(), + Encoding::Xz => { + let mut writer = xz2::write::XzEncoder::new(Vec::new(), 6); + writer.write_all(data).unwrap(); + writer.finish().unwrap() + } + Encoding::Zlib => { + let mut writer = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + writer.write_all(data).unwrap(); + writer.finish().unwrap() + } + _ => { + let mut bytes = Vec::new(); + lzma_rs::lzma_compress(&mut std::io::Cursor::new(data), &mut bytes).unwrap(); + match encoding { + Encoding::CompactLzma => { + bytes.drain(5..13); + } + Encoding::MarkedLzma => { + bytes.splice(..13, b"7zip".iter().copied()); + } + _ => {} + } + bytes + } + }; + let size = encoded.len() as u16 + | if matches!(encoding, Encoding::Plain) { + 0x8000 + } else { + 0 + }; + let mut result = if big { + size.to_be_bytes().to_vec() + } else { + size.to_le_bytes().to_vec() + }; + result.extend_from_slice(&encoded); + result +} + +fn base(version: u16, big: bool, kind: u64, mode: u64) -> Vec { + let mut bytes = vec![ + 0; + match version { + 4 => 16, + 3 => 12, + _ => 4, + } + ]; + if version == 4 { + put(&mut bytes, 0, 16, kind, big); + put(&mut bytes, 16, 16, mode, big); + } else { + put(&mut bytes, 0, 4, kind, big); + put(&mut bytes, 4, 12, mode, big); + } + bytes +} + +fn tree_image(version: u16, big: bool, encoding: Encoding) -> Vec { + let content = b"hello firmware\n"; + let mut image = vec![0; 128]; + image.extend_from_slice(content); + let inode_start = image.len(); + let mut inodes = base(version, big, 2, 0o644); + let mut tail = vec![0; if version == 4 { 16 } else { 20 }]; + match version { + 4 => { + put(&mut tail, 0, 32, 128, big); + put(&mut tail, 32, 32, u32::MAX as u64, big); + put(&mut tail, 96, 32, content.len() as u64, big); + } + 3 => { + put(&mut tail, 0, 64, 128, big); + put(&mut tail, 64, 32, u32::MAX as u64, big); + put(&mut tail, 128, 32, content.len() as u64, big); + } + _ => { + put(&mut tail, 32, 32, 128, big); + put(&mut tail, 64, 32, u32::MAX as u64, big); + put(&mut tail, 128, 32, content.len() as u64, big); + } + } + inodes.extend_from_slice(&tail); + let descriptor = 0x0100_0000 | content.len() as u32; + inodes.extend_from_slice(&if big { + descriptor.to_be_bytes() + } else { + descriptor.to_le_bytes() + }); + let symlink_offset = inodes.len(); + inodes.extend(base(version, big, 3, 0o777)); + let mut tail = vec![ + 0; + match version { + 4 => 8, + 3 => 6, + _ => 2, + } + ]; + put( + &mut tail, + if version == 2 { 0 } else { 32 }, + if version == 4 { 32 } else { 16 }, + 9, + big, + ); + inodes.extend(tail); + inodes.extend_from_slice(b"hello.txt"); + let root_offset = inodes.len(); + let mut directories = vec![ + 0; + match version { + 4 => 12, + 3 => 9, + _ => 4, + } + ]; + put( + &mut directories, + 0, + if version == 4 { 32 } else { 8 }, + 1, + big, + ); + for (name, offset, kind) in [ + (b"hello.txt".as_slice(), 0, 2), + (b"link".as_slice(), symlink_offset, 3), + ] { + let mut entry = vec![ + 0; + match version { + 4 => 8, + 3 => 5, + _ => 3, + } + ]; + if version == 4 { + put(&mut entry, 0, 16, offset as u64, big); + put(&mut entry, 32, 16, kind, big); + put(&mut entry, 48, 16, name.len() as u64 - 1, big); + } else { + put(&mut entry, 0, 13, offset as u64, big); + put(&mut entry, 13, 3, kind, big); + entry[2] = name.len() as u8 - 1; + } + directories.extend(entry); + directories.extend_from_slice(name); + } + inodes.extend(base(version, big, 1, 0o755)); + let mut tail = vec![0; if version == 2 { 11 } else { 16 }]; + match version { + 4 => put(&mut tail, 64, 16, directories.len() as u64 + 3, big), + 3 => put(&mut tail, 32, 19, directories.len() as u64 + 3, big), + _ => put(&mut tail, 0, 19, directories.len() as u64, big), + } + inodes.extend(tail); + image.extend(metadata(&inodes, big, encoding)); + let directory_start = image.len(); + image.extend(metadata(&directories, big, encoding)); + let used = image.len(); + image[..4].copy_from_slice(if big { b"sqsh" } else { b"hsqs" }); + put(&mut image, 32, 32, 3, big); + put(&mut image, 224, 16, version as u64, big); + if version == 4 { + put(&mut image, 96, 32, 4096, big); + put(&mut image, 160, 16, 1, big); + put(&mut image, 176, 16, 12, big); + put(&mut image, 256, 64, root_offset as u64, big); + put(&mut image, 320, 64, used as u64, big); + put(&mut image, 512, 64, inode_start as u64, big); + put(&mut image, 576, 64, directory_start as u64, big); + } else { + put(&mut image, 272, 16, 12, big); + put(&mut image, 344, 64, root_offset as u64, big); + put(&mut image, 408, 32, 4096, big); + if version == 3 { + put(&mut image, 504, 64, used as u64, big); + put(&mut image, 696, 64, inode_start as u64, big); + put(&mut image, 760, 64, directory_start as u64, big); + } else { + put(&mut image, 64, 32, used as u64, big); + put(&mut image, 160, 32, inode_start as u64, big); + put(&mut image, 192, 32, directory_start as u64, big); + } + } + image +} + +#[test] +fn rejects_directory_cycles() { + let mut image = tree_image(4, false, Encoding::Plain); + let root = u64::from_le_bytes(image[32..40].try_into().unwrap()) as u16; + let directory = u64::from_le_bytes(image[72..80].try_into().unwrap()) as usize; + image[directory + 14..directory + 16].copy_from_slice(&root.to_le_bytes()); + let temp = tempfile::tempdir().unwrap(); + let mut output = + fat_extract::output::OutputTree::new(&temp.path().join("root"), Default::default()) + .unwrap(); + assert!(squashfs::extract(&image, &mut output) + .unwrap_err() + .to_string() + .contains("cycle")); +} + +#[test] +fn rejects_directory_path_escape() { + let mut image = tree_image(4, false, Encoding::Plain); + let directory = u64::from_le_bytes(image[72..80].try_into().unwrap()) as usize; + image[directory + 22] = b'/'; + let temp = tempfile::tempdir().unwrap(); + let mut output = + fat_extract::output::OutputTree::new(&temp.path().join("root"), Default::default()) + .unwrap(); + assert!(squashfs::extract(&image, &mut output) + .unwrap_err() + .to_string() + .contains("filename")); +} + +#[test] +fn rejects_decompressed_metadata_larger_than_its_limit() { + let mut image = tree_image(4, false, Encoding::Plain); + let inode = u64::from_le_bytes(image[64..72].try_into().unwrap()) as usize; + let oversized = metadata(&vec![0; 8193], false, Encoding::Zlib); + let directory = inode + oversized.len(); + image.truncate(inode); + image.extend(oversized); + image.extend(metadata(&[0; 32], false, Encoding::Plain)); + let size = image.len() as u64; + image[32..40].fill(0); + image[40..48].copy_from_slice(&size.to_le_bytes()); + image[72..80].copy_from_slice(&(directory as u64).to_le_bytes()); + let temp = tempfile::tempdir().unwrap(); + let mut output = + fat_extract::output::OutputTree::new(&temp.path().join("root"), Default::default()) + .unwrap(); + assert!(squashfs::extract(&image, &mut output) + .unwrap_err() + .to_string() + .contains("decompression limit")); +} + +#[test] +fn recovers_fragment_tail() { + let mut image = tree_image(4, false, Encoding::Plain); + let inode = u64::from_le_bytes(image[64..72].try_into().unwrap()) as usize; + image[inode + 22..inode + 26].fill(0); + let mut fragment = vec![0; 16]; + fragment[..8].copy_from_slice(&128u64.to_le_bytes()); + fragment[8..12].copy_from_slice(&(0x0100_0000u32 | 15).to_le_bytes()); + let fragment_block = image.len(); + image.extend(metadata(&fragment, false, Encoding::Plain)); + let table = image.len(); + image.extend_from_slice(&(fragment_block as u64).to_le_bytes()); + let used = image.len() as u64; + image[16..20].copy_from_slice(&1u32.to_le_bytes()); + image[40..48].copy_from_slice(&used.to_le_bytes()); + image[80..88].copy_from_slice(&(table as u64).to_le_bytes()); + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("root"); + let mut output = + fat_extract::output::OutputTree::new(&destination, Default::default()).unwrap(); + squashfs::extract(&image, &mut output).unwrap(); + output.finish().unwrap(); + assert_eq!( + std::fs::read(destination.join("hello.txt")).unwrap(), + b"hello firmware\n" + ); +} + +#[test] +fn honors_v4_compression_identifier() { + for (encoding, identifier, name) in [ + (Encoding::Zlib, 1u16, "zlib"), + (Encoding::CompactLzma, 2, "lzma"), + (Encoding::Xz, 4, "xz"), + ] { + let mut image = tree_image(4, false, encoding); + image[20..22].copy_from_slice(&identifier.to_le_bytes()); + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("root"); + let mut output = + fat_extract::output::OutputTree::new(&destination, Default::default()).unwrap(); + assert_eq!( + squashfs::extract(&image, &mut output).unwrap().compression, + name + ); + output.finish().unwrap(); + assert_eq!( + std::fs::read(destination.join("hello.txt")).unwrap(), + b"hello firmware\n" + ); + } +} + +#[test] +fn distinguishes_unsupported_compression_from_corrupt_input() { + let mut image = tree_image(4, false, Encoding::Plain); + image[20..22].copy_from_slice(&99u16.to_le_bytes()); + assert_eq!(squashfs::probe(&image).unwrap().compression, "unknown"); + let temp = tempfile::tempdir().unwrap(); + let mut output = + fat_extract::output::OutputTree::new(&temp.path().join("root"), Default::default()) + .unwrap(); + assert_eq!( + squashfs::extract(&image, &mut output).unwrap_err().kind(), + ErrorKind::Unsupported + ); +} + +#[test] +fn rejects_file_data_inside_the_superblock() { + for version in [2, 3, 4] { + for big in [false, true] { + let minimum = match version { + 4 => 96, + 3 => 119, + _ => 63, + }; + let image = file_start_image(version, big, minimum - 1, false); + let temp = tempfile::tempdir().unwrap(); + let mut output = + fat_extract::output::OutputTree::new(&temp.path().join("root"), Default::default()) + .unwrap(); + let error = squashfs::extract(&image, &mut output).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidData); + assert!(error.to_string().contains("outside data region")); + } + } +} + +#[test] +fn sparse_blocks_do_not_require_a_payload_address() { + for version in [2, 3, 4] { + let image = file_start_image(version, false, 0, true); + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("root"); + let mut output = + fat_extract::output::OutputTree::new(&destination, Default::default()).unwrap(); + squashfs::extract(&image, &mut output).unwrap(); + output.finish().unwrap(); + assert_eq!( + std::fs::read(destination.join("hello.txt")).unwrap(), + vec![0; 15] + ); + } +} + +fn file_start_image(version: u16, big: bool, start: u64, sparse: bool) -> Vec { + let mut image = tree_image(version, big, Encoding::Plain); + let inode = 128 + b"hello firmware\n".len() + 2; + let offset = match version { + 4 => 16, + 3 => 12, + _ => 8, + }; + let width = if version == 3 { 8 } else { 4 }; + let bytes = if big { + start.to_be_bytes() + } else { + start.to_le_bytes() + }; + let value = if big { + &bytes[8 - width..] + } else { + &bytes[..width] + }; + image[inode + offset..inode + offset + width].copy_from_slice(value); + if sparse { + let descriptor = inode + if version == 2 { 24 } else { 32 }; + image[descriptor..descriptor + 4].fill(0); + } + image +} diff --git a/docs/architecture.md b/docs/architecture.md index 0a22314..3e6a22c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -5,7 +5,7 @@ The workspace contains thirteen crates. `fat_cli` is the only binary crate; the ```text fat_cli ├── fat_core shared models and SQLite persistence -├── fat_extract binwalk and unblob extraction wrapper +├── fat_extract recursive native extraction and fallback orchestration ├── fat_analyze analyzer engine and registry ├── fat_emulate emulation planning and session orchestration │ ├── fat_backend backend abstraction and registry @@ -26,3 +26,15 @@ The main extension points are: - the evidence IR in `fat_query`, shared by static, runtime, and source adapters. For build, test, and contribution workflow, see [DEVELOPMENT.md](../DEVELOPMENT.md) and [CONTRIBUTING.md](../CONTRIBUTING.md). + +## Extraction + +`fat_extract::pipeline` runs a deterministic queue of container members and embedded regions. Archive handlers unwrap ZIP, TAR, and gzip; filesystem handlers produce terminal trees. Extracted filesystem contents retain their original layout rather than gaining analysis-generated files. A single signature scanner skips successfully decoded spans. + +`output::OutputTree` owns path validation, cumulative output accounting, private staging, mode preservation, and deferred symlinks. Publication happens only after the decoder finishes. SquashFS header, inode, compression, and cached metadata readers are separate modules. CramFS reuses its bounded decoder. New formats can reuse the writer and return child paths for further inspection or a terminal tree. + +Artifact records retain their source, parent, source-relative offset, decoder, output, counts, timing, and failure reason. Failed children preserve successful siblings. Rootfs content evidence and unresolved branches determine whether fallback extraction continues. The manifest adds `artifacts` and `recovery_status` without removing earlier fields. + +Default native limits are 512 MiB per input, 256 MiB per file, 1 GiB cumulative output including symlink targets, 262,144 output entries, 4,096 artifacts, eight container levels, and 64 path components. Metadata and fragment caches share a 16 MiB cap. Filesystem decoders never execute recovered programs. + +Supported SquashFS codecs are zlib, LZMA, and XZ. Other codecs produce explicit unsupported outcomes. Special device entries are counted and omitted; ownership, timestamps, and extended attributes are not restored. TAR hard links and sparse entries are currently unsupported. Case-distinct filenames require case-sensitive output storage; collisions fail the tree instead of overwriting a file. diff --git a/docs/commands.md b/docs/commands.md index 471efca..fbc7540 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -8,12 +8,16 @@ | --- | --- | | `new`, `list`, `info`, `delete` | Manage project workspaces under `.fat-projects/` | | `identify` | Classify evidence in an unknown file or directory | -| `extract` | Run binwalk or unblob and record carved files and filesystem candidates | +| `extract` | Recursively unpack firmware and record filesystem recovery and per-artifact outcomes | | `extract-payload`, `carve` | Extract a known byte range or format-sized object | | `analyze` | Derive project signals, findings, and the bootloader snapshot | | `doctor` | Report external-tool and backend readiness | | `data`, `kernel` | Install and verify runtime data and maintained kernels | +`fat extract firmware.zip --extractor native --json` uses the built-in container and filesystem decoders. The default `auto` selection retains fallback extraction when native recovery is incomplete. `recovery_status` is `rootfs_recovered`, `files_only`, or `partial`; the `artifacts` array explains each recognized branch. `work/native-artifacts.json` also preserves native outcomes when the command cannot produce a final manifest. + +For firmware with case-distinct names, create the project on a case-sensitive volume: `fat new firmware.zip --projects-dir /path/to/case-sensitive/projects`, then extract that project. On macOS, use a case-sensitive APFS volume or disk image. Existing project and extractor-selection arguments remain available. + ## Firmware structure and classification | Command | Purpose | diff --git a/tests/integration/test_analyzer_triggers.rs b/tests/integration/test_analyzer_triggers.rs index 859234e..64cb26b 100644 --- a/tests/integration/test_analyzer_triggers.rs +++ b/tests/integration/test_analyzer_triggers.rs @@ -306,6 +306,8 @@ while [ $i -le 12 ]; do dd if=/dev/zero of="_many.bin.extracted/squashfs-root/opt/app$i/bin/busybox" bs=1024 count=8 2>/dev/null i=$((i + 1)) done +mkdir -p _many.bin.extracted/squashfs-root/bin +ln -s ../opt/app1/bin/busybox _many.bin.extracted/squashfs-root/bin/busybox exit 0 "#, ); @@ -352,7 +354,7 @@ exit 0 "rootfs line must carry size and filesystem kind: {rootfs_line}" ); assert!( - extract_stdout.contains("top-level: opt"), + extract_stdout.contains("top-level: bin opt"), "{extract_stdout}" ); diff --git a/tests/integration/test_extract_cli.rs b/tests/integration/test_extract_cli.rs index 8a7213c..93cc112 100644 --- a/tests/integration/test_extract_cli.rs +++ b/tests/integration/test_extract_cli.rs @@ -421,14 +421,12 @@ exit 0 } #[test] -fn fat_extract_all_runs_engines_that_auto_would_suppress() { +fn fat_extract_auto_and_all_continue_past_carved_images() { let workspace = tempdir().expect("workspace"); let fake_bin_dir = tempdir().expect("fake bin dir"); let firmware_path = workspace.path().join("suppressed.bin"); fs::write(&firmware_path, b"firmware-bytes").expect("firmware file"); - // binwalk carves an image it cannot unpack, which in auto mode suppresses - // the engine that would have recovered the rootfs. write_script( &fake_bin_dir.path().join("binwalk"), r#"#!/bin/sh @@ -470,11 +468,10 @@ exit 0 assert!(auto.status.success(), "{auto:?}"); let auto_stdout = String::from_utf8_lossy(&auto.stdout); assert!( - auto_stdout - .contains("- unblob: skipped (binwalk already carved candidate filesystem images)"), + auto_stdout.contains("- unblob: succeeded (carved a rootfs)"), "{auto_stdout}" ); - assert!(auto_stdout.contains("rootfs: not found"), "{auto_stdout}"); + assert!(!auto_stdout.contains("rootfs: not found"), "{auto_stdout}"); let project_dir = only_project_dir(workspace.path()); let forced = Command::new(env!("CARGO_BIN_EXE_fat")) @@ -708,8 +705,6 @@ fn fat_extract_reports_the_winning_engine_and_skip_reasons() { let firmware_path = workspace.path().join("provenance.bin"); fs::write(&firmware_path, b"firmware-bytes-with-no-native-regions").expect("firmware file"); - // binwalk carves a filesystem image, which suppresses unblob — the exact - // decision that used to be visible only in work/unblob.log. write_script( &fake_bin_dir.path().join("binwalk"), r#"#!/bin/sh @@ -720,7 +715,7 @@ exit 0 ); write_script( &fake_bin_dir.path().join("unblob"), - "#!/bin/sh\necho 'unblob should have been skipped' >&2\nexit 9\n", + "#!/bin/sh\necho 'fallback attempted' >&2\nexit 9\n", ); write_script( &fake_bin_dir.path().join("unsquashfs"), @@ -754,7 +749,7 @@ exit 0 "{stdout}" ); assert!( - stdout.contains("- unblob: skipped (binwalk already carved candidate filesystem images)"), + stdout.contains("- unblob: failed (see work/unblob.log)"), "{stdout}" ); @@ -779,8 +774,7 @@ exit 0 ); assert!(rerun_stdout.contains("engine: binwalk"), "{rerun_stdout}"); assert!( - rerun_stdout - .contains("- unblob: skipped (binwalk already carved candidate filesystem images)"), + rerun_stdout.contains("- unblob: failed (see work/unblob.log)"), "{rerun_stdout}" ); } @@ -1375,7 +1369,7 @@ exit 0 } #[test] -fn fat_extract_skips_unblob_when_binwalk_already_carved_filesystem_images() { +fn fat_extract_continues_when_only_filesystem_images_were_carved() { let projects_dir = tempdir().expect("projects dir"); let firmware_dir = tempdir().expect("firmware dir"); let fake_bin_dir = tempdir().expect("fake bin dir"); @@ -1393,7 +1387,7 @@ exit 0 write_script( &fake_bin_dir.path().join("unblob"), r#"#!/bin/sh -echo "unblob should have been skipped" >&2 +echo "fallback attempted" >&2 exit 9 "#, ); @@ -1436,11 +1430,11 @@ exit 0 let unblob_log = fs::read_to_string(project_dir.join("work").join("unblob.log")).expect("unblob log"); - assert!(unblob_log.contains("skipped unblob"), "{unblob_log}"); + assert!(unblob_log.contains("exit=9"), "{unblob_log}"); } #[test] -fn fat_extract_skips_unblob_when_binwalk_already_carved_boot_artifacts() { +fn fat_extract_continues_when_only_boot_artifacts_were_carved() { let projects_dir = tempdir().expect("projects dir"); let firmware_dir = tempdir().expect("firmware dir"); let fake_bin_dir = tempdir().expect("fake bin dir"); @@ -1459,7 +1453,7 @@ exit 0 write_script( &fake_bin_dir.path().join("unblob"), r#"#!/bin/sh -echo "unblob should have been skipped for carved boot artifacts" >&2 +echo "fallback attempted" >&2 exit 9 "#, ); @@ -1501,10 +1495,7 @@ exit 9 let unblob_log = fs::read_to_string(project_dir.join("work").join("unblob.log")).expect("unblob log"); - assert!( - unblob_log.contains("skipped unblob because binwalk already carved boot artifacts"), - "{unblob_log}" - ); + assert!(unblob_log.contains("exit=9"), "{unblob_log}"); } #[test] @@ -2691,3 +2682,75 @@ exit 0 "text mode must stay human output: {text_stdout}" ); } + +#[test] +fn native_extract_recurses_through_containers_and_records_lineage() { + let workspace = tempdir().unwrap(); + let image = firmware_formats::cramfs_fixture(firmware_formats::FixtureEndian::Big).image; + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(image.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "payload", image.as_slice()) + .unwrap(); + let input = workspace.path().join("nested.bin"); + fs::write(&input, archive.into_inner().unwrap()).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_fat")) + .args([ + "extract", + input.to_str().unwrap(), + "--extractor", + "native", + "--json", + ]) + .current_dir(workspace.path()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + let json: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(json["recovery_status"], "rootfs_recovered"); + assert_eq!(json["artifacts"][0]["format"], "tar"); + assert_eq!(json["artifacts"][1]["format"], "cramfs"); + assert_eq!(json["artifacts"][1]["parent"], 0); + assert_eq!(json["engine"], "native"); +} + +#[test] +fn native_selection_does_not_run_external_rootfs_fallback() { + let workspace = tempdir().unwrap(); + let tools = tempdir().unwrap(); + let marker = workspace.path().join("external-ran"); + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "rootfs.squashfs", &b"data"[..]) + .unwrap(); + let input = workspace.path().join("native-only.tar"); + fs::write(&input, archive.into_inner().unwrap()).unwrap(); + let script = format!("#!/bin/sh\n/usr/bin/touch '{}'\nexit 1\n", marker.display()); + for name in ["sasquatch", "unsquashfs"] { + write_script(&tools.path().join(name), &script); + } + let output = Command::new(env!("CARGO_BIN_EXE_fat")) + .args([ + "extract", + input.to_str().unwrap(), + "--extractor", + "native", + "--json", + ]) + .current_dir(workspace.path()) + .env("PATH", tools.path()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + assert!(!marker.exists()); + let json: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(json["recovery_status"], "files_only"); + assert!(json["rootfs"].is_null()); +} diff --git a/tests/integration/test_extraction_manifest.rs b/tests/integration/test_extraction_manifest.rs index 84ef387..834472b 100644 --- a/tests/integration/test_extraction_manifest.rs +++ b/tests/integration/test_extraction_manifest.rs @@ -104,6 +104,8 @@ fn find_rootfs_discovers_nested_rootfs_directory() { let root = unique_temp_dir("fat-extract-rootfs"); let nested_rootfs = root.join("firmware").join("extracted").join("rootfs"); fs::create_dir_all(&nested_rootfs).unwrap(); + fs::create_dir_all(nested_rootfs.join("bin")).unwrap(); + fs::write(nested_rootfs.join("bin/busybox"), b"ELF").unwrap(); let found = fat_extract::rootfs::find_rootfs(root.join("firmware")); @@ -115,6 +117,8 @@ fn find_rootfs_accepts_a_rootfs_directory_as_input() { let root = unique_temp_dir("fat-extract-rootfs-direct"); let rootfs = root.join("rootfs"); fs::create_dir_all(&rootfs).unwrap(); + fs::create_dir_all(rootfs.join("bin")).unwrap(); + fs::write(rootfs.join("bin/busybox"), b"ELF").unwrap(); let found = fat_extract::rootfs::find_rootfs(&rootfs); @@ -126,6 +130,8 @@ fn find_rootfs_accepts_a_jffs2_root_directory_as_input() { let root = unique_temp_dir("fat-extract-jffs2-rootfs-direct"); let rootfs = root.join("jffs2-root"); fs::create_dir_all(&rootfs).unwrap(); + fs::create_dir_all(rootfs.join("bin")).unwrap(); + fs::write(rootfs.join("bin/busybox"), b"ELF").unwrap(); let found = fat_extract::rootfs::find_rootfs(&rootfs); @@ -141,6 +147,8 @@ fn find_rootfs_skips_symlink_cycles() { let nested = root.join("firmware").join("extracted"); let nested_rootfs = nested.join("rootfs"); fs::create_dir_all(&nested_rootfs).unwrap(); + fs::create_dir_all(nested_rootfs.join("bin")).unwrap(); + fs::write(nested_rootfs.join("bin/busybox"), b"ELF").unwrap(); symlink(root.join("firmware"), nested.join("loop")).unwrap(); let found = fat_extract::rootfs::find_rootfs(root.join("firmware")); @@ -225,3 +233,24 @@ fn unique_temp_dir(prefix: &str) -> PathBuf { fs::create_dir_all(&dir).unwrap(); dir } + +#[test] +fn empty_named_directories_are_not_recovered_filesystems() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir(temp.path().join("squashfs-root")).unwrap(); + assert!(fat_extract::rootfs::find_rootfs(temp.path()).is_none()); + assert!(fat_extract::rootfs::find_all_trees(temp.path()).is_empty()); +} + +#[test] +fn a_directory_name_and_readme_do_not_prove_rootfs_recovery() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("rootfs"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("readme"), b"firmware package").unwrap(); + assert!(fat_extract::rootfs::find_rootfs(temp.path()).is_none()); + assert_eq!( + fat_extract::rootfs::find_all_trees(temp.path())[0].0, + "filesystem" + ); +} diff --git a/tests/integration/test_extraction_strategy.rs b/tests/integration/test_extraction_strategy.rs index b8cdb20..ca8de8b 100644 --- a/tests/integration/test_extraction_strategy.rs +++ b/tests/integration/test_extraction_strategy.rs @@ -18,6 +18,7 @@ struct StubExtractor { evidence: CarvedEvidence, sufficiency: Sufficiency, runs: Arc, + incomplete: bool, } impl StubExtractor { @@ -28,6 +29,7 @@ impl StubExtractor { evidence, sufficiency: Sufficiency::AnyCarvedEvidence, runs: Arc::new(AtomicUsize::new(0)), + incomplete: false, } } @@ -65,9 +67,11 @@ impl Extractor for StubExtractor { _on_tick: &mut dyn FnMut(Duration), ) -> ExtractionOutcome { self.runs.fetch_add(1, Ordering::SeqCst); - ExtractionOutcome::new(self.id, self.status) + let mut outcome = ExtractionOutcome::new(self.id, self.status) .with_detail(self.evidence.summary()) - .with_evidence(self.evidence.clone()) + .with_evidence(self.evidence.clone()); + outcome.incomplete = self.incomplete; + outcome } } @@ -79,6 +83,19 @@ fn rootfs_evidence() -> CarvedEvidence { } } +#[test] +fn a_recovered_tree_with_an_unresolved_sibling_does_not_stop_fallback() { + let mut first = StubExtractor::new("first", rootfs_evidence()); + first.incomplete = true; + let second = StubExtractor::new("second", rootfs_evidence()); + let second_runs = second.runs(); + let mut registry = ExtractorRegistry::new(); + registry.register(first); + registry.register(second); + run(®istry, &ExtractorSelection::Auto, &harness()); + assert_eq!(second_runs.load(Ordering::SeqCst), 1); +} + fn image_evidence() -> CarvedEvidence { CarvedEvidence { filesystem_images: 1,