diff --git a/Cargo.lock b/Cargo.lock index d1e5d3cec..8a947ec53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2788,6 +2788,13 @@ dependencies = [ "kurbo 0.13.1", ] +[[package]] +name = "craft-tar" +version = "0.1.0" +dependencies = [ + "serde", +] + [[package]] name = "craft-tcraft" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bd08fa725..2a8fcc771 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,6 +174,7 @@ craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } +craft-tar = { path = "crates/tar" } craft-epub = { path = "crates/epub" } craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } diff --git a/README.md b/README.md index 8150dfea9..24204a96e 100644 --- a/README.md +++ b/README.md @@ -386,6 +386,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit text-format crates read into and write from. WordCraft's model without its editing code. - [`craft-xlsx`](crates/xlsx/README.md): Excel workbooks (`.xlsx`, `.xlsm`, SpreadsheetML, transitional and strict) and binary workbooks (`.xlsb`) on `craft-opc`: the workbook part, shared strings, the styles tables and worksheet rows streamed one at a time, with limits on the package, the XML and the binary records. - [`craft-ora`](crates/ora/README.md): OpenRaster documents (`.ora`): the ZIP archive and the `stack.xml` layer tree as written, read with bounded sizes and written. From PhotoCraft. +- [`craft-tar`](crates/tar/src/lib.rs): TAR (v7, ustar, GNU long names and base-256 sizes, local and global pax overrides) and cpio (newc, crc, odc) member walkers for hostile input, from Craft Anything: list each member, its kind, mode, link target and where its data lies, without extracting. Bounded by the input and a member cap; an optional `serde` feature serializes listings. - [`craft-y4m`](crates/y4m/src/lib.rs): YUV4MPEG2 (`.y4m`) reader, from FilmCraft: the stream header, colour space layouts at 8 to 16 bits, a frame index built from the `FRAME` lines without loading the file, and plane access. Bounded on hostile input. diff --git a/crates/tar/Cargo.toml b/crates/tar/Cargo.toml new file mode 100644 index 000000000..e0628a35a --- /dev/null +++ b/crates/tar/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "craft-tar" +description = "TAR (v7, ustar, GNU long names and base-256 sizes, pax overrides) and cpio (newc, crc, odc) member walkers for hostile input: list members and where their data lies, without extracting. From Craft Anything." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +# `serde`: `Serialize` on the listing types, for apps that report listings as JSON. +serde = { workspace = true, optional = true } + +[features] +serde = ["dep:serde"] diff --git a/crates/tar/src/lib.rs b/crates/tar/src/lib.rs new file mode 100644 index 000000000..7712fc2aa --- /dev/null +++ b/crates/tar/src/lib.rs @@ -0,0 +1,911 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! Member walkers for Unix archives: tar (v7, ustar, GNU long names and base-256 sizes, pax path +//! local/global overrides) and cpio (newc, crc and portable odc). They list members and where their data +//! lies; nothing is extracted here, and no input can make them panic or loop forever. + +use std::collections::BTreeSet; + +/// Bounds-checked reads from the archive bytes. +struct Reader<'a> { + data: &'a [u8], +} + +impl<'a> Reader<'a> { + fn new(data: &'a [u8]) -> Self { + Reader { data } + } + + /// `len` bytes at `off`, or `None` when they don't all lie inside the data. + fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + self.data.get(off..off.checked_add(len)?) + } +} + +/// What a member is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum MemberKind { + File, + Dir, + Symlink, + Hardlink, + Device, + Fifo, + Other, +} + +/// One archive member. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct Member { + pub name: String, + pub kind: MemberKind, + pub size: u64, + /// Where the member's data starts, relative to the archive start. + pub data_offset: u64, + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub link_target: Option, + pub mode: u32, +} + +/// Why tar name metadata stopped a walk that still retains earlier members. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum StopReason { + GnuMetadataTooLong, + GnuMetadataTruncated, + GlobalNameBudget, +} + +/// The members of an archive. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct Listing { + /// `tar`, `cpio-newc`, `cpio-crc` or `cpio-odc`. + pub format: &'static str, + pub members: Vec, + /// Bytes from the archive start to the end of its last structure (end marker included). + pub length: u64, + /// True when the end-of-archive marker was reached. + pub complete: bool, + /// True when `max_members` stopped the walk. + pub truncated: bool, + /// Name metadata that prevented reading the remaining member identities. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub stop_reason: Option, + pub warnings: Vec, +} + +const BLOCK: usize = 512; +const MAX_NAME: usize = 4096; +const MAX_GLOBAL_NAME_COPIES: usize = 64 << 20; + +fn trim_field(f: &[u8]) -> &[u8] { + let end = f.iter().position(|&b| b == 0).unwrap_or(f.len()); + f.get(..end).unwrap_or(&[]) +} + +fn text(f: &[u8]) -> String { + String::from_utf8_lossy(trim_field(f)).into_owned() +} + +/// An octal number field (spaces and NULs around it are ignored); GNU base-256 when the high bit +/// of the first byte is set. +fn octal(f: &[u8]) -> Option { + if let Some((&first, rest)) = f.split_first() + && first & 0x80 != 0 + { + if first & 0x40 != 0 { + return None; // This field holds an unsigned value. + } + let mut v = u64::from(first & 0x7f); + for &b in rest { + v = v.checked_mul(256)?.checked_add(u64::from(b))?; + } + return Some(v); + } + let padding = |&b: &u8| b == 0 || b == b' '; + let mut digits = f.iter().copied().skip_while(padding).peekable(); + let mut v = 0u64; + while let Some(b) = digits.next_if(|b| !padding(b)) { + if !(b'0'..=b'7').contains(&b) { + return None; + } + v = v.checked_mul(8)?.checked_add(u64::from(b - b'0'))?; + } + digits.all(|b| padding(&b)).then_some(v) +} + +/// True when a 512-byte block is a tar header with a correct checksum. +pub fn tar_header_ok(h: &[u8]) -> bool { + let Some(h) = h.get(..BLOCK) else { return false }; + let Some(stored) = h.get(148..156).and_then(octal) else { return false }; + let mut unsigned = 0u64; + let mut signed = 0i64; + for (i, &b) in h.iter().enumerate() { + let v = if (148..156).contains(&i) { b' ' } else { b }; + unsigned += u64::from(v); + signed += i64::from(v as i8); + } + // An all-zero block sums to 256 (the blank checksum field) and is the end marker, not a header. + unsigned != 256 && (stored == unsigned || i64::try_from(stored).is_ok_and(|s| s == signed)) +} + +fn round_up(v: u64, to: u64) -> Option { + let r = v % to; + if r == 0 { Some(v) } else { v.checked_add(to - r) } +} + +/// Walk a tar archive. `None` when the first block is not a valid header. +pub fn tar(data: &[u8], max_members: usize) -> Option { + tar_with_name_copy_limit(data, max_members, MAX_GLOBAL_NAME_COPIES) +} + +fn tar_with_name_copy_limit(data: &[u8], max_members: usize, mut name_bytes_left: usize) -> Option { + let r = Reader::new(data); + if !tar_header_ok(r.slice(0, BLOCK)?) { + return None; + } + let mut out = Listing { format: "tar", ..Listing::default() }; + let mut pos = 0usize; + let mut long_name: Option = None; + let mut long_link: Option = None; + let mut pax = PaxOverrides::default(); + // Every iteration advances by at least one block, so the walk is bounded by the input size. + 'headers: loop { + let Some(h) = r.slice(pos, BLOCK) else { + out.warnings.push(format!("archive ends inside a header at {pos:#x}")); + out.length = data.len() as u64; + break; + }; + if h.iter().all(|&b| b == 0) { + // End of archive: one or two zero blocks. + let second = r.slice(pos + BLOCK, BLOCK).is_some_and(|b| b.iter().all(|&x| x == 0)); + out.length = (pos + if second { 2 * BLOCK } else { BLOCK }) as u64; + out.complete = true; + break; + } + if !tar_header_ok(h) { + out.warnings.push(format!("invalid header checksum at {pos:#x}; stopping")); + out.length = pos as u64; + break; + } + let typeflag = h.get(156).copied().unwrap_or(0); + let Some(mut size) = h.get(124..136).and_then(octal) else { + out.warnings.push(format!("unreadable size at {pos:#x}")); + out.length = pos as u64; + break; + }; + if !matches!(typeflag, b'x' | b'g' | b'L' | b'K') + && let Some(s) = pax.size.take().or(pax.global_size) + { + size = s; + } + let data_off = pos + BLOCK; + let Some(next) = + round_up(size, BLOCK as u64).and_then(|s| u64::try_from(data_off).ok()?.checked_add(s)).and_then(|n| usize::try_from(n).ok()) + else { + if matches!(typeflag, b'L' | b'K') { + out.stop_reason = Some(StopReason::GnuMetadataTruncated); + } + out.warnings.push(format!("size overflows at {pos:#x}")); + out.length = pos as u64; + break; + }; + let body = r.slice(data_off, usize::try_from(size).unwrap_or(usize::MAX)); + match typeflag { + b'L' | b'K' => { + let Some(body) = body.filter(|_| next <= data.len()) else { + out.stop_reason = Some(StopReason::GnuMetadataTruncated); + out.warnings.push(format!("GNU long-name record ends early at {pos:#x}")); + out.length = data.len() as u64; + break; + }; + let end = body.iter().take(MAX_NAME + 1).position(|&byte| byte == 0).unwrap_or(body.len()); + if end > MAX_NAME { + out.stop_reason = Some(StopReason::GnuMetadataTooLong); + out.warnings.push(format!("GNU long name or link exceeds {MAX_NAME} bytes at {pos:#x}")); + out.length = pos as u64; + break; + } + let value = Some(String::from_utf8_lossy(&body[..end]).into_owned()); + if typeflag == b'L' { + long_name = value; + } else { + long_link = value; + } + } + b'x' | b'g' => { + if let Some(records) = body + && let Err(warning) = pax.apply(records, typeflag == b'g', pos) + { + out.warnings.push(warning); + out.length = pos as u64; + break 'headers; + } + } + _ => { + if out.members.len() >= max_members { + out.truncated = true; + out.length = pos as u64; + break; + } + let linked = matches!(typeflag, b'1' | b'2'); + let copied_path = if pax.path.is_none() { pax.global_path.as_ref().map_or(0, String::len) } else { 0 }; + let copied_link = if linked && pax.link.is_none() { pax.global_link.as_ref().map_or(0, String::len) } else { 0 }; + let Some(remaining) = name_bytes_left.checked_sub(copied_path.saturating_add(copied_link)) else { + out.stop_reason = Some(StopReason::GlobalNameBudget); + out.warnings.push("the retained global pax name-byte budget was reached; later members were omitted".into()); + out.length = pos as u64; + break; + }; + name_bytes_left = remaining; + // Old GNU headers use these bytes for timestamps and sparse-file metadata. + let ustar = h.get(257..263) == Some(b"ustar\0"); + let base = text(h.get(0..100).unwrap_or(&[])); + let prefix = if ustar { text(h.get(345..500).unwrap_or(&[])) } else { String::new() }; + let joined = if prefix.is_empty() { base } else { format!("{prefix}/{base}") }; + // PAX values override header names, including GNU's long-name replacement. + // Consume the one-member GNU records even when a PAX value takes precedence. + let gnu_name = long_name.take(); + let gnu_link = long_link.take(); + let name = pax.path.take().or_else(|| pax.global_path.clone()).or(gnu_name).unwrap_or(joined); + let link = pax + .link + .take() + .or_else(|| if linked { pax.global_link.clone() } else { None }) + .or(gnu_link) + .or_else(|| Some(text(h.get(157..257).unwrap_or(&[])))) + .filter(|l| !l.is_empty()); + let mut kind = tar_kind(typeflag, &name); + if std::mem::take(&mut pax.sparse) || !pax.global_sparse.is_empty() { + kind = MemberKind::Other; + out.warnings.push(format!("{name}: GNU sparse reconstruction is unsupported; stored extents are not the complete file")); + } + let mode = h.get(100..108).and_then(octal).and_then(|m| u32::try_from(m).ok()).unwrap_or(0); + if body.is_none() && kind == MemberKind::File { + out.warnings.push(format!("{name}: data runs past the end of the input")); + } + out.members.push(Member { + name, + kind, + size, + data_offset: data_off as u64, + link_target: if matches!(kind, MemberKind::Symlink | MemberKind::Hardlink) { link } else { None }, + mode, + }); + } + } + if next > data.len() { + out.length = data.len() as u64; + out.warnings.push("archive ends early (no end-of-archive marker)".into()); + break; + } + pos = next; + } + Some(out) +} + +/// What a tar header's typeflag says the member is. +fn tar_kind(typeflag: u8, name: &str) -> MemberKind { + match typeflag { + b'0' | 0 | b'7' => { + if name.ends_with('/') { + MemberKind::Dir + } else { + MemberKind::File + } + } + b'1' => MemberKind::Hardlink, + b'2' => MemberKind::Symlink, + b'3' | b'4' => MemberKind::Device, + b'5' => MemberKind::Dir, + b'6' => MemberKind::Fifo, + _ => MemberKind::Other, + } +} + +/// The pax values that override the next member (local) or every later one (global). +#[derive(Default)] +struct PaxOverrides<'a> { + path: Option, + link: Option, + size: Option, + sparse: bool, + global_path: Option, + global_link: Option, + global_size: Option, + /// Borrow keys from the input: global sparse attributes persist until their own deletion. + global_sparse: BTreeSet<&'a [u8]>, +} + +impl<'a> PaxOverrides<'a> { + /// Apply one extended header's records. `Err` is the warning that stops the walk. + fn apply(&mut self, mut remaining: &'a [u8], global: bool, pos: usize) -> Result<(), String> { + for _ in 0..256 { + if remaining.is_empty() { + break; + } + let Some((key, value, rest)) = pax_record(remaining) else { + return Err(format!("invalid pax record at {pos:#x}")); + }; + if key.starts_with(b"GNU.sparse.") { + if !global { + self.sparse = true; + } else if value.is_empty() { + self.global_sparse.remove(key); + } else { + self.global_sparse.insert(key); + } + } + match key { + b"path" | b"linkpath" | b"GNU.sparse.name" => self.set_name(key, value, global, pos)?, + b"size" => self.set_size(value, global, pos)?, + _ => {} + } + remaining = rest; + } + if remaining.is_empty() { Ok(()) } else { Err(format!("pax record limit reached at {pos:#x}")) } + } + + fn set_name(&mut self, key: &[u8], value: &[u8], global: bool, pos: usize) -> Result<(), String> { + if value.len() > MAX_NAME { + return Err(format!("pax name exceeds {MAX_NAME} bytes at {pos:#x}")); + } + let slot = match (global, key == b"linkpath") { + (true, true) => &mut self.global_link, + (true, false) => &mut self.global_path, + (false, true) => &mut self.link, + (false, false) => &mut self.path, + }; + *slot = if global && value.is_empty() { None } else { Some(String::from_utf8_lossy(value).into_owned()) }; + Ok(()) + } + + fn set_size(&mut self, value: &[u8], global: bool, pos: usize) -> Result<(), String> { + let size = std::str::from_utf8(value).ok().and_then(|s| s.parse().ok()); + if !value.is_empty() && (size.is_none() || !value.iter().all(u8::is_ascii_digit)) { + let scope = if global { "global" } else { "local" }; + return Err(format!("invalid {scope} pax size at {pos:#x}")); + } + if global { + self.global_size = size; + } else { + self.size = size; + } + Ok(()) + } +} + +/// Read one length-prefixed pax record, borrowing even unknown keys and values from the header. +fn pax_record(data: &[u8]) -> Option<(&[u8], &[u8], &[u8])> { + let sp = data.iter().position(|&byte| byte == b' ')?; + let digits = data.get(..sp)?; + if !digits.iter().all(u8::is_ascii_digit) { + return None; + } + let len: usize = std::str::from_utf8(digits).ok()?.parse().ok()?; + let field = data.get(sp + 1..len)?.strip_suffix(b"\n")?; + let eq = field.iter().position(|&byte| byte == b'=')?; + if eq == 0 { + return None; + } + Some((field.get(..eq)?, field.get(eq + 1..)?, data.get(len..)?)) +} + +fn hex_field(f: &[u8]) -> Option { + if !f.iter().all(u8::is_ascii_hexdigit) { + return None; + } + let s = std::str::from_utf8(f).ok()?; + u64::from_str_radix(s, 16).ok() +} + +fn oct_field(f: &[u8]) -> Option { + if !f.iter().all(|b| (b'0'..=b'7').contains(b)) { + return None; + } + let s = std::str::from_utf8(f).ok()?; + u64::from_str_radix(s, 8).ok() +} + +fn mode_kind(mode: u32) -> MemberKind { + match mode & 0o170_000 { + 0o100_000 => MemberKind::File, + 0o040_000 => MemberKind::Dir, + 0o120_000 => MemberKind::Symlink, + 0o020_000 | 0o060_000 => MemberKind::Device, + 0o010_000 => MemberKind::Fifo, + _ => MemberKind::Other, + } +} + +/// Walk a cpio archive (newc `070701`, crc `070702`, odc `070707`). `None` when the first header +/// is not one of these. +pub fn cpio(data: &[u8], max_members: usize) -> Option { + cpio_with_checksum(data, max_members, &mut |body| CpioChecksum::Value(body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte))))) +} + +/// A checksum sum, or the reason a caller stopped verification before completing it. +#[derive(Clone, Copy)] +pub enum CpioChecksum { + Value(u32), + Skipped, + Cancelled, +} + +/// Let a scanner share its checksum work allowance and cancellation across overlapping probes. +pub fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn FnMut(&[u8]) -> CpioChecksum) -> Option { + let r = Reader::new(data); + let format = match r.slice(0, 6)? { + b"070701" => "cpio-newc", + b"070702" => "cpio-crc", + b"070707" => "cpio-odc", + _ => return None, + }; + let odc = format == "cpio-odc"; + let mut hard_links = std::collections::HashMap::<(u64, u64, u64), Vec>::new(); + let mut out = Listing { format, ..Listing::default() }; + let mut pos = 0usize; + let align = |v: usize| if odc { Some(v) } else { v.checked_add(3).map(|x| x & !3) }; + loop { + let hdr_len = if odc { 76 } else { 110 }; + let Some(h) = r.slice(pos, hdr_len) else { + out.warnings.push(format!("archive ends inside a header at {pos:#x}")); + out.length = data.len() as u64; + break; + }; + if h.get(..6) != r.slice(0, 6) { + out.warnings.push(format!("bad header magic at {pos:#x}; stopping")); + out.length = pos as u64; + break; + } + let fields = if odc { + (h.get(18..24).and_then(oct_field), h.get(59..65).and_then(oct_field), h.get(65..76).and_then(oct_field)) + } else { + (h.get(14..22).and_then(hex_field), h.get(94..102).and_then(hex_field), h.get(54..62).and_then(hex_field)) + }; + let (Some(mode), Some(namesize), Some(filesize)) = fields else { + out.warnings.push(format!("unreadable header fields at {pos:#x}")); + out.length = pos as u64; + break; + }; + let namesize = usize::try_from(namesize).unwrap_or(usize::MAX); + if namesize == 0 || namesize > MAX_NAME { + out.warnings.push(format!("implausible name size at {pos:#x}")); + out.length = pos as u64; + break; + } + let Some(name_bytes) = r.slice(pos + hdr_len, namesize) else { + out.warnings.push(format!("name runs past the end at {pos:#x}")); + out.length = data.len() as u64; + break; + }; + // newc permits extra trailing NULs in namesize for alignment. + let Some(end) = name_bytes.iter().position(|&b| b == 0) else { + out.warnings.push(format!("unterminated member name at {pos:#x}")); + out.length = pos as u64; + break; + }; + if end == 0 || name_bytes.get(end..).is_some_and(|pad| pad.iter().any(|&b| b != 0)) { + out.warnings.push(format!("invalid member name at {pos:#x}")); + out.length = pos as u64; + break; + } + let name = text(name_bytes); + let Some(data_off) = (pos + hdr_len).checked_add(namesize).and_then(align) else { break }; + let size = usize::try_from(filesize).unwrap_or(usize::MAX); + let Some(next) = data_off.checked_add(size).and_then(align) else { + out.warnings.push(format!("size overflows at {pos:#x}")); + out.length = pos as u64; + break; + }; + if name == "TRAILER!!!" { + out.length = next.min(data.len()) as u64; + if size == 0 && next <= data.len() { + out.complete = true; + } else { + out.warnings.push(format!("invalid or truncated trailer at {pos:#x}")); + } + break; + } + let mode = u32::try_from(mode).unwrap_or(0); + let kind = mode_kind(mode); + let link_target = if kind == MemberKind::Symlink { r.slice(data_off, size.min(MAX_NAME)).map(text) } else { None }; + if out.members.len() >= max_members { + out.truncated = true; + out.length = pos as u64; + break; + } + if format == "cpio-crc" { + match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) { + Some(expected) => { + if let Some(body) = r.slice(data_off, size) { + match checksum(body) { + CpioChecksum::Value(actual) => { + if actual != expected { + out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})")); + } + } + CpioChecksum::Skipped => out.warnings.push(format!("{name}: data checksum not verified (scan byte budget exhausted)")), + CpioChecksum::Cancelled => return None, + } + } + } + None => out.warnings.push(format!("{name}: unreadable data checksum")), + } + } + // newc stores hard-linked files by device and inode. GNU cpio may put the + // payload on the last name, so identify aliases after scanning the group. + if !odc + && kind == MemberKind::File + && h.get(38..46).and_then(hex_field).is_some_and(|n| n > 1) + && let (Some(ino), Some(major), Some(minor)) = + (h.get(6..14).and_then(hex_field), h.get(62..70).and_then(hex_field), h.get(70..78).and_then(hex_field)) + { + hard_links.entry((major, minor, ino)).or_default().push(out.members.len()); + } + out.members.push(Member { name, kind, size: filesize, data_offset: data_off as u64, link_target, mode }); + if next > data.len() { + out.warnings.push("archive ends early (no trailer)".into()); + out.length = data.len() as u64; + break; + } + pos = next; + } + for group in hard_links.values().filter(|group| group.len() > 1) { + let source = group.iter().rev().copied().find(|&i| out.members.get(i).is_some_and(|m| m.size > 0)).or_else(|| group.first().copied()); + let Some(source) = source else { continue }; + let Some(name) = out.members.get(source).map(|m| m.name.clone()) else { continue }; + for &index in group { + if index != source + && let Some(alias) = out.members.get_mut(index) + && alias.size == 0 + { + alias.kind = MemberKind::Hardlink; + alias.link_target = Some(name.clone()); + } + } + } + Some(out) +} + +/// The data of a member, when it lies inside `data`. +pub fn member_data<'a>(data: &'a [u8], m: &Member) -> Option<&'a [u8]> { + Reader::new(data).slice(usize::try_from(m.data_offset).ok()?, usize::try_from(m.size).ok()?) +} + +#[cfg(test)] +mod tar_tests; +#[cfg(test)] +mod testkit; + +#[cfg(test)] +mod tests { + use super::*; + use crate::testkit as fx; + + #[test] + fn plus_sign_in_cpio_fields_is_rejected() { + assert_eq!(hex_field(b"+000000A"), None); + assert_eq!(oct_field(b"+00012"), None); + assert_eq!(hex_field(b"0000000A"), Some(10)); + assert_eq!(oct_field(b"00012"), Some(10)); + } + + fn global_record(key: &str, value: &str) -> String { + let tail = format!(" {key}={value}\n"); + let mut len = tail.len() + 1; + loop { + let record = format!("{len}{tail}"); + if record.len() == len { + return record; + } + len = record.len(); + } + } + + #[test] + fn global_name_copies_stop_before_spending_the_budget() { + let global = global_record("path", "shared"); + let data = + fx::tar_entries(&[("global", b'g', global.as_bytes(), ""), ("one", b'0', b"", ""), ("two", b'0', b"", ""), ("three", b'0', b"", "")]); + let listing = tar_with_name_copy_limit(&data, 10, 12).unwrap(); + assert_eq!(listing.members.len(), 2); + assert!(listing.members.iter().all(|m| m.name == "shared")); + assert_eq!(listing.stop_reason, Some(StopReason::GlobalNameBudget)); + assert!(!listing.complete && !listing.truncated); + assert!(listing.warnings.iter().any(|w| w.contains("name-byte budget"))); + assert_eq!(listing.length, 2048); + assert!(tar_with_name_copy_limit(&data, 10, 18).unwrap().complete); + } + + #[test] + fn local_names_and_unused_global_links_do_not_spend_copy_bytes() { + let global = global_record("path", "name") + &global_record("linkpath", "unused-link"); + let local = global_record("path", "local-name"); + let data = fx::tar_entries(&[ + ("global", b'g', global.as_bytes(), ""), + ("local", b'x', local.as_bytes(), ""), + ("one", b'0', b"", ""), + ("two", b'0', b"", ""), + ]); + let listing = tar_with_name_copy_limit(&data, 10, 4).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members.iter().map(|m| m.name.as_str()).collect::>(), ["local-name", "name"]); + assert!(listing.members.iter().all(|m| m.link_target.is_none())); + } + + #[test] + fn global_names_and_links_share_the_decoded_byte_budget() { + let global = global_record("path", "é") + &global_record("linkpath", "中"); + let data = fx::tar_entries(&[("global", b'g', global.as_bytes(), ""), ("one", b'2', b"", "fallback"), ("two", b'1', b"", "fallback")]); + let listing = tar_with_name_copy_limit(&data, 10, 5).unwrap(); + assert_eq!(listing.members.len(), 1); + assert_eq!(listing.members[0].link_target.as_deref(), Some("中")); + assert_eq!(listing.stop_reason, Some(StopReason::GlobalNameBudget)); + assert!(tar_with_name_copy_limit(&data, 10, 10).unwrap().complete); + } + + #[test] + fn oversized_gnu_names_do_not_alias_their_truncated_prefix() { + for kind in *b"LK" { + let mut long = vec![b'n'; MAX_NAME + 1]; + long.push(0); + let data = fx::tar_entries(&[ + ("before", b'0', b"kept", ""), + ("././@LongLink", kind, &long, ""), + ("short", if kind == b'K' { b'2' } else { b'0' }, b"", "fallback"), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(!listing.complete); + assert_eq!(listing.stop_reason, Some(StopReason::GnuMetadataTooLong)); + assert_eq!(listing.members.len(), 1); + assert_eq!(listing.members[0].name, "before"); + assert_eq!(member_data(&data, &listing.members[0]).unwrap(), b"kept"); + assert!(listing.warnings.iter().any(|warning| warning.contains("GNU long name or link exceeds"))); + } + } + + #[test] + fn gnu_long_names_keep_exact_limits_and_short_terminated_values() { + for kind in *b"LK" { + for terminated in [false, true] { + let mut long = vec![b'n'; MAX_NAME]; + if terminated { + long.push(0); + } + let data = fx::tar_entries(&[("././@LongLink", kind, &long, ""), ("short", if kind == b'K' { b'2' } else { b'0' }, b"", "fallback")]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete, "{:?}", listing.warnings); + let member = &listing.members[0]; + let name = if kind == b'L' { member.name.as_str() } else { member.link_target.as_deref().unwrap() }; + assert_eq!(name, "n".repeat(MAX_NAME)); + } + } + let mut padded = b"short\0".to_vec(); + padded.resize(MAX_NAME + 100, 0); + let data = fx::tar_entries(&[("././@LongLink", b'L', &padded, ""), ("fallback", b'0', b"payload", "")]); + assert_eq!(tar(&data, 10).unwrap().members[0].name, "short"); + } + + fn sparse_pax_record(key: &str, value: &str) -> Vec { + let field = format!(" {key}={value}\n"); + let mut length = field.len() + 1; + loop { + let actual = field.len() + length.to_string().len(); + if actual == length { + return format!("{length}{field}").into_bytes(); + } + length = actual; + } + } + + #[test] + fn gnu_sparse_pax_members_are_not_regular_file_bytes() { + for fields in [ + vec![("GNU.sparse.size", "100"), ("GNU.sparse.numblocks", "1"), ("GNU.sparse.offset", "50"), ("GNU.sparse.numbytes", "3")], + vec![("GNU.sparse.size", "100"), ("GNU.sparse.map", "50,3"), ("GNU.sparse.name", "original.bin")], + vec![("GNU.sparse.major", "1"), ("GNU.sparse.minor", "0"), ("GNU.sparse.realsize", "100"), ("GNU.sparse.name", "original.bin")], + ] { + let metadata: Vec = fields.iter().flat_map(|(key, value)| sparse_pax_record(key, value)).collect(); + let version_one = fields.iter().any(|(key, _)| *key == "GNU.sparse.major"); + let mut sparse_body = if version_one { b"1\n50\n3\n".to_vec() } else { Vec::new() }; + if version_one { + sparse_body.resize(512, 0); + } + sparse_body.extend_from_slice(b"abc"); + let data = + fx::tar_entries(&[("metadata", b'x', &metadata, ""), ("stored-name", b'0', &sparse_body, ""), ("normal", b'0', b"ordinary", "")]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete, "{:?}", listing.warnings); + assert_eq!(listing.members.len(), 2); + assert_eq!(listing.members[0].kind, MemberKind::Other); + let named = fields.iter().any(|(key, _)| *key == "GNU.sparse.name"); + assert_eq!(listing.members[0].name, if named { "original.bin" } else { "stored-name" }); + assert_eq!(listing.members[1].kind, MemberKind::File); + assert_eq!(member_data(&data, &listing.members[1]).unwrap(), b"ordinary"); + assert_eq!(listing.warnings.len(), 1); + assert!(listing.warnings[0].contains("sparse reconstruction is unsupported")); + } + } + + #[test] + fn sparse_metadata_survives_other_extension_headers() { + let sparse = sparse_pax_record("GNU.sparse.map", "0,3"); + let path = sparse_pax_record("path", "renamed"); + let data = fx::tar_entries(&[ + ("sparse", b'x', &sparse, ""), + ("name", b'L', b"long-name\0", ""), + ("path", b'x', &path, ""), + ("header", b'0', b"abc", ""), + ("next", b'0', b"ok", ""), + ]); + let listing = tar(&data, 10).unwrap(); + assert_eq!(listing.members[0].name, "renamed"); + assert_eq!(listing.members[0].kind, MemberKind::Other); + assert_eq!(listing.members[1].kind, MemberKind::File); + } + + #[test] + fn gnu_metadata_stop_reasons_keep_partial_listings_and_marker_compatibility() { + for kind in *b"LK" { + let mut data = + fx::tar_entries(&[("before", b'0', b"kept", ""), ("././@LongLink", kind, b"long\0", ""), ("fallback", b'0', b"later", "")]); + data.truncate(3 * BLOCK + 2); + let listing = tar(&data, 10).unwrap(); + assert_eq!(listing.stop_reason, Some(StopReason::GnuMetadataTruncated)); + assert!(!listing.complete); + assert!(!listing.truncated); + assert_eq!(listing.members.len(), 1); + assert_eq!(member_data(&data, &listing.members[0]).unwrap(), b"kept"); + } + let mut data = fx::tar_entries(&[("file", b'0', b"kept", "")]); + data.truncate(2 * BLOCK); + let listing = tar(&data, 10).unwrap(); + assert!(!listing.complete); + assert_eq!(listing.stop_reason, None); + assert_eq!(member_data(&data, &listing.members[0]).unwrap(), b"kept"); + let cpio_listing = cpio(&fx::cpio_newc(&[("file", b"kept", 0o100644)]), 10).unwrap(); + assert_eq!(cpio_listing.stop_reason, None); + } + + #[test] + fn gnu_metadata_requires_its_complete_padded_record() { + let body = b"next\0"; + for kind in *b"LK" { + let data = fx::tar_entries(&[ + ("before", b'0', b"kept", ""), + ("././@LongLink", kind, body, ""), + ("fallback", if kind == b'L' { b'0' } else { b'2' }, b"", "fallback"), + ]); + for padding in [0, 1, BLOCK - body.len() - 1] { + let short = &data[..3 * BLOCK + body.len() + padding]; + let listing = tar(short, 10).unwrap(); + assert_eq!(listing.stop_reason, Some(StopReason::GnuMetadataTruncated)); + assert!(!listing.complete); + assert_eq!(listing.members.len(), 1); + assert_eq!(member_data(short, &listing.members[0]).unwrap(), b"kept"); + } + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.stop_reason, None); + assert_eq!(listing.members.len(), 2); + } + } + + #[test] + fn walks_tar() { + let t = fx::tar(&[("etc/passwd", b"root:x:0:0\n"), ("bin/", b""), ("long/".repeat(30).as_str(), b"deep")]); + let l = tar(&t, 100).unwrap(); + assert!(l.complete, "{:?}", l.warnings); + assert_eq!(l.members.len(), 3); + assert_eq!(l.members[0].name, "etc/passwd"); + assert_eq!(member_data(&t, &l.members[0]).unwrap(), b"root:x:0:0\n"); + assert_eq!(l.members[1].kind, MemberKind::Dir); + assert!(l.members[2].name.starts_with("long/long/")); + assert_eq!(l.length, t.len() as u64); + let l2 = tar(&t, 1).unwrap(); + assert!(l2.truncated); + assert_eq!(l2.members.len(), 1); + } + + #[test] + fn cpio_names_and_trailers_must_be_complete() { + let bytes = fx::cpio_newc(&[("abc", b"x", 0o100644)]); + let trailer = bytes.windows(6).rposition(|b| b == b"070701").unwrap(); + for (offset, byte) in [(113, b'x'), (111, 0)] { + let mut damaged = bytes.clone(); + damaged[offset] = byte; + let listed = cpio(&damaged, 10).unwrap(); + assert!(!listed.complete); + assert!(listed.members.is_empty()); + } + let mut padded_name = bytes.clone(); + padded_name[112] = 0; + assert_eq!(cpio(&padded_name, 10).unwrap().members[0].name, "ab"); + let mut with_payload = bytes.clone(); + with_payload[trailer + 61] = b'1'; + assert!(!cpio(&with_payload, 10).unwrap().complete); + assert!(!cpio(&bytes[..bytes.len() - 1], 10).unwrap().complete); + } + + #[test] + fn cpio_hard_link_aliases_do_not_appear_as_empty_files() { + for content_first in [true, false] { + let data = if content_first { [b"abc".as_slice(), b""] } else { [b"".as_slice(), b"abc"] }; + let mut archive = fx::cpio_newc(&[("one", data[0], 0o100644), ("two", data[1], 0o100644), ("separate", b"", 0o100644)]); + let starts: Vec<_> = archive.windows(6).enumerate().filter(|(_, bytes)| *bytes == b"070701").map(|(i, _)| i).collect(); + for &start in &starts[..3] { + archive[start + 6..start + 14].copy_from_slice(b"00000015"); + archive[start + 38..start + 46].copy_from_slice(b"00000002"); + } + archive[starts[2] + 70..starts[2] + 78].copy_from_slice(b"00000001"); + let (file, alias) = if content_first { (0, 1) } else { (1, 0) }; + for crc in [false, true] { + if crc { + for &start in &starts { + archive[start..start + 6].copy_from_slice(b"070702"); + } + archive[starts[file] + 102..starts[file] + 110].copy_from_slice(b"00000126"); + } + let listing = cpio(&archive, 10).unwrap(); + assert_eq!(listing.members[file].kind, MemberKind::File); + assert_eq!(member_data(&archive, &listing.members[file]), Some(b"abc".as_slice())); + assert_eq!(listing.members[alias].kind, MemberKind::Hardlink); + assert_eq!(listing.members[alias].link_target.as_deref(), Some(listing.members[file].name.as_str())); + assert_eq!(listing.members[2].kind, MemberKind::File); + assert!(listing.warnings.is_empty()); + } + } + } + + #[test] + fn walks_cpio() { + let c = fx::cpio_newc(&[("init", b"#!/bin/sh\n", 0o100755), ("dev", b"", 0o040755), ("sh", b"busybox", 0o120777)]); + let l = cpio(&c, 100).unwrap(); + assert!(l.complete); + assert_eq!(l.members.len(), 3); + assert_eq!(member_data(&c, &l.members[0]).unwrap(), b"#!/bin/sh\n"); + assert_eq!(l.members[2].kind, MemberKind::Symlink); + assert_eq!(l.members[2].link_target.as_deref(), Some("busybox")); + assert_eq!(l.length, c.len() as u64); + } + + #[test] + fn octal_fields() { + assert_eq!(octal(b"0000644 \0"), Some(0o644)); + assert_eq!(octal(b" \0"), Some(0)); + assert_eq!(octal(b"9"), None); + assert_eq!(octal(&[0x80, 0, 0, 1, 0]), Some(256)); + assert_eq!(octal(&[0xff; 8]), None); + assert_eq!(octal(&[0xff; 12]), None); + } + + #[test] + fn hostile_archives_do_not_panic() { + let t = fx::tar(&[("a", b"hello"), ("b/c", b"world")]); + let c = fx::cpio_newc(&[("a", b"hello", 0o100644)]); + for s in [&t, &c] { + for m in fx::mutations(s) { + let _ = tar(&m, 10); + let _ = cpio(&m, 10); + } + for n in 0..s.len().min(800) { + let _ = tar(&s[..n], 10); + let _ = cpio(&s[..n], 10); + } + } + for seed in 0..50 { + let n = fx::noise(seed, 2048); + let _ = tar(&n, 10); + let _ = cpio(&n, 10); + } + assert!(pax_record(b"12 path=abc\n").is_some()); + assert!(pax_record(b"9999999999999999999999 x").is_none()); + } +} diff --git a/crates/tar/src/tar_tests.rs b/crates/tar/src/tar_tests.rs new file mode 100644 index 000000000..38f72abd9 --- /dev/null +++ b/crates/tar/src/tar_tests.rs @@ -0,0 +1,404 @@ +// ---- from Craft Anything crates/formats/tests/tar_gnu_names.rs +mod tar_gnu_names { + use crate::testkit as fx; + use crate::{member_data, tar}; + + fn checksum(header: &mut [u8]) { + header[148..156].fill(b' '); + let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + } + + #[test] + fn old_gnu_timestamps_are_not_posix_path_prefixes() { + let mut archive = fx::tar(&[("program", b"payload")]); + let header = &mut archive[..512]; + header[257..265].copy_from_slice(b"ustar \0"); + header[345..357].copy_from_slice(b"00000000123\0"); + header[357..369].copy_from_slice(b"00000000456\0"); + checksum(header); + let listing = tar(&archive, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members[0].name, "program"); + assert_eq!(member_data(&archive, &listing.members[0]), Some(b"payload".as_slice())); + } + + #[test] + fn posix_ustar_prefixes_are_still_joined() { + let mut archive = fx::tar(&[("program", b"payload")]); + let header = &mut archive[..512]; + header[345..352].copy_from_slice(b"usr/bin"); + checksum(header); + assert_eq!(tar(&archive, 10).unwrap().members[0].name, "usr/bin/program"); + } +} + +// ---- from Craft Anything crates/formats/tests/tar_local_pax_size.rs +mod tar_local_pax_size { + use crate::testkit::tar_entries; + use crate::{member_data, tar}; + + /// Encode the decimal byte count that includes the PAX record itself. + fn record(value: &str) -> String { + let body = format!(" size={value}\n"); + let mut size = body.len() + 1; + loop { + let result = format!("{size}{body}"); + if result.len() == size { + return result; + } + size = result.len(); + } + } + + /// Invalid local sizes cannot silently fall back to a plausible member header. + #[test] + fn malformed_local_sizes_stop_before_the_affected_member() { + for value in ["no", "-1", "+3", " 3", "3 ", "3.0", "18446744073709551616"] { + let pax = record(value); + let bytes = tar_entries(&[("keep", b'0', b"ok", ""), ("PaxHeader", b'x', pax.as_bytes(), ""), ("affected", b'0', b"abc", "")]); + let report = tar(&bytes, 10).unwrap(); + assert!(!report.complete, "size={value}"); + assert_eq!(report.members.len(), 1); + assert_eq!(report.members[0].name, "keep"); + assert_eq!(member_data(&bytes, &report.members[0]), Some(b"ok".as_slice())); + assert!(report.warnings.iter().any(|w| w.contains("pax size"))); + } + } + + /// Decimal sizes, zero, and the existing empty override behavior still work. + #[test] + fn valid_local_sizes_remain_readable() { + for (size, body) in [("3", b"abc".as_slice()), ("0003", b"abc"), ("0", b""), ("", b"abc")] { + let pax = record(size); + let bytes = tar_entries(&[("PaxHeader", b'x', pax.as_bytes(), ""), ("file", b'0', body, "")]); + let report = tar(&bytes, 10).unwrap(); + assert!(report.complete, "{:?}", report.warnings); + assert_eq!(member_data(&bytes, &report.members[0]), Some(body)); + } + } +} + +// ---- from Craft Anything crates/formats/tests/tar_pax_size.rs +mod tar_pax_size { + use crate::testkit as fx; + use crate::{member_data, tar}; + + fn clear_member_size(archive: &mut [u8], offset: usize) { + let header = &mut archive[offset..offset + 512]; + header[124..136].copy_from_slice(b"00000000000\0"); + header[148..156].fill(b' '); + let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + } + + #[test] + fn pax_size_survives_a_gnu_long_name_record() { + let mut archive = fx::tar_entries(&[ + ("PaxHeader", b'x', b"10 size=3\n", ""), + ("././@LongLink", b'L', b"renamed\0", ""), + ("short", b'0', b"abc", ""), + ("next", b'0', b"later", ""), + ]); + clear_member_size(&mut archive, 2048); + let listing = tar(&archive, 10).unwrap(); + assert!(listing.complete, "{:?}", listing.warnings); + assert_eq!(listing.members.len(), 2); + assert_eq!(listing.members[0].name, "renamed"); + assert_eq!(listing.members[0].size, 3); + assert_eq!(member_data(&archive, &listing.members[0]), Some(b"abc".as_slice())); + assert_eq!(listing.members[1].size, 5); + assert_eq!(member_data(&archive, &listing.members[1]), Some(b"later".as_slice())); + } + + #[test] + fn pax_size_without_intervening_metadata_still_applies_once() { + let mut archive = fx::tar_entries(&[("PaxHeader", b'x', b"10 size=3\n", ""), ("short", b'0', b"abc", ""), ("next", b'0', b"later", "")]); + clear_member_size(&mut archive, 1024); + let listing = tar(&archive, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members.iter().map(|m| m.size).collect::>(), vec![3, 5]); + } + + fn pax_record(key: &str, value: &str) -> String { + let content = format!(" {key}={value}\n"); + let mut len = content.len() + 1; + loop { + let record = format!("{len}{content}"); + if record.len() == len { + return record; + } + len = record.len(); + } + } + + #[test] + fn malformed_pax_framing_stops_the_listing() { + for invalid in [b"10 size=3!".as_slice(), b"10 size=3!\n", b"20 size=3\n", b"+12 path=no\n", b"5 =x\n", b"12 path=a=b\n11 path=bad"] { + let archive = fx::tar_entries(&[("keep", b'0', b"one", ""), ("PaxHeader", b'x', invalid, ""), ("after", b'0', b"two", "")]); + let listing = tar(&archive, 10).unwrap(); + assert!(!listing.complete); + assert_eq!(listing.members.len(), 1); + assert_eq!(listing.members[0].name, "keep"); + assert!(listing.warnings.iter().any(|s| s.contains("pax"))); + } + } + + #[test] + fn pax_path_can_contain_an_equals_sign() { + let archive = fx::tar_entries(&[("PaxHeader", b'x', b"12 path=a=b\n", ""), ("short", b'0', b"ok", "")]); + let listing = tar(&archive, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members[0].name, "a=b"); + } + + #[test] + fn the_name_limit_applies_only_to_pax_fields_kept_as_names() { + let exact = "a".repeat(4096); + let large = "b".repeat(100_000); + let pax = pax_record("comment", &large) + &pax_record("path", &exact); + let archive = fx::tar_entries(&[("PaxHeader", b'x', pax.as_bytes(), ""), ("short", b'0', b"ok", "")]); + let listing = tar(&archive, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members[0].name, exact); + for key in ["path", "linkpath"] { + let pax = pax_record(key, &large); + let archive = fx::tar_entries(&[("PaxHeader", b'x', pax.as_bytes(), ""), ("short", b'2', b"", "link")]); + let listing = tar(&archive, 10).unwrap(); + assert!(!listing.complete); + assert!(listing.members.is_empty()); + assert!(listing.warnings.iter().any(|s| s.contains("4096"))); + } + } + + #[test] + fn empty_pax_values_remove_an_earlier_override() { + let pax = pax_record("path", "other") + &pax_record("path", "") + &pax_record("linkpath", "first") + &pax_record("linkpath", ""); + let archive = fx::tar_entries(&[("PaxHeader", b'x', pax.as_bytes(), ""), ("original", b'2', b"", "target")]); + let listing = tar(&archive, 10).unwrap(); + assert!(listing.complete); + assert!(listing.members[0].name.is_empty()); + assert!(listing.members[0].link_target.is_none()); + } + + #[test] + fn tar_sizes_allow_outer_padding_but_do_not_join_separated_digits() { + for (field, valid) in [(b" 000000001\0 ", true), (b" 0000 0001\0 ", false), (b" 0000\x000001\0 ", false)] { + let mut archive = fx::tar_entries(&[("one", b'0', b"1", ""), ("two", b'0', b"2", "")]); + let header = &mut archive[..512]; + header[124..136].copy_from_slice(field); + header[148..156].fill(b' '); + let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + let listing = tar(&archive, 10).unwrap(); + assert_eq!(listing.complete, valid); + assert_eq!(listing.members.len(), if valid { 2 } else { 0 }); + } + } + + #[test] + fn excess_pax_records_do_not_hide_a_later_override() { + let mut records = pax_record("comment", "").repeat(256); + for excess in [false, true] { + if excess { + records.push_str(&pax_record("size", "9999")); + } + let archive = fx::tar_entries(&[("PaxHeader", b'x', records.as_bytes(), ""), ("member", b'0', b"ok", "")]); + let listing = tar(&archive, 10).unwrap(); + assert_eq!(listing.complete, !excess); + assert_eq!(listing.members.len(), usize::from(!excess)); + } + } +} + +// ---- from Craft Anything crates/formats/tests/tar_global_pax.rs +mod tar_global_pax { + use crate::testkit as fx; + use crate::{MemberKind, member_data, tar}; + + fn record(key: &str, value: &str) -> String { + let tail = format!(" {key}={value}\n"); + let mut len = tail.len() + 1; + loop { + let record = format!("{len}{tail}"); + if record.len() == len { + return record; + } + len = record.len(); + } + } + + fn clear_size(data: &mut [u8], at: usize) { + let header = &mut data[at..at + 512]; + header[124..136].copy_from_slice(b"00000000000\0"); + header[148..156].fill(b' '); + let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + } + + #[test] + fn global_path_and_size_persist_across_members_and_metadata() { + let global = record("path", "global-name") + &record("size", "3"); + let local = record("comment", "unrelated"); + let mut data = fx::tar_entries(&[ + ("global", b'g', global.as_bytes(), ""), + ("first", b'0', b"abc", ""), + ("metadata", b'x', local.as_bytes(), ""), + ("second", b'0', b"def", ""), + ]); + clear_size(&mut data, 1024); + clear_size(&mut data, 3072); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete, "{:?}", listing.warnings); + assert_eq!(listing.members.len(), 2); + assert!(listing.members.iter().all(|member| member.name == "global-name" && member.size == 3)); + assert_eq!(member_data(&data, &listing.members[0]), Some(b"abc".as_slice())); + assert_eq!(member_data(&data, &listing.members[1]), Some(b"def".as_slice())); + } + + #[test] + fn local_attributes_override_only_one_member() { + let global = record("path", "inherited") + &record("size", "3"); + let local = record("path", "one-member") + &record("size", "0"); + let data = fx::tar_entries(&[ + ("global", b'g', global.as_bytes(), ""), + ("local", b'x', local.as_bytes(), ""), + ("first", b'0', b"", ""), + ("second", b'0', b"abc", ""), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members.iter().map(|m| (m.name.as_str(), m.size)).collect::>(), [("one-member", 0), ("inherited", 3)]); + } + + #[test] + fn later_global_records_update_and_delete_only_their_keys() { + let first = record("path", "first-global") + &record("size", "3"); + let second = record("path", "second-global"); + let remove = record("path", "") + &record("size", ""); + let data = fx::tar_entries(&[ + ("global", b'g', first.as_bytes(), ""), + ("one", b'0', b"abc", ""), + ("global", b'g', second.as_bytes(), ""), + ("two", b'0', b"def", ""), + ("global", b'g', remove.as_bytes(), ""), + ("plain", b'0', b"normal", ""), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete); + assert_eq!( + listing.members.iter().map(|m| (m.name.as_str(), m.size)).collect::>(), + [("first-global", 3), ("second-global", 3), ("plain", 6)] + ); + } + + #[test] + fn global_link_names_persist_until_deleted() { + let global = record("linkpath", "shared-target"); + let local = record("linkpath", "local-target"); + let remove = record("linkpath", ""); + let data = fx::tar_entries(&[ + ("global", b'g', global.as_bytes(), ""), + ("one", b'2', b"", "fallback"), + ("local", b'x', local.as_bytes(), ""), + ("two", b'1', b"", "fallback"), + ("three", b'2', b"", "fallback"), + ("global", b'g', remove.as_bytes(), ""), + ("four", b'2', b"", "original"), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete); + assert_eq!( + listing.members.iter().map(|m| m.link_target.as_deref()).collect::>(), + [Some("shared-target"), Some("local-target"), Some("shared-target"), Some("original")] + ); + } + + #[test] + fn malformed_or_excess_global_records_do_not_claim_complete_members() { + for metadata in [String::from("10 size=3!"), record("comment", "").repeat(257), record("path", &"a".repeat(4097))] { + let data = fx::tar_entries(&[("keep", b'0', b"ok", ""), ("global", b'g', metadata.as_bytes(), ""), ("after", b'0', b"later", "")]); + let listing = tar(&data, 10).unwrap(); + assert!(!listing.complete); + assert_eq!(listing.members.len(), 1); + assert_eq!(listing.members[0].name, "keep"); + assert!(listing.warnings.iter().any(|warning| warning.contains("pax"))); + } + } + + #[test] + fn global_sparse_metadata_keeps_stored_extents_out_of_regular_files() { + let global = record("GNU.sparse.map", "0,3"); + let remove = record("GNU.sparse.map", ""); + let data = fx::tar_entries(&[ + ("global", b'g', global.as_bytes(), ""), + ("one", b'0', b"abc", ""), + ("two", b'0', b"def", ""), + ("global", b'g', remove.as_bytes(), ""), + ("plain", b'0', b"ghi", ""), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members.iter().map(|m| m.kind).collect::>(), [MemberKind::Other, MemberKind::Other, MemberKind::File]); + assert_eq!(listing.warnings.iter().filter(|warning| warning.contains("sparse")).count(), 2); + } + + #[test] + fn unrelated_global_fields_preserve_local_and_gnu_names() { + let global = record("comment", "not a member name"); + let local = record("path", "local"); + let data = fx::tar_entries(&[ + ("local", b'x', local.as_bytes(), ""), + ("global", b'g', global.as_bytes(), ""), + ("first", b'0', b"one", ""), + ("././@LongLink", b'L', b"gnu-name\0", ""), + ("second", b'0', b"two", ""), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete && listing.warnings.is_empty()); + assert_eq!(listing.members.iter().map(|m| m.name.as_str()).collect::>(), ["local", "gnu-name"]); + } + + #[test] + fn global_names_obey_the_existing_member_limit() { + let path = "a".repeat(4096); + let global = record("path", &path); + let data = fx::tar_entries(&[("global", b'g', global.as_bytes(), ""), ("one", b'0', b"abc", ""), ("two", b'0', b"def", "")]); + let listing = tar(&data, 1).unwrap(); + assert!(listing.truncated && !listing.complete); + assert_eq!(listing.members.len(), 1); + assert_eq!(listing.members[0].name, path); + } + + #[test] + fn global_pax_names_override_and_consume_gnu_name_records() { + let global = record("path", "global") + &record("linkpath", "global-target"); + let remove = record("path", "") + &record("linkpath", ""); + let data = fx::tar_entries(&[ + ("global", b'g', global.as_bytes(), ""), + ("LongLink", b'L', b"gnu-name\0", ""), + ("LongLink", b'K', b"gnu-target\0", ""), + ("first", b'2', b"", "fallback"), + ("global", b'g', remove.as_bytes(), ""), + ("plain", b'2', b"", "original"), + ]); + let listing = tar(&data, 10).unwrap(); + assert!(listing.complete); + assert_eq!( + listing.members.iter().map(|m| (m.name.as_str(), m.link_target.as_deref())).collect::>(), + [("global", Some("global-target")), ("plain", Some("original"))] + ); + } + + #[test] + fn nonempty_global_sizes_must_be_unsigned_decimal_numbers() { + for size in ["+3", "-3", "3x", " 3", "18446744073709551616"] { + let global = record("size", size); + let data = fx::tar_entries(&[("keep", b'0', b"abc", ""), ("global", b'g', global.as_bytes(), ""), ("after", b'0', b"def", "")]); + let listing = tar(&data, 10).unwrap(); + assert!(!listing.complete); + assert_eq!(listing.members.len(), 1); + assert!(listing.warnings.iter().any(|warning| warning.contains("global pax size"))); + } + } +} diff --git a/crates/tar/src/testkit.rs b/crates/tar/src/testkit.rs new file mode 100644 index 000000000..828644a4b --- /dev/null +++ b/crates/tar/src/testkit.rs @@ -0,0 +1,135 @@ +//! Archive builders and damage generators for the tests, from Craft Anything's testkit. + +#![allow(clippy::indexing_slicing, clippy::cast_possible_truncation)] + +fn tar_header(name: &str, typeflag: u8, size: usize, link: &str) -> Vec { + let mut h = vec![0u8; 512]; + let n = name.as_bytes(); + h[..n.len().min(100)].copy_from_slice(&n[..n.len().min(100)]); + h[100..108].copy_from_slice(b"0000644\0"); + h[108..116].copy_from_slice(b"0000000\0"); + h[116..124].copy_from_slice(b"0000000\0"); + h[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); + h[136..148].copy_from_slice(b"00000000000\0"); + h[156] = typeflag; + let l = link.as_bytes(); + h[157..157 + l.len().min(100)].copy_from_slice(&l[..l.len().min(100)]); + h[257..263].copy_from_slice(b"ustar\0"); + h[263..265].copy_from_slice(b"00"); + h[148..156].copy_from_slice(b" "); + let sum: u32 = h.iter().map(|&b| u32::from(b)).sum(); + h[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + h +} + +fn pad512(v: &mut Vec) { + while !v.len().is_multiple_of(512) { + v.push(0); + } +} + +/// A tar entry for [`tar_entries`]: (name, typeflag, data, link target). +pub type TarEntry<'a> = (&'a str, u8, &'a [u8], &'a str); + +/// A ustar archive from raw entries; names longer than 100 bytes get a GNU long-name record. +pub fn tar_entries(entries: &[TarEntry<'_>]) -> Vec { + let mut out = Vec::new(); + for (name, flag, data, link) in entries { + if name.len() > 100 { + let mut long = name.as_bytes().to_vec(); + long.push(0); + out.extend(tar_header("././@LongLink", b'L', long.len(), "")); + out.extend_from_slice(&long); + pad512(&mut out); + } + out.extend(tar_header(name, *flag, data.len(), link)); + out.extend_from_slice(data); + pad512(&mut out); + } + out.extend(vec![0u8; 1024]); + out +} + +/// A ustar archive of files (names ending in `/` become folders). +pub fn tar(files: &[(&str, &[u8])]) -> Vec { + let e: Vec> = files.iter().map(|(n, d)| (*n, if n.ends_with('/') { b'5' } else { b'0' }, *d, "")).collect(); + tar_entries(&e) +} + +/// A cpio newc archive from (name, data, mode). Symlinks (mode 0o120000) store the target as data. +pub fn cpio_newc(files: &[(&str, &[u8], u32)]) -> Vec { + let mut out = Vec::new(); + let mut push = |name: &str, data: &[u8], mode: u32, ino: u32| { + let hdr = format!( + "070701{ino:08x}{mode:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}", + 0, + 0, + 1, + 0, + data.len(), + 0, + 0, + 0, + 0, + name.len() + 1, + 0 + ); + out.extend_from_slice(hdr.as_bytes()); + out.extend_from_slice(name.as_bytes()); + out.push(0); + while !out.len().is_multiple_of(4) { + out.push(0); + } + out.extend_from_slice(data); + while !out.len().is_multiple_of(4) { + out.push(0); + } + }; + for (i, (n, d, m)) in files.iter().enumerate() { + push(n, d, *m, i as u32 + 1); + } + push("TRAILER!!!", b"", 0, 0); + out +} + +/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops. +pub fn noise(seed: u64, len: usize) -> Vec { + let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() +} + +/// Every way to damage a sample used by robustness tests: truncations at many lengths and +/// single-byte corruptions spread across the file. +pub fn mutations(sample: &[u8]) -> Vec> { + let mut out = Vec::new(); + let n = sample.len(); + let step = (n / 64).max(1); + let mut len = 0; + while len < n { + out.push(sample[..len].to_vec()); + len += step; + } + for (i, seed) in (0..n).step_by(step).zip(1u64..) { + let mut m = sample.to_vec(); + let noise = noise(seed, 8); + for (j, v) in noise.iter().enumerate() { + if let Some(b) = m.get_mut(i + j) { + *b = *v; + } + } + out.push(m); + let mut m = sample.to_vec(); + if let Some(b) = m.get_mut(i) { + *b = 0xff; + } + out.push(m); + } + out +}