diff --git a/crates/format/src/build.rs b/crates/format/src/build.rs index c40fa345b..8b4432c1a 100644 --- a/crates/format/src/build.rs +++ b/crates/format/src/build.rs @@ -83,7 +83,7 @@ fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut Strin /// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of /// code: a mention in a comment or a note is not a declaration. fn declares_step(text: &str) -> bool { - text.lines().map(str::trim_start).filter(|line| !line.starts_with("//")).any(|line| { + without_comments(text).lines().map(str::trim_start).any(|line| { let line = line.strip_prefix("pub(crate)").or_else(|| line.strip_prefix("pub")).unwrap_or(line).trim_start(); line.strip_prefix("fn").is_some_and(|rest| { rest.starts_with(char::is_whitespace) && rest.trim_start().strip_prefix("step").is_some_and(|rest| rest.trim_start().starts_with('(')) @@ -91,6 +91,38 @@ fn declares_step(text: &str) -> bool { }) } +/// `text` with its `//` and (nested) `/* */` comments taken out; a comment's line breaks stay, so +/// what follows a block comment is still on its own line. +fn without_comments(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars().peekable(); + let mut depth = 0usize; + while let Some(c) = chars.next() { + match (c, chars.peek().copied()) { + ('/', Some('*')) => { + chars.next(); + depth += 1; + } + ('*', Some('/')) if depth > 0 => { + chars.next(); + depth -= 1; + } + ('/', Some('/')) if depth == 0 => { + for c in chars.by_ref() { + if c == '\n' { + out.push('\n'); + break; + } + } + } + ('\n', _) => out.push('\n'), + (c, _) if depth == 0 => out.push(c), + _ => {} + } + } + out +} + /// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing /// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error /// reading it fails the build rather than leave a change out of the lists. diff --git a/crates/format/src/build_tests.rs b/crates/format/src/build_tests.rs index d993a572c..0045c7baf 100644 --- a/crates/format/src/build_tests.rs +++ b/crates/format/src/build_tests.rs @@ -79,6 +79,11 @@ fn a_step_is_a_declaration_not_a_mention() { assert!(!declares_step("//! No fn step() is needed.\n")); assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n")); assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n")); + // a declaration inside a block comment, nested or not, is not one + assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n")); + assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n")); + assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n")); + assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n")); assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n")); assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n")); assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n")); diff --git a/crates/format/src/container.rs b/crates/format/src/container.rs index e5955c066..e578c59aa 100644 --- a/crates/format/src/container.rs +++ b/crates/format/src/container.rs @@ -144,7 +144,12 @@ impl Writer { if cancelled.is_some_and(|c| c()) { return Err(FormatError::Cancelled); } - let compression = if compressed { craft_zip::Compression::Deflated } else { craft_zip::Compression::Stored }; + // the configured level, clamped to the Deflate levels craft-zip writes (1 to 9) + let compression = if compressed { + craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8) + } else { + craft_zip::Compression::Stored + }; match cancelled { Some(cancelled) if bytes.len() > PIECE => { let mut stopped = false; @@ -180,16 +185,22 @@ impl Writer { /// file within `budget` and its expanded size within the reader's budget? The parts already /// written count at their expanded size (an upper bound on what they take in the file). pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool { - self.names.len().saturating_add(entries) <= self.limits.max_entries && extra <= self.space(entries, budget) + self.remaining(entries, budget).is_some_and(|left| extra <= left) } /// The bytes `entries` more optional parts may hold together ([`Writer::room`]). pub fn space(&self, entries: usize, budget: usize) -> usize { - if self.names.len().saturating_add(entries) > self.limits.max_entries { - return 0; - } - let overhead = self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536); - budget.saturating_sub(overhead.saturating_add(self.expanded)).min((self.limits.max_expanded)().saturating_sub(self.expanded)) + self.remaining(entries, budget).unwrap_or(0) + } + + /// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is + /// written already is past `budget` or the reader's budget. + fn remaining(&self, entries: usize, budget: usize) -> Option { + let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?; + let overhead = count.saturating_mul(256).saturating_add(65_536); + let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?; + let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?; + Some(in_file.min(expanded)) } /// Add a payload as a blob, `blobs/.`, and say its name, for the JSON that @@ -241,6 +252,10 @@ impl Reader { craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 }, ) .map_err(invalid)?; + // the directory lists distinct names, so a repeated name is caught here + if archive.has_duplicate_names() { + return Err(invalid("duplicate or invalid archive entry")); + } // Validate entry names match format rules: no empty, no backslashes, // no path components that are "." or "..", and no duplicates; and every part's declared @@ -272,6 +287,8 @@ impl Reader { let bytes = self.limited_part("document.json", limit)?; self.document = Some(bytes.into()); } + // a cached document still keeps the current part limit + let limit = limit.min((self.limits.part_limit)("document.json")); self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget")) } @@ -310,6 +327,8 @@ impl Reader { /// reference counts against the resolved budget. pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> { let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?; + // a cached blob still keeps the part's current limit + let limit = limit.min((self.limits.part_limit)(name)); let hash = name .strip_prefix("blobs/") .and_then(|n| n.strip_suffix(&format!(".{extension}"))) diff --git a/crates/format/src/container_tests.rs b/crates/format/src/container_tests.rs index 9f6c8455f..7fbabf155 100644 --- a/crates/format/src/container_tests.rs +++ b/crates/format/src/container_tests.rs @@ -113,6 +113,9 @@ fn room_counts_written_parts_and_reads_keep_part_limits() { writer.part("large.big", &[0; 500 * 1024], false).unwrap(); assert!(!writer.room(1, 200 * 1024, 600 * 1024)); assert!(writer.room(1, 1024, 1 << 20)); + // what is written already is past the budget: not even an empty part fits + assert!(!writer.room(1, 0, 400 * 1024)); + assert_eq!(writer.space(1, 400 * 1024), 0); let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS }; let mut writer = Writer::new(wide); writer.part("document.json", &[b' '; 2048], false).unwrap(); @@ -161,3 +164,59 @@ fn a_source_that_grows_while_read_is_refused() { let bytes = writer.finish().unwrap(); assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed"))); } + +/// Compressed parts use the configured Deflate level. +#[test] +fn compressed_parts_use_the_configured_level() { + // words in a varied order, so the levels compress them differently + let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "]; + let mut seed = 7u32; + let text: Vec = (0..150) + .flat_map(|_| { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + words[(seed >> 16) as usize % words.len()].bytes() + }) + .take(1000) + .collect(); + let size = |level: fn(&str, usize) -> i64| { + let mut writer = Writer::new(Limits { level, ..LIMITS }); + writer.part("document.json", &text, true).unwrap(); + writer.finish().unwrap().len() + }; + assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used"); + assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped"); + assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped"); +} + +/// An archive naming one part twice is refused. +#[test] +fn a_repeated_part_name_is_refused() { + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{}", false).unwrap(); + writer.part("documenX.json", b"[]", false).unwrap(); + let mut bytes = writer.finish().unwrap(); + let (from, to) = (b"documenX.json", b"document.json"); + let mut at = 0; + while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) { + bytes[at + i..at + i + to.len()].copy_from_slice(to); + at += i + to.len(); + } + assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err()); +} + +/// Cached documents and blobs keep the current part limits. +#[test] +fn cached_parts_keep_the_current_part_limits() { + let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS }; + let mut writer = Writer::new(wide); + writer.part("document.json", &[b' '; 512], false).unwrap(); + let blob = writer.blob(&[7; 512], "bin", false).unwrap(); + let bytes = writer.finish().unwrap(); + let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap(); + let reference = serde_json::json!({"blob": blob}); + assert!(reader.document(4096).is_ok()); + assert!(reader.blob(&reference, "blob", "bin").is_ok()); + reader.limits = Limits { part_limit: |_| 128, ..wide }; + assert!(reader.document(4096).is_err()); + assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err()); +} diff --git a/crates/format/src/newer.rs b/crates/format/src/newer.rs index a66ef8674..a35426f5d 100644 --- a/crates/format/src/newer.rs +++ b/crates/format/src/newer.rs @@ -153,12 +153,17 @@ impl NewerFile { /// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was /// read of it (`known`, its typed record written out): what `known` lacks is kept and named. - /// An object whose `type` is `placeholder_tag` (an element the app kept whole) and the fields - /// in `container_only` (what a container adds beside the file's own) are not compared; nor - /// are the top level's `format` and `format_pending`. + /// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not + /// compared; nor are these top-level fields: those in `container_only` (what a container adds + /// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go + /// back whole). pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) { let mut paths = Vec::new(); - self.kept = vanished(file, known, "", &mut paths, placeholder_tag, container_only); + let mut top_only: Vec<&str> = container_only.to_vec(); + top_only.extend(["format", "format_pending"]); + top_only.extend(self.sections.iter().map(|s| s.key.as_str())); + let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only); + self.kept = kept; self.not_understood.extend(paths); for section in &mut self.sections { section.empty = known.get(§ion.key).cloned(); @@ -182,8 +187,11 @@ impl NewerFile { } for (key, element) in &self.elements { let name = element.get("name"); + // the identity records are matched by: an id first, else a name + let anchor = anchor_of(element); + let taken = |list: &[Value]| anchor.as_ref().is_some_and(|a| Positions::of(list).find(list.len(), a).is_some()); match top.entry(key.clone()).or_insert_with(|| json!([])) { - Value::Array(list) if name.is_none() || !list.iter().any(|x| x.get("name") == name) => list.push(element.clone()), + Value::Array(list) if !taken(list) => list.push(element.clone()), _ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))), } } @@ -285,7 +293,8 @@ pub fn looks_default(v: &Value) -> bool { } /// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names. -fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, placeholder_tag: &str, container_only: &[&str]) -> Option { +/// `top_only` are top-level keys left out of the comparison. +fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, placeholder_tag: &str, top_only: &[&str]) -> Option { match (raw, known) { (Value::Object(raw), Value::Object(known)) => { if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) { @@ -293,7 +302,7 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, pla } let mut kept = BTreeMap::new(); for (key, value) in raw { - if container_only.contains(&key.as_str()) || (path.is_empty() && matches!(key.as_str(), "format" | "format_pending")) { + if path.is_empty() && top_only.contains(&key.as_str()) { continue; } let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") }; @@ -305,7 +314,7 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, pla kept.insert(key.clone(), Kept::Field(value.clone())); } Some(known) => { - if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, container_only) { + if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) { kept.insert(key.clone(), k); } } @@ -324,7 +333,7 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, pla (None, Anchor::Id(id)) => format!("{path}[{id}]"), _ => format!("{path}[{at}]"), }; - if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, container_only) { + if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) { items.push((anchor, k)); } } diff --git a/crates/format/src/newer_tests.rs b/crates/format/src/newer_tests.rs index 9627daf70..d4578dce7 100644 --- a/crates/format/src/newer_tests.rs +++ b/crates/format/src/newer_tests.rs @@ -126,3 +126,41 @@ fn records_match_by_id_name_or_unchanged_index() { by_name.restore_into(&mut renamed); assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1})); } + +/// A section kept whole goes back whole, even when this build's record of it has fields. +#[test] +fn a_section_kept_whole_is_not_compared_field_by_field() { + let mut said = NewerFile::new("App", &newer()); + said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable"); + let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}}); + said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]); + let mut saved = json!({"name": "A", "plugins": {"mode": "none"}}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, file); +} + +/// A kept element goes back only while no record has its id or name. +#[test] +fn a_kept_element_keeps_record_identity() { + let mut said = NewerFile::new("App", &newer()); + said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into()); + said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into()); + let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]}); + assert_eq!(said.restore_into(&mut clash).len(), 1); + assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]})); + let mut free = json!({"items": []}); + assert!(said.restore_into(&mut free).is_empty()); + assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]})); +} + +/// Container-only fields are left out at the top level only. +#[test] +fn container_only_fields_are_top_level() { + let mut said = NewerFile::new("App", &newer()); + let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}}); + said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]); + assert_eq!(said.not_understood, ["field `layer.thumbnail`"]); + let mut saved = json!({"layer": {}}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}})); +}