mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 20:25:02 +00:00
craft-format: block comments in step detection, room past budget, configured Deflate level, duplicate names, cached limits, whole-section/identity/top-level retention
This commit is contained in:
@@ -83,7 +83,7 @@ fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut Strin
|
||||
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
|
||||
/// code: a mention in a comment or a note is not a declaration.
|
||||
fn declares_step(text: &str) -> bool {
|
||||
text.lines().map(str::trim_start).filter(|line| !line.starts_with("//")).any(|line| {
|
||||
without_comments(text).lines().map(str::trim_start).any(|line| {
|
||||
let line = line.strip_prefix("pub(crate)").or_else(|| line.strip_prefix("pub")).unwrap_or(line).trim_start();
|
||||
line.strip_prefix("fn").is_some_and(|rest| {
|
||||
rest.starts_with(char::is_whitespace) && rest.trim_start().strip_prefix("step").is_some_and(|rest| rest.trim_start().starts_with('('))
|
||||
@@ -91,6 +91,38 @@ fn declares_step(text: &str) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// `text` with its `//` and (nested) `/* */` comments taken out; a comment's line breaks stay, so
|
||||
/// what follows a block comment is still on its own line.
|
||||
fn without_comments(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut chars = text.chars().peekable();
|
||||
let mut depth = 0usize;
|
||||
while let Some(c) = chars.next() {
|
||||
match (c, chars.peek().copied()) {
|
||||
('/', Some('*')) => {
|
||||
chars.next();
|
||||
depth += 1;
|
||||
}
|
||||
('*', Some('/')) if depth > 0 => {
|
||||
chars.next();
|
||||
depth -= 1;
|
||||
}
|
||||
('/', Some('/')) if depth == 0 => {
|
||||
for c in chars.by_ref() {
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
('\n', _) => out.push('\n'),
|
||||
(c, _) if depth == 0 => out.push(c),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
|
||||
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
|
||||
/// reading it fails the build rather than leave a change out of the lists.
|
||||
|
||||
@@ -79,6 +79,11 @@ fn a_step_is_a_declaration_not_a_mention() {
|
||||
assert!(!declares_step("//! No fn step() is needed.\n"));
|
||||
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
// a declaration inside a block comment, nested or not, is not one
|
||||
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
|
||||
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
|
||||
|
||||
@@ -144,7 +144,12 @@ impl Writer {
|
||||
if cancelled.is_some_and(|c| c()) {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
let compression = if compressed { craft_zip::Compression::Deflated } else { craft_zip::Compression::Stored };
|
||||
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
|
||||
let compression = if compressed {
|
||||
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
|
||||
} else {
|
||||
craft_zip::Compression::Stored
|
||||
};
|
||||
match cancelled {
|
||||
Some(cancelled) if bytes.len() > PIECE => {
|
||||
let mut stopped = false;
|
||||
@@ -180,16 +185,22 @@ impl Writer {
|
||||
/// file within `budget` and its expanded size within the reader's budget? The parts already
|
||||
/// written count at their expanded size (an upper bound on what they take in the file).
|
||||
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
|
||||
self.names.len().saturating_add(entries) <= self.limits.max_entries && extra <= self.space(entries, budget)
|
||||
self.remaining(entries, budget).is_some_and(|left| extra <= left)
|
||||
}
|
||||
|
||||
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
|
||||
pub fn space(&self, entries: usize, budget: usize) -> usize {
|
||||
if self.names.len().saturating_add(entries) > self.limits.max_entries {
|
||||
return 0;
|
||||
}
|
||||
let overhead = self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536);
|
||||
budget.saturating_sub(overhead.saturating_add(self.expanded)).min((self.limits.max_expanded)().saturating_sub(self.expanded))
|
||||
self.remaining(entries, budget).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
|
||||
/// written already is past `budget` or the reader's budget.
|
||||
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
|
||||
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
|
||||
let overhead = count.saturating_mul(256).saturating_add(65_536);
|
||||
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
|
||||
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
|
||||
Some(in_file.min(expanded))
|
||||
}
|
||||
|
||||
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
|
||||
@@ -241,6 +252,10 @@ impl Reader {
|
||||
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
|
||||
)
|
||||
.map_err(invalid)?;
|
||||
// the directory lists distinct names, so a repeated name is caught here
|
||||
if archive.has_duplicate_names() {
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
}
|
||||
|
||||
// Validate entry names match format rules: no empty, no backslashes,
|
||||
// no path components that are "." or "..", and no duplicates; and every part's declared
|
||||
@@ -272,6 +287,8 @@ impl Reader {
|
||||
let bytes = self.limited_part("document.json", limit)?;
|
||||
self.document = Some(bytes.into());
|
||||
}
|
||||
// a cached document still keeps the current part limit
|
||||
let limit = limit.min((self.limits.part_limit)("document.json"));
|
||||
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
|
||||
}
|
||||
|
||||
@@ -310,6 +327,8 @@ impl Reader {
|
||||
/// reference counts against the resolved budget.
|
||||
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
|
||||
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
|
||||
// a cached blob still keeps the part's current limit
|
||||
let limit = limit.min((self.limits.part_limit)(name));
|
||||
let hash = name
|
||||
.strip_prefix("blobs/")
|
||||
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
|
||||
|
||||
@@ -113,6 +113,9 @@ fn room_counts_written_parts_and_reads_keep_part_limits() {
|
||||
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
|
||||
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
|
||||
assert!(writer.room(1, 1024, 1 << 20));
|
||||
// what is written already is past the budget: not even an empty part fits
|
||||
assert!(!writer.room(1, 0, 400 * 1024));
|
||||
assert_eq!(writer.space(1, 400 * 1024), 0);
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 2048], false).unwrap();
|
||||
@@ -161,3 +164,59 @@ fn a_source_that_grows_while_read_is_refused() {
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
|
||||
}
|
||||
|
||||
/// Compressed parts use the configured Deflate level.
|
||||
#[test]
|
||||
fn compressed_parts_use_the_configured_level() {
|
||||
// words in a varied order, so the levels compress them differently
|
||||
let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "];
|
||||
let mut seed = 7u32;
|
||||
let text: Vec<u8> = (0..150)
|
||||
.flat_map(|_| {
|
||||
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
words[(seed >> 16) as usize % words.len()].bytes()
|
||||
})
|
||||
.take(1000)
|
||||
.collect();
|
||||
let size = |level: fn(&str, usize) -> i64| {
|
||||
let mut writer = Writer::new(Limits { level, ..LIMITS });
|
||||
writer.part("document.json", &text, true).unwrap();
|
||||
writer.finish().unwrap().len()
|
||||
};
|
||||
assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used");
|
||||
assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped");
|
||||
assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped");
|
||||
}
|
||||
|
||||
/// An archive naming one part twice is refused.
|
||||
#[test]
|
||||
fn a_repeated_part_name_is_refused() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
writer.part("documenX.json", b"[]", false).unwrap();
|
||||
let mut bytes = writer.finish().unwrap();
|
||||
let (from, to) = (b"documenX.json", b"document.json");
|
||||
let mut at = 0;
|
||||
while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) {
|
||||
bytes[at + i..at + i + to.len()].copy_from_slice(to);
|
||||
at += i + to.len();
|
||||
}
|
||||
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
|
||||
}
|
||||
|
||||
/// Cached documents and blobs keep the current part limits.
|
||||
#[test]
|
||||
fn cached_parts_keep_the_current_part_limits() {
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 512], false).unwrap();
|
||||
let blob = writer.blob(&[7; 512], "bin", false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
|
||||
let reference = serde_json::json!({"blob": blob});
|
||||
assert!(reader.document(4096).is_ok());
|
||||
assert!(reader.blob(&reference, "blob", "bin").is_ok());
|
||||
reader.limits = Limits { part_limit: |_| 128, ..wide };
|
||||
assert!(reader.document(4096).is_err());
|
||||
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
|
||||
}
|
||||
|
||||
@@ -153,12 +153,17 @@ impl NewerFile {
|
||||
|
||||
/// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was
|
||||
/// read of it (`known`, its typed record written out): what `known` lacks is kept and named.
|
||||
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) and the fields
|
||||
/// in `container_only` (what a container adds beside the file's own) are not compared; nor
|
||||
/// are the top level's `format` and `format_pending`.
|
||||
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not
|
||||
/// compared; nor are these top-level fields: those in `container_only` (what a container adds
|
||||
/// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go
|
||||
/// back whole).
|
||||
pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) {
|
||||
let mut paths = Vec::new();
|
||||
self.kept = vanished(file, known, "", &mut paths, placeholder_tag, container_only);
|
||||
let mut top_only: Vec<&str> = container_only.to_vec();
|
||||
top_only.extend(["format", "format_pending"]);
|
||||
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
|
||||
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
|
||||
self.kept = kept;
|
||||
self.not_understood.extend(paths);
|
||||
for section in &mut self.sections {
|
||||
section.empty = known.get(§ion.key).cloned();
|
||||
@@ -182,8 +187,11 @@ impl NewerFile {
|
||||
}
|
||||
for (key, element) in &self.elements {
|
||||
let name = element.get("name");
|
||||
// the identity records are matched by: an id first, else a name
|
||||
let anchor = anchor_of(element);
|
||||
let taken = |list: &[Value]| anchor.as_ref().is_some_and(|a| Positions::of(list).find(list.len(), a).is_some());
|
||||
match top.entry(key.clone()).or_insert_with(|| json!([])) {
|
||||
Value::Array(list) if name.is_none() || !list.iter().any(|x| x.get("name") == name) => list.push(element.clone()),
|
||||
Value::Array(list) if !taken(list) => list.push(element.clone()),
|
||||
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
|
||||
}
|
||||
}
|
||||
@@ -285,7 +293,8 @@ pub fn looks_default(v: &Value) -> bool {
|
||||
}
|
||||
|
||||
/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names.
|
||||
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, container_only: &[&str]) -> Option<Kept> {
|
||||
/// `top_only` are top-level keys left out of the comparison.
|
||||
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, top_only: &[&str]) -> Option<Kept> {
|
||||
match (raw, known) {
|
||||
(Value::Object(raw), Value::Object(known)) => {
|
||||
if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) {
|
||||
@@ -293,7 +302,7 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, pla
|
||||
}
|
||||
let mut kept = BTreeMap::new();
|
||||
for (key, value) in raw {
|
||||
if container_only.contains(&key.as_str()) || (path.is_empty() && matches!(key.as_str(), "format" | "format_pending")) {
|
||||
if path.is_empty() && top_only.contains(&key.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
|
||||
@@ -305,7 +314,7 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, pla
|
||||
kept.insert(key.clone(), Kept::Field(value.clone()));
|
||||
}
|
||||
Some(known) => {
|
||||
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, container_only) {
|
||||
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) {
|
||||
kept.insert(key.clone(), k);
|
||||
}
|
||||
}
|
||||
@@ -324,7 +333,7 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, pla
|
||||
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
|
||||
_ => format!("{path}[{at}]"),
|
||||
};
|
||||
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, container_only) {
|
||||
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) {
|
||||
items.push((anchor, k));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,3 +126,41 @@ fn records_match_by_id_name_or_unchanged_index() {
|
||||
by_name.restore_into(&mut renamed);
|
||||
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
|
||||
}
|
||||
|
||||
/// A section kept whole goes back whole, even when this build's record of it has fields.
|
||||
#[test]
|
||||
fn a_section_kept_whole_is_not_compared_field_by_field() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable");
|
||||
let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}});
|
||||
said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]);
|
||||
let mut saved = json!({"name": "A", "plugins": {"mode": "none"}});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, file);
|
||||
}
|
||||
|
||||
/// A kept element goes back only while no record has its id or name.
|
||||
#[test]
|
||||
fn a_kept_element_keeps_record_identity() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into());
|
||||
said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into());
|
||||
let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]});
|
||||
assert_eq!(said.restore_into(&mut clash).len(), 1);
|
||||
assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]}));
|
||||
let mut free = json!({"items": []});
|
||||
assert!(said.restore_into(&mut free).is_empty());
|
||||
assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]}));
|
||||
}
|
||||
|
||||
/// Container-only fields are left out at the top level only.
|
||||
#[test]
|
||||
fn container_only_fields_are_top_level() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}});
|
||||
said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]);
|
||||
assert_eq!(said.not_understood, ["field `layer.thumbnail`"]);
|
||||
let mut saved = json!({"layer": {}});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user