Merge pull request #1598 from storytold/codex/vectorcraft-streaming-envelope

Share VectorCraft streaming envelope helpers
This commit is contained in:
Dylan M. Taylor
2026-11-15 11:48:48 -05:00
committed by GitHub
5 changed files with 500 additions and 1 deletions
Generated
+1
View File
@@ -2866,6 +2866,7 @@ name = "craft-vectorcraft-format"
version = "0.1.0"
dependencies = [
"flate2",
"proptest",
"serde",
"serde_json",
"thiserror 2.0.21",
+3
View File
@@ -15,3 +15,6 @@ flate2 = { version = "1", default-features = false, features = ["rust_backend"]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+12 -1
View File
@@ -34,6 +34,7 @@
//! [`sniff`] reads: a reader of a compressed file decompresses it with its own limits first.
pub mod base64;
pub mod stream;
use std::collections::{BTreeMap, HashMap};
use std::fmt;
@@ -769,10 +770,20 @@ impl<'a> Tail<'a> {
/// Write what follows the JSON: [`BLOB_MAGIC`] and the entries, or nothing when there are none.
pub fn write_to(&self, mut w: impl Write) -> io::Result<()> {
self.write_to_with(&mut w, |_, _| Ok(()))
}
/// Write the marker and parts, calling `before_part` before each payload. A compressed
/// writer can use the MIME type to switch between compressed and stored deflate blocks.
pub fn write_to_with<W: Write>(&self, mut w: W, mut before_part: impl FnMut(&mut W, &str) -> io::Result<()>) -> io::Result<()> {
if self.parts.is_empty() {
return Ok(());
}
w.write_all(BLOB_MAGIC)?;
self.parts.iter().try_for_each(|p| w.write_all(p.bytes))
for part in &self.parts {
before_part(&mut w, &part.mime)?;
w.write_all(part.bytes)?;
}
Ok(())
}
}
+302
View File
@@ -0,0 +1,302 @@
//! Streaming helpers for the native JSON envelope.
//!
//! Field values are emitted in bounded chunks. The decoded JSON and tail framing stay the same
//! as VectorCraft's previous writer, but compressed bytes can differ when a deflate backend's
//! output depends on input chunking. Gzip readers still validate the CRC and size trailer.
//!
//! Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors.
//! Extracted from VectorCraft, licensed under MIT OR Apache-2.0.
use serde::Serialize;
use std::collections::{BTreeMap, HashSet};
use std::io::{self, Write};
/// Native file metadata for streaming a JSON envelope. Asset entries should be prepared using
/// the same [`crate::Tail`] whose parts will be written after the JSON.
pub struct FileHeader<'a> {
/// Native format version to write.
pub version: u32,
/// Application generator string.
pub generator: &'a str,
/// Optional saved PDF compatibility preference.
pub pdf_compatible: Option<bool>,
/// Optional inline preview.
pub preview: Option<&'a crate::Image>,
/// Embedded profiles keyed by profile name.
pub profiles: &'a BTreeMap<&'a str, crate::Image>,
/// Optional PDF entry.
pub pdf: Option<&'a crate::Image>,
}
/// Write the native JSON envelope, streaming the document through `document`. The callback
/// returns the image entries after serializing the document, allowing it to collect image keys
/// without keeping a copy of the document JSON. It must write one complete JSON value.
pub fn write_file_json<'a>(
mut out: impl Write,
header: &FileHeader<'a>,
pretty: bool,
document: impl FnOnce(&mut dyn Write) -> io::Result<BTreeMap<&'a str, crate::Image>>,
) -> io::Result<()> {
if !(crate::MIN_VERSION..=crate::VERSION).contains(&header.version) {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "unsupported native file version"));
}
let mut env = Envelope::new(pretty);
let format = if header.version < 3 { crate::LEGACY_EXTENSION } else { crate::EXTENSION };
env.field(&mut out, "format", &format)?;
env.field(&mut out, "version", &header.version)?;
env.field(&mut out, "generator", &header.generator)?;
if let Some(compatible) = header.pdf_compatible {
env.field(&mut out, "pdfCompatible", &compatible)?;
}
if let Some(preview) = header.preview {
env.field(&mut out, "preview", preview)?;
}
if !header.profiles.is_empty() {
env.field(&mut out, "profiles", header.profiles)?;
}
if let Some(pdf) = header.pdf {
env.field(&mut out, "pdf", pdf)?;
}
env.key(&mut out, "document")?;
let images = document(&mut Indent { inner: &mut out, on: pretty })?;
env.field(&mut out, "images", &images)?;
env.finish(&mut out)
}
/// Indents JSON text written through it one level (two spaces after each newline), as a member
/// of the file's top-level object; newlines only occur between tokens, inside strings they are
/// escaped.
pub struct Indent<W: Write> {
pub inner: W,
pub on: bool,
}
impl<W: Write> Write for Indent<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.write_all(buf)?;
Ok(buf.len())
}
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
if !self.on {
return self.inner.write_all(buf);
}
let mut lines = buf.split(|&b| b == b'\n');
self.inner.write_all(lines.next().unwrap_or_default())?;
for line in lines {
self.inner.write_all(b"\n ")?;
self.inner.write_all(line)?;
}
Ok(())
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// The member that names an image.
const TAG: &[u8] = b"\"key\":";
/// Passes JSON text through, collecting the string values of every `"key": "…"` member at any
/// depth (what the images a document uses are named by). Quotes inside JSON strings are escaped,
/// so every match is a real member.
pub struct KeyScan<W: Write> {
pub inner: W,
pub keys: HashSet<String>,
/// Looking for keys (else it only passes the text on).
pub active: bool,
/// How much of [`TAG`] the last bytes matched.
matched: usize,
/// Right after a tag: skipping whitespace to its value.
after_tag: bool,
/// The value string so far, quotes included.
value: Option<Vec<u8>>,
escaped: bool,
}
impl<W: Write> KeyScan<W> {
pub fn new(inner: W) -> Self {
Self { inner, keys: HashSet::new(), active: true, matched: 0, after_tag: false, value: None, escaped: false }
}
fn scan(&mut self, buf: &[u8]) {
if !self.active {
return;
}
let mut i = 0;
while let Some(&b) = buf.get(i) {
// Nothing in progress: skip to the next quote.
if self.matched == 0 && !self.after_tag && self.value.is_none() {
match buf.get(i..).and_then(|r| r.iter().position(|&c| c == b'"')) {
Some(at) => i += at,
None => return,
}
self.matched = 1;
i += 1;
continue;
}
self.step(b);
i += 1;
}
}
fn step(&mut self, b: u8) {
if let Some(v) = self.value.as_mut() {
v.push(b);
if self.escaped {
self.escaped = false;
} else if b == b'\\' {
self.escaped = true;
} else if b == b'"'
&& let Some(v) = self.value.take()
&& let Ok(s) = serde_json::from_slice::<String>(&v)
{
self.keys.insert(s);
}
} else if self.after_tag && !b.is_ascii_whitespace() {
self.after_tag = false;
if b == b'"' {
self.value = Some(vec![b]);
}
}
if TAG.get(self.matched) == Some(&b) {
self.matched += 1;
if self.matched == TAG.len() {
self.matched = 0;
self.after_tag = true;
self.value = None;
self.escaped = false;
}
} else {
self.matched = usize::from(b == b'"');
}
}
}
impl<W: Write> Write for KeyScan<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.write_all(buf)?;
Ok(buf.len())
}
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
self.scan(buf);
self.inner.write_all(buf)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// The file's top-level object, written member by member (the document is streamed in).
pub struct Envelope {
pretty: bool,
first: bool,
}
impl Envelope {
/// Start an object with optional indentation.
pub fn new(pretty: bool) -> Self {
Self { pretty, first: true }
}
/// Serialize a member directly to the output without buffering the full value.
pub fn field(&mut self, out: &mut impl Write, key: &str, value: &impl Serialize) -> io::Result<()> {
self.key(out, key)?;
// serde_json emits small writes; keep compression and underlying I/O efficient while
// bounding temporary memory independently of the serialized table size.
let mut buffered = io::BufWriter::with_capacity(64 << 10, out);
let indented = Indent { inner: &mut buffered, on: self.pretty };
if self.pretty { serde_json::to_writer_pretty(indented, value) } else { serde_json::to_writer(indented, value) }.map_err(io::Error::other)?;
buffered.into_inner().map(|_| ()).map_err(|e| e.into_error())
}
/// The start of member `key` (a plain name): its value follows, indented one level when
/// pretty (as serde_json would: see [`Indent`]).
pub fn key(&mut self, out: &mut impl Write, key: &str) -> io::Result<()> {
let sep: &[u8] = if std::mem::take(&mut self.first) { b"{" } else { b"," };
let indent: &[u8] = if self.pretty { b"\n " } else { b"" };
out.write_all(sep)?;
out.write_all(indent)?;
serde_json::to_writer(&mut *out, key).map_err(io::Error::other)?;
out.write_all(if self.pretty { b": " } else { b":" })
}
/// Close the object.
pub fn finish(self, out: &mut impl Write) -> io::Result<()> {
if self.first {
return out.write_all(b"{}");
}
out.write_all(if self.pretty { b"\n}" } else { b"}" })
}
}
#[cfg(test)]
mod tests {
use proptest::prelude::*;
use super::*;
/// How the keys were found before saves were streamed: in the whole JSON text at once.
fn string_members(json: &[u8]) -> HashSet<String> {
let mut out = HashSet::new();
let mut rest = json;
while let Some(at) = rest.windows(TAG.len()).position(|w| w == TAG) {
rest = rest.get(at + TAG.len()..).unwrap_or_default();
let value = rest.trim_ascii_start();
if value.first() == Some(&b'"')
&& let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::<String>().next()
{
out.insert(s);
}
}
out
}
/// The keys `KeyScan` finds in `json` written in pieces of `chunk` bytes.
fn scanned(json: &[u8], chunk: usize) -> HashSet<String> {
let mut scan = KeyScan::new(Vec::new());
for piece in json.chunks(chunk.max(1)) {
scan.write_all(piece).unwrap();
}
assert_eq!(scan.inner, json, "the text passes through");
scan.keys
}
#[test]
fn finds_the_keys_the_whole_text_search_found() {
let v = serde_json::json!({"key": "a", "x": [{"key": "b \"q\" \\ é"}, {"monkey": "no"}, {"k": "key"}], "s": "\"key\": \"no\"", "key2": "n", "o": {"key": 3}});
for json in [serde_json::to_vec(&v).unwrap(), serde_json::to_vec_pretty(&v).unwrap()] {
let want = string_members(&json);
assert_eq!(want, HashSet::from(["a".to_string(), "b \"q\" \\ é".to_string()]));
for chunk in [1, 2, 3, 7, 64, json.len()] {
assert_eq!(scanned(&json, chunk), want, "chunk {chunk}");
}
}
}
fn arb_json() -> impl Strategy<Value = serde_json::Value> {
let key = prop::sample::select(vec!["key", "k", "kkey", "key ", "x"]).prop_map(str::to_string);
let leaf = prop_oneof![
Just(serde_json::Value::Null),
any::<i32>().prop_map(serde_json::Value::from),
"[a-z\"\\\\: ké\n]{0,8}".prop_map(serde_json::Value::from),
];
leaf.prop_recursive(4, 32, 6, move |inner| {
prop_oneof![
prop::collection::vec(inner.clone(), 0..5).prop_map(serde_json::Value::from),
prop::collection::vec((key.clone(), inner), 0..5).prop_map(|m| serde_json::Value::Object(m.into_iter().collect())),
]
})
}
proptest! {
#[test]
fn scanning_in_pieces_finds_what_the_whole_text_search_found(v in arb_json(), chunk in 1usize..40, pretty in any::<bool>()) {
let json = if pretty { serde_json::to_vec_pretty(&v) } else { serde_json::to_vec(&v) }.unwrap();
prop_assert_eq!(scanned(&json, chunk), string_members(&json));
}
}
}
+182
View File
@@ -0,0 +1,182 @@
use craft_vectorcraft_format::stream::{Envelope, Indent, KeyScan};
use std::io::{self, Write};
#[test]
fn envelope_streams_nested_json_and_escapes_keys() {
for pretty in [false, true] {
let mut bytes = Vec::new();
let mut env = Envelope::new(pretty);
env.field(&mut bytes, "quoted\"key", &vec![1, 2]).unwrap();
env.key(&mut bytes, "document").unwrap();
let mut output = Indent { inner: &mut bytes, on: pretty };
if pretty {
serde_json::to_writer_pretty(&mut output, &serde_json::json!({"nested": [3,4]})).unwrap();
} else {
serde_json::to_writer(&mut output, &serde_json::json!({"nested": [3,4]})).unwrap();
}
env.finish(&mut bytes).unwrap();
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(value, serde_json::json!({"quoted\"key": [1, 2], "document": {"nested": [3,4]}}));
let mut expected = Vec::new();
let empty = Envelope::new(pretty);
empty.finish(&mut expected).unwrap();
assert_eq!(expected, b"{}");
}
}
#[test]
fn scan_survives_every_chunk_boundary_and_forwards_data() {
let json = br#"{"key":"one","nested":[{"key":"two \\\"quoted\\\""},{"key":42}],"other":"x"}"#;
for chunk in 1..=json.len() {
let mut scan = KeyScan::new(Vec::new());
for part in json.chunks(chunk) {
scan.write_all(part).unwrap();
}
assert_eq!(scan.inner, json);
assert!(scan.keys.contains("one"));
assert_eq!(scan.keys.len(), 2);
}
}
struct Broken;
impl Write for Broken {
fn write(&mut self, _: &[u8]) -> io::Result<usize> {
Err(io::Error::other("failure"))
}
fn flush(&mut self) -> io::Result<()> {
Err(io::Error::other("failure"))
}
}
#[test]
fn forwards_writer_errors() {
assert!(Envelope::new(false).field(&mut Broken, "x", &1).is_err());
assert!(KeyScan::new(Broken).write_all(b"{}").is_err());
assert!(Indent { inner: Broken, on: true }.write_all(b"x\ny").is_err());
}
#[test]
fn tail_calls_hook_in_order_and_propagates_errors() {
use craft_vectorcraft_format::{BLOB_MAGIC, INLINE_MAX, Tail};
let first = vec![1; INLINE_MAX];
let second = vec![2; INLINE_MAX];
let mut tail = Tail::new(4);
tail.image("image/png", &first, false);
tail.image("application/pdf", &second, false);
let mut out = Vec::new();
let mut seen = Vec::new();
tail.write_to_with(&mut out, |_, mime| {
seen.push(mime.to_owned());
Ok(())
})
.unwrap();
assert_eq!(seen, ["image/png", "application/pdf"]);
assert_eq!(out, [BLOB_MAGIC, &first, &second].concat());
assert!(tail.write_to_with(Vec::new(), |_, _| Err(io::Error::other("failure"))).is_err());
let mut called = false;
Tail::new(4)
.write_to_with(Vec::new(), |_, _| {
called = true;
Ok(())
})
.unwrap();
assert!(!called);
}
#[test]
fn native_schema_order_and_versions() {
use craft_vectorcraft_format::stream::{FileHeader, write_file_json};
let profiles = Default::default();
for (version, name) in [(1, "drawcraft"), (4, "vectorcraft")] {
for pretty in [false, true] {
let header = FileHeader { version, generator: "test", pdf_compatible: Some(false), preview: None, profiles: &profiles, pdf: None };
let mut out = Vec::new();
write_file_json(&mut out, &header, pretty, |w| {
w.write_all(if pretty { b"{\n \"key\": \"image\"\n}" } else { b"{\"key\":\"image\"}" })?;
Ok(Default::default())
})
.unwrap();
let value: serde_json::Value = serde_json::from_slice(&out).unwrap();
assert_eq!(value["format"], name);
assert_eq!(value["pdfCompatible"], false);
let text = std::str::from_utf8(&out).unwrap();
assert!(text.find("\"format\"").unwrap() < text.find("\"version\"").unwrap());
assert!(text.find("\"document\"").unwrap() < text.find("\"images\"").unwrap());
}
}
let header = FileHeader { version: 0, generator: "test", pdf_compatible: None, preview: None, profiles: &profiles, pdf: None };
assert!(write_file_json(Vec::new(), &header, false, |_| Ok(Default::default())).is_err());
}
// Oracle for the previous app writer: each member was first serialized to a Vec, then
// indented and appended. Compare decoded file bytes, not backend-dependent gzip bytes.
#[test]
fn streamed_native_file_matches_previous_framing() {
use craft_vectorcraft_format::{
Tail,
stream::{FileHeader, write_file_json},
};
use flate2::{Compression, read::GzDecoder, write::GzEncoder};
use std::collections::BTreeMap;
use std::io::Read;
fn old_field(out: &mut Vec<u8>, first: &mut bool, pretty: bool, key: &str, value: &impl serde::Serialize) {
out.extend_from_slice(if std::mem::take(first) { b"{" } else { b"," });
if pretty {
out.extend_from_slice(b"\n ");
}
out.extend_from_slice(format!("\"{key}\"{}", if pretty { ": " } else { ":" }).as_bytes());
let json = if pretty { serde_json::to_vec_pretty(value) } else { serde_json::to_vec(value) }.unwrap();
let mut indent = Indent { inner: out, on: pretty };
indent.write_all(&json).unwrap();
}
let payload = vec![42; craft_vectorcraft_format::INLINE_MAX];
for version in [1, 3, 4] {
for pretty in [false, true] {
let mut tail = Tail::new(version);
let image = tail.image("image/png", &payload, false);
let profile = tail.image("application/vnd.iccprofile", &payload, false);
let pdf = tail.image("application/pdf", &payload, false);
let profiles = BTreeMap::from([("profile", profile)]);
let images = BTreeMap::from([("image", image)]);
let document = serde_json::json!({"layers": [{"key": "image", "name": "A"}]});
let header = FileHeader {
version,
generator: "VectorCraft test",
pdf_compatible: Some(true),
preview: None,
profiles: &profiles,
pdf: Some(&pdf),
};
let mut actual = Vec::new();
write_file_json(&mut actual, &header, pretty, |w| {
if pretty { serde_json::to_writer_pretty(w, &document) } else { serde_json::to_writer(w, &document) }.map_err(io::Error::other)?;
Ok(images.clone())
})
.unwrap();
tail.write_to(&mut actual).unwrap();
let mut old = Vec::new();
let mut first = true;
let format = if version < 3 { "drawcraft" } else { "vectorcraft" };
old_field(&mut old, &mut first, pretty, "format", &format);
old_field(&mut old, &mut first, pretty, "version", &version);
old_field(&mut old, &mut first, pretty, "generator", &"VectorCraft test");
old_field(&mut old, &mut first, pretty, "pdfCompatible", &true);
old_field(&mut old, &mut first, pretty, "profiles", &profiles);
old_field(&mut old, &mut first, pretty, "pdf", &pdf);
old_field(&mut old, &mut first, pretty, "document", &document);
old_field(&mut old, &mut first, pretty, "images", &images);
old.extend_from_slice(if pretty { b"\n}" } else { b"}" });
tail.write_to(&mut old).unwrap();
assert_eq!(actual, old, "version {version}, pretty {pretty}");
for bytes in [&old, &actual] {
let mut gzip = GzEncoder::new(Vec::new(), Compression::default());
gzip.write_all(bytes).unwrap();
let mut decoded = Vec::new();
GzDecoder::new(gzip.finish().unwrap().as_slice()).read_to_end(&mut decoded).unwrap();
assert_eq!(decoded, old);
}
}
}
}