mirror of
https://github.com/storytold/vectorcraft.git
synced 2026-11-16 12:03:31 +00:00
Merge pull request #7866 from storytold/codex/shared-gzip-consumer
Use shared streaming native format writer
This commit is contained in:
Generated
+10
-1
@@ -1016,6 +1016,14 @@ name = "craft-fonts-build"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=c4b8405d5fd8f29cf77de528206bd2b38bb0ecb6#c4b8405d5fd8f29cf77de528206bd2b38bb0ecb6"
|
||||
|
||||
[[package]]
|
||||
name = "craft-gzip"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=b0d84cd15db39f2af9869cfb47b8b72cbeca3f20#b0d84cd15db39f2af9869cfb47b8b72cbeca3f20"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-i18n"
|
||||
version = "0.1.0"
|
||||
@@ -1212,7 +1220,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "craft-vectorcraft-format"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=e98299546f5c5c3c22388d16507f8b5e61a5f280#e98299546f5c5c3c22388d16507f8b5e61a5f280"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=b0d84cd15db39f2af9869cfb47b8b72cbeca3f20#b0d84cd15db39f2af9869cfb47b8b72cbeca3f20"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"serde",
|
||||
@@ -6741,6 +6749,7 @@ dependencies = [
|
||||
name = "vectorcraft-format"
|
||||
version = "0.24.1"
|
||||
dependencies = [
|
||||
"craft-gzip",
|
||||
"craft-vectorcraft-format",
|
||||
"flate2",
|
||||
"proptest",
|
||||
|
||||
+2
-1
@@ -21,7 +21,8 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = "9be9a637
|
||||
# desktop. The `egui` feature adds `craft_window::decorate`. (craft-libs#217.)
|
||||
craft-window = { git = "https://github.com/storytold/craft-libs", rev = "9be9a63782298ab59cd9b7a42ee9cba4f42a5ccb", features = ["egui"] }
|
||||
# The `.vectorcraft` envelope: its members, version, embedded images and the data after the JSON.
|
||||
craft-vectorcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "e98299546f5c5c3c22388d16507f8b5e61a5f280" }
|
||||
craft-vectorcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "b0d84cd15db39f2af9869cfb47b8b72cbeca3f20" }
|
||||
craft-gzip = { git = "https://github.com/storytold/craft-libs", rev = "b0d84cd15db39f2af9869cfb47b8b72cbeca3f20" }
|
||||
# craft-update: app-specific pretend versions apply to startup and manual checks.
|
||||
# Native Affinity documents (craft-libs#914).
|
||||
craft-affinity = { git = "https://github.com/storytold/craft-libs", rev = "228f6d0c50f1fdee4c5124603602993fc79d4c9d" }
|
||||
|
||||
@@ -12,6 +12,7 @@ workspace = true
|
||||
vectorcraft-doc = { workspace = true }
|
||||
vectorcraft-brush = { workspace = true }
|
||||
craft-vectorcraft-format = { workspace = true }
|
||||
craft-gzip = { workspace = true }
|
||||
serde ={ workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
+42
-68
@@ -57,12 +57,14 @@ use vectorcraft_doc::{Document, ImageBlob};
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use atomic::{read_only_location, sync_file, write_atomic, write_atomic_with, write_new_with};
|
||||
use craft_gzip::Gz;
|
||||
pub use craft_vectorcraft_format::base64::{decode as base64_decode, decoded_len as base64_decoded_len, encode as base64_encode};
|
||||
use craft_vectorcraft_format::stream::{FileHeader, Indent, KeyScan, write_file_json};
|
||||
pub use craft_vectorcraft_format::{
|
||||
BLOB_MAGIC, COMPRESSED_SINCE, EXTENSION, ICC_MIME, INLINE_MAX, LEGACY_EXTENSION, MIN_VERSION, PREVIEW_MAX, VERSION, is_compressed, json_part,
|
||||
sniff,
|
||||
};
|
||||
use stream::{Gz, Indent, KeyScan, Out};
|
||||
use stream::Out;
|
||||
|
||||
/// Most bytes the JSON of a compressed file may unpack to (a guard against decompression bombs,
|
||||
/// as are [`MAX_BLOB`] and [`MAX_BLOBS`]). A compressed save refuses a document past any of them,
|
||||
@@ -367,35 +369,44 @@ pub fn save_to(doc: &Document, o: &SaveOptions, w: impl Write) -> Result<(), For
|
||||
let profiles: BTreeMap<&str, Image> = o.profiles.iter().map(|(name, icc)| (name.as_str(), out_blobs.image(ICC_MIME, icc, false))).collect();
|
||||
let pdf = o.pdf.as_deref().map(|p| out_blobs.image("application/pdf", p, false));
|
||||
let mut out = if o.compress { Out::Gz(Box::new(Gz::new(w).map_err(encode_err)?)) } else { Out::Plain(w) };
|
||||
let mut env = Envelope { pretty, first: true };
|
||||
// Older apps only know the name from before the rename.
|
||||
env.field(&mut out, "format", &if o.version < 3 { LEGACY_EXTENSION } else { EXTENSION })?;
|
||||
env.field(&mut out, "version", &o.version)?;
|
||||
env.field(&mut out, "generator", &format!("VectorCraft {}", env!("CARGO_PKG_VERSION")))?;
|
||||
if let Some(compatible) = o.pdf_compatible {
|
||||
env.field(&mut out, "pdfCompatible", &compatible)?;
|
||||
}
|
||||
if let Some(png) = &o.preview {
|
||||
env.field(&mut out, "preview", &Image::inline("image/png", png))?;
|
||||
}
|
||||
if !profiles.is_empty() {
|
||||
env.field(&mut out, "profiles", &profiles)?;
|
||||
}
|
||||
if let Some(pdf) = &pdf {
|
||||
env.field(&mut out, "pdf", pdf)?;
|
||||
}
|
||||
env.key(&mut out, "document")?;
|
||||
let mut scan = KeyScan::new(Indent { inner: &mut out, on: pretty });
|
||||
// Without blobs no key matters.
|
||||
scan.active = images.is_none() && !blobs.is_empty();
|
||||
let mut buffered = std::io::BufWriter::with_capacity(JSON_CHUNK, &mut scan);
|
||||
write_body(&mut buffered, v1.as_ref(), &d, pretty)?;
|
||||
buffered.flush().map_err(encode_err)?;
|
||||
drop(buffered);
|
||||
let used = std::mem::take(&mut scan.keys);
|
||||
let images = images.unwrap_or_else(|| entries.images(&used, &mut out_blobs));
|
||||
env.field(&mut out, "images", &images)?;
|
||||
env.finish(&mut out)?;
|
||||
let generator = format!("VectorCraft {}", env!("CARGO_PKG_VERSION"));
|
||||
let preview = o.preview.as_deref().map(|png| Image::inline("image/png", png));
|
||||
let header = FileHeader {
|
||||
version: o.version,
|
||||
generator: &generator,
|
||||
pdf_compatible: o.pdf_compatible,
|
||||
preview: preview.as_ref(),
|
||||
profiles: &profiles,
|
||||
pdf: pdf.as_ref(),
|
||||
};
|
||||
write_file_json(&mut out, &header, pretty, |document| {
|
||||
let mut scan = KeyScan::new(document);
|
||||
// Without blobs no key matters.
|
||||
scan.active = images.is_none() && !blobs.is_empty();
|
||||
let mut buffered = std::io::BufWriter::with_capacity(JSON_CHUNK, &mut scan);
|
||||
match v1.as_ref() {
|
||||
Some(v) => {
|
||||
if pretty {
|
||||
serde_json::to_writer_pretty(&mut buffered, v)
|
||||
} else {
|
||||
serde_json::to_writer(&mut buffered, v)
|
||||
}
|
||||
}
|
||||
None => {
|
||||
if pretty {
|
||||
serde_json::to_writer_pretty(&mut buffered, &d)
|
||||
} else {
|
||||
serde_json::to_writer(&mut buffered, &d)
|
||||
}
|
||||
}
|
||||
}
|
||||
.map_err(std::io::Error::other)?;
|
||||
buffered.flush()?;
|
||||
drop(buffered);
|
||||
let used = std::mem::take(&mut scan.keys);
|
||||
Ok(images.unwrap_or_else(|| entries.images(&used, &mut out_blobs)))
|
||||
})
|
||||
.map_err(encode_err)?;
|
||||
if o.compress {
|
||||
let json = out.written().unwrap_or(0);
|
||||
let tail: Vec<u64> = out_blobs.parts().iter().map(|p| p.bytes.len() as u64).collect();
|
||||
@@ -409,13 +420,7 @@ pub fn save_to(doc: &Document, o: &SaveOptions, w: impl Write) -> Result<(), For
|
||||
return Err(FormatError::TooBigToCompress(format!("{} MB of images, at most {} MB", out_blobs.len() >> 20, MAX_BLOBS >> 20)));
|
||||
}
|
||||
}
|
||||
if !out_blobs.is_empty() {
|
||||
out.write_all(BLOB_MAGIC).map_err(encode_err)?;
|
||||
for part in out_blobs.parts() {
|
||||
out.packed(is_packed(&part.mime)).map_err(encode_err)?;
|
||||
out.write_all(part.bytes).map_err(encode_err)?;
|
||||
}
|
||||
}
|
||||
out_blobs.write_to_with(&mut out, |out, mime| out.packed(is_packed(mime))).map_err(encode_err)?;
|
||||
out.finish().map_err(encode_err)
|
||||
}
|
||||
|
||||
@@ -450,10 +455,6 @@ fn encode_err(e: impl std::fmt::Display) -> FormatError {
|
||||
FormatError::Encode(e.to_string())
|
||||
}
|
||||
|
||||
fn to_json(v: &impl Serialize, pretty: bool) -> Result<Vec<u8>, FormatError> {
|
||||
if pretty { serde_json::to_vec_pretty(v) } else { serde_json::to_vec(v) }.map_err(encode_err)
|
||||
}
|
||||
|
||||
/// The document's JSON: its version 1 form `v1`, else `d` (generic, not `dyn Write`: serde_json
|
||||
/// writes token by token).
|
||||
fn write_body(w: impl Write, v1: Option<&serde_json::Value>, d: &Document, pretty: bool) -> Result<(), FormatError> {
|
||||
@@ -467,33 +468,6 @@ fn write_json(w: impl Write, v: &impl Serialize, pretty: bool) -> Result<(), For
|
||||
if pretty { serde_json::to_writer_pretty(w, v) } else { serde_json::to_writer(w, v) }.map_err(encode_err)
|
||||
}
|
||||
|
||||
/// The file's top-level object, written member by member (the document is streamed in).
|
||||
struct Envelope {
|
||||
pretty: bool,
|
||||
first: bool,
|
||||
}
|
||||
|
||||
impl Envelope {
|
||||
fn field(&mut self, out: &mut impl Write, key: &str, value: &impl Serialize) -> Result<(), FormatError> {
|
||||
let json = to_json(value, self.pretty)?;
|
||||
self.key(out, key)?;
|
||||
Indent { inner: out, on: self.pretty }.write_all(&json).map_err(encode_err)
|
||||
}
|
||||
|
||||
/// The start of member `key` (a plain name): its value follows, indented one level when
|
||||
/// pretty (as serde_json would: see [`Indent`]).
|
||||
fn key(&mut self, out: &mut impl Write, key: &str) -> Result<(), FormatError> {
|
||||
let sep: &[u8] = if std::mem::take(&mut self.first) { b"{" } else { b"," };
|
||||
let indent: &[u8] = if self.pretty { b"\n " } else { b"" };
|
||||
let colon: &[u8] = if self.pretty { b"\": " } else { b"\":" };
|
||||
[sep, indent, b"\"", key.as_bytes(), colon].iter().try_for_each(|part| out.write_all(part)).map_err(encode_err)
|
||||
}
|
||||
|
||||
fn finish(self, out: &mut impl Write) -> Result<(), FormatError> {
|
||||
out.write_all(if self.pretty { b"\n}" } else { b"}" }).map_err(encode_err)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(bytes: &[u8]) -> Result<Document, FormatError> {
|
||||
load_info(bytes).map(|(doc, _)| doc)
|
||||
}
|
||||
|
||||
+2
-299
@@ -2,10 +2,9 @@
|
||||
//! not) and the data after it is copied in from the document's own buffers, so a save holds no
|
||||
//! copy of the file in memory besides what the caller's writer keeps.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io::{self, Write};
|
||||
|
||||
use flate2::{Compress, Compression, Crc, FlushCompress, Status};
|
||||
use craft_gzip::Gz;
|
||||
|
||||
/// The output of a save: plain, or gzip ([`Gz`]).
|
||||
pub(crate) enum Out<W: Write> {
|
||||
@@ -26,7 +25,7 @@ impl<W: Write> Out<W> {
|
||||
pub(crate) fn written(&self) -> Option<u64> {
|
||||
match self {
|
||||
Out::Plain(_) => None,
|
||||
Out::Gz(gz) => Some(gz.total),
|
||||
Out::Gz(gz) => Some(gz.written()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,299 +60,3 @@ impl<W: Write> Write for Out<W> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes of compressed output gathered before they go to the writer.
|
||||
const OUT_CHUNK: usize = 64 << 10;
|
||||
|
||||
/// A gzip stream, written as [`flate2::write::GzEncoder`] at the default level writes it (the
|
||||
/// same header and the same data; the deflate blocks are the same too where flate2 deflates with
|
||||
/// miniz_oxide, not with zlib-rs, which writes them by how the data is fed), except that data which
|
||||
/// is compressed already ([`Gz::packed`]) is stored as it is: the default level spends a second
|
||||
/// or more per photo on it for next to nothing, and makes it slower to unpack. Where the level
|
||||
/// changes the deflate blocks so far end with a sync flush and a fresh compressor writes on,
|
||||
/// which makes one valid deflate stream that every reader unpacks.
|
||||
pub(crate) struct Gz<W: Write> {
|
||||
out: W,
|
||||
deflate: Compress,
|
||||
/// What `deflate` writes: stored blocks (packed data), else the default level.
|
||||
stored: bool,
|
||||
crc: Crc,
|
||||
buf: Vec<u8>,
|
||||
/// Bytes written (unpacked).
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl<W: Write> Gz<W> {
|
||||
pub(crate) fn new(mut out: W) -> io::Result<Self> {
|
||||
// flate2's header for the default level: no name, no time, unknown OS.
|
||||
out.write_all(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255])?;
|
||||
Ok(Self {
|
||||
out,
|
||||
deflate: Compress::new(Compression::default(), false),
|
||||
stored: false,
|
||||
crc: Crc::new(),
|
||||
buf: Vec::with_capacity(OUT_CHUNK),
|
||||
total: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Run the compressor over `input` with `flush` until it has taken it all (and, flushing,
|
||||
/// written everything out).
|
||||
fn deflate(&mut self, mut input: &[u8], flush: FlushCompress) -> io::Result<()> {
|
||||
loop {
|
||||
if self.buf.len() == self.buf.capacity() {
|
||||
self.out.write_all(&self.buf)?;
|
||||
self.buf.clear();
|
||||
}
|
||||
let (before_in, before_out) = (self.deflate.total_in(), self.deflate.total_out());
|
||||
let status = self.deflate.compress_vec(input, &mut self.buf, flush).map_err(io::Error::other)?;
|
||||
let taken = usize::try_from(self.deflate.total_in() - before_in).unwrap_or(input.len()).min(input.len());
|
||||
input = input.get(taken..).unwrap_or_default();
|
||||
let made = self.deflate.total_out() - before_out;
|
||||
let full = self.buf.len() == self.buf.capacity();
|
||||
match flush {
|
||||
FlushCompress::None if input.is_empty() => return Ok(()),
|
||||
FlushCompress::Finish if status == Status::StreamEnd => return Ok(()),
|
||||
// A flush is done once it left room in the output.
|
||||
FlushCompress::Sync if input.is_empty() && !full => return Ok(()),
|
||||
_ => {}
|
||||
}
|
||||
if taken == 0 && made == 0 && !full {
|
||||
return Err(io::Error::other("the compressor made no progress"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What is written next is compressed already (stored as it is), or not (compressed at the
|
||||
/// default level).
|
||||
fn packed(&mut self, packed: bool) -> io::Result<()> {
|
||||
if packed == self.stored {
|
||||
return Ok(());
|
||||
}
|
||||
self.deflate(&[], FlushCompress::Sync)?;
|
||||
self.deflate = Compress::new(if packed { Compression::none() } else { Compression::default() }, false);
|
||||
self.stored = packed;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(mut self) -> io::Result<()> {
|
||||
self.deflate(&[], FlushCompress::Finish)?;
|
||||
self.out.write_all(&self.buf)?;
|
||||
self.out.write_all(&self.crc.sum().to_le_bytes())?;
|
||||
self.out.write_all(&self.crc.amount().to_le_bytes())?;
|
||||
self.out.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> Write for Gz<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.write_all(buf)?;
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.crc.update(buf);
|
||||
self.total += buf.len() as u64;
|
||||
self.deflate(buf, FlushCompress::None)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
// Flushing would end a deflate block: the stream is flushed when it is finished.
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Indents JSON text written through it one level (two spaces after each newline), as a member
|
||||
/// of the file's top-level object; newlines only occur between tokens, inside strings they are
|
||||
/// escaped.
|
||||
pub(crate) struct Indent<W: Write> {
|
||||
pub(crate) inner: W,
|
||||
pub(crate) on: bool,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for Indent<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.write_all(buf)?;
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
if !self.on {
|
||||
return self.inner.write_all(buf);
|
||||
}
|
||||
let mut lines = buf.split(|&b| b == b'\n');
|
||||
self.inner.write_all(lines.next().unwrap_or_default())?;
|
||||
for line in lines {
|
||||
self.inner.write_all(b"\n ")?;
|
||||
self.inner.write_all(line)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// The member that names an image.
|
||||
const TAG: &[u8] = b"\"key\":";
|
||||
|
||||
/// Passes JSON text through, collecting the string values of every `"key": "…"` member at any
|
||||
/// depth (what the images a document uses are named by). Quotes inside JSON strings are escaped,
|
||||
/// so every match is a real member.
|
||||
pub(crate) struct KeyScan<W: Write> {
|
||||
pub(crate) inner: W,
|
||||
pub(crate) keys: HashSet<String>,
|
||||
/// Looking for keys (else it only passes the text on).
|
||||
pub(crate) active: bool,
|
||||
/// How much of [`TAG`] the last bytes matched.
|
||||
matched: usize,
|
||||
/// Right after a tag: skipping whitespace to its value.
|
||||
after_tag: bool,
|
||||
/// The value string so far, quotes included.
|
||||
value: Option<Vec<u8>>,
|
||||
escaped: bool,
|
||||
}
|
||||
|
||||
impl<W: Write> KeyScan<W> {
|
||||
pub(crate) fn new(inner: W) -> Self {
|
||||
Self { inner, keys: HashSet::new(), active: true, matched: 0, after_tag: false, value: None, escaped: false }
|
||||
}
|
||||
|
||||
fn scan(&mut self, buf: &[u8]) {
|
||||
if !self.active {
|
||||
return;
|
||||
}
|
||||
let mut i = 0;
|
||||
while let Some(&b) = buf.get(i) {
|
||||
// Nothing in progress: skip to the next quote.
|
||||
if self.matched == 0 && !self.after_tag && self.value.is_none() {
|
||||
match buf.get(i..).and_then(|r| r.iter().position(|&c| c == b'"')) {
|
||||
Some(at) => i += at,
|
||||
None => return,
|
||||
}
|
||||
self.matched = 1;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
self.step(b);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn step(&mut self, b: u8) {
|
||||
if let Some(v) = self.value.as_mut() {
|
||||
v.push(b);
|
||||
if self.escaped {
|
||||
self.escaped = false;
|
||||
} else if b == b'\\' {
|
||||
self.escaped = true;
|
||||
} else if b == b'"'
|
||||
&& let Some(v) = self.value.take()
|
||||
&& let Ok(s) = serde_json::from_slice::<String>(&v)
|
||||
{
|
||||
self.keys.insert(s);
|
||||
}
|
||||
} else if self.after_tag && !b.is_ascii_whitespace() {
|
||||
self.after_tag = false;
|
||||
if b == b'"' {
|
||||
self.value = Some(vec![b]);
|
||||
}
|
||||
}
|
||||
if TAG.get(self.matched) == Some(&b) {
|
||||
self.matched += 1;
|
||||
if self.matched == TAG.len() {
|
||||
self.matched = 0;
|
||||
self.after_tag = true;
|
||||
self.value = None;
|
||||
self.escaped = false;
|
||||
}
|
||||
} else {
|
||||
self.matched = usize::from(b == b'"');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> Write for KeyScan<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.write_all(buf)?;
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.scan(buf);
|
||||
self.inner.write_all(buf)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// How the keys were found before saves were streamed: in the whole JSON text at once.
|
||||
fn string_members(json: &[u8]) -> HashSet<String> {
|
||||
let mut out = HashSet::new();
|
||||
let mut rest = json;
|
||||
while let Some(at) = rest.windows(TAG.len()).position(|w| w == TAG) {
|
||||
rest = rest.get(at + TAG.len()..).unwrap_or_default();
|
||||
let value = rest.trim_ascii_start();
|
||||
if value.first() == Some(&b'"')
|
||||
&& let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::<String>().next()
|
||||
{
|
||||
out.insert(s);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The keys `KeyScan` finds in `json` written in pieces of `chunk` bytes.
|
||||
fn scanned(json: &[u8], chunk: usize) -> HashSet<String> {
|
||||
let mut scan = KeyScan::new(Vec::new());
|
||||
for piece in json.chunks(chunk.max(1)) {
|
||||
scan.write_all(piece).unwrap();
|
||||
}
|
||||
assert_eq!(scan.inner, json, "the text passes through");
|
||||
scan.keys
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_the_keys_the_whole_text_search_found() {
|
||||
let v = serde_json::json!({"key": "a", "x": [{"key": "b \"q\" \\ é"}, {"monkey": "no"}, {"k": "key"}], "s": "\"key\": \"no\"", "key2": "n", "o": {"key": 3}});
|
||||
for json in [serde_json::to_vec(&v).unwrap(), serde_json::to_vec_pretty(&v).unwrap()] {
|
||||
let want = string_members(&json);
|
||||
assert_eq!(want, HashSet::from(["a".to_string(), "b \"q\" \\ é".to_string()]));
|
||||
for chunk in [1, 2, 3, 7, 64, json.len()] {
|
||||
assert_eq!(scanned(&json, chunk), want, "chunk {chunk}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn arb_json() -> impl Strategy<Value = serde_json::Value> {
|
||||
let key = prop::sample::select(vec!["key", "k", "kkey", "key ", "x"]).prop_map(str::to_string);
|
||||
let leaf = prop_oneof![
|
||||
Just(serde_json::Value::Null),
|
||||
any::<i32>().prop_map(serde_json::Value::from),
|
||||
"[a-z\"\\\\: ké\n]{0,8}".prop_map(serde_json::Value::from),
|
||||
];
|
||||
leaf.prop_recursive(4, 32, 6, move |inner| {
|
||||
prop_oneof![
|
||||
prop::collection::vec(inner.clone(), 0..5).prop_map(serde_json::Value::from),
|
||||
prop::collection::vec((key.clone(), inner), 0..5).prop_map(|m| serde_json::Value::Object(m.into_iter().collect())),
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn scanning_in_pieces_finds_what_the_whole_text_search_found(v in arb_json(), chunk in 1usize..40, pretty in any::<bool>()) {
|
||||
let json = if pretty { serde_json::to_vec_pretty(&v) } else { serde_json::to_vec(&v) }.unwrap();
|
||||
prop_assert_eq!(scanned(&json, chunk), string_members(&json));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user