diff --git a/Cargo.lock b/Cargo.lock index 7d56959d..307b428c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1088,6 +1088,10 @@ dependencies = [ "tempfile", ] +[[package]] +name = "craft-kiwi" +version = "0.1.0" + [[package]] name = "craft-lottie" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 96438def..c449f4f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } craft-idml = { path = "crates/idml" } craft-jpeg = { path = "crates/jpeg" } +craft-kiwi = { path = "crates/kiwi" } craft-keymap = { path = "crates/keymap" } craft-audio = { path = "crates/audio" } # No default features, so craft-audio's `encode` builds craft-media without `isobmff`. diff --git a/README.md b/README.md index 9a82818c..c004b1e4 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = ") -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for KiwiError {} + +fn err(msg: impl Into) -> Result { + Err(KiwiError(msg.into())) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + Enum, + Struct, + Message, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Type { + Bool, + Byte, + Int, + Uint, + Float, + Str, + Int64, + Uint64, + /// Index of a definition in the schema. + Def(usize), +} + +/// Built-in types in their schema order (encoded as `!index`). +const BUILTINS: [Type; 8] = [Type::Bool, Type::Byte, Type::Int, Type::Uint, Type::Float, Type::Str, Type::Int64, Type::Uint64]; + +#[derive(Clone, Debug)] +pub struct Field { + pub name: String, + /// `None` for enum values. + pub ty: Option, + pub array: bool, + /// The field id (messages) or the value (enums). + pub value: u32, +} + +#[derive(Clone, Debug)] +pub struct Def { + pub name: String, + pub kind: Kind, + pub fields: Vec, +} + +#[derive(Clone, Debug, Default)] +pub struct Schema { + pub defs: Vec, +} + +/// A decoded value. Field and enum names borrow from the schema. +#[derive(Clone, Debug, PartialEq)] +pub enum Kv<'s> { + Bool(bool), + Int(i64), + Uint(u64), + Float(f32), + Str(String), + /// A `byte[]` field. + Bytes(Vec), + Enum(&'s str), + Array(Vec>), + /// A struct or message: its fields in the order read. + Object(Vec<(&'s str, Kv<'s>)>), +} + +impl<'s> Kv<'s> { + /// Field `name` of an object. + pub fn get(&self, name: &str) -> Option<&Kv<'s>> { + match self { + Kv::Object(f) => f.iter().find(|(n, _)| *n == name).map(|(_, v)| v), + _ => None, + } + } + + pub fn f64(&self) -> Option { + let v = match self { + Kv::Float(f) => f64::from(*f), + Kv::Int(i) => *i as f64, + Kv::Uint(u) => *u as f64, + Kv::Bool(b) => f64::from(u8::from(*b)), + _ => return None, + }; + v.is_finite().then_some(v) + } + + pub fn bool(&self) -> Option { + match self { + Kv::Bool(b) => Some(*b), + _ => None, + } + } + + pub fn str(&self) -> Option<&str> { + match self { + Kv::Str(s) => Some(s), + Kv::Enum(s) => Some(s), + _ => None, + } + } + + pub fn list(&self) -> &[Kv<'s>] { + match self { + Kv::Array(a) => a, + _ => &[], + } + } + + pub fn bytes(&self) -> Option<&[u8]> { + match self { + Kv::Bytes(b) => Some(b), + _ => None, + } + } +} + +/// A bounds-checked reader over kiwi bytes. +struct Reader<'b> { + data: &'b [u8], + pos: usize, +} + +impl<'b> Reader<'b> { + fn left(&self) -> usize { + self.data.len().saturating_sub(self.pos) + } + + fn byte(&mut self) -> Result { + let b = *self.data.get(self.pos).ok_or_else(|| KiwiError("unexpected end of data".into()))?; + self.pos += 1; + Ok(b) + } + + fn take(&mut self, n: usize) -> Result<&'b [u8], KiwiError> { + let end = self.pos.checked_add(n).filter(|e| *e <= self.data.len()).ok_or_else(|| KiwiError("unexpected end of data".into()))?; + let s = self.data.get(self.pos..end).ok_or_else(|| KiwiError("unexpected end of data".into()))?; + self.pos = end; + Ok(s) + } + + fn var_uint(&mut self) -> Result { + let mut value = 0u32; + let mut shift = 0; + loop { + let b = self.byte()?; + let part = b & 127; + if shift == 28 && (part > 0x0f || b & 128 != 0) { + return err("u32 varint overflow"); + } + value |= u32::from(part) << shift; + shift += 7; + if b & 128 == 0 { + return Ok(value); + } + } + } + + fn var_int(&mut self) -> Result { + let v = self.var_uint()?; + Ok(if v & 1 != 0 { !((v >> 1) as i32) } else { (v >> 1) as i32 }) + } + + fn var_uint64(&mut self) -> Result { + let mut value = 0u64; + let mut shift = 0u32; + loop { + let b = self.byte()?; + if shift == 56 { + // The ninth byte carries a full eight bits. + return Ok(value | (u64::from(b) << 56)); + } + value |= u64::from(b & 127) << shift; + shift += 7; + if b & 128 == 0 { + return Ok(value); + } + } + } + + fn var_int64(&mut self) -> Result { + let v = self.var_uint64()?; + Ok(if v & 1 != 0 { !((v >> 1) as i64) } else { (v >> 1) as i64 }) + } + + fn var_float(&mut self) -> Result { + if self.data.get(self.pos) == Some(&0) { + self.pos += 1; + return Ok(0.0); + } + let bits = self.take(4)?.try_into().map(u32::from_le_bytes).map_err(|_| KiwiError("unexpected end of data".into()))?; + // The exponent is stored first so zero fits one byte; move it back into place. + Ok(f32::from_bits(bits.rotate_left(23))) + } + + fn string(&mut self) -> Result { + let rest = self.data.get(self.pos..).unwrap_or_default(); + let len = rest.iter().position(|b| *b == 0).ok_or_else(|| KiwiError("unterminated string".into()))?; + let s = String::from_utf8_lossy(self.take(len)?).into_owned(); + self.pos += 1; + Ok(s) + } +} + +impl Schema { + /// Decode a binary schema. + pub fn decode(data: &[u8]) -> Result { + let mut r = Reader { data, pos: 0 }; + let count = r.var_uint()? as usize; + if count > MAX_DEFS || count > r.left() { + return err("schema: too many definitions"); + } + let mut defs = Vec::with_capacity(count); + let mut raw_types = Vec::new(); + let mut total_fields = 0usize; + for _ in 0..count { + let name = r.string()?; + let kind = match r.byte()? { + 0 => Kind::Enum, + 1 => Kind::Struct, + 2 => Kind::Message, + k => return err(format!("schema: unknown definition kind {k}")), + }; + let n = r.var_uint()? as usize; + total_fields = total_fields.saturating_add(n); + if n > MAX_FIELDS || n > r.left() || total_fields > MAX_SCHEMA_FIELDS { + return err("schema: too many fields"); + } + let mut fields = Vec::with_capacity(n); + // A set, not a scan per field: wide definitions would make the scan quadratic. + let mut ids = HashSet::new(); + for _ in 0..n { + let fname = r.string()?; + let ty = r.var_int()?; + let array = r.byte()? & 1 != 0; + let value = r.var_uint()?; + if kind == Kind::Message && !ids.insert(value) { + return err(format!("schema: duplicate field id {value} in {name}")); + } + raw_types.push(ty); + fields.push(Field { name: fname, ty: None, array, value }); + } + defs.push(Def { name, kind, fields }); + } + let total = defs.len(); + let mut types = raw_types.into_iter(); + for d in &mut defs { + for f in &mut d.fields { + let raw = types.next().unwrap_or(0); + if d.kind == Kind::Enum { + continue; + } + f.ty = Some(if raw < 0 { + *BUILTINS.get((!raw) as usize).ok_or_else(|| KiwiError(format!("schema: unknown type {raw}")))? + } else { + let i = raw as usize; + if i >= total { + return err(format!("schema: field {} names missing definition {i}", f.name)); + } + Type::Def(i) + }); + } + } + Ok(Schema { defs }) + } + + pub fn find(&self, name: &str) -> Option { + self.defs.iter().position(|d| d.name == name) + } + + /// Decode a message of definition `root`, within [`DEFAULT_MAX_VALUES`] values. + pub fn decode_message<'s>(&'s self, root: usize, data: &[u8]) -> Result, KiwiError> { + self.decode_message_within(root, data, DEFAULT_MAX_VALUES) + } + + /// [`Self::decode_message`] within `budget` values. + pub fn decode_message_within<'s>(&'s self, root: usize, data: &[u8], budget: usize) -> Result, KiwiError> { + let mut d = Decoder { schema: self, r: Reader { data, pos: 0 }, budget, ids: vec![None; self.defs.len()] }; + d.def(root, 0) + } +} + +struct Decoder<'s, 'b> { + schema: &'s Schema, + r: Reader<'b>, + budget: usize, + /// Per definition, the index of the first field with each id (message field ids, enum + /// values), built the first time the definition is decoded: a message can carry many tags + /// against a wide definition, and a scan per tag would multiply the two. + ids: Vec>>, +} + +impl<'s> Decoder<'s, '_> { + fn spend(&mut self, n: usize) -> Result<(), KiwiError> { + self.budget = self.budget.checked_sub(n).ok_or_else(|| KiwiError("message: too many values".into()))?; + Ok(()) + } + + /// The first field of definition `i` with id (or enum value) `id`. + fn field_by_id(&mut self, i: usize, id: u32) -> Option<&'s Field> { + let fields = &self.schema.defs.get(i)?.fields; + let ids = self.ids.get_mut(i)?.get_or_insert_with(|| { + let mut m = HashMap::with_capacity(fields.len()); + for (j, f) in fields.iter().enumerate() { + m.entry(f.value).or_insert(j); + } + m + }); + fields.get(*ids.get(&id)?) + } + + fn def(&mut self, i: usize, depth: usize) -> Result, KiwiError> { + if depth > MAX_DEPTH { + return err("message: nested too deeply"); + } + self.spend(1)?; + let schema = self.schema; + let d = schema.defs.get(i).ok_or_else(|| KiwiError("message: missing definition".into()))?; + match d.kind { + Kind::Enum => { + let v = self.r.var_uint()?; + Ok(self.field_by_id(i, v).map_or(Kv::Uint(u64::from(v)), |f| Kv::Enum(f.name.as_str()))) + } + Kind::Struct => { + let mut out = Vec::with_capacity(d.fields.len()); + for f in &d.fields { + out.push((f.name.as_str(), self.field(f, depth)?)); + } + Ok(Kv::Object(out)) + } + Kind::Message => { + let mut out = Vec::new(); + loop { + let id = self.r.var_uint()?; + if id == 0 { + // Grown by doubling: trimmed, a large file's tree takes a good part less. + out.shrink_to_fit(); + return Ok(Kv::Object(out)); + } + let f = self.field_by_id(i, id).ok_or_else(|| KiwiError(format!("message: {} has no field {id}", d.name)))?; + out.push((f.name.as_str(), self.field(f, depth)?)); + } + } + } + } + + fn field(&mut self, f: &Field, depth: usize) -> Result, KiwiError> { + let ty = f.ty.ok_or_else(|| KiwiError("message: untyped field".into()))?; + if !f.array { + // A nested definition spends its own value in `def`. + if !matches!(ty, Type::Def(_)) { + self.spend(1)?; + } + return self.value(ty, depth); + } + let n = self.r.var_uint()? as usize; + // Every element takes at least one byte. + if n > self.r.left() { + return err("message: array longer than the data"); + } + // The field itself counts, so empty arrays can't pile up for free. + self.spend(1)?; + if ty == Type::Byte { + return Ok(Kv::Bytes(self.r.take(n)?.to_vec())); + } + self.spend(n)?; + let mut out = Vec::with_capacity(n); + for _ in 0..n { + out.push(self.value(ty, depth)?); + } + Ok(Kv::Array(out)) + } + + fn value(&mut self, ty: Type, depth: usize) -> Result, KiwiError> { + Ok(match ty { + Type::Bool => Kv::Bool(self.r.byte()? != 0), + Type::Byte => Kv::Uint(u64::from(self.r.byte()?)), + Type::Int => Kv::Int(i64::from(self.r.var_int()?)), + Type::Uint => Kv::Uint(u64::from(self.r.var_uint()?)), + Type::Float => Kv::Float(self.r.var_float()?), + Type::Str => Kv::Str(self.r.string()?), + Type::Int64 => Kv::Int(self.r.var_int64()?), + Type::Uint64 => Kv::Uint(self.r.var_uint64()?), + Type::Def(i) => self.def(i, depth + 1)?, + }) + } +} + +/// An encoder for the same format. Values are checked against the schema; it never panics. +pub mod encode { + use super::{BUILTINS, Field, Kind, KiwiError, Kv, MAX_DEPTH, Schema, Type, err}; + + #[derive(Default)] + pub struct Writer(pub Vec); + + impl Writer { + pub fn byte(&mut self, b: u8) { + self.0.push(b); + } + + pub fn var_uint(&mut self, mut v: u32) { + loop { + let b = (v & 127) as u8; + v >>= 7; + if v == 0 { + self.0.push(b); + return; + } + self.0.push(b | 128); + } + } + + pub fn var_int(&mut self, v: i32) { + self.var_uint(((v << 1) ^ (v >> 31)) as u32); + } + + pub fn var_uint64(&mut self, mut v: u64) { + for _ in 0..8 { + let b = (v & 127) as u8; + v >>= 7; + if v == 0 { + self.0.push(b); + return; + } + self.0.push(b | 128); + } + self.0.push(v as u8); + } + + pub fn var_int64(&mut self, v: i64) { + self.var_uint64(((v << 1) ^ (v >> 63)) as u64); + } + + pub fn var_float(&mut self, f: f32) { + let bits = f.to_bits().rotate_right(23); + if bits & 255 == 0 { + self.0.push(0); + } else { + self.0.extend_from_slice(&bits.to_le_bytes()); + } + } + + /// A NUL-terminated string (NULs inside are dropped: they would end it early). + pub fn string(&mut self, s: &str) { + self.0.extend(s.bytes().filter(|b| *b != 0)); + self.0.push(0); + } + } + + pub fn schema(s: &Schema) -> Vec { + let mut w = Writer::default(); + w.var_uint(s.defs.len() as u32); + for d in &s.defs { + w.string(&d.name); + w.byte(match d.kind { + Kind::Enum => 0, + Kind::Struct => 1, + Kind::Message => 2, + }); + w.var_uint(d.fields.len() as u32); + for f in &d.fields { + w.string(&f.name); + w.var_int(match f.ty { + None => 0, + Some(Type::Def(i)) => i32::try_from(i).unwrap_or(i32::MAX), + Some(t) => !(BUILTINS.iter().position(|b| *b == t).unwrap_or(0) as i32), + }); + w.byte(u8::from(f.array)); + w.var_uint(f.value); + } + } + w.0 + } + + /// Encode `v` as definition `def` (objects may leave message fields out; struct fields are + /// required, in order). Fails on what doesn't fit the schema: unknown fields or enum names, + /// missing struct fields, wrong types, numbers out of range, non-finite floats. + pub fn message(s: &Schema, def: usize, v: &Kv) -> Result, KiwiError> { + let mut w = Writer::default(); + encode_def(s, def, v, &mut w, 0)?; + Ok(w.0) + } + + fn encode_def(s: &Schema, i: usize, v: &Kv, w: &mut Writer, depth: usize) -> Result<(), KiwiError> { + if depth > MAX_DEPTH { + return err("encode: nested too deeply"); + } + let d = s.defs.get(i).ok_or_else(|| KiwiError("encode: missing definition".into()))?; + match d.kind { + Kind::Enum => { + let value = match v { + Kv::Uint(u) => u32::try_from(*u).ok(), + _ => v.str().and_then(|name| d.fields.iter().find(|f| f.name == name)).map(|f| f.value), + }; + w.var_uint(value.ok_or_else(|| KiwiError(format!("encode: {v:?} is not a value of {}", d.name)))?); + } + Kind::Struct => { + for f in &d.fields { + let fv = v.get(&f.name).ok_or_else(|| KiwiError(format!("encode: {} needs field {}", d.name, f.name)))?; + encode_field(s, f, fv, w, depth)?; + } + } + Kind::Message => { + let Kv::Object(fields) = v else { return err(format!("encode: {} needs an object", d.name)) }; + for (name, fv) in fields { + let f = d.fields.iter().find(|f| f.name == *name).ok_or_else(|| KiwiError(format!("encode: {} has no field {name}", d.name)))?; + w.var_uint(f.value); + encode_field(s, f, fv, w, depth)?; + } + w.var_uint(0); + } + } + Ok(()) + } + + fn encode_field(s: &Schema, f: &Field, v: &Kv, w: &mut Writer, depth: usize) -> Result<(), KiwiError> { + let ty = f.ty.ok_or_else(|| KiwiError("encode: untyped field".into()))?; + if !f.array { + return encode_value(s, ty, v, w, depth); + } + let len = |n: usize| u32::try_from(n).map_err(|_| KiwiError(format!("encode: {} is too long", f.name))); + if let (Type::Byte, Kv::Bytes(b)) = (ty, v) { + w.var_uint(len(b.len())?); + w.0.extend_from_slice(b); + return Ok(()); + } + let Kv::Array(items) = v else { return err(format!("encode: {} needs a list", f.name)) }; + w.var_uint(len(items.len())?); + for it in items { + encode_value(s, ty, it, w, depth)?; + } + Ok(()) + } + + /// An integer value (whole finite floats count). + fn int(v: &Kv) -> Option { + match v { + Kv::Int(i) => Some(i128::from(*i)), + Kv::Uint(u) => Some(i128::from(*u)), + Kv::Bool(b) => Some(i128::from(*b)), + Kv::Float(f) if f.is_finite() && f.fract() == 0.0 => Some(*f as i128), + _ => None, + } + } + + fn encode_value(s: &Schema, ty: Type, v: &Kv, w: &mut Writer, depth: usize) -> Result<(), KiwiError> { + let bad = || KiwiError(format!("encode: {v:?} is not a {ty:?}")); + let ranged = |lo: i128, hi: i128| int(v).filter(|i| (lo..=hi).contains(i)).ok_or_else(bad); + match ty { + Type::Bool => w.byte(u8::from(v.bool().ok_or_else(bad)?)), + Type::Byte => w.byte(ranged(0, 255)? as u8), + Type::Int => w.var_int(ranged(i32::MIN.into(), i32::MAX.into())? as i32), + Type::Uint => w.var_uint(ranged(0, u32::MAX.into())? as u32), + // Infinity is a value of its own (an axis with no maximum size); NaN is not. + Type::Float => w.var_float(match v { + Kv::Float(f) if f.is_infinite() => *f, + _ => v.f64().map(|x| x as f32).filter(|x| x.is_finite()).ok_or_else(bad)?, + }), + Type::Str => w.string(v.str().ok_or_else(bad)?), + Type::Int64 => w.var_int64(ranged(i64::MIN.into(), i64::MAX.into())? as i64), + Type::Uint64 => w.var_uint64(ranged(0, u64::MAX.into())? as u64), + Type::Def(i) => encode_def(s, i, v, w, depth + 1)?, + } + Ok(()) + } +} + +/// The built-in type names, in [`BUILTINS`] order. +const BUILTIN_NAMES: [&str; 8] = ["bool", "byte", "int", "uint", "float", "string", "int64", "uint64"]; + +impl Schema { + /// A schema from a compact text form, one definition per line: `enum Name A B …` (values from + /// 0), `struct Name type field, type field[] …` or `message Name …` (field ids from 1), the + /// types being built-in names or other definitions'. `A=3` or `type field=7` gives a value or + /// a field its own number. The limits of [`Schema::decode`] apply, so what parses also reads + /// back; field names are distinct within a definition, and so are a message's field ids + /// (never 0, which ends a message). + pub fn parse(text: &str) -> Result { + let lines: Vec> = text.lines().map(|l| l.split_whitespace().collect::>()).filter(|l| !l.is_empty()).collect(); + if lines.len() > MAX_DEFS { + return err("schema text: too many definitions"); + } + // The first definition of a name is the one a type names. + let mut index = HashMap::with_capacity(lines.len()); + for (i, l) in lines.iter().enumerate() { + if let Some(name) = l.get(1) { + index.entry(*name).or_insert(i); + } + } + let mut defs = Vec::with_capacity(lines.len()); + let mut total_fields = 0usize; + for l in &lines { + let (Some(kind), Some(name)) = (l.first(), l.get(1)) else { return err("schema text: a definition needs a kind and a name") }; + let rest = l.get(2..).unwrap_or_default(); + let (mut names, mut ids) = (HashSet::new(), HashSet::new()); + let (kind, fields) = match *kind { + "enum" => { + let mut fields = Vec::new(); + for (n, i) in rest.iter().zip(0..) { + let (field, value) = numbered(n, i)?; + if !names.insert(field.clone()) { + return err(format!("schema text: duplicate value {field} in {name}")); + } + fields.push(Field { name: field, ty: None, array: false, value }); + } + (Kind::Enum, fields) + } + "struct" | "message" => { + let mut fields = Vec::new(); + let list = rest.join(" "); + // A definition may have no fields. + for (f, id) in list.split(',').filter(|_| !rest.is_empty()).zip(1..) { + let Some((t, n)) = f.trim().split_once(' ') else { return err(format!("schema text: bad field `{f}` in {name}")) }; + let base = t.trim_end_matches("[]"); + let ty = match BUILTIN_NAMES.iter().position(|b| *b == base) { + Some(i) => BUILTINS.get(i).copied(), + None => index.get(base).copied().map(Type::Def), + } + .ok_or_else(|| KiwiError(format!("schema text: unknown type {base}")))?; + let (field, value) = numbered(n.trim(), id)?; + if !names.insert(field.clone()) { + return err(format!("schema text: duplicate field {field} in {name}")); + } + if *kind == "message" { + // Zero ends a message on the wire, so no field can have it. + if value == 0 { + return err(format!("schema text: field id 0 in {name}")); + } + if !ids.insert(value) { + return err(format!("schema text: duplicate field id {value} in {name}")); + } + } + fields.push(Field { name: field, ty: Some(ty), array: t.ends_with("[]"), value }); + } + (if *kind == "struct" { Kind::Struct } else { Kind::Message }, fields) + } + k => return err(format!("schema text: unknown kind {k}")), + }; + total_fields = total_fields.saturating_add(fields.len()); + if fields.len() > MAX_FIELDS || total_fields > MAX_SCHEMA_FIELDS { + return err("schema text: too many fields"); + } + defs.push(Def { name: name.to_string(), kind, fields }); + } + Ok(Schema { defs }) + } +} + +/// `name=number` as its parts, or `name` with `default`. +fn numbered(token: &str, default: u32) -> Result<(String, u32), KiwiError> { + match token.split_once('=') { + None => Ok((token.to_string(), default)), + Some((name, n)) => Ok((name.to_string(), n.parse().map_err(|_| KiwiError(format!("schema text: bad number in `{token}`")))?)), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/kiwi/src/tests.rs b/crates/kiwi/src/tests.rs new file mode 100644 index 00000000..dafc2add --- /dev/null +++ b/crates/kiwi/src/tests.rs @@ -0,0 +1,363 @@ +//! Tests on schemas written in the compact text form and messages built in code. +#![allow(clippy::unwrap_used, clippy::panic)] + +use super::{DEFAULT_MAX_VALUES, Def, Field, Kind, Kv, Reader, Schema, Type, encode}; + +const SCHEMA: &str = " +enum Kind NONE BOX TEXT +struct Id uint session, uint local +struct Point float x, float y +message Blob byte[] bytes +message Node Id id, Kind kind, string name, bool visible, Point size, Node[] children, float[] dashes, int64 stamp +message Message Node[] nodes, Blob[] blobs +"; + +fn schema() -> Schema { + Schema::parse(SCHEMA).unwrap() +} + +fn o(fields: Vec<(&'static str, Kv<'static>)>) -> Kv<'static> { + Kv::Object(fields) +} + +fn node(local: u64, name: &str, children: Vec>) -> Kv<'static> { + o(vec![ + ("id", o(vec![("session", Kv::Uint(1)), ("local", Kv::Uint(local))])), + ("kind", Kv::Enum("BOX")), + ("name", Kv::Str(name.into())), + ("visible", Kv::Bool(true)), + ("size", o(vec![("x", Kv::Float(10.5)), ("y", Kv::Float(-2.0))])), + ("children", Kv::Array(children)), + ("dashes", Kv::Array(vec![Kv::Float(1.0), Kv::Float(0.0)])), + ("stamp", Kv::Int(-5_000_000_000)), + ]) +} + +fn message() -> Kv<'static> { + o(vec![ + ("nodes", Kv::Array(vec![node(1, "a", vec![node(2, "héllo", vec![])]), node(3, "", vec![])])), + ("blobs", Kv::Array(vec![o(vec![("bytes", Kv::Bytes(vec![0, 1, 2, 255]))])])), + ]) +} + +struct Lcg(u64); +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + self.0 >> 33 + } +} + +#[test] +fn overflowing_u32_varint_is_rejected() { + let mut boundary = Reader { data: &[0xff, 0xff, 0xff, 0xff, 0x0f], pos: 0 }; + assert_eq!(boundary.var_uint(), Ok(u32::MAX)); + + let mut overflowing = Reader { data: &[0xff, 0xff, 0xff, 0xff, 0x7f], pos: 0 }; + assert!(overflowing.var_uint().is_err(), "overflowing u32 varint was accepted"); +} + +#[test] +fn the_wire_form_is_the_published_one() { + // message M { uint a = 1; string s = 2; float f = 3; int i = 4; }: field id, value, …, 0. + let sc = Schema::parse("message M uint a, string s, float f, int i").unwrap(); + let v = o(vec![("a", Kv::Uint(300)), ("s", Kv::Str("a".into())), ("f", Kv::Float(0.0)), ("i", Kv::Int(-1))]); + // 300 as a varint is AC 02; a string ends in NUL; a zero float is one zero byte; -1 zigzags to 1. + let bytes = encode::message(&sc, 0, &v).unwrap(); + assert_eq!(bytes, [1, 0xAC, 0x02, 2, b'a', 0, 3, 0, 4, 1, 0]); + assert_eq!(sc.decode_message(0, &bytes).unwrap(), v); + // 1.0f32 (0x3F800000): the exponent byte comes first. + let one = encode::message(&sc, 0, &o(vec![("f", Kv::Float(1.0))])).unwrap(); + assert_eq!(one, [3, 0x7F, 0, 0, 0, 0]); +} + +#[test] +fn duplicate_explicit_field_ids_are_ambiguous() { + assert!(Schema::parse("message M uint a=1, uint b=1").is_err()); +} + +#[test] +fn schema_and_message_round_trip() { + let sc = schema(); + let back = Schema::decode(&encode::schema(&sc)).unwrap(); + assert_eq!(back.defs.len(), sc.defs.len()); + for (a, b) in sc.defs.iter().zip(&back.defs) { + assert_eq!((a.name.as_str(), a.kind), (b.name.as_str(), b.kind)); + let fields = |d: &Def| d.fields.iter().map(|f| (f.name.clone(), f.ty, f.array, f.value)).collect::>(); + assert_eq!(fields(a), fields(b)); + } + let msg = message(); + let root = sc.find("Message").unwrap(); + assert_eq!(back.decode_message(root, &encode::message(&sc, root, &msg).unwrap()).unwrap(), msg); +} + +#[test] +fn scalars_round_trip() { + let names = ["f", "i", "u", "i64", "u64", "s", "b"]; + let types = [Type::Float, Type::Int, Type::Uint, Type::Int64, Type::Uint64, Type::Str, Type::Bool]; + let fields = names + .iter() + .zip(types) + .enumerate() + .map(|(i, (n, t))| Field { name: n.to_string(), ty: Some(t), array: false, value: i as u32 + 1 }) + .collect(); + let sc = Schema { defs: vec![Def { name: "M".into(), kind: Kind::Message, fields }] }; + let cases = [ + (0.0f32, 0i32, 0u32, 0i64, 0u64), + (-1.5, -7, 300, i64::MIN, u64::MAX), + (3.25e7, i32::MAX, u32::MAX, i64::MAX, 1 << 60), + (1e-40, i32::MIN, 1, -1, 127), + ]; + for (fv, iv, uv, i64v, u64v) in cases { + let mut w = encode::Writer::default(); + w.var_uint(1); + w.var_float(fv); + w.var_uint(2); + w.var_int(iv); + w.var_uint(3); + w.var_uint(uv); + w.var_uint(4); + w.var_int64(i64v); + w.var_uint(5); + w.var_uint64(u64v); + w.var_uint(6); + w.string("héllo"); + w.var_uint(7); + w.byte(1); + w.var_uint(0); + let got = sc.decode_message(0, &w.0).unwrap(); + // Denormals are stored as zero. + let want_f = if fv.abs() < f32::MIN_POSITIVE { 0.0 } else { fv }; + assert_eq!(got.get("f"), Some(&Kv::Float(want_f))); + assert_eq!(got.get("i"), Some(&Kv::Int(i64::from(iv)))); + assert_eq!(got.get("u"), Some(&Kv::Uint(u64::from(uv)))); + assert_eq!(got.get("i64"), Some(&Kv::Int(i64v))); + assert_eq!(got.get("u64"), Some(&Kv::Uint(u64v))); + assert_eq!(got.get("s"), Some(&Kv::Str("héllo".into()))); + assert_eq!(got.get("b"), Some(&Kv::Bool(true))); + } +} + +#[test] +fn the_encoder_refuses_what_does_not_fit_the_schema() { + let sc = schema(); + let root = sc.find("Message").unwrap(); + let bad = [ + o(vec![("missing", Kv::Uint(1))]), + o(vec![("nodes", Kv::Uint(1))]), + o(vec![("nodes", Kv::Array(vec![o(vec![("kind", Kv::Enum("NOPE"))])]))]), + o(vec![("nodes", Kv::Array(vec![o(vec![("id", o(vec![("session", Kv::Uint(1))]))])]))]), + o(vec![("nodes", Kv::Array(vec![o(vec![("size", o(vec![("x", Kv::Float(f32::NAN)), ("y", Kv::Float(0.0))]))])]))]), + o(vec![("nodes", Kv::Array(vec![o(vec![("id", o(vec![("session", Kv::Int(-1)), ("local", Kv::Uint(0))]))])]))]), + ]; + for v in bad { + assert!(encode::message(&sc, root, &v).is_err(), "{v:?}"); + } + // Infinity is a value; a message nested past the depth cap is not written. + let inf = o(vec![("nodes", Kv::Array(vec![o(vec![("dashes", Kv::Array(vec![Kv::Float(f32::INFINITY)]))])]))]); + assert!(encode::message(&sc, root, &inf).is_ok()); + let mut deep = o(vec![]); + for _ in 0..super::MAX_DEPTH + 2 { + deep = o(vec![("children", Kv::Array(vec![deep]))]); + } + assert!(encode::message(&sc, root, &o(vec![("nodes", Kv::Array(vec![deep]))])).is_err()); +} + +#[test] +fn mangled_input_never_panics() { + let sc = schema(); + let root = sc.find("Message").unwrap(); + let schema_bytes = encode::schema(&sc); + let msg = encode::message(&sc, root, &message()).unwrap(); + let mut r = Lcg(11); + for cut in 0..schema_bytes.len() { + let _ = Schema::decode(&schema_bytes[..cut]); + } + for cut in 0..msg.len() { + let _ = sc.decode_message(root, &msg[..cut]); + } + for _ in 0..2000 { + let mut m = msg.clone(); + for _ in 0..1 + r.next() % 4 { + let i = (r.next() as usize) % m.len(); + m[i] = r.next() as u8; + } + let _ = sc.decode_message(root, &m); + let mut sbytes = schema_bytes.clone(); + let i = (r.next() as usize) % sbytes.len(); + sbytes[i] = r.next() as u8; + if let Ok(s2) = Schema::decode(&sbytes) + && let Some(rt) = s2.find("Message") + { + let _ = s2.decode_message(rt, &msg); + } + } + for _ in 0..500 { + let junk: Vec = (0..(r.next() % 200)).map(|_| r.next() as u8).collect(); + let _ = Schema::decode(&junk); + let _ = sc.decode_message(root, &junk); + let _ = Schema::parse(&String::from_utf8_lossy(&junk)); + } +} + +#[test] +fn scalar_fields_count_against_the_value_budget() { + // A message of 1000 scalar fields (id 1, `true`) is 1001 values: a budget of 10 refuses it. + let fields = vec![Field { name: "b".into(), ty: Some(Type::Bool), array: false, value: 1 }]; + let sc = Schema { defs: vec![Def { name: "Message".into(), kind: Kind::Message, fields }] }; + let mut data = [1u8, 1].repeat(1000); + data.push(0); + let e = sc.decode_message_within(0, &data, 10).unwrap_err(); + assert_eq!(e.0, "message: too many values"); + assert!(sc.decode_message_within(0, &data, 1001).is_ok()); + assert!(sc.decode_message_within(0, &data, 1000).is_err()); + assert!(DEFAULT_MAX_VALUES >= 1001 && sc.decode_message(0, &data).is_ok()); +} + +#[test] +fn empty_arrays_count_against_the_value_budget() { + // 1000 empty `uint[]` fields: each field is a value even with no elements, so a small budget + // can't be outgrown by piling up empty containers. + let fields = vec![Field { name: "a".into(), ty: Some(Type::Uint), array: true, value: 1 }]; + let sc = Schema { defs: vec![Def { name: "Message".into(), kind: Kind::Message, fields }] }; + let mut data = [1u8, 0].repeat(1000); + data.push(0); + let e = sc.decode_message_within(0, &data, 10).unwrap_err(); + assert_eq!(e.0, "message: too many values"); + assert!(sc.decode_message_within(0, &data, 1001).is_ok()); +} + +#[test] +fn message_tags_find_their_field_by_id_in_a_wide_definition() { + // Every tag is looked up by id, and the first field with an id wins. + let mut fields: Vec = (1..=5000).map(|i| Field { name: format!("f{i}"), ty: Some(Type::Bool), array: false, value: i }).collect(); + fields.push(Field { name: "shadow".into(), ty: Some(Type::Bool), array: false, value: 7 }); + let sc = Schema { defs: vec![Def { name: "Message".into(), kind: Kind::Message, fields }] }; + let mut w = encode::Writer::default(); + for _ in 0..20_000 { + w.var_uint(5000); + w.byte(1); + } + w.var_uint(7); + w.byte(0); + w.var_uint(0); + let Kv::Object(out) = sc.decode_message(0, &w.0).unwrap() else { panic!() }; + assert_eq!(out.len(), 20_001); + assert!(out.iter().take(20_000).all(|(n, v)| *n == "f5000" && *v == Kv::Bool(true))); + assert_eq!(out.last().map(|(n, _)| *n), Some("f7")); +} + +#[test] +fn a_schema_is_refused_past_its_limits() { + // 25 definitions of 10 000 fields each: every one is within the per-definition cap, but + // together they pass the schema's total. + let mut w = encode::Writer::default(); + w.var_uint(25); + for d in 0..25 { + w.string(&format!("D{d}")); + w.byte(0); + w.var_uint(10_000); + for i in 0..10_000u32 { + w.string(""); + w.var_int(0); + w.byte(0); + w.var_uint(i); + } + } + assert_eq!(Schema::decode(&w.0).unwrap_err().0, "schema: too many fields"); + // More definitions, or more fields in one, than the caps (and than the data could hold). + let mut w = encode::Writer::default(); + w.var_uint(super::MAX_DEFS as u32 + 1); + w.0.resize(20_000, 0); + assert_eq!(Schema::decode(&w.0).unwrap_err().0, "schema: too many definitions"); + let mut w = encode::Writer::default(); + w.var_uint(1); + w.string("D"); + w.byte(0); + w.var_uint(super::MAX_FIELDS as u32 + 1); + w.0.resize(20_000, 0); + assert_eq!(Schema::decode(&w.0).unwrap_err().0, "schema: too many fields"); +} + +#[test] +fn hostile_input_is_refused() { + // A message that contains itself, nested past the depth cap. + let field = |ty, array| vec![Field { name: "m".into(), ty: Some(ty), array, value: 1 }]; + let nested = Schema { defs: vec![Def { name: "Message".into(), kind: Kind::Message, fields: field(Type::Def(0), false) }] }; + assert_eq!(nested.decode_message(0, &[1u8; 10_000]).unwrap_err().0, "message: nested too deeply"); + // An array count larger than the data. + let array = Schema { defs: vec![Def { name: "Message".into(), kind: Kind::Message, fields: field(Type::Uint, true) }] }; + assert!(array.decode_message(0, &[1, 0xff, 0xff, 0xff, 0xff, 0x0f]).is_err()); + // A field type naming a missing definition. + let mut w = encode::Writer::default(); + w.var_uint(1); + w.string("Message"); + w.byte(2); + w.var_uint(1); + w.string("x"); + w.var_int(5); + w.byte(0); + w.var_uint(1); + assert!(Schema::decode(&w.0).is_err()); + // A message field id the definition doesn't have. + assert!(array.decode_message(0, &[9, 0]).is_err()); +} + +#[test] +fn a_wide_schema_decodes_without_a_scan_per_field() { + // 20 messages of 10 000 distinct field ids each: within every cap. A scan of the earlier + // fields for each new one made this a billion comparisons. + let mut w = encode::Writer::default(); + w.var_uint(20); + for d in 0..20 { + w.string(&format!("D{d}")); + w.byte(2); + w.var_uint(10_000); + for i in 1..=10_000u32 { + w.string("f"); + w.var_int(-1); + w.byte(0); + w.var_uint(i); + } + } + let started = std::time::Instant::now(); + let sc = Schema::decode(&w.0).unwrap(); + assert_eq!(sc.defs.iter().map(|d| d.fields.len()).sum::(), 200_000); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + // A repeated id is still refused. + let mut w = encode::Writer::default(); + w.var_uint(1); + w.string("M"); + w.byte(2); + w.var_uint(2); + for _ in 0..2 { + w.string("f"); + w.var_int(-1); + w.byte(0); + w.var_uint(3); + } + assert_eq!(Schema::decode(&w.0).unwrap_err().0, "schema: duplicate field id 3 in M"); +} + +#[test] +fn the_text_form_keeps_the_limits_and_refuses_ambiguous_definitions() { + // Definitions without fields are definitions. + let sc = Schema::parse("message Empty\nstruct Nothing\nenum None\nmessage M Empty e, Nothing n").unwrap(); + assert_eq!(sc.defs.iter().map(|d| d.fields.len()).collect::>(), [0, 0, 0, 2]); + assert_eq!(Schema::decode(&encode::schema(&sc)).unwrap().defs.len(), 4); + // Two fields of one name, two values of one name, a message field with id 0 or a repeated id. + for bad in ["message M uint x, uint x", "struct S uint x, float x", "enum E A A", "message M uint x=0", "message M uint a=2, uint b=2"] { + assert!(Schema::parse(bad).is_err(), "{bad}"); + } + // A struct's fields have no ids on the wire: their numbers are free. + assert!(Schema::parse("struct S uint a=0, uint b=0").is_ok()); + // The caps of the binary form. + let wide = |n: usize| format!("message M {}", (0..n).map(|i| format!("bool f{i}")).collect::>().join(", ")); + assert!(Schema::parse(&wide(super::MAX_FIELDS)).is_ok()); + assert_eq!(Schema::parse(&wide(super::MAX_FIELDS + 1)).unwrap_err().0, "schema text: too many fields"); + let many = (0..=super::MAX_DEFS).map(|i| format!("enum E{i}")).collect::>().join("\n"); + assert_eq!(Schema::parse(&many).unwrap_err().0, "schema text: too many definitions"); + let line = (0..super::MAX_FIELDS).map(|i| format!("bool f{i}")).collect::>().join(", "); + let total = (0..=super::MAX_SCHEMA_FIELDS / super::MAX_FIELDS).map(|i| format!("struct S{i} {line}")).collect::>().join("\n"); + assert_eq!(Schema::parse(&total).unwrap_err().0, "schema text: too many fields"); +}