diff --git a/Cargo.lock b/Cargo.lock index 8d0c9f2c9..4aeb50e79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2154,6 +2154,14 @@ dependencies = [ "serde_json", ] +[[package]] +name = "craft-mcp-server" +version = "0.1.0" +dependencies = [ + "proptest", + "serde_json", +] + [[package]] name = "craft-media" version = "0.1.0" @@ -2737,6 +2745,10 @@ dependencies = [ "zeroize", ] +[[package]] +name = "craft-sniff" +version = "0.1.0" + [[package]] name = "craft-stencilcraft-document" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0176a86e0..5d1e085df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ craft-kra = { path = "crates/kra" } craft-audio = { path = "crates/audio" } craft-latex = { path = "crates/latex" } craft-markdown = { path = "crates/markdown" } +craft-macho = { path = "crates/macho" } craft-math = { path = "crates/math" } # No default features: craft-audio turns on the craft-media modules each of its features needs. craft-media = { path = "crates/media", default-features = false } @@ -174,7 +175,7 @@ craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } -craft-macho = { path = "crates/macho" } +craft-sniff = { path = "crates/sniff" } craft-epub = { path = "crates/epub" } craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } diff --git a/README.md b/README.md index 48b989df3..7cf947180 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit AC-3/E-AC-3, MP1/MP2, MP3 encoding), streaming and seeking, and packet decoding for any demuxer. See its README for formats and features. - [`craft-latex`](crates/latex/README.md): LaTeX documents read (the everyday subset) and written (a standalone article), to and from the craft-flow model. +- [`craft-macho`](crates/macho/src/lib.rs): Mach-O header pieces from untrusted bytes, from Craft Anything: universal (fat and fat64) slice tables checked against the file, CPU and load command names, the fixed size each load command needs before its fields are read, and the thread-state program counter. - [`craft-markdown`](crates/markdown/README.md): Markdown read and written (a CommonMark subset with GitHub tables and strikethrough), to and from the craft-flow model. - [`craft-math`](crates/math/README.md): equations as one tree (the Office Math object model) with its syntaxes, read and written: OMML, presentation MathML, LaTeX math and the UnicodeMath linear format. - [`craft-media`](crates/media/README.md): bit-level I/O (`bitstream`), clean-room MP4/MOV @@ -386,7 +387,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit text-format crates read into and write from. WordCraft's model without its editing code. - [`craft-xlsx`](crates/xlsx/README.md): Excel workbooks (`.xlsx`, `.xlsm`, SpreadsheetML, transitional and strict) and binary workbooks (`.xlsb`) on `craft-opc`: the workbook part, shared strings, the styles tables and worksheet rows streamed one at a time, with limits on the package, the XML and the binary records. - [`craft-ora`](crates/ora/README.md): OpenRaster documents (`.ora`): the ZIP archive and the `stack.xml` layer tree as written, read with bounded sizes and written. From PhotoCraft. -- [`craft-macho`](crates/macho/src/lib.rs): Mach-O header pieces from untrusted bytes, from Craft Anything: universal (fat and fat64) slice tables checked against the file, CPU and load command names, the fixed size each load command needs before its fields are read, and the thread-state program counter. +- [`craft-sniff`](crates/sniff/src/lib.rs): what a file is from its first bytes: one dependency-free, wasm-safe signature table naming the format, its media type, extension and family for images, audio, video, 3D models, archives, compressed streams, executables and databases. Identifies only; parsing stays with each format's reader. - [`craft-y4m`](crates/y4m/src/lib.rs): YUV4MPEG2 (`.y4m`) reader, from FilmCraft: the stream header, colour space layouts at 8 to 16 bits, a frame index built from the `FRAME` lines without loading the file, and plane access. Bounded on hostile input. diff --git a/crates/docx/src/write/story/drawing.rs b/crates/docx/src/write/story/drawing.rs index 7c03cca6e..94154f649 100644 --- a/crates/docx/src/write/story/drawing.rs +++ b/crates/docx/src/write/story/drawing.rs @@ -69,9 +69,18 @@ impl Writer<'_> { } /// A shape's or text box's `wps:wsp`, at `off` in its group (or 0, 0). In a group it carries - /// its own `wps:cNvPr` with drawing id `id`. - pub(super) fn wsp(&mut self, w: &mut W, o: &InlineObject, id: Option<&str>, off: (f32, f32), rels: &mut PartRels, depth: usize) { - let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, effects, float, freeform, extra } = o else { return }; + /// its own `wps:cNvPr` with drawing id `id`. Returns where in `w` its `w:txbxContent` was + /// written, for a VML fallback to repeat. + pub(super) fn wsp( + &mut self, + w: &mut W, + o: &InlineObject, + id: Option<&str>, + off: (f32, f32), + rels: &mut PartRels, + depth: usize, + ) -> Option> { + let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, effects, float, freeform, extra } = o else { return None }; let geom = freeform.as_deref().filter(|_| *kind == ShapeKind::Freeform); w.open("wps:wsp", &[]); if let Some(id) = id { @@ -181,6 +190,7 @@ impl Writer<'_> { // its place in it instead. let part = (*story).and_then(|id| self.doc.parts.get(&id).filter(|p| p.kind == craft_wordmodel::PartKind::TextBox).map(|p| (id, p))); let link = part.and_then(|(id, _)| self.txbx_links.get(&id).copied()); + let mut content = None; if let Some((n, seq)) = link.filter(|(_, seq)| *seq > 0) { w.empty("wps:linkedTxbx", &[("id", &n.to_string()), ("seq", &seq.to_string())]); } else if let Some((id, part)) = part @@ -190,6 +200,7 @@ impl Writer<'_> { Some((n, _)) => w.open("wps:txbx", &[("id", &n.to_string())]), None => w.open("wps:txbx", &[]), } + let start = w.s.len(); w.open("w:txbxContent", &[]); if self.written_boxes.insert(id) { let blocks = part.blocks.clone(); @@ -198,6 +209,7 @@ impl Writer<'_> { w.empty("w:p", &[]); } w.close("w:txbxContent"); + content = Some(start..w.s.len()); w.close("wps:txbx"); self.boxes.leave(); } @@ -231,6 +243,7 @@ impl Writer<'_> { w.close("wps:bodyPr"); } w.close("wps:wsp"); + content } /// A `w:drawing` holding `graphic` (an `a:graphic`) in a frame of the object's size and place. diff --git a/crates/docx/src/write/story/object.rs b/crates/docx/src/write/story/object.rs index 6cdd54f46..820b85ee7 100644 --- a/crates/docx/src/write/story/object.rs +++ b/crates/docx/src/write/story/object.rs @@ -1,7 +1,7 @@ use super::drawing::{DrawingInfo, shape_name, spin_attrs, write_fill, xfrm_spin_only}; use super::field::{FieldData, field_run}; use super::section::comment_reference_style; -use super::{CharProps, Fill, InlineObject, NoteKind, PartRels, TextColor, W, Wrap, Writer, emu, n, rpr}; +use super::{Anchor, CharProps, Fill, Float, InlineObject, NoteKind, PartRels, ShapeKind, TextColor, W, Wrap, Writer, emu, n, rpr}; impl Writer<'_> { /// Write an inline object (`Bookmark`, `Comment`, `Equation`, `Field`, `Shape`, etc.). @@ -366,6 +366,16 @@ impl Writer<'_> { self.rev_open(w, props); w.open("w:r", &[]); self.rpr(w, props); + // A Word 2010 shape (`wps`) with a VML form goes in `mc:AlternateContent`, as Word writes + // it: a reader that doesn't know `wps` (Word 2007, the Office Compatibility Pack) takes the + // VML fallback instead of refusing the file. A shape without one stays bare: guarded with + // no fallback, that reader would open the file without the shape and drop it on its next + // save, which is worse than refusing the file. + let fallback = freeform.is_none() && vml_tag(*kind).is_some(); + if fallback { + w.open("mc:AlternateContent", &[]); + w.open("mc:Choice", &[("Requires", "wps")]); + } w.open("w:drawing", &[]); let docpr = self.shape_docpr(extra.id); let link = Self::drawing_link(props, rels); @@ -384,18 +394,164 @@ impl Writer<'_> { } let float = &float; self.drawing_open(w, DrawingInfo { float, size: (*sw, *sh), docpr: &docpr, name: &name, alt: o.alt_text(), link: link.as_deref() }); + // This shape's stacking order: floating objects in its text box move `self.z` on. + let z = self.z; w.empty("wp:cNvGraphicFramePr", &[]); w.open("a:graphic", &[]); w.open("a:graphicData", &[("uri", "http://schemas.microsoft.com/office/word/2010/wordprocessingShape")]); - self.wsp(w, o, None, (0.0, 0.0), rels, depth); + let content = self.wsp(w, o, None, (0.0, 0.0), rels, depth); w.close("a:graphicData"); w.close("a:graphic"); w.close(if float.wrap == Wrap::Inline { "wp:inline" } else { "wp:anchor" }); w.close("w:drawing"); + if fallback { + w.close("mc:Choice"); + // The same text again: the fallback is a copy of the choice, not more of the story. + let content = content.and_then(|r| w.s.get(r)).map(str::to_owned); + w.open("mc:Fallback", &[]); + self.vml_shape(w, o, float, z, content.as_deref()); + w.close("mc:Fallback"); + w.close("mc:AlternateContent"); + } w.close("w:r"); self.rev_close(w, props); } + /// A shape as VML (`w:pict`), for readers older than Word 2010 shapes: its frame, place, + /// wrapping, fill, outline and text box. Only for a kind [`vml_tag`] names. + fn vml_shape(&self, w: &mut W, o: &InlineObject, float: &Float, z: u32, content: Option<&str>) { + let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, extra, .. } = o else { return }; + let Some(tag) = vml_tag(*kind) else { return }; + let pt = |v: f32| format!("{:.2}pt", craft_wordmodel::geom::finite(v)); + let mut style = Vec::new(); + if float.wrap != Wrap::Inline { + style.push("position:absolute".to_string()); + style.push(format!("margin-left:{}", pt(float.x))); + style.push(format!("margin-top:{}", pt(float.y))); + } + style.push(format!("width:{}", pt(sw.max(0.0)))); + style.push(format!("height:{}", pt(sh.max(0.0)))); + let spin = float.spin(); + if spin.deg != 0.0 { + style.push(format!("rotation:{}", spin.deg)); + } + match (spin.flip_h, spin.flip_v) { + (true, true) => style.push("flip:x y".into()), + (true, false) => style.push("flip:x".into()), + (false, true) => style.push("flip:y".into()), + (false, false) => {} + } + if float.wrap != Wrap::Inline { + // The stacking order the choice was given, as a signed 32-bit `z-index`. Behind the text + // is negative, shifted down rather than negated, so a higher `z` still stacks higher. + let z = i64::from(z.min(i32::MAX as u32)); + let z = if float.wrap == Wrap::BehindText { z - (i64::from(i32::MAX) + 1) } else { z }; + style.push(format!("z-index:{z}")); + // The same clearances from the text as the choice's `distL`/`distR`/`distT`/`distB`. + let d = pt(float.dist.clamp(0.0, 1584.0)); + style.push(format!("mso-wrap-distance-left:{d}")); + style.push(format!("mso-wrap-distance-right:{d}")); + style.push(format!("mso-wrap-distance-top:{}", pt(float.dist_top.clamp(0.0, 1584.0)))); + style.push(format!("mso-wrap-distance-bottom:{}", pt(float.dist_bottom.clamp(0.0, 1584.0)))); + if let Some(a) = float.h_align { + style.push(format!("mso-position-horizontal:{}", a.ooxml(true))); + } + style.push(format!("mso-position-horizontal-relative:{}", vml_rel(float.h_rel, true))); + if let Some(a) = float.v_align { + style.push(format!("mso-position-vertical:{}", a.ooxml(false))); + } + style.push(format!("mso-position-vertical-relative:{}", vml_rel(float.v_rel, false))); + } + let body = (*story).and_then(|id| self.doc.parts.get(&id)).map(|p| p.body).unwrap_or_default(); + style.push( + match body.anchor { + craft_wordmodel::props::VAlign::Top => "v-text-anchor:top", + craft_wordmodel::props::VAlign::Center => "v-text-anchor:middle", + craft_wordmodel::props::VAlign::Bottom => "v-text-anchor:bottom", + } + .into(), + ); + if float.hidden { + style.push("visibility:hidden".into()); + } + let style = style.join(";"); + let mut attrs: Vec<(&str, String)> = vec![("style", style)]; + if *kind == ShapeKind::RoundedRectangle { + // The same fraction of the shorter side: VML's in 1/65536ths, DrawingML's in 1/100000ths. + let c = match extra.corner { + Some(c) => c.min(craft_wordmodel::connector::MAX_CORNER) as f32 / 100_000.0, + None => craft_wordmodel::connector::DEFAULT_CORNER, + }; + attrs.push(("arcsize", format!("{}f", (c * 65_536.0).round() as u32))); + } + if float.wrap != Wrap::Inline { + attrs.push(("o:allowincell", (if float.outside_cell { "f" } else { "t" }).into())); + } + let opacity = match fill.as_ref() { + Some(f) => { + attrs.push(("fillcolor", format!("#{}", f.main_color().hex()))); + match f { + Fill::Solid { transparency, .. } if *transparency > 0.0 => Some(format!("{:.4}", (1.0 - transparency / 100.0).clamp(0.0, 1.0))), + _ => None, + } + } + None => { + attrs.push(("filled", "f".into())); + None + } + }; + match stroke { + Some(c) => { + attrs.push(("strokecolor", format!("#{}", c.hex()))); + attrs.push(("strokeweight", pt(stroke_width.clamp(0.0, 100.0)))); + } + None => attrs.push(("stroked", "f".into())), + } + let attrs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect(); + w.open("w:pict", &[]); + w.open(tag, &attrs); + if let Some(opacity) = &opacity { + w.empty("v:fill", &[("opacity", opacity)]); + } + if let Some(content) = content { + let inset = body.inset.map(|i| i.map(|v| pt(v.min(craft_wordmodel::MAX_TEXT_BOX_INSET) as f32 / 12_700.0)).join(",")); + let mut tb: Vec<(&str, &str)> = Vec::new(); + if let Some(inset) = &inset { + tb.push(("inset", inset)); + } + // Text direction as the VML reader takes it (stacked letters have no VML form). + let mut tb_style = match body.vert { + craft_wordmodel::props::TextVert::Vert => vec!["layout-flow:vertical"], + craft_wordmodel::props::TextVert::Vert270 => vec!["layout-flow:vertical", "mso-layout-flow-alt:bottom-to-top"], + craft_wordmodel::props::TextVert::Horz | craft_wordmodel::props::TextVert::Stacked => Vec::new(), + }; + if body.auto_fit { + tb_style.push("mso-fit-shape-to-text:t"); + } + let tb_style = tb_style.join(";"); + if !tb_style.is_empty() { + tb.push(("style", &tb_style)); + } + w.open("v:textbox", &tb); + w.raw(content); + w.close("v:textbox"); + } + let wrap = match float.wrap { + Wrap::Square => Some("square"), + Wrap::Tight => Some("tight"), + Wrap::Through => Some("through"), + Wrap::TopAndBottom => Some("topAndBottom"), + Wrap::Inline | Wrap::BehindText | Wrap::InFrontOfText => None, + }; + if let Some(wrap) = wrap { + let side = float.wrap_text.ooxml(); + let attrs: &[(&str, &str)] = if float.wrap_text.is_both() { &[("type", wrap)] } else { &[("type", wrap), ("side", side)] }; + w.empty("w10:wrap", attrs); + } + w.close(tag); + w.close("w:pict"); + } + fn write_group(&mut self, w: &mut W, o: &InlineObject, props: &CharProps, rels: &mut PartRels, depth: usize) { let InlineObject::Group { w: gw, h: gh, float, ch_w, ch_h, children, canvas } = o else { return }; self.rev_open(w, props); @@ -487,6 +643,33 @@ impl Writer<'_> { } } +/// The VML element that draws a shape kind, for the kinds VML has one for. +fn vml_tag(kind: ShapeKind) -> Option<&'static str> { + match kind { + ShapeKind::Rectangle | ShapeKind::TextBox => Some("v:rect"), + ShapeKind::RoundedRectangle => Some("v:roundrect"), + ShapeKind::Ellipse => Some("v:oval"), + _ => None, + } +} + +/// A floating object's anchor as VML names it (`mso-position-horizontal-relative`, `-vertical-`). +fn vml_rel(a: Anchor, horiz: bool) -> &'static str { + match (a, horiz) { + (Anchor::Margin, _) => "margin", + (Anchor::Page, _) => "page", + (Anchor::InsideMargin, _) => "inner-margin-area", + (Anchor::OutsideMargin, _) => "outer-margin-area", + (Anchor::LeftMargin, true) => "left-margin-area", + (Anchor::RightMargin, true) => "right-margin-area", + (Anchor::Character, true) => "char", + (Anchor::TopMargin, false) => "top-margin-area", + (Anchor::BottomMargin, false) => "bottom-margin-area", + (Anchor::Line, false) => "line", + _ => "text", + } +} + pub(super) fn ink_name(tool: craft_wordmodel::freeform::InkTool) -> &'static str { match tool { craft_wordmodel::freeform::InkTool::Pen => "Pen", diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index b0bab5833..d1def3f8c 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -49,3 +49,4 @@ mod undefined_style; mod vml_shapes; mod word_xml; mod wordml2003_write; +mod wps_fallback; diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 8f3bbdd0b..65a01e6b4 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -956,7 +956,8 @@ fn toc_field_skips_nested_stories() { let mut xml = String::new(); std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap(); let boxes: Vec<(usize, usize)> = xml.match_indices("").map(|(a, _)| (a, a + xml[a..].find("").unwrap())).collect(); - assert_eq!(boxes.len(), 2, "{xml}"); + // Each box twice: its shape and its VML fallback. + assert_eq!(boxes.len(), 4, "{xml}"); let in_box = |i: usize| boxes.iter().any(|(a, b)| (*a..*b).contains(&i)); for kind in ["begin", "separate", "end"] { let at: Vec = xml.match_indices(&format!(r#"fldCharType="{kind}""#)).map(|(i, _)| i).collect(); @@ -964,7 +965,7 @@ fn toc_field_skips_nested_stories() { assert!(!in_box(at[0]), "field {kind} written inside a text box: {xml}"); } let (begin, end) = (xml.find(r#"fldCharType="begin""#).unwrap(), xml.find(r#"fldCharType="end""#).unwrap()); - assert!(begin > boxes[0].1 && end > boxes[1].1 && end < xml.find("").unwrap(), "{xml}"); + assert!(begin > boxes[1].1 && end > boxes[3].1 && end < xml.find("").unwrap(), "{xml}"); } #[test] diff --git a/crates/docx/tests/it/text_boxes.rs b/crates/docx/tests/it/text_boxes.rs index 608d22d33..a5824cafc 100644 --- a/crates/docx/tests/it/text_boxes.rs +++ b/crates/docx/tests/it/text_boxes.rs @@ -257,5 +257,6 @@ fn story_shared_by_many_shapes_is_written_once() { } let xml = document_xml(&craft_docx::write(&d).unwrap()); assert!(xml.len() < 2_000_000, "document.xml is {} bytes", xml.len()); - assert_eq!(xml.matches(text.as_str()).count(), 1); + // Once in the first shape and once in its VML fallback; the other shapes don't repeat it. + assert_eq!(xml.matches(text.as_str()).count(), 2); } diff --git a/crates/docx/tests/it/wps_fallback.rs b/crates/docx/tests/it/wps_fallback.rs new file mode 100644 index 000000000..e2d960cf9 --- /dev/null +++ b/crates/docx/tests/it/wps_fallback.rs @@ -0,0 +1,158 @@ +//! Word 2010 shapes (`wps:wsp`) are written inside `mc:AlternateContent` with a VML fallback +//! (ECMA-376 Part 3, Markup Compatibility; Part 4 §14.1, VML), so a reader that doesn't know +//! `wps` — Word 2007, or Word 2000-2003 with the Office Compatibility Pack — takes the fallback +//! instead of refusing the whole file. A shape with no VML form is left bare. + +use std::io::Read; + +use craft_wordmodel::para::{Anchor, FloatAlign, ShapeKind, Wrap}; +use craft_wordmodel::props::TextVert; +use craft_wordmodel::{Document, InlineObject, PartKind, TextBody, para_block}; + +fn document_xml(bytes: &[u8]) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut s = String::new(); + z.by_name("word/document.xml").unwrap().read_to_string(&mut s).unwrap(); + s +} + +fn shape(kind: ShapeKind, story: Option) -> InlineObject { + let float = craft_wordmodel::para::Float { + wrap: Wrap::Square, + h_rel: Anchor::Margin, + h_align: Some(FloatAlign::End), + v_rel: Anchor::Paragraph, + y: 0.05, + ..Default::default() + }; + InlineObject::Shape { + kind, + w: 10.7, + h: 12.2, + fill: None, + stroke: None, + stroke_width: 0.75, + float, + story, + effects: Default::default(), + freeform: None, + extra: Default::default(), + } +} + +/// A page number in a floating text box, as in a footer: the shape that made the +/// Compatibility Pack refuse the file. +#[test] +fn a_text_box_gets_a_vml_fallback_with_its_text() { + let mut d = Document::from_text("a"); + let id = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text("29", Default::default()))]); + d.parts.get_mut(&id).unwrap().body = TextBody { auto_fit: true, inset: Some([0; 4]), ..Default::default() }; + let mut o = shape(ShapeKind::TextBox, Some(id)); + if let InlineObject::Shape { float, .. } = &mut o { + (float.dist, float.dist_top, float.dist_bottom) = (18.0, 12.0, 6.0); + } + d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap(); + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains(r#""#), "{xml}"); + assert!(!xml.contains(""), "no bare Word 2010 shape: {xml}"); + let fallback = &xml[xml.find("").expect("a fallback")..]; + assert!(fallback.starts_with(""#, + r#""#, + ] { + assert!(fallback.contains(want), "{want} in {fallback}"); + } + assert_eq!(xml.matches(">29").count(), 2, "the text in the choice and again in the fallback"); + + // Read back: the choice, once. + let r = craft_docx::read(&bytes).unwrap(); + let mut boxes = Vec::new(); + r.objects_in_reading_order(&mut |o| boxes.extend(o.text_boxes())); + boxes.retain(|id| r.parts.get(id).is_some_and(|p| p.kind == PartKind::TextBox)); + assert_eq!(boxes.len(), 1, "one text box, not the fallback as well"); + assert_eq!(r.plain_text(craft_wordmodel::StoryRef::Part(boxes[0])), "29"); +} + +/// The fallback keeps the box's own stacking order (not that of a floating object inside it), +/// its hidden flag and its text direction. +#[test] +fn the_fallback_keeps_order_visibility_and_direction() { + let mut d = Document::from_text("a"); + let inner = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text("inner", Default::default()))]); + let mut nested = shape(ShapeKind::TextBox, Some(inner)); + if let InlineObject::Shape { float, .. } = &mut nested { + float.z = 100; + } + let mut p = craft_wordmodel::Paragraph::with_text("outer", Default::default()); + p.insert_object(0, nested, &Default::default()).unwrap(); + let outer = d.add_part(PartKind::TextBox, vec![para_block(p)]); + d.parts.get_mut(&outer).unwrap().body = TextBody { vert: TextVert::Vert270, ..Default::default() }; + let mut o = shape(ShapeKind::TextBox, Some(outer)); + if let InlineObject::Shape { float, .. } = &mut o { + float.z = 10; + float.hidden = true; + } + d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap(); + let xml = document_xml(&craft_docx::write(&d).unwrap()); + // The outer box's VML: the `v:rect` holding the vertical text box (the inner box's fallback + // is copied into the outer one's text, so it comes after it). + let tb = xml.find(r#""#).expect("the outer box's text, turned"); + let rect = &xml[xml[..tb].rfind(" = xml + .match_indices("z-index:") + .map(|(i, m)| { + let v = &xml[i + m.len()..]; + v[..v.find([';', '"']).unwrap()].parse().unwrap() + }) + .collect(); + got.sort_unstable(); + let low = -(1i64 << 31); + assert_eq!(got, vec![low + 10, low + 20, -1], "{xml}"); +} + +/// A shape VML has no simple form for stays a bare `wps:wsp`: guarded without a fallback, an old +/// reader would open the file without it and drop it on its next save. +#[test] +fn a_shape_without_a_vml_form_is_not_guarded() { + let mut d = Document::from_text("a"); + d.insert_object(&craft_wordmodel::Pos::body(0, 1), shape(ShapeKind::Star, None), &Default::default()).unwrap(); + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains("") && xml.contains(r#""#), "{xml}"); + assert!(!xml.contains(""), "{xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let mut kinds = Vec::new(); + r.objects_in_reading_order(&mut |o| { + if let InlineObject::Shape { kind, .. } = o { + kinds.push(*kind); + } + }); + assert_eq!(kinds, vec![ShapeKind::Star]); +} diff --git a/crates/exr/src/deep.rs b/crates/exr/src/deep.rs index b86d497c8..ad2ab00b5 100644 --- a/crates/exr/src/deep.rs +++ b/crates/exr/src/deep.rs @@ -391,19 +391,14 @@ fn decode_deep_meta(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Result Result { Ok(cur.position()) } +/// Where part `header_index`'s offset table starts and where the last part's table ends. The +/// tables follow the headers, `chunk_count` u64 per header in header order, and the chunks +/// follow the last table. Sizes that overflow, or tables that run past the end of the file, +/// are errors. +fn offset_tables(meta: &MetaData, bytes: &[u8], header_index: usize) -> Result<(u64, u64)> { + let mut table_pos = header_end(bytes)?; + let mut tables_end = table_pos; + for (i, h) in meta.headers.iter().enumerate() { + let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8)); + tables_end = len.and_then(|len| tables_end.checked_add(len)).ok_or_else(|| err("the offset tables are too large"))?; + if i < header_index { + table_pos = tables_end; + } + } + if tables_end > bytes.len() as u64 { + return Err(err(format!("the offset tables end at byte {tables_end}, past the end of the file"))); + } + Ok((table_pos, tables_end)) +} + /// Composite the deep samples into a flat image: per pixel, samples sorted by Z (stable; the /// file is assumed messy), front-to-back `over` with premultiplied colour, un-premultiplied /// into straight alpha at the end. Without any alpha channel the front-most sample wins, diff --git a/crates/exr/src/deep_tests.rs b/crates/exr/src/deep_tests.rs index cb1721809..bdbfa0d68 100644 --- a/crates/exr/src/deep_tests.rs +++ b/crates/exr/src/deep_tests.rs @@ -886,6 +886,106 @@ fn inconsistent_count_table_errors() { assert!(matches!(decode_deep_exr(&bytes, &Limits::none()), Err(CodecError::Malformed(_)))); } +/// A two-part file made from the single-part deep file `deep` and a 1×1 flat scan-line part, +/// the deep part first when `deep_first`: each header gains `name` and `chunkCount`, the +/// version field the multipart bit, and each chunk its part number. The offset tables follow +/// the headers in header order, then the chunks. Returns the file, where the first table +/// starts, where the second table starts and where the last table ends. +fn two_part(deep: &[u8], deep_first: bool) -> (Vec, usize, usize, usize) { + let end = header_end_of(deep); + let first = u64::from_le_bytes(deep[end..end + 8].try_into().expect("8")) as usize; + let offsets: Vec = + (0..(first - end) / 8).map(|i| u64::from_le_bytes(deep[end + 8 * i..end + 8 * i + 8].try_into().expect("8")) as usize).collect(); + let deep_chunks: Vec<&[u8]> = offsets.iter().enumerate().map(|(i, &o)| &deep[o..offsets.get(i + 1).copied().unwrap_or(deep.len())]).collect(); + + let mut deep_header = deep[8..end - 1].to_vec(); + attr(&mut deep_header, "name", "string", b"deep"); + attr(&mut deep_header, "chunkCount", "int", &(deep_chunks.len() as i32).to_le_bytes()); + let mut flat_header = Vec::new(); + let mut chlist = b"R\0".to_vec(); + chlist.extend_from_slice(&1i32.to_le_bytes()); // half + chlist.extend_from_slice(&[0u8; 4]); // pLinear, reserved + chlist.extend_from_slice(&1i32.to_le_bytes()); + chlist.extend_from_slice(&1i32.to_le_bytes()); + chlist.push(0); + attr(&mut flat_header, "channels", "chlist", &chlist); + attr(&mut flat_header, "compression", "compression", &[0]); + let dw: Vec = [0i32; 4].iter().flat_map(|v| v.to_le_bytes()).collect(); + attr(&mut flat_header, "dataWindow", "box2i", &dw); + attr(&mut flat_header, "displayWindow", "box2i", &dw); + attr(&mut flat_header, "lineOrder", "lineOrder", &[0]); + attr(&mut flat_header, "pixelAspectRatio", "float", &1f32.to_le_bytes()); + attr(&mut flat_header, "screenWindowCenter", "v2f", &[0f32.to_le_bytes(), 0f32.to_le_bytes()].concat()); + attr(&mut flat_header, "screenWindowWidth", "float", &1f32.to_le_bytes()); + attr(&mut flat_header, "name", "string", b"flat"); + attr(&mut flat_header, "type", "string", b"scanlineimage"); + attr(&mut flat_header, "chunkCount", "int", &1i32.to_le_bytes()); + // The flat part's one chunk: y, the data size, one half sample. + let mut flat_chunk = 0i32.to_le_bytes().to_vec(); + flat_chunk.extend_from_slice(&2i32.to_le_bytes()); + flat_chunk.extend_from_slice(&f16::from_f32(0.5).to_le_bytes()); + let flat_chunks: Vec<&[u8]> = vec![&flat_chunk]; + + let parts: [(&[u8], &[&[u8]]); 2] = if deep_first { + [(&deep_header, &deep_chunks), (&flat_header, &flat_chunks)] + } else { + [(&flat_header, &flat_chunks), (&deep_header, &deep_chunks)] + }; + let mut out = deep[..4].to_vec(); + out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes()); + for (header, _) in &parts { + out.extend_from_slice(header); + out.push(0); // end of this header + } + out.push(0); // end of the headers + let first_table = out.len(); + let second_table = first_table + parts[0].1.len() * 8; + let tables_end = second_table + parts[1].1.len() * 8; + let mut pos = tables_end; + for (_, chunks) in &parts { + for c in chunks.iter() { + out.extend_from_slice(&(pos as u64).to_le_bytes()); + pos += 4 + c.len(); + } + } + for (part, (_, chunks)) in parts.iter().enumerate() { + for c in chunks.iter() { + out.extend_from_slice(&(part as i32).to_le_bytes()); + out.extend_from_slice(c); + } + } + (out, first_table, second_table, tables_end) +} + +/// The deep part of a two-part file decodes as the single-part file does, first or second. +/// Its chunks start after every part's offset table: a chunk offset into the other part's +/// table, or one byte before the end of the last table, is refused as table data, and a file +/// that ends inside the last table is refused even when the deep part's own table is whole. +#[test] +fn multipart_deep_chunks_start_after_every_offset_table() { + let single = gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba); + let want = decode_deep_exr(&single, &Limits::none()).expect("single part"); + for deep_first in [true, false] { + let (bytes, ..) = two_part(&single, deep_first); + let deep = decode_deep_exr(&bytes, &Limits::none()).expect("two parts"); + assert_eq!(deep.counts, want.counts); + assert_eq!(deep.channel("Z").expect("Z").samples, want.channel("Z").expect("Z").samples); + } + + let refused = |bytes: &[u8], what: &str| match decode_deep_exr(bytes, &Limits::none()) { + Err(e @ CodecError::Malformed(_)) => assert!(e.to_string().contains(what), "{e}"), + Err(e) => panic!("{e}, want an error about {what}"), + Ok(_) => panic!("decoded, want an error about {what}"), + }; + let (bytes, first_table, second_table, tables_end) = two_part(&single, true); + for off in [second_table, tables_end - 1] { + let mut into_table = bytes.clone(); + into_table[first_table..first_table + 8].copy_from_slice(&(off as u64).to_le_bytes()); + refused(&into_table, "outside the chunk area"); + } + refused(&bytes[..tables_end - 4], "past the end of the file"); +} + /// `gen_deep` for a cropped render: the data window sits at `(dwx, dwy)` inside a larger /// display window of `sw × sh` (the stage the renderer composed onto). Scanline chunk Y /// coordinates are absolute display rows, so they carry the data-window offset. diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index dbd7dca4c..bcd42e4e8 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -43,7 +43,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "< | TIFF | yes | yes | U8, U16, F32 | all six (CMYK, CMYK+A included) | yes | yes (tag 34675) | no | yes (tag 700) | yes | yes (Description, Make, Model, Software, DateTime, Artist, Copyright) | no | `tiff` | | WebP | yes | yes | U8 | RGB, RGBA | yes | yes | yes | yes | no | no | yes (VP8, when `webp_lossless` is off) | `image-webp` / built-in VP8 | | GIF | yes | yes | U8 | RGBA | 1-bit | no | no | no | no | no | yes (256-colour palette) | `image` | -| BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` | +| BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | yes (header pixels per metre) | no | no | `image` | | TGA | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | no | `image` (read) / built in (RLE write, one scanline per packet) | | ICO | yes | yes | U8 | RGBA (at most 256×256) | yes | no | no | no | no | no | no | `image` | | DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC3, when `EncodeOptions::dds` asks for one) | built in | @@ -79,7 +79,8 @@ the same. Palette BMP and PNG pixels decode as RGB(A) in `Image`; `bmp_palette` and `png_palette` expose the embedded table (and a PNG's transparent entry) so `photocraft-io` can open them in Indexed Color mode. `bmp_palette` returns the colours the header declares, not the decoder's 256-entry padding. -`encode_bmp_indexed` writes the inverse: indices plus a table as an 8-bit palette BMP. +`encode_bmp_indexed` writes the inverse: indices plus a table as an 8-bit palette BMP; +`encode_bmp_indexed_with_dpi` also writes a resolution in the header. ## Documented asymmetries and limitations diff --git a/crates/image-codecs/src/codecs.rs b/crates/image-codecs/src/codecs.rs index ef607d378..e2faaac4c 100644 --- a/crates/image-codecs/src/codecs.rs +++ b/crates/image-codecs/src/codecs.rs @@ -1,6 +1,7 @@ pub(crate) mod avif; #[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] pub(crate) mod avif_decode; +pub(crate) mod bmp; pub(crate) mod dds; pub(crate) mod dds_bc; pub(crate) mod deep_exr; diff --git a/crates/image-codecs/src/codecs/bmp.rs b/crates/image-codecs/src/codecs/bmp.rs new file mode 100644 index 000000000..239794a2c --- /dev/null +++ b/crates/image-codecs/src/codecs/bmp.rs @@ -0,0 +1,106 @@ +//! A BMP's resolution: `biXPelsPerMeter` and `biYPelsPerMeter`, the pixels per metre of the +//! BITMAPINFOHEADER and every later header (V4, V5, OS/2 2.x), at file offsets 38 and 42. The +//! 12-byte BITMAPCOREHEADER of OS/2 1.x has no such fields. The `image` crate reads and writes +//! the pixels only, and writes 0 here, which readers take as "no resolution". + +use super::png::dpi_of_ppm; + +/// Bytes before the fields: the 14-byte BITMAPFILEHEADER, then the DIB header's size, width, +/// height, planes, bit count, compression and image size. +const X_PPM: usize = 38; +const Y_PPM: usize = 42; +/// The shortest DIB header that carries the resolution (BITMAPINFOHEADER). +const INFO_HEADER: u32 = 40; +const METERS_PER_INCH: f64 = 0.0254; + +/// The little-endian 4 bytes at `at`, or `None` past the end of `bytes`. +fn le4(bytes: &[u8], at: usize) -> Option<[u8; 4]> { + bytes.get(at..at.checked_add(4)?)?.try_into().ok() +} + +/// Whether `bytes` start a BMP whose header has the resolution fields. +fn has_resolution_fields(bytes: &[u8]) -> bool { + bytes.starts_with(b"BM") && le4(bytes, Y_PPM).is_some() && le4(bytes, 14).map(u32::from_le_bytes).is_some_and(|dib| dib >= INFO_HEADER) +} + +/// The resolution in pixels per inch a BMP records, or `None` when its header has no fields for it +/// or either is 0 or negative (most writers leave them 0). A whole number of ppi stored rounded to +/// whole pixels per metre (300 ppi = 11811) reads back whole, as a PNG's `pHYs` does. +pub(crate) fn resolution(bytes: &[u8]) -> Option<(f32, f32)> { + if !has_resolution_fields(bytes) { + return None; + } + let field = |at: usize| le4(bytes, at).map(i32::from_le_bytes).and_then(|v| u32::try_from(v).ok()); + let (x, y) = (field(X_PPM)?, field(Y_PPM)?); + (x > 0 && y > 0).then(|| (dpi_of_ppm(x), dpi_of_ppm(y))) +} + +/// Pixels per metre for `ppi`, rounded as Photoshop and the PNG `pHYs` writer do (300 ppi = 11811), +/// or `None` for a resolution that has no positive whole number of pixels per metre. +fn ppm_of_dpi(ppi: f32) -> Option { + let ppm = (f64::from(ppi) / METERS_PER_INCH).round(); + (ppm.is_finite() && ppm >= 1.0).then(|| ppm.min(f64::from(i32::MAX)) as i32) +} + +/// Writes `dpi` into the resolution fields of the BMP in `bytes`. A header without the fields, or +/// a resolution that cannot be stored, leaves the bytes as they are. +pub(crate) fn set_resolution(bytes: &mut [u8], dpi: Option<(f32, f32)>) { + let Some((x, y)) = dpi.and_then(|(x, y)| Some((ppm_of_dpi(x)?, ppm_of_dpi(y)?))) else { return }; + if !has_resolution_fields(bytes) { + return; + } + for (at, ppm) in [(X_PPM, x), (Y_PPM, y)] { + if let Some(field) = bytes.get_mut(at..at + 4) { + field.copy_from_slice(&ppm.to_le_bytes()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A 46-byte BMP start: file header, a DIB header of `dib` bytes, 11811 px/m each way. + fn start(dib: u32) -> Vec { + let mut b = vec![0u8; Y_PPM + 4]; + b[..2].copy_from_slice(b"BM"); + b[14..18].copy_from_slice(&dib.to_le_bytes()); + b[X_PPM..X_PPM + 4].copy_from_slice(&11811i32.to_le_bytes()); + b[Y_PPM..Y_PPM + 4].copy_from_slice(&11811i32.to_le_bytes()); + b + } + + #[test] + fn only_headers_with_the_fields_are_read_or_written() { + assert_eq!(resolution(&start(40)), Some((300.0, 300.0))); + assert_eq!(resolution(&start(124)), Some((300.0, 300.0)), "BITMAPV5HEADER"); + // OS/2 1.x's 12-byte core header has no resolution: those bytes are pixels or palette. + let mut core = start(12); + assert_eq!(resolution(&core), None); + set_resolution(&mut core, Some((72.0, 72.0))); + assert_eq!(core, start(12), "left as it is"); + // Too short, or not a BMP: nothing read, nothing written, no panic. + let mut short = start(40)[..Y_PPM + 3].to_vec(); + assert_eq!(resolution(&short), None); + set_resolution(&mut short, Some((72.0, 72.0))); + assert_eq!(short, start(40)[..Y_PPM + 3]); + let mut other = start(40); + other[0] = b'X'; + assert_eq!(resolution(&other), None); + set_resolution(&mut other, Some((72.0, 72.0))); + assert_eq!(&other[X_PPM..X_PPM + 4], &11811i32.to_le_bytes(), "not a BMP: left as it is"); + } + + #[test] + fn resolutions_without_a_whole_pixel_per_metre_are_not_written() { + for bad in [0.0, -300.0, 0.01, f32::NAN, f32::INFINITY] { + let mut b = start(40); + set_resolution(&mut b, Some((bad, 300.0))); + assert_eq!(b, start(40), "{bad}"); + } + // A huge one is capped at the field's maximum rather than wrapping negative. + let mut b = start(40); + set_resolution(&mut b, Some((1e12, 300.0))); + assert_eq!(&b[X_PPM..X_PPM + 4], &i32::MAX.to_le_bytes()); + } +} diff --git a/crates/image-codecs/src/codecs/deep_exr.rs b/crates/image-codecs/src/codecs/deep_exr.rs index 0cc69e261..634737816 100644 --- a/crates/image-codecs/src/codecs/deep_exr.rs +++ b/crates/image-codecs/src/codecs/deep_exr.rs @@ -304,19 +304,14 @@ pub(crate) fn decode_deep(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Res let bytes_per_sample: usize = exr_types.iter().map(|t| sample_bytes(*t)).sum(); let multipart = meta.requirements.has_multiple_layers; - // The offset tables follow the headers: `chunk_count` u64 per header, in header order. - // Our part's table starts after the tables of all earlier parts. - let mut table_pos = header_end(bytes)?; - for h_ in meta.headers.iter().take(header_index) { - table_pos = table_pos.saturating_add((h_.chunk_count.min(1 << 24) as u64).saturating_mul(8)); - } + // A chunk offset before the end of the last offset table points into table data. + let (table_pos, tables_end) = offset_tables(meta, bytes, header_index)?; let mut cur = Cursor::new(bytes); seek_to(&mut cur, table_pos)?; let mut offsets = Vec::with_capacity(header.chunk_count.min(1 << 20)); for _ in 0..header.chunk_count { offsets.push(u64(&mut cur)?); } - let tables_end = cur.position(); // Block geometry. let (tiles_x, tile_w, tile_h, slots) = match &header.blocks { @@ -565,6 +560,26 @@ fn header_end(bytes: &[u8]) -> Result { Ok(cur.position()) } +/// Where part `header_index`'s offset table starts and where the last part's table ends. The +/// tables follow the headers, `chunk_count` u64 per header in header order, and the chunks +/// follow the last table. Sizes that overflow, or tables that run past the end of the file, +/// are errors. +fn offset_tables(meta: &MetaData, bytes: &[u8], header_index: usize) -> Result<(u64, u64), CodecError> { + let mut table_pos = header_end(bytes)?; + let mut tables_end = table_pos; + for (i, h) in meta.headers.iter().enumerate() { + let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8)); + tables_end = len.and_then(|len| tables_end.checked_add(len)).ok_or_else(|| err("the offset tables are too large"))?; + if i < header_index { + table_pos = tables_end; + } + } + if tables_end > bytes.len() as u64 { + return Err(err(format!("the offset tables end at byte {tables_end}, past the end of the file"))); + } + Ok((table_pos, tables_end)) +} + /// Composite the deep samples into a flat image: per pixel, samples sorted by Z (stable; the /// file is assumed messy), front-to-back `over` with premultiplied colour, un-premultiplied /// into straight alpha at the end. Without any alpha channel the front-most sample wins, diff --git a/crates/image-codecs/src/format.rs b/crates/image-codecs/src/format.rs index 9e55dc3d2..983dab3dd 100644 --- a/crates/image-codecs/src/format.rs +++ b/crates/image-codecs/src/format.rs @@ -308,7 +308,8 @@ pub fn caps(format: Format) -> FormatCaps { // Resolution travels in EXIF (WebP has no field of its own). Format::WebP => FormatCaps { icc: true, exif: true, xmp: true, dpi: true, animation: true, ..base }, Format::Gif => FormatCaps { layouts: &[L::Rgba], animation: true, lossy: true, ..base }, - Format::Bmp => base, + // Resolution in the header's pixels per metre (BITMAPINFOHEADER and later). + Format::Bmp => FormatCaps { dpi: true, ..base }, Format::Tga => FormatCaps { layouts: RGB_GRAY, ..base }, Format::Ico => FormatCaps { layouts: &[L::Rgba], ..base }, Format::Pnm => FormatCaps { depths: &[S::U8, S::U16, S::F32], layouts: ALL_LAYOUTS, ..base }, diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index 24fd32deb..8c29ed52d 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -74,7 +74,7 @@ pub use half::f16; use crate::codecs::sgi; #[cfg(feature = "sunraster")] use crate::codecs::sunraster; -use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm}; +use crate::codecs::{bmp, dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm}; /// Detect the format and decode with default [`Limits`]. pub fn decode(bytes: &[u8]) -> Result { @@ -142,7 +142,11 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res Format::Avif => codecs::avif_decode::decode(bytes, l, opts.keep_orientation), #[cfg(not(all(feature = "avif-decode", not(target_arch = "wasm32"))))] Format::Avif => via_image::decode(format, bytes, l), - Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l), + Format::Bmp => via_image::decode(format, bytes, l).map(|mut img| { + img.meta.dpi = bmp::resolution(bytes); + img + }), + Format::Gif | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l), }?; finish_format(format, bytes, img, opts) } @@ -325,10 +329,26 @@ pub fn bmp_palette(bytes: &[u8]) -> Result>, CodecError> { /// Encodes `indices` (row-major, one byte per pixel) as an 8-bit palette BMP with the colour /// table `palette` (1 to 256 entries). Used for Indexed Color documents; BMP has no transparency. +/// The header's resolution is left unset; [`encode_bmp_indexed_with_dpi`] writes one. pub fn encode_bmp_indexed(width: u32, height: u32, indices: &[u8], palette: &[[u8; 3]]) -> Result, CodecError> { via_image::encode_bmp_indexed(width, height, indices, palette) } +/// [`encode_bmp_indexed`] with the resolution `dpi` (pixels per inch, horizontal then vertical) +/// in the header's pixels per metre, as [`encode`] writes [`Metadata::dpi`] for a BMP. `None`, or +/// a resolution with no positive whole number of pixels per metre, leaves the fields unset. +pub fn encode_bmp_indexed_with_dpi( + width: u32, + height: u32, + indices: &[u8], + palette: &[[u8; 3]], + dpi: Option<(f32, f32)>, +) -> Result, CodecError> { + let mut bytes = via_image::encode_bmp_indexed(width, height, indices, palette)?; + bmp::set_resolution(&mut bytes, dpi); + Ok(bytes) +} + /// Lists the pages (image directories) of a TIFF or BigTIFF file without decoding pixels: the /// main IFD chain and its SubIFDs, each with its size, kind (page, reduced-resolution copy or /// mask) and storage. [`TiffInfo::default_page`] is the page [`decode`] opens. @@ -546,8 +566,12 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result Err(CodecError::unsupported(format, "encoding is not available in this build")), Format::Jp2 => Err(CodecError::unsupported(format, "JPEG 2000 is import-only; save as PNG or TIFF instead")), Format::Jxl => jxl::encode(image, plan), - Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => { - via_image::encode(format, image, plan, opts) - } + // The `image` crate leaves a BMP's resolution fields 0; Photoshop fills them in. They are + // part of the header, not metadata, so they are written whatever `embed_metadata` says. + Format::Bmp => via_image::encode(format, image, plan, opts).map(|mut bytes| { + bmp::set_resolution(&mut bytes, image.meta.dpi); + bytes + }), + Format::Gif | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => via_image::encode(format, image, plan, opts), } } diff --git a/crates/image-codecs/tests/it/bmp_resolution.rs b/crates/image-codecs/tests/it/bmp_resolution.rs new file mode 100644 index 000000000..2e35f222c --- /dev/null +++ b/crates/image-codecs/tests/it/bmp_resolution.rs @@ -0,0 +1,77 @@ +//! A BMP keeps its resolution in the header's `biXPelsPerMeter` and `biYPelsPerMeter`, which +//! Photoshop writes and reads (storytold/photocraft#13630). BMP exports left both 0, so a 300 ppi +//! document saved as BMP opened at 72 ppi elsewhere, and an opened BMP's resolution was ignored. + +use crate::common::test_image; +use craft_image_codecs::*; + +/// The DIB header's size and its pixels per metre, horizontal then vertical (file offsets 14, 38 +/// and 42, little-endian). +fn header(bytes: &[u8]) -> (u32, i32, i32) { + let at = |i: usize| [bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]]; + (u32::from_le_bytes(at(14)), i32::from_le_bytes(at(38)), i32::from_le_bytes(at(42))) +} + +/// A test image of `layout` at resolution `dpi`. +fn image(layout: ChannelLayout, dpi: Option<(f32, f32)>) -> Image { + let mut img = test_image(layout, SampleType::U8); + img.meta.dpi = dpi; + img +} + +#[test] +fn bmp_export_writes_the_resolution_in_pixels_per_metre_and_reads_it_back() { + // RGB is written with a BITMAPINFOHEADER, RGBA with a BITMAPV4HEADER: same field offsets. + for (layout, dib) in [(ChannelLayout::Rgb, 40), (ChannelLayout::Rgba, 108)] { + // The fields are part of the header, not metadata: embed_metadata off keeps them. + for embed_metadata in [true, false] { + let opts = EncodeOptions { embed_metadata, ..Default::default() }; + let img = image(layout, Some((300.0, 150.0))); + let bytes = encode(&img, Format::Bmp, &opts).unwrap(); + // 300 / 0.0254 = 11811.02 and 150 / 0.0254 = 5905.51, rounded. + assert_eq!(header(&bytes), (dib, 11811, 5906), "{layout:?} metadata {embed_metadata}"); + assert_eq!(decode(&bytes).unwrap().meta.dpi, Some((300.0, 150.0)), "{layout:?} metadata {embed_metadata}"); + assert!(!fidelity_warnings_with(&img, Format::Bmp, &opts).contains(&FidelityWarning::DpiDropped), "{layout:?}"); + } + } +} + +#[test] +fn bmp_without_a_resolution_leaves_the_fields_unset() { + let bytes = encode(&image(ChannelLayout::Rgb, None), Format::Bmp, &EncodeOptions::default()).unwrap(); + assert_eq!(header(&bytes), (40, 0, 0)); + assert_eq!(decode(&bytes).unwrap().meta.dpi, None, "0 means no resolution: the caller's default applies"); +} + +#[test] +fn bmp_opens_at_the_resolution_its_header_records() { + let bytes = encode(&image(ChannelLayout::Rgb, None), Format::Bmp, &EncodeOptions::default()).unwrap(); + let with = |x: i32, y: i32| { + let mut b = bytes.clone(); + b[38..42].copy_from_slice(&x.to_le_bytes()); + b[42..46].copy_from_slice(&y.to_le_bytes()); + decode(&b).unwrap().meta.dpi + }; + // What other writers store for 96 and 72 ppi reads back whole. + assert_eq!(with(3780, 3780), Some((96.0, 96.0))); + assert_eq!(with(2835, 2835), Some((72.0, 72.0))); + // 40000 px/m is exactly 1016 ppi; a value no whole ppi rounds to stays as it is (71.98). + let (x, y) = with(40000, 2834).unwrap(); + assert_eq!(x, 1016.0); + assert!((y - 71.9836).abs() < 1e-3, "{y}"); + // An axis that is 0 or negative leaves the resolution unknown. + assert_eq!(with(11811, 0), None); + assert_eq!(with(-11811, 11811), None); +} + +#[test] +fn indexed_bmp_writes_the_resolution_it_is_given() { + let (idx, pal) = ([0u8, 1, 1, 0], [[0, 0, 0], [255, 0, 0]]); + let bytes = encode_bmp_indexed_with_dpi(2, 2, &idx, &pal, Some((300.0, 300.0))).unwrap(); + assert_eq!(header(&bytes), (40, 11811, 11811)); + assert_eq!(decode(&bytes).unwrap().meta.dpi, Some((300.0, 300.0))); + let unset = encode_bmp_indexed_with_dpi(2, 2, &idx, &pal, None).unwrap(); + assert_eq!(header(&unset), (40, 0, 0)); + // The existing function keeps writing no resolution, so its callers' files don't change. + assert_eq!(encode_bmp_indexed(2, 2, &idx, &pal).unwrap(), unset); +} diff --git a/crates/image-codecs/tests/it/deep_exr.rs b/crates/image-codecs/tests/it/deep_exr.rs index e614f865d..2e1105634 100644 --- a/crates/image-codecs/tests/it/deep_exr.rs +++ b/crates/image-codecs/tests/it/deep_exr.rs @@ -799,6 +799,106 @@ fn inconsistent_count_table_errors() { assert!(matches!(decode_deep_exr(&bytes, &Limits::none()), Err(CodecError::Malformed { .. }))); } +/// A two-part file made from the single-part deep file `deep` and a 1×1 flat scan-line part, +/// the deep part first when `deep_first`: each header gains `name` and `chunkCount`, the +/// version field the multipart bit, and each chunk its part number. The offset tables follow +/// the headers in header order, then the chunks. Returns the file, where the first table +/// starts, where the second table starts and where the last table ends. +fn two_part(deep: &[u8], deep_first: bool) -> (Vec, usize, usize, usize) { + let end = header_end_of(deep); + let first = u64::from_le_bytes(deep[end..end + 8].try_into().expect("8")) as usize; + let offsets: Vec = + (0..(first - end) / 8).map(|i| u64::from_le_bytes(deep[end + 8 * i..end + 8 * i + 8].try_into().expect("8")) as usize).collect(); + let deep_chunks: Vec<&[u8]> = offsets.iter().enumerate().map(|(i, &o)| &deep[o..offsets.get(i + 1).copied().unwrap_or(deep.len())]).collect(); + + let mut deep_header = deep[8..end - 1].to_vec(); + attr(&mut deep_header, "name", "string", b"deep"); + attr(&mut deep_header, "chunkCount", "int", &(deep_chunks.len() as i32).to_le_bytes()); + let mut flat_header = Vec::new(); + let mut chlist = b"R\0".to_vec(); + chlist.extend_from_slice(&1i32.to_le_bytes()); // half + chlist.extend_from_slice(&[0u8; 4]); // pLinear, reserved + chlist.extend_from_slice(&1i32.to_le_bytes()); + chlist.extend_from_slice(&1i32.to_le_bytes()); + chlist.push(0); + attr(&mut flat_header, "channels", "chlist", &chlist); + attr(&mut flat_header, "compression", "compression", &[0]); + let dw: Vec = [0i32; 4].iter().flat_map(|v| v.to_le_bytes()).collect(); + attr(&mut flat_header, "dataWindow", "box2i", &dw); + attr(&mut flat_header, "displayWindow", "box2i", &dw); + attr(&mut flat_header, "lineOrder", "lineOrder", &[0]); + attr(&mut flat_header, "pixelAspectRatio", "float", &1f32.to_le_bytes()); + attr(&mut flat_header, "screenWindowCenter", "v2f", &[0f32.to_le_bytes(), 0f32.to_le_bytes()].concat()); + attr(&mut flat_header, "screenWindowWidth", "float", &1f32.to_le_bytes()); + attr(&mut flat_header, "name", "string", b"flat"); + attr(&mut flat_header, "type", "string", b"scanlineimage"); + attr(&mut flat_header, "chunkCount", "int", &1i32.to_le_bytes()); + // The flat part's one chunk: y, the data size, one half sample. + let mut flat_chunk = 0i32.to_le_bytes().to_vec(); + flat_chunk.extend_from_slice(&2i32.to_le_bytes()); + flat_chunk.extend_from_slice(&f16::from_f32(0.5).to_le_bytes()); + let flat_chunks: Vec<&[u8]> = vec![&flat_chunk]; + + let parts: [(&[u8], &[&[u8]]); 2] = if deep_first { + [(&deep_header, &deep_chunks), (&flat_header, &flat_chunks)] + } else { + [(&flat_header, &flat_chunks), (&deep_header, &deep_chunks)] + }; + let mut out = deep[..4].to_vec(); + out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes()); + for (header, _) in &parts { + out.extend_from_slice(header); + out.push(0); // end of this header + } + out.push(0); // end of the headers + let first_table = out.len(); + let second_table = first_table + parts[0].1.len() * 8; + let tables_end = second_table + parts[1].1.len() * 8; + let mut pos = tables_end; + for (_, chunks) in &parts { + for c in chunks.iter() { + out.extend_from_slice(&(pos as u64).to_le_bytes()); + pos += 4 + c.len(); + } + } + for (part, (_, chunks)) in parts.iter().enumerate() { + for c in chunks.iter() { + out.extend_from_slice(&(part as i32).to_le_bytes()); + out.extend_from_slice(c); + } + } + (out, first_table, second_table, tables_end) +} + +/// The deep part of a two-part file decodes as the single-part file does, first or second. +/// Its chunks start after every part's offset table: a chunk offset into the other part's +/// table, or one byte before the end of the last table, is refused as table data, and a file +/// that ends inside the last table is refused even when the deep part's own table is whole. +#[test] +fn multipart_deep_chunks_start_after_every_offset_table() { + let single = gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba); + let want = decode_deep_exr(&single, &Limits::none()).expect("single part"); + for deep_first in [true, false] { + let (bytes, ..) = two_part(&single, deep_first); + let deep = decode_deep_exr(&bytes, &Limits::none()).expect("two parts"); + assert_eq!(deep.counts, want.counts); + assert_eq!(deep.channel("Z").expect("Z").samples, want.channel("Z").expect("Z").samples); + } + + let refused = |bytes: &[u8], what: &str| match decode_deep_exr(bytes, &Limits::none()) { + Err(CodecError::Malformed { message, .. }) => assert!(message.contains(what), "{message}"), + Err(e) => panic!("{e}, want an error about {what}"), + Ok(_) => panic!("decoded, want an error about {what}"), + }; + let (bytes, first_table, second_table, tables_end) = two_part(&single, true); + for off in [second_table, tables_end - 1] { + let mut into_table = bytes.clone(); + into_table[first_table..first_table + 8].copy_from_slice(&(off as u64).to_le_bytes()); + refused(&into_table, "outside the chunk area"); + } + refused(&bytes[..tables_end - 4], "past the end of the file"); +} + // --------------------------------------------------------------------------------------------- // Real reference files: the deep test images of the OpenEXR repository at commit // `69b2604fc76e370615438bdc8d2cd95b9349c12e`, read from `corpus/exr/` at the repo root with the diff --git a/crates/image-codecs/tests/it/main.rs b/crates/image-codecs/tests/it/main.rs index 380e37565..69be177db 100644 --- a/crates/image-codecs/tests/it/main.rs +++ b/crates/image-codecs/tests/it/main.rs @@ -1,5 +1,6 @@ //! The crate's integration tests, linked into one test binary. +mod bmp_resolution; mod camera_raw; mod common; mod dds; diff --git a/crates/mcp-server/Cargo.toml b/crates/mcp-server/Cargo.toml new file mode 100644 index 000000000..9b034c7cc --- /dev/null +++ b/crates/mcp-server/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-mcp-server" +description = "MCP server machinery any app may share: JSON-RPC 2.0 over stdio, the MCP lifecycle, tool and resource methods, tool schemas and argument checks, image results" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde_json = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/mcp-server/src/content.rs b/crates/mcp-server/src/content.rs new file mode 100644 index 000000000..a2a017dec --- /dev/null +++ b/crates/mcp-server/src/content.rs @@ -0,0 +1,64 @@ +//! The content blocks a tool result is made of (MCP `CallToolResult`), and the contents a resource +//! read sends (MCP `ReadResourceResult`). + +use serde_json::{Value, json}; + +/// What `resources/read` sends for one resource: text of a MIME type, or bytes of a MIME type, +/// which go out base64-encoded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Contents { + Text { mime_type: String, text: String }, + Blob { mime_type: String, data: Vec }, +} + +impl Contents { + /// `value` as JSON text (`application/json`). + pub fn json(value: &Value) -> Self { + Self::Text { mime_type: "application/json".into(), text: value.to_string() } + } + + /// The MCP contents block of resource `uri`. + pub(crate) fn block(&self, uri: &str) -> Value { + match self { + Self::Text { mime_type, text } => json!({"uri": uri, "mimeType": mime_type, "text": text}), + Self::Blob { mime_type, data } => json!({"uri": uri, "mimeType": mime_type, "blob": base64(data)}), + } + } +} + +/// A tool result of one text block, flagged as an error when `is_error` (the model sees the +/// message; an error is still a result, not a JSON-RPC error). +pub fn text_result(text: impl Into, is_error: bool) -> Value { + let mut r = json!({"content": [{"type": "text", "text": text.into()}]}); + if is_error { + r["isError"] = json!(true); + } + r +} + +/// A tool result of an image (`data` of type `mime`, base64-encoded here) followed by a text +/// block saying what it shows. +pub fn image_result(data: &[u8], mime: &str, text: impl Into) -> Value { + json!({"content": [ + {"type": "image", "data": base64(data), "mimeType": mime}, + {"type": "text", "text": text.into()}, + ]}) +} + +/// Standard base64 with padding (RFC 4648), as MCP image content carries it. +pub fn base64(data: &[u8]) -> String { + const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)]; + let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() { + if i <= chunk.len() { + out.push(char::from(T[((n >> shift) & 63) as usize])); + } else { + out.push('='); + } + } + } + out +} diff --git a/crates/mcp-server/src/lib.rs b/crates/mcp-server/src/lib.rs new file mode 100644 index 000000000..17e6827c4 --- /dev/null +++ b/crates/mcp-server/src/lib.rs @@ -0,0 +1,21 @@ +//! MCP server machinery any app may share: JSON-RPC 2.0 over stdio (newline-delimited), the MCP +//! lifecycle, the tools, resources, prompts and completion methods, tool schemas with the check of +//! a call's arguments against them, and the content blocks tools reply with (text, images). +//! +//! An app implements [`Tools`]: what its tools are, how each one runs, its instructions, and the +//! resources, resource templates and prompts it serves, if any. [`Server`] speaks the protocol +//! around them and tells a client at `initialize` only what the app serves. Nothing here knows any +//! app's design, engine or renderer. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +mod content; +mod schema; +mod server; + +pub use content::{Contents, base64, image_result, text_result}; +pub use schema::{Effect, check_arguments, object, string, tool}; +pub use server::{Incoming, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response}; + +#[cfg(test)] +mod tests; diff --git a/crates/mcp-server/src/schema.rs b/crates/mcp-server/src/schema.rs new file mode 100644 index 000000000..3cc72da34 --- /dev/null +++ b/crates/mcp-server/src/schema.rs @@ -0,0 +1,132 @@ +//! Tool definitions (MCP `Tool`: name, title, description, input schema, annotations) and the +//! check of a call's arguments against the definition it was listed with. + +use serde_json::{Value, json}; + +/// What a tool may do, for the client's approval prompts (MCP tool annotations). Clients that +/// auto-approve on these hints must not be told less than the truth. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Effect { + /// Only looks. + ReadOnly, + /// Changes the app's document in memory, undoably. + Edits, + /// Replaces the app's document in memory (unsaved changes are lost). + Replaces, + /// May read, write, overwrite or delete files on disk. + Files, +} + +/// A tool's definition, its annotations from `effect`. +pub fn tool(name: &str, title: &str, desc: &str, schema: Value, effect: Effect) -> Value { + let (read_only, destructive, open_world) = match effect { + Effect::ReadOnly => (true, false, false), + Effect::Edits => (false, false, false), + Effect::Replaces => (false, true, false), + Effect::Files => (false, true, true), + }; + json!({ + "name": name, + "title": title, + "description": desc, + "inputSchema": schema, + "annotations": {"title": title, "readOnlyHint": read_only, "destructiveHint": destructive, "idempotentHint": read_only, "openWorldHint": open_world}, + }) +} + +/// An object schema with these `props`, the `required` ones, and no other keys. +pub fn object(props: Value, required: &[&str]) -> Value { + let mut o = json!({"type": "object", "properties": props, "additionalProperties": false}); + if !required.is_empty() { + o["required"] = json!(required); + } + o +} + +/// A string property described by `desc`. +pub fn string(desc: &str) -> Value { + json!({"type": "string", "description": desc}) +} + +fn type_matches(t: &str, v: &Value) -> bool { + match t { + "string" => v.is_string(), + "number" => v.is_number(), + "integer" => v.is_i64() || v.is_u64(), + "boolean" => v.is_boolean(), + "array" => v.is_array(), + "object" => v.is_object(), + "null" => v.is_null(), + _ => true, + } +} + +/// Check `args` against tool `name`'s input schema as `defs` list it (top level): unknown tool, +/// missing required arguments, unknown arguments, type and enum mismatches. +pub fn check_arguments(defs: &[Value], name: &str, args: &Value) -> Result<(), String> { + let def = defs.iter().find(|d| d["name"] == name).ok_or_else(|| format!("unknown tool `{name}`"))?; + let schema = &def["inputSchema"]; + let Some(a) = args.as_object() else { return Err(format!("{name}: arguments must be an object")) }; + for r in schema["required"].as_array().into_iter().flatten().filter_map(Value::as_str) { + if a.get(r).is_none_or(Value::is_null) { + return Err(format!("{name}: missing required argument `{r}`")); + } + } + let props = schema["properties"].as_object(); + for (k, v) in a { + let Some(p) = props.and_then(|p| p.get(k)) else { + let valid: Vec<&str> = props.map(|p| p.keys().map(String::as_str).collect()).unwrap_or_default(); + let valid = if valid.is_empty() { "none".to_string() } else { valid.join(", ") }; + let k: String = k.chars().take(64).collect(); + return Err(format!("{name}: unknown argument `{k}`; its arguments are: {valid}")); + }; + if v.is_null() { + continue; + } + // An argument of several types lists them as `anyOf` (Gemini refuses a list of types). + let types: Vec<&Value> = match p["anyOf"].as_array() { + Some(alts) => alts.iter().map(|alt| &alt["type"]).collect(), + None => vec![&p["type"]], + }; + let ok = types.iter().any(|t| match t { + Value::String(t) => type_matches(t, v), + Value::Array(ts) => ts.iter().filter_map(Value::as_str).any(|t| type_matches(t, v)), + _ => true, + }); + if !ok { + let expected = if types.len() == 1 { types[0].clone() } else { Value::Array(types.into_iter().cloned().collect()) }; + return Err(format!("{name}: argument `{k}` must be of type {expected}")); + } + if let Some(allowed) = p["enum"].as_array() + && !allowed.contains(v) + { + return Err(format!("{name}: `{k}` must be one of {}", Value::Array(allowed.clone()))); + } + } + Ok(()) +} + +/// Check `args` against prompt `name` as `defs` list it (MCP `Prompt`): unknown prompt, missing +/// required arguments, unknown arguments, values that are not strings. +pub(crate) fn check_prompt_arguments(defs: &[Value], name: &str, args: &Value) -> Result<(), String> { + let def = defs.iter().find(|d| d["name"] == name).ok_or_else(|| format!("unknown prompt `{name}`"))?; + let declared = def["arguments"].as_array().map(Vec::as_slice).unwrap_or_default(); + let Some(a) = args.as_object() else { return Err(format!("{name}: arguments must be an object")) }; + for r in declared.iter().filter(|d| d["required"] == true).filter_map(|d| d["name"].as_str()) { + if !a.contains_key(r) { + return Err(format!("{name}: missing required argument `{r}`")); + } + } + for (k, v) in a { + if !declared.iter().any(|d| d["name"] == k.as_str()) { + let valid: Vec<&str> = declared.iter().filter_map(|d| d["name"].as_str()).collect(); + let valid = if valid.is_empty() { "none".to_string() } else { valid.join(", ") }; + let k: String = k.chars().take(64).collect(); + return Err(format!("{name}: unknown argument `{k}`; its arguments are: {valid}")); + } + if !v.is_string() { + return Err(format!("{name}: argument `{k}` must be a string")); + } + } + Ok(()) +} diff --git a/crates/mcp-server/src/server.rs b/crates/mcp-server/src/server.rs new file mode 100644 index 000000000..b7ed05027 --- /dev/null +++ b/crates/mcp-server/src/server.rs @@ -0,0 +1,330 @@ +//! JSON-RPC 2.0 framing and the MCP lifecycle / tools / resources / prompts / completion methods. + +use std::io::{BufRead, Write}; + +use serde_json::{Value, json}; + +use crate::content::Contents; +use crate::schema::{check_arguments, check_prompt_arguments}; + +/// The newest protocol revision we speak; offered when the client asks for one we don't know. +pub const PROTOCOL_VERSION: &str = "2025-06-18"; +pub const SUPPORTED_VERSIONS: &[&str] = &[PROTOCOL_VERSION, "2025-03-26", "2024-11-05"]; + +const PARSE_ERROR: i64 = -32700; +const INVALID_REQUEST: i64 = -32600; +const METHOD_NOT_FOUND: i64 = -32601; +const INVALID_PARAMS: i64 = -32602; +const INTERNAL_ERROR: i64 = -32603; +const RESOURCE_NOT_FOUND: i64 = -32002; + +/// Most messages in one JSON-RPC batch. +const MAX_BATCH: usize = 1000; +/// Most suggestions in one `completion/complete` reply (the MCP limit). +const MAX_COMPLETIONS: usize = 100; + +/// What `completion/complete` asks about: a prompt by its name, or a resource by its URI or URI +/// template. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reference<'a> { + Prompt(&'a str), + Resource(&'a str), +} + +/// An app's tools, served by [`Server`]. +pub trait Tools { + /// What every client is told at `initialize`. + fn instructions(&self) -> String; + /// Every tool's definition ([`crate::tool`]), as `tools/list` serves it now. + fn list(&mut self) -> Vec; + /// Run tool `name`. The server calls it right after the [`Tools::list`] its arguments were + /// checked against ([`crate::check_arguments`]), so it runs the tool as that listing defined + /// it. Failures are tool results with `isError: true` ([`crate::text_result`]); it never + /// panics on hostile arguments. + fn call(&mut self, name: &str, args: &Value) -> Value; + /// The resources `resources/list` serves (MCP `Resource`: uri, name, description, mimeType). + fn resources(&self) -> Vec { + Vec::new() + } + /// The resource templates `resources/templates/list` serves (MCP `ResourceTemplate`: + /// uriTemplate, name, description, mimeType). A URI made from one is read with + /// [`Tools::read_resource`]. + fn resource_templates(&self) -> Vec { + Vec::new() + } + /// The contents of resource `uri`: `None` when there is no such resource, `Err` when it could + /// not be read now. + fn read_resource(&mut self, uri: &str) -> Option> { + let _ = uri; + None + } + /// The prompts `prompts/list` serves (MCP `Prompt`: name, title, description, arguments). + fn prompts(&self) -> Vec { + Vec::new() + } + /// Prompt `name` filled in with `args` (MCP `GetPromptResult`: description, messages). The + /// server calls it only for a listed prompt, with `args` checked against that listing: an + /// object of strings with every required argument and no other. `Err` is an internal error. + fn get_prompt(&mut self, name: &str, args: &Value) -> Result { + let _ = args; + Err(format!("prompt `{name}` cannot be filled in")) + } + /// What argument `argument` of `reference` could be, given what has been typed (`value`). + /// Asked only of an app that serves prompts or resource templates; a prompt reference is + /// always a listed prompt. The reply holds the first 100. + fn complete(&mut self, reference: Reference<'_>, argument: &str, value: &str) -> Vec { + let _ = (reference, argument, value); + Vec::new() + } +} + +/// Who the server says it is at `initialize`, and the longest request line it reads. +#[derive(Clone, Copy, Debug)] +pub struct Info { + pub name: &'static str, + pub title: &'static str, + pub version: &'static str, + /// Longest accepted request line, in bytes; a longer one is refused in bounded space. + pub max_line: usize, +} + +/// An MCP server over the app's [`Tools`]. +pub struct Server { + info: Info, + tools: T, + initialized: bool, +} + +fn error(id: Value, code: i64, message: impl Into) -> Value { + json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message.into()}}) +} + +/// One message read from a newline-delimited stream by [`read_message`]. +#[derive(Debug, PartialEq, Eq)] +pub enum Incoming { + /// A line to handle ([`Server::handle_line`]), without its newline. + Line(String), + /// A line refused unread (too long, not UTF-8): the error reply to send for it. + Refused(String), +} + +/// Read the next line of newline-delimited JSON-RPC from `input`; `None` once it closes. A line +/// longer than `max_line` is consumed in bounded space, so a client cannot exhaust memory by +/// sending an arbitrarily long request without a newline, and an unterminated last line counts. +pub fn read_message(input: &mut impl BufRead, max_line: usize) -> std::io::Result> { + let mut line = Vec::new(); + let mut oversized = false; + let mut received = false; + loop { + let chunk = input.fill_buf()?; + if chunk.is_empty() { + break; + } + let count = chunk.iter().position(|&byte| byte == b'\n').map_or(chunk.len(), |i| i + 1); + let newline = chunk[count - 1] == b'\n'; + let content_len = count - usize::from(newline); + if !oversized { + if line.len().saturating_add(content_len) > max_line { + oversized = true; + line.clear(); + } else { + line.extend_from_slice(&chunk[..content_len]); + } + } + input.consume(count); + received = true; + if newline { + break; + } + } + if !received { + return Ok(None); + } + if oversized { + return Ok(Some(Incoming::Refused(error(Value::Null, INVALID_REQUEST, "request too large").to_string()))); + } + Ok(Some(match String::from_utf8(line) { + Ok(line) => Incoming::Line(line), + // Not UTF-8: answer with a parse error and keep serving. + Err(_) => Incoming::Refused(error(Value::Null, PARSE_ERROR, "parse error: input is not UTF-8").to_string()), + })) +} + +/// The JSON-RPC response to request `id`, as [`Server::handle`] sends it. +pub fn response(id: Value, result: Value) -> Value { + json!({"jsonrpc": "2.0", "id": id, "result": result}) +} + +impl Server { + pub fn new(info: Info, tools: T) -> Self { + Self { info, tools, initialized: false } + } + pub fn tools(&mut self) -> &mut T { + &mut self.tools + } + pub fn is_initialized(&self) -> bool { + self.initialized + } + + /// Serve newline-delimited JSON-RPC until `input` closes. Only protocol messages go to + /// `output`; diagnostics belong on stderr. Consume oversized lines in bounded space so a + /// client cannot exhaust memory by sending an arbitrarily long request without a newline. + pub fn serve(&mut self, mut input: impl BufRead, mut output: impl Write) -> std::io::Result<()> { + while let Some(incoming) = read_message(&mut input, self.info.max_line)? { + let reply = match incoming { + Incoming::Line(line) => self.handle_line(&line), + Incoming::Refused(reply) => Some(reply), + }; + if let Some(reply) = reply { + output.write_all(reply.as_bytes())?; + output.write_all(b"\n")?; + output.flush()?; + } + } + Ok(()) + } + + /// One line in, at most one line out (notifications get no reply). + pub fn handle_line(&mut self, line: &str) -> Option { + // Check the raw input before trimming: a huge whitespace prefix is still a huge line. + if line.len() > self.info.max_line { + return Some(error(Value::Null, INVALID_REQUEST, "request too large").to_string()); + } + let line = line.trim(); + if line.is_empty() { + return None; + } + let reply = match serde_json::from_str::(line) { + Ok(Value::Array(batch)) if batch.is_empty() => Some(error(Value::Null, INVALID_REQUEST, "empty batch")), + Ok(Value::Array(batch)) if batch.len() > MAX_BATCH => Some(error(Value::Null, INVALID_REQUEST, "batch too large")), + Ok(Value::Array(batch)) => { + let replies: Vec = batch.into_iter().filter_map(|m| self.handle(m)).collect(); + (!replies.is_empty()).then_some(Value::Array(replies)) + } + Ok(msg) => self.handle(msg), + Err(e) => Some(error(Value::Null, PARSE_ERROR, format!("parse error: {e}"))), + }; + reply.map(|r| r.to_string()) + } + + pub fn handle(&mut self, msg: Value) -> Option { + let Value::Object(o) = &msg else { return Some(error(Value::Null, INVALID_REQUEST, "message must be an object")) }; + let id = o.get("id").cloned(); + let Some(method) = o.get("method").and_then(Value::as_str) else { + // A response to a request of ours (we send none) or junk. + if o.contains_key("result") || o.contains_key("error") { + return None; + } + return Some(error(id.unwrap_or(Value::Null), INVALID_REQUEST, "missing `method`")); + }; + let params = o.get("params").cloned().unwrap_or(Value::Null); + let Some(id) = id else { + if method == "notifications/initialized" { + self.initialized = true; + } + return None; + }; + Some(match self.request(method, ¶ms) { + Ok(r) => response(id, r), + Err((code, m)) => error(id, code, m), + }) + } + + fn request(&mut self, method: &str, params: &Value) -> Result { + match method { + "initialize" => { + let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(PROTOCOL_VERSION); + let version = if SUPPORTED_VERSIONS.contains(&asked) { asked } else { PROTOCOL_VERSION }; + let Info { name, title, version: server_version, .. } = self.info; + Ok(json!({ + "protocolVersion": version, + "capabilities": self.capabilities(), + "serverInfo": {"name": name, "title": title, "version": server_version}, + "instructions": self.tools.instructions(), + })) + } + "ping" => Ok(json!({})), + "tools/list" => Ok(json!({"tools": self.tools.list()})), + "tools/call" => { + let name = params.get("name").and_then(Value::as_str).ok_or((INVALID_PARAMS, "missing tool `name`".to_string()))?; + let args = match params.get("arguments") { + None | Some(Value::Null) => json!({}), + Some(a) => a.clone(), + }; + check_arguments(&self.tools.list(), name, &args).map_err(|e| (INVALID_PARAMS, e))?; + Ok(self.tools.call(name, &args)) + } + "resources/list" => Ok(json!({"resources": self.tools.resources()})), + "resources/templates/list" => Ok(json!({"resourceTemplates": self.tools.resource_templates()})), + "prompts/list" => Ok(json!({"prompts": self.tools.prompts()})), + "resources/read" => { + let uri = params.get("uri").and_then(Value::as_str).unwrap_or(""); + let contents = self + .tools + .read_resource(uri) + .ok_or_else(|| (RESOURCE_NOT_FOUND, format!("unknown resource `{uri}`")))? + .map_err(|e| (INTERNAL_ERROR, e))?; + Ok(json!({"contents": [contents.block(uri)]})) + } + "prompts/get" if !self.tools.prompts().is_empty() => { + let name = params.get("name").and_then(Value::as_str).ok_or((INVALID_PARAMS, "missing prompt `name`".to_string()))?; + let args = match params.get("arguments") { + None | Some(Value::Null) => json!({}), + Some(a) => a.clone(), + }; + check_prompt_arguments(&self.tools.prompts(), name, &args).map_err(|e| (INVALID_PARAMS, e))?; + self.tools.get_prompt(name, &args).map_err(|e| (INTERNAL_ERROR, e)) + } + "completion/complete" if self.completes() => { + let reference = params.get("ref").unwrap_or(&Value::Null); + let reference = match (reference.get("type").and_then(Value::as_str), reference.get("name"), reference.get("uri")) { + (Some("ref/prompt"), Some(Value::String(name)), _) => { + if !self.tools.prompts().iter().any(|p| p["name"] == name.as_str()) { + return Err((INVALID_PARAMS, format!("unknown prompt `{name}`"))); + } + Reference::Prompt(name) + } + (Some("ref/resource"), _, Some(Value::String(uri))) => Reference::Resource(uri), + _ => { + return Err(( + INVALID_PARAMS, + "`ref` must be a prompt's {type: ref/prompt, name} or a resource's {type: ref/resource, uri}".into(), + )); + } + }; + let argument = params.get("argument"); + let (Some(argument), Some(value)) = + (argument.and_then(|a| a.get("name")).and_then(Value::as_str), argument.and_then(|a| a.get("value")).and_then(Value::as_str)) + else { + return Err((INVALID_PARAMS, "`argument` must have a `name` and a `value`, both strings".into())); + }; + let mut values = self.tools.complete(reference, argument, value); + let total = values.len(); + values.truncate(MAX_COMPLETIONS); + Ok(json!({"completion": {"values": values, "total": total, "hasMore": total > MAX_COMPLETIONS}})) + } + _ => Err((METHOD_NOT_FOUND, format!("method `{method}` not found"))), + } + } + + /// What the server tells a client at `initialize` that it serves: the tools always, and the + /// rest only when the app has some. + fn capabilities(&self) -> Value { + let mut capabilities = json!({"tools": {"listChanged": false}}); + if !self.tools.resources().is_empty() || !self.tools.resource_templates().is_empty() { + capabilities["resources"] = json!({"listChanged": false}); + } + if !self.tools.prompts().is_empty() { + capabilities["prompts"] = json!({"listChanged": false}); + } + if self.completes() { + capabilities["completions"] = json!({}); + } + capabilities + } + + /// Whether the app has arguments to complete: those of its prompts and resource templates. + fn completes(&self) -> bool { + !self.tools.prompts().is_empty() || !self.tools.resource_templates().is_empty() + } +} diff --git a/crates/mcp-server/src/tests.rs b/crates/mcp-server/src/tests.rs new file mode 100644 index 000000000..b687ee2e8 --- /dev/null +++ b/crates/mcp-server/src/tests.rs @@ -0,0 +1,368 @@ +use serde_json::{Value, json}; + +use crate::{ + Contents, Effect, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result, tool, +}; + +/// One app's tools: `echo` and `level`, nothing behind them (no engine, no design). `log` records +/// every listing and call in order. +#[derive(Default)] +struct Echo { + calls: Vec, + log: Vec, +} + +impl Tools for Echo { + fn instructions(&self) -> String { + "Echo test tools.".into() + } + fn list(&mut self) -> Vec { + self.log.push("list".into()); + vec![ + tool("echo", "Echo", "Say `text` back.", object(json!({"text": string("What to say")}), &["text"]), Effect::ReadOnly), + tool("level", "Level", "Set the level.", object(json!({"level": {"type": "string", "enum": ["low", "high"]}}), &[]), Effect::Edits), + ] + } + fn call(&mut self, name: &str, args: &Value) -> Value { + self.log.push(format!("call {name}")); + self.calls.push(json!({"name": name, "args": args})); + text_result(args["text"].as_str().unwrap_or_default(), false) + } +} + +fn info() -> Info { + Info { name: "echo", title: "Echo", version: "1.0.0", max_line: 1 << 16 } +} + +fn serve(lines: &[Value]) -> (Server, Vec) { + serve_with(Echo::default(), lines) +} + +fn serve_with(tools: T, lines: &[Value]) -> (Server, Vec) { + let input: String = lines.iter().map(|l| format!("{l}\n")).collect(); + let mut output = Vec::new(); + let mut s = Server::new(info(), tools); + s.serve(std::io::Cursor::new(input), &mut output).unwrap(); + let replies = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + (s, replies) +} + +/// A registered tool is listed and called over the stdio transport, with no engine behind it. +#[test] +fn a_registered_tool_is_served_over_stdio() { + let (mut s, replies) = serve(&[ + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-03-26"}}), + json!({"jsonrpc": "2.0", "method": "notifications/initialized"}), + json!({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}), + json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "hi"}}}), + ]); + assert_eq!(replies.len(), 3, "the notification gets no reply: {replies:?}"); + let init = &replies[0]["result"]; + assert_eq!(init["protocolVersion"], "2025-03-26"); + assert_eq!(init["serverInfo"], json!({"name": "echo", "title": "Echo", "version": "1.0.0"})); + assert_eq!(init["instructions"], "Echo test tools."); + assert!(s.is_initialized()); + assert_eq!(replies[1]["result"]["tools"][0]["name"], "echo"); + assert_eq!(replies[1]["result"]["tools"][0]["annotations"]["readOnlyHint"], true); + assert_eq!(replies[2], json!({"jsonrpc": "2.0", "id": 3, "result": {"content": [{"type": "text", "text": "hi"}]}})); + assert_eq!(s.tools().calls, vec![json!({"name": "echo", "args": {"text": "hi"}})]); +} + +/// Arguments are checked against the tool's schema before the tool runs; unknown tools and +/// methods are JSON-RPC errors. +#[test] +fn arguments_break_the_schema_before_the_tool_runs() { + let (mut s, replies) = serve(&[ + json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "echo", "arguments": {}}}), + json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": 3}}}), + json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "a", "loud": true}}}), + json!({"jsonrpc": "2.0", "id": 4, "method": "tools/call", "params": {"name": "shout", "arguments": {}}}), + json!({"jsonrpc": "2.0", "id": 5, "method": "nope"}), + json!({"jsonrpc": "2.0", "id": 6, "method": "resources/read", "params": {"uri": "x://y"}}), + ]); + let codes: Vec<&Value> = replies.iter().map(|r| &r["error"]["code"]).collect(); + assert_eq!(codes, [&json!(-32602), &json!(-32602), &json!(-32602), &json!(-32602), &json!(-32601), &json!(-32002)], "{replies:?}"); + assert_eq!(replies[0]["error"]["message"], "echo: missing required argument `text`"); + assert_eq!(replies[2]["error"]["message"], "echo: unknown argument `loud`; its arguments are: text"); + assert!(s.tools().calls.is_empty()); + let defs = s.tools().list(); + assert!(check_arguments(&defs, "echo", &json!({"text": "a"})).is_ok()); +} + +/// Hostile framing: oversized lines, bad UTF-8 and bad JSON are answered and the server goes on. +#[test] +fn hostile_lines_are_answered_and_serving_goes_on() { + let mut input = vec![b'x'; (1 << 16) + 10]; + input.push(b'\n'); + input.extend_from_slice(b"\xff\xfe\n{not json\n[]\n"); + input.extend_from_slice(json!({"jsonrpc": "2.0", "id": 9, "method": "ping"}).to_string().as_bytes()); + let mut output = Vec::new(); + Server::new(info(), Echo::default()).serve(std::io::Cursor::new(input), &mut output).unwrap(); + let replies: Vec = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + let codes: Vec<&Value> = replies.iter().map(|r| r.get("error").map_or(&r["result"], |e| &e["code"])).collect(); + assert_eq!(codes, [&json!(-32600), &json!(-32700), &json!(-32700), &json!(-32600), &json!({})], "{replies:?}"); +} + +#[test] +fn base64_matches_rfc4648() { + for (i, o) in [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] { + assert_eq!(crate::base64(i.as_bytes()), o); + } + let r = crate::image_result(b"foo", "image/png", "info"); + assert_eq!(r, json!({"content": [{"type": "image", "data": "Zm9v", "mimeType": "image/png"}, {"type": "text", "text": "info"}]})); +} + +fn ping(id: usize) -> Value { + json!({"jsonrpc": "2.0", "id": id, "method": "ping"}) +} + +fn notification() -> Value { + json!({"jsonrpc": "2.0", "method": "notifications/progress"}) +} + +fn reply(s: &mut Server, msg: &Value) -> Option { + s.handle_line(&msg.to_string()).map(|r| serde_json::from_str(&r).unwrap()) +} + +/// A batch answers each request in it and none of its notifications; a batch of notifications +/// only gets no reply at all; a batch holds at most 1000 messages. +#[test] +fn batches_answer_their_requests_up_to_the_limit() { + let mut s = Server::new(info(), Echo::default()); + let mixed = + json!([ping(1), notification(), {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "b"}}}]); + let replies = reply(&mut s, &mixed).unwrap(); + assert_eq!( + replies, + json!([{"jsonrpc": "2.0", "id": 1, "result": {}}, {"jsonrpc": "2.0", "id": 2, "result": {"content": [{"type": "text", "text": "b"}]}}]) + ); + assert_eq!(reply(&mut s, &json!([notification(), notification()])), None); + let full: Vec = (0..1000).map(ping).collect(); + assert_eq!(reply(&mut s, &Value::Array(full)).unwrap().as_array().map(Vec::len), Some(1000)); + let over: Vec = (0..1001).map(ping).collect(); + let r = reply(&mut s, &Value::Array(over)).unwrap(); + assert_eq!(r, json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "batch too large"}})); +} + +/// We send no requests, so a response is never answered, alone or in a batch; a message with +/// neither a method nor a result, or one that is not an object, is refused; a blank line is skipped. +#[test] +fn a_response_we_never_asked_for_is_ignored() { + let mut s = Server::new(info(), Echo::default()); + assert_eq!(reply(&mut s, &json!({"jsonrpc": "2.0", "id": 1, "result": {}})), None); + assert_eq!(reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "error": {"code": -1, "message": "no"}})), None); + assert_eq!( + reply(&mut s, &json!([{"jsonrpc": "2.0", "id": 3, "result": {}}, ping(4)])).unwrap(), + json!([{"jsonrpc": "2.0", "id": 4, "result": {}}]) + ); + let r = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 5})).unwrap(); + assert_eq!(r, json!({"jsonrpc": "2.0", "id": 5, "error": {"code": -32600, "message": "missing `method`"}})); + let r = reply(&mut s, &json!(42)).unwrap(); + assert_eq!(r, json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "message must be an object"}})); + assert_eq!(s.handle_line(" "), None); + assert!(s.tools().log.is_empty()); +} + +/// A version we speak is echoed; one we don't, or none, gets ours. +#[test] +fn an_unknown_protocol_version_gets_ours() { + let mut s = Server::new(info(), Echo::default()); + let version = |s: &mut Server, params: Value| { + reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": params})).unwrap()["result"]["protocolVersion"].clone() + }; + for v in SUPPORTED_VERSIONS { + assert_eq!(version(&mut s, json!({"protocolVersion": v})), json!(v)); + } + assert_eq!(version(&mut s, json!({"protocolVersion": "1999-01-01"})), json!(PROTOCOL_VERSION)); + assert_eq!(version(&mut s, json!({"protocolVersion": 7})), json!(PROTOCOL_VERSION)); + assert_eq!(version(&mut s, json!({})), json!(PROTOCOL_VERSION)); + assert!(!s.is_initialized(), "initialized only by the client's notification"); +} + +/// `arguments: null` is no arguments; a value outside a property's `enum` is refused before the +/// tool runs. +#[test] +fn null_arguments_are_none_and_enums_are_checked() { + let mut s = Server::new(info(), Echo::default()); + let call = |s: &mut Server, id: u64, args: Value| { + reply(s, &json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": "level", "arguments": args}})).unwrap() + }; + assert!(call(&mut s, 1, Value::Null).get("result").is_some()); + let none = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "level"}})).unwrap(); + assert!(none.get("result").is_some(), "{none}"); + assert!(call(&mut s, 3, json!({"level": "high"})).get("result").is_some()); + let r = call(&mut s, 4, json!({"level": "loud"})); + assert_eq!(r["error"], json!({"code": -32602, "message": "level: `level` must be one of [\"low\",\"high\"]"})); + let r = call(&mut s, 5, json!([])); + assert_eq!(r["error"], json!({"code": -32602, "message": "level: arguments must be an object"})); + let args: Vec<&Value> = s.tools().calls.iter().map(|c| &c["args"]).collect(); + assert_eq!(args, [&json!({}), &json!({}), &json!({"level": "high"})]); +} + +/// An app that serves no resources, templates or prompts lists none of each, says it serves only +/// tools, and has no prompt to fill in or argument to complete. +#[test] +fn the_default_lists_are_empty() { + let mut s = Server::new(info(), Echo::default()); + let result = |s: &mut Server, method: &str| reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": method})).unwrap()["result"].clone(); + assert_eq!(result(&mut s, "resources/list"), json!({"resources": []})); + assert_eq!(result(&mut s, "resources/templates/list"), json!({"resourceTemplates": []})); + assert_eq!(result(&mut s, "prompts/list"), json!({"prompts": []})); + assert_eq!(result(&mut s, "initialize")["capabilities"], json!({"tools": {"listChanged": false}})); + for method in ["prompts/get", "completion/complete"] { + let r = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "method": method, "params": {"name": "describe"}})).unwrap(); + assert_eq!(r["error"]["code"], -32601, "{method}: {r}"); + } +} + +/// An app that serves more than tools: a prompt with a required and an optional argument, a +/// resource template, a Markdown resource and a PNG, and suggestions for any argument. `asked` +/// records every prompt filled in and every completion asked for. +#[derive(Default)] +struct Library { + asked: Vec, +} + +impl Tools for Library { + fn instructions(&self) -> String { + "Library test.".into() + } + fn list(&mut self) -> Vec { + Vec::new() + } + fn call(&mut self, name: &str, _: &Value) -> Value { + text_result(format!("no tool `{name}`"), true) + } + fn resources(&self) -> Vec { + vec![ + json!({"uri": "lib://readme", "name": "Readme", "mimeType": "text/markdown"}), + json!({"uri": "lib://logo", "name": "Logo", "mimeType": "image/png"}), + ] + } + fn resource_templates(&self) -> Vec { + vec![json!({"uriTemplate": "lib://parts/{name}", "name": "Part", "mimeType": "text/markdown"})] + } + fn read_resource(&mut self, uri: &str) -> Option> { + let markdown = |text: String| Contents::Text { mime_type: "text/markdown".into(), text }; + Some(Ok(match uri { + "lib://readme" => markdown("# Parts\n\nA \"quoted\" line.".into()), + "lib://logo" => Contents::Blob { mime_type: "image/png".into(), data: b"foo".to_vec() }, + "lib://parts/broken" => return Some(Err("the part is being rebuilt".into())), + _ => markdown(format!("# {}", uri.strip_prefix("lib://parts/")?)), + })) + } + fn prompts(&self) -> Vec { + vec![json!({"name": "describe", "title": "Describe a part", "arguments": [{"name": "part", "required": true}, {"name": "tone"}]})] + } + fn get_prompt(&mut self, name: &str, args: &Value) -> Result { + self.asked.push(format!("{name} {args}")); + let text = format!("Describe {} in a {} tone.", args["part"].as_str().unwrap_or_default(), args["tone"].as_str().unwrap_or("plain")); + Ok(json!({"messages": [{"role": "user", "content": {"type": "text", "text": text}}]})) + } + fn complete(&mut self, reference: Reference<'_>, argument: &str, value: &str) -> Vec { + self.asked.push(format!("{reference:?} {argument}={value}")); + (0..150).map(|i| format!("{value}{i}")).collect() + } +} + +/// The shared crate serves an app's prompt, resource template and Markdown and binary resources, +/// and tells the client at `initialize` that it does (#13336). +#[test] +fn an_app_serves_prompts_templates_and_markdown_or_binary_resources() { + let request = |id: u64, method: &str, params: Value| json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}); + let (mut s, replies) = serve_with( + Library::default(), + &[ + request(1, "initialize", json!({"protocolVersion": PROTOCOL_VERSION})), + request(2, "prompts/list", json!({})), + request(3, "prompts/get", json!({"name": "describe", "arguments": {"part": "bracket"}})), + request(4, "resources/templates/list", json!({})), + request(5, "resources/read", json!({"uri": "lib://readme"})), + request(6, "resources/read", json!({"uri": "lib://logo"})), + request(7, "resources/read", json!({"uri": "lib://parts/bracket"})), + request(8, "resources/read", json!({"uri": "lib://parts/broken"})), + request(9, "resources/read", json!({"uri": "lib://nope"})), + ], + ); + let result = |i: usize| &replies[i]["result"]; + assert_eq!( + result(0)["capabilities"], + json!({"tools": {"listChanged": false}, "resources": {"listChanged": false}, "prompts": {"listChanged": false}, "completions": {}}) + ); + assert_eq!(result(1)["prompts"][0]["name"], "describe"); + assert_eq!(result(2), &json!({"messages": [{"role": "user", "content": {"type": "text", "text": "Describe bracket in a plain tone."}}]})); + assert_eq!(result(3), &json!({"resourceTemplates": [{"uriTemplate": "lib://parts/{name}", "name": "Part", "mimeType": "text/markdown"}]})); + // The Markdown goes out as it is, not as a JSON string of it. + assert_eq!(result(4), &json!({"contents": [{"uri": "lib://readme", "mimeType": "text/markdown", "text": "# Parts\n\nA \"quoted\" line."}]})); + assert_eq!(result(5), &json!({"contents": [{"uri": "lib://logo", "mimeType": "image/png", "blob": "Zm9v"}]})); + assert_eq!(result(6), &json!({"contents": [{"uri": "lib://parts/bracket", "mimeType": "text/markdown", "text": "# bracket"}]})); + assert_eq!(replies[7]["error"], json!({"code": -32603, "message": "the part is being rebuilt"})); + assert_eq!(replies[8]["error"]["code"], -32002); + assert_eq!(s.tools().asked, [r#"describe {"part":"bracket"}"#]); +} + +/// A prompt is filled in only with the arguments its listing declares, each a string, the +/// required ones present; anything else is invalid params and the app is never asked. +#[test] +fn prompt_arguments_are_checked_before_the_prompt_is_filled_in() { + let mut s = Server::new(info(), Library::default()); + let get = + |s: &mut Server, params: Value| reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "prompts/get", "params": params})).unwrap(); + for (params, message) in [ + (json!({"arguments": {"part": "a"}}), "missing prompt `name`"), + (json!({"name": "summarise"}), "unknown prompt `summarise`"), + (json!({"name": "describe"}), "describe: missing required argument `part`"), + (json!({"name": "describe", "arguments": ["a"]}), "describe: arguments must be an object"), + (json!({"name": "describe", "arguments": {"part": 3}}), "describe: argument `part` must be a string"), + (json!({"name": "describe", "arguments": {"part": null}}), "describe: argument `part` must be a string"), + (json!({"name": "describe", "arguments": {"part": "a", "size": "big"}}), "describe: unknown argument `size`; its arguments are: part, tone"), + ] { + assert_eq!(get(&mut s, params.clone())["error"], json!({"code": -32602, "message": message}), "{params}"); + } + assert!(s.tools().asked.is_empty()); + let filled = get(&mut s, json!({"name": "describe", "arguments": {"part": "hinge", "tone": "dry"}})); + assert_eq!(filled["result"]["messages"][0]["content"]["text"], "Describe hinge in a dry tone."); +} + +/// Completion asks the app about a listed prompt or a resource, and sends the first 100 +/// suggestions with how many there are; a bad reference or argument is invalid params. +#[test] +fn completion_suggests_up_to_a_hundred_values() { + let mut s = Server::new(info(), Library::default()); + let complete = |s: &mut Server, params: Value| { + reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "completion/complete", "params": params})).unwrap() + }; + let r = complete(&mut s, json!({"ref": {"type": "ref/prompt", "name": "describe"}, "argument": {"name": "part", "value": "br"}})); + let completion = &r["result"]["completion"]; + assert_eq!(completion["values"].as_array().map(Vec::len), Some(100), "{r}"); + assert_eq!((&completion["values"][0], &completion["total"], &completion["hasMore"]), (&json!("br0"), &json!(150), &json!(true))); + let r = complete(&mut s, json!({"ref": {"type": "ref/resource", "uri": "lib://parts/{name}"}, "argument": {"name": "name", "value": ""}})); + assert_eq!(r["result"]["completion"]["values"][99], "99"); + assert_eq!(s.tools().asked, [r#"Prompt("describe") part=br"#, r#"Resource("lib://parts/{name}") name="#]); + let bad_ref = "`ref` must be a prompt's {type: ref/prompt, name} or a resource's {type: ref/resource, uri}"; + let bad_argument = "`argument` must have a `name` and a `value`, both strings"; + for (params, message) in [ + (json!({"ref": {"type": "ref/prompt", "name": "summarise"}, "argument": {"name": "a", "value": ""}}), "unknown prompt `summarise`"), + (json!({"ref": {"type": "ref/tool", "name": "x"}, "argument": {"name": "a", "value": ""}}), bad_ref), + (json!({"ref": {"type": "ref/resource"}, "argument": {"name": "a", "value": ""}}), bad_ref), + (json!({"argument": {"name": "a", "value": ""}}), bad_ref), + (json!({"ref": {"type": "ref/prompt", "name": "describe"}, "argument": {"name": "part"}}), bad_argument), + (json!({"ref": {"type": "ref/prompt", "name": "describe"}}), bad_argument), + ] { + assert_eq!(complete(&mut s, params.clone())["error"], json!({"code": -32602, "message": message}), "{params}"); + } + assert_eq!(s.tools().asked.len(), 2, "a refused completion never reaches the app"); +} + +/// `Tools::call` runs right after the listing its arguments were checked against, every time, +/// whatever was listed before it; a refused call is listed and never run. +#[test] +fn every_call_runs_right_after_its_listing() { + let mut s = Server::new(info(), Echo::default()); + let call = |name: &str, args: Value| json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": name, "arguments": args}}); + reply(&mut s, &json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list"})); + reply(&mut s, &call("echo", json!({"text": "a"}))); + reply(&mut s, &call("echo", json!({}))); + reply(&mut s, &json!([call("level", json!({})), call("echo", json!({"text": "b"}))])); + assert_eq!(s.tools().log, ["list", "list", "call echo", "list", "list", "call level", "list", "call echo"]); +} diff --git a/crates/print/src/cups/mod.rs b/crates/print/src/cups/mod.rs index 1a0a9360b..d7bcd478e 100644 --- a/crates/print/src/cups/mod.rs +++ b/crates/print/src/cups/mod.rs @@ -33,6 +33,8 @@ const CUPS_SOCKET: &str = "/run/cups/cups.sock"; const MAX_BODY: usize = 1 << 20; const MAX_HEADERS: usize = 8 << 10; const MAX_PRINTERS: usize = 256; +// Even opted-in fleet discovery has a fixed record ceiling in addition to MAX_BODY. +const MAX_DISCOVERY_QUEUES: usize = 65_536; const MAX_NAME: usize = 256; /// A job's title on the wire; IPP values are u16-length, and no real title is this long. const MAX_JOB_NAME: usize = 255; @@ -338,9 +340,24 @@ pub fn destinations_at(endpoints: &[Server]) -> Result, Error> { /// explicitly choose whether an optional class/default query failure should hide otherwise /// usable printer queues. No error is swallowed here except CUPS's absent/unsupported list. pub fn queues_at(endpoints: &[Server], is_class: bool) -> Result, Error> { + queues_at_with_limit(endpoints, is_class, MAX_PRINTERS) +} + +/// List queues or classes with an explicit fleet-size budget (1 through 65,536). +/// +/// The existing [`queues_at`] API keeps its 256-queue default. Increasing this budget does +/// not increase the one-MiB response, HTTP header, name-length or exchange-time limits. +/// An excessive response is refused in full, never silently truncated. This changes only +/// local parsing: it does not invent pagination or retry a possibly submitted print job. +/// +/// # Errors +/// Returns an error for an invalid budget, an over-budget or malformed response, or a +/// transport/server error, with the same absent-list handling as [`queues_at`]. +pub fn queues_at_with_limit(endpoints: &[Server], is_class: bool, max_queues: usize) -> Result, Error> { + check_queue_limit(max_queues)?; let operation = if is_class { 0x4005 } else { OPERATION_CUPS_GET_PRINTERS }; let reply = post(endpoints, "/", &discovery_request(operation), DISCOVERY_TIMEOUT, true)?; - let parsed = parse_reply(&reply)?; + let parsed = parse_reply_with_limit(&reply, max_queues)?; if !status_ok(parsed.status) { if parsed.status == 0x0406 || (is_class && parsed.status == 0x0501) { return Ok(Vec::new()); @@ -834,7 +851,22 @@ impl<'a> Reader<'a> { } } +fn check_queue_limit(max_queues: usize) -> Result<(), Error> { + if !(1..=MAX_DISCOVERY_QUEUES).contains(&max_queues) { + return Err(Error::Spool("CUPS discovery queue limit must be 1 through 65536".into())); + } + Ok(()) +} + fn parse_reply(reply: &[u8]) -> Result { + parse_reply_with_limit(reply, MAX_PRINTERS) +} + +fn parse_reply_with_limit(reply: &[u8], max_queues: usize) -> Result { + check_queue_limit(max_queues)?; + if reply.len() > MAX_BODY { + return Err(Error::Spool("CUPS reply exceeds limit".into())); + } let malformed = || Error::Spool("the print server sent a malformed reply".into()); let mut r = Reader { bytes: reply, pos: 0 }; let _version = r.array::<2>().ok_or_else(malformed)?; @@ -854,7 +886,7 @@ fn parse_reply(reply: &[u8]) -> Result { } GROUP_OPERATION | GROUP_JOB | GROUP_PRINTER | 0x05 => { if !queue.name.is_empty() { - if queues.len() >= MAX_PRINTERS { + if queues.len() >= max_queues { return Err(malformed()); } queues.push(std::mem::take(&mut queue)); @@ -879,7 +911,7 @@ fn parse_reply(reply: &[u8]) -> Result { match (group, name, tag) { (GROUP_PRINTER, "printer-name", TAG_NAME) => { if let Some(printer) = std::str::from_utf8(value).ok().filter(|n| !n.is_empty() && n.len() <= MAX_NAME) { - if names.len() >= MAX_PRINTERS { + if names.len() >= max_queues { return Err(Error::Spool("too many CUPS destinations".into())); } if !queue.name.is_empty() { diff --git a/crates/print/src/cups/tests.rs b/crates/print/src/cups/tests.rs index 978246e56..252e1bba8 100644 --- a/crates/print/src/cups/tests.rs +++ b/crates/print/src/cups/tests.rs @@ -591,3 +591,37 @@ fn stalled_unix_peer_still_fails_within_the_absolute_read_deadline() { assert!(matches!(error.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock)); assert!(start.elapsed() < Duration::from_secs(3)); } + +#[test] +fn fleet_discovery_opt_in_preserves_all_257_queues_and_default_limit() { + let mut body = vec![1, 1, 0, 0, 0, 0, 0, 1]; + for index in 0..257 { + body.push(4); + body.extend(printer_name(&format!("Printer{index}"))); + } + body.push(3); + assert!(parse_reply(&body).is_err()); + assert_eq!(parse_reply_with_limit(&body, 257).unwrap().queues.len(), 257); + assert!(parse_reply_with_limit(&body, 256).is_err()); + for is_class in [false, true] { + let server = FakeCups::on_tcp(vec![http_reply(&body)]).unwrap(); + let queues = queues_at_with_limit(server.endpoint(), is_class, 257).unwrap(); + assert_eq!(queues.len(), 257); + assert_eq!(queues[256].name, "Printer256"); + assert!(queues.iter().all(|q| q.is_class == is_class)); + let request = server.request().unwrap(); + assert!(has(&request, b"requested-attributes")); + } +} + +#[test] +fn fleet_discovery_keeps_absolute_resource_limits() { + for limit in [0, MAX_DISCOVERY_QUEUES + 1, usize::MAX] { + assert!(queues_at_with_limit(&[], false, limit).is_err()); + } + assert!(parse_reply_with_limit(&vec![0; MAX_BODY + 1], MAX_DISCOVERY_QUEUES).is_err()); + let mut body = vec![1, 1, 0, 0, 0, 0, 0, 1, 4]; + body.extend(attr(0x42, &"x".repeat(MAX_NAME + 1), b"value")); + body.push(3); + assert!(parse_reply_with_limit(&body, MAX_DISCOVERY_QUEUES).is_err()); +} diff --git a/crates/raw/src/dng.rs b/crates/raw/src/dng.rs index 13bf252f3..efa1ec4f5 100644 --- a/crates/raw/src/dng.rs +++ b/crates/raw/src/dng.rs @@ -500,6 +500,12 @@ const FRAME_RATE: u16 = 51044; /// CinemaDNG `ReelName` (tag 51081). const REEL_NAME: u16 = 51081; +/// A CinemaDNG `FrameRate` of `n / d` frames a second between 1/1000 and 1000 (shared with the +/// bounded `dng_metadata` reader). +pub(crate) fn plausible_frame_rate(n: i64, d: i64) -> bool { + n > 0 && d > 0 && n <= 1000 * d && n * 1000 >= d +} + /// The CinemaDNG tags of a frame, from IFD 0 (where the specification puts them), else the raw IFD. A frame rate /// outside 1/1000 to 1000 frames a second and a time code that is not valid BCD are ignored. fn cinema_tags(ifd0: &Ifd, raw: &Ifd) -> CinemaTags { @@ -510,7 +516,7 @@ fn cinema_tags(ifd0: &Ifd, raw: &Ifd) -> CinemaTags { _ => None, }); CinemaTags { - frame_rate: frame_rate.filter(|&(n, d)| n > 0 && d > 0 && n <= 1000 * d && n * 1000 >= d), + frame_rate: frame_rate.filter(|&(n, d)| plausible_frame_rate(n, d)), timecode: pick(TIME_CODES).bytes(TIME_CODES).and_then(Timecode::from_bytes), reel_name: pick(REEL_NAME).string(REEL_NAME).map(|s| s.trim().to_owned()).filter(|s| !s.is_empty()), } diff --git a/crates/raw/src/dng_metadata.rs b/crates/raw/src/dng_metadata.rs new file mode 100644 index 000000000..1d3a02b16 --- /dev/null +++ b/crates/raw/src/dng_metadata.rs @@ -0,0 +1,259 @@ +//! Bounded DNG processing and camera metadata, before application rendering. +//! +//! This is the [`crate::dng_lines`] companion on the same `craft_tiff::classic` IFDs. The full +//! reader's `dng::color_data` works on `craft_tiff::Tiff` and returns the values it will render +//! with (raw IFD first, implausible or singular values dropped, `BaselineExposureOffset` added). +//! This reader returns the tags as written, IFD 0 first, and fails on a value outside the file, +//! so a damaged file does not silently lose its white balance. For a well-formed file both read the +//! same matrices and neutral (see `tests/it/dng_metadata.rs`). +use crate::dng_lines::{self, Error, PHOTOMETRIC_CFA, Result}; +use craft_tiff::classic::{Entry, Ifd, Tiff, tags}; +type M3 = [[f64; 3]; 3]; +type V3 = [f64; 3]; +/// Camera-plane colour filter pattern. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Pattern { + pub rows: usize, + pub cols: usize, + pub colors: Vec, +} +/// Parsed image processing tags. +#[derive(Clone, Debug, Default)] +pub struct Params { + pub active_area: Option<[usize; 4]>, + pub linearization: Option>, + pub black_repeat: (usize, usize), + pub black: Vec, + pub black_delta_h: Vec, + pub black_delta_v: Vec, + pub white: Vec, + pub crop_origin: Option<(f64, f64)>, + pub crop_size: Option<(f64, f64)>, + pub cfa: Option, +} +/// Parsed camera colour calibration. +#[derive(Clone, Debug, Default)] +pub struct Calibration { + pub color_matrix: [Option; 2], + pub camera_calibration: [Option; 2], + pub forward_matrix: [Option; 2], + pub illuminant: [Option; 2], + pub analog_balance: Option, + pub as_shot_neutral: Option, + pub as_shot_white_xy: Option<(f64, f64)>, + pub baseline_exposure: f64, +} +/// SMPTE ST 12-1 time code of a CinemaDNG frame: the crate's [`crate::Timecode`], which the full +/// reader's `CinemaTags` uses too. +pub use crate::Timecode; + +/// File-level capture fields, without image geometry or rendering policy. +#[derive(Clone, Debug, PartialEq)] +pub struct CaptureInfo { + pub dng_version: [u8; 4], + pub frame_rate: Option<(i64, i64)>, + pub timecode: Option, + pub camera: Option, + pub reel: Option, +} +/// Metadata reader with the file's main and raw IFDs. +pub struct Metadata<'a, 'i> { + pub tiff: Tiff<'a>, + pub main: &'i Ifd, + pub raw: &'i Ifd, +} +impl Metadata<'_, '_> { + /// A raw-image tag: from the raw IFD, else IFD 0. + fn raw_tag(&self, tag: u16) -> Option<&Entry> { + self.raw.get(tag).or_else(|| self.main.get(tag)) + } + + /// A camera-wide tag: from IFD 0, else the raw IFD. + fn main_tag(&self, tag: u16) -> Option<&Entry> { + self.main.get(tag).or_else(|| self.raw.get(tag)) + } + + /// A numeric tag's values; a value outside the file (or a zero denominator) is an error, so a + /// damaged file does not silently lose its white balance. + fn floats(&self, e: Option<&Entry>) -> Result>> { + Ok(e.map(|e| self.tiff.floats(e)).transpose()?) + } + + /// A 3 × 3 matrix tag (`None` when absent or not 3 × 3, e.g. for other colour counts). + fn matrix(&self, tag: u16) -> Result> { + let Some(v) = self.floats(self.main_tag(tag))? else { return Ok(None) }; + let Ok(v) = <[f64; 9]>::try_from(v.as_slice()) else { return Ok(None) }; + Ok(Some([[v[0], v[1], v[2]], [v[3], v[4], v[5]], [v[6], v[7], v[8]]])) + } + + fn vec3(&self, tag: u16) -> Result> { + Ok(self.floats(self.main_tag(tag))?.and_then(|v| <[f64; 3]>::try_from(v.as_slice()).ok())) + } + + /// The CFA pattern in camera planes (`None` for LinearRaw). + pub fn cfa(&self) -> Result> { + let l = dng_lines::layout(&self.tiff, self.raw)?; + if l.photometric != PHOTOMETRIC_CFA { + return Ok(None); + } + let dims = self.raw_tag(tags::CFA_REPEAT_PATTERN_DIM).map(|e| self.tiff.uints(e)).transpose()?.unwrap_or_else(|| vec![2, 2]); + let (rows, cols) = match dims[..] { + [r, c, ..] => (r as usize, c as usize), + _ => return Err(Error::Malformed("CFARepeatPatternDim")), + }; + if !(1..=8).contains(&rows) || !(1..=8).contains(&cols) { + return Err(Error::Unsupported(format!("{rows} × {cols} CFA pattern"))); + } + let pattern = self.tiff.uints(self.raw_tag(tags::CFA_PATTERN).ok_or(Error::Malformed("no CFAPattern"))?)?; + if pattern.len() < rows * cols { + return Err(Error::Malformed("CFAPattern shorter than its repeat")); + } + let planes = match self.raw_tag(tags::CFA_PLANE_COLOR) { + Some(e) => self.tiff.uints(e)?, + None => vec![0, 1, 2], + }; + let mut sorted = planes.clone(); + sorted.sort_unstable(); + if sorted != [0, 1, 2] { + return Err(Error::Unsupported("CFA colours other than red, green and blue".into())); + } + if let Some(e) = self.raw_tag(tags::CFA_LAYOUT) + && self.tiff.uint(e)? != 1 + { + return Err(Error::Unsupported("a non-rectangular CFA layout".into())); + } + let colors = pattern[..rows * cols] + .iter() + .map(|&v| planes.iter().position(|&p| p == v).map(|i| i as u8).ok_or(Error::Malformed("CFAPattern colour not in CFAPlaneColor"))) + .collect::>>()?; + Ok(Some(Pattern { rows, cols, colors })) + } + + pub fn params(&self) -> Result { + let uint_pair = |tag| -> Result> { + match self.raw_tag(tag) { + Some(e) => match self.tiff.uints(e)?[..] { + [a, b, ..] => Ok(Some((a as usize, b as usize))), + _ => Err(Error::Malformed("short field")), + }, + None => Ok(None), + } + }; + let active_area = match self.raw_tag(tags::ACTIVE_AREA) { + Some(e) => match self.tiff.uints(e)?[..] { + [t, l, b, r, ..] => Some([t, l, b, r].map(|v| usize::try_from(v).unwrap_or(usize::MAX))), + _ => return Err(Error::Malformed("ActiveArea")), + }, + None => None, + }; + let linearization = match self.raw_tag(tags::LINEARIZATION_TABLE) { + Some(e) => { + // checked before `uints` allocates a u64 per entry + if e.count > 65_536 { + return Err(Error::Malformed("LinearizationTable longer than 65536")); + } + let v = self.tiff.uints(e)?; + Some(v.into_iter().map(|x| x.min(65_535) as u16).collect()) + } + None => None, + }; + let pair = |tag| -> Result> { + match self.raw_tag(tag) { + Some(e) => match self.tiff.floats(e)?[..] { + [a, b, ..] => Ok(Some((a, b))), + _ => Err(Error::Malformed("short field")), + }, + None => Ok(None), + } + }; + let list = |tag| -> Result> { Ok(self.raw_tag(tag).map(|e| self.tiff.floats(e)).transpose()?.unwrap_or_default()) }; + Ok(Params { + active_area, + linearization, + black_repeat: uint_pair(tags::BLACK_LEVEL_REPEAT_DIM)?.unwrap_or((1, 1)), + black: list(tags::BLACK_LEVEL)?, + black_delta_h: list(tags::BLACK_LEVEL_DELTA_H)?, + black_delta_v: list(tags::BLACK_LEVEL_DELTA_V)?, + white: list(tags::WHITE_LEVEL)?, + crop_origin: pair(tags::DEFAULT_CROP_ORIGIN)?, + crop_size: pair(tags::DEFAULT_CROP_SIZE)?, + cfa: self.cfa()?, + }) + } + + fn text(&self, tag: u16) -> Option { + self.main_tag(tag).and_then(|e| self.tiff.ascii(e).ok()).filter(|s| !s.is_empty()) + } + /// Read file-level CinemaDNG and camera fields. + pub fn capture_info(&self) -> CaptureInfo { + let dng_version = self + .main_tag(tags::DNG_VERSION) + .and_then(|e| self.tiff.uints(e).ok()) + .and_then(|v| v.get(..4).map(|v| [0, 1, 2, 3].map(|i| v[i] as u8))) + .unwrap_or_default(); + let frame_rate = self.main_tag(tags::FRAME_RATE).and_then(|e| { + let b = self.tiff.value_bytes(e).ok()?; + if !matches!(e.ty, craft_tiff::classic::Type::SRational | craft_tiff::classic::Type::Rational) { + return None; + } + let n = i64::from(self.tiff.u32_of(b.get(0..4)?.try_into().ok()?) as i32); + let d = i64::from(self.tiff.u32_of(b.get(4..8)?.try_into().ok()?) as i32); + crate::dng::plausible_frame_rate(n, d).then_some((n, d)) + }); + let timecode = self.main_tag(tags::TIME_CODES).and_then(|e| self.tiff.value_bytes(e).ok()).and_then(Timecode::from_bytes); + let camera = self.text(tags::UNIQUE_CAMERA_MODEL).or_else(|| { + let mm = [self.text(tags::MAKE), self.text(tags::MODEL)].into_iter().flatten().collect::>().join(" "); + (!mm.is_empty()).then_some(mm) + }); + CaptureInfo { dng_version, frame_rate, timecode, camera, reel: self.text(tags::REEL_NAME) } + } + + /// The colour tags. + pub fn calibration(&self) -> Result { + let illum = |tag| self.main_tag(tag).map(|e| self.tiff.uint(e)).transpose(); + let white_xy = self.floats(self.main_tag(tags::AS_SHOT_WHITE_XY))?.and_then(|v| match v[..] { + [x, y] => Some((x, y)), + _ => None, + }); + Ok(Calibration { + color_matrix: [self.matrix(tags::COLOR_MATRIX_1)?, self.matrix(tags::COLOR_MATRIX_2)?], + camera_calibration: [self.matrix(tags::CAMERA_CALIBRATION_1)?, self.matrix(tags::CAMERA_CALIBRATION_2)?], + forward_matrix: [self.matrix(tags::FORWARD_MATRIX_1)?, self.matrix(tags::FORWARD_MATRIX_2)?], + illuminant: [illum(tags::CALIBRATION_ILLUMINANT_1)?, illum(tags::CALIBRATION_ILLUMINANT_2)?], + analog_balance: self.vec3(tags::ANALOG_BALANCE)?, + as_shot_neutral: self.vec3(tags::AS_SHOT_NEUTRAL)?, + as_shot_white_xy: white_xy, + baseline_exposure: self.floats(self.main_tag(tags::BASELINE_EXPOSURE))?.and_then(|v| v.first().copied()).unwrap_or(0.0), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn timecode_is_the_crate_timecode() { + let tc = Timecode { frames: 12, seconds: 34, minutes: 12, hours: 9, drop_frame: true }; + assert_eq!(Timecode::from_bytes(&[0x52, 0x34, 0x12, 0x09]), Some(tc)); + let shared: crate::Timecode = tc; + assert_eq!(Timecode::from_bytes(&shared.to_bytes()), Some(tc)); + } + + #[test] + fn calibration_defaults_and_rejects_bad_rational() -> Result<()> { + let empty = Ifd::default(); + let tiff = Tiff { data: &[], big_endian: false }; + let metadata = Metadata { tiff, main: &empty, raw: &empty }; + let calibration = metadata.calibration()?; + assert_eq!(calibration.baseline_exposure, 0.0); + assert_eq!(calibration.color_matrix, [None, None]); + + // IFD with one BaselineExposure SRational pointing past the end of the file. + let data = [b'I', b'I', 42, 0, 8, 0, 0, 0, 1, 0, 42, 198, 10, 0, 1, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0]; + let (tiff, first) = Tiff::new(&data)?; + let (ifd, _) = tiff.read_ifd(first, 0)?; + assert!(Metadata { tiff, main: &ifd, raw: &ifd }.calibration().is_err()); + Ok(()) + } +} diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index 3187ebdd2..83a1165ca 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -96,6 +96,9 @@ mod arw_lens_rm6; /// Bounded DNG strip and tile sample reader. #[cfg(feature = "line-dng")] pub mod dng_lines; +/// Bounded DNG camera and processing metadata. +#[cfg(feature = "line-dng")] +pub mod dng_metadata; #[allow(dead_code)] mod huffman; #[cfg(feature = "jxl")] @@ -119,6 +122,7 @@ mod nef; mod nef_lens; #[cfg(feature = "nef")] mod nefc; +pub mod opcode_params; /// Bounded raw DNG opcode-list envelopes. pub mod opcode_raw; pub mod opcodes; diff --git a/crates/raw/src/opcode_params.rs b/crates/raw/src/opcode_params.rs new file mode 100644 index 000000000..4118d6cef --- /dev/null +++ b/crates/raw/src/opcode_params.rs @@ -0,0 +1,418 @@ +//! Context-aware parsing of DNG opcode parameters. Image processing remains the caller's job. +//! +//! The typed [`crate::opcodes`] model is the file's own view (owned, serialisable, lenient: a +//! malformed opcode becomes `Unknown`, a zero pitch reads as 1, coordinates stay as written). This +//! module answers a different question for a caller about to apply an opcode to a given image: +//! parameters borrowed from the record (bad-pixel lists are not copied), checked strictly (a zero +//! pitch, a count larger than its bytes or a warp on a mosaic is an error) and areas clamped to the +//! image. The record envelope comes from [`crate::opcode_raw`]; the rectangle on the wire is +//! [`crate::opcodes::Area`], which [`Area::clamp`] turns into image coordinates. + +/// Image properties used to validate and clamp opcode parameters. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Context { + pub width: usize, + pub height: usize, + pub planes: usize, + pub mosaic: bool, + pub allow_trim: bool, +} +/// A parameter failure, retaining the diagnostic used by the caller. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParseError { + Malformed(&'static str), + Unsupported(&'static str), +} +/// A clamped image area and its sample pitches. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Area { + pub top: usize, + pub left: usize, + pub bottom: usize, + pub right: usize, + pub plane: usize, + pub planes: usize, + pub row_pitch: usize, + pub col_pitch: usize, +} +impl Area { + /// Iterate sampled rows, rejecting a zero pitch even for caller-constructed areas. + pub fn rows(&self) -> Result + use<>, ParseError> { + if self.row_pitch == 0 { + return Err(ParseError::Malformed("opcode area with a zero pitch")); + } + Ok((self.top..self.bottom.max(self.top)).step_by(self.row_pitch)) + } + /// Iterate sampled columns, rejecting a zero pitch even for caller-constructed areas. + pub fn cols(&self) -> Result + use<>, ParseError> { + if self.col_pitch == 0 { + return Err(ParseError::Malformed("opcode area with a zero pitch")); + } + Ok((self.left..self.right.max(self.left)).step_by(self.col_pitch)) + } +} +/// Borrowed bad-pixel coordinates, decoded lazily without copying the payload. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PointRecords<'a>(&'a [u8]); +impl PointRecords<'_> { + pub fn iter(&self) -> impl Iterator + '_ { + self.0.as_chunks::<8>().0.iter().map(|b| (u32::from_be_bytes([b[0], b[1], b[2], b[3]]), u32::from_be_bytes([b[4], b[5], b[6], b[7]]))) + } +} +/// Borrowed bad-pixel rectangles, decoded lazily without copying the payload. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct RectangleRecords<'a>(&'a [u8]); +impl RectangleRecords<'_> { + pub fn iter(&self) -> impl Iterator + '_ { + self.0.as_chunks::<16>().0.iter().map(|b| { + [ + u32::from_be_bytes([b[0], b[1], b[2], b[3]]), + u32::from_be_bytes([b[4], b[5], b[6], b[7]]), + u32::from_be_bytes([b[8], b[9], b[10], b[11]]), + u32::from_be_bytes([b[12], b[13], b[14], b[15]]), + ] + }) + } +} +/// Parameters for the fourteen defined opcodes. +#[derive(Clone, Debug, PartialEq)] +pub enum Params<'a> { + WarpRectilinear { planes: Vec<[f64; 6]>, center: [f64; 2] }, + WarpFisheye { planes: Vec<[f64; 4]>, center: [f64; 2] }, + FixVignetteRadial { k: [f64; 5], center: [f64; 2] }, + FixBadPixelsConstant { constant: u32, bayer_phase: u32 }, + FixBadPixelsList { bayer_phase: u32, points: PointRecords<'a>, rects: RectangleRecords<'a> }, + TrimBounds { top: usize, left: usize, bottom: usize, right: usize }, + MapTable { area: Area, table: Vec }, + MapPolynomial { area: Area, coefficients: Vec }, + GainMap { area: Area, points_v: usize, points_h: usize, spacing: [f64; 2], origin: [f64; 2], map_planes: usize, gains: Vec }, + DeltaPerRow { area: Area, deltas: Vec }, + DeltaPerColumn { area: Area, deltas: Vec }, + ScalePerRow { area: Area, scales: Vec }, + ScalePerColumn { area: Area, scales: Vec }, + WarpRectilinear2 { planes: Vec<[f64; 19]>, center: [f64; 2], reciprocal: bool }, +} +struct Reader<'a> { + bytes: &'a [u8], + pos: usize, +} +impl<'a> Reader<'a> { + fn take(&mut self, n: usize) -> Result<&'a [u8], ParseError> { + let end = self.pos.checked_add(n).ok_or(ParseError::Malformed("opcode parameters"))?; + let b = self.bytes.get(self.pos..end).ok_or(ParseError::Malformed("opcode parameters too short"))?; + self.pos = end; + Ok(b) + } + fn u32(&mut self) -> Result { + let b = self.take(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + fn f32(&mut self) -> Result { + let v = f32::from_bits(self.u32()?); + if v.is_finite() { Ok(v) } else { Err(ParseError::Malformed("non-finite opcode parameter")) } + } + fn f64(&mut self) -> Result { + let b = self.take(8)?; + let v = f64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]); + if v.is_finite() { Ok(v) } else { Err(ParseError::Malformed("non-finite opcode parameter")) } + } + fn remaining(&self) -> usize { + self.bytes.len().saturating_sub(self.pos) + } + fn count(&mut self, size: usize) -> Result { + let n = self.u32()? as usize; + if n.saturating_mul(size) > self.remaining() { Err(ParseError::Malformed("opcode count larger than its parameters")) } else { Ok(n) } + } + fn array(&mut self) -> Result<[f64; N], ParseError> { + let mut a = [0.0; N]; + for x in &mut a { + *x = self.f64()?; + } + Ok(a) + } + fn area(&mut self, c: Context) -> Result { + let wire = crate::opcodes::Area { + top: self.u32()?, + left: self.u32()?, + bottom: self.u32()?, + right: self.u32()?, + plane: self.u32()?, + planes: self.u32()?, + row_pitch: self.u32()?, + col_pitch: self.u32()?, + }; + Area::clamp(wire, c) + } +} +impl Area { + /// The area of an opcode as written ([`crate::opcodes::Area`]) clamped to the image of `c`: + /// coordinates to its size, the plane range to its planes. A zero pitch is an error here (the + /// typed reader reads it as 1). + pub fn clamp(wire: crate::opcodes::Area, c: Context) -> Result { + if wire.row_pitch == 0 || wire.col_pitch == 0 { + return Err(ParseError::Malformed("opcode area with a zero pitch")); + } + let px = |v: u32| v as usize; + let plane = px(wire.plane).min(c.planes); + Ok(Area { + top: px(wire.top).min(c.height), + left: px(wire.left).min(c.width), + bottom: px(wire.bottom).min(c.height), + right: px(wire.right).min(c.width), + plane, + planes: px(wire.planes).min(c.planes - plane), + row_pitch: px(wire.row_pitch), + col_pitch: px(wire.col_pitch), + }) + } +} +/// Parse a known opcode's parameters. Unknown identifiers return an unsupported error. +pub fn parse(id: u32, bytes: &[u8], context: Context) -> Result, ParseError> { + let mut r = Reader { bytes, pos: 0 }; + match id { + 1 | 2 | 14 => { + if context.mosaic { + return Err(ParseError::Unsupported("a warp opcode on a CFA mosaic")); + } + let per = match id { + 1 => 6, + 2 => 4, + _ => 19, + }; + let n = r.count(per * 8)?; + if n != 1 && n != context.planes { + return Err(ParseError::Malformed("warp opcode coefficient sets")); + } + match id { + 1 => { + let planes = (0..n).map(|_| r.array()).collect::>()?; + let center = r.array()?; + Ok(Params::WarpRectilinear { planes, center }) + } + 2 => { + let planes = (0..n).map(|_| r.array()).collect::>()?; + let center = r.array()?; + Ok(Params::WarpFisheye { planes, center }) + } + _ => { + let mut planes = Vec::with_capacity(n); + for _ in 0..n { + let mut p: [f64; 19] = r.array()?; + p[17] = p[17].clamp(0.0, 1.0); + p[18] = p[18].clamp(0.0, 1.0); + if p[17] >= p[18] { + return Err(ParseError::Malformed("WarpRectilinear2 valid radius range")); + } + planes.push(p); + } + let center = r.array()?; + let reciprocal = r.u32()? != 0; + Ok(Params::WarpRectilinear2 { planes, center, reciprocal }) + } + } + } + 3 => Ok(Params::FixVignetteRadial { k: r.array()?, center: r.array()? }), + 4 => Ok(Params::FixBadPixelsConstant { constant: r.u32()?, bayer_phase: r.u32()? }), + 5 => { + let bayer_phase = r.u32()?; + let points = r.u32()? as usize; + let rects = r.u32()? as usize; + if points.saturating_mul(8).saturating_add(rects.saturating_mul(16)) > r.remaining() { + return Err(ParseError::Malformed("FixBadPixelsList counts larger than its parameters")); + } + let points = + PointRecords(r.take(points.checked_mul(8).ok_or(ParseError::Malformed("FixBadPixelsList counts larger than its parameters"))?)?); + let rects = + RectangleRecords(r.take(rects.checked_mul(16).ok_or(ParseError::Malformed("FixBadPixelsList counts larger than its parameters"))?)?); + Ok(Params::FixBadPixelsList { bayer_phase, points, rects }) + } + 6 => { + let top = r.u32()? as usize; + let left = r.u32()? as usize; + let bottom = r.u32()? as usize; + let right = r.u32()? as usize; + if !context.allow_trim { + return Err(ParseError::Unsupported("TrimBounds before the demosaic")); + } + let bottom = bottom.min(context.height); + let right = right.min(context.width); + if top >= bottom || left >= right { + return Err(ParseError::Malformed("TrimBounds outside the picture")); + } + Ok(Params::TrimBounds { top, left, bottom, right }) + } + 7..=13 => { + let area = r.area(context)?; + match id { + 7 => { + let n = r.count(2)?; + if n == 0 { + return Err(ParseError::Malformed("MapTable without entries")); + } + let table = (0..n) + .map(|_| { + let b = r.take(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + }) + .collect::>()?; + Ok(Params::MapTable { area, table }) + } + 8 => { + let degree = r.u32()? as usize; + if degree > 8 { + return Err(ParseError::Malformed("MapPolynomial degree above 8")); + } + let coefficients = (0..=degree).map(|_| r.f64()).collect::>()?; + Ok(Params::MapPolynomial { area, coefficients }) + } + 9 => { + let points_v = r.u32()? as usize; + let points_h = r.u32()? as usize; + let spacing = r.array()?; + let origin = r.array()?; + let map_planes = r.u32()? as usize; + let n = points_v.saturating_mul(points_h).saturating_mul(map_planes); + if points_v == 0 || points_h == 0 || map_planes == 0 || n.saturating_mul(4) > r.remaining() { + return Err(ParseError::Malformed("GainMap size")); + } + if (points_v > 1 && spacing[0] <= 0.0) || (points_h > 1 && spacing[1] <= 0.0) { + return Err(ParseError::Malformed("GainMap spacing")); + } + let gains = (0..n).map(|_| r.f32()).collect::>()?; + Ok(Params::GainMap { area, points_v, points_h, spacing, origin, map_planes, gains }) + } + _ => { + let n = r.count(4)?; + let vals = (0..n).map(|_| r.f32()).collect::, _>>()?; + let need = if matches!(id, 10 | 12) { + area.bottom.saturating_sub(area.top).div_ceil(area.row_pitch) + } else { + area.right.saturating_sub(area.left).div_ceil(area.col_pitch) + }; + if vals.len() < need { + return Err(ParseError::Malformed("opcode with fewer values than rows or columns")); + } + Ok(match id { + 10 => Params::DeltaPerRow { area, deltas: vals }, + 11 => Params::DeltaPerColumn { area, deltas: vals }, + 12 => Params::ScalePerRow { area, scales: vals }, + _ => Params::ScalePerColumn { area, scales: vals }, + }) + } + } + } + _ => Err(ParseError::Unsupported("unknown DNG opcode")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn u(v: &[u32]) -> Vec { + v.iter().flat_map(|x| x.to_be_bytes()).collect() + } + fn d(v: &[f64]) -> Vec { + v.iter().flat_map(|x| x.to_be_bytes()).collect() + } + fn f(v: &[f32]) -> Vec { + v.iter().flat_map(|x| x.to_be_bytes()).collect() + } + fn ctx() -> Context { + Context { width: 4, height: 3, planes: 2, mosaic: false, allow_trim: true } + } + #[test] + fn caller_constructed_areas_reject_zero_pitch() -> Result<(), ParseError> { + let mut area = Area { top: 1, left: 2, bottom: 7, right: 9, plane: 0, planes: 1, row_pitch: 2, col_pitch: 3 }; + assert_eq!(area.rows()?.collect::>(), vec![1, 3, 5]); + assert_eq!(area.cols()?.collect::>(), vec![2, 5, 8]); + area.row_pitch = 0; + area.col_pitch = 0; + assert!(area.rows().is_err()); + assert!(area.cols().is_err()); + // Empty/reversed bounds must not allow an invalid pitch through either. + area.bottom = 0; + area.right = 0; + assert!(area.rows().is_err()); + assert!(area.cols().is_err()); + Ok(()) + } + + #[test] + fn parses_all_variants_and_clamps_areas() -> Result<(), ParseError> { + let area = u(&[0, 1, 8, 9, 1, 5, 1, 2]); + let cases = vec![ + (1, [u(&[1]), d(&[1., 0., 0., 0., 0., 0., 0.5, 0.5])].concat()), + (2, [u(&[1]), d(&[1., 0., 0., 0., 0.5, 0.5])].concat()), + (3, d(&[0., 0., 0., 0., 0., 0.5, 0.5])), + (4, u(&[0, 0])), + (5, u(&[0, 1, 1, 1, 2, 0, 0, 1, 1])), + (6, u(&[0, 1, 8, 9])), + (7, [area.clone(), u(&[1]), vec![0, 1]].concat()), + (8, [area.clone(), u(&[0]), d(&[1.])].concat()), + (9, [area.clone(), u(&[1, 1]), d(&[0., 0., 0., 0.]), u(&[1]), f(&[1.])].concat()), + (10, [area.clone(), u(&[3]), f(&[1., 2., 3.])].concat()), + (11, [area.clone(), u(&[2]), f(&[1., 2.])].concat()), + (12, [area.clone(), u(&[3]), f(&[1., 2., 3.])].concat()), + (13, [area.clone(), u(&[2]), f(&[1., 2.])].concat()), + (14, [u(&[1]), d(&[0.; 17]), d(&[-1., 2., 0.5, 0.5]), u(&[1])].concat()), + ]; + for (id, bytes) in cases { + let p = parse(id, &bytes, ctx())?; + assert!(matches!( + (id, p), + (1, Params::WarpRectilinear { .. }) + | (2, Params::WarpFisheye { .. }) + | (3, Params::FixVignetteRadial { .. }) + | (4, Params::FixBadPixelsConstant { .. }) + | (5, Params::FixBadPixelsList { .. }) + | (6, Params::TrimBounds { .. }) + | (7, Params::MapTable { .. }) + | (8, Params::MapPolynomial { .. }) + | (9, Params::GainMap { .. }) + | (10, Params::DeltaPerRow { .. }) + | (11, Params::DeltaPerColumn { .. }) + | (12, Params::ScalePerRow { .. }) + | (13, Params::ScalePerColumn { .. }) + | (14, Params::WarpRectilinear2 { .. }) + )); + } + assert!(matches!( + parse(7, &[area, u(&[1]), vec![0, 1]].concat(), ctx()), + Ok(Params::MapTable { area: Area { bottom: 3, right: 4, plane: 1, planes: 1, .. }, .. }) + )); + Ok(()) + } + #[test] + fn validates_in_application_order() { + assert_eq!(parse(1, &[], Context { mosaic: true, ..ctx() }), Err(ParseError::Unsupported("a warp opcode on a CFA mosaic"))); + assert_eq!(parse(6, &[], Context { allow_trim: false, ..ctx() }), Err(ParseError::Malformed("opcode parameters too short"))); + assert_eq!( + parse(6, &u(&[0, 0, 1, 1]), Context { allow_trim: false, ..ctx() }), + Err(ParseError::Unsupported("TrimBounds before the demosaic")) + ); + assert_eq!(parse(5, &u(&[0, u32::MAX, u32::MAX]), ctx()), Err(ParseError::Malformed("FixBadPixelsList counts larger than its parameters"))); + let a = u(&[0, 0, 3, 4, 0, 1, 1, 1]); + assert_eq!(parse(10, &[a, u(&[1]), f(&[f32::NAN])].concat(), ctx()), Err(ParseError::Malformed("non-finite opcode parameter"))); + } + + #[test] + fn typed_areas_clamp_to_the_image() { + let c = ctx(); + let wire = crate::opcodes::Area { top: 1, left: 2, bottom: 9_999, right: 9_999, plane: 9, planes: 9, row_pitch: 2, col_pitch: 1 }; + let a = Area::clamp(wire, c).unwrap(); + assert_eq!((a.bottom, a.right), (c.height, c.width)); + assert_eq!((a.plane, a.planes), (c.planes, 0)); + assert!(Area::clamp(crate::opcodes::Area { row_pitch: 0, ..wire }, c).is_err()); + // the typed reader and this one agree on a well-formed area + let typed = crate::opcodes::Opcode::MapTable { area: crate::opcodes::Area { col_pitch: 1, ..wire }, table: vec![1, 2] }; + let bytes = crate::opcodes::write_list(&[typed]); + let record = &crate::opcode_raw::parse(&bytes, 1).unwrap()[0]; + match parse(record.id, &record.params, c).unwrap() { + Params::MapTable { area, table } => { + assert_eq!(area, a); + assert_eq!(table, [1, 2]); + } + other => panic!("MapTable expected, got {other:?}"), + } + } +} diff --git a/crates/raw/tests/it/dng.rs b/crates/raw/tests/it/dng.rs index 450b8ea96..21074a828 100644 --- a/crates/raw/tests/it/dng.rs +++ b/crates/raw/tests/it/dng.rs @@ -214,7 +214,7 @@ pub(super) fn dng(raw: IfdBuilder, order: ByteOrder) -> Vec { } /// [`dng`] with extra IFD 0 tags (where DNG writers put camera-profile tags). -fn dng_with(raw: IfdBuilder, order: ByteOrder, extra: impl FnOnce(&mut IfdBuilder)) -> Vec { +pub(super) fn dng_with(raw: IfdBuilder, order: ByteOrder, extra: impl FnOnce(&mut IfdBuilder)) -> Vec { let mut ifd0 = IfdBuilder::new(); ifd0.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![1])); ifd0.set(t::DNG_VERSION, Value::Byte(vec![1, 6, 0, 0])); diff --git a/crates/raw/tests/it/dng_metadata.rs b/crates/raw/tests/it/dng_metadata.rs new file mode 100644 index 000000000..a2f5e92ef --- /dev/null +++ b/crates/raw/tests/it/dng_metadata.rs @@ -0,0 +1,150 @@ +//! The bounded DNG metadata reader (`dng_metadata`) on hand-built DNGs, checked against the tags +//! written and, for the colour tags, against the full reader. + +use super::dng::{base_ifd, dng, dng_with}; +use craft_raw::dng_lines::{self, Error}; +use craft_raw::dng_metadata::{Metadata, Pattern}; +use craft_tiff::classic::{Tiff, tags as ct}; +use craft_tiff::{ByteOrder, IfdBuilder, ImageData, Value}; + +fn cfa_raw(edit: impl FnOnce(&mut IfdBuilder)) -> IfdBuilder { + let mut raw = base_ifd(8, 4, 16, 1, true); + raw.set(ct::COMPRESSION, Value::Short(vec![1])); + raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0; 64]] }); + edit(&mut raw); + raw +} + +fn with_metadata(bytes: &[u8], f: impl FnOnce(&Metadata) -> T) -> T { + let (tiff, first) = Tiff::new(bytes).unwrap(); + let ifds = tiff.all_ifds(first).unwrap(); + let raw = dng_lines::find_raw_ifd(&tiff, &ifds).unwrap(); + f(&Metadata { tiff, main: &ifds[0], raw }) +} + +#[test] +fn processing_tags_are_read_from_the_raw_ifd() { + let raw = cfa_raw(|r| { + r.set(ct::ACTIVE_AREA, Value::Long(vec![1, 2, 3, 7])); + r.set(ct::LINEARIZATION_TABLE, Value::Short(vec![0, 10, 20])); + r.set(ct::BLACK_LEVEL_REPEAT_DIM, Value::Short(vec![2, 2])); + r.set(ct::BLACK_LEVEL, Value::Rational(vec![(64, 1), (65, 1), (66, 1), (67, 1)])); + r.set(ct::BLACK_LEVEL_DELTA_H, Value::SRational(vec![(1, 2); 8])); + r.set(ct::BLACK_LEVEL_DELTA_V, Value::SRational(vec![(-1, 2); 4])); + r.set(ct::WHITE_LEVEL, Value::Long(vec![4000])); + r.set(ct::DEFAULT_CROP_ORIGIN, Value::Rational(vec![(1, 2), (1, 1)])); + r.set(ct::DEFAULT_CROP_SIZE, Value::Long(vec![6, 2])); + }); + for order in [ByteOrder::Little, ByteOrder::Big] { + let p = with_metadata(&dng(raw.clone(), order), |m| m.params().unwrap()); + assert_eq!(p.active_area, Some([1, 2, 3, 7])); + assert_eq!(p.linearization, Some(vec![0, 10, 20])); + assert_eq!(p.black_repeat, (2, 2)); + assert_eq!(p.black, [64.0, 65.0, 66.0, 67.0]); + assert_eq!(p.black_delta_h, [0.5; 8]); + assert_eq!(p.black_delta_v, [-0.5; 4]); + assert_eq!(p.white, [4000.0]); + assert_eq!((p.crop_origin, p.crop_size), (Some((0.5, 1.0)), Some((6.0, 2.0)))); + // base_ifd writes CFAPattern 1 0 2 1 (GRBG) + assert_eq!(p.cfa, Some(Pattern { rows: 2, cols: 2, colors: vec![1, 0, 2, 1] })); + } + // a LinearizationTable longer than 65536 entries is refused + let long = cfa_raw(|r| { + r.set(ct::LINEARIZATION_TABLE, Value::Short(vec![1; 65_537])); + }); + assert!(matches!(with_metadata(&dng(long, ByteOrder::Little), |m| m.params()), Err(Error::Malformed(_)))); + // absent tags take the DNG defaults + let p = with_metadata(&dng(cfa_raw(|_| {}), ByteOrder::Little), |m| m.params().unwrap()); + assert_eq!((p.active_area, p.linearization, p.black_repeat), (None, None, (1, 1))); + assert!(p.black.is_empty() && p.white.is_empty() && p.crop_origin.is_none()); +} + +#[test] +fn cfa_patterns_map_plane_colours_and_refuse_odd_layouts() { + let pattern = |edit: fn(&mut IfdBuilder)| with_metadata(&dng(cfa_raw(edit), ByteOrder::Little), |m| m.cfa()); + // CFAPlaneColor reorders the planes: file colour 2 is plane 0 + let swapped = pattern(|r| { + r.set(ct::CFA_PLANE_COLOR, Value::Byte(vec![2, 1, 0])); + r.set(ct::CFA_LAYOUT, Value::Short(vec![1])); + }); + assert_eq!(swapped.unwrap(), Some(Pattern { rows: 2, cols: 2, colors: vec![1, 2, 0, 1] })); + let unsupported = |r: Result, Error>| matches!(r, Err(Error::Unsupported(_))); + let malformed = |r: Result, Error>| matches!(r, Err(Error::Malformed(_))); + assert!(unsupported(pattern(|r| { + r.set(ct::CFA_LAYOUT, Value::Short(vec![2])); + }))); + assert!(unsupported(pattern(|r| { + r.set(ct::CFA_PLANE_COLOR, Value::Byte(vec![0, 1, 3])); + }))); + assert!(unsupported(pattern(|r| { + r.set(ct::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![9, 2])); + }))); + assert!(malformed(pattern(|r| { + r.set(ct::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2])); + }))); + assert!(malformed(pattern(|r| { + r.set(ct::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![4, 4])); + }))); + assert!(malformed(pattern(|r| { + r.set(ct::CFA_PATTERN, Value::Byte(vec![0, 1, 1, 3])); + }))); + // LinearRaw has no pattern + let mut linear = base_ifd(4, 2, 16, 3, false); + linear.set(ct::COMPRESSION, Value::Short(vec![1])); + linear.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![vec![0; 48]] }); + assert_eq!(with_metadata(&dng(linear, ByteOrder::Big), |m| m.cfa()).unwrap(), None); +} + +#[test] +fn calibration_matches_the_full_reader() { + let bytes = dng_with(cfa_raw(|_| {}), ByteOrder::Little, |ifd0| { + ifd0.set(ct::COLOR_MATRIX_2, Value::SRational(vec![(2, 1), (0, 1), (0, 1), (0, 1), (2, 1), (0, 1), (0, 1), (0, 1), (2, 1)])); + ifd0.set(ct::CALIBRATION_ILLUMINANT_2, Value::Short(vec![17])); + ifd0.set(ct::ANALOG_BALANCE, Value::Rational(vec![(1, 1); 3])); + ifd0.set(ct::AS_SHOT_NEUTRAL, Value::Rational(vec![(1, 2), (1, 1), (3, 4)])); + ifd0.set(ct::BASELINE_EXPOSURE, Value::SRational(vec![(1, 4)])); + // not 3 x 3 (a four-colour camera): not a matrix here + ifd0.set(ct::FORWARD_MATRIX_1, Value::SRational(vec![(1, 1); 12])); + }); + let c = with_metadata(&bytes, |m| m.calibration().unwrap()); + let full = craft_raw::decode(&bytes).unwrap().color; + assert_eq!(c.color_matrix, full.color_matrix); + assert_eq!(c.illuminant, [Some(21), Some(17)]); + assert_eq!(c.as_shot_neutral, full.as_shot_neutral); + assert_eq!(c.analog_balance, Some([1.0; 3])); + assert_eq!(c.as_shot_white_xy, Some((0.3127, 0.329))); + assert_eq!(c.baseline_exposure, 0.25); + assert_eq!(c.forward_matrix, [None, None]); + assert_eq!(c.camera_calibration, [None, None]); +} + +#[test] +fn capture_info_matches_the_full_readers_cinema_tags() { + use craft_raw::dng_metadata::Timecode; + let tc = Timecode { hours: 10, minutes: 59, seconds: 7, frames: 23, drop_frame: true }; + let bytes = dng_with(cfa_raw(|_| {}), ByteOrder::Big, |ifd0| { + ifd0.set(ct::FRAME_RATE, Value::SRational(vec![(24000, 1001)])); + ifd0.set(ct::TIME_CODES, Value::Byte(tc.to_bytes().to_vec())); + ifd0.set(ct::REEL_NAME, Value::Ascii("A001".into())); + ifd0.set(ct::MODEL, Value::Ascii("Cam".into())); + }); + let info = with_metadata(&bytes, |m| m.capture_info()); + let cinema = craft_raw::decode(&bytes).unwrap().cinema; + assert_eq!(info.dng_version, [1, 6, 0, 0]); + assert_eq!(info.frame_rate, Some((24000, 1001))); + assert_eq!((info.frame_rate, info.timecode), (cinema.frame_rate, cinema.timecode)); + assert_eq!(info.timecode, Some(tc)); + assert_eq!(info.reel.as_deref(), cinema.reel_name.as_deref()); + assert_eq!(info.camera.as_deref(), Some("Hand Cam")); + // an implausible rate and a broken time code are dropped by both; UniqueCameraModel wins + let bytes = dng_with(cfa_raw(|_| {}), ByteOrder::Little, |ifd0| { + ifd0.set(ct::FRAME_RATE, Value::Rational(vec![(5000, 1)])); + ifd0.set(ct::TIME_CODES, Value::Byte(vec![0x0a, 0, 0, 0, 0, 0, 0, 0])); + ifd0.set(ct::UNIQUE_CAMERA_MODEL, Value::Ascii("Hand Cam Mk II".into())); + }); + let info = with_metadata(&bytes, |m| m.capture_info()); + assert_eq!((info.frame_rate, info.timecode, info.reel), (None, None, None)); + assert_eq!(info.camera.as_deref(), Some("Hand Cam Mk II")); + let cinema = craft_raw::decode(&bytes).unwrap().cinema; + assert_eq!((cinema.frame_rate, cinema.timecode), (None, None)); +} diff --git a/crates/raw/tests/it/main.rs b/crates/raw/tests/it/main.rs index 2f1b1061f..5a9386eb0 100644 --- a/crates/raw/tests/it/main.rs +++ b/crates/raw/tests/it/main.rs @@ -4,6 +4,8 @@ mod dng; #[cfg(feature = "line-dng")] mod dng_lines; +#[cfg(feature = "line-dng")] +mod dng_metadata; #[cfg(feature = "dng")] mod robust; mod vendor_robust; diff --git a/crates/sniff/Cargo.toml b/crates/sniff/Cargo.toml new file mode 100644 index 000000000..5fd0955d5 --- /dev/null +++ b/crates/sniff/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-sniff" +description = "What a file is from its first bytes: a dependency-free signature table naming the format, its media type, extension and family, for images, audio, video, 3D models, archives, executables and databases." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true diff --git a/crates/sniff/src/detect.rs b/crates/sniff/src/detect.rs new file mode 100644 index 000000000..d97baf126 --- /dev/null +++ b/crates/sniff/src/detect.rs @@ -0,0 +1,264 @@ +//! The signature checks behind [`identify`]. + +use crate::Kind; + +/// How many leading bytes [`identify`] reads at most. Every signature it knows lies within them. +pub const HEAD: usize = 4096; + +/// The largest universal Mach-O architecture count taken as Mach-O rather than a Java class. +/// Both start `CAFEBABE`; a class file's next four bytes are its minor and major version, and +/// every released class version (major 45, JDK 1.0, onwards) reads as at least 45 there, while +/// real universal binaries hold a handful of architectures. +const MAX_FAT_ARCHES: u32 = 44; + +/// The ISO BMFF brands read from an `ftyp` box: the major brand and up to this many compatible ones. +const MAX_BRANDS: usize = 32; + +/// The MPEG transport stream packet size and the sync byte each packet starts with. +const TS_PACKET: usize = 188; +const TS_SYNC: u8 = 0x47; + +/// Name the format of a file from its leading bytes, or `None` when no signature matches. +/// +/// Only the first [`HEAD`] bytes are read. Signatures at the start of the file are checked +/// before the weaker ones further in or built from few bits (TAR at byte 257, MP3 and AAC frame +/// headers, MPEG transport stream packets), so a file is never named by a weak signature when a +/// strong one matches. +/// +/// `CAFEBABE` starts both a universal Mach-O and a Java class file. The next big-endian 32-bit +/// value is a Mach-O architecture count or a class file's version: up to 44 is taken as +/// [`Kind::MachOFat`], 45 and over as [`Kind::JavaClass`]. A caller that knows the file name +/// (`.class`) may prefer that. +pub fn identify(bytes: &[u8]) -> Option { + let head = bytes.get(..HEAD).unwrap_or(bytes); + leading(head).or_else(|| container(head)).or_else(|| trailing(head)) +} + +/// Signatures at byte 0 that need nothing more than a fixed prefix. +fn leading(b: &[u8]) -> Option { + const PREFIXES: &[(&[u8], Kind)] = &[ + (b"\x89PNG\r\n\x1a\n", Kind::Png), + (b"\xff\xd8\xff", Kind::Jpeg), + (b"GIF87a", Kind::Gif), + (b"GIF89a", Kind::Gif), + (b"\xff\x0a", Kind::Jxl), + (b"\x00\x00\x00\x0cJXL \r\n\x87\n", Kind::Jxl), + (b"qoif", Kind::Qoi), + (b"\x76\x2f\x31\x01", Kind::OpenExr), + (b"fLaC", Kind::Flac), + (b"MThd\x00\x00\x00\x06", Kind::Midi), + (b"#!AMR", Kind::Amr), + (b"FLV\x01", Kind::Flv), + (b"\x00\x00\x01\xba", Kind::MpegPs), + (b"glTF", Kind::Glb), + (b"Kaydara FBX Binary ", Kind::Fbx), + (b"ply\n", Kind::Ply), + (b"ply\r", Kind::Ply), + (b"PK\x03\x04", Kind::Zip), + (b"PK\x05\x06", Kind::Zip), + (b"7z\xbc\xaf\x27\x1c", Kind::SevenZip), + (b"Rar!\x1a\x07\x00", Kind::Rar), + (b"Rar!\x1a\x07\x01\x00", Kind::Rar), + (b"\x1f\x8b\x08", Kind::Gzip), + (b"\xfd7zXZ\x00", Kind::Xz), + (b"\x28\xb5\x2f\xfd", Kind::Zstd), + (b"070701", Kind::Cpio), + (b"070702", Kind::Cpio), + (b"070707", Kind::Cpio), + (b"\xc7\x71", Kind::Cpio), + (b"\x71\xc7", Kind::Cpio), + (b"\x7fELF", Kind::Elf), + (b"\xfe\xed\xfa\xce", Kind::MachO), + (b"\xfe\xed\xfa\xcf", Kind::MachO), + (b"\xce\xfa\xed\xfe", Kind::MachO), + (b"\xcf\xfa\xed\xfe", Kind::MachO), + (b"\xca\xfe\xba\xbf", Kind::MachOFat), + (b"\0asm", Kind::Wasm), + (b"dex\n", Kind::Dex), + (b"SQLite format 3\0", Kind::Sqlite), + (b"bplist00", Kind::BinaryPlist), + (b"%PDF-", Kind::Pdf), + (b"ID3", Kind::Mp3), + ]; + PREFIXES.iter().find(|(magic, _)| b.starts_with(magic)).map(|&(_, kind)| kind).or_else(|| checked(b)) +} + +/// Signatures at byte 0 whose prefix is short or shared, confirmed by a field after it. +fn checked(b: &[u8]) -> Option { + if b.starts_with(b"MZ") { + return Some(dos(b)); + } + match b.get(..4)? { + [0xca, 0xfe, 0xba, 0xbe] => Some(if be32(b, 4)? <= MAX_FAT_ARCHES { Kind::MachOFat } else { Kind::JavaClass }), + b"II*\0" | b"MM\0*" | b"II+\0" | b"MM\0+" => Some(Kind::Tiff), + b"8BPS" => matches!(be16(b, 4)?, 1 | 2).then_some(Kind::Psd), + [0, 0, 1, 0] => (le16(b, 4)? > 0 && b.get(9).copied()? == 0).then_some(Kind::Ico), + [b'B', b'Z', b'h', level] => (b'1'..=b'9').contains(level).then_some(Kind::Bzip2), + [b'B', b'M', ..] => bmp(b), + _ => None, + } +} + +/// `MZ`: a PE file when `e_lfanew` (at 0x3C) points at `PE\0\0` within the bytes given. +fn dos(b: &[u8]) -> Kind { + let pe = le32(b, 0x3c) + .and_then(|offset| usize::try_from(offset).ok()) + .and_then(|offset| b.get(offset..offset.checked_add(4)?)) + .is_some_and(|sig| sig == b"PE\0\0"); + if pe { Kind::Pe } else { Kind::DosExecutable } +} + +/// `BM` alone is two letters of text, so a BMP also needs zero reserved fields and a known +/// DIB header size. +fn bmp(b: &[u8]) -> Option { + let reserved = b.get(6..10)? == [0, 0, 0, 0]; + let dib = matches!(le32(b, 14)?, 12 | 16 | 40 | 52 | 56 | 64 | 108 | 124); + (reserved && dib).then_some(Kind::Bmp) +} + +/// Container formats named by what the container says it holds. +fn container(b: &[u8]) -> Option { + match (b.get(..4)?, b.get(8..12)) { + (b"RIFF", Some(b"WAVE")) => Some(Kind::Wav), + (b"RIFF", Some(b"WEBP")) => Some(Kind::WebP), + (b"RIFF", Some(b"AVI ")) => Some(Kind::Avi), + (b"FORM", Some(b"AIFF" | b"AIFC")) => Some(Kind::Aiff), + (b"OggS", _) => ogg(b), + ([0x1a, 0x45, 0xdf, 0xa3], _) => Some(ebml(b)), + _ => iso_bmff(b), + } +} + +/// An Ogg stream is named by its first packet: Opus and Theora have their own types. +fn ogg(b: &[u8]) -> Option { + let segments = usize::from(*b.get(26)?); + let packet = b.get(27usize.checked_add(segments)?..).unwrap_or_default(); + Some(if packet.starts_with(b"OpusHead") { + Kind::Opus + } else if packet.starts_with(b"\x80theora") { + Kind::OggVideo + } else { + Kind::Ogg + }) +} + +/// EBML: WebM and Matroska differ by the `DocType` element (ID 0x4282) in the EBML header. +fn ebml(b: &[u8]) -> Kind { + let header = b.get(..64).unwrap_or(b); + let doc_type = header.windows(2).position(|w| w == [0x42, 0x82]).and_then(|at| { + let size = header.get(at.checked_add(2)?).copied()?; + // A one-byte EBML size has its top bit set; the value is the remaining seven bits. + let len = usize::from(size & 0x7f); + let start = at.checked_add(3)?; + if size & 0x80 == 0 { + return None; + } + header.get(start..start.checked_add(len)?) + }); + if doc_type == Some(b"webm") { Kind::WebM } else { Kind::Matroska } +} + +/// ISO base media files: an `ftyp` box whose brands say what it holds, or an old QuickTime +/// movie starting with one of its own top-level atoms. +fn iso_bmff(b: &[u8]) -> Option { + let box_type = b.get(4..8)?; + let size = usize::try_from(be32(b, 0)?).ok()?; + if box_type != b"ftyp" { + let atom = matches!(box_type, b"moov" | b"mdat" | b"wide") && size >= 8; + return atom.then_some(Kind::QuickTime); + } + if size < 16 { + return None; + } + let end = size.min(b.len()).min(16usize.saturating_add(MAX_BRANDS.saturating_mul(4))); + let major: &[u8; 4] = b.get(8..12)?.try_into().ok()?; + let compatible = b.get(16..end).unwrap_or_default().as_chunks::<4>().0; + let brands = core::iter::once(major).chain(compatible.iter()); + // A still image also lists generic brands, so image brands win wherever they are listed: + // AVIF and JPEG XL over HEVC-coded HEIF, over the bare HEIF structure brands, over video. + let mut best: Option<(u8, Kind)> = None; + for kind in brands.filter_map(brand_kind) { + let rank = brand_rank(kind); + if best.is_none_or(|(top, _)| rank > top) { + best = Some((rank, kind)); + } + } + best.map(|(_, kind)| kind) +} + +fn brand_rank(kind: Kind) -> u8 { + match kind { + Kind::Avif | Kind::Jxl => 3, + Kind::Heif => 2, + _ => 0, + } +} + +fn brand_kind(brand: &[u8; 4]) -> Option { + Some(match brand { + b"avif" | b"avis" => Kind::Avif, + b"heic" | b"heix" | b"heim" | b"heis" | b"hevc" | b"hevx" => Kind::Heif, + // The HEIF structure brands. AVIF files list them too, and AVIF outranks them. + b"mif1" | b"msf1" => Kind::Heif, + b"jxl " => Kind::Jxl, + b"qt " => Kind::QuickTime, + b"M4A " | b"M4B " | b"M4P " => Kind::M4a, + b"3gp4" | b"3gp5" | b"3gp6" | b"3gp7" | b"3gg6" | b"3g2a" | b"3g2b" | b"3g2c" => Kind::ThreeGp, + b"isom" | b"iso2" | b"iso3" | b"iso4" | b"iso5" | b"iso6" | b"mp41" | b"mp42" | b"avc1" | b"dash" | b"mmp4" | b"M4V " | b"M4VH" | b"M4VP" + | b"MSNV" | b"f4v " => Kind::Mp4, + _ => return None, + }) +} + +/// Signatures that sit past byte 0 or are made of a few bits, checked last. +fn trailing(b: &[u8]) -> Option { + if let Some(magic) = b.get(257..263) + && (magic == b"ustar\0" || magic == b"ustar ") + { + return Some(Kind::Tar); + } + if transport_stream(b) { + return Some(Kind::MpegTs); + } + frame_sync(b) +} + +/// Three transport stream packets in a row, each starting with the sync byte. +fn transport_stream(b: &[u8]) -> bool { + (0..3).all(|n| b.get(n * TS_PACKET).copied() == Some(TS_SYNC)) +} + +/// An MPEG audio frame header: 11 sync bits, then the layer bits tell MP3 (layer III) from AAC +/// in ADTS (layer 00). Bitrate and sampling indexes must not be the reserved ones. +fn frame_sync(b: &[u8]) -> Option { + let [0xff, second, third, ..] = *b else { + return None; + }; + if second & 0xe0 != 0xe0 { + return None; + } + let version = (second >> 3) & 0b11; + let layer = (second >> 1) & 0b11; + match layer { + // ADTS uses only the MPEG-4 (0) and MPEG-2 (1) id bit, so the second byte is 0xFx. + 0b00 if second & 0xf0 == 0xf0 && (third >> 2) & 0x0f < 13 => Some(Kind::Aac), + 0b01 if version != 0b01 && third >> 4 != 0x0f && (third >> 2) & 0b11 != 0b11 => Some(Kind::Mp3), + _ => None, + } +} + +fn be16(b: &[u8], at: usize) -> Option { + Some(u16::from_be_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?)) +} + +fn le16(b: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?)) +} + +fn be32(b: &[u8], at: usize) -> Option { + Some(u32::from_be_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?)) +} + +fn le32(b: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?)) +} diff --git a/crates/sniff/src/kind.rs b/crates/sniff/src/kind.rs new file mode 100644 index 000000000..da98b15dc --- /dev/null +++ b/crates/sniff/src/kind.rs @@ -0,0 +1,270 @@ +//! The kinds [`identify`](crate::identify) names, and what each one is. + +/// The broad kind of content a file holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Family { + /// Still or layered pictures. + Image, + /// Sound with no pictures. + Audio, + /// Moving pictures, with or without sound. Container formats that may hold only sound + /// (Matroska, MP4, Ogg Theora) are named by what their signature promises. + Video, + /// 3D scenes and meshes. + Model, + /// Several files packed together. + Archive, + /// One compressed stream. + Compressed, + /// Machine code or bytecode a runtime loads. + Executable, + /// Structured stores: databases and binary property lists. + Database, + /// Page-description documents. + Document, +} + +/// A file format named by its signature. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Kind { + // Images. + Png, + Jpeg, + Gif, + WebP, + Bmp, + Tiff, + /// Windows icon (`.ico`). + Ico, + /// HEIF still images, HEVC-coded (`.heic`) or generic. + Heif, + Avif, + /// JPEG XL, as a bare codestream or in its ISO BMFF container. + Jxl, + /// Photoshop document or large document (`.psd`, `.psb`). + Psd, + Qoi, + OpenExr, + // Audio. + /// MPEG-1/2 Audio Layer III, with or without an ID3v2 tag. + Mp3, + /// AAC in ADTS framing. + Aac, + Flac, + /// RIFF WAVE. + Wav, + /// AIFF or AIFF-C. + Aiff, + /// Ogg holding anything but Opus or Theora (usually Vorbis). + Ogg, + /// Ogg Opus. + Opus, + /// MPEG-4 audio (`.m4a`, `.m4b`). + M4a, + /// Standard MIDI file. + Midi, + /// AMR narrowband or wideband speech. + Amr, + // Video. + /// MPEG-4 Part 14 and the ISO base media brands it shares. + Mp4, + /// QuickTime movie. + QuickTime, + /// 3GPP or 3GPP2 movie. + ThreeGp, + WebM, + Matroska, + /// Ogg holding Theora video. + OggVideo, + /// RIFF AVI. + Avi, + /// Flash video. + Flv, + /// MPEG program stream. + MpegPs, + /// MPEG transport stream (188-byte packets). + MpegTs, + // 3D models. + /// Binary glTF. + Glb, + /// Binary FBX. ASCII FBX has no signature. + Fbx, + /// PLY, ASCII or binary. + Ply, + // Archives and compressed streams. + Zip, + /// POSIX or GNU tar. + Tar, + /// cpio, binary or ASCII ("newc", CRC and "odc"). + Cpio, + SevenZip, + Rar, + Gzip, + Bzip2, + Xz, + Zstd, + // Executables. + Elf, + /// Portable Executable: a DOS header whose `e_lfanew` reaches a `PE\0\0` signature. + Pe, + /// A DOS `MZ` header with no PE signature in the bytes given (a DOS program, or a PE file + /// whose header lies past them). + DosExecutable, + /// A single-architecture Mach-O file. + MachO, + /// A universal ("fat") Mach-O file. + MachOFat, + /// A Java class file. It shares `CAFEBABE` with universal Mach-O; see [`identify`](crate::identify). + JavaClass, + /// WebAssembly binary module. + Wasm, + /// Dalvik executable. + Dex, + // Databases. + Sqlite, + /// Apple binary property list. + BinaryPlist, + // Documents. + Pdf, +} + +impl Kind { + /// Every kind, in the order [`identify`](crate::identify) documents them. + pub const ALL: [Kind; 56] = [ + Kind::Png, + Kind::Jpeg, + Kind::Gif, + Kind::WebP, + Kind::Bmp, + Kind::Tiff, + Kind::Ico, + Kind::Heif, + Kind::Avif, + Kind::Jxl, + Kind::Psd, + Kind::Qoi, + Kind::OpenExr, + Kind::Mp3, + Kind::Aac, + Kind::Flac, + Kind::Wav, + Kind::Aiff, + Kind::Ogg, + Kind::Opus, + Kind::M4a, + Kind::Midi, + Kind::Amr, + Kind::Mp4, + Kind::QuickTime, + Kind::ThreeGp, + Kind::WebM, + Kind::Matroska, + Kind::OggVideo, + Kind::Avi, + Kind::Flv, + Kind::MpegPs, + Kind::MpegTs, + Kind::Glb, + Kind::Fbx, + Kind::Ply, + Kind::Zip, + Kind::Tar, + Kind::Cpio, + Kind::SevenZip, + Kind::Rar, + Kind::Gzip, + Kind::Bzip2, + Kind::Xz, + Kind::Zstd, + Kind::Elf, + Kind::Pe, + Kind::DosExecutable, + Kind::MachO, + Kind::MachOFat, + Kind::JavaClass, + Kind::Wasm, + Kind::Dex, + Kind::Sqlite, + Kind::BinaryPlist, + Kind::Pdf, + ]; + + /// The media type to send or store with the file: the IANA registration where one exists, + /// otherwise the name in common use (`audio/wav`, `video/x-matroska`, `model/fbx`). + pub fn mime_type(self) -> &'static str { + self.info().0 + } + + /// The usual file extension, lower case, without the leading dot. + pub fn extension(self) -> &'static str { + self.info().1 + } + + /// The broad kind of content. + pub fn family(self) -> Family { + self.info().2 + } + + fn info(self) -> (&'static str, &'static str, Family) { + use Family::{Archive, Audio, Compressed, Database, Document, Executable, Image, Model, Video}; + match self { + Kind::Png => ("image/png", "png", Image), + Kind::Jpeg => ("image/jpeg", "jpg", Image), + Kind::Gif => ("image/gif", "gif", Image), + Kind::WebP => ("image/webp", "webp", Image), + Kind::Bmp => ("image/bmp", "bmp", Image), + Kind::Tiff => ("image/tiff", "tif", Image), + Kind::Ico => ("image/vnd.microsoft.icon", "ico", Image), + Kind::Heif => ("image/heif", "heic", Image), + Kind::Avif => ("image/avif", "avif", Image), + Kind::Jxl => ("image/jxl", "jxl", Image), + Kind::Psd => ("image/vnd.adobe.photoshop", "psd", Image), + Kind::Qoi => ("image/qoi", "qoi", Image), + Kind::OpenExr => ("image/x-exr", "exr", Image), + Kind::Mp3 => ("audio/mpeg", "mp3", Audio), + Kind::Aac => ("audio/aac", "aac", Audio), + Kind::Flac => ("audio/flac", "flac", Audio), + Kind::Wav => ("audio/wav", "wav", Audio), + Kind::Aiff => ("audio/aiff", "aiff", Audio), + Kind::Ogg => ("audio/ogg", "ogg", Audio), + Kind::Opus => ("audio/opus", "opus", Audio), + Kind::M4a => ("audio/mp4", "m4a", Audio), + Kind::Midi => ("audio/midi", "mid", Audio), + Kind::Amr => ("audio/amr", "amr", Audio), + Kind::Mp4 => ("video/mp4", "mp4", Video), + Kind::QuickTime => ("video/quicktime", "mov", Video), + Kind::ThreeGp => ("video/3gpp", "3gp", Video), + Kind::WebM => ("video/webm", "webm", Video), + Kind::Matroska => ("video/x-matroska", "mkv", Video), + Kind::OggVideo => ("video/ogg", "ogv", Video), + Kind::Avi => ("video/x-msvideo", "avi", Video), + Kind::Flv => ("video/x-flv", "flv", Video), + Kind::MpegPs => ("video/mpeg", "mpg", Video), + Kind::MpegTs => ("video/mp2t", "ts", Video), + Kind::Glb => ("model/gltf-binary", "glb", Model), + Kind::Fbx => ("model/fbx", "fbx", Model), + Kind::Ply => ("model/ply", "ply", Model), + Kind::Zip => ("application/zip", "zip", Archive), + Kind::Tar => ("application/x-tar", "tar", Archive), + Kind::Cpio => ("application/x-cpio", "cpio", Archive), + Kind::SevenZip => ("application/x-7z-compressed", "7z", Archive), + Kind::Rar => ("application/vnd.rar", "rar", Archive), + Kind::Gzip => ("application/gzip", "gz", Compressed), + Kind::Bzip2 => ("application/x-bzip2", "bz2", Compressed), + Kind::Xz => ("application/x-xz", "xz", Compressed), + Kind::Zstd => ("application/zstd", "zst", Compressed), + Kind::Elf => ("application/x-elf", "elf", Executable), + Kind::Pe => ("application/vnd.microsoft.portable-executable", "exe", Executable), + Kind::DosExecutable => ("application/x-msdownload", "exe", Executable), + Kind::MachO | Kind::MachOFat => ("application/x-mach-binary", "macho", Executable), + Kind::JavaClass => ("application/java-vm", "class", Executable), + Kind::Wasm => ("application/wasm", "wasm", Executable), + Kind::Dex => ("application/vnd.android.dex", "dex", Executable), + Kind::Sqlite => ("application/vnd.sqlite3", "sqlite", Database), + Kind::BinaryPlist => ("application/x-bplist", "plist", Database), + Kind::Pdf => ("application/pdf", "pdf", Document), + } + } +} diff --git a/crates/sniff/src/lib.rs b/crates/sniff/src/lib.rs new file mode 100644 index 000000000..b95e11591 --- /dev/null +++ b/crates/sniff/src/lib.rs @@ -0,0 +1,32 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![deny(clippy::indexing_slicing)] +#![forbid(unsafe_code)] + +//! What a file is, from its first bytes. +//! +//! [`identify`] reads a file's leading bytes and names its [`Kind`]: the format, with its media +//! type ([`Kind::mime_type`]), usual extension ([`Kind::extension`]) and [`Family`]. It covers the +//! formats apps most often have to tell apart before they open anything: images, audio, video, +//! 3D models, archives and compressed streams, executables and databases. +//! +//! Only signatures are read. Nothing is parsed, validated or decoded, so a file that `identify` +//! names can still be damaged, and a file it doesn't name may still be readable by its own +//! reader. Formats with no signature (plain text, CSV, ASCII FBX, binary STL) are never named. +//! Which kinds an app supports, and what it does when the name and the bytes disagree, stay +//! with the app. +//! +//! [`HEAD`] bytes are enough for every signature; passing more changes nothing, passing fewer +//! only loses the signatures that sit further in (TAR's at byte 257, MPEG transport streams' +//! third packet). Any input is fine: nothing here allocates, panics or reads past the slice. +//! +//! Written from the formats' public specifications and registrations (IANA media types, ISO/IEC +//! 14496-12 brands, the EBML and Matroska specifications, RIFF, the ELF, PE and Mach-O headers). + +mod detect; +mod kind; + +pub use detect::{HEAD, identify}; +pub use kind::{Family, Kind}; + +#[cfg(test)] +mod tests; diff --git a/crates/sniff/src/tests.rs b/crates/sniff/src/tests.rs new file mode 100644 index 000000000..2c5d2da8d --- /dev/null +++ b/crates/sniff/src/tests.rs @@ -0,0 +1,311 @@ +use crate::{Family, HEAD, Kind, identify}; + +/// `prefix` followed by zeros up to `len` bytes. +fn padded(prefix: &[u8], len: usize) -> Vec { + let mut bytes = prefix.to_vec(); + bytes.resize(len.max(prefix.len()), 0); + bytes +} + +/// An ISO BMFF `ftyp` box with the given major and compatible brands. +fn ftyp(major: &[u8; 4], compatible: &[&[u8; 4]]) -> Vec { + let size = u32::try_from(16 + compatible.len() * 4).unwrap(); + let mut bytes = size.to_be_bytes().to_vec(); + bytes.extend_from_slice(b"ftyp"); + bytes.extend_from_slice(major); + bytes.extend_from_slice(&[0, 0, 0, 0]); + for brand in compatible { + bytes.extend_from_slice(*brand); + } + bytes +} + +/// A RIFF or FORM container header of the given form type. +fn riff(chunk: &[u8; 4], form: &[u8; 4]) -> Vec { + let mut bytes = chunk.to_vec(); + bytes.extend_from_slice(&[0x24, 0, 0, 0]); + bytes.extend_from_slice(form); + bytes +} + +/// An Ogg page whose first packet starts with `packet`. +fn ogg(packet: &[u8]) -> Vec { + let mut bytes = b"OggS\0\x02".to_vec(); + bytes.resize(26, 0); + bytes.push(1); + bytes.push(u8::try_from(packet.len()).unwrap()); + bytes.extend_from_slice(packet); + bytes +} + +/// An EBML header naming `doc_type`. +fn ebml(doc_type: &[u8]) -> Vec { + let mut bytes = vec![0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x86, 0x81, 0x01, 0x42, 0x82]; + bytes.push(0x80 | u8::try_from(doc_type.len()).unwrap()); + bytes.extend_from_slice(doc_type); + bytes +} + +#[test] +fn names_fixed_prefixes() { + let cases: &[(&[u8], Kind)] = &[ + (b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR", Kind::Png), + (b"\xff\xd8\xff\xe0\0\x10JFIF", Kind::Jpeg), + (b"GIF89a\x01\0\x01\0", Kind::Gif), + (b"GIF87a", Kind::Gif), + (b"\xff\x0a\xfa", Kind::Jxl), + (b"\x00\x00\x00\x0cJXL \r\n\x87\n", Kind::Jxl), + (b"qoif\0\0\0\x01", Kind::Qoi), + (b"\x76\x2f\x31\x01\x02\0\0\0", Kind::OpenExr), + (b"fLaC\0\0\0\x22", Kind::Flac), + (b"MThd\0\0\0\x06\0\x01", Kind::Midi), + (b"#!AMR\n", Kind::Amr), + (b"#!AMR-WB\n", Kind::Amr), + (b"FLV\x01\x05", Kind::Flv), + (b"\x00\x00\x01\xba\x44", Kind::MpegPs), + (b"glTF\x02\0\0\0", Kind::Glb), + (b"Kaydara FBX Binary \0\x1a\0", Kind::Fbx), + (b"ply\nformat ascii 1.0\n", Kind::Ply), + (b"ply\r\nformat binary_little_endian 1.0\r\n", Kind::Ply), + (b"PK\x03\x04\x14\0", Kind::Zip), + (b"PK\x05\x06\0\0\0\0", Kind::Zip), + (b"7z\xbc\xaf\x27\x1c\0\x04", Kind::SevenZip), + (b"Rar!\x1a\x07\x00", Kind::Rar), + (b"Rar!\x1a\x07\x01\x00", Kind::Rar), + (b"\x1f\x8b\x08\0", Kind::Gzip), + (b"\xfd7zXZ\x00\x00", Kind::Xz), + (b"\x28\xb5\x2f\xfd\x24", Kind::Zstd), + (b"070701000000", Kind::Cpio), + (b"070707000000", Kind::Cpio), + (b"\xc7\x71\0\0", Kind::Cpio), + (b"\x7fELF\x02\x01\x01", Kind::Elf), + (b"\xcf\xfa\xed\xfe\x07\0\0\x01", Kind::MachO), + (b"\xfe\xed\xfa\xce", Kind::MachO), + (b"\xca\xfe\xba\xbf\0\0\0\x02", Kind::MachOFat), + (b"\0asm\x01\0\0\0", Kind::Wasm), + (b"dex\n035\0", Kind::Dex), + (b"SQLite format 3\0\x10\0", Kind::Sqlite), + (b"bplist00\xd1", Kind::BinaryPlist), + (b"%PDF-1.7\n", Kind::Pdf), + (b"ID3\x04\0\0\0\0\0\0", Kind::Mp3), + ]; + for &(bytes, kind) in cases { + assert_eq!(identify(bytes), Some(kind), "{bytes:02x?}"); + } +} + +#[test] +fn java_class_and_universal_mach_o_share_cafebabe() { + assert_eq!(identify(b"\xca\xfe\xba\xbe\0\0\0\x02"), Some(Kind::MachOFat)); + assert_eq!(identify(b"\xca\xfe\xba\xbe\0\0\0\x2c"), Some(Kind::MachOFat)); + // JDK 1.1 (45.3) and Java 21 (65.0). + assert_eq!(identify(b"\xca\xfe\xba\xbe\0\x03\0\x2d"), Some(Kind::JavaClass)); + assert_eq!(identify(b"\xca\xfe\xba\xbe\0\0\0\x41"), Some(Kind::JavaClass)); + assert_eq!(identify(b"\xca\xfe\xba\xbe"), None); +} + +#[test] +fn checks_fields_after_short_prefixes() { + assert_eq!(identify(b"II*\0\x08\0\0\0"), Some(Kind::Tiff)); + assert_eq!(identify(b"MM\0*\0\0\0\x08"), Some(Kind::Tiff)); + assert_eq!(identify(b"II+\0\x08\0\0\0"), Some(Kind::Tiff)); + assert_eq!(identify(b"8BPS\0\x01\0\0"), Some(Kind::Psd)); + assert_eq!(identify(b"8BPS\0\x02\0\0"), Some(Kind::Psd)); + assert_eq!(identify(b"8BPS\0\x03\0\0"), None); + assert_eq!(identify(b"\0\0\x01\0\x01\0\x10\x10\0\0"), Some(Kind::Ico)); + assert_eq!(identify(b"\0\0\x01\0\0\0\x10\x10\0\0"), None); + assert_eq!(identify(b"BZh91AY&SY"), Some(Kind::Bzip2)); + assert_eq!(identify(b"BZh0"), None); +} + +#[test] +fn tells_pe_from_dos_programs() { + let mut pe = padded(b"MZ\x90\0", 0x84); + pe[0x3c] = 0x80; + pe[0x80..0x84].copy_from_slice(b"PE\0\0"); + assert_eq!(identify(&pe), Some(Kind::Pe)); + // The PE header lies past the bytes given, or isn't there at all. + pe[0x3c] = 0xf0; + assert_eq!(identify(&pe), Some(Kind::DosExecutable)); + let mut far = padded(b"MZ", 0x40); + far[0x3c..0x40].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(identify(&far), Some(Kind::DosExecutable)); + assert_eq!(identify(b"MZ"), Some(Kind::DosExecutable)); +} + +#[test] +fn bmp_needs_reserved_zeros_and_a_known_dib_header() { + let mut bmp = padded(b"BM\x36\0\0\0", 54); + bmp[10] = 54; + bmp[14] = 40; + assert_eq!(identify(&bmp), Some(Kind::Bmp)); + bmp[14] = 41; + assert_eq!(identify(&bmp), None); + assert_eq!(identify(b"BMW is a car brand, not a bitmap."), None); +} + +#[test] +fn names_riff_and_form_containers_by_form_type() { + assert_eq!(identify(&riff(b"RIFF", b"WAVE")), Some(Kind::Wav)); + assert_eq!(identify(&riff(b"RIFF", b"WEBP")), Some(Kind::WebP)); + assert_eq!(identify(&riff(b"RIFF", b"AVI ")), Some(Kind::Avi)); + assert_eq!(identify(&riff(b"FORM", b"AIFF")), Some(Kind::Aiff)); + assert_eq!(identify(&riff(b"FORM", b"AIFC")), Some(Kind::Aiff)); + assert_eq!(identify(&riff(b"RIFF", b"CDXA")), None); +} + +#[test] +fn names_ogg_streams_by_first_packet() { + assert_eq!(identify(&ogg(b"OpusHead\x01\x02")), Some(Kind::Opus)); + assert_eq!(identify(&ogg(b"\x80theora\x03\x02")), Some(Kind::OggVideo)); + assert_eq!(identify(&ogg(b"\x01vorbis\0\0")), Some(Kind::Ogg)); + // A page cut off before its segment table is still Ogg. + assert_eq!(identify(&ogg(b"")[..27]), Some(Kind::Ogg)); + assert_eq!(identify(b"OggS\0\x02"), None); +} + +#[test] +fn names_ebml_by_doc_type() { + assert_eq!(identify(&ebml(b"webm")), Some(Kind::WebM)); + assert_eq!(identify(&ebml(b"matroska")), Some(Kind::Matroska)); + // No readable DocType: still Matroska's container. + assert_eq!(identify(&[0x1a, 0x45, 0xdf, 0xa3, 0x42, 0x82, 0x04]), Some(Kind::Matroska)); + assert_eq!(identify(&[0x1a, 0x45, 0xdf, 0xa3, 0x42, 0x82, 0x84, b'w']), Some(Kind::Matroska)); +} + +#[test] +fn names_iso_bmff_by_brand() { + assert_eq!(identify(&ftyp(b"isom", &[b"isom", b"iso2", b"avc1", b"mp41"])), Some(Kind::Mp4)); + assert_eq!(identify(&ftyp(b"mp42", &[])), Some(Kind::Mp4)); + assert_eq!(identify(&ftyp(b"qt ", &[b"qt "])), Some(Kind::QuickTime)); + assert_eq!(identify(&ftyp(b"M4A ", &[b"isom"])), Some(Kind::M4a)); + assert_eq!(identify(&ftyp(b"3gp5", &[b"isom"])), Some(Kind::ThreeGp)); + assert_eq!(identify(&ftyp(b"heic", &[b"mif1"])), Some(Kind::Heif)); + assert_eq!(identify(&ftyp(b"jxl ", &[])), Some(Kind::Jxl)); + // A still image lists generic brands too; the image brand wins wherever it is. + assert_eq!(identify(&ftyp(b"mif1", &[b"avif", b"miaf"])), Some(Kind::Avif)); + assert_eq!(identify(&ftyp(b"isom", &[b"heic"])), Some(Kind::Heif)); + // Unknown brands (a camera's raw container, say) are not guessed at. + assert_eq!(identify(&ftyp(b"crx ", &[b"isom"])), Some(Kind::Mp4)); + assert_eq!(identify(&ftyp(b"crx ", &[])), None); + let mut short = ftyp(b"isom", &[]); + short[3] = 12; + assert_eq!(identify(&short), None); +} + +#[test] +fn names_old_quicktime_atoms() { + assert_eq!(identify(b"\0\0\0\x08wide\0\0\0\0mdat"), Some(Kind::QuickTime)); + assert_eq!(identify(b"\0\0\x10\0moov"), Some(Kind::QuickTime)); + assert_eq!(identify(b"\0\0\0\x04moov"), None); + assert_eq!(identify(b"\0\0\0\x10free"), None); +} + +#[test] +fn names_tar_by_its_ustar_magic() { + let mut tar = padded(b"hello.txt", 512); + tar[257..263].copy_from_slice(b"ustar\0"); + assert_eq!(identify(&tar), Some(Kind::Tar)); + tar[257..263].copy_from_slice(b"ustar "); + assert_eq!(identify(&tar), Some(Kind::Tar)); + tar[257..263].copy_from_slice(b"ustaR "); + assert_eq!(identify(&tar), None); +} + +#[test] +fn transport_streams_need_three_packets() { + let mut ts = vec![0u8; 3 * 188]; + for n in 0..3 { + ts[n * 188] = 0x47; + } + assert_eq!(identify(&ts), Some(Kind::MpegTs)); + assert_eq!(identify(&ts[..377]), Some(Kind::MpegTs)); + assert_eq!(identify(&ts[..376]), None); +} + +#[test] +fn reads_mpeg_audio_frame_headers() { + // MPEG-1, MPEG-2 and MPEG-2.5 layer III. + assert_eq!(identify(b"\xff\xfb\x90\x64"), Some(Kind::Mp3)); + assert_eq!(identify(b"\xff\xf3\x48\xc4"), Some(Kind::Mp3)); + assert_eq!(identify(b"\xff\xe3\x18\xc4"), Some(Kind::Mp3)); + // Reserved version, bad bitrate, reserved sampling rate, layers I and II. + assert_eq!(identify(b"\xff\xeb\x90\x64"), None); + assert_eq!(identify(b"\xff\xfb\xf0\x64"), None); + assert_eq!(identify(b"\xff\xfb\x9c\x64"), None); + assert_eq!(identify(b"\xff\xff\x90\x64"), None); + assert_eq!(identify(b"\xff\xfd\x90\x64"), None); + // ADTS AAC, MPEG-4 and MPEG-2 ids; a reserved sampling index is not AAC. + assert_eq!(identify(b"\xff\xf1\x50\x80"), Some(Kind::Aac)); + assert_eq!(identify(b"\xff\xf9\x4c\x80"), Some(Kind::Aac)); + assert_eq!(identify(b"\xff\xf1\x7c\x80"), None); + assert_eq!(identify(b"\xff\x81\x50"), None); + assert_eq!(identify(b"\xff\xf1"), None); +} + +#[test] +fn names_nothing_without_a_signature() { + for bytes in [&b""[..], b"\0", b"hello, world\n", b"; FBX 7.3.0 project file", b"plywood", b"name,value\n1,2\n", b""] { + assert_eq!(identify(bytes), None, "{bytes:?}"); + } +} + +#[test] +fn reads_only_the_head() { + let mut late = vec![0u8; HEAD + 600]; + late[HEAD + 257..HEAD + 263].copy_from_slice(b"ustar\0"); + assert_eq!(identify(&late), None); + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + png.resize(HEAD * 4, 0xff); + assert_eq!(identify(&png), Some(Kind::Png)); +} + +#[test] +fn every_prefix_of_every_sample_is_safe() { + let mut samples = vec![ftyp(b"mif1", &[b"avif"]), ebml(b"webm"), ogg(b"OpusHead"), riff(b"RIFF", b"WAVE")]; + samples.push(padded(b"MZ", 0x100)); + samples.push((0..=255).collect()); + for sample in &samples { + for end in 0..=sample.len() { + let _ = identify(&sample[..end]); + } + } +} + +#[test] +fn every_kind_has_a_type_extension_and_family() { + let mut seen = std::collections::HashSet::new(); + for kind in Kind::ALL { + assert!(seen.insert(kind), "{kind:?} listed twice"); + let mime = kind.mime_type(); + let (top, sub) = mime.split_once('/').unwrap(); + assert!(!top.is_empty() && !sub.is_empty(), "{kind:?}: {mime}"); + assert!(!kind.extension().is_empty() && !kind.extension().starts_with('.')); + assert_eq!(kind.extension(), kind.extension().to_ascii_lowercase()); + let expected = match top { + "image" => Family::Image, + "audio" => Family::Audio, + "video" => Family::Video, + "model" => Family::Model, + _ => kind.family(), + }; + assert_eq!(kind.family(), expected, "{kind:?}"); + } +} + +#[test] +fn media_types_apps_send() { + assert_eq!(Kind::Png.mime_type(), "image/png"); + assert_eq!(Kind::Jpeg.extension(), "jpg"); + assert_eq!(Kind::Wav.mime_type(), "audio/wav"); + assert_eq!(Kind::Flac.mime_type(), "audio/flac"); + assert_eq!(Kind::Glb.mime_type(), "model/gltf-binary"); + assert_eq!(Kind::Fbx.mime_type(), "model/fbx"); + assert_eq!(Kind::Ply.mime_type(), "model/ply"); + assert_eq!(Kind::Mp4.family(), Family::Video); + assert_eq!(Kind::Zip.family(), Family::Archive); + assert_eq!(Kind::Gzip.family(), Family::Compressed); + assert_eq!(Kind::Elf.family(), Family::Executable); + assert_eq!(Kind::Sqlite.family(), Family::Database); + assert_eq!(Kind::Pdf.family(), Family::Document); +} diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml index 3b778da47..b35c4de41 100644 --- a/crates/svg/Cargo.toml +++ b/crates/svg/Cargo.toml @@ -13,14 +13,16 @@ workspace = true [dependencies] craft-xml = { workspace = true } quick-xml = { version = "0.41", default-features = false } -craft-image-codecs = { workspace = true } +# Probing and bounding raster images embedded in a parsed tree (`tree`). +craft-image-codecs = { workspace = true, optional = true } # SVGZ (gzip), pure-Rust backend. flate2 = { version = "1", default-features = false, features = ["rust_backend"] } kurbo = { workspace = true } roxmltree = "0.21.2" thiserror = { workspace = true } -vectorcraft-geom = { workspace = true } -vectorcraft-pathops = { workspace = true } +# The geometry reader (`paths`) builds and cleans its outlines with these. +vectorcraft-geom = { workspace = true, optional = true } +vectorcraft-pathops = { workspace = true, optional = true } # The full SVG parser (Apache-2.0 OR MIT). The one version every Craft app uses: apps depend on # craft-svg's `tree` feature and take usvg from its re-export, never directly. usvg = { version = "=0.48.1", default-features = false, optional = true } @@ -31,8 +33,13 @@ resvg = { version = "=0.48.1", default-features = false, features = ["text", "ra data-url = { version = "0.3.2", optional = true } [features] +default = ["paths"] +# `craft_svg::paths`: the geometry reader (Bézier paths in millimetres) for cutters, plotters and +# CAD import. Off, the crate is SVG text, guards, path data, colours and the writer only, with no +# app crates (vectorcraft) among its dependencies. +paths = ["dep:vectorcraft-geom", "dep:vectorcraft-pathops"] # `craft_svg::tree`: usvg behind the bounded entry point. -tree = ["dep:usvg", "dep:data-url"] +tree = ["paths", "dep:usvg", "dep:data-url", "dep:craft-image-codecs"] # Text converted to paths (needs a font database in the options). tree-text = ["tree", "usvg/text"] # The system's fonts in `usvg::fontdb`. diff --git a/crates/svg/src/color.rs b/crates/svg/src/color.rs index 14588bc90..610ca6aa1 100644 --- a/crates/svg/src/color.rs +++ b/crates/svg/src/color.rs @@ -65,12 +65,20 @@ pub struct ColorParseError { /// and SVG 2 paint both defer to): the 148 named colours, `#rgb`, `#rgba`, `#rrggbb`, /// `#rrggbbaa`, `rgb()`/`rgba()` with integer 0–255 or percentage channels, and /// `hsl()`/`hsla()`, in comma or space syntax with an optional alpha (`/ a` or a fourth -/// comma value). Alpha is checked and then dropped. +/// comma value). Alpha is checked and then dropped; [`parse_color_alpha`] keeps it. /// /// Anything else — `transparent`, `currentColor`, `color()`, out-of-range channels — /// is an error, not a guess: silent color loss is worse than a reported failure. /// Callers that understand `none`/`currentColor` (SVG paint) resolve them first. pub fn parse_color(input: &str) -> Result { + parse_color_alpha(input).map(|(color, _)| color) +} + +/// [`parse_color`], with the colour's alpha in `[0, 1]`: from `#rgba` and `#rrggbbaa` (in +/// 255ths), from the fourth value of `rgb()`/`hsl()` (a number or a percentage), else 1. A +/// colour with an alpha of its own (a fill of `rgba(…)`) multiplies with `fill-opacity`; that +/// is the caller's to do. +pub fn parse_color_alpha(input: &str) -> Result<(Color, f32), ColorParseError> { let err = || ColorParseError { input: input.to_string() }; // Cap input length before work: a caller feeding untrusted text cannot // make this quadratic. @@ -84,30 +92,27 @@ pub fn parse_color(input: &str) -> Result { if let Some((name, rest)) = lower.split_once('(') { return parse_function(name.trim(), rest).ok_or_else(err); } - NAMED.iter().find(|(name, _)| *name == lower).map(|(_, rgb)| Color::new((rgb >> 16) as u8, (rgb >> 8) as u8, *rgb as u8)).ok_or_else(err) + NAMED.iter().find(|(name, _)| *name == lower).map(|(_, rgb)| (Color::new((rgb >> 16) as u8, (rgb >> 8) as u8, *rgb as u8), 1.0)).ok_or_else(err) } -/// Hex digits after `#`: 3 or 4 (each doubled), 6 or 8; the alpha digits are dropped. -fn parse_hex(digits: &str) -> Option { +/// Hex digits after `#`: 3 or 4 (each doubled), 6 or 8; the fourth (or fourth pair) is alpha. +fn parse_hex(digits: &str) -> Option<(Color, f32)> { if !digits.bytes().all(|b| b.is_ascii_hexdigit()) { return None; } let value = |s: &str| u8::from_str_radix(s, 16).ok(); - match digits.len() { - 3 | 4 => { - let d = |i: usize| value(digits.get(i..=i)?).map(|v| v * 17); - Some(Color::new(d(0)?, d(1)?, d(2)?)) - } - 6 | 8 => { - let d = |i: usize| value(digits.get(i..i + 2)?); - Some(Color::new(d(0)?, d(2)?, d(4)?)) - } - _ => None, - } + let channels: Vec = match digits.len() { + 3 | 4 => (0..digits.len()).map(|i| value(digits.get(i..=i)?).map(|v| v * 17)).collect::>()?, + 6 | 8 => (0..digits.len() / 2).map(|i| value(digits.get(2 * i..2 * i + 2)?)).collect::>()?, + _ => return None, + }; + let (&r, &g, &b) = (channels.first()?, channels.get(1)?, channels.get(2)?); + let alpha = channels.get(3).map_or(1.0, |a| f32::from(*a) / 255.0); + Some((Color::new(r, g, b), alpha)) } /// `rgb(…)`, `rgba(…)`, `hsl(…)`, `hsla(…)` after the `(`, which must close with `)`. -fn parse_function(name: &str, rest: &str) -> Option { +fn parse_function(name: &str, rest: &str) -> Option<(Color, f32)> { let body = rest.trim().strip_suffix(')')?; // Legacy comma syntax, or modern space syntax with `/ alpha`. let parts: Vec<&str> = if body.contains(',') { @@ -127,26 +132,29 @@ fn parse_function(name: &str, rest: &str) -> Option { if !(parts.len() == 3 || parts.len() == 4) { return None; } - if let Some(alpha) = parts.get(3) { - let a = match alpha.strip_suffix('%') { + let mut alpha = 1.0; + if let Some(text) = parts.get(3) { + let a = match text.strip_suffix('%') { Some(p) => number(p)? / 100.0, - None => number(alpha)?, + None => number(text)?, }; if !(0.0..=1.0).contains(&a) { return None; } + alpha = a as f32; } let (Some(a), Some(b), Some(c)) = (parts.first(), parts.get(1), parts.get(2)) else { return None; }; - match name { - "rgb" | "rgba" => Some(Color::new(rgb_channel(a)?, rgb_channel(b)?, rgb_channel(c)?)), + let color = match name { + "rgb" | "rgba" => Color::new(rgb_channel(a)?, rgb_channel(b)?, rgb_channel(c)?), "hsl" | "hsla" => { let hue = number(a.strip_suffix("deg").unwrap_or(a))?; - Some(hsl_to_rgb(hue.rem_euclid(360.0), percent(b)?, percent(c)?)) + hsl_to_rgb(hue.rem_euclid(360.0), percent(b)?, percent(c)?) } - _ => None, - } + _ => return None, + }; + Some((color, alpha)) } /// A finite plain decimal number. @@ -372,6 +380,24 @@ mod tests { assert_eq!(c("#abc"), Color::new(0xaa, 0xbb, 0xcc)); } + #[test] + fn alpha_is_kept_by_parse_color_alpha() { + let a = |input: &str| parse_color_alpha(input).ok().expect("test input should parse"); + assert_eq!(a("#ff000080"), (Color::new(255, 0, 0), 128.0 / 255.0)); + assert_eq!(a("#f008"), (Color::new(255, 0, 0), 136.0 / 255.0)); + assert_eq!(a("#f00"), (Color::new(255, 0, 0), 1.0)); + assert_eq!(a("rgba(255, 0, 0, 0.25)"), (Color::new(255, 0, 0), 0.25)); + assert_eq!(a("rgb(0 0 255 / 50%)"), (Color::new(0, 0, 255), 0.5)); + assert_eq!(a("hsla(120, 100%, 50%, 0)"), (Color::new(0, 255, 0), 0.0)); + assert_eq!(a("SteelBlue"), (Color::new(0x46, 0x82, 0xb4), 1.0)); + // parse_color keeps dropping it, and both refuse the same input. + assert_eq!(c("#ff000080"), Color::new(255, 0, 0)); + for bad in ["rgba(255, 0, 0, 1.5)", "#ff00000", "rgb(0 0 0 / -1)", "transparent", "#ggg"] { + assert!(parse_color_alpha(bad).is_err(), "{bad}"); + assert!(parse_color(bad).is_err(), "{bad}"); + } + } + #[test] fn parses_css_keywords() { assert_eq!(c("black"), Color::new(0, 0, 0)); diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs index 25bf880cd..7cd36692e 100644 --- a/crates/svg/src/lib.rs +++ b/crates/svg/src/lib.rs @@ -5,8 +5,8 @@ //! UTF-16 and ISO 8859-1 decoded; and SVGZ compression. //! - [`guard`]: checks on the text before any XML reader or SVG converter sees it: how deep the //! elements nest, and what the document type's entities expand to. -//! - [`paths`]: the geometry of a document as Bézier paths in millimetres, for cutters, plotters -//! and CAD import. +//! - `paths` (feature `paths`, on by default): the geometry of a document as Bézier paths in +//! millimetres, for cutters, plotters and CAD import. //! - [`color`]: CSS colours. //! - [`write`]: a writer for SVG documents, their path data and numbers. //! - `tree` (feature `tree`): the full parser. usvg at the one version the Craft apps share, @@ -22,6 +22,7 @@ pub mod drawing_write; pub mod guard; pub mod neutral_document; pub mod path_data; +#[cfg(feature = "paths")] pub mod paths; pub mod preflight; pub mod sniff; diff --git a/crates/svg/src/write.rs b/crates/svg/src/write.rs index 66cb1d6a0..4051c081f 100644 --- a/crates/svg/src/write.rs +++ b/crates/svg/src/write.rs @@ -362,6 +362,7 @@ mod tests { } #[test] + #[cfg(feature = "paths")] fn writes_a_document_the_reader_reads() { let mut w = SvgWriter::new(); assert!(!w.is_empty()); diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 9d8e1b785..998f8ed88 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -36,8 +36,9 @@ were. Part names and relationship ids are kept, so a file read and written witho where it did, and writing it twice gives the same bytes. The calculation chain is dropped (Excel rebuilds it). `XlsxKind` is the workbook part's content type (`.xlsx`, `.xlsm`, `.xltx`, `.xltm`). Typed worksheet features (`SheetPart::features`), each mirroring its element with optional -attributes and the unnamed ones kept: sheet views (panes, selections), print setup (options, -margins, page setup, header and footer, page breaks), data validation (also read from the `x14` +attributes and the unnamed ones kept: sheet views (panes, selections), the default sizes +(`sheetFormatPr`), print setup (options, margins, page setup, header and footer, page breaks), +data validation (also read from the `x14` extension), conditional formats (colour scales, data bars, icon sets, rule extensions kept), hyperlinks (with their URLs), the legacy drawing. Comments (`Comments`, from the comments part) and the note shapes of the VML drawing (`NoteShape`). A sheet that gains comments and has no VML @@ -83,13 +84,15 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1 The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. -A binary workbook gives the same `Workbook`, strings, rows and merged cells, and the same `Styles`: +A binary workbook gives the same `Workbook`, strings, rows (with their heights, outline levels, +hidden flags and styles), columns, default sizes and merged cells, and the same `Styles`: number formats, fonts, fills, borders, cell and cell style formats, cell styles, differential formats and the indexed colours, as Excel writes them in `styles.xml` (default fill colours and empty settings left out, a cell style format's `apply*="0"`, a number format with a newer locale tag in its Excel 2007 spelling, as `formatCode` has it). Custom table styles and recent colours are not read; a damaged styles part is a warning and no styles, while one past a cap (65,536 entries in a -table, 1,024 gradient stops) or past `binary_text` is too large. Its cell formulas are decoded from +table, 1,024 gradient stops) or past `binary_text` is too large. The descent Excel records for rows +and the sheet (`x14ac:dyDescent`) is in alternate-content blocks, which are skipped. Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the formula, numbers as Excel spells them), with `FormulaKind::Shared` or @@ -111,7 +114,8 @@ are this crate's own, apart from `craft-xls`'s. past it, formulas keep their saved results. A formula's tokens are bounded at 16 KB, its nesting at 256 levels and its array constants at 65,536 values in all, each charged before it is held; once the formula text allowance is spent, no formula is decoded. Fixed caps: 1024 - sheets, 100,000 merged ranges per sheet, 64 nested extension blocks. + sheets, 100,000 merged ranges and 16,384 column records per sheet, 64 nested extension blocks. A + row taller than 8,192 twips keeps the default height. - Cells and rows outside the 1,048,576 × 16,384 grid are dropped and counted. SpreadsheetML is read leniently: a damaged optional part is a warning, a damaged worksheet keeps diff --git a/crates/xlsx/src/features.rs b/crates/xlsx/src/features.rs index 17c2c2aab..9bae562b5 100644 --- a/crates/xlsx/src/features.rs +++ b/crates/xlsx/src/features.rs @@ -1,6 +1,7 @@ -//! The worksheet's typed features, each mirroring its SpreadsheetML element: sheet views, print -//! setup, data validation, conditional formatting, hyperlinks (ECMA-376 Part 1, 18.3), and the -//! comments part (18.7) with the legacy note shapes Excel draws comments with. +//! The worksheet's typed features, each mirroring its SpreadsheetML element: sheet views, the +//! sheet's default sizes, print setup, data validation, conditional formatting, hyperlinks +//! (ECMA-376 Part 1, 18.3), and the comments part (18.7) with the legacy note shapes Excel draws +//! comments with. //! //! An attribute the file may leave out is an `Option` (`None`: not written, the schema's //! default applies), so a feature reads and writes back as it was. Attributes a type does not @@ -213,6 +214,27 @@ impl SheetView { } } +// ------------------------------------------------------------------------------- sheet format + +element! { + /// ``: the sheet's default column width and row height, and its outline depth. + pub struct SheetFormat { + /// Characters of the default font's maximum digit width, padding excluded (8 when absent). + base_col_width: u32 = "baseColWidth", + /// Characters, padding included, like [`crate::Column::width`]. + default_col_width: f64 = "defaultColWidth", + /// Points. + default_row_height: f64 = "defaultRowHeight", + custom_height: bool = "customHeight", + /// Rows are hidden unless they say otherwise. + zero_height: bool = "zeroHeight", + thick_top: bool = "thickTop", + thick_bottom: bool = "thickBottom", + outline_level_row: u32 = "outlineLevelRow", + outline_level_col: u32 = "outlineLevelCol", + } +} + // ------------------------------------------------------------------------------- printing element! { @@ -643,6 +665,8 @@ impl Hyperlink { #[derive(Clone, Debug, Default, PartialEq)] pub struct Features { pub views: Vec, + /// `sheetFormatPr`. + pub format: Option, pub print: Print, pub validations: Option, pub conditional_formats: Vec, @@ -660,6 +684,7 @@ impl Features { let attrs_of = |n: &str| root.child(n); Features { views: root.child("sheetViews").map(|v| v.children("sheetView").map(SheetView::read).collect()).unwrap_or_default(), + format: attrs_of("sheetFormatPr").map(SheetFormat::read_attrs), print: Print { options: attrs_of("printOptions").map(PrintOptions::read_attrs), margins: attrs_of("pageMargins").map(PageMargins::read_attrs), @@ -708,6 +733,7 @@ impl Features { let p = &self.print; vec![ ("sheetViews", Vec::from_iter((!self.views.is_empty()).then_some(views))), + ("sheetFormatPr", Vec::from_iter(self.format.as_ref().map(|f| el("sheetFormatPr", &|e| f.write_attrs(e))))), ("conditionalFormatting", self.conditional_formats.iter().map(ConditionalFormatting::to_el).collect()), ("dataValidations", Vec::from_iter(validations)), ("hyperlinks", Vec::from_iter((!self.hyperlinks.is_empty()).then_some(links))), diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index 5a64ae0ab..9105ba886 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -75,8 +75,8 @@ pub use document::{Document, DrawingPart, KeptPart, PivotCachePart, PivotTablePa pub use drawing::{Anchor, Drawing, Item, Marker, ObjectKind, Placement}; pub use features::{ Break, CfRule, Cfvo, ColorScale, Comment, Comments, ConditionalFormatting, DataBar, DataValidation, DataValidations, Features, HeaderFooter, - Hyperlink, IconSet, NoteShape, PageMargins, PageSetup, Pane, Print, PrintOptions, Selection, SheetView, Table, TableColumn, TableStyleInfo, - notes_vml, parse_sqref, range_ref, + Hyperlink, IconSet, NoteShape, PageMargins, PageSetup, Pane, Print, PrintOptions, Selection, SheetFormat, SheetView, Table, TableColumn, + TableStyleInfo, notes_vml, parse_sqref, range_ref, }; pub use package::{Package, is_xlsb}; pub use pivot::{ diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index c8fdba69f..de623537a 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,8 +1,8 @@ -//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from -//! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an -//! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why. -//! The styles part is read into the same [`crate::styles::Styles`] ([`styles`]). A record that -//! does not parse fails the read. +//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheets (rows, cells, +//! columns, default sizes, merged cells), from their records. Cell formulas are decoded from +//! their tokens ([`formula`]) into the text an `.xlsx` file holds; one that can't be keeps only +//! its saved result, with a warning saying why. The styles part is read into the same +//! [`crate::styles::Styles`] ([`styles`]). A record that does not parse fails the read. mod expr; mod formula; @@ -15,7 +15,8 @@ use std::sync::Arc; use craft_opc::{ContentTypes, Rels}; use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS, Range, Visibility}; -use crate::sheet::{Cell, CellFormula, CellValue, FormulaKind, Row, SheetPart}; +use crate::features::SheetFormat; +use crate::sheet::{Cell, CellFormula, CellValue, Column, FormulaKind, Row, SheetPart}; use crate::strings::StringItem; use crate::workbook::{SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; @@ -35,6 +36,10 @@ const MAX_SHEETS: usize = 1024; const MAX_BOOKS: usize = 65_536; /// Most merged ranges in one worksheet. const MAX_MERGES: usize = 100_000; +/// Most column records in one worksheet: one per column. +const MAX_COLUMNS: usize = MAX_COLS as usize; +/// The tallest row, in twips (409.6 points). +const MAX_ROW_TWIPS: u16 = 8192; /// Defined names (Excel's hidden ones for newer functions included) read at most; past it, the /// workbook's formulas keep their saved results. const MAX_NAMES: usize = 65_536; @@ -766,7 +771,87 @@ fn value(id: u16, b: &mut Bytes<'_>, strings: usize, limits: &Limits, budget: &B Ok(value) } -/// A worksheet part: rows (BrtRowHdr) and cells inside BrtBeginSheetData, merged cells after. +/// `BrtRowHdr` after `rw`, up to its column spans: the row's height, flags and style, as the +/// `.xlsx` row Excel writes. `miyRw` is the height of every row: it is `ht` when set by hand +/// (`fUnsynced`) or when it differs from the sheet's default, and past 8192 twips it is ignored. +fn row_header(index: u32, b: &mut Bytes<'_>, default_twips: Option) -> Result { + let ixfe = b.u32()?; + let twips = b.u16()?; + let [borders, flags, phonetic] = [b.u8()?, b.u8()?, b.u8()?]; + let custom = flags & 0x20 != 0; + let height = (twips <= MAX_ROW_TWIPS && (custom || default_twips.is_some_and(|d| d != twips))).then(|| f64::from(twips) / 20.0); + let mut attrs = vec![]; + for (bit, byte, name) in [(1, borders, "thickTop"), (2, borders, "thickBot"), (1, phonetic, "ph")] { + if byte & bit != 0 { + attrs.push((name.to_string(), "1".to_string())); + } + } + Ok(Row { + index, + height, + custom_height: custom && height.is_some(), + hidden: flags & 0x10 != 0, + outline_level: flags & 7, + collapsed: flags & 0x08 != 0, + style: (flags & 0x40 != 0).then_some(ixfe), + cells: vec![], + attrs, + }) +} + +/// `BrtWsFmtInfo`: `sheetFormatPr` as Excel writes it, and the default row height in twips. +/// `dxGCol` is the default width in 256ths of a character (`0xFFFFFFFF`: none set); +/// `cchDefColWidth` is the base width, left out at its default of 8. +fn sheet_format(b: &mut Bytes<'_>) -> Result<(SheetFormat, u16), Error> { + let width = b.u32()?; + let base = b.u16()?; + let twips = b.u16()?; + let flags = b.u16()?; + let rows = b.u8()?; + let cols = b.u8()?; + let set = |bit: u16| (flags & bit != 0).then_some(true); + let level = |n: u8| (n > 0).then_some(u32::from(n.min(7))); + let format = SheetFormat { + base_col_width: (base != 8).then_some(u32::from(base)), + default_col_width: (width != u32::MAX).then(|| f64::from(width) / 256.0), + default_row_height: Some(f64::from(twips) / 20.0), + custom_height: set(1), + zero_height: set(2), + thick_top: set(4), + thick_bottom: set(8), + outline_level_row: level(rows), + outline_level_col: level(cols), + attrs: vec![], + }; + Ok((format, twips)) +} + +/// `BrtColInfo`: `None` for a range outside the grid or backwards. `coldx` is the width in 256ths +/// of a character, which is the `.xlsx` width exactly. +fn column(b: &mut Bytes<'_>) -> Result, Error> { + let first = b.u32()?; + let last = b.u32()?; + let width = b.u32()?; + let style = b.u32()?; + let flags = b.u16()?; + if first > last || last >= MAX_COLS { + return Ok(None); + } + Ok(Some(Column { + first, + last, + width: Some(f64::from(width) / 256.0), + custom_width: flags & 2 != 0, + best_fit: flags & 4 != 0, + hidden: flags & 1 != 0, + outline_level: ((flags >> 8) & 7) as u8, + collapsed: flags & 0x1000 != 0, + style: (style != 0).then_some(style), + })) +} + +/// A worksheet part: the sheet's default sizes (BrtWsFmtInfo) and columns (BrtColInfo) before +/// BrtBeginSheetData, rows (BrtRowHdr) and cells inside it, merged cells after. /// Formulas refer to the workbook's `links`; with none, each keeps its saved result. #[allow(clippy::too_many_arguments)] pub(crate) fn worksheet( @@ -783,8 +868,13 @@ pub(crate) fn worksheet( let mut sheet = SheetPart::default(); let mut data = false; let mut data_seen = false; + // Every BrtColInfo counts toward the cap, a skipped one too. + let mut column_records = 0usize; let mut row: Option = None; let mut col = None; + // The default row height in twips: a row of another height has `ht` even when it was not set + // by hand (a large font, a thick border), as Excel writes it in `.xlsx`. + let mut default_twips: Option = None; // BrtCellMeta applies to the cell record that follows it (a dynamic-array formula), as `cm` // does in `.xlsx`: a one-based index into the metadata part's cell metadata. Excel may write // the cell's BrtValueMeta (a rich value, such as a spill error) in between. @@ -853,15 +943,14 @@ pub(crate) fn worksheet( if r >= MAX_ROWS || row.as_ref().is_some_and(|previous| r <= previous.index) { return Err(invalid("row is out of bounds or unordered")); } - // ixfe, miyRw and the flags are not read. - b.take(9)?; + let next = row_header(r, &mut b, default_twips)?; let spans = b.u32()?; if spans > 16 { return Err(invalid("too many row spans")); } b.take(spans as usize * 8)?; b.finish()?; - if let Some(previous) = row.replace(Row { index: r, ..Row::default() }) { + if let Some(previous) = row.replace(next) { each(previous)?; } sheet.rows += 1; @@ -920,6 +1009,31 @@ pub(crate) fn worksheet( }); col = Some(c); } + 485 => { + if data_seen { + return Err(invalid("sheet format after worksheet data")); + } + let (format, twips) = sheet_format(&mut b)?; + b.finish()?; + default_twips = Some(twips); + sheet.features.format = Some(format); + } + 60 => { + if data_seen { + return Err(invalid("column after worksheet data")); + } + if column_records >= MAX_COLUMNS { + return Err(Error::TooLarge(format!("XLSB exceeds {MAX_COLUMNS} column records"))); + } + column_records += 1; + let column = column(&mut b)?; + b.finish()?; + let Some(column) = column else { + warnings.warn("invalid XLSB column ranges were skipped"); + continue; + }; + sheet.columns.push(column); + } 176 => { if data { return Err(invalid("merge inside worksheet data")); diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 18a833388..1205c1e68 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -6,8 +6,8 @@ mod common; use common::{read, read_with}; use craft_xlsx::testing::zip; use craft_xlsx::{ - CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, SheetKind, Styles, - Value, Visibility, is_xlsb, + CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, Row, SheetKind, + SheetPart, Styles, Value, Visibility, is_xlsb, }; const ROOT: &str = r#""#; @@ -934,6 +934,136 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() { assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings); } +/// `BrtRowHdr` with its style, height and three flag bytes, and no column spans. +fn row_hdr(r: u32, ixfe: u32, twips: u16, flags: [u8; 3]) -> Vec { + record(0, &[&r.to_le_bytes()[..], &ixfe.to_le_bytes(), &twips.to_le_bytes(), &flags, &0u32.to_le_bytes()].concat()) +} +/// `BrtColInfo`. +fn col_info(first: u32, last: u32, coldx: u32, ixfe: u32, flags: u16) -> Vec { + record(60, &[&first.to_le_bytes()[..], &last.to_le_bytes(), &coldx.to_le_bytes(), &ixfe.to_le_bytes(), &flags.to_le_bytes()].concat()) +} +/// `BrtWsFmtInfo`. +fn ws_fmt_info(dx: u32, base: u16, twips: u16, flags: u16, levels: [u8; 2]) -> Vec { + record(485, &[&dx.to_le_bytes()[..], &base.to_le_bytes(), &twips.to_le_bytes(), &flags.to_le_bytes(), &levels].concat()) +} +/// A worksheet with `head` before its data (the sheet format and columns) and `rows` in it. +fn sized_sheet(head: &[u8], rows: &[u8]) -> Vec { + [&record(129, &[])[..], head, &record(145, &[]), rows, &record(146, &[]), &record(130, &[])].concat() +} +fn sizes_of(bytes: &[u8]) -> Result<(Vec, SheetPart, Vec), Error> { + let mut p = Package::open(bytes, &Limits::default())?; + let wb = p.workbook()?; + let mut rows = vec![]; + let part = p.sheet(&wb.sheets[0], |r| { + rows.push(r); + Ok(()) + })?; + Ok((rows, part, p.warnings().to_vec())) +} +fn xml_sheet(xml: &str) -> (Vec, SheetPart) { + let types = r#""#; + let root = r#""#; + let rels = r#""#; + let book = r#""#; + let bytes = zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", root.as_bytes()), + ("xl/workbook.xml", book.as_bytes()), + ("xl/_rels/workbook.xml.rels", rels.as_bytes()), + ("xl/sheet.xml", xml.as_bytes()), + ]); + let (rows, part, _) = sizes_of(&bytes).unwrap(); + (rows, part) +} + +/// The binary sheet below as Excel spells it in `.xlsx`. Excel writes `ht` for a row whose +/// height differs from the default even when it was not set by hand. +const SIZES_XML: &str = r#"1"#; + +fn sizes_bin() -> Vec { + let head = [ + ws_fmt_info(3225, 10, 285, 0x0A, [2, 1]), + record(390, &[]), + col_info(2, 2, 5273, 0, 0x0002), + col_info(3, 5, 0, 0, 0x0103), + col_info(7, 7, 2321, 7, 0), + col_info(8, 8, 6075, 0, 0x0006), + col_info(9, 9, 2321, 0, 0x1000), + col_info(16_383, 16_383, 7833, 0, 0x0002), + record(391, &[]), + ] + .concat(); + let rows = [ + row_hdr(0, 0, 285, [0, 0, 0]), + cell(2, 0, &((1u32 << 2) | 2).to_le_bytes()), + row_hdr(1, 0, 600, [0, 0x20, 0]), + row_hdr(2, 0, 533, [0, 0, 0]), + row_hdr(3, 0, 285, [0, 0x10, 0]), + row_hdr(4, 0, 300, [3, 0, 0]), + row_hdr(5, 0, 285, [0, 2, 0]), + row_hdr(6, 0, 285, [0, 0x11, 0]), + row_hdr(7, 0, 285, [0, 0x08, 0]), + row_hdr(8, 3, 285, [0, 0x40, 0]), + // A style without fGhostDirty is not applied. + row_hdr(9, 5, 285, [0, 0, 0]), + row_hdr(10, 0, 285, [0, 0, 1]), + // Past 8192 twips the height is ignored, set by hand or not. + row_hdr(11, 0, 9000, [0, 0x20, 0]), + ] + .concat(); + sized_sheet(&head, &rows) +} + +#[test] +fn row_heights_and_column_widths_are_read_as_their_xlsx_spelling() { + let (rows, part, _) = sizes_of(&file(&sizes_bin())).unwrap(); + let (xml_rows, xml_part) = xml_sheet(SIZES_XML); + assert_eq!(rows, xml_rows); + assert_eq!(part.columns, xml_part.columns); + assert_eq!(part.features.format, xml_part.features.format); + // A few of the conversions, spelled out. + assert_eq!((rows[1].height, rows[1].custom_height), (Some(30.0), true)); + assert_eq!((rows[2].height, rows[2].custom_height), (Some(26.65), false)); + assert_eq!(rows[8].style, Some(3)); + assert_eq!(part.columns[0].width, Some(5273.0 / 256.0)); + let format = part.features.format.clone().unwrap(); + assert_eq!((format.default_col_width, format.default_row_height), (Some(12.59765625), Some(14.25))); + // Written as SpreadsheetML, the binary sheet has its default sizes and columns. + let written = String::from_utf8(part.to_xml(&rows)).unwrap(); + assert!(written.contains(r#""#), "{written}"); + assert!(written.contains(r#""#), "{written}"); +} + +#[test] +fn damaged_sizes_are_skipped_or_refused() { + // A column range outside the grid, or backwards, is skipped with a warning. + let head = [record(390, &[]), col_info(0, 16_384, 2321, 0, 0), col_info(5, 4, 2321, 0, 0), col_info(1, 1, 512, 0, 2), record(391, &[])].concat(); + let (_, part, warnings) = sizes_of(&file(&sized_sheet(&head, &row(0)))).unwrap(); + assert_eq!(part.columns.len(), 1); + assert_eq!((part.columns[0].first, part.columns[0].width), (1, Some(2.0))); + assert!(warnings.iter().any(|w| w.contains("column ranges")), "{warnings:?}"); + // Without a sheet format, only a height set by hand is a row's `ht`. + let (rows, _, _) = sizes_of(&file(&sized_sheet(&[], &[row_hdr(0, 0, 300, [0, 0, 0]), row_hdr(1, 0, 400, [0, 0x20, 0])].concat()))).unwrap(); + assert_eq!((rows[0].height, rows[1].height), (None, Some(20.0))); + // More column records than columns are refused. + let many = [&record(390, &[])[..], &col_info(0, 0, 2321, 0, 0).repeat(16_385), &record(391, &[])].concat(); + assert!(matches!(sizes_of(&file(&sized_sheet(&many, &[]))), Err(Error::TooLarge(_)))); + // Skipped (reversed) column records count toward that cap too. + let reversed = [&record(390, &[])[..], &col_info(5, 4, 2321, 0, 0).repeat(16_385), &record(391, &[])].concat(); + assert!(matches!(sizes_of(&file(&sized_sheet(&reversed, &[]))), Err(Error::TooLarge(_)))); + // A sheet format or column after the data, or a record of the wrong size, fails the read. + for (why, sheet) in [ + ("format after data", worksheet(&[], &ws_fmt_info(u32::MAX, 8, 285, 0, [0, 0]))), + ("column after data", worksheet(&[], &col_info(0, 0, 2321, 0, 0))), + ("short column", sized_sheet(&record(60, &[0; 16]), &[])), + ("short format", sized_sheet(&record(485, &[0; 11]), &[])), + ("long row header", sized_sheet(&[], &record(0, &[0; 18]))), + ] { + assert!(matches!(sizes_of(&file(&sheet)), Err(Error::Invalid(_))), "{why}"); + } +} + // Styles ([MS-XLSB] 2.1.7.50): a styles part assembled from records, and the `styles.xml` Excel // writes for the same formats when it saves the workbook as `.xlsx`.