From ce819906a58fc8ce0482ed159f11c4d1e7fa2d3b Mon Sep 17 00:00:00 2001 From: Zach Shallbetter Date: Tue, 13 Oct 2026 12:28:12 -0700 Subject: [PATCH 001/209] craft-cms: public constructors for matrix/TRC and gray profiles (closes #61) - Add Curve::gamma, Curve::parametric, and Curve::from_table constructors with finite and range validation. - Add Profile::matrix_shaper accepting PCS D50-adapted matrices and populating chad for non-D50 white points. - Add Profile::from_source_matrix adapting unadapted source primaries to PCS D50 via Bradford transform. - Add Profile::gray constructor for monochrome display profiles. - Validate curve samples remain within [0.0, 1.0] and verify colorant matrix invertibility post-quantization. - Re-export math::Mat3 from the crate root. - Add independent ICC oracle validation tests against moxcms verifying both matrix conventions produce exact color transforms (diff <= 1). --- crates/cms/src/curve.rs | 31 +++++++ crates/cms/src/lib.rs | 3 +- crates/cms/src/profile.rs | 113 ++++++++++++++++++++++++++ crates/cms/tests/constructors.rs | 135 +++++++++++++++++++++++++++++++ crates/cms/tests/fogra39.rs | 9 ++- 5 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 crates/cms/tests/constructors.rs diff --git a/crates/cms/src/curve.rs b/crates/cms/src/curve.rs index f50f2d663..9e2795cd8 100644 --- a/crates/cms/src/curve.rs +++ b/crates/cms/src/curve.rs @@ -26,6 +26,37 @@ pub fn srgb_trc() -> Curve { } impl Curve { + /// Creates a gamma curve `y = x^g`. Returns `Identity` if `g == 1.0` (within 1e-6). + pub fn gamma(g: f64) -> Result { + if !g.is_finite() || g <= 0.0 { + return Err(crate::CmsError::InvalidArgument("gamma must be positive and finite".into())); + } + if (g - 1.0).abs() < 1e-6 { Ok(Curve::Identity) } else { Ok(Curve::Gamma(g)) } + } + + /// Creates an ICC parametric curve (types 0 to 4). + pub fn parametric(kind: u16, p: [f64; 7]) -> Result { + if kind > 4 { + return Err(crate::CmsError::InvalidArgument(format!("parametric curve kind {kind} must be <= 4"))); + } + let count = Self::param_count(kind); + if !p.iter().take(count).all(|v| v.is_finite()) { + return Err(crate::CmsError::InvalidArgument("parametric curve parameters must be finite".into())); + } + Ok(Curve::Parametric { kind, p }) + } + + /// Creates a sampled lookup table curve over evenly spaced inputs in `[0, 1]`. + pub fn from_table(samples: Vec) -> Result { + if samples.len() < 2 { + return Err(crate::CmsError::InvalidArgument("sampled curve must contain at least 2 entries".into())); + } + if !samples.iter().all(|s| s.is_finite() && (0.0..=1.0).contains(s)) { + return Err(crate::CmsError::InvalidArgument("sampled curve values must be finite and within [0, 1]".into())); + } + Ok(Curve::Table(samples)) + } + /// Number of parameters stored for a parametric function type. pub fn param_count(kind: u16) -> usize { match kind { diff --git a/crates/cms/src/lib.rs b/crates/cms/src/lib.rs index 613d0db6b..31dc6b1ab 100644 --- a/crates/cms/src/lib.rs +++ b/crates/cms/src/lib.rs @@ -13,7 +13,7 @@ //! * [`Lut3d`] samples a display transform into a 3D texture for the GPU canvas. //! //! Implemented from the public ICC.1:2010 (v4.3) and ICC.1:2001-04 (v2) specifications, -//! CIE 15, and Adobe's published black point compensation paper. +//! CIE 15, and the ISO 18619 black point compensation standard. #![forbid(unsafe_code)] pub mod builtin; @@ -34,6 +34,7 @@ pub use clut::Clut; pub use curve::Curve; pub use gamut::GamutCheck; pub use lut3d::Lut3d; +pub use math::Mat3; pub use profile::{ColorSpace, Pcs, Profile, ProfileClass}; pub use transform::{SampleKind, Transform, TransformOptions, black_point, cached}; diff --git a/crates/cms/src/profile.rs b/crates/cms/src/profile.rs index ef8cbcee6..9c428179d 100644 --- a/crates/cms/src/profile.rs +++ b/crates/cms/src/profile.rs @@ -182,6 +182,119 @@ pub struct Profile { } impl Profile { + /// Builds a matrix/TRC display profile from an already PCS D50-adapted colorant matrix, + /// a white point, and per-channel tone reproduction curves. + /// + /// # Matrix Coordinate Space + /// In accordance with ICC.1:2010 §9.2.35–37, the colorant matrix tags (`rXYZ`, `gXYZ`, `bXYZ`) + /// stored in an ICC profile must be chromatically adapted to the D50 Profile Connection Space (PCS). + /// `pcs_matrix` must therefore be expressed relative to D50 (e.g. as read directly from ICC profile tags). + /// + /// If `white` is not D50, the profile's `chad` tag is populated with the Bradford chromatic adaptation + /// matrix mapping `white` to D50. + pub fn matrix_shaper(white: [f64; 3], pcs_matrix: Mat3, trc: [Curve; 3]) -> Result { + if !white.iter().all(|&c| c.is_finite() && c > 0.0) { + return Err(CmsError::InvalidArgument("white point must be positive and finite".into())); + } + if !pcs_matrix.iter().flatten().all(|&v| v.is_finite()) { + return Err(CmsError::InvalidArgument("matrix elements must be finite".into())); + } + if math::invert(&pcs_matrix).is_none() { + return Err(CmsError::InvalidArgument("colorant matrix must be invertible".into())); + } + + let d50 = math::d50_quantized(); + let is_d50 = (white[0] - math::D50[0]).abs() < 1e-4 && (white[1] - math::D50[1]).abs() < 1e-4 && (white[2] - math::D50[2]).abs() < 1e-4; + + let chad = if is_d50 { None } else { Some(math::bradford(white, math::D50)) }; + + // Quantize and normalize to ensure (1, 1, 1) maps exactly to PCS D50 + let mut q = pcs_matrix.map(|row| row.map(math::s15f16_round)); + for (i, row) in q.iter_mut().enumerate() { + let err = d50[i] - row.iter().sum::(); + let k = (0..3).max_by(|a, b| row[*a].abs().total_cmp(&row[*b].abs())).unwrap_or(0); + row[k] = math::s15f16_round(row[k] + err); + } + if math::invert(&q).is_none() { + return Err(CmsError::InvalidArgument("colorant matrix became singular after quantization/normalization".into())); + } + let chad_quantized = chad.map(|a| a.map(|row| row.map(math::s15f16_round))); + + Ok(Profile { + version: (4, 0x30), + class: ProfileClass::Display, + color_space: ColorSpace::Rgb, + pcs: Pcs::Xyz, + rendering_intent: Intent::RelativeColorimetric, + description: "Parametric Matrix/TRC".into(), + copyright: "".into(), + white_point: if is_d50 { d50 } else { white }, + black_point: None, + chad: chad_quantized, + matrix: Some(q), + trc: Some(trc), + gray_trc: None, + a2b: [None, None, None], + b2a: [None, None, None], + bytes: None, + hash: Default::default(), + }) + } + + /// Builds a matrix/TRC display profile from unadapted source primaries relative to `white`. + /// + /// The unadapted `source_matrix` (whose columns sum to `white`) is chromatically adapted to PCS D50 + /// using the Bradford transform before storage in the profile's matrix tags. + pub fn from_source_matrix(white: [f64; 3], source_matrix: Mat3, trc: [Curve; 3]) -> Result { + if !white.iter().all(|&c| c.is_finite() && c > 0.0) { + return Err(CmsError::InvalidArgument("white point must be positive and finite".into())); + } + if !source_matrix.iter().flatten().all(|&v| v.is_finite()) { + return Err(CmsError::InvalidArgument("matrix elements must be finite".into())); + } + if math::invert(&source_matrix).is_none() { + return Err(CmsError::InvalidArgument("colorant matrix must be invertible".into())); + } + + let is_d50 = (white[0] - math::D50[0]).abs() < 1e-4 && (white[1] - math::D50[1]).abs() < 1e-4 && (white[2] - math::D50[2]).abs() < 1e-4; + let pcs_matrix = if is_d50 { + source_matrix + } else { + let adapt = math::bradford(white, math::D50); + math::mul(&adapt, &source_matrix) + }; + + Self::matrix_shaper(white, pcs_matrix, trc) + } + + /// Builds a monochrome profile from a white point and tone reproduction curve. + pub fn gray(white: [f64; 3], trc: Curve) -> Result { + if !white.iter().all(|&c| c.is_finite() && c > 0.0) { + return Err(CmsError::InvalidArgument("white point must be positive and finite".into())); + } + let is_d50 = (white[0] - math::D50[0]).abs() < 1e-4 && (white[1] - math::D50[1]).abs() < 1e-4 && (white[2] - math::D50[2]).abs() < 1e-4; + + Ok(Profile { + version: (4, 0x30), + class: ProfileClass::Display, + color_space: ColorSpace::Gray, + pcs: Pcs::Xyz, + rendering_intent: Intent::RelativeColorimetric, + description: "Parametric Gray".into(), + copyright: "".into(), + white_point: if is_d50 { math::d50_quantized() } else { white }, + black_point: None, + chad: None, + matrix: None, + trc: None, + gray_trc: Some(trc), + a2b: [None, None, None], + b2a: [None, None, None], + bytes: None, + hash: Default::default(), + }) + } + /// Parses ICC profile bytes. pub fn parse(bytes: &[u8]) -> Result { let mut p = parse_profile(bytes)?; diff --git a/crates/cms/tests/constructors.rs b/crates/cms/tests/constructors.rs new file mode 100644 index 000000000..eecd8883e --- /dev/null +++ b/crates/cms/tests/constructors.rs @@ -0,0 +1,135 @@ +use craft_cms::{Builtin, Curve, Intent, Mat3, Profile, Transform}; + +#[test] +fn test_curve_constructors() { + // Gamma + let g22 = Curve::gamma(2.2).expect("valid gamma"); + assert_eq!(g22, Curve::Gamma(2.2)); + + let g10 = Curve::gamma(1.0).expect("gamma 1.0 is identity"); + assert_eq!(g10, Curve::Identity); + + let g_near = Curve::gamma(1.0 + 1e-7).expect("near-identity gamma normalized"); + assert_eq!(g_near, Curve::Identity); + + assert!(Curve::gamma(0.0).is_err()); + assert!(Curve::gamma(-2.2).is_err()); + assert!(Curve::gamma(f64::NAN).is_err()); + assert!(Curve::gamma(f64::INFINITY).is_err()); + + // Parametric + let p0 = Curve::parametric(0, [2.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).expect("valid parametric type 0"); + assert_eq!(p0, Curve::Parametric { kind: 0, p: [2.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] }); + + assert!(Curve::parametric(5, [0.0; 7]).is_err()); + assert!(Curve::parametric(0, [f64::NAN, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).is_err()); + + // Sampled table + let table = Curve::from_table(vec![0.0, 0.5, 1.0]).expect("valid sampled table"); + assert_eq!(table, Curve::Table(vec![0.0, 0.5, 1.0])); + + assert!(Curve::from_table(vec![0.5]).is_err()); + assert!(Curve::from_table(vec![0.0, f32::NAN]).is_err()); + assert!(Curve::from_table(vec![0.0, -0.1, 1.0]).is_err()); + assert!(Curve::from_table(vec![0.0, 2.0]).is_err()); + assert!(Curve::from_table(vec![0.0, 1.0001]).is_err()); +} + +#[test] +fn test_profile_constructors() { + let srgb = Builtin::Srgb.profile(); + let white_d65 = [0.95047, 1.0, 1.08883]; + + // 1. Matrix shaper with already PCS D50-adapted matrix (e.g. read from ICC tags) + let pcs_d50_matrix: Mat3 = [[0.4360747, 0.3850649, 0.1430804], [0.2225045, 0.7168786, 0.0606169], [0.0139322, 0.0971045, 0.7141733]]; + let trc = [craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc()]; + let custom_from_pcs = Profile::matrix_shaper(white_d65, pcs_d50_matrix, trc.clone()).expect("matrix shaper from PCS D50"); + assert_eq!(custom_from_pcs.color_space, craft_cms::ColorSpace::Rgb); + assert!(custom_from_pcs.same_colors(srgb), "custom profile from PCS D50 matrix should match reference sRGB"); + + // 2. Matrix shaper from unadapted D65 source primaries (undergoes single Bradford adaptation) + let source_d65_matrix: Mat3 = [[0.4124564, 0.3575761, 0.1804375], [0.2126729, 0.7151522, 0.0721750], [0.0193339, 0.1191920, 0.9503041]]; + let custom_from_source = Profile::from_source_matrix(white_d65, source_d65_matrix, trc).expect("matrix shaper from source"); + assert!(custom_from_source.same_colors(srgb), "custom profile adapted from source primaries should match reference sRGB"); + + // Evaluate transform + let xform = Transform::new(&custom_from_pcs, srgb, Intent::RelativeColorimetric, true).expect("transform from custom RGB to sRGB"); + let mut out = [0.0f32; 3]; + xform.eval(&[0.5, 0.5, 0.5], &mut out).expect("eval"); + assert!((out[0] - 0.5).abs() < 1e-2); + assert!((out[1] - 0.5).abs() < 1e-2); + assert!((out[2] - 0.5).abs() < 1e-2); + + // Invalid matrix shaper inputs + assert!(Profile::matrix_shaper([0.0, 1.0, 1.0], pcs_d50_matrix, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err()); + assert!(Profile::matrix_shaper([f64::NAN, 1.0, 1.0], pcs_d50_matrix, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err()); + let singular: Mat3 = [[0.0; 3]; 3]; + assert!(Profile::matrix_shaper(white_d65, singular, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err()); + + // Near-degenerate matrix that is invertible before quantization but becomes singular after + let d50 = [0.9642, 1.0, 0.8249]; + let near_degenerate: Mat3 = [[d50[0] - 0.000001, 0.000001, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, d50[2]]]; + assert!(Profile::matrix_shaper(d50, near_degenerate, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err()); + + // Gray profile + let gray = Profile::gray(white_d65, Curve::gamma(2.2).expect("curve")).expect("gray profile"); + assert_eq!(gray.color_space, craft_cms::ColorSpace::Gray); + + let gray_to_srgb = Transform::new(&gray, srgb, Intent::RelativeColorimetric, true).expect("transform from gray to sRGB"); + let mut srgb_out = [0.0f32; 3]; + gray_to_srgb.eval(&[0.5], &mut srgb_out).expect("eval"); + assert!((srgb_out[0] - srgb_out[1]).abs() < 1e-3); + assert!((srgb_out[1] - srgb_out[2]).abs() < 1e-3); +} + +#[test] +fn test_matrix_constructors_against_independent_moxcms_oracle() { + use moxcms::{ColorProfile, Layout, TransformOptions as MoxOpts}; + + let srgb = Builtin::Srgb.profile(); + let white_d65 = [0.95047, 1.0, 1.08883]; + + // 1. PCS D50-adapted matrix profile + let pcs_d50_matrix: Mat3 = [[0.4360747, 0.3850649, 0.1430804], [0.2225045, 0.7168786, 0.0606169], [0.0139322, 0.0971045, 0.7141733]]; + let trc = [craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc()]; + let custom_from_pcs = Profile::matrix_shaper(white_d65, pcs_d50_matrix, trc.clone()).expect("matrix shaper from PCS D50"); + + // 2. Unadapted source matrix profile + let source_d65_matrix: Mat3 = [[0.4124564, 0.3575761, 0.1804375], [0.2126729, 0.7151522, 0.0721750], [0.0193339, 0.1191920, 0.9503041]]; + let custom_from_source = Profile::from_source_matrix(white_d65, source_d65_matrix, trc).expect("matrix shaper from source"); + + // Convert both to standard ICC bytes and parse with independent oracle moxcms + let mox_from_pcs = ColorProfile::new_from_slice(&custom_from_pcs.to_bytes()).expect("moxcms parse custom PCS profile"); + let mox_from_source = ColorProfile::new_from_slice(&custom_from_source.to_bytes()).expect("moxcms parse custom source profile"); + let mox_dst_srgb = ColorProfile::new_from_slice(&srgb.to_bytes()).expect("moxcms parse builtin sRGB"); + + let test_pixels: Vec = (0..512u32).flat_map(|i| [(i * 17 % 256) as u8, (i * 53 % 256) as u8, (i * 97 % 256) as u8]).collect(); + + // Verify 1: moxcms transforming custom_from_pcs -> builtin sRGB produces identity (diff <= 1) + let mox_xform_pcs = mox_from_pcs.create_transform_8bit(Layout::Rgb, &mox_dst_srgb, Layout::Rgb, MoxOpts::default()).unwrap(); + let mut mox_out_pcs = vec![0u8; test_pixels.len()]; + mox_xform_pcs.transform(&test_pixels, &mut mox_out_pcs).unwrap(); + let max_diff_pcs = test_pixels.iter().zip(&mox_out_pcs).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap(); + assert!(max_diff_pcs <= 1, "moxcms PCS-adapted profile -> sRGB max diff was {max_diff_pcs}"); + + // Verify 2: moxcms transforming custom_from_source -> builtin sRGB produces identity (diff <= 1) + let mox_xform_src = mox_from_source.create_transform_8bit(Layout::Rgb, &mox_dst_srgb, Layout::Rgb, MoxOpts::default()).unwrap(); + let mut mox_out_src = vec![0u8; test_pixels.len()]; + mox_xform_src.transform(&test_pixels, &mut mox_out_src).unwrap(); + let max_diff_src = test_pixels.iter().zip(&mox_out_src).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap(); + assert!(max_diff_src <= 1, "moxcms source-adapted profile -> sRGB max diff was {max_diff_src}"); + + // Verify 3: craft_cms and moxcms agree when converting from custom_from_source to DisplayP3 + let p3 = Builtin::DisplayP3.profile(); + let mox_dst_p3 = ColorProfile::new_from_slice(&p3.to_bytes()).expect("moxcms parse Display P3"); + let mox_xform_p3 = mox_from_source.create_transform_8bit(Layout::Rgb, &mox_dst_p3, Layout::Rgb, MoxOpts::default()).unwrap(); + let mut mox_p3_out = vec![0u8; test_pixels.len()]; + mox_xform_p3.transform(&test_pixels, &mut mox_p3_out).unwrap(); + + let ours_xform_p3 = Transform::new(&custom_from_source, p3, Intent::RelativeColorimetric, false).unwrap(); + let mut ours_p3_out = vec![0u8; test_pixels.len()]; + ours_xform_p3.convert_u8(&test_pixels, 3, &mut ours_p3_out, 3, false).unwrap(); + + let max_p3_diff = ours_p3_out.iter().zip(&mox_p3_out).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap(); + assert!(max_p3_diff <= 1, "craft_cms vs moxcms custom source profile -> P3 max diff was {max_p3_diff}"); +} diff --git a/crates/cms/tests/fogra39.rs b/crates/cms/tests/fogra39.rs index f7fcdd0ca..f958ffc51 100644 --- a/crates/cms/tests/fogra39.rs +++ b/crates/cms/tests/fogra39.rs @@ -545,8 +545,13 @@ fn stand_in_separations_respect_the_ink_limit() { let t = Transform::new(Builtin::Srgb.profile(), cmyk, intent, true).unwrap(); let mut out = vec![0.0f32; N * N * N * 4]; t.convert_f32(&grid, 3, &mut out, 4, false).unwrap(); - let (i, max) = - out.chunks_exact(4).map(|c| c.iter().sum::()).enumerate().fold((0, 0.0f32), |m, (i, s)| if s > m.1 { (i, s) } else { m }); + let (i, max) = out + .as_chunks::<4>() + .0 + .iter() + .map(|c| c.iter().sum::()) + .enumerate() + .fold((0, 0.0f32), |m, (i, s)| if s > m.1 { (i, s) } else { m }); eprintln!("{name}, {intent:?}: max total ink {:.2} % at sRGB {:?}", max * 100.0, &grid[i * 3..i * 3 + 3]); assert!(max <= LIMIT, "{name}, {intent:?}: total ink {max} at sRGB {:?}", &grid[i * 3..i * 3 + 3]); From e7914038f339325f9e51807b6f2cc7fae4585eff Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 09:55:35 -0500 Subject: [PATCH 002/209] craft-xlsx: read the defined names of a binary workbook Each BrtName becomes a DefinedName as Excel writes it in .xlsx: built-in names as _xlnm.Print_Area, sheet scope as localSheetId, the formula decoded with relative references as offsets from A1, the comment, hidden, and a macro's function, vbProcedure or xlm, functionGroupId, shortcutKey, publishToServer and workbookParameter. The hidden names for newer functions and LET parameters are left out, as .xlsx does. Cell formulas using a name that was read now decode instead of keeping their saved results, PtgNameX through the workbook's own link is 'Other Sheet'!Local, and a deleted 3-D reference is Sheet2!#REF!, as in .xlsx. A name whose formula can't be read is left out with a warning, and formulas using it keep their saved results. Closes #1250. --- crates/xlsx/README.md | 9 +- crates/xlsx/src/xlsb.rs | 153 +++++++++++++++++++++++++++++--- crates/xlsx/src/xlsb/expr.rs | 3 + crates/xlsx/src/xlsb/formula.rs | 105 ++++++++++++++++------ crates/xlsx/tests/xlsb.rs | 135 +++++++++++++++++++++++++++- 5 files changed, 360 insertions(+), 45 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 1dc30574d..4e90ca830 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -84,12 +84,17 @@ The parts this crate writes use the transitional namespaces; a strict file's kep they were. A binary workbook is not written. A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not -read (each cell keeps its format index). Its cell formulas are decoded from their tokens into the +read (each cell keeps its format index). Its defined names are those of `definedNames` in the +`.xlsx` Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their +`localSheetId`, relative references as the cell they reach from A1, macro names with their +`function`, `xlm`, `functionGroupId` and `shortcutKey`. Excel's hidden names for newer functions +and LET parameters are left out, as in `.xlsx`; so is a name whose formula can't be read, with a +warning. Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, no implicit `@`), with `FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: user defined names, table references, +keeps only its saved result, with a warning saying why: a defined name left out, table references, other workbooks, Python code stored outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 2fdde59be..1941ea61e 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,5 +1,5 @@ -//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from -//! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an +//! Binary workbooks ([MS-XLSB]): the workbook part with its defined names, shared strings and +//! worksheet cells, from their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an //! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why. //! Styles are not read. A record that does not parse fails the read. @@ -15,7 +15,7 @@ use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS, Range, Visibility}; use crate::sheet::{Cell, CellFormula, CellValue, FormulaKind, Row, SheetPart}; use crate::strings::StringItem; -use crate::workbook::{SheetEntry, SheetKind, Workbook}; +use crate::workbook::{DefinedName, SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; @@ -266,6 +266,9 @@ pub(crate) fn workbook( records.first(131, warnings)?; let mut wb = Workbook::default(); let mut links = Some(Links::default()); + let mut pending = vec![]; + // Each BrtBundleSh's place in `wb.sheets`, `None` for a skipped one. + let mut bundle_sheets: Vec> = vec![]; let mut bundle = false; let mut bundle_seen = false; while let Some((id, payload)) = records.next(warnings)? { @@ -279,14 +282,15 @@ pub(crate) fn workbook( if wb.sheets.is_empty() { return Err(invalid("no supported worksheets")); } - if links.is_none() { - warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"); + match links.as_mut() { + Some(l) => wb.defined_names = defined_names(pending, &bundle_sheets, l, limits, budget, warnings), + None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), } return Ok((wb, links)); } 39 | 355 | 357 | 358 | 362 | 667 => { if let Some(l) = links.as_mut() - && link_record(id, payload, l).is_err() + && link_record(id, payload, l, &mut pending).is_err() { links = None; } @@ -341,6 +345,7 @@ pub(crate) fn workbook( if let Some(l) = links.as_mut() { l.sheets.push(None); } + bundle_sheets.push(None); continue; }; let rel = rels.by_id(&rel_id).ok_or_else(|| invalid("worksheet relationship is missing"))?; @@ -355,6 +360,7 @@ pub(crate) fn workbook( if let Some(l) = links.as_mut() { l.sheets.push((kind == SheetKind::Worksheet).then(|| name.clone())); } + bundle_sheets.push(u32::try_from(wb.sheets.len()).ok()); wb.sheets.push(SheetEntry { name, sheet_id, visibility, rel_id, kind, part: Some(rel.target.clone()) }); } _ => {} @@ -363,9 +369,110 @@ pub(crate) fn workbook( Err(invalid("missing workbook end record")) } +/// A `BrtName`'s parts that only its [`DefinedName`] needs, until the workbook part is read. +struct PendingName { + /// Its place in [`Links`]'s names. + index: usize, + flags: u32, + /// `chKey`: a macro's shortcut key. + key: u8, + rgce: Vec, + rgcb: Vec, + comment: Option, + menu: Option, + description: Option, +} + +/// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the +/// hidden ones for newer functions and LET parameters, which Excel doesn't write there. `sheets` +/// maps a name's `itab` (its sheet's place in `BrtBundleSh` order) to [`Workbook::sheets`]. +/// A name whose formula can't be read, or that belongs to a sheet the reader skips, is left out +/// with a warning, and a cell formula using it keeps its saved result. +fn defined_names( + pending: Vec, + sheets: &[Option], + links: &mut Links, + limits: &Limits, + budget: &Budget, + warnings: &mut Warnings, +) -> Vec { + let mut out = vec![]; + for p in pending { + let Some(name) = links.names.get(p.index) else { continue }; + if name.future || formula::is_param(&name.text) { + continue; + } + let local_sheet_id = match name.scope { + None => None, + Some(itab) => match usize::try_from(itab).ok().and_then(|i| sheets.get(i)) { + Some(Some(i)) => Some(*i), + _ => { + warnings.warn("XLSB: some defined names were left out (names of sheets the reader skips)"); + continue; + } + }, + }; + let formula = match formula::decode(&p.rgce, &p.rgcb, links, Mode::Name) { + Ok(e) => expr::print(&e), + Err(formula::Unsupported(why)) => { + warnings.warn(format!("XLSB: some defined names were left out ({why})")); + continue; + } + }; + if !budget.try_formula_text(formula.len(), limits) { + warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); + continue; + } + let mut attrs = vec![]; + let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); + // In `CT_DefinedName`'s order. + if let Some(m) = p.menu { + put("customMenu", m); + } + if let Some(d) = p.description { + put("description", d); + } + let (func, vba, proc) = (p.flags & 0x02 != 0, p.flags & 0x04 != 0, p.flags & 0x08 != 0); + if func { + put("function", "1".into()); + } + if vba { + put("vbProcedure", "1".into()); + } else if proc { + put("xlm", "1".into()); + } + let group = (p.flags >> 6) & 0x1FF; + if proc && group != 0 { + put("functionGroupId", group.to_string()); + } + if p.key != 0 { + put("shortcutKey", char::from(p.key).to_string()); + } + if p.flags & (1 << 15) != 0 { + put("publishToServer", "1".into()); + } + if p.flags & (1 << 16) != 0 { + put("workbookParameter", "1".into()); + } + out.push(DefinedName { + name: name.text.clone(), + local_sheet_id, + formula, + comment: p.comment.unwrap_or_default(), + hidden: p.flags & 1 != 0, + attrs, + }); + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + } + out +} + /// A supporting link (`BrtSupSelf`, `BrtSupSame`, `BrtSupAddin`, `BrtSupBookSrc`), the sheet links -/// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`. -fn link_record(id: u16, payload: &[u8], links: &mut Links) -> Result<(), Error> { +/// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`; a name's other parts go to +/// `pending`. +fn link_record(id: u16, payload: &[u8], links: &mut Links, pending: &mut Vec) -> Result<(), Error> { let mut b = Bytes::new(payload); match id { 357 | 358 | 667 => { @@ -391,15 +498,37 @@ fn link_record(id: u16, payload: &[u8], links: &mut Links) -> Result<(), Error> b.finish()?; } _ => { - // BrtName ([MS-XLSB] 2.4.718): flags, chKey, itab, then the name; the formula and the - // rest aren't needed to call a function or tell a name from one. + // BrtName ([MS-XLSB] 2.4.718). Its formula is decoded once the workbook part is read. if links.names.len() >= MAX_NAMES { return Err(invalid("too many defined names")); } let flags = b.u32()?; - b.take(5)?; + let key = b.u8()?; + let itab = b.u32()?; let text = b.text(255)?; - links.names.push(Name { text, future: flags & (1 << 17) != 0 }); + let cce = b.u32()? as usize; + let rgce = b.take(cce)?.to_vec(); + let cb = b.u32()? as usize; + let rgcb = b.take(cb)?.to_vec(); + let comment = b.string(255, true)?; + // A macro's name (fProc) has four more strings: one unused, its custom menu, its + // description and another unused. + let (menu, description) = if flags & 0x08 != 0 { + b.string(32767, true)?; + let menu = b.string(32767, true)?; + let description = b.string(32767, true)?; + b.string(32767, true)?; + (menu, description) + } else { + (None, None) + }; + b.finish()?; + let scope = (itab != u32::MAX).then_some(itab); + // A built-in name (fBuiltin) is stored bare; `.xlsx` spells it `_xlnm.Print_Area`, in + // `definedNames` and in formulas alike. + let text = if flags & 0x20 != 0 { format!("_xlnm.{text}") } else { text }; + pending.push(PendingName { index: links.names.len(), flags, key, rgce, rgcb, comment, menu, description }); + links.names.push(Name { text, future: flags & (1 << 17) != 0, scope, read: false }); } } Ok(()) diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index 3dd643a6a..17f5086ae 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -24,6 +24,8 @@ pub(crate) enum RefKind { Rows(u32, bool, u32, bool), /// Whole columns `A:C`. Cols(u32, bool, u32, bool), + /// A deleted cell or range on a named sheet: `Sheet2!#REF!`. + Deleted, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -173,6 +175,7 @@ fn push_reference(s: &mut String, r: &Reference) { s.push_str(dollar(*a1)); push_col(s, *c1); } + RefKind::Deleted => s.push_str(CellError::Ref.as_str()), } } diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 327ffcc25..2e7f7cc96 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -4,8 +4,9 @@ //! called through hidden names), so this is its own decoder, with its own function table //! ([`super::ftab`]) and its own copies of craft-xls's depth guards and limits. Anything outside the //! token set it knows is reported as unsupported, and the caller keeps the cell's cached value. -//! So is a formula that needs something the reader doesn't read: a user's defined name (XLSB -//! names are not read yet) or Python code stored outside the cell. +//! So is a formula that needs something the reader doesn't read: a defined name whose own +//! formula could not be read, or Python code stored outside the cell. Defined names' formulas +//! are decoded here too ([`Mode::Name`]). //! //! Names keep their file spelling (`_xlfn.CONCAT`, `_xlpm.x`), as `.xlsx` formula text has them. @@ -92,13 +93,26 @@ pub(super) struct Xti { pub last: i32, } -/// A `BrtName` as far as cell formulas need it. +/// A `BrtName` as far as formulas need it. #[derive(Clone, Debug)] pub(super) struct Name { - /// The name as written (`_xlfn.CONCAT`, `_xlpm.x`, `Rate`). + /// The name as `.xlsx` spells it (`_xlfn.CONCAT`, `_xlpm.x`, `_xlnm.Print_Area`, `Rate`). pub text: String, /// `fFutureFunction`: a hidden name standing for a function newer than the file format. pub future: bool, + /// `itab`: the sheet a sheet-scoped name belongs to, by its place in `BrtBundleSh` order. + pub scope: Option, + /// Its own formula was read, so it is one of the workbook's defined names. + pub read: bool, +} + +impl Name { + /// Whether a formula may use the name: one the formula text itself defines (a newer + /// function, a LET parameter), or a defined name the reader gives. A name's formula may use + /// any other name: its text is the same either way. + fn usable(&self, mode: Mode) -> bool { + self.future || is_param(&self.text) || self.read || mode == Mode::Name + } } /// Workbook-level tables formulas refer to. @@ -114,20 +128,37 @@ pub(crate) struct Links { } /// Whether a name stands for a LET parameter (`_xlpm.x`), which only the formula itself defines. -fn is_param(name: &str) -> bool { +pub(super) fn is_param(name: &str) -> bool { name.get(..6).is_some_and(|p| p.eq_ignore_ascii_case("_xlpm.")) } impl Links { - /// The sheets a 3-D reference names, or `None` for a deleted sheet (the reference is `#REF!`). - fn sheet_sel(&self, ixti: u16) -> Result, Unsupported> { + /// `PtgName`'s and `PtgNameX`'s name: a 1-based index into every `BrtName`. A cell formula + /// using a defined name the reader doesn't give would calculate to `#NAME?`, so it keeps its + /// saved result instead. + fn name(&self, index: u32, mode: Mode) -> Result<&Name, Unsupported> { + let i = usize::try_from(index).ok().and_then(|i| i.checked_sub(1)); + let name = i.and_then(|i| self.names.get(i)).ok_or(Unsupported("reference to a missing defined name"))?; + if !name.usable(mode) { + return Err(Unsupported("defined names whose formula could not be read")); + } + Ok(name) + } + + /// Sheet link `ixti`, which must be through this workbook's own supporting link. + fn this_book(&self, ixti: u16) -> Result<&Xti, Unsupported> { let x = self.xti.get(usize::from(ixti)).ok_or(Unsupported("reference to a missing sheet link"))?; match usize::try_from(x.book).ok().and_then(|b| self.books.get(b)) { - Some(Book::This) => {} - Some(Book::External) => return Err(Unsupported("references to other workbooks (external links)")), - Some(_) => return Err(Unsupported("references through add-in or same-sheet links")), - None => return Err(Unsupported("reference to a missing workbook link")), + Some(Book::This) => Ok(x), + Some(Book::External) => Err(Unsupported("references to other workbooks (external links)")), + Some(_) => Err(Unsupported("references through add-in or same-sheet links")), + None => Err(Unsupported("reference to a missing workbook link")), } + } + + /// The sheets a 3-D reference names, or `None` for a deleted sheet (the reference is `#REF!`). + fn sheet_sel(&self, ixti: u16) -> Result, Unsupported> { + let x = self.this_book(ixti)?; // -1: "the first sheet in the reference cannot be found" (a deleted sheet). if x.first == -1 || x.last == -1 { return Ok(None); @@ -194,10 +225,15 @@ const LAST_COL: u32 = MAX_COLS - 1; /// Where a formula lives: a cell formula's references hold coordinates; a shared formula's /// relative tokens (`PtgRefN`, `PtgAreaN`) hold offsets from the shared formula's first cell. +/// +/// A defined name's relative references are offsets from A1 that wrap around the grid +/// (`RgceLocRel`), in its 3-D references too: `.xlsx` spells them as the cell they reach from A1 +/// (one column left is `Data!XFD1`). #[derive(Clone, Copy, PartialEq)] pub(super) enum Mode { Cell, Shared(CellRef), + Name, } /// RgceLoc ([MS-XLSB] 2.5.98.91): a row and a `ColRelShort` (14-bit column, then the @@ -225,9 +261,15 @@ fn loc(row: u32, col: u16, mode: Mode) -> Result { match mode { Mode::Cell => anchor(row, col), Mode::Shared(base) => relative_anchor(row, col, base), + Mode::Name => relative_anchor(row, col, CellRef::new(0, 0)), } } +/// A 3-D reference's cell: a coordinate, except in a defined name. +fn loc3d(row: u32, col: u16, mode: Mode) -> Result { + if mode == Mode::Name { loc(row, col, mode) } else { anchor(row, col) } +} + /// RgceArea: a range spanning every row (or column) is a whole-column (row) reference. fn area(r0: u32, r1: u32, c0: u16, c1: u16) -> Result { area_of(anchor(r0, c0)?, anchor(r1, c1)?) @@ -380,14 +422,7 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res } 0x23 => { // PtgName: a 1-based index into every BrtName, hidden ones included. - let i = usize::try_from(t.u32()?).ok().and_then(|i| i.checked_sub(1)); - let name = i.and_then(|i| links.names.get(i)).ok_or(Unsupported("reference to a missing defined name"))?; - // Defined names are not imported from XLSB yet: a cell using one would calculate - // to #NAME?, so it keeps its saved result. Newer functions and LET parameters are - // hidden names that the formula text itself defines. - if !name.future && !is_param(&name.text) { - return Err(Unsupported("defined names")); - } + let name = links.name(t.u32()?, mode)?; push(&mut stack, kids, Expr::Name(name.text.clone()))?; } 0x24 => { @@ -420,36 +455,52 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res t.take(12)?; push(&mut stack, kids, Expr::Error(CellError::Ref))?; } - 0x39 => return Err(Unsupported("names in other workbooks or on other sheets")), + 0x39 => { + // PtgNameX: a name through a supporting link. Through this workbook's own link + // it is a sheet-scoped name used from another sheet, which `.xlsx` spells + // `'Other Sheet'!Local`: Excel's files show it, [MS-XLSB] doesn't say. + let ixti = t.u16()?; + let index = t.u32()?; + links.this_book(ixti)?; + let name = links.name(index, mode)?; + let sheet = name.scope.and_then(|s| links.sheets.get(usize::try_from(s).ok()?).cloned().flatten()); + let sheet = sheet.ok_or(Unsupported("names in other workbooks or on other sheets"))?; + push(&mut stack, kids, Expr::Name(format!("{sheet}!{}", name.text)))?; + } 0x3A => { let ixti = t.u16()?; let (r, c) = (t.u32()?, t.u16()?); - let kind = RefKind::Cell(anchor(r, c)?); + let kind = RefKind::Cell(loc3d(r, c, mode)?); push(&mut stack, kids, reference(links.sheet_sel(ixti)?, kind))?; } 0x3B => { let ixti = t.u16()?; let (r0, r1, c0, c1) = (t.u32()?, t.u32()?, t.u16()?, t.u16()?); - let kind = area(r0, r1, c0, c1)?; + let kind = area_of(loc3d(r0, c0, mode)?, loc3d(r1, c1, mode)?)?; push(&mut stack, kids, reference(links.sheet_sel(ixti)?, kind))?; } 0x3C | 0x3D => { + // PtgRefErr3d, PtgAreaErr3d: a deleted cell or range, which `.xlsx` spells + // `Sheet2!#REF!` (`#REF!` when the sheet is gone too). The sheet link must still + // be one the reader follows: an external #REF! stays cached. let ixti = t.u16()?; t.take(if id == 0x3C { 6 } else { 12 })?; - // The sheet link must still be one the reader follows: an external #REF! stays cached. - links.sheet_sel(ixti)?; - push(&mut stack, kids, Expr::Error(CellError::Ref))?; + push(&mut stack, kids, reference(links.sheet_sel(ixti)?, RefKind::Deleted))?; } 0x01 => return Err(Unsupported("the shared or array formula form")), 0x02 => return Err(Unsupported("what-if data tables")), 0x18 => return Err(Unsupported("structured table references")), 0x2C => { - let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; + if mode == Mode::Cell { + return Err(Unsupported("relative references in a cell formula")); + } let (r, c) = (t.u32()?, t.u16()?); push(&mut stack, kids, Expr::Ref(Reference { sheet: SheetSel::Current, kind: RefKind::Cell(loc(r, c, mode)?) }))?; } 0x2D => { - let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; + if mode == Mode::Cell { + return Err(Unsupported("relative references in a cell formula")); + } let (r0, r1, c0, c1) = (t.u32()?, t.u32()?, t.u16()?, t.u16()?); let kind = area_of(loc(r0, c0, mode)?, loc(r1, c1, mode)?)?; push(&mut stack, kids, Expr::Ref(Reference { sheet: SheetSel::Current, kind }))?; diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index ee9ac3afd..73bd23218 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -5,7 +5,7 @@ mod common; use common::{read, read_with}; use craft_xlsx::testing::zip; -use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb}; +use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Package, Range, SheetKind, Value, Visibility, is_xlsb}; const ROOT: &str = r#""#; const RELS: &str = r#""#; @@ -463,6 +463,9 @@ fn name(flags: u32, text: &str) -> Vec { data.extend(wide(text)); data.extend([2, 0, 0, 0, 0x1c, 0x1d, 0, 0, 0, 0]); // Formula `#NAME?`, no extra data. data.extend(u32::MAX.to_le_bytes()); // No comment. + if flags & 0x08 != 0 { + data.extend(u32::MAX.to_le_bytes().repeat(4)); // A macro's four strings, all null. + } record(39, &data) } @@ -543,8 +546,8 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { ), (9, ref3d(3, 0, 0), vec![]), (10, [&[0x25u8][..], &0u32.to_le_bytes(), &1u32.to_le_bytes(), &REL.to_le_bytes(), &REL.to_le_bytes(), &[0x41, 20, 0]].concat(), vec![]), - // Kept as their saved results: (11, vec![0x43, 4, 0, 0, 0, 0x1E, 2, 0, 0x05], vec![]), + // Kept as their saved results: (12, ref3d(4, 0, REL), vec![]), (13, vec![0x01, 0, 0, 0, 0], vec![0, 0, 0, 0]), // Part of a shared formula at A1 that isn't one. ]; @@ -572,7 +575,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { Some("_xlfn.LET(_xlpm.x,1,_xlpm.x+1)"), Some("#REF!"), Some("SQRT(A1:A2)"), - None, + Some("Rate*2"), None, None, // A BrtCellMeta before a cell gives it cell metadata, as `cm` does in .xlsx. @@ -584,7 +587,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&CellRef::new(0, 14), &1)]); // Every formula cell keeps its saved result as its value. assert!((1..=15).all(|c| sheet.value(CellRef::new(0, c)) == Value::Number(0.0))); - for why in ["defined names", "other workbooks", "missing from the file", "structured table references"] { + for why in ["other workbooks", "missing from the file", "structured table references"] { assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); } } @@ -930,3 +933,127 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&at("C1"), &1)]); assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings); } + +// Defined names ([MS-XLSB] 2.4.718): `BrtName` records, and the `definedNames` Excel writes for +// the same names when it saves the workbook as `.xlsx`. + +/// `BrtName`. A macro's name (`fProc`, 0x08) has four more strings, null here. +fn brt_name(flags: u32, key: u8, itab: u32, text: &str, rgce: &[u8], comment: Option<&str>) -> Vec { + let mut data = flags.to_le_bytes().to_vec(); + data.push(key); + data.extend(itab.to_le_bytes()); + data.extend(wide(text)); + data.extend((rgce.len() as u32).to_le_bytes()); + data.extend(rgce); + data.extend(0u32.to_le_bytes()); + data.extend(comment.map_or_else(|| u32::MAX.to_le_bytes().to_vec(), wide)); + if flags & 0x08 != 0 { + data.extend(u32::MAX.to_le_bytes().repeat(4)); + } + record(39, &data) +} +fn ref3d_name(ixti: u16, row: u32, col: u16) -> Vec { + [&[0x3Au8][..], &ixti.to_le_bytes(), &loc(row, col)].concat() +} +const WORKBOOK: u32 = u32::MAX; + +/// `book(false)` with the supporting links of `book_with_links` (sheet links Data, Hidden, +/// Data:Hidden, a deleted sheet, the other workbook, and this workbook as a whole, which +/// `PtgNameX` uses), and `names`. +fn book_with_names(names: &[Vec]) -> Vec { + let plain = book(false); + let end = record(132, &[]); + let mut xti = 6u32.to_le_bytes().to_vec(); + for (book, first, last) in [(0u32, 0i32, 0i32), (0, 1, 1), (0, 0, 1), (0, -1, -1), (1, 0, 0), (0, -2, -2)] { + xti.extend([book.to_le_bytes(), first.to_le_bytes(), last.to_le_bytes()].concat()); + } + let links = [record(353, &[]), record(357, &[]), record(355, &wide("rId9")), record(362, &xti), record(354, &[])]; + [&plain[..plain.len() - end.len()], &links.concat(), &names.concat(), &end].concat() +} + +/// The names below, in record order, as `PtgName` numbers them from 1. +fn names() -> Vec> { + vec![ + // 1: a newer function's hidden name: not a defined name in `.xlsx`. + brt_name(0x0002_000b, 0, WORKBOOK, "_xlfn.CONCAT", &[0x1C, 0x1D], None), + // 2: built-in and hidden, on Data. + brt_name(0x21, 0, 0, "_FilterDatabase", &[&[0x3Bu8, 0, 0][..], &0u32.to_le_bytes(), &9u32.to_le_bytes(), &[0, 0, 2, 0]].concat(), None), + // 3: refers to another workbook: left out. + brt_name(0, 0, WORKBOOK, "External", &ref3d_name(4, 0, 0), None), + // 4: a deleted cell. + brt_name(0, 0, WORKBOOK, "Gone", &[0x3C, 0, 0, 0, 0, 0, 0, 0, 0], None), + brt_name(1, 0, WORKBOOK, "Hid", &[0x1E, 42, 0], None), + // 6, 7: the same name on two sheets. + brt_name(0, 0, 0, "Local", &ref3d_name(0, 1, 1), None), + brt_name(0, 0, 1, "Local", &ref3d_name(1, 0, 1), None), + // 8: an XLM macro with a shortcut key, in the "User Defined" group (14). + brt_name(0x388, b'k', WORKBOOK, "MacroName", &ref3d_name(0, 0, 0), None), + // 9 + brt_name(0, 0, WORKBOOK, "Rate", &[&[0x1Fu8][..], &0.05f64.to_le_bytes()].concat(), Some("a note")), + // 10, 11: relative references are offsets from A1, wrapping around the grid. + brt_name(0, 0, WORKBOOK, "Rel", &ref3d_name(0, 0, 0xFFFF), None), + brt_name(0, 0, WORKBOOK, "RelN", &[&[0x2Cu8][..], &loc(u32::MAX, REL)].concat(), None), + // 12: on a sheet the workbook doesn't have: left out. + brt_name(0, 0, 5, "Skipped", &[0x1E, 1, 0], None), + // 13: a name of a name. + brt_name(0, 0, WORKBOOK, "Twice", &[0x23, 9, 0, 0, 0, 0x1E, 2, 0, 0x05], None), + ] +} + +const NAMES_XML: &str = r#"Data!#REF!Data!$B$2Hidden!$B$1Data!$A$10.05Data!XFD1A1048576Rate*2"#; + +fn xml_names(xml: &str) -> Vec { + let types = r#""#; + let root = r#""#; + let bytes = zip(&[("[Content_Types].xml", types.as_bytes()), ("_rels/.rels", root.as_bytes()), ("xl/workbook.xml", xml.as_bytes())]); + Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap().defined_names +} + +#[test] +fn defined_names_are_read_as_their_xlsx_spelling() { + let cells: Vec> = vec![ + vec![0x23, 9, 0, 0, 0, 0x1E, 100, 0, 0x05], + vec![0x23, 6, 0, 0, 0], + // PtgNameX through this workbook's own link: a sheet-scoped name used from another sheet. + vec![0x39, 5, 0, 7, 0, 0, 0], + vec![0x23, 2, 0, 0, 0], + vec![0x3C, 1, 0, 0, 0, 0, 0, 0, 0], + // Kept as their saved results: a name left out, a name in another workbook, and a + // workbook name through PtgNameX. + vec![0x23, 3, 0, 0, 0], + vec![0x39, 4, 0, 9, 0, 0, 0], + vec![0x39, 5, 0, 9, 0, 0, 0], + ]; + let mut data = row(0); + for (i, rgce) in cells.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, rgce, &[])); + } + let bytes = package(&book_with_names(&names()), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert_eq!(wb.defined_names, xml_names(NAMES_XML)); + let book = read(&bytes).unwrap(); + let got: Vec<_> = (1..=8).map(|c| book.sheets[0].formula(CellRef::new(0, c))).collect(); + let want = [Some("Rate*100"), Some("Local"), Some("Hidden!Local"), Some("_xlnm._FilterDatabase"), Some("Hidden!#REF!"), None, None, None]; + assert_eq!(got, want); + for why in ["defined names whose formula could not be read", "other workbooks", "names in other workbooks or on other sheets"] { + assert!(warned(&book, why), "{why}: {:?}", book.warnings); + } + for why in ["(references to other workbooks (external links))", "(names of sheets the reader skips)"] { + assert!(book.warnings.iter().any(|w| w.contains("defined names were left out") && w.contains(why)), "{why}: {:?}", book.warnings); + } +} + +#[test] +fn a_damaged_name_leaves_names_and_formulas_as_values() { + // A truncated BrtName: the workbook opens with no names, and every formula keeps its value. + let mut list = names(); + let full = brt_name(0, 0, WORKBOOK, "Rate", &[0x1E, 1, 0], None); + list[8] = record(39, &full[2..full.len() - 2]); // The payload without its record header and last two bytes. + let data = [row(0), formula_cell(1, &[0x1E, 1, 0], &[])].concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert!(wb.defined_names.is_empty()); + let book = read(&bytes).unwrap(); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 1)), None); + assert!(book.warnings.iter().any(|w| w.contains("names could not be read")), "{:?}", book.warnings); +} From 2a786e27a2826e9a67cfe0af167410409717b0d3 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 11:30:32 -0500 Subject: [PATCH 003/209] craft-xlsx: names on macro sheets, spans over chart sheets, formula-less built-ins A second Excel probe showed three cases where the .xlsb reader left out what .xlsx has: - Macro names point at an XLM macro sheet (Macro1!$A$1), which the sheet links treated as a sheet with no cells, so every such name was left out. Macro sheets have cells; references to them are now named. - A 3-D span over a chart sheet ('Data:After Sheet'!$A$1) was refused. A span is spelled by its ends; a sheet with no cells between them adds nothing, and .xlsx spells it the same. - Excel writes a workbook-level built-in with no formula (Database) beside a sheet's own. .xlsx leaves it out of definedNames, but formulas using it spell _xlnm.Database; they now do instead of keeping their saved results. --- crates/xlsx/README.md | 7 +++-- crates/xlsx/src/xlsb.rs | 13 ++++++++-- crates/xlsx/src/xlsb/formula.rs | 13 ++++------ crates/xlsx/tests/xlsb.rs | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 4e90ca830..3b636a2fa 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -88,8 +88,11 @@ read (each cell keeps its format index). Its defined names are those of `defined `.xlsx` Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their `localSheetId`, relative references as the cell they reach from A1, macro names with their `function`, `xlm`, `functionGroupId` and `shortcutKey`. Excel's hidden names for newer functions -and LET parameters are left out, as in `.xlsx`; so is a name whose formula can't be read, with a -warning. Its cell formulas are decoded from their tokens into the +and LET parameters are left out, as in `.xlsx`, and so is the formula-less workbook-level built-in +Excel writes beside a sheet's (`Database`), which formulas still spell `_xlnm.Database`; a name +whose formula can't be read is left out with a warning. A reference may name an XLM macro sheet +(`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends whatever chart +sheets lie between them. Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, no implicit `@`), with `FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 1941ea61e..dd52a55b4 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -356,9 +356,10 @@ pub(crate) fn workbook( if let SheetKind::Other(_) = kind { return Err(invalid("unknown sheet relationship type")); } - // Only a worksheet has cells a formula can refer to. + // Only worksheets and macro sheets have cells a formula can refer to (a macro's + // name refers to its macro sheet: `Macro1!$A$1`). if let Some(l) = links.as_mut() { - l.sheets.push((kind == SheetKind::Worksheet).then(|| name.clone())); + l.sheets.push(matches!(kind, SheetKind::Worksheet | SheetKind::Macrosheet).then(|| name.clone())); } bundle_sheets.push(u32::try_from(wb.sheets.len()).ok()); wb.sheets.push(SheetEntry { name, sheet_id, visibility, rel_id, kind, part: Some(rel.target.clone()) }); @@ -412,6 +413,14 @@ fn defined_names( } }, }; + // Excel writes a workbook-level built-in with no formula (`Database`) beside a sheet's + // own; `.xlsx` leaves it out of `definedNames`, and formulas using it spell its name. + if p.rgce.is_empty() { + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + continue; + } let formula = match formula::decode(&p.rgce, &p.rgcb, links, Mode::Name) { Ok(e) => expr::print(&e), Err(formula::Unsupported(why)) => { diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 2e7f7cc96..be6f47bfb 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -118,8 +118,8 @@ impl Name { /// Workbook-level tables formulas refer to. #[derive(Default)] pub(crate) struct Links { - /// Every sheet's name in `BrtBundleSh` order, `None` for sheets the reader skips or that hold - /// no cells (chart, dialog and macro sheets). + /// Every sheet's name in `BrtBundleSh` order, `None` for the sheets that hold no cells (chart, + /// dialog and module sheets). pub(super) sheets: Vec>, /// Every `BrtName`, in order: `PtgName` indexes them from 1. pub(super) names: Vec, @@ -167,14 +167,11 @@ impl Links { let (Ok(first), Ok(last)) = (usize::try_from(x.first), usize::try_from(x.last)) else { return Err(Unsupported("reference to a missing sheet")); }; - // A span takes every sheet between its ends, so a skipped one anywhere in it changes - // what the reference means. + // A span is spelled by its ends. A sheet between them with no cells (a chart sheet) adds + // nothing to it, and `.xlsx` spells the span the same; an end with no cells can't be named. let span = self.sheets.get(first..=last).filter(|s| !s.is_empty()).ok_or(Unsupported("reference to a missing sheet"))?; - if span.iter().any(Option::is_none) { - return Err(Unsupported("references to sheets the reader skips")); - } let (Some(Some(a)), Some(Some(b))) = (span.first(), span.last()) else { - return Err(Unsupported("reference to a missing sheet")); + return Err(Unsupported("references to sheets with no cells")); }; Ok(Some(if first == last { SheetSel::Named(a.clone()) } else { SheetSel::Span(a.clone(), b.clone()) })) } diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 73bd23218..78e2d89b1 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1057,3 +1057,49 @@ fn a_damaged_name_leaves_names_and_formulas_as_values() { assert_eq!(book.sheets[0].formula(CellRef::new(0, 1)), None); assert!(book.warnings.iter().any(|w| w.contains("names could not be read")), "{:?}", book.warnings); } + +#[test] +fn names_on_macro_sheets_spans_over_chart_sheets_and_empty_built_ins() { + // Sheets Data, a chart sheet, an XLM macro sheet, Hidden; sheet links Macro1, Data:Hidden + // (over both) and Data. + let mut book = record(131, &[]); + book.extend(record(153, &[[0u8; 8].as_slice(), &wide("")].concat())); + book.extend(record(143, &[])); + for (id, rel, name) in [(1u32, "one", "Data"), (3, "c", "Chart1"), (4, "m", "Macro1"), (2, "two", "Hidden")] { + book.extend(record(156, &[&0u32.to_le_bytes()[..], &id.to_le_bytes(), &wide(rel), &wide(name)].concat())); + } + book.extend(record(144, &[])); + let mut xti = 3u32.to_le_bytes().to_vec(); + for (first, last) in [(2i32, 2i32), (0, 3), (0, 0)] { + xti.extend([0u32.to_le_bytes(), first.to_le_bytes(), last.to_le_bytes()].concat()); + } + book.extend([record(353, &[]), record(357, &[]), record(362, &xti), record(354, &[])].concat()); + let area = [&[0x3Bu8][..], &2u16.to_le_bytes(), &0u32.to_le_bytes(), &1u32.to_le_bytes(), &0u16.to_le_bytes(), &1u16.to_le_bytes()].concat(); + // A sheet's built-in Database, then the workbook-level one Excel writes with no formula. + book.extend(brt_name(0x20, 0, 0, "Database", &area, None)); + book.extend(brt_name(0x20, 0, WORKBOOK, "Database", &[], None)); + book.extend(brt_name(0x388, b'k', WORKBOOK, "Cmd", &ref3d_name(0, 0, 0), None)); + book.extend(brt_name(0, 0, WORKBOOK, "Span", &ref3d_name(1, 0, 0), None)); + book.extend(record(132, &[])); + let rels = RELS.replace( + "", + r#""#, + ); + let ref3d_cell = |ixti: u16| [&[0x5Au8][..], &ixti.to_le_bytes(), &loc(0, 0)].concat(); + let data = [ + row(0), + formula_cell(1, &[0x23, 2, 0, 0, 0], &[]), + formula_cell(2, &[0x23, 4, 0, 0, 0], &[]), + formula_cell(3, &ref3d_cell(0), &[]), + formula_cell(4, &ref3d_cell(1), &[]), + ] + .concat(); + let bytes = package(&book, &worksheet(&data, &[]), &sst(), ROOT, &rels, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"Data!$A$1:$B$2Macro1!$A$1Data:Hidden!$A$1"#; + assert_eq!(wb.defined_names, xml_names(xml)); + let read = read(&bytes).unwrap(); + let got: Vec<_> = (1..=4).map(|c| read.sheets[0].formula(CellRef::new(0, c))).collect(); + assert_eq!(got, [Some("_xlnm.Database"), Some("Span"), Some("Macro1!$A$1"), Some("Data:Hidden!$A$1")]); + assert!(read.warnings.iter().all(|w| !w.contains("left out") && !w.contains("saved results")), "{:?}", read.warnings); +} From 4a250888020df77cb7bdac49252c6d79ae15c1cc Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 11:18:42 -0500 Subject: [PATCH 004/209] craft-xlsx: read the tables of a binary workbook Package::tables reads a sheet's tableN.bin parts, through its BrtListParts, into the Table the XML reader builds, as Excel writes the table in .xlsx: BrtBeginList and its columns, the calculated and totals formulas decoded as cell formulas are, an XML map's column properties, the table style, and the AutoFilter and sort state built as the autoFilter and sortState elements. A filter column whose criteria are in an extension (date groups) or on a Boolean value is left out with a warning, as is a sort by stroke or PinYin order. PtgList now decodes to the structured reference .xlsx holds (Sales[Val], Sales[[#Headers],[Key]], escaped column names, Excel's spacing flags), from the workbook's tables, indexed when the workbook part is read. A reference to a table that can't be read, a deleted table or another workbook keeps the cell's saved result. Closes #1251. --- crates/xlsx/README.md | 21 +- crates/xlsx/src/package.rs | 59 ++- crates/xlsx/src/xlsb.rs | 24 +- crates/xlsx/src/xlsb/expr.rs | 3 + crates/xlsx/src/xlsb/formula.rs | 98 +++- crates/xlsx/src/xlsb/table.rs | 770 ++++++++++++++++++++++++++++++++ crates/xlsx/tests/xlsb.rs | 266 ++++++++++- 7 files changed, 1228 insertions(+), 13 deletions(-) create mode 100644 crates/xlsx/src/xlsb/table.rs diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index e03128e81..c99218085 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -91,9 +91,24 @@ the formula, numbers as Excel spells them), with `FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: user defined names, table references, -other workbooks, Python code stored outside the cell, data tables. The XLSB function table -(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. +keeps only its saved result, with a warning saying why: user defined names, other workbooks, +Python code stored outside the cell, data tables. + +A binary workbook's tables (`tableN.bin`, listed by the sheet's `BrtListPart`s) read through +`Package::tables` into the same `Table`, as Excel writes the table in `.xlsx`: the calculated and +totals formulas decoded as cell formulas are, an XML map's column properties, and the AutoFilter and +sort state built as the `autoFilter` and `sortState` elements, with numbers in a filter spelled as +Excel writes them there (17 significant digits; Excel writes a few values shorter, such as `0.1`, +which read back the same). A filter column the reader can't spell is left out with a warning (Excel +keeps date-group filters in an extension the reader skips; a custom filter on a Boolean value has no +`.xlsx` spelling Excel has shown), as is a sort by stroke or PinYin order. The revision ids Excel +writes (`xr:uid`) are in alternate-content blocks, which are skipped. A structured reference in a +formula (`PtgList`) is spelled as `.xlsx` has it (`Sales[Val]`, `Sales[[#Headers],[Key]]`, +`Sales[[#This Row],[A]:[B]]`, with Excel's spacing flags), from the table's id, which is +workbook-wide: the workbook's table parts are indexed when its workbook part is read. A reference to +a table that can't be read, to a deleted table or into another workbook keeps the cell's saved +result. The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, +apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index 8f2c5efc6..820132836 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -12,7 +12,7 @@ use crate::pivot::{CacheRecord, PivotCacheDefinition, PivotTableDefinition, read use crate::sheet::{Row, RowReader, SheetPart}; use crate::strings::StringItem; use crate::styles::Styles; -use crate::workbook::{SheetEntry, Workbook}; +use crate::workbook::{SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings, xlsb}; /// Most bytes of `[Content_Types].xml` read from a binary workbook. @@ -149,7 +149,10 @@ impl<'a> Package<'a> { if self.binary.is_some() { let bytes = self.required(&self.workbook_part.clone())?; let rels = self.workbook_rels()?; - let (wb, links) = xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings)?; + let (wb, mut links) = xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings)?; + if let Some(links) = links.as_mut() { + self.index_tables(&wb, links)?; + } self.binary_links = Some(links); return Ok(wb); } @@ -296,10 +299,31 @@ impl<'a> Package<'a> { Ok(Some((part, Drawing::read(&root), rels))) } - /// The tables of a worksheet read by [`Package::sheet`], in the order of its `tableParts`, - /// each with its part name. A table part that does not read is skipped with a warning. + /// The tables of a worksheet read by [`Package::sheet`], in the order of its `tableParts` + /// (its `BrtListPart`s in a binary workbook), each with its part name. A table part that does + /// not read is skipped with a warning. pub fn tables(&mut self, sheet: &SheetPart) -> Result, Error> { let mut out = Vec::new(); + if self.binary.is_some() { + let rels = self.opc.rels(&sheet.part)?; + for id in &sheet.features.table_parts { + let Some(rel) = rels.by_id(id).filter(|r| !r.external && r.kind() == "table") else { + self.warn(format!("XLSB: {}: a table part relationship is missing", sheet.part)); + continue; + }; + let part = rel.target.clone(); + let read = self.binary_part(&part, xlsb::TABLE).and_then(|bytes| { + let links = self.binary_links.as_ref().and_then(Option::as_ref); + xlsb::table(&bytes, links, &self.limits, &self.budget, &mut self.warnings) + }); + match read { + Ok(t) => out.push((part, t)), + Err(Error::TooLarge(m)) => return Err(Error::TooLarge(m)), + Err(e) => self.warn(format!("skipped {part}: {e}")), + } + } + return Ok(out); + } for id in &sheet.features.table_parts { let Some(rel) = sheet.rels.by_id(id).filter(|r| !r.external) else { continue }; let part = rel.target.clone(); @@ -392,6 +416,33 @@ impl<'a> Package<'a> { Ok(root.map(|r| craft_drawingml::theme::read_theme(&mut craft_drawingml::part::NoSource, &r))) } + /// A part of a binary workbook, checked against its content type. + fn binary_part(&mut self, part: &str, content_type: &str) -> Result, Error> { + if let Some(types) = &self.binary { + xlsb::check_type(types, part, content_type)?; + } + self.required(part) + } + + /// Indexes the table parts of a binary workbook's worksheets, for the structured references + /// in its formulas. A table part that does not read is left out with a warning: formulas + /// using it keep their saved results. + fn index_tables(&mut self, wb: &Workbook, links: &mut xlsb::Links) -> Result<(), Error> { + for entry in wb.sheets.iter().filter(|e| e.kind == SheetKind::Worksheet) { + let Some(part) = &entry.part else { continue }; + let rels = self.opc.rels(part)?; + for rel in rels.list.iter().filter(|r| r.kind() == "table" && !r.external) { + let read = self.binary_part(&rel.target, xlsb::TABLE).and_then(|bytes| xlsb::index_table(links, &bytes, &self.limits, &self.budget)); + match read { + Ok(()) => {} + Err(Error::TooLarge(m)) => return Err(Error::TooLarge(m)), + Err(e) => self.warn(format!("XLSB: {} could not be read ({e}); formulas using its table keep their saved results", rel.target)), + } + } + } + Ok(()) + } + fn required(&mut self, part: &str) -> Result, Error> { self.opc.read(part)?.ok_or_else(|| Error::Invalid(format!("missing part {part}"))) } diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 65aab0eb8..13e687d2c 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,11 +1,13 @@ //! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from //! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an //! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why. -//! Styles are not read. A record that does not parse fails the read. +//! Table parts are read in `table`. Styles are not read. A record that does not parse fails +//! the read. mod expr; mod formula; mod ftab; +mod table; use std::collections::HashMap; use std::sync::Arc; @@ -19,6 +21,7 @@ use crate::workbook::{SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; +pub(crate) use table::{TABLE, index_table, table}; pub(crate) const MAIN: &str = "application/vnd.ms-excel.main"; const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main"; @@ -28,6 +31,8 @@ pub(crate) const STRINGS: &str = "application/vnd.ms-excel.sharedStrings"; const MAX_SHEETS: usize = 1024; /// Most merged ranges in one worksheet. const MAX_MERGES: usize = 100_000; +/// Table parts one worksheet lists at most. +const MAX_TABLE_PARTS: usize = 65_536; /// Defined names (Excel's hidden ones for newer functions included) read at most; past it, the /// workbook's formulas keep their saved results. const MAX_NAMES: usize = 65_536; @@ -172,11 +177,13 @@ struct Records<'a, 'l> { limits: &'l Limits, budget: &'l Budget, blocks: Vec, + /// Future-record and alternate-content blocks skipped so far. + skipped: usize, } impl<'a, 'l> Records<'a, 'l> { fn new(bytes: &'a [u8], limits: &'l Limits, budget: &'l Budget) -> Self { - Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![] } + Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], skipped: 0 } } fn next(&mut self, warnings: &mut Warnings) -> Result, Error> { @@ -219,6 +226,7 @@ impl<'a, 'l> Records<'a, 'l> { return Err(invalid("extension blocks nested too deeply")); } self.blocks.push(id + 1); + self.skipped += 1; warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped"); } // BrtFRTEnd, BrtACEnd. @@ -886,6 +894,18 @@ pub(crate) fn worksheet( }); col = Some(c); } + // BrtListPart: one of the sheet's tables, by relationship id. + 661 => { + if data { + return Err(invalid("table part inside worksheet data")); + } + let rel = b.text(255)?; + b.finish()?; + if sheet.features.table_parts.len() >= MAX_TABLE_PARTS { + return Err(Error::TooLarge(format!("XLSB exceeds {MAX_TABLE_PARTS} tables on a sheet"))); + } + sheet.features.table_parts.push(rel); + } 176 => { if data { return Err(invalid("merge inside worksheet data")); diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index 197644a33..96dc8f03a 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -123,6 +123,8 @@ pub(crate) enum Expr { /// Spaces and line breaks typed before and after an expression (`PtgAttrSpace`), which /// `.xlsx` formula text keeps. Spaced(String, Box, String), + /// A structured reference to a table, as written (`Sales[[#Headers],[Key]]`). + Structured(String), Unary(UnOp, Box), Binary(BinOp, Box, Box), Call(String, Vec), @@ -310,6 +312,7 @@ fn write_expr(s: &mut String, e: &Expr) { } None => s.push_str(n), }, + Expr::Structured(t) => s.push_str(t), Expr::Unary(UnOp::Neg, x) => { s.push('-'); write_expr(s, x); diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 3a6960b39..7cc820ff8 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -5,10 +5,12 @@ //! ([`super::ftab`]) and its own copies of craft-xls's depth guards and limits. Anything outside the //! token set it knows is reported as unsupported, and the caller keeps the cell's cached value. //! So is a formula that needs something the reader doesn't read: a user's defined name (XLSB -//! names are not read yet) or Python code stored outside the cell. +//! names are not read yet), a table whose part could not be read, or Python code stored outside +//! the cell. Structured references (`PtgList`) are spelled from the tables in [`Links`]. //! //! Names keep their file spelling (`_xlfn.CONCAT`, `_xlpm.x`), as `.xlsx` formula text has them. +use std::collections::HashMap; use std::sync::Arc; use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS}; @@ -154,6 +156,15 @@ pub(crate) struct Links { pub(super) names: Vec, pub(super) books: Vec, pub(super) xti: Vec, + /// Every table by its `idList`, which `PtgList` names: `None` for an id two tables share. + pub(super) tables: HashMap>, +} + +/// A table as structured references need it: its display name and its columns' names. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct ListInfo { + pub name: String, + pub columns: Vec, } /// Whether a name stands for a LET parameter (`_xlpm.x`), which only the formula itself defines. @@ -530,7 +541,13 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res } 0x01 => return Err(Unsupported("the shared or array formula form")), 0x02 => return Err(Unsupported("what-if data tables")), - 0x18 => return Err(Unsupported("structured table references")), + 0x18 => { + if t.u8()? != 0x19 { + return Err(Unsupported("unknown formula token")); + } + let text = list(&mut t, links)?; + push(&mut stack, kids, Expr::Structured(text))?; + } 0x2C => { let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; let (r, c) = (t.u32()?, t.u16()?); @@ -570,6 +587,83 @@ fn bare_function(name: String) -> String { } } +/// `PtgList` ([MS-XLSB] 2.5.98.52), after its two token bytes: a structured reference, spelled +/// as `.xlsx` has it (`Sales[Val]`, `Sales[[#Headers],[Key]]`, `Sales[[#This Row],[A]:[B]]`). +/// The two spacing flags put a space inside the outer brackets and after each comma, as Excel +/// shows the reference. +fn list(t: &mut Bytes<'_>, links: &Links) -> Result { + let ixti = t.u16()?; + let bits = t.u16()?; + let index = t.u32()?; + let (first, last) = (usize::from(t.u16()?), usize::from(t.u16()?)); + if bits & 0x3000 != 0 { + return Err(Unsupported("structured references to deleted tables or to tables in other workbooks")); + } + // The table's sheet link: a table is always in this workbook, on one of its sheets. + links.sheet_sel(ixti)?; + let table = links.tables.get(&index).and_then(Option::as_ref).ok_or(Unsupported("structured references to tables the reader could not read"))?; + let rows: &[&str] = match (bits >> 2) & 0x1F { + 0x00 => &[], + 0x01 => &["#All"], + 0x02 => &["#Headers"], + 0x04 => &["#Data"], + 0x06 => &["#Headers", "#Data"], + 0x08 => &["#Totals"], + 0x0C => &["#Data", "#Totals"], + 0x10 => &["#This Row"], + _ => return Err(Unsupported("unknown structured reference rows")), + }; + let column = |i: usize| table.columns.get(i).map(|c| escape_column(c)).ok_or(Unsupported("structured reference to a missing table column")); + let columns = match bits & 3 { + 0 => None, + 1 => Some((column(first)?, None)), + 2 if first <= last => Some((column(first)?, Some(column(last)?))), + _ => return Err(Unsupported("invalid structured reference columns")), + }; + let mut s = table.name.clone(); + match (rows, &columns) { + ([], None) => s.push_str("[]"), + ([], Some((c, None))) => { + s.push('['); + s.push_str(c); + s.push(']'); + } + ([row], None) => { + s.push('['); + s.push_str(row); + s.push(']'); + } + _ => { + let mut items: Vec = rows.iter().map(|r| format!("[{r}]")).collect(); + match &columns { + Some((a, Some(b))) => items.push(format!("[{a}]:[{b}]")), + Some((a, None)) => items.push(format!("[{a}]")), + None => {} + } + let pad = if bits & 0x80 != 0 { " " } else { "" }; + let comma = if bits & 0x100 != 0 { ", " } else { "," }; + s.push('['); + s.push_str(pad); + s.push_str(&items.join(comma)); + s.push_str(pad); + s.push(']'); + } + } + Ok(s) +} + +/// A column name in a structured reference: `'` escapes `[`, `]`, `#` and `'` (`It''s '['#1']`). +fn escape_column(name: &str) -> String { + let mut s = String::with_capacity(name.len()); + for c in name.chars() { + if matches!(c, '[' | ']' | '#' | '\'') { + s.push('\''); + } + s.push(c); + } + s +} + /// Text written out in the formula: a string, or strings joined with `&` (how Python code /// longer than a string constant is written). fn literal_text(e: &Expr) -> bool { diff --git a/crates/xlsx/src/xlsb/table.rs b/crates/xlsx/src/xlsb/table.rs new file mode 100644 index 000000000..71e8f3d92 --- /dev/null +++ b/crates/xlsx/src/xlsb/table.rs @@ -0,0 +1,770 @@ +//! Table parts ([MS-XLSB] 2.1.7.51, `tableN.bin`) into the [`Table`] a `.xlsx` table part reads +//! as. The table's AutoFilter and sort state are held as `.xlsx` elements, so they are built here +//! as Excel writes them. A filter column the reader can't spell is left out with a warning, as +//! is a sort state; the table keeps the rest. + +use craft_sheet::{CellRef, MAX_COLS, MAX_ROWS, Range}; +use craft_xml::El; + +use super::formula::{self, ListInfo, Mode}; +use super::{Budget, Bytes, Links, Records, expr, invalid}; +use crate::features::{Table, TableColumn, TableStyleInfo}; +use crate::{Error, Limits, Warnings}; + +/// The content type of a table part. +pub(crate) const TABLE: &str = "application/vnd.ms-excel.table"; + +/// `RfX` / `UncheckedRfX` ([MS-XLSB] 2.5.118, 2.5.154): first and last row, first and last column. +fn rfx(b: &mut Bytes<'_>) -> Result { + let (r0, r1, c0, c1) = (b.u32()?, b.u32()?, b.u32()?, b.u32()?); + if r0 > r1 || c0 > c1 || r1 >= MAX_ROWS || c1 >= MAX_COLS { + return Err(invalid("invalid table range")); + } + Ok(Range::new(CellRef::new(r0, c0), CellRef::new(r1, c1))) +} + +/// A `DXFId`, `None` for 0xFFFFFFFF (no differential format). +fn dxf(id: u32) -> Option { + (id != u32::MAX).then_some(id) +} + +/// An `XLNullableWideString` of at most `max` characters, with the empty string as none: Excel +/// writes an empty comment where `.xlsx` has no attribute. +fn text(b: &mut Bytes<'_>, max: u32) -> Result, Error> { + Ok(b.string(max, true)?.filter(|s| !s.is_empty())) +} + +/// What a table part holds of formulas, for the reader's index of tables: its id, display name +/// and column names. +fn list_info(bytes: &[u8], limits: &Limits, budget: &Budget) -> Result<(u32, ListInfo), Error> { + let t = table(bytes, None, limits, budget, &mut Warnings::default())?; + let id = t.id.ok_or_else(|| invalid("table without an id"))?; + let name = t.display_name.or(t.name).ok_or_else(|| invalid("table without a name"))?; + // A column without a name can't be spelled in a structured reference. + let columns = t + .columns + .into_iter() + .map(|c| c.name.filter(|n| !n.is_empty())) + .collect::>() + .ok_or_else(|| invalid("table column without a name"))?; + Ok((id, ListInfo { name, columns })) +} + +/// Adds a table part to `links`, so formulas can name it. Two tables with one id name neither. +pub(crate) fn index_table(links: &mut Links, bytes: &[u8], limits: &Limits, budget: &Budget) -> Result<(), Error> { + let (id, info) = list_info(bytes, limits, budget)?; + links.tables.entry(id).and_modify(|t| *t = None).or_insert(Some(info)); + Ok(()) +} + +/// A table part. `links` decodes its calculated-column and totals-row formulas; without them +/// (or when one can't be decoded) a formula is left out, with a warning. +pub(crate) fn table(bytes: &[u8], links: Option<&Links>, limits: &Limits, budget: &Budget, warnings: &mut Warnings) -> Result { + let mut records = Records::new(bytes, limits, budget); + let mut table = match records.next(warnings)? { + Some((343, payload)) => begin_list(payload)?, + _ => return Err(invalid("missing table begin record")), + }; + let mut cols_open = false; + let mut cols_seen = false; + let mut column: Option = None; + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + match id { + 161 if !cols_seen && table.auto_filter.is_none() => { + let range = rfx(&mut b)?; + b.finish()?; + table.auto_filter = Some(auto_filter(range, &mut records, warnings)?); + } + 530 if !cols_seen && table.sort_state.is_none() => table.sort_state = sort_state(payload, &mut records, warnings)?, + 345 if !cols_seen => { + b.u32()?; + b.finish()?; + cols_open = true; + cols_seen = true; + } + 346 if cols_open && column.is_none() => { + b.finish()?; + cols_open = false; + } + 347 if cols_open && column.is_none() => { + if table.columns.len() >= MAX_COLS as usize { + return Err(invalid("too many table columns")); + } + column = Some(begin_list_col(payload)?); + } + 351 | 352 if column.is_some() => { + let f = list_formula(payload, links, limits, budget, warnings)?; + if let Some(c) = column.as_mut() { + *(if id == 351 { &mut c.calculated_formula } else { &mut c.totals_row_formula }) = f; + } + } + 348 => { + b.finish()?; + table.columns.push(column.take().ok_or_else(|| invalid("table column end without a column"))?); + } + 349 => { + let c = column.as_mut().ok_or_else(|| invalid("XML column properties outside a table column"))?; + c.other.push(xml_column(payload)?); + } + 513 if !cols_open && cols_seen => { + let flags = b.u16()?; + let name = b.string(255, true)?; + b.finish()?; + let bit = |n: u16| Some(flags & (1 << n) != 0); + table.style = Some(TableStyleInfo { + name, + show_first_column: bit(0), + show_last_column: bit(1), + show_row_stripes: bit(2), + show_column_stripes: bit(3), + ..TableStyleInfo::default() + }); + } + 344 => { + if cols_open || !cols_seen || column.is_some() { + return Err(invalid("incomplete table")); + } + records.end(payload, warnings)?; + return Ok(table); + } + // The end of an XML map's column; anything else out of place is damage. + 350 => b.finish()?, + _ => return Err(invalid(format!("unexpected record {id} in a table"))), + } + } + Err(invalid("missing table end record")) +} + +/// `BrtBeginList` ([MS-XLSB] 2.4.100). `.xlsx` leaves out what is the default: one header row, +/// no totals row, a totals row that has been shown, a worksheet table. +fn begin_list(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let range = rfx(&mut b)?; + let table_type = match b.u32()? { + 0 => None, + 2 => Some("xml".to_string()), + 3 => Some("queryTable".to_string()), + _ => return Err(invalid("invalid table type")), + }; + let id = b.u32()?; + if id == 0 || id == u32::MAX { + return Err(invalid("invalid table id")); + } + let header = b.u32()?; + let totals = b.u32()?; + let flags = b.u32()?; + // fSingleCell: a single-cell table belongs in its own part type, not here. + if flags & 2 != 0 { + return Err(invalid("single-cell table in a table part")); + } + let dxfs: Vec> = (0..6).map(|_| b.u32().map(dxf)).collect::>()?; + let connection = b.u32()?; + let name = b.string(255, true)?; + let display_name = b.string(255, true)?; + let comment = text(&mut b, 255)?; + let header_row_cell_style = b.string(255, true)?; + let data_cell_style = b.string(255, true)?; + let totals_row_cell_style = b.string(255, true)?; + b.finish()?; + let set = |bit: u32| (flags & bit != 0).then_some(true); + Ok(Table { + id: Some(id), + name, + display_name, + comment, + table_type, + header_row_count: (header != 1).then_some(header), + insert_row: set(4), + insert_row_shift: set(8), + totals_row_count: (totals != 0).then_some(totals), + totals_row_shown: (flags & 1 == 0).then_some(false), + published: set(16), + header_row_dxf_id: dxfs[0], + data_dxf_id: dxfs[1], + totals_row_dxf_id: dxfs[2], + table_border_dxf_id: dxfs[3], + header_row_border_dxf_id: dxfs[4], + totals_row_border_dxf_id: dxfs[5], + header_row_cell_style, + data_cell_style, + totals_row_cell_style, + connection_id: (connection != 0).then_some(connection), + range, + ..Table::default() + }) +} + +/// `BrtBeginListCol` ([MS-XLSB] 2.4.101): `stName` is `.xlsx`'s `uniqueName`, `stCaption` its +/// `name`, and the insert row's format and style are the data's. +fn begin_list_col(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let id = b.u32()?; + let function = match b.u32()? { + 0 => None, + 1 => Some("average"), + 2 => Some("count"), + 3 => Some("countNums"), + 4 => Some("max"), + 5 => Some("min"), + 6 => Some("sum"), + 7 => Some("stdDev"), + 8 => Some("var"), + 9 => Some("custom"), + _ => return Err(invalid("invalid totals row function")), + }; + let header_row_dxf_id = dxf(b.u32()?); + let data_dxf_id = dxf(b.u32()?); + let totals_row_dxf_id = dxf(b.u32()?); + let query = b.u32()?; + let unique_name = b.string(255, true)?; + let name = b.string(255, true)?; + let totals_row_label = b.string(8189, true)?; + let header_row_cell_style = b.string(255, true)?; + let data_cell_style = b.string(255, true)?; + let totals_row_cell_style = b.string(255, true)?; + b.finish()?; + Ok(TableColumn { + id: Some(id), + unique_name, + name, + totals_row_function: function.map(str::to_string), + totals_row_label, + query_table_field_id: (query != 0).then_some(query), + header_row_dxf_id, + data_dxf_id, + totals_row_dxf_id, + header_row_cell_style, + data_cell_style, + totals_row_cell_style, + ..TableColumn::default() + }) +} + +/// `XmlDataType` ([MS-XLSB] 2.5.170) from 1, as `.xlsx` names the XML Schema types. +const XML_TYPES: [&str; 45] = [ + "string", + "normalizedString", + "token", + "byte", + "unsignedByte", + "base64Binary", + "hexBinary", + "integer", + "positiveInteger", + "negativeInteger", + "nonPositiveInteger", + "nonNegativeInteger", + "int", + "unsignedInt", + "long", + "unsignedLong", + "short", + "unsignedShort", + "decimal", + "float", + "double", + "boolean", + "time", + "dateTime", + "duration", + "date", + "gMonth", + "gYear", + "gYearMonth", + "gDay", + "gMonthDay", + "Name", + "QName", + "NCName", + "anyURI", + "language", + "ID", + "IDREF", + "IDREFS", + "ENTITY", + "ENTITIES", + "NOTATION", + "NMTOKEN", + "NMTOKENS", + "anyType", +]; + +/// `BrtBeginListXmlCPr` ([MS-XLSB] 2.4.104): a column's place in an XML map, as `xmlColumnPr`. +/// `fCanBeSingle` (one node for every row) is `.xlsx`'s `denormalized`. +fn xml_column(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let map = b.u32()?; + let flags = b.u32()?; + let kind = b.u32()?; + let xpath = b.text(31_999)?; + b.finish()?; + let kind = kind.checked_sub(1).and_then(|k| XML_TYPES.get(k as usize)).ok_or_else(|| invalid("invalid XML data type"))?; + let mut el = El::new("xmlColumnPr").with_attr("mapId", map).with_attr("xpath", xpath); + if flags & 2 != 0 { + el.set_attr("denormalized", 1); + } + el.set_attr("xmlDataType", kind); + Ok(el) +} + +/// `BrtListCCFmla` / `BrtListTrFmla` ([MS-XLSB] 2.4.706, 2.4.708): a flag byte (`fArray`, bit 1) +/// and a `ListParsedFormula`. Its references hold coordinates, as a cell formula's do. +fn list_formula( + payload: &[u8], + links: Option<&Links>, + limits: &Limits, + budget: &Budget, + warnings: &mut Warnings, +) -> Result, Error> { + let mut b = Bytes::new(payload); + let array = b.u8()? & 2 != 0; + let cce = b.u32()?; + if !(1..=16384).contains(&cce) { + return Err(invalid("invalid formula token length")); + } + let rgce = b.take(cce as usize)?; + let cb = b.u32()?; + let rgcb = b.take(cb as usize)?; + b.finish()?; + let Some(links) = links else { return Ok(None) }; + let text = match formula::decode(rgce, rgcb, links, Mode::Cell) { + Ok(e) => expr::print(&e), + Err(formula::Unsupported(why)) => { + warnings.warn(format!("XLSB: some table column formulas were left out ({why})")); + return Ok(None); + } + }; + if !budget.try_formula_text(text.len(), limits) { + warnings.warn("XLSB: some table column formulas were left out (more formula text than the limits allow)"); + return Ok(None); + } + Ok(Some((text, array))) +} + +/// A number in a filter, as Excel writes it in `.xlsx`, which differs from a formula's: 17 +/// significant digits (`1234.5678901234501`, `1.0000000000000001E-5`), written out from 0.01 up to +/// 10^16 unless that takes more than 17 decimals, and otherwise with an exponent of no fixed +/// width (`1E-4`, `1E+20`). Excel writes a few values with fewer digits (`0.1`, not +/// `0.10000000000000001`); both read back as the same number. +fn filter_number(n: f64) -> String { + if n == 0.0 || !n.is_finite() { + return "0".into(); + } + let sci = format!("{:.16e}", n.abs()); + let Some((mantissa, exp)) = sci.split_once('e') else { return sci }; + let e: i32 = exp.parse().unwrap_or(0); + let digits = mantissa.replace('.', ""); + let digits = digits.trim_end_matches('0'); + let digits = if digits.is_empty() { "0" } else { digits }; + let decimals = i32::try_from(digits.len()).unwrap_or(i32::MAX).saturating_sub(1).saturating_sub(e); + let mut s = String::from(if n < 0.0 { "-" } else { "" }); + if !(-2..=15).contains(&e) || decimals > 17 { + s.push_str(&digits[..1]); + if digits.len() > 1 { + s.push('.'); + s.push_str(&digits[1..]); + } + s.push_str(&format!("E{}{}", if e < 0 { '-' } else { '+' }, e.unsigned_abs())); + } else if e >= 0 { + let whole = e as usize + 1; + if digits.len() <= whole { + s.push_str(digits); + s.push_str(&"0".repeat(whole - digits.len())); + } else { + s.push_str(&digits[..whole]); + s.push('.'); + s.push_str(&digits[whole..]); + } + } else { + s.push_str("0."); + s.push_str(&"0".repeat((-e - 1) as usize)); + s.push_str(digits); + } + s +} + +/// Warns that a table's filter column was left out, and why. +fn left_out(warnings: &mut Warnings, why: &str) { + warnings.warn(format!("XLSB: some table filters were left out ({why})")); +} + +/// The records after `BrtBeginAFilter` ([MS-XLSB] 2.4.8) up to `BrtEndAFilter`, as `autoFilter`. +fn auto_filter(range: Range, records: &mut Records<'_, '_>, warnings: &mut Warnings) -> Result { + let mut el = El::new("autoFilter").with_attr("ref", range.a1()); + while let Some((id, payload)) = records.next(warnings)? { + match id { + 163 => { + if let Some(c) = filter_column(payload, records, warnings)? { + el.push(c); + } + } + 162 => { + Bytes::new(payload).finish()?; + return Ok(el); + } + _ => return Err(invalid(format!("unexpected record {id} in a table filter"))), + } + } + Err(invalid("missing table filter end record")) +} + +/// A `BrtBeginFilterColumn` ([MS-XLSB] 2.4.84) and its criteria, up to `BrtEndFilterColumn`, as +/// `filterColumn`. `None` when its criteria can't be spelled as `.xlsx` has them, or are kept +/// in an extension the reader skips (Excel's date-group filters). +fn filter_column(payload: &[u8], records: &mut Records<'_, '_>, warnings: &mut Warnings) -> Result, Error> { + let mut b = Bytes::new(payload); + let col = b.u32()?; + let flags = b.u16()?; + b.finish()?; + let mut el = El::new("filterColumn").with_attr("colId", col); + if flags & 1 != 0 { + el.set_attr("hiddenButton", 1); + } + if flags & 2 != 0 { + el.set_attr("showButton", 0); + } + let skipped = records.skipped; + let mut keep = true; + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + match id { + 165 => { + let blank = b.u32()?; + b.u32()?; + b.finish()?; + let mut filters = El::new("filters"); + if blank == 1 { + filters.set_attr("blank", 1); + } + loop { + let (id, payload) = records.next(warnings)?.ok_or_else(|| invalid("missing filter list end record"))?; + let mut b = Bytes::new(payload); + match id { + 167 => { + let val = b.text(255)?; + b.finish()?; + filters.push(El::new("filter").with_attr("val", val)); + } + 175 => filters.push(date_group_item(payload)?), + 166 => { + b.finish()?; + break; + } + _ => return Err(invalid(format!("unexpected record {id} in a filter list"))), + } + } + el.push(filters); + } + 172 => { + let and = b.u32()?; + b.finish()?; + // Excel writes 1 where `.xlsx` has `and="1"` ([MS-XLSB]'s table has them the other way round). + let mut custom = El::new("customFilters"); + if and == 1 { + custom.set_attr("and", 1); + } + loop { + let (id, payload) = records.next(warnings)?.ok_or_else(|| invalid("missing custom filter end record"))?; + match id { + 174 => match custom_filter(payload)? { + Some(f) => custom.push(f), + None => { + left_out(warnings, "custom filters on a Boolean value"); + keep = false; + } + }, + 173 => { + Bytes::new(payload).finish()?; + break; + } + _ => return Err(invalid(format!("unexpected record {id} in custom filters"))), + } + } + el.push(custom); + } + 170 => { + let flags = b.u8()?; + let val = b.f64()?; + let filter_val = b.f64()?; + b.finish()?; + let mut top = El::new("top10"); + if flags & 1 == 0 { + top.set_attr("top", 0); + } + if flags & 2 != 0 { + top.set_attr("percent", 1); + } + top.set_attr("val", filter_number(val)); + if flags & 4 != 0 { + top.set_attr("filterVal", filter_number(filter_val)); + } + el.push(top); + } + 171 => el.push(dynamic_filter(payload)?), + 168 => { + let dxf_id = b.u32()?; + let cell = b.u32()?; + b.finish()?; + let mut color = El::new("colorFilter").with_attr("dxfId", dxf_id); + if cell == 0 { + color.set_attr("cellColor", 0); + } + el.push(color); + } + 169 => { + let set = b.u32()?; + let icon = b.u32()?; + b.finish()?; + // KPINIL: no filter. + if set != u32::MAX { + let name = ICON_SETS.get(set as usize).ok_or_else(|| invalid("invalid filter icon set"))?; + el.push(El::new("iconFilter").with_attr("iconSet", name).with_attr("iconId", icon)); + } + } + 164 => { + b.finish()?; + if records.skipped != skipped { + left_out(warnings, "filters kept in extensions the reader skips, such as date groups"); + return Ok(None); + } + return Ok(keep.then_some(el)); + } + _ => return Err(invalid(format!("unexpected record {id} in a filter column"))), + } + } + Err(invalid("missing filter column end record")) +} + +/// `KPISets` ([MS-XLSB] 2.5.86), as `.xlsx` names them. +const ICON_SETS: [&str; 17] = [ + "3Arrows", + "3ArrowsGray", + "3Flags", + "3TrafficLights1", + "3TrafficLights2", + "3Signs", + "3Symbols", + "3Symbols2", + "4Arrows", + "4ArrowsGray", + "4RedToBlack", + "4Rating", + "4TrafficLights", + "5Arrows", + "5ArrowsGray", + "5Rating", + "5Quarters", +]; + +/// `BrtCustomFilter` ([MS-XLSB] 2.4.348) as `customFilter`; `None` for a Boolean value, whose +/// `.xlsx` spelling no Excel file has shown. A blank or non-blank test is `val=" "`, as Excel +/// writes it. +fn custom_filter(payload: &[u8]) -> Result, Error> { + let mut b = Bytes::new(payload); + let vts = b.u8()?; + let operator = match b.u8()? { + 1 => Some("lessThan"), + 2 => None, + 3 => Some("lessThanOrEqual"), + 4 => Some("greaterThan"), + 5 => Some("notEqual"), + 6 => Some("greaterThanOrEqual"), + _ => return Err(invalid("invalid custom filter operator")), + }; + let union = b.take(8)?; + let val = match vts { + 4 => { + let n = f64::from_le_bytes(union.try_into().map_err(|_| invalid("truncated number"))?); + if !n.is_finite() { + return Err(invalid("nonfinite numeric value")); + } + filter_number(n) + } + 6 => b.text(255)?, + 8 => return Ok(None), + 0x0C | 0x0E => " ".to_string(), + _ => return Err(invalid("invalid custom filter type")), + }; + b.finish()?; + let mut el = El::new("customFilter"); + if let Some(op) = operator { + el.set_attr("operator", op); + } + el.set_attr("val", val); + Ok(Some(el)) +} + +/// `BrtDynamicFilter` ([MS-XLSB] 2.4.362) as `dynamicFilter`: the computed values only once the +/// filter has been applied, and the maximum only for date periods. +fn dynamic_filter(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let cft = b.u32()?; + let applied = b.u8()? & 1 != 0; + let val = b.f64()?; + let max = b.f64()?; + b.finish()?; + const PERIODS: [&str; 16] = [ + "tomorrow", + "today", + "yesterday", + "nextWeek", + "thisWeek", + "lastWeek", + "nextMonth", + "thisMonth", + "lastMonth", + "nextQuarter", + "thisQuarter", + "lastQuarter", + "nextYear", + "thisYear", + "lastYear", + "yearToDate", + ]; + let kind = match cft { + 0 => "null".to_string(), + 1 => "aboveAverage".to_string(), + 2 => "belowAverage".to_string(), + 8..=0x17 => PERIODS[(cft - 8) as usize].to_string(), + 0x18..=0x1B => format!("Q{}", cft - 0x17), + 0x1C..=0x27 => format!("M{}", cft - 0x1B), + _ => return Err(invalid("invalid dynamic filter type")), + }; + let mut el = El::new("dynamicFilter").with_attr("type", kind); + if applied && (1..=0x17).contains(&cft) { + el.set_attr("val", filter_number(val)); + if cft >= 8 { + el.set_attr("maxVal", filter_number(max)); + } + } + Ok(el) +} + +/// `BrtAFilterDateGroupItem` ([MS-XLSB] 2.4.5) as `dateGroupItem`, with the parts of the date +/// its grouping compares. +fn date_group_item(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let year = b.u16()?; + let month = b.u16()?; + let day = b.u32()?; + let (hour, minute, second) = (b.u16()?, b.u16()?, b.u16()?); + b.take(6)?; + let grouping = b.u32()?; + b.finish()?; + const GROUPS: [&str; 6] = ["year", "month", "day", "hour", "minute", "second"]; + let name = GROUPS.get(grouping as usize).ok_or_else(|| invalid("invalid date group"))?; + let mut el = El::new("dateGroupItem").with_attr("year", year); + let parts = [("month", u32::from(month)), ("day", day), ("hour", u32::from(hour)), ("minute", u32::from(minute)), ("second", u32::from(second))]; + for (name, value) in parts.iter().take(grouping as usize) { + el.set_attr(name, value); + } + el.set_attr("dateTimeGrouping", name); + Ok(el) +} + +/// `BrtBeginSortState` ([MS-XLSB] 2.4.227) and its `BrtBeginSortCond`s, up to `BrtEndSortState`, +/// as `sortState`. `None`, with a warning, for a sort by another method than character order, +/// which `.xlsx` names (stroke or PinYin) and the record does not. +fn sort_state(payload: &[u8], records: &mut Records<'_, '_>, warnings: &mut Warnings) -> Result, Error> { + let mut b = Bytes::new(payload); + let flags = b.u16()?; + let range = rfx(&mut b)?; + b.finish()?; + let mut el = El::new("sortState"); + if flags & 1 != 0 { + el.set_attr("columnSort", 1); + } + if flags & 2 != 0 { + el.set_attr("caseSensitive", 1); + } + el.set_attr("ref", range.a1()); + let keep = flags & 4 == 0; + if !keep { + warnings.warn("XLSB: some table sort states were left out (sorts by stroke or PinYin order)"); + } + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + match id { + 532 => { + let flags = b.u16()?; + let range = rfx(&mut b)?; + let (first, second) = (b.u32()?, b.u32()?); + let list = b.string(32_767, true)?; + b.finish()?; + let mut cond = El::new("sortCondition"); + if flags & 1 != 0 { + cond.set_attr("descending", 1); + } + let on = (flags >> 1) & 0xF; + match on { + 0 => {} + 1 => cond.set_attr("sortBy", "cellColor"), + 2 => cond.set_attr("sortBy", "fontColor"), + 3 => cond.set_attr("sortBy", "icon"), + _ => return Err(invalid("invalid sort condition")), + } + cond.set_attr("ref", range.a1()); + match on { + 0 => { + if let Some(list) = list { + cond.set_attr("customList", list); + } + } + 1 | 2 => cond.set_attr("dxfId", first), + _ => { + let name = ICON_SETS.get(first as usize).ok_or_else(|| invalid("invalid sort icon set"))?; + cond.set_attr("iconSet", name); + cond.set_attr("iconId", second); + } + } + el.push(cond); + } + 533 => b.finish()?, + 531 => { + b.finish()?; + return Ok(keep.then_some(el)); + } + _ => return Err(invalid(format!("unexpected record {id} in a sort state"))), + } + } + Err(invalid("missing sort state end record")) +} + +#[cfg(test)] +mod tests { + use super::filter_number; + + #[test] + fn filter_numbers_are_spelled_as_excel_writes_them() { + // Excel's `.xlsx` text for each value. + let cases = [ + (20.0, "20"), + (23.25, "23.25"), + (46340.0, "46340"), + (-1.5, "-1.5"), + (1e15, "1000000000000000"), + (999_999_999_999_999.0, "999999999999999"), + (1.234_567_890_123_45e17, "1.2345678901234499E+17"), + (1e16, "1E+16"), + (-1e16, "-1E+16"), + (1e20, "1E+20"), + (2.5e300, "2.5000000000000001E+300"), + (1_234.567_890_123_45, "1234.5678901234501"), + (12_345_678_901_234.5, "12345678901234.5"), + (99_999_999_999_999.9, "99999999999999.906"), + (1.1, "1.1000000000000001"), + (0.57, "0.56999999999999995"), + (0.01, "0.01"), + (0.035, "3.5000000000000003E-2"), + (0.001, "1E-3"), + (1e-4, "1E-4"), + (1e-5, "1.0000000000000001E-5"), + (1.5e-10, "1.5E-10"), + (1e-19, "9.9999999999999998E-20"), + ]; + for (n, want) in cases { + assert_eq!(filter_number(n), want, "{n}"); + } + } +} diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 950dc8847..3a0228275 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -5,7 +5,7 @@ mod common; use common::{read, read_with}; use craft_xlsx::testing::zip; -use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb}; +use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Package, Range, SheetKind, Table, Value, Visibility, is_xlsb}; const ROOT: &str = r#""#; const RELS: &str = r#""#; @@ -584,7 +584,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&CellRef::new(0, 14), &1)]); // Every formula cell keeps its saved result as its value. assert!((1..=15).all(|c| sheet.value(CellRef::new(0, c)) == Value::Number(0.0))); - for why in ["defined names", "other workbooks", "missing from the file", "structured table references"] { + for why in ["defined names", "other workbooks", "missing from the file", "tables the reader could not read"] { assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); } } @@ -1035,3 +1035,265 @@ fn references_to_macro_sheets_and_spans_over_chart_sheets_are_named() { assert_eq!(got, [Some("Macro1!$A$1"), Some("Data:Hidden!$A$1"), None]); assert!(warned(&wb, "references to sheets with no cells"), "{:?}", wb.warnings); } + +// Tables ([MS-XLSB] 2.1.7.51): table parts assembled from the specification's records, and the +// `` Excel writes for the same table in `.xlsx`. + +fn rfx(r0: u32, r1: u32, c0: u32, c1: u32) -> Vec { + [r0, r1, c0, c1].iter().flat_map(|n| n.to_le_bytes()).collect() +} +/// `BrtBeginList`: a worksheet table with no formats or cell styles, its name as its display name. +fn begin_list(range: &[u8], id: u32, header: u32, totals: u32, flags: u32, name: &str) -> Vec { + begin_list_of(0, range, id, header, totals, flags, name) +} +/// `BrtBeginList` of a table type (`lt`): 0 a worksheet table, 2 an XML table. +fn begin_list_of(lt: u32, range: &[u8], id: u32, header: u32, totals: u32, flags: u32, name: &str) -> Vec { + let mut data = range.to_vec(); + for n in [lt, id, header, totals, flags] { + data.extend(n.to_le_bytes()); + } + data.extend(u32::MAX.to_le_bytes().repeat(6)); + data.extend(0u32.to_le_bytes()); + data.extend([wide(name), wide(name), wide("")].concat()); + data.extend(u32::MAX.to_le_bytes().repeat(3)); + record(343, &data) +} +/// `BrtBeginListCol` with its totals function (`ilta`), data format and totals label. +fn list_col(id: u32, ilta: u32, data_dxf: u32, name: &str, total: Option<&str>) -> Vec { + let mut data: Vec = [id, ilta, u32::MAX, data_dxf, u32::MAX, 0, u32::MAX].iter().flat_map(|n| n.to_le_bytes()).collect(); + data.extend(wide(name)); + data.extend(total.map_or_else(|| u32::MAX.to_le_bytes().to_vec(), wide)); + data.extend(u32::MAX.to_le_bytes().repeat(3)); + record(347, &data) +} +/// `BrtListCCFmla` (351) or `BrtListTrFmla` (352). +fn list_formula(id: u16, rgce: &[u8]) -> Vec { + record(id, &[&[0u8][..], &(rgce.len() as u32).to_le_bytes(), rgce, &0u32.to_le_bytes()].concat()) +} +/// `PtgList`: the flags hold the columns (bits 0-1), the rows (2-6) and the two spacing bits. +fn ptg_list(ixti: u16, flags: u16, list: u32, first: u16, last: u16) -> Vec { + [&[0x18u8, 0x19][..], &ixti.to_le_bytes(), &flags.to_le_bytes(), &list.to_le_bytes(), &first.to_le_bytes(), &last.to_le_bytes()].concat() +} +fn filter_column(col: u32, flags: u16, criteria: &[Vec]) -> Vec { + [record(163, &[&col.to_le_bytes()[..], &flags.to_le_bytes()].concat()), criteria.concat(), record(164, &[])].concat() +} +fn custom_filter(vts: u8, op: u8, union: [u8; 8], text: Option<&str>) -> Vec { + record(174, &[&[vts, op][..], &union, &text.map(wide).unwrap_or_default()].concat()) +} +fn table_style(flags: u16, name: &str) -> Vec { + record(513, &[&flags.to_le_bytes()[..], &wide(name)].concat()) +} +/// The rows of a `PtgList` (bits 2-6). +const fn rows(t: u16) -> u16 { + t << 2 +} + +/// The filters of Sales: a value list with blanks, two custom criteria (OR), a bottom-percent +/// filter applied with its button hidden, and "today" applied. +fn sales_filters() -> Vec> { + vec![ + filter_column(0, 0, &[record(165, &[1, 0, 0, 0, 0, 0, 0, 0]), record(167, &wide("k3")), record(167, &wide("k4")), record(166, &[])]), + filter_column( + 1, + 0, + &[ + record(172, &0u32.to_le_bytes()), + custom_filter(4, 1, 20f64.to_le_bytes(), None), + custom_filter(6, 2, [0; 8], Some("k*")), + record(173, &[]), + ], + ), + filter_column(2, 1, &[record(170, &[&[0b110u8][..], &10f64.to_le_bytes(), &23.25f64.to_le_bytes()].concat())]), + filter_column(3, 0, &[record(171, &[&9u32.to_le_bytes()[..], &[1], &46340f64.to_le_bytes(), &46341f64.to_le_bytes()].concat())]), + ] +} + +/// Sales (B2:E8, id 1) with a totals row, a calculated column, a column name needing escapes, +/// `filters` and a sort. +fn sales(filters: &[Vec]) -> Vec { + [ + begin_list(&rfx(1, 7, 1, 4), 1, 1, 1, 1, "Sales"), + record(161, &rfx(1, 6, 1, 4)), + filters.concat(), + record(162, &[]), + record(530, &[&0u16.to_le_bytes()[..], &rfx(2, 6, 1, 4)].concat()), + record(532, &[&1u16.to_le_bytes()[..], &rfx(1, 7, 1, 1), &[0; 8], &u32::MAX.to_le_bytes()].concat()), + record(533, &[]), + record(531, &[]), + record(345, &4u32.to_le_bytes()), + list_col(1, 0, u32::MAX, "Key", Some("Total")), + record(348, &[]), + list_col(2, 6, u32::MAX, "Val", None), + record(348, &[]), + list_col(3, 0, 2, "It's [#1]", None), + record(348, &[]), + list_col(4, 9, u32::MAX, "Calc", None), + list_formula(351, &[ptg_list(0, 1 | rows(0x10), 1, 1, 1), vec![0x1E, 2, 0, 0x05]].concat()), + list_formula(352, &[ptg_list(0, 1, 1, 3, 3), vec![0x42, 1, 7, 0]].concat()), + record(348, &[]), + record(346, &[]), + table_style(0b101, "TableStyleMedium9"), + record(344, &[]), + ] + .concat() +} + +/// NoHead (H10:I11, id 2): no header row, a totals row never shown, no stripes. +fn no_head() -> Vec { + [ + begin_list(&rfx(9, 10, 7, 8), 2, 0, 0, 0, "NoHead"), + record(345, &2u32.to_le_bytes()), + list_col(1, 0, u32::MAX, "Column1", None), + record(348, &[]), + list_col(2, 0, u32::MAX, "Column2", None), + record(348, &[]), + record(346, &[]), + table_style(0, "TableStyleMedium2"), + record(344, &[]), + ] + .concat() +} + +/// XmlT (K1:L2, id 3): an XML table, its columns mapped to an XML map's elements. +fn xml_table_part() -> Vec { + let col = |id: u32, name: &str, kind: u32, xpath: &str| { + let mut data: Vec = [id, 0, u32::MAX, u32::MAX, u32::MAX, 0].iter().flat_map(|n| n.to_le_bytes()).collect(); + data.extend([wide(name), wide(name)].concat()); + data.extend(u32::MAX.to_le_bytes().repeat(4)); + let map = [&1u32.to_le_bytes()[..], &(if id == 2 { 2u32 } else { 0 }).to_le_bytes(), &kind.to_le_bytes(), &wide(xpath)].concat(); + [record(347, &data), record(349, &map), record(350, &[]), record(348, &[])].concat() + }; + [ + begin_list_of(2, &rfx(0, 1, 10, 11), 3, 1, 0, 0, "XmlT"), + record(345, &2u32.to_le_bytes()), + col(1, "a", 0x0D, "/rows/r/a"), + col(2, "b", 0x01, "/rows/@b"), + record(346, &[]), + table_style(0b100, "TableStyleMedium2"), + record(344, &[]), + ] + .concat() +} + +const XML_TABLE_XML: &str = r#"
"#; +const SALES_XML: &str = r#"Sales[[#This Row],[Val]]*2MAX(Sales[Calc])
"#; +const NO_HEAD_XML: &str = r#"
"#; + +fn xml_table(xml: &str) -> Table { + let root = craft_xml::Options::canonical(&craft_xlsx::ns::CANONICAL).parse(xml.as_bytes()).unwrap(); + Table::read(&root).unwrap() +} + +/// `book_with_links(&xti())` whose sheet one lists `tables` (`BrtListPart`, relationship ids +/// t1, t2, ...) after `data`. +fn book_with_tables(data: &[u8], tables: &[Vec]) -> Vec { + let mut list = vec![record(660, &(tables.len() as u32).to_le_bytes())]; + let mut rels = String::from(r#""#); + let mut types = TYPES.trim_end_matches("").to_string(); + for i in 1..=tables.len() { + list.push(record(661, &wide(&format!("t{i}")))); + rels.push_str(&format!(r#""#)); + types.push_str(&format!(r#""#)); + } + list.push(record(662, &[])); + rels.push_str(""); + types.push_str(""); + let names: Vec = (1..=tables.len()).map(|i| format!("tables/t{i}.bin")).collect(); + let (book, one, two, strings) = (book_with_links(&xti()), worksheet(data, &list.concat()), worksheet(&[], &[]), sst()); + let mut entries: Vec<(&str, &[u8])> = vec![ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT.as_bytes()), + ("records/book.bin", &book), + ("records/_rels/book.bin.rels", RELS.as_bytes()), + ("data/one.bin", &one), + ("data/_rels/one.bin.rels", rels.as_bytes()), + ("data/two.bin", &two), + ("text/strings.bin", &strings), + ]; + entries.extend(names.iter().map(String::as_str).zip(tables.iter().map(Vec::as_slice))); + zip(&entries) +} + +/// Sheet one's tables through `Package`, and its warnings. +fn tables_of(bytes: &[u8]) -> (Vec, Vec) { + let mut p = Package::open(bytes, &Limits::default()).unwrap(); + let wb = p.workbook().unwrap(); + let part = p.sheet(&wb.sheets[0], |_| Ok(())).unwrap(); + let tables = p.tables(&part).unwrap().into_iter().map(|(_, t)| t).collect(); + (tables, p.warnings().to_vec()) +} + +#[test] +fn tables_are_read_as_their_xlsx_spelling() { + // Spacing flags: 0x80 inside the outer brackets, 0x100 after commas. + let cells: Vec> = vec![ + ptg_list(0, 1, 1, 1, 1), + ptg_list(0, 1 | rows(2), 1, 0, 0), + ptg_list(0, 2 | rows(4), 1, 1, 3), + ptg_list(0, rows(1), 1, 0, 0), + ptg_list(0, 0, 1, 0, 0), + ptg_list(0, 2 | rows(0x10), 1, 1, 3), + ptg_list(0, 1, 1, 2, 2), + ptg_list(0, 1 | rows(4) | 0x180, 1, 1, 1), + ptg_list(0, rows(0x10), 2, 0, 0), + ptg_list(0, 1 | rows(0xC) | 0x100, 1, 0, 0), + ptg_list(0, rows(6), 1, 0, 0), + // Kept as their saved results: a table that isn't there, a deleted table, a column past + // the table's last. + ptg_list(0, 0, 9, 0, 0), + ptg_list(0, 0x1000, 1, 0, 0), + ptg_list(0, 1, 1, 4, 4), + ]; + let mut data = row(0); + for (i, rgce) in cells.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, rgce, &[])); + } + let bytes = book_with_tables(&data, &[sales(&sales_filters()), no_head(), xml_table_part()]); + let (tables, _) = tables_of(&bytes); + assert_eq!(tables, [xml_table(SALES_XML), xml_table(NO_HEAD_XML), xml_table(XML_TABLE_XML)]); + let wb = read(&bytes).unwrap(); + let got: Vec<_> = (1..=14).map(|c| wb.sheets[0].formula(CellRef::new(0, c))).collect(); + let want = [ + Some("Sales[Val]"), + Some("Sales[[#Headers],[Key]]"), + Some("Sales[[#Data],[Val]:[Calc]]"), + Some("Sales[#All]"), + Some("Sales[]"), + Some("Sales[[#This Row],[Val]:[Calc]]"), + Some("Sales[It''s '['#1']]"), + Some("Sales[ [#Data], [Val] ]"), + Some("NoHead[#This Row]"), + Some("Sales[[#Data], [#Totals], [Key]]"), + Some("Sales[[#Headers],[#Data]]"), + None, + None, + None, + ]; + assert_eq!(got, want); + for why in ["tables the reader could not read", "deleted tables", "missing table column"] { + assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); + } +} + +#[test] +fn table_filters_and_parts_the_reader_cannot_spell_are_left_out() { + // A filter column whose criteria are in a future-record block (where Excel keeps date + // groups), and one on a Boolean value: both left out, the table kept. + let frt = [record(35, &[3, 0, 2, 0]), record(5094, &[0; 32]), record(36, &[])].concat(); + let mut filters = sales_filters(); + filters.push(filter_column(4, 0, &[frt])); + filters.push(filter_column(5, 0, &[record(172, &0u32.to_le_bytes()), custom_filter(8, 2, [1, 0, 0, 0, 0, 0, 0, 0], None), record(173, &[])])); + // A truncated table part: skipped, and formulas using it keep their saved results. + let mut broken = no_head(); + broken.truncate(20); + let data = [row(0), formula_cell(1, &ptg_list(0, 0, 2, 0, 0), &[]), formula_cell(2, &ptg_list(0, 1, 1, 1, 1), &[])].concat(); + let bytes = book_with_tables(&data, &[sales(&filters), broken]); + let (tables, warnings) = tables_of(&bytes); + assert_eq!(tables, [xml_table(SALES_XML)]); + for why in ["filters kept in extensions", "custom filters on a Boolean value", "skipped tables/t2.bin"] { + assert!(warnings.iter().any(|w| w.contains(why)), "{why}: {warnings:?}"); + } + let wb = read(&bytes).unwrap(); + assert_eq!([1, 2].map(|c| wb.sheets[0].formula(CellRef::new(0, c))), [None, Some("Sales[Val]")]); + assert!(warned(&wb, "tables the reader could not read"), "{:?}", wb.warnings); +} From b5eda42a33893ec17143532cef6fc1d97accb9d5 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 12:21:06 -0500 Subject: [PATCH 005/209] craft-xlsx: VBA function names and macro descriptions in binary workbooks An Excel probe with VBA enabled showed two things the names reader got wrong: - Excel writes a VBA function a cell calls as a BrtName with fFunc, fOB and fProc, no formula, and none of the four strings fProc names otherwise carry. The reader refused it, which dropped every name and left every formula as its saved result. Such a name now ends after its comment; it stays out of the names, as .xlsm lists none, and cells calling it spell it. - The two middle strings of a macro's name are its description and help topic, as .xlsm writes them (description="Function A" help="!0"), not the custom menu and description the specification's labels say. --- crates/xlsx/README.md | 39 ++++++++++++++++----------------- crates/xlsx/src/xlsb.rs | 23 +++++++++++--------- crates/xlsx/tests/xlsb.rs | 45 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index b873c87f6..e79fdbd55 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -83,25 +83,26 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1 The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. -A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not -read (each cell keeps its format index). Its defined names are those of `definedNames` in the -`.xlsx` Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their -`localSheetId`, relative references as the cell they reach from A1, macro names with their -`function`, `xlm`, `functionGroupId` and `shortcutKey`. Excel's hidden names for newer functions -and LET parameters are left out, as in `.xlsx`, and so is the formula-less workbook-level built-in -Excel writes beside a sheet's (`Database`), which formulas still spell `_xlnm.Database`; a name -whose formula can't be read is left out with a warning. A reference may name an XLM macro sheet -(`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends whatever chart -sheets lie between them. Its cell formulas are decoded from their tokens into the text an `.xlsx` -file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC 29500 functions -`.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the formula, numbers as -Excel spells them), with -`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell -metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the -file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: a defined name left out, table references, -other workbooks, Python code stored outside the cell, data tables. The XLSB function table -(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. +A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not read +(each cell keeps its format index). Its defined names are those of `definedNames` in the `.xlsx` +Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their `localSheetId`, +relative references as the cell they reach from A1, macro names with their `function`, `xlm` or +`vbProcedure`, `functionGroupId`, `shortcutKey`, `description` and `help`. Excel's hidden names for +newer functions and LET parameters are left out, as in `.xlsx`, and so is the formula-less +workbook-level built-in Excel writes beside a sheet's (`Database`), which formulas still spell +`_xlnm.Database`, and the formula-less name of a VBA function a cell calls (`.xlsm` lists none; the +cell spells it); a name whose formula can't be read is left out with a warning. A reference may name +an XLM macro sheet (`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends +whatever chart sheets lie between them. Its cell formulas are decoded from their tokens into the +text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC +29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the +formula, numbers as Excel spells them), with `FormulaKind::Shared` or `FormulaKind::Array` and their +ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own +text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs +something the reader does not read keeps only its saved result, with a warning saying why: a defined +name left out, table references, other workbooks, Python code stored outside the cell, data tables. +The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from +`craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index dd52a55b4..e6246f4be 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -380,8 +380,8 @@ struct PendingName { rgce: Vec, rgcb: Vec, comment: Option, - menu: Option, description: Option, + help: Option, } /// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the @@ -435,12 +435,12 @@ fn defined_names( let mut attrs = vec![]; let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); // In `CT_DefinedName`'s order. - if let Some(m) = p.menu { - put("customMenu", m); - } if let Some(d) = p.description { put("description", d); } + if let Some(h) = p.help { + put("help", h); + } let (func, vba, proc) = (p.flags & 0x02 != 0, p.flags & 0x04 != 0, p.flags & 0x08 != 0); if func { put("function", "1".into()); @@ -520,14 +520,17 @@ fn link_record(id: u16, payload: &[u8], links: &mut Links, pending: &mut VecData!$A$1"#; + assert_eq!(wb.defined_names, xml_names(xml)); +} From 0f944f62d5330c58bfe2cfdeac3d74881ec67748 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 12:44:03 -0500 Subject: [PATCH 006/209] craft-xlsx: review fixes: names using a left-out name are left out; quoted sheet-scoped calls - A name whose formula uses a name that is left out (another workbook, a skipped sheet, an unreadable formula) was still published, so a cell using it decoded and would calculate to #NAME?. Each name's formula now records the names it uses, and every name reached from a left-out one is left out too, whatever their order: once each, so a long chain costs no more than its uses. A cell using one keeps its saved result. - A sheet-scoped name called as a function through PtgNameX prints its sheet quoted where needed ('My Sheet'!Fn(1)), as a sheet-qualified name does. --- crates/xlsx/README.md | 24 +++++++++---------- crates/xlsx/src/xlsb.rs | 42 +++++++++++++++++++++++++++++++-- crates/xlsx/src/xlsb/expr.rs | 24 ++++++++++++++++++- crates/xlsx/src/xlsb/formula.rs | 10 +++++++- crates/xlsx/tests/xlsb.rs | 22 +++++++++++++++++ 5 files changed, 106 insertions(+), 16 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 75dd8602a..6e6291934 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -91,18 +91,18 @@ relative references as the cell they reach from A1, macro names with their `func newer functions and LET parameters are left out, as in `.xlsx`, and so is the formula-less workbook-level built-in Excel writes beside a sheet's (`Database`), which formulas still spell `_xlnm.Database`, and the formula-less name of a VBA function a cell calls (`.xlsm` lists none; the -cell spells it); a name whose formula can't be read is left out with a warning. A reference may name -an XLM macro sheet (`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends -whatever chart sheets lie between them. Its cell formulas are decoded from their tokens into the -text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC -29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the -formula, numbers as Excel spells them), with `FormulaKind::Shared` or `FormulaKind::Array` and their -ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own -text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs -something the reader does not read keeps only its saved result, with a warning saying why: a defined -name left out, table references, other workbooks, Python code stored outside the cell, data tables. -The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from -`craft-xls`'s. +cell spells it); a name whose formula can't be read is left out with a warning, and so is every name +whose formula uses a left-out name. A reference may name an XLM macro sheet (`Macro1!$A$1`, where +macro names point), and a 3-D span is spelled by its ends whatever chart sheets lie between them. +Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and +`_xlpm.` names as written, apart from the four ISO/IEC 29500 functions `.xlsx` names bare, no +implicit `@`, the spaces and line breaks typed in the formula, numbers as Excel spells them), with +`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell +metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the +file keeps the tokens once. A formula whose decoding needs something the reader does not read keeps +only its saved result, with a warning saying why: a defined name left out, table references, other +workbooks, Python code stored outside the cell, data tables. The XLSB function table +(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 1eb7cddc1..22976d826 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -397,7 +397,10 @@ fn defined_names( budget: &Budget, warnings: &mut Warnings, ) -> Vec { - let mut out = vec![]; + // Each name's formula, and the names it uses. A name whose formula uses a name that is left + // out is left out too, whatever their order, so a cell using it keeps its saved result. + let mut omitted = vec![false; links.names.len()]; + let mut kept = vec![]; for p in pending { let Some(name) = links.names.get(p.index) else { continue }; if name.future || formula::is_param(&name.text) { @@ -409,6 +412,7 @@ fn defined_names( Some(Some(i)) => Some(*i), _ => { warnings.warn("XLSB: some defined names were left out (names of sheets the reader skips)"); + omit(&mut omitted, p.index); continue; } }, @@ -421,17 +425,46 @@ fn defined_names( } continue; } - let formula = match formula::decode(&p.rgce, &p.rgcb, links, Mode::Name) { + let mut used = vec![]; + let formula = match formula::decode_using(&p.rgce, &p.rgcb, links, Mode::Name, &mut used) { Ok(e) => expr::print(&e), Err(formula::Unsupported(why)) => { warnings.warn(format!("XLSB: some defined names were left out ({why})")); + omit(&mut omitted, p.index); continue; } }; if !budget.try_formula_text(formula.len(), limits) { warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); + omit(&mut omitted, p.index); continue; } + kept.push((p, local_sheet_id, formula, used)); + } + // Who uses each name, then every name reached from a left-out one: once each, so a chain + // of any length costs what its uses do. + let mut users: HashMap> = HashMap::new(); + for (k, (.., used)) in kept.iter().enumerate() { + for &u in used { + users.entry(u).or_default().push(k); + } + } + let mut queue: Vec = (0..omitted.len()).filter(|&i| omitted[i]).collect(); + while let Some(i) = queue.pop() { + for &k in users.get(&i).map_or(&[][..], Vec::as_slice) { + let Some((p, ..)) = kept.get(k) else { continue }; + if omit(&mut omitted, p.index) { + warnings.warn("XLSB: some defined names were left out (names using a name that was left out)"); + queue.push(p.index); + } + } + } + let mut out = vec![]; + for (p, local_sheet_id, formula, _) in kept { + if omitted.get(p.index).copied().unwrap_or(true) { + continue; + } + let Some(name) = links.names.get(p.index) else { continue }; let mut attrs = vec![]; let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); // In `CT_DefinedName`'s order. @@ -478,6 +511,11 @@ fn defined_names( out } +/// Marks name `i` left out; whether it wasn't already. +fn omit(omitted: &mut [bool], i: usize) -> bool { + omitted.get_mut(i).is_some_and(|o| !std::mem::replace(o, true)) +} + /// A supporting link (`BrtSupSelf`, `BrtSupSame`, `BrtSupAddin`, `BrtSupBookSrc`), the sheet links /// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`; a name's other parts go to /// `pending`. diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index eee43a557..4c5efe4f7 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -283,6 +283,20 @@ fn write_args(s: &mut String, args: &[Expr], sep: char) { } } +/// A function's name; a sheet-scoped one (`My Sheet!Fn`) is quoted as a sheet-qualified name is. +/// Kept out of [`write_expr`], whose frame every nesting level of a formula repeats. +#[inline(never)] +fn push_function_name(s: &mut String, name: &str) { + match sheet_split(name) { + Some((sheet, name)) => { + s.push_str("e_sheet(sheet)); + s.push('!'); + s.push_str(name); + } + None => s.push_str(name), + } +} + fn write_expr(s: &mut String, e: &Expr) { match e { Expr::Number(n) => s.push_str(&number(*n)), @@ -331,7 +345,7 @@ fn write_expr(s: &mut String, e: &Expr) { write_expr(s, b); } Expr::Call(name, args) => { - s.push_str(name); + push_function_name(s, name); s.push('('); write_args(s, args, ','); s.push(')'); @@ -377,4 +391,12 @@ mod tests { assert_eq!(number_to_text(1e100), "1E+100"); assert!(number(f64::MAX).parse::().is_ok_and(f64::is_finite)); } + + #[test] + fn a_sheet_scoped_function_name_is_quoted() { + let call = |name: &str| print(&Expr::Call(name.into(), vec![Expr::Number(1.0)])); + assert_eq!(call("My Sheet!Fn"), "'My Sheet'!Fn(1)"); + assert_eq!(call("Data!Fn"), "Data!Fn(1)"); + assert_eq!(call("SUM"), "SUM(1)"); + } } diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 2c5712d33..46f7b5acd 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -338,6 +338,11 @@ fn reference(sheet: Option, kind: RefKind) -> Expr { /// represented. A shared formula's tokens are decoded once for each of its cells /// (`Mode::Shared` with that cell), whose relative references are offsets from it. pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Result { + decode_using(rgce, rgcb, links, mode, &mut Vec::new()) +} + +/// [`decode`], adding to `used` the 0-based index of every defined name the formula uses. +pub(super) fn decode_using(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode, used: &mut Vec) -> Result { if rgce.len() > MAX_RGCE { return Err(Unsupported("formula too long")); } @@ -512,7 +517,9 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res } 0x23 => { // PtgName: a 1-based index into every BrtName, hidden ones included. - let name = links.name(t.u32()?, mode)?; + let index = t.u32()?; + let name = links.name(index, mode)?; + used.extend(usize::try_from(index).ok().and_then(|i| i.checked_sub(1))); push(&mut stack, kids, Expr::Name(name.text.clone()))?; } 0x24 => { @@ -553,6 +560,7 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res let index = t.u32()?; links.this_book(ixti)?; let name = links.name(index, mode)?; + used.extend(usize::try_from(index).ok().and_then(|i| i.checked_sub(1))); let sheet = name.scope.and_then(|s| links.sheets.get(usize::try_from(s).ok()?).cloned().flatten()); let sheet = sheet.ok_or(Unsupported("names in other workbooks or on other sheets"))?; push(&mut stack, kids, Expr::Name(format!("{sheet}!{}", name.text)))?; diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 6502c1e5d..4d8f12d75 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1280,3 +1280,25 @@ fn a_spent_formula_allowance_skips_decoding() { assert!(warned(&wb, "more formula text than the limits allow"), "{:?}", wb.warnings); assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings); } + +#[test] +fn a_name_using_a_left_out_name_is_left_out_too() { + // 14 Early = Later (a later name), 15 Alias = External (left out: another workbook), + // 16 Later = Alias, 17 Fine = Rate. Early, Alias and Later are left out whatever their order, + // and cells using them keep their saved results; Fine is read. + let mut list = names(); + for (text, index) in [("Early", 16u32), ("Alias", 3), ("Later", 15), ("Fine", 9)] { + list.push(brt_name(0, 0, WORKBOOK, text, &[&[0x23u8][..], &index.to_le_bytes()].concat(), None)); + } + let cell = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + let data = + [row(0), formula_cell(1, &cell(14), &[]), formula_cell(2, &cell(15), &[]), formula_cell(3, &cell(16), &[]), formula_cell(4, &cell(17), &[])] + .concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = NAMES_XML.replace("", r#"Rate"#); + assert_eq!(wb.defined_names, xml_names(&xml)); + let book = read(&bytes).unwrap(); + assert_eq!((1..=4).map(|c| book.sheets[0].formula(CellRef::new(0, c))).collect::>(), [None, None, None, Some("Fine")]); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} From 3bd1794251a5cfdc72c99e2fa9d63c28ab827ba7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 13:24:04 +0100 Subject: [PATCH 007/209] DWG: keep field formats when saving an older drawing as DWG 2007 or later acadrust reads a pre-2007 DXF FIELD's format only into Field::format, but its DWG writer writes that for versions before 2007 only and value.format from 2007 on. Copy it over before writing. Co-Authored-By: Claude Sonnet 5.5 --- crates/dwg/src/lib.rs | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/crates/dwg/src/lib.rs b/crates/dwg/src/lib.rs index 05c1a673c..216ed49a0 100644 --- a/crates/dwg/src/lib.rs +++ b/crates/dwg/src/lib.rs @@ -703,6 +703,23 @@ mod native { } } + /// A FIELD's format string between the two places acadrust 0.6.6 keeps it. Its DXF reader + /// puts a pre-2007 field's format (group 4) only in `Field::format`, and its DWG writer + /// writes that one only for versions before 2007; from 2007 it writes `value.format`. A field + /// read from an older drawing and written as DWG 2007 or later would lose its format + /// (`dd/MM/yyyy`, `%lu2`). The child values hold no format of their own in the old DXF. + fn field_formats(doc: &mut acadrust::CadDocument) { + if doc.version < acadrust::DxfVersion::AC1021 { + return; + } + for o in doc.objects.values_mut() { + let acadrust::objects::ObjectType::Field(f) = o else { continue }; + if f.value.format.is_empty() && !f.format.is_empty() { + f.value.format = f.format.clone(); + } + } + } + /// The DWG dimension flags byte. Its bit 1 is DXF group 70's bit 32 (ODA DWG specification, /// common dimension data), which acadrust keeps no copy of from DXF: it is set here as /// CADCraft's DXF writer always sets it (#1094). Bit 0, the opposite of group 70's bit 128, @@ -1238,6 +1255,7 @@ mod native { }; other_layout_owners(&mut doc); table_style_rows(&mut doc, true); + field_formats(&mut doc); dimension_flags(&mut doc); let mut unwritten = drop_unwritable(&mut doc); for (kind, n) in proxies { @@ -2103,4 +2121,43 @@ mod tests { assert_eq!(frames, [vec![0xDE, 0xAD, 0xBE, 0xEF]], "{version}"); } } + + /// A FIELD read from a pre-2007 DXF keeps its format (group 4) only in `Field::format`, which + /// acadrust's DWG writer writes just for versions before 2007: saved as DWG 2007 or later the + /// format (`dd/MM/yyyy`) was lost. A 2007+ field with its own format is written as it is. + #[cfg(not(no_dwg_bridge))] + #[test] + fn field_formats_survive_saving_an_older_drawing_as_dwg_2007_or_later() { + let formats = |dwg: Vec| -> Vec { + let back = acadrust::DwgReader::from_stream(std::io::Cursor::new(dwg)).read().unwrap(); + back.objects + .values() + .filter_map(|o| match o { + acadrust::objects::ObjectType::Field(f) => Some(f.value.format.clone()), + _ => None, + }) + .collect() + }; + for source in ["AC1015", "AC1018"] { + let dxf = format!( + " 0\nSECTION\n 2\nHEADER\n 9\n$ACADVER\n 1\n{source}\n 0\nENDSEC\n 0\nSECTION\n 2\nCLASSES\n 0\nCLASS\n 1\nFIELD\n 2\nAcDbField\n 3\nObjectDBX Classes\n 90\n0\n 91\n0\n 0\nENDSEC\n 0\nSECTION\n 2\nOBJECTS\n 0\nFIELD\n 5\nA1F\n330\n1F\n 1\nAcVar\n 2\n\\AcVar Date\n 4\ndd/MM/yyyy\n 90\n0\n 97\n0\n 91\n0\n 92\n0\n 94\n0\n 95\n0\n 96\n0\n300\n\n 0\nENDSEC\n 0\nEOF\n" + ); + assert_eq!(formats(dxf_to_dwg_as(dxf.as_bytes(), "AC1032").unwrap()), ["dd/MM/yyyy"], "{source}"); + } + let mut doc = acadrust::CadDocument::new(); + doc.classes.add_or_update(acadrust::classes::DxfClass::new("FIELD", "AcDbField")); + let handle = doc.allocate_handle(); + let field = acadrust::objects::Field { + handle, + owner: doc.header.named_objects_dict_handle, + evaluator_id: "AcVar".into(), + code: "\\AcVar Date".into(), + value: acadrust::entities::CellValue { format: "%lu2".into(), ..Default::default() }, + ..Default::default() + }; + doc.objects.insert(handle, acadrust::objects::ObjectType::Field(field)); + let mut dxf = Vec::new(); + acadrust::DxfWriter::new(&doc).write_to_writer(&mut dxf).unwrap(); + assert_eq!(formats(dxf_to_dwg_as(&dxf, "AC1032").unwrap()), ["%lu2"]); + } } From 0dba1ed64ebe764ffea4a97d801d65a9e708431e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 15:03:42 +0100 Subject: [PATCH 008/209] DWG: keep a field's format when the DXF is read as a newer kept version dxf_to_dwg_kept_as declares the kept version before reading, and acadrust 0.6.6 reads a FIELD's group 4 format only below AC1021, so an older drawing's own field lost its format. Collect the group 4 formats by handle before reading and put them back into an empty Field::format; field_formats then copies them into value.format for 2007+. Carried-over fields keep their own value.format. Co-Authored-By: Claude Opus 5.5 --- crates/dwg/src/lib.rs | 150 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 10 deletions(-) diff --git a/crates/dwg/src/lib.rs b/crates/dwg/src/lib.rs index 216ed49a0..2d173b325 100644 --- a/crates/dwg/src/lib.rs +++ b/crates/dwg/src/lib.rs @@ -708,7 +708,19 @@ mod native { /// writes that one only for versions before 2007; from 2007 it writes `value.format`. A field /// read from an older drawing and written as DWG 2007 or later would lose its format /// (`dd/MM/yyyy`, `%lu2`). The child values hold no format of their own in the old DXF. - fn field_formats(doc: &mut acadrust::CadDocument) { + /// + /// `legacy`: the group 4 formats of a DXF that was read as a newer version than it declares + /// ([`legacy_field_formats`]), which the reader then skipped. They go back into an empty + /// `Field::format` first, as the reader puts them there under the DXF's own version, so a + /// DWG before 2007 keeps them too. + fn field_formats(doc: &mut acadrust::CadDocument, legacy: &[(acadrust::Handle, String)]) { + for (handle, format) in legacy { + if let Some(acadrust::objects::ObjectType::Field(f)) = doc.objects.get_mut(handle) + && f.format.is_empty() + { + f.format.clone_from(format); + } + } if doc.version < acadrust::DxfVersion::AC1021 { return; } @@ -920,8 +932,9 @@ mod native { /// read from a file of version `kept` ("AC1032"), and acadrust reads several of those /// differently by version: read as R2000, an AC1032 file's mental ray render settings came out /// a class version too high, for which the ODA rejects the whole DWG, and a field lost its - /// cached value. So the DXF is read as `kept` declares. A `kept` version that is unknown, or - /// not newer than the DXF's own, changes nothing. + /// cached value. So the DXF is read as `kept` declares; the formats of the DXF's own pre-2007 + /// fields, which the reader skips then, are restored ([`legacy_field_formats`]). A `kept` + /// version that is unknown, or not newer than the DXF's own, changes nothing. pub fn dxf_to_dwg_kept_as(dxf: &[u8], version: &str, kept: &str) -> Result, String> { dxf_to_dwg_kept_noted(dxf, version, kept).map(|(dwg, _)| dwg) } @@ -987,6 +1000,35 @@ mod native { Some(out) } + /// The format (group 4) of each FIELD record in an ASCII DXF, by handle. acadrust 0.6.6 reads + /// it only from a DXF that declares a version before 2007 (AC1021), where it follows the + /// field code or the object handles, before group 91 (`read_field_object_dxf`). When + /// [`declare_version`] made an older DXF read as 2007 or later, a field the DXF wrote itself + /// would lose its format; a field carried over from a 2007+ file has none and keeps its own + /// `value.format`. + fn legacy_field_formats(dxf: &[u8]) -> Vec<(acadrust::Handle, String)> { + let Some(pairs) = super::proxy_transport::text_pairs(dxf) else { return Vec::new() }; + let mut formats = Vec::new(); + let (mut in_field, mut handle, mut format, mut past) = (false, None, None, false); + for pair in pairs { + let Some((code, value)) = pair else { break }; + match code { + 0 => { + if let (Some(h), Some(f)) = (handle.take(), format.take()) { + formats.push((h, f)); + } + in_field = value == "FIELD" || value == "ACDBFIELD"; + (handle, format, past) = (None, None, false); + } + 5 if in_field && handle.is_none() => handle = u64::from_str_radix(value.trim(), 16).ok().map(acadrust::Handle::new), + 4 if in_field && !past && format.is_none() => format = Some(value), + 91 if in_field => past = true, + _ => {} + } + } + formats + } + /// The byte offset and the trimmed value of header variable `name` in an ASCII DXF: the value /// of the group that follows the `9` group naming it. The file is read as (group code, /// value) line pairs, in one pass with no buffering, up to the first `0`/`ENDSEC`, the end @@ -1215,19 +1257,23 @@ mod native { )); } // Decoding a code page to UTF-8 can grow the text, so the limit applies again. - let data = match kept.and_then(|k| declare_version(dxf, k)) { - Some(d) if d.len() > Limits::DEFAULT.max_dxf_bytes => { + let (data, legacy) = match kept.and_then(|k| declare_version(dxf, k).map(|d| (k, d))) { + Some((_, d)) if d.len() > Limits::DEFAULT.max_dxf_bytes => { return Err(format!( "DXF: the input is {} as UTF-8, larger than the {} accepted for DWG conversion", mb(d.len() as u64), mb(Limits::DEFAULT.max_dxf_bytes as u64) )); } - Some(d) => d, - None => dxf.to_vec(), + // Read as 2007 or later, the DXF's own fields lose their formats: restored after. + Some((k, d)) => (d, k >= acadrust::DxfVersion::AC1021), + None => (dxf.to_vec(), false), }; - let (classes, source) = guarded(AssertUnwindSafe(|| (dxf_class_names(&data), super::proxy_transport::source_classes(&data)))) - .map_err(|_| "the DXF proxy class conversion failed".to_string())?; + let (classes, source, legacy) = guarded(AssertUnwindSafe(|| { + let legacy = if legacy { legacy_field_formats(&data) } else { Vec::new() }; + (dxf_class_names(&data), super::proxy_transport::source_classes(&data), legacy) + })) + .map_err(|_| "the DXF proxy class conversion failed".to_string())?; let mut doc = std::panic::catch_unwind(move || acadrust::DxfReader::from_reader(Cursor::new(data)).and_then(|r| r.read())) .map_err(|_| "the DXF→DWG conversion failed".to_string())? .map_err(|e| format!("DXF: {e}"))?; @@ -1255,7 +1301,7 @@ mod native { }; other_layout_owners(&mut doc); table_style_rows(&mut doc, true); - field_formats(&mut doc); + field_formats(&mut doc, &legacy); dimension_flags(&mut doc); let mut unwritten = drop_unwritable(&mut doc); for (kind, n) in proxies { @@ -2160,4 +2206,88 @@ mod tests { acadrust::DxfWriter::new(&doc).write_to_writer(&mut dxf).unwrap(); assert_eq!(formats(dxf_to_dwg_as(&dxf, "AC1032").unwrap()), ["%lu2"]); } + + /// A FIELD record as an AC1015/AC1018 DXF writes it, its format `dd/MM/yyyy` in group 4. + #[cfg(not(no_dwg_bridge))] + fn legacy_field(handle: acadrust::Handle, owner: acadrust::Handle) -> String { + format!( + " 0\r\nFIELD\r\n 5\r\n{handle:X}\r\n330\r\n{owner:X}\r\n 1\r\nAcVar\r\n 2\r\n\\AcVar Date\r\n 4\r\ndd/MM/yyyy\r\n 90\r\n0\r\n 97\r\n0\r\n 91\r\n0\r\n 92\r\n0\r\n 94\r\n0\r\n 95\r\n0\r\n 96\r\n0\r\n300\r\n\r\n" + ) + } + + /// The FIELD formats of a DWG by handle: `value.format` from 2007, `Field::format` before. + #[cfg(not(no_dwg_bridge))] + fn dwg_field_formats(dwg: Vec) -> Vec<(u64, String)> { + let back = acadrust::DwgReader::from_stream(std::io::Cursor::new(dwg)).read().unwrap(); + let mut formats: Vec<(u64, String)> = back + .objects + .values() + .filter_map(|o| match o { + acadrust::objects::ObjectType::Field(f) => Some((f.handle.into(), f.pre2007_format().to_string())), + _ => None, + }) + .collect(); + formats.sort(); + formats + } + + /// CADCraft's R2000 DXF of a drawing with fields declares AC1015 (or the drawing's AC1018) + /// and is read as the `kept` version, for which acadrust skips group 4: an older drawing's own + /// field lost its format (`dd/MM/yyyy`) in the DWG (#1580 review). + #[cfg(not(no_dwg_bridge))] + #[test] + fn field_formats_survive_a_kept_version_newer_than_the_drawing() { + for source in ["AC1015", "AC1018"] { + for kept in ["AC1024", "AC1032"] { + let owner = acadrust::Handle::new(0x1F); + let legacy = legacy_field(acadrust::Handle::new(0xA1F), owner); + let dxf = format!( + " 0\r\nSECTION\r\n 2\r\nHEADER\r\n 9\r\n$ACADVER\r\n 1\r\n{source}\r\n 0\r\nENDSEC\r\n 0\r\nSECTION\r\n 2\r\nCLASSES\r\n 0\r\nCLASS\r\n 1\r\nFIELD\r\n 2\r\nAcDbField\r\n 3\r\nObjectDBX Classes\r\n 90\r\n0\r\n 91\r\n0\r\n 0\r\nENDSEC\r\n 0\r\nSECTION\r\n 2\r\nOBJECTS\r\n{legacy} 0\r\nENDSEC\r\n 0\r\nEOF\r\n" + ); + for version in ["AC1032", "AC1018"] { + let got = dwg_field_formats(dxf_to_dwg_kept_as(dxf.as_bytes(), version, kept).unwrap()); + assert_eq!(got, [(0xA1F, "dd/MM/yyyy".to_string())], "{source} kept {kept} as {version}"); + } + } + } + } + + /// The same DXF with a field carried over from a 2007+ file next to the drawing's own: the + /// carried-over one keeps its own `value.format` (`%lu2`), the drawing's own its group 4 + /// format (`dd/MM/yyyy`). + #[cfg(not(no_dwg_bridge))] + #[test] + fn field_formats_survive_a_kept_version_with_carried_over_and_own_fields() { + for source in ["AC1015", "AC1018"] { + for kept in ["AC1024", "AC1032"] { + // The carried-over field as acadrust writes 2007+ DXF, the drawing's own field in + // the old layout, under the older version. + let mut doc = acadrust::CadDocument::new(); + doc.classes.add_or_update(acadrust::classes::DxfClass::new("FIELD", "AcDbField")); + let (carried, own) = (doc.allocate_handle(), doc.allocate_handle()); + let field = acadrust::objects::Field { + handle: carried, + owner: doc.header.named_objects_dict_handle, + evaluator_id: "AcVar".into(), + code: "\\AcVar Date".into(), + value: acadrust::entities::CellValue { format: "%lu2".into(), ..Default::default() }, + ..Default::default() + }; + doc.objects.insert(carried, acadrust::objects::ObjectType::Field(field)); + let mut dxf = Vec::new(); + acadrust::DxfWriter::new(&doc).write_to_writer(&mut dxf).unwrap(); + let dxf = String::from_utf8(dxf).unwrap(); + let dxf = dxf.replacen("$ACADVER\r\n 1\r\nAC1032\r\n", &format!("$ACADVER\r\n 1\r\n{source}\r\n"), 1); + let at = dxf.find(" 0\r\nFIELD\r\n").unwrap(); + let dxf = format!("{}{}{}", &dxf[..at], legacy_field(own, doc.header.named_objects_dict_handle), &dxf[at..]); + assert!(dxf.contains(&format!("$ACADVER\r\n 1\r\n{source}\r\n"))); + for version in ["AC1032", "AC1018"] { + let got = dwg_field_formats(dxf_to_dwg_kept_as(dxf.as_bytes(), version, kept).unwrap()); + let mut want = vec![(carried.into(), "%lu2".to_string()), (own.into(), "dd/MM/yyyy".to_string())]; + want.sort(); + assert_eq!(got, want, "mixed {source} kept {kept} as {version}"); + } + } + } + } } From 6fde933469b8e86fc9d6d0f034fc7b680bb819d0 Mon Sep 17 00:00:00 2001 From: Diego Guerini Date: Sun, 15 Nov 2026 18:01:57 +0100 Subject: [PATCH 009/209] craft-docx: read a DrawingML locked canvas as a group of its members Word 2007 writes shapes, pictures and WordArt in a drawing as an lc:lockedCanvas (ECMA-376 Part 1 20.3.2.1) in compatibility mode. Only its first picture was read; its shapes and their text were dropped. The canvas is now rewritten as the wpg group the reader already handles, its DrawingML text as text-box paragraphs with their run formatting. Moved from storytold/wordcraft#5067 (wordcraft#5060), whose DOCX reader now lives here. --- crates/docx/src/read/story.rs | 1 + crates/docx/src/read/story/drawing.rs | 4 + crates/docx/src/read/story/locked_canvas.rs | 254 ++++++++++++++++++++ crates/docx/src/xml.rs | 4 +- crates/docx/tests/it/locked_canvas.rs | 143 +++++++++++ crates/docx/tests/it/main.rs | 1 + 6 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 crates/docx/src/read/story/locked_canvas.rs create mode 100644 crates/docx/tests/it/locked_canvas.rs diff --git a/crates/docx/src/read/story.rs b/crates/docx/src/read/story.rs index 770375365..86ceb1651 100644 --- a/crates/docx/src/read/story.rs +++ b/crates/docx/src/read/story.rs @@ -22,6 +22,7 @@ use craft_wordmodel::wordart::TextWarp; mod blocks; mod drawing; mod fields; +mod locked_canvas; mod runs; mod vml; diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..0b7f7e78a 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -82,6 +82,10 @@ impl Reader<'_> { self.read_canvas(sc, c, rels, w, h, float) } else if let Some(g) = gd.child("wpg:wgp") { self.read_group(sc, g, rels, w, h, float)? + } else if let Some(lc) = gd.child("lc:lockedCanvas") { + // Word 2007's shapes, pictures and text in a drawing: read as the group they are. + let g = super::locked_canvas::as_group(lc, &self.doc.settings.theme_colors); + self.read_group(sc, &g, rels, w, h, float)? } else if direct_shape.is_none() && gd.find("a:blip").is_some() { self.read_pic(gd, rels, w, h, alt.clone(), float)? } else { diff --git a/crates/docx/src/read/story/locked_canvas.rs b/crates/docx/src/read/story/locked_canvas.rs new file mode 100644 index 000000000..01e8f2872 --- /dev/null +++ b/crates/docx/src/read/story/locked_canvas.rs @@ -0,0 +1,254 @@ +//! A DrawingML locked canvas (`lc:lockedCanvas`, ECMA-376 Part 1 §20.3.2.1): the shapes, +//! pictures and text Word 2007 writes in a drawing in compatibility mode (wordcraft#5060). It is a group +//! shape in the DrawingML namespace (`a:sp`, `a:pic`, `a:grpSp`, `a:cxnSp`, text in `a:txSp`), so +//! it is read as the `wpg:wgp` group the reader already knows: each member renamed into its +//! WordprocessingML drawing form, and its DrawingML text into `w:txbxContent` paragraphs. + +use super::{El, Node, Rgb}; + +/// Groups nested deeper than this are left out, as `group_members` does. +const MAX_NEST: usize = 8; +/// DrawingML's text size when neither the run nor the list style gives one (18 pt, in +/// hundredths of a point). +const DEFAULT_SZ: i64 = 1800; + +/// The locked canvas `lc` as a `wpg:wgp` group. `theme` resolves its text colours. +pub(super) fn as_group(lc: &El, theme: &[Rgb]) -> El { + el("wpg:wgp", Vec::new(), members(lc, theme, 0)) +} + +/// The members of DrawingML group `g`, in their WordprocessingML drawing form. +fn members(g: &El, theme: &[Rgb], depth: usize) -> Vec { + g.els() + .filter_map(|e| match e.name.as_str() { + "a:grpSpPr" => Some(renamed(e, "wpg:grpSpPr")), + "a:grpSp" if depth < MAX_NEST => Some(el("wpg:grpSp", Vec::new(), members(e, theme, depth + 1))), + "a:sp" => Some(shape(e, theme)), + "a:cxnSp" => Some(connector(e)), + "a:pic" => Some(picture(e)), + "a:txSp" => Some(text_shape(e, theme)), + _ => None, + }) + .map(Node::El) + .collect() +} + +/// `a:sp` as `wps:wsp`: its name, properties and style, and its text as a text box's. +fn shape(sp: &El, theme: &[Rgb]) -> El { + let nv = sp.child("a:nvSpPr"); + let mut kids: Vec = [ + nv.and_then(|n| n.child("a:cNvPr")).map(|e| renamed(e, "wps:cNvPr")), + nv.and_then(|n| n.child("a:cNvSpPr")).map(|e| renamed(e, "wps:cNvSpPr")), + sp.child("a:spPr").map(|e| renamed(e, "wps:spPr")), + sp.child("a:style").map(|e| renamed(e, "wps:style")), + ] + .into_iter() + .flatten() + .map(Node::El) + .collect(); + // The text sits in the shape's own rectangle (`a:useSpRect`); a text rectangle of its own + // (`a:txSp/a:xfrm`) is not kept apart. + if let Some(body) = sp.child("a:txSp").and_then(|t| t.child("a:txBody")) { + kids.extend(text_box(body, theme)); + } + el("wps:wsp", Vec::new(), kids) +} + +/// A group's text shape (`a:txSp` as a member, text with no shape of its own) as a `wps:wsp` text +/// box without fill or outline at its own rectangle (`a:xfrm`). +fn text_shape(tx: &El, theme: &[Rgb]) -> El { + let none = || Node::El(el("a:noFill", Vec::new(), Vec::new())); + let geometry = el("a:prstGeom", attrs(&[("prst", "rect")]), vec![Node::El(el("a:avLst", Vec::new(), Vec::new()))]); + let mut sp_pr: Vec = tx.child("a:xfrm").map(|x| Node::El(x.clone())).into_iter().collect(); + sp_pr.extend([Node::El(geometry), none(), Node::El(el("a:ln", Vec::new(), vec![none()]))]); + let mut kids = vec![Node::El(el("wps:cNvSpPr", attrs(&[("txBox", "1")]), Vec::new())), Node::El(el("wps:spPr", Vec::new(), sp_pr))]; + if let Some(body) = tx.child("a:txBody") { + kids.extend(text_box(body, theme)); + } + el("wps:wsp", Vec::new(), kids) +} + +/// DrawingML text body `body` as a text box's content and body properties. +fn text_box(body: &El, theme: &[Rgb]) -> Vec { + let content = el("w:txbxContent", Vec::new(), paragraphs(body, theme)); + let mut kids = vec![Node::El(el("wps:txbx", Vec::new(), vec![Node::El(content)]))]; + if let Some(bp) = body.child("a:bodyPr") { + kids.push(Node::El(renamed(bp, "wps:bodyPr"))); + } + kids +} + +/// `a:cxnSp` as a `wps:wsp` line. Its glued ends name shapes by the canvas's own ids, which +/// aren't the document's, so they are left out. +fn connector(cx: &El) -> El { + let kids = [ + cx.child("a:nvCxnSpPr").and_then(|n| n.child("a:cNvPr")).map(|e| renamed(e, "wps:cNvPr")), + cx.child("a:spPr").map(|e| renamed(e, "wps:spPr")), + cx.child("a:style").map(|e| renamed(e, "wps:style")), + ]; + el("wps:wsp", Vec::new(), kids.into_iter().flatten().map(Node::El).collect()) +} + +/// `a:pic` as `pic:pic`. +fn picture(pic: &El) -> El { + let nv = pic.child("a:nvPicPr").and_then(|n| n.child("a:cNvPr")).map(|e| el("pic:nvPicPr", Vec::new(), vec![Node::El(renamed(e, "pic:cNvPr"))])); + let kids = [nv, pic.child("a:blipFill").map(|e| renamed(e, "pic:blipFill")), pic.child("a:spPr").map(|e| renamed(e, "pic:spPr"))]; + el("pic:pic", Vec::new(), kids.into_iter().flatten().map(Node::El).collect()) +} + +/// The paragraphs of DrawingML text body `body` as `w:p`s. DrawingML text has no space before +/// or after a paragraph and single lines, so each says so rather than taking the document's +/// defaults. +fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { + body.children("a:p") + .map(|p| { + // A run's properties default to its paragraph's own (`a:pPr/a:defRPr`), then to the + // list style's for the paragraph's level (`lvl`, 0–8), property by property. + let ppr_el = p.child("a:pPr"); + let level = ppr_el.and_then(|pp| pp.attr("lvl")).and_then(|v| v.trim().parse::().ok()).filter(|l| *l <= 8).unwrap_or(0); + let level_style = body.child("a:lstStyle").and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))); + let defaults: Vec<&El> = + [ppr_el.and_then(|pp| pp.child("a:defRPr")), level_style.and_then(|l| l.child("a:defRPr"))].into_iter().flatten().collect(); + let defaults = defaults.as_slice(); + let mut ppr = + vec![Node::El(el("w:spacing", attrs(&[("w:before", "0"), ("w:after", "0"), ("w:line", "240"), ("w:lineRule", "auto")]), Vec::new()))]; + let jc = match ppr_el.and_then(|pp| pp.attr("algn")) { + Some("ctr") => Some("center"), + Some("r") => Some("right"), + Some("just") => Some("both"), + Some("dist") => Some("distribute"), + Some("l") => Some("left"), + _ => None, + }; + if let Some(jc) = jc { + ppr.push(Node::El(el("w:jc", attrs(&[("w:val", jc)]), Vec::new()))); + } + // The paragraph mark's own size and font (`a:endParaRPr`), which an empty one's + // height follows. + ppr.push(Node::El(run_props(p.child("a:endParaRPr"), defaults, theme))); + let mut kids = vec![Node::El(el("w:pPr", Vec::new(), ppr))]; + for r in p.els() { + let text = match r.name.as_str() { + "a:r" | "a:fld" => Node::El(el( + "w:t", + attrs(&[("xml:space", "preserve")]), + vec![Node::Text(r.child("a:t").map(|t| t.text()).unwrap_or_default())], + )), + "a:br" => Node::El(el("w:br", Vec::new(), Vec::new())), + _ => continue, + }; + let rpr = run_props(r.child("a:rPr"), defaults, theme); + kids.push(Node::El(el("w:r", Vec::new(), vec![Node::El(rpr), text]))); + } + Node::El(el("w:p", Vec::new(), kids)) + }) + .collect() +} + +/// A DrawingML run's properties (`a:rPr`, each falling back on the first of `defaults`, the +/// `a:defRPr`s in order, that gives it) as `w:rPr`: font, size, bold, italic, underline, +/// strikethrough, caps, baseline and colour. +fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { + let get = |n: &str| rpr.and_then(|r| r.attr(n)).or_else(|| defaults.iter().find_map(|d| d.attr(n))); + let child = |n: &str| rpr.and_then(|r| r.child(n)).or_else(|| defaults.iter().find_map(|d| d.child(n))); + let on = |n: &str| get(n).is_some_and(|v| v == "1" || v == "true"); + let mut kids = Vec::new(); + let mut push = |name: &str, a: &[(&str, &str)]| kids.push(Node::El(el(name, attrs(a), Vec::new()))); + // Theme fonts (`+mn-lt`) are left to the document's. + let face = |n: &str| child(n).and_then(|f| f.attr("typeface")).filter(|t| !t.is_empty() && !t.starts_with('+')); + let mut fonts = Vec::new(); + if let Some(f) = face("a:latin") { + fonts.extend([("w:ascii", f), ("w:hAnsi", f)]); + } + if let Some(f) = face("a:ea") { + fonts.push(("w:eastAsia", f)); + } + if let Some(f) = face("a:cs") { + fonts.push(("w:cs", f)); + } + if !fonts.is_empty() { + push("w:rFonts", &fonts); + } + if on("b") { + push("w:b", &[]); + } + if on("i") { + push("w:i", &[]); + } + match get("cap") { + Some("all") => push("w:caps", &[]), + Some("small") => push("w:smallCaps", &[]), + _ => {} + } + match get("strike") { + Some("sngStrike") => push("w:strike", &[]), + Some("dblStrike") => push("w:dstrike", &[]), + _ => {} + } + match get("u") { + None | Some("none") => {} + Some("dbl") => push("w:u", &[("w:val", "double")]), + Some(_) => push("w:u", &[("w:val", "single")]), + } + match get("baseline").and_then(|v| v.trim().parse::().ok()) { + Some(b) if b > 0 => push("w:vertAlign", &[("w:val", "superscript")]), + Some(b) if b < 0 => push("w:vertAlign", &[("w:val", "subscript")]), + _ => {} + } + if let Some(rgb) = child("a:solidFill").and_then(|f| crate::read::drawing_color::color(f, theme)) { + push("w:color", &[("w:val", &rgb.hex())]); + } + // Hundredths of a point into half-points, within Word's 1–1638 pt. + let sz = get("sz").and_then(|v| v.trim().parse::().ok()).unwrap_or(DEFAULT_SZ).clamp(100, 163_800) / 50; + let sz = sz.to_string(); + push("w:sz", &[("w:val", &sz)]); + push("w:szCs", &[("w:val", &sz)]); + // WordArt's outline, shadow, glow and reflection, as Word 2010 writes them in `w:rPr`. + kids.extend(text_effects(child("a:ln"), child("a:effectLst"), theme).into_iter().map(Node::El)); + el("w:rPr", Vec::new(), kids) +} + +/// A DrawingML run's text outline (`a:ln`) and effects (`a:effectLst`: outer shadow, glow, +/// reflection) as the `w14:` run properties ([MS-DOCX] §2.6.1) the run reader takes for WordArt. +/// Colours are resolved here, theme and modifiers included, into a `w14:srgbClr` with its +/// transparency. Lengths (EMUs), angles and percentages carry over as they are. +fn text_effects(ln: Option<&El>, fx: Option<&El>, theme: &[Rgb]) -> Vec { + let color = |e: &El| -> Option { + let (rgb, transparency) = crate::read::drawing_color::color_alpha(e, theme, None)?; + let alpha = ((transparency.clamp(0.0, 100.0) * 1000.0).round() as i64).to_string(); + let kids = if transparency > 0.0 { vec![Node::El(el("w14:alpha", attrs(&[("w14:val", &alpha)]), Vec::new()))] } else { Vec::new() }; + Some(Node::El(el("w14:srgbClr", attrs(&[("w14:val", &rgb.hex())]), kids))) + }; + // The attributes `names` of `e`, under the `w14:` prefix. + let copied = |e: &El, names: &[&str]| -> Vec<(String, String)> { + names.iter().filter_map(|n| e.attr(n).map(|v| (format!("w14:{n}"), v.to_string()))).collect() + }; + let mut out = Vec::new(); + if let Some(ln) = ln + && let Some(c) = ln.child("a:solidFill").and_then(color) + { + out.push(el("w14:textOutline", copied(ln, &["w"]), vec![Node::El(el("w14:solidFill", Vec::new(), vec![c]))])); + } + for e in fx.into_iter().flat_map(|f| f.els()) { + match e.name.as_str() { + "a:outerShdw" => out.push(el("w14:shadow", copied(e, &["blurRad", "dist", "dir"]), color(e).into_iter().collect())), + "a:glow" => out.push(el("w14:glow", copied(e, &["rad"]), color(e).into_iter().collect())), + "a:reflection" => out.push(el("w14:reflection", copied(e, &["blurRad", "stA", "endPos", "dist"]), Vec::new())), + _ => {} + } + } + out +} + +fn el(name: &str, attrs: Vec<(String, String)>, kids: Vec) -> El { + El { name: name.to_string(), attrs, kids } +} + +fn attrs(a: &[(&str, &str)]) -> Vec<(String, String)> { + a.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() +} + +/// `e` under another name, with its attributes and children. +fn renamed(e: &El, name: &str) -> El { + El { name: name.to_string(), ..e.clone() } +} diff --git a/crates/docx/src/xml.rs b/crates/docx/src/xml.rs index bb86755d9..fa50d217a 100644 --- a/crates/docx/src/xml.rs +++ b/crates/docx/src/xml.rs @@ -52,8 +52,10 @@ pub const NAMESPACES: &[(&str, &str)] = &[ pub const DECORATIVE_EXT: &str = "{C183D7F6-B498-43B3-948B-1728B52AA6E4}"; pub const DECORATIVE_NS: &str = "http://schemas.microsoft.com/office/drawing/2017/decorative"; -/// Namespaces that only appear in non-body parts. +/// Namespaces that only appear in non-body parts, or that the reader reads but never writes. const OTHER_NAMESPACES: &[(&str, &str)] = &[ + // A locked canvas, read into a drawing group (wordcraft#5060). + ("lc", "http://schemas.openxmlformats.org/drawingml/2006/lockedCanvas"), ("b", "http://schemas.openxmlformats.org/officeDocument/2006/bibliography"), ("rel", "http://schemas.openxmlformats.org/package/2006/relationships"), ("ct", "http://schemas.openxmlformats.org/package/2006/content-types"), diff --git a/crates/docx/tests/it/locked_canvas.rs b/crates/docx/tests/it/locked_canvas.rs new file mode 100644 index 000000000..8fa7a0a20 --- /dev/null +++ b/crates/docx/tests/it/locked_canvas.rs @@ -0,0 +1,143 @@ +//! A DrawingML locked canvas (`lc:lockedCanvas`, ECMA-376 Part 1 §20.3.2.1), as Word 2007 writes +//! shapes, pictures and WordArt in a drawing in compatibility mode (wordcraft#5060). Fixtures are +//! hand-written from ECMA-376 Part 1 §20.1 (DrawingML main) and §20.3 (locked canvas). + +use std::io::Write; + +use craft_wordmodel::para::ShapeKind; +use craft_wordmodel::props::{Rgb, TextColor}; +use craft_wordmodel::{Document, InlineObject, StoryRef}; + +const NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main""#; + +fn docx(body: &str) -> Vec { + let doc = format!(r#"{body}"#); + let types = + r#""#; + let root = r#""#; + let rels = r#""#; + let entries: [(&str, &[u8]); 5] = [ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", root.as_bytes()), + ("word/document.xml", doc.as_bytes()), + ("word/_rels/document.xml.rels", rels.as_bytes()), + ("word/media/image1.png", "\u{89}PNG stand-in".as_bytes()), + ]; + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, bytes) in entries { + zw.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +/// A floating drawing, 200 × 100 pt, holding `canvas` (the locked canvas's members) in a canvas +/// whose members' space starts at 10 pt, 10 pt. +fn drawing(canvas: &str) -> String { + format!( + r#"00{canvas}"# + ) +} + +const OVAL: &str = r#""#; + +/// WordArt in a nested group: bold, all-caps red text with a green outline and a reflection, +/// 32 pt, centred, bent by a chevron warp; a second paragraph takes the list style's size, and an +/// empty third one its mark's. +const WORD_ART: &str = r#"Youngkeepers"#; + +const PICTURE: &str = r#""#; + +/// A line glued to the oval by the canvas's own id, which isn't a document shape's. +const LINE: &str = r#""#; + +fn group(d: &Document) -> (f32, f32, Vec) { + let p = d.body.iter().find_map(|b| b.as_para()).unwrap(); + match p.objects.first() { + Some(InlineObject::Group { w, h, children, canvas: None, .. }) => (*w, *h, children.clone()), + o => panic!("{o:?}"), + } +} + +/// wordcraft#5060: the locked canvas is a group of its shapes, text, picture and line, each where the +/// canvas (and the group inside it) puts it, and nothing is lost on save. +#[test] +fn a_locked_canvas_reads_as_a_group_of_its_members() { + let d = craft_docx::read(&docx(&drawing(&[OVAL, WORD_ART, PICTURE, LINE].concat()))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (w, h, children) = group(d); + assert_eq!((w, h), (200.0, 100.0)); + let at: Vec<(f32, f32)> = children.iter().map(|c| (c.x, c.y)).collect(); + assert_eq!(at, [(0.0, 0.0), (110.0, 60.0), (0.0, 50.0), (0.0, 80.0)]); + let [oval, art, picture, line] = &children[..] else { panic!("{children:?}") }; + let InlineObject::Shape { kind: ShapeKind::Ellipse, w: 100.0, h: 50.0, fill: Some(fill), stroke: Some(stroke), .. } = &oval.obj else { + panic!("{:?}", oval.obj) + }; + assert_eq!((fill.main_color(), *stroke), (Rgb(0xFF, 0xFF, 0x66), Rgb(0, 0, 0))); + let InlineObject::Shape { kind: ShapeKind::TextBox, w: 20.0, h: 10.0, story: Some(id), extra, .. } = &art.obj else { + panic!("{:?}", art.obj) + }; + assert_eq!(extra.warp.as_ref().map(|w| w.preset.as_str()), Some("textChevron")); + assert_eq!(d.plain_text(StoryRef::Part(*id)).trim_end(), "Young\nkeepers"); + let blocks = &d.parts.get(id).unwrap().blocks; + let first = blocks[0].as_para().unwrap(); + assert_eq!(first.props.align, Some(craft_wordmodel::props::Align::Center)); + assert_eq!((first.props.space_before, first.props.space_after), (Some(0.0), Some(0.0)), "DrawingML text has no paragraph spacing"); + let run = &first.runs[0].props; + assert_eq!((run.size, run.bold, run.caps, run.color), (Some(32.0), Some(true), Some(true), Some(TextColor::Rgb(Rgb(0xFF, 0, 0))))); + assert_eq!(run.font.as_deref(), Some("Corsiva Stand-in")); + let fx = run.text_effects.as_deref().unwrap(); + let outline = fx.outline.unwrap(); + assert_eq!((outline.color, outline.width), (Some(Rgb(0, 0xFF, 0)), 0.75)); + assert!(fx.reflection.is_some(), "{fx:?}"); + assert_eq!(blocks[1].as_para().unwrap().runs[0].props.size, Some(14.0), "the list style's size"); + assert_eq!(blocks[2].as_para().unwrap().mark.size, Some(10.0), "an empty paragraph's mark"); + let InlineObject::Image { w: 50.0, h: 50.0, alt, .. } = &picture.obj else { panic!("{:?}", picture.obj) }; + assert_eq!(alt, "A logo"); + let InlineObject::Shape { kind: ShapeKind::Line, stroke: Some(blue), extra, .. } = &line.obj else { panic!("{:?}", line.obj) }; + assert_eq!((*blue, extra.start), (Rgb(0, 0, 0xFF), None), "no glue to a canvas id"); + } +} + +/// A text shape as a member of the canvas (`a:txSp`, text with no shape of its own) is a text box +/// at its own rectangle. Runs take their paragraph's own defaults (`a:pPr/a:defRPr`) before the +/// list style's for the paragraph's level. +#[test] +fn a_locked_canvas_text_shape_and_paragraph_defaults() { + let text = r#"OwnLevelFirst"#; + let d = craft_docx::read(&docx(&drawing(text))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (_, _, children) = group(d); + let [shape] = &children[..] else { panic!("{children:?}") }; + assert_eq!((shape.x, shape.y), (40.0, 20.0)); + let InlineObject::Shape { kind: ShapeKind::TextBox, w: 100.0, h: 40.0, fill: None, stroke: None, story: Some(id), .. } = &shape.obj else { + panic!("{:?}", shape.obj) + }; + assert_eq!(d.plain_text(StoryRef::Part(*id)).trim_end(), "Own\nLevel\nFirst"); + let blocks = &d.parts.get(id).unwrap().blocks; + let run = |i: usize| blocks[i].as_para().unwrap().runs[0].props.clone(); + assert_eq!((run(0).size, run(0).italic), (Some(32.0), Some(true)), "the paragraph's own size, the level's italic"); + assert_eq!((run(1).size, run(1).italic), (Some(20.0), None), "level 2's"); + assert_eq!((run(2).size, run(2).italic), (Some(14.0), Some(true)), "level 1's"); + } +} + +/// Before wordcraft#5060 only the canvas's first picture showed: the shapes and their text were dropped. +#[test] +fn a_locked_canvas_is_not_read_as_its_picture_alone() { + let d = craft_docx::read(&docx(&drawing(&[OVAL, PICTURE].concat()))).unwrap(); + let (_, _, children) = group(&d); + assert_eq!(children.len(), 2); +} + +/// Hostile canvases never panic: no members, groups nested past the limit, text with no runs, +/// huge and broken sizes. +#[test] +fn hostile_locked_canvases_never_panic() { + let deep = format!("{}{}{}", r#""#.repeat(40), OVAL, "".repeat(40)); + let odd = r#"f"#; + for canvas in ["", deep.as_str(), odd] { + let d = craft_docx::read(&docx(&drawing(canvas))).unwrap(); + let _ = craft_docx::write(&d).unwrap(); + } +} diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index b0bab5833..7d706e3d8 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -23,6 +23,7 @@ mod heading_levels; mod legacy_forms; mod ligatures; mod line_numbering; +mod locked_canvas; mod malformed; mod missing_pictures; mod nested_block_controls; From 88c52213b45509b3f1c7aec43029592b2af2acd7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 18:38:29 +0100 Subject: [PATCH 010/209] craft-drawingml: keep theme sysClr, custClrLst, theme family id, font panose and a:defPPr on save Co-Authored-By: Claude Sonnet 5.5 --- crates/drawingml/src/part.rs | 80 +++++++++++++++++++++++++++++ crates/drawingml/src/read.rs | 5 ++ crates/drawingml/src/text.rs | 6 ++- crates/drawingml/src/theme.rs | 45 +++++++++++++++- crates/drawingml/src/theme/read.rs | 16 ++++-- crates/drawingml/src/theme/write.rs | 22 ++++++-- crates/drawingml/src/write.rs | 4 ++ 7 files changed, 167 insertions(+), 11 deletions(-) diff --git a/crates/drawingml/src/part.rs b/crates/drawingml/src/part.rs index 27215f85d..c3e530dd8 100644 --- a/crates/drawingml/src/part.rs +++ b/crates/drawingml/src/part.rs @@ -239,4 +239,84 @@ mod tests { assert_eq!(back.extra.len(), MAX_EXTRA_COLORS); assert_eq!(back.extra[0].tints.as_deref(), Some(&long[..MAX_TINT_ROWS])); } + + const THEME_FILE: &str = r#" + + + + + +"#; + + fn theme_round_trip(t: &Theme) -> (String, Theme) { + let xml = String::from_utf8(crate::theme::theme_xml(&mut Part::new(), t)).unwrap(); + let back = crate::theme::read_theme(&mut Part::new(), &parse(xml.as_bytes()).unwrap()); + (xml, back) + } + + #[test] + fn a_theme_keeps_system_colours_custom_colours_family_id_and_font_panose() { + let t = crate::theme::read_theme(&mut Part::new(), &parse(THEME_FILE.as_bytes()).unwrap()); + assert_eq!(t.sys_colors.len(), 2); + let (xml, back) = theme_round_trip(&t); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains("Brand red") && xml.contains("C00000") && xml.contains("Brand teal") && xml.contains("008080"), "{xml}"); + assert!(xml.contains("thm15:themeFamily") && xml.contains("{62F939B6-93AF-4DB8-9C6B-D6C7DFDC589F}") && xml.contains("{4A3C46E8-61CC-4603-A589-7422A47A8E4A}"), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + assert_eq!((&back.sys_colors, &back.colors.colors, &back.fonts, &back.raw_extra), (&t.sys_colors, &t.colors.colors, &t.fonts, &t.raw_extra), "read, write, read changes nothing"); + // Order: the colour list comes before the extension list, both after the theme elements. + let at = |n: &str| xml.find(n).unwrap_or_else(|| panic!("{n} in {xml}")); + assert!(at("") < at("") && at("") < at(""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + let (plain, back) = theme_round_trip(&Theme::default()); + assert!(!plain.contains("sysClr") && !plain.contains("custClrLst") && !plain.contains("themeFamily") && !plain.contains("panose"), "{plain}"); + assert!(back.sys_colors.is_empty() && back.fonts.minor.latin_attrs.is_empty()); + } + + #[test] + fn list_styles_keep_def_ppr_in_every_place_they_are_written() { + use crate::text::{ListStyle, TextBody}; + let src = r#""#; + let ls = crate::read::list_style(&mut Part::new(), &parse(src.as_bytes()).unwrap()); + let d = ls.default_level.as_ref().expect("defPPr read"); + assert_eq!((d.para.margin_left, d.para.align, d.run.lang.as_deref(), d.run.size), (Some(36.0), Some(crate::text::Align::Center), Some("de-DE"), Some(18.0))); + assert_eq!(ls.level(0).and_then(|l| l.run.size), Some(44.0)); + for tag in ["p:defaultTextStyle", "p:titleStyle", "p:bodyStyle", "p:otherStyle", "p:notesStyle", "a:lstStyle"] { + let mut w = craft_xml::Writer::new(); + crate::write::list_style(&mut w, &mut Part::new(), tag, &ls); + let xml = format!(r#"{}"#, w.finish()); + assert!(xml.find("{}"#, w.finish()); + let back = crate::read::text_body(&mut Part::new(), parse(xml.as_bytes()).unwrap().child(craft_xml::Local("txBody")).unwrap()); + assert_eq!(back.list_style, ls); + let empty = ListStyle { default_level: Some(Default::default()), ..Default::default() }; + assert!(!empty.is_empty()); + let mut w = craft_xml::Writer::new(); + crate::write::list_style(&mut w, &mut Part::new(), "p:defaultTextStyle", &empty); + assert_eq!(w.finish(), ""); + // Without `a:defPPr` none is written. + let mut w = craft_xml::Writer::new(); + crate::write::list_style(&mut w, &mut Part::new(), "p:defaultTextStyle", &ListStyle { default_level: None, ..ls }); + assert!(!w.finish().contains("defPPr")); + } } diff --git a/crates/drawingml/src/read.rs b/crates/drawingml/src/read.rs index 9e9f8ff3e..012c20552 100644 --- a/crates/drawingml/src/read.rs +++ b/crates/drawingml/src/read.rs @@ -623,6 +623,11 @@ pub fn list_style(src: &mut dyn Source, ls: &El) -> ListStyle { let mut out = ListStyle::default(); for e in ls.elements() { let n = e.local(); + if n == "defPPr" { + let (_, para, run) = ppr(src, e); + out.default_level = Some(LevelStyle { para, run: run.unwrap_or_default() }); + continue; + } let Some(l) = n.strip_prefix("lvl").and_then(|r| r.strip_suffix("pPr")).and_then(|d| d.parse::().ok()) else { continue }; if !(1..=9).contains(&l) { continue; diff --git a/crates/drawingml/src/text.rs b/crates/drawingml/src/text.rs index f60f23236..8fe0244e1 100644 --- a/crates/drawingml/src/text.rs +++ b/crates/drawingml/src/text.rs @@ -706,6 +706,10 @@ pub struct LevelStyle { pub struct ListStyle { /// Levels 1–9 (index 0–8); `None` = not specified. pub levels: Vec>, + /// The default paragraph style (`a:defPPr`) that sits under levels 1–9; `None` = the list + /// style has no `a:defPPr`. + #[serde(skip_serializing_if = "Option::is_none")] + pub default_level: Option, } impl ListStyle { @@ -722,7 +726,7 @@ impl ListStyle { } } pub fn is_empty(&self) -> bool { - self.levels.iter().all(Option::is_none) + self.default_level.is_none() && self.levels.iter().all(Option::is_none) } } diff --git a/crates/drawingml/src/theme.rs b/crates/drawingml/src/theme.rs index 69d7c2af6..9b663b86e 100644 --- a/crates/drawingml/src/theme.rs +++ b/crates/drawingml/src/theme.rs @@ -11,11 +11,35 @@ pub struct FontSet { pub latin: String, pub ea: String, pub cs: String, + /// The `panose`, `pitchFamily` and `charset` the file gives the Latin, East Asian and complex + /// script typefaces; `None` = the attribute is absent. + #[serde(skip_serializing_if = "TypefaceAttrs::is_empty")] + pub latin_attrs: TypefaceAttrs, + #[serde(skip_serializing_if = "TypefaceAttrs::is_empty")] + pub ea_attrs: TypefaceAttrs, + #[serde(skip_serializing_if = "TypefaceAttrs::is_empty")] + pub cs_attrs: TypefaceAttrs, /// Per-script typefaces (`a:font script="Arab" typeface="…"`), in document order. #[serde(skip_serializing_if = "Vec::is_empty")] pub scripts: Vec, } +/// The font-matching attributes of a theme typeface (`a:latin`, `a:ea`, `a:cs`). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TypefaceAttrs { + /// Ten hex byte pairs, as written. + pub panose: Option, + pub pitch_family: Option, + pub charset: Option, +} + +impl TypefaceAttrs { + pub fn is_empty(&self) -> bool { + self.panose.is_none() && self.pitch_family.is_none() && self.charset.is_none() + } +} + /// A theme font for one script (ISO 15924 tag, e.g. `Arab`). #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] @@ -53,6 +77,17 @@ pub struct FormatScheme { pub bg_fills: Vec, } +/// A colour slot written as a system colour (``). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SysColor { + pub slot: SchemeSlot, + /// The `val` (`windowText`, `window`…). + pub name: String, + /// The `lastClr`. + pub last: Rgba, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] pub struct Theme { @@ -60,7 +95,14 @@ pub struct Theme { pub colors: ColorScheme, pub fonts: FontScheme, pub format: FormatScheme, - /// Raw theme XML extras kept for round-trip (object defaults, extra colour lists). + /// The colour slots the file wrote as `a:sysClr` (`dk1` `windowText`, `lt1` `window`), in + /// document order. The writer keeps a slot as `a:sysClr` while its colour still equals + /// `last`; a slot whose colour was changed is written as `a:srgbClr`. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub sys_colors: Vec, + /// Raw theme XML extras kept for round-trip, in document order: object defaults, extra + /// colour lists, custom colours (`a:custClrLst`) and the extension list (`a:extLst`, which + /// holds the theme family id). #[serde(skip_serializing_if = "Option::is_none")] pub raw_extra: Option, } @@ -76,6 +118,7 @@ impl Default for Theme { colors: ColorScheme::new("Default", colors.map(hex)), fonts: FontScheme::default(), format: default_format(), + sys_colors: Vec::new(), raw_extra: None, } } diff --git a/crates/drawingml/src/theme/read.rs b/crates/drawingml/src/theme/read.rs index 29e49dc93..24b882b3a 100644 --- a/crates/drawingml/src/theme/read.rs +++ b/crates/drawingml/src/theme/read.rs @@ -2,7 +2,7 @@ use crate::color::{ColorScheme, Rgba, SchemeSlot}; use crate::style::{ColorBase, Effects}; -use crate::theme::{FontScheme, FontSet, FormatScheme, ScriptFont, Theme, default_format}; +use crate::theme::{FontScheme, FontSet, FormatScheme, ScriptFont, SysColor, Theme, TypefaceAttrs, default_format}; use craft_xml::{El, Local}; @@ -21,7 +21,12 @@ fn font_set(e: Option<&El>) -> FontSet { .collect() }) .unwrap_or_default(); - FontSet { latin: tf("latin"), ea: tf("ea"), cs: tf("cs"), scripts } + let attrs = |n: &str| { + let c = e.and_then(|e| e.child(Local(n))); + let at = |a: &str| c.and_then(|c| c.attr_local(a)).map(String::from); + TypefaceAttrs { panose: at("panose"), pitch_family: at("pitchFamily"), charset: at("charset") } + }; + FontSet { latin: tf("latin"), ea: tf("ea"), cs: tf("cs"), latin_attrs: attrs("latin"), ea_attrs: attrs("ea"), cs_attrs: attrs("cs"), scripts } } pub fn read_theme(src: &mut dyn Source, root: &El) -> Theme { @@ -33,7 +38,10 @@ pub fn read_theme(src: &mut dyn Source, root: &El) -> Theme { if let Some(c) = cs.child(Local(slot.xml_name())).and_then(dml::color) { let rgb = match c.base { ColorBase::Rgb { rgb } => rgb, - ColorBase::System { last, .. } => last, + ColorBase::System { name, last } => { + t.sys_colors.push(SysColor { slot, name, last }); + last + } ColorBase::Preset { name } => crate::color::preset(&name).unwrap_or(Rgba::BLACK), ColorBase::Scheme { .. } => continue, }; @@ -90,7 +98,7 @@ pub fn read_theme(src: &mut dyn Source, root: &El) -> Theme { } let extra: String = root .elements() - .filter(|e| e.is(Local("objectDefaults")) || e.is(Local("extraClrSchemeLst"))) + .filter(|e| ["objectDefaults", "extraClrSchemeLst", "custClrLst", "extLst"].iter().any(|n| e.is(Local(n)))) .filter(|e| !crate::xmlx::has_rel_refs(e)) .map(|e| src.keep(e)) .collect(); diff --git a/crates/drawingml/src/theme/write.rs b/crates/drawingml/src/theme/write.rs index 5e95e57eb..98a87f3a2 100644 --- a/crates/drawingml/src/theme/write.rs +++ b/crates/drawingml/src/theme/write.rs @@ -2,7 +2,7 @@ use crate::color::{ColorScheme, Rgba, SchemeSlot}; use crate::style::{ColorRef, Effects, Fill}; -use crate::theme::{FontSet, Theme, default_format}; +use crate::theme::{FontSet, Theme, TypefaceAttrs, default_format}; use craft_xml::Writer; @@ -12,9 +12,17 @@ use crate::write as dml; fn font_set(w: &mut Writer, tag: &str, f: &FontSet) { w.open(tag, &[]); - w.tag("a:latin").attr("typeface", if f.latin.is_empty() { "Arial" } else { &f.latin }).empty(); - w.tag("a:ea").attr("typeface", &f.ea).empty(); - w.tag("a:cs").attr("typeface", &f.cs).empty(); + let face = |w: &mut Writer, tag: &str, name: &str, a: &TypefaceAttrs| { + w.tag(tag) + .attr("typeface", name) + .attr_opt("panose", a.panose.as_deref()) + .attr_opt("pitchFamily", a.pitch_family.as_deref()) + .attr_opt("charset", a.charset.as_deref()) + .empty(); + }; + face(w, "a:latin", if f.latin.is_empty() { "Arial" } else { &f.latin }, &f.latin_attrs); + face(w, "a:ea", &f.ea, &f.ea_attrs); + face(w, "a:cs", &f.cs, &f.cs_attrs); for sf in &f.scripts { w.tag("a:font").attr("script", &sf.script).attr("typeface", &sf.typeface).empty(); } @@ -74,7 +82,11 @@ pub fn theme_xml(out: &mut dyn Sink, t: &Theme) -> Vec { for slot in SchemeSlot::THEME { let tag = format!("a:{}", slot.xml_name()); w.open(&tag, &[]); - w.tag("a:srgbClr").attr("val", t.colors.get(slot).hex()).empty(); + let rgb = t.colors.get(slot); + match t.sys_colors.iter().find(|s| s.slot == slot && s.last == rgb && crate::color::is_system_name(&s.name)) { + Some(s) => w.tag("a:sysClr").attr("val", &s.name).attr("lastClr", rgb.hex()).empty(), + None => w.tag("a:srgbClr").attr("val", rgb.hex()).empty(), + } w.close(); } extra_colors(&mut w, &t.colors); diff --git a/crates/drawingml/src/write.rs b/crates/drawingml/src/write.rs index 25bfb89db..da3f10d79 100644 --- a/crates/drawingml/src/write.rs +++ b/crates/drawingml/src/write.rs @@ -777,6 +777,10 @@ pub fn list_style(w: &mut Writer, out: &mut dyn Sink, tag: &str, ls: &ListStyle) return; } w.open(tag, &[]); + if let Some(l) = &ls.default_level { + let def = (l.run != RunProps::default()).then_some(&l.run); + ppr(w, out, "a:defPPr", None, &l.para, def); + } for (i, l) in ls.levels.iter().enumerate().take(9) { if let Some(l) = l { let t = format!("a:lvl{}pPr", i + 1); From a2d2eefa3cf5d53410575784824ace6abbf0a0b7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 18:40:20 +0100 Subject: [PATCH 011/209] craft-drawingml: drop theme font panose when the typeface changes; more theme tests Co-Authored-By: Claude Sonnet 5.5 --- crates/drawingml/src/part.rs | 32 +++++++++++++++++++++++++++++ crates/drawingml/src/theme.rs | 2 ++ crates/drawingml/src/theme/read.rs | 2 +- crates/drawingml/src/theme/write.rs | 3 +++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/drawingml/src/part.rs b/crates/drawingml/src/part.rs index c3e530dd8..37d99481d 100644 --- a/crates/drawingml/src/part.rs +++ b/crates/drawingml/src/part.rs @@ -286,6 +286,38 @@ mod tests { assert!(back.sys_colors.is_empty() && back.fonts.minor.latin_attrs.is_empty()); } + #[test] + fn font_panose_is_dropped_when_the_typeface_changes() { + let mut t = crate::theme::read_theme(&mut Part::new(), &parse(THEME_FILE.as_bytes()).unwrap()); + t.fonts.minor.latin = "Arial".into(); + let (xml, _) = theme_round_trip(&t); + assert!(xml.contains(r#""#), "{xml}"); + assert!(!xml.contains("020F0502020204030204"), "{xml}"); + assert!(xml.contains(r#""#, r#""#) + .replace(r#""#, r#""#); + let t = crate::theme::read_theme(&mut Part::new(), &parse(file.as_bytes()).unwrap()); + let (xml, _) = theme_round_trip(&t); + assert!(xml.contains(r#""#), "{xml}"); + assert!(xml.contains(r#""#), "{xml}"); + } + + #[test] + fn custom_colours_and_family_id_without_object_defaults_follow_an_empty_one() { + let file = THEME_FILE.replace("", ""); + let t = crate::theme::read_theme(&mut Part::new(), &parse(file.as_bytes()).unwrap()); + let (xml, back) = theme_round_trip(&t); + let at = |n: &str| xml.find(n).unwrap_or_else(|| panic!("{n} in {xml}")); + assert!(at("") < at(""), "{xml}"); + assert!(at("") < at(", pub pitch_family: Option, diff --git a/crates/drawingml/src/theme/read.rs b/crates/drawingml/src/theme/read.rs index 24b882b3a..0f4e4b2f0 100644 --- a/crates/drawingml/src/theme/read.rs +++ b/crates/drawingml/src/theme/read.rs @@ -24,7 +24,7 @@ fn font_set(e: Option<&El>) -> FontSet { let attrs = |n: &str| { let c = e.and_then(|e| e.child(Local(n))); let at = |a: &str| c.and_then(|c| c.attr_local(a)).map(String::from); - TypefaceAttrs { panose: at("panose"), pitch_family: at("pitchFamily"), charset: at("charset") } + TypefaceAttrs { typeface: at("typeface").unwrap_or_default(), panose: at("panose"), pitch_family: at("pitchFamily"), charset: at("charset") } }; FontSet { latin: tf("latin"), ea: tf("ea"), cs: tf("cs"), latin_attrs: attrs("latin"), ea_attrs: attrs("ea"), cs_attrs: attrs("cs"), scripts } } diff --git a/crates/drawingml/src/theme/write.rs b/crates/drawingml/src/theme/write.rs index 98a87f3a2..bc46d59e3 100644 --- a/crates/drawingml/src/theme/write.rs +++ b/crates/drawingml/src/theme/write.rs @@ -13,6 +13,9 @@ use crate::write as dml; fn font_set(w: &mut Writer, tag: &str, f: &FontSet) { w.open(tag, &[]); let face = |w: &mut Writer, tag: &str, name: &str, a: &TypefaceAttrs| { + // Panose and the like describe one font: a changed typeface does not keep them. + let none = TypefaceAttrs::default(); + let a = if a.typeface == name { a } else { &none }; w.tag(tag) .attr("typeface", name) .attr_opt("panose", a.panose.as_deref()) From d88b7542c4709fb0b3d98604e1a50bbc7cdf9948 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 18:43:13 +0100 Subject: [PATCH 012/209] craft-drawingml: format theme tests Co-Authored-By: Claude Sonnet 5.5 --- crates/drawingml/src/part.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/drawingml/src/part.rs b/crates/drawingml/src/part.rs index 37d99481d..a251d4f3c 100644 --- a/crates/drawingml/src/part.rs +++ b/crates/drawingml/src/part.rs @@ -263,12 +263,21 @@ mod tests { assert!(xml.contains(r#""#), "{xml}"); assert!(xml.contains(r#""#), "{xml}"); assert!(xml.contains("Brand red") && xml.contains("C00000") && xml.contains("Brand teal") && xml.contains("008080"), "{xml}"); - assert!(xml.contains("thm15:themeFamily") && xml.contains("{62F939B6-93AF-4DB8-9C6B-D6C7DFDC589F}") && xml.contains("{4A3C46E8-61CC-4603-A589-7422A47A8E4A}"), "{xml}"); + assert!( + xml.contains("thm15:themeFamily") + && xml.contains("{62F939B6-93AF-4DB8-9C6B-D6C7DFDC589F}") + && xml.contains("{4A3C46E8-61CC-4603-A589-7422A47A8E4A}"), + "{xml}" + ); assert!(xml.contains(r#""#), "{xml}"); assert!(xml.contains(r#""#), "{xml}"); assert!(xml.contains(r#""#), "{xml}"); assert!(xml.contains(r#""#), "{xml}"); - assert_eq!((&back.sys_colors, &back.colors.colors, &back.fonts, &back.raw_extra), (&t.sys_colors, &t.colors.colors, &t.fonts, &t.raw_extra), "read, write, read changes nothing"); + assert_eq!( + (&back.sys_colors, &back.colors.colors, &back.fonts, &back.raw_extra), + (&t.sys_colors, &t.colors.colors, &t.fonts, &t.raw_extra), + "read, write, read changes nothing" + ); // Order: the colour list comes before the extension list, both after the theme elements. let at = |n: &str| xml.find(n).unwrap_or_else(|| panic!("{n} in {xml}")); assert!(at("") < at("") && at("") < at("") < at(""), "{xml}"); assert!(at("") < at(""#; let ls = crate::read::list_style(&mut Part::new(), &parse(src.as_bytes()).unwrap()); let d = ls.default_level.as_ref().expect("defPPr read"); - assert_eq!((d.para.margin_left, d.para.align, d.run.lang.as_deref(), d.run.size), (Some(36.0), Some(crate::text::Align::Center), Some("de-DE"), Some(18.0))); + assert_eq!( + (d.para.margin_left, d.para.align, d.run.lang.as_deref(), d.run.size), + (Some(36.0), Some(crate::text::Align::Center), Some("de-DE"), Some(18.0)) + ); assert_eq!(ls.level(0).and_then(|l| l.run.size), Some(44.0)); for tag in ["p:defaultTextStyle", "p:titleStyle", "p:bodyStyle", "p:otherStyle", "p:notesStyle", "a:lstStyle"] { let mut w = craft_xml::Writer::new(); From 92fb3fa8ac2981c4c31961cfce7f7fdfbb03540d Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sun, 15 Nov 2026 10:27:00 -0800 Subject: [PATCH 013/209] craft-texture: add per-axis wrapping and MaterialCraft's stored sampling MaterialCraft's `crates/image` gained per-axis wrap modes and the texture read of its function graphs. This adds them to craft-texture so the app can switch to the crate. - `get_axes`, `sample_axes` and `sample_nearest_axes` take `tiling: [bool; 2]`, the wrap mode of `u` and of `v` on their own. With equal entries they give the same results as `get`, `sample` and `sample_nearest`, which now call them. - `sample_stored` rounds the bilinear weights to multiples of 1/256 and blends by linear interpolation in `f32`. With the image's integer-sampling flag on, it clamps each channel to 0..=1 and rounds it to a multiple of 1/65535. A channel whose interpolation is not finite is taken from the `f64` weighted sum, so finite texels give a finite sample. - `with_integer_sampling` and `integer_sampling` set and read the flag. New images start with it off, clones keep it, and `PartialEq` compares it. --- crates/texture/README.md | 16 +- crates/texture/src/lib.rs | 367 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 368 insertions(+), 15 deletions(-) diff --git a/crates/texture/README.md b/crates/texture/README.md index a41b42666..8c82e5d0b 100644 --- a/crates/texture/README.md +++ b/crates/texture/README.md @@ -25,7 +25,16 @@ network I/O and has no `unsafe` code. (). Repeat keeps its period at any finite coordinate. A NaN or infinite coordinate gives `[0, 0, 0, 1]`. Bilinear sums are taken in `f64` and clamped to the `f32` range, so finite - texels give a finite sample. + texels give a finite sample. `sample_axes`, `sample_nearest_axes` and `get_axes` take + `tiling: [bool; 2]`, the wrap mode of `u` and of `v` on their own, as glTF's `wrapS` and + `wrapT`; with equal entries they give the same results as the forms with one `tile`. +- **Stored sampling.** `sample_stored` is the texture read of MaterialCraft's function graphs. It + rounds the bilinear weights to multiples of 1/256 and blends the texels by linear interpolation + in `f32`. When the image's integer-sampling flag is on (`with_integer_sampling(true)`, read with + `integer_sampling()`), it then clamps each channel to `0..=1` and rounds it to a multiple of + 1/65535. A channel whose interpolation is not finite is taken from the `f64` weighted sum + instead, so finite texels give a finite sample. New images start with the flag off, clones keep + it, and two images are equal only when their flags are equal. - **Other operations.** `get` reads one texel, `map`, `to_gray`, `to_color` and `to_kind` convert, `resize` box-averages whole-factor shrinks and samples bilinearly otherwise, and `to_rgba8_display` encodes 8-bit sRGB RGBA. `linear_to_srgb`, `srgb_to_linear` and `luminance` @@ -58,7 +67,8 @@ Without `parallel` the crate builds for `wasm32-unknown-unknown`. | Part | Source | |---|---| | `Image`, `Kind`, sampling, resizing, conversions, sRGB helpers, resource keys and the six tests `sample_survives_huge_coordinates`, `resource_keys_round_trip`, `sample_wraps_and_clamps`, `resize_box_and_bilinear`, `hostile_sizes_are_clamped`, `conversions` | MaterialCraft `crates/image/src/lib.rs` at [3133062](https://github.com/storytold/materialcraft/commit/3133062968b8a1566eda12a357cb06062db16687) | -| `Limits`, the `try_` constructors, `from_data`, private fields with getters, the `parallel` feature | written here | +| `get_axes`, `sample_axes`, `sample_nearest_axes`, `sample_stored` and the integer-sampling flag | MaterialCraft `crates/image/src/lib.rs` at [6c3b35c](https://github.com/storytold/materialcraft/commit/6c3b35c39632feafcda3d7a90ff18242f3ad910f), on this crate's coordinate reduction and `f64` bilinear sum | +| `Limits`, the `try_` constructors, `from_data`, private fields with getters, the `parallel` feature, `with_integer_sampling` and `integer_sampling` | written here | ## Migration @@ -73,5 +83,7 @@ For MaterialCraft: | --- | --- | | `image.w`, `image.h`, `image.kind`, `image.data` | `width()`, `height()`, `kind()`, `data()` and `shared_data()` | | `Image { w, h, kind, data }` built by hand | `Image::from_data(w, h, kind, data, &Limits::default())` | +| `image.integer_sampling = on` and `Image { integer_sampling, .. }` | `image.with_integer_sampling(on)`; read it with `integer_sampling()` | +| `sample_axes` blends with `a + (b − a)t` in `f32` | it sums the weighted texels in `f64`, as `sample` does; the results differ by rounding | | `rayon` always on | the `parallel` feature | | `linear_to_srgb(NaN)` returned −0.055 | it returns NaN | diff --git a/crates/texture/src/lib.rs b/crates/texture/src/lib.rs index 2aaef3e8b..04bc292f5 100644 --- a/crates/texture/src/lib.rs +++ b/crates/texture/src/lib.rs @@ -12,6 +12,10 @@ //! [`Image::sample_nearest`] and [`Image::sample`] (bilinear) use the texel coordinates, wrapping //! and weights of the Vulkan specification, sections 20.4.1, 20.6.1 and 20.6.3 //! (). +//! [`Image::get_axes`], [`Image::sample_axes`] and [`Image::sample_nearest_axes`] take the wrap +//! mode of each axis on its own. [`Image::sample_stored`] is MaterialCraft's function-graph texture +//! read: bilinear with weights rounded to eight fractional bits, and with its result rounded to 16 +//! bits when the image's integer-sampling flag is on ([`Image::with_integer_sampling`]). //! //! Sizes are bounded. [`Limits`] caps the side, the pixel count and the bytes, and the `try_` //! constructors return an [`ImageError`] before allocating anything over them. A plane is also @@ -130,15 +134,18 @@ pub struct Image { w: u32, h: u32, kind: Kind, + /// Whether [`Image::sample_stored`] rounds its result to 16 bits, as for a normalized integer + /// texture. Off for every new image. + integer_sampling: bool, /// Row-major, `kind.channels()` floats per pixel. data: Arc>, } -/// Images are equal when their size, kind and floats are equal. Floats compare as `f32` does, so -/// an image holding NaN is unequal to every image, its clones included. +/// Images are equal when their size, kind, integer-sampling flag and floats are equal. Floats +/// compare as `f32` does, so an image holding NaN is unequal to every image, its clones included. impl PartialEq for Image { fn eq(&self, o: &Image) -> bool { - self.w == o.w && self.h == o.h && self.kind == o.kind && self.data == o.data + self.w == o.w && self.h == o.h && self.kind == o.kind && self.integer_sampling == o.integer_sampling && self.data == o.data } } @@ -187,6 +194,13 @@ fn reduce(t: f32, tile: bool) -> Option { Some(if tile { t.rem_euclid(1.0).min(BELOW_ONE) } else { t.clamp(-1.0, 2.0) }) } +/// A bilinear sum taken in `f64` as an `f32`: a finite sum clamped to the `f32` range, and an +/// infinite or NaN sum as it is. +#[inline] +fn f32_of_sum(v: f64) -> f32 { + if v.is_finite() { v.clamp(f64::from(f32::MIN), f64::from(f32::MAX)) as f32 } else { v as f32 } +} + /// Fill `data` row by row with `f(x, y)`, keeping the first `c` channels of each result. fn fill [f32; 4] + Sync>(data: &mut [f32], w: u32, c: usize, f: &F) { let row_len = (w as usize).max(1) * c; @@ -209,7 +223,7 @@ impl Image { let (w, h, len) = bounded_size(w, h, kind, MAX_ALLOC_BYTES); let mut data = vec![0.0f32; len]; fill(&mut data, w, kind.channels(), &f); - Image { w, h, kind, data: Arc::new(data) } + Image { w, h, kind, integer_sampling: false, data: Arc::new(data) } } /// An image checked against `limits`, with its memory reserved before any pixel is generated. @@ -219,7 +233,7 @@ impl Image { data.try_reserve_exact(len).map_err(|_| ImageError::OutOfMemory(w, h))?; data.resize(len, 0.0); fill(&mut data, w, kind.channels(), &f); - Ok(Image { w, h, kind, data: Arc::new(data) }) + Ok(Image { w, h, kind, integer_sampling: false, data: Arc::new(data) }) } /// A constant image, each side clamped to `1..=MAX_SIDE`. @@ -272,7 +286,21 @@ impl Image { if data.len() != expected { return Err(ImageError::BadDataLength { expected, found: data.len() }); } - Ok(Image { w, h, kind, data }) + Ok(Image { w, h, kind, integer_sampling: false, data }) + } + + /// This image with the integer-sampling flag set to `on`. The flag only changes + /// [`Image::sample_stored`], which then rounds its result to 16 bits. Clones keep the flag. + /// [`Image::map`], [`Image::resize`] to a new size and the other operations that make a new + /// buffer return an image with the flag off. The pixels are not changed or copied. + pub fn with_integer_sampling(mut self, on: bool) -> Image { + self.integer_sampling = on; + self + } + + /// Whether [`Image::sample_stored`] rounds its result to 16 bits. + pub fn integer_sampling(&self) -> bool { + self.integer_sampling } pub fn width(&self) -> u32 { @@ -308,7 +336,14 @@ impl Image { /// The pixel at integer coordinates (wrapped or clamped), as RGBA (gray → `[v, v, v, 1]`). #[inline] pub fn get(&self, x: i64, y: i64, tile: bool) -> [f32; 4] { - let (x, y) = (wrap(x, self.w, tile), wrap(y, self.h, tile)); + self.get_axes(x, y, [tile; 2]) + } + + /// [`Image::get`] with the wrap mode of each axis given on its own: `tiling[0]` repeats `x` + /// and `tiling[1]` repeats `y`; an axis whose entry is `false` is clamped to its edge texels. + #[inline] + pub fn get_axes(&self, x: i64, y: i64, tiling: [bool; 2]) -> [f32; 4] { + let (x, y) = (wrap(x, self.w, tiling[0]), wrap(y, self.h, tiling[1])); let i = y * self.w as usize + x; match self.kind { Kind::Gray => { @@ -339,7 +374,15 @@ impl Image { /// leaves a sample at a texel center unchanged. NaN or infinite coordinates give `[0, 0, 0, 1]`. #[inline] pub fn sample(&self, u: f32, v: f32, tile: bool) -> [f32; 4] { - let (Some(u), Some(v)) = (reduce(u, tile), reduce(v, tile)) else { + self.sample_axes(u, v, [tile; 2]) + } + + /// [`Image::sample`] with the wrap mode of each axis given on its own: `tiling[0]` for `u` and + /// `tiling[1]` for `v`, as glTF samplers set `wrapS` and `wrapT` separately. With both entries + /// equal the result is the same as [`Image::sample`] with that `tile`. + #[inline] + pub fn sample_axes(&self, u: f32, v: f32, tiling: [bool; 2]) -> [f32; 4] { + let (Some(u), Some(v)) = (reduce(u, tiling[0]), reduce(v, tiling[1])) else { return [0.0, 0.0, 0.0, 1.0]; }; let fx = u * self.w as f32 - 0.5; @@ -356,12 +399,12 @@ impl Image { if weight == 0.0 { continue; } - let p = self.get(x, y, tile); + let p = self.get_axes(x, y, tiling); for (o, p) in o.iter_mut().zip(p) { *o += f64::from(weight) * f64::from(p); } } - o.map(|v| if v.is_finite() { v.clamp(f64::from(f32::MIN), f64::from(f32::MAX)) as f32 } else { v as f32 }) + o.map(f32_of_sum) } #[inline] @@ -373,18 +416,73 @@ impl Image { } } + /// Bilinear sample at normalized `(u, v)` with filter weights of eight fractional bits, the + /// texture read of MaterialCraft's function graphs. The texel coordinate and wrapping are those + /// of [`Image::sample`]. Each fractional part `α` and `β` is rounded to the nearest multiple of + /// 1/256 (a half rounds away from zero), and the four texels are blended by linear + /// interpolation in `f32`: `a + (b − a)α` along each of the two rows, then the same between + /// the rows with `β`. When the image's integer-sampling flag is on + /// ([`Image::with_integer_sampling`]), each channel is then clamped to `0..=1` and rounded to + /// the nearest multiple of 1/65535, as a filtered 16-bit texel. A NaN channel stays NaN. + /// + /// A channel whose interpolation is infinite or NaN is computed again as the weighted sum of + /// [`Image::sample`], skipping texels of weight 0 and clamping a finite sum to the `f32` + /// range, so finite texels give a finite sample. NaN or infinite coordinates give + /// `[0, 0, 0, 1]`. + #[inline] + pub fn sample_stored(&self, u: f32, v: f32, tile: bool) -> [f32; 4] { + let pixel = self.sample_fixed_weights(u, v, tile); + if self.integer_sampling { pixel.map(|value| (value.clamp(0.0, 1.0) * 65535.0).round() / 65535.0) } else { pixel } + } + + /// The bilinear interpolation of [`Image::sample_stored`] before its 16-bit rounding. + #[inline] + fn sample_fixed_weights(&self, u: f32, v: f32, tile: bool) -> [f32; 4] { + let (Some(u), Some(v)) = (reduce(u, tile), reduce(v, tile)) else { + return [0.0, 0.0, 0.0, 1.0]; + }; + let fx = u * self.w as f32 - 0.5; + let fy = v * self.h as f32 - 0.5; + let (x0, y0) = (fx.floor(), fy.floor()); + let tx = ((fx - x0) * 256.0).round() / 256.0; + let ty = ((fy - y0) * 256.0).round() / 256.0; + let (x0, y0) = (x0 as i64, y0 as i64); + let (a, b, c, d) = (self.get(x0, y0, tile), self.get(x0 + 1, y0, tile), self.get(x0, y0 + 1, tile), self.get(x0 + 1, y0 + 1, tile)); + let mut o = [0.0f32; 4]; + for (k, o) in o.iter_mut().enumerate() { + let top = a[k] + (b[k] - a[k]) * tx; + let bot = c[k] + (d[k] - c[k]) * tx; + *o = top + (bot - top) * ty; + if !o.is_finite() { + let taps = [((1.0 - tx) * (1.0 - ty), a[k]), (tx * (1.0 - ty), b[k]), ((1.0 - tx) * ty, c[k]), (tx * ty, d[k])]; + let sum = taps.iter().filter(|(weight, _)| *weight != 0.0).map(|(weight, p)| f64::from(*weight) * f64::from(*p)).sum(); + *o = f32_of_sum(sum); + } + } + o + } + /// Nearest sample at normalized `(u, v)`: the texel `floor(u × w)`, wrapped or clamped. NaN or /// infinite coordinates give `[0, 0, 0, 1]`. #[inline] pub fn sample_nearest(&self, u: f32, v: f32, tile: bool) -> [f32; 4] { - let (Some(u), Some(v)) = (reduce(u, tile), reduce(v, tile)) else { + self.sample_nearest_axes(u, v, [tile; 2]) + } + + /// [`Image::sample_nearest`] with the wrap mode of each axis given on its own: `tiling[0]` for + /// `u` and `tiling[1]` for `v`. With both entries equal the result is the same as + /// [`Image::sample_nearest`] with that `tile`. + #[inline] + pub fn sample_nearest_axes(&self, u: f32, v: f32, tiling: [bool; 2]) -> [f32; 4] { + let (Some(u), Some(v)) = (reduce(u, tiling[0]), reduce(v, tiling[1])) else { return [0.0, 0.0, 0.0, 1.0]; }; let (x, y) = ((u * self.w as f32).floor() as i64, (v * self.h as f32).floor() as i64); // A repeated coordinate is below 1, so its texel is below the size even where `u × w` // rounds up to it. - let (x, y) = if tile { (x.min(i64::from(self.w) - 1), y.min(i64::from(self.h) - 1)) } else { (x, y) }; - self.get(x, y, tile) + let x = if tiling[0] { x.min(i64::from(self.w) - 1) } else { x }; + let y = if tiling[1] { y.min(i64::from(self.h) - 1) } else { y }; + self.get_axes(x, y, tiling) } /// Map every pixel (as RGBA) to a new image of `kind`, same size. When a result of that size @@ -892,4 +990,247 @@ mod tests { assert_eq!(half.data(), &[0.5, 0.5, 0.0, 1.0, 2.5, 0.5, 0.0, 1.0]); assert_eq!(img.to_gray().width(), 4); } + + /// The four wrap combinations of the two axes. + const TILINGS: [[bool; 2]; 4] = [[false, false], [true, true], [true, false], [false, true]]; + + /// Gray texels of 8-bit values, `k / 255`. + fn gray8() -> Image { + Image::gray(5, 3, |x, y| ((x * 37 + y * 91 + 13) % 256) as f32 / 255.0) + } + + /// Color texels from −2.5 to 3.42, with negative values and values above 1. + fn color() -> Image { + Image::generate(4, 3, Kind::Color, |x, y| { + let s = |k: u32| ((x * 7 + y * 13 + k * 5) % 17) as f32 * 0.37 - 2.5; + [s(0), s(1), s(2), s(3)] + }) + } + + /// 65 even steps from `lo` to `hi`, then the texel centers and edges of sides 3, 4 and 5. + fn coordinates(lo: f32, hi: f32) -> Vec { + let mut v: Vec = (0..=64).map(|i| lo + (hi - lo) * i as f32 / 64.0).collect(); + for side in [3u32, 4, 5] { + v.extend((0..=2 * side).map(|i| i as f32 / (2 * side) as f32)); + } + v + } + + /// FNV-1a over the bits of every channel. + fn hash(h: &mut u64, p: [f32; 4]) { + for b in p.iter().flat_map(|v| v.to_bits().to_le_bytes()) { + *h ^= u64::from(b); + *h = h.wrapping_mul(0x100_0000_01b3); + } + } + + const FNV_START: u64 = 0xcbf2_9ce4_8422_2325; + + /// Hash of `f(u, v)` over every pair of `coordinates`. + fn hash_grid(us: &[f32], f: impl Fn(f32, f32) -> [f32; 4]) -> u64 { + let mut h = FNV_START; + for &u in us { + for &v in us { + hash(&mut h, f(u, v)); + } + } + h + } + + /// Hashes of `sample_stored`, `sample_nearest_axes` and `get_axes` computed with MaterialCraft's + /// `crates/image` at storytold/materialcraft `6c3b35c396`, over the fixtures and grids above. + #[test] + fn stored_and_axis_reads_match_materialcraft() { + // (range, image, integer sampling, tile, sample_stored hash) + let stored = [ + ("unit", "gray8", false, false, 0x702b_0f81_2833_5e06u64), + ("unit", "gray8", false, true, 0x4ac5_b29c_45b2_d0ca), + ("unit", "gray8", true, false, 0x8400_a867_00fc_389a), + ("unit", "gray8", true, true, 0x47ea_befd_7fd6_cc5e), + ("unit", "color", false, false, 0x25f6_2544_f084_4182), + ("unit", "color", false, true, 0x607f_3c9f_896b_5975), + ("unit", "color", true, false, 0x0261_02eb_451b_36d6), + ("unit", "color", true, true, 0x51f8_9417_f5e9_2341), + ("wide", "gray8", false, false, 0xb0e7_096c_c64b_0f72), + ("wide", "gray8", false, true, 0x7a8a_4059_68ac_2a07), + ("wide", "gray8", true, false, 0x418c_9c08_2718_2171), + ("wide", "gray8", true, true, 0xe0b7_0088_256f_ccb2), + ("wide", "color", false, false, 0x51b4_8bb3_2885_f844), + ("wide", "color", false, true, 0xab4b_72d5_25ff_aaed), + ("wide", "color", true, false, 0x6570_eaa2_25f3_4d29), + ("wide", "color", true, true, 0x15a3_5aa7_7f50_4f92), + ]; + // (range, image, sample_nearest_axes hash for each entry of TILINGS) + let nearest = [ + ("unit", "gray8", [0xe427_9012_572d_b173u64, 0x76b9_e33c_849f_069d, 0x42ab_5ecd_8f1a_5dd2, 0x190c_23b8_f3e9_a2e5]), + ("unit", "color", [0x6a0e_0370_810d_1ad9, 0x0ab0_d62b_8308_4f9d, 0xa839_3cdf_db7d_844a, 0x1fc4_027e_bb8d_7c45]), + ("wide", "gray8", [0xd1cf_daaf_d7db_239c, 0x1d22_a6db_ad03_0960, 0x754e_fabf_5203_d6a0, 0xfe73_7385_df89_4dd4]), + ("wide", "color", [0xb894_834b_96fe_684d, 0x7eb4_0a21_3a51_c6bc, 0x2443_5503_2fea_4068, 0x4705_30d0_9afc_dc75]), + ]; + // (image, get_axes hash over x and y in -12..12 for each entry of TILINGS) + let get = [ + ("gray8", [0x6c95_1e05_c9c6_8f21u64, 0xac4a_e19a_4da7_b625, 0xcee5_f582_facb_f043, 0x7f1d_ff0e_357b_5eb5]), + ("color", [0x21af_a81f_327a_d3af, 0x9cf4_2557_9880_5345, 0x9b58_bde1_8021_2779, 0x6bb0_a98f_13d5_6635]), + ]; + let image = |name: &str| if name == "gray8" { gray8() } else { color() }; + let grid = |range: &str| if range == "unit" { coordinates(0.0, 0.99) } else { coordinates(-2.5, 3.5) }; + for (range, name, integer, tile, expected) in stored { + let img = image(name).with_integer_sampling(integer); + let h = hash_grid(&grid(range), |u, v| img.sample_stored(u, v, tile)); + assert_eq!(h, expected, "sample_stored {range} {name} integer {integer} tile {tile}"); + } + for (range, name, expected) in nearest { + let img = image(name); + for (tiling, expected) in TILINGS.into_iter().zip(expected) { + let h = hash_grid(&grid(range), |u, v| img.sample_nearest_axes(u, v, tiling)); + assert_eq!(h, expected, "sample_nearest_axes {range} {name} {tiling:?}"); + } + } + for (name, expected) in get { + let img = image(name); + for (tiling, expected) in TILINGS.into_iter().zip(expected) { + let mut h = FNV_START; + for x in -12..12 { + for y in -12..12 { + hash(&mut h, img.get_axes(x, y, tiling)); + } + } + assert_eq!(h, expected, "get_axes {name} {tiling:?}"); + } + } + } + + /// Values of MaterialCraft's `sample_axes` at storytold/materialcraft `6c3b35c396`. It blends + /// by linear interpolation in `f32`, and this crate sums the weighted texels in `f64`, so the + /// results differ by rounding only. + #[test] + fn axis_bilinear_matches_materialcraft_up_to_rounding() { + let img = color(); + let cases = [ + (0.3, 0.7, [true, false], [1.473_799_7, -0.072_800_055, 0.645_000_2, 0.733_799_8]), + (0.3, 0.7, [false, true], [1.473_799_7, -0.072_800_055, 0.645_000_2, 0.733_799_8]), + (1.2, -0.4, [true, false], [-1.722_999_5, 0.127_000_57, 0.089_999_32, 1.939_999_3]), + (1.2, -0.4, [false, true], [-1.057_000_3, -1.094, 0.756_000_1, 2.606_000_2]), + (-0.15, 1.1, [true, false], [2.051, -1.76, 0.090_000_15, 1.311_000_1]), + (-0.15, 1.1, [false, true], [-1.834_000_6, 0.015_999_317, 0.608_000_5, 2.458_000_7]), + (0.95, 0.05, [true, false], [-1.464, 0.386_000_04, 2.236_000_3, -0.317_000_15]), + (0.95, 0.05, [false, true], [0.145_500_18, -0.206_000_21, 1.644_000_2, -0.594_499_8]), + ]; + for (u, v, tiling, expected) in cases { + let p = img.sample_axes(u, v, tiling); + for (p, e) in p.iter().zip(expected) { + assert!((p - e).abs() <= 2.0e-6, "u {u} v {v} {tiling:?}: {p} against {e}"); + } + } + } + + #[test] + fn equal_axes_give_the_shared_results() { + let us = coordinates(-2.5, 3.5); + for img in [gray8(), color()] { + for tile in [false, true] { + for &u in &us { + for &v in &us { + assert_eq!(img.sample_axes(u, v, [tile; 2]).map(f32::to_bits), img.sample(u, v, tile).map(f32::to_bits)); + assert_eq!(img.sample_nearest_axes(u, v, [tile; 2]).map(f32::to_bits), img.sample_nearest(u, v, tile).map(f32::to_bits)); + } + } + for x in -12..12 { + assert_eq!(img.get_axes(x, -x, [tile; 2]), img.get(x, -x, tile)); + } + } + } + } + + #[test] + fn axes_wrap_independently() { + // Texel (x, y) holds x + 10y. + let img = Image::gray(4, 2, |x, y| (x + 10 * y) as f32); + assert_eq!(img.get_axes(-1, -1, [true, false])[0], 3.0); + assert_eq!(img.get_axes(-1, -1, [false, true])[0], 10.0); + assert_eq!(img.get_axes(5, 3, [true, false])[0], 11.0); + assert_eq!(img.sample_nearest_axes(1.1, 1.5, [true, false])[0], 10.0); + assert_eq!(img.sample_nearest_axes(1.1, 1.5, [false, true])[0], 13.0); + // v = 0.25 is the center of row 0; u = 1 is halfway between texels 3 and 0 when repeating. + assert!((img.sample_axes(1.0, 0.25, [true, false])[0] - 1.5).abs() < 1e-6); + assert_eq!(img.sample_axes(1.0, 0.25, [false, true])[0], 3.0); + assert_eq!(img.sample_axes(f32::NAN, 0.5, [false, true]), [0.0, 0.0, 0.0, 1.0]); + assert_eq!(img.sample_nearest_axes(0.5, f32::INFINITY, [true, false]), [0.0, 0.0, 0.0, 1.0]); + } + + #[test] + fn stored_sampling_rounds_weights_to_eight_bits() { + let img = Image::gray(2, 1, |x, _| x as f32); + // u × 2 − 0.5 = 0.3: the weight 0.3 rounds to 77/256. + assert_eq!(img.sample_stored(0.4, 0.5, false)[0], 77.0 / 256.0); + assert!((img.sample(0.4, 0.5, false)[0] - 0.3).abs() < 1e-6); + // A fraction of 255.5/256 or more rounds up to the next texel. + assert_eq!(img.sample_stored((1.5 - 0.25 / 256.0) / 2.0, 0.5, false)[0], 1.0); + assert_eq!(img.sample_stored(f32::NAN, 0.5, true), [0.0, 0.0, 0.0, 1.0]); + assert_eq!(img.sample_stored(0.5, f32::NEG_INFINITY, false), [0.0, 0.0, 0.0, 1.0]); + } + + #[test] + fn integer_sampling_rounds_stored_samples_to_sixteen_bits() { + let img = Image::gray(2, 1, |x, _| x as f32); + assert!(!img.integer_sampling()); + let int = img.clone().with_integer_sampling(true); + assert!(int.integer_sampling()); + // 77/256 × 65535 = 19711.64, which rounds to 19712. + assert_eq!(int.sample_stored(0.4, 0.5, false)[0], 19_712.0 / 65_535.0); + assert_eq!(int.sample(0.4, 0.5, false), img.sample(0.4, 0.5, false)); + assert_eq!(int.sample_nearest(0.4, 0.5, false), img.sample_nearest(0.4, 0.5, false)); + let l = Limits::default(); + let Ok(wide) = Image::from_data(3, 1, Kind::Gray, vec![-0.5, 2.0, f32::NAN], &l) else { panic!("valid 3 x 1 image") }; + let wide = wide.with_integer_sampling(true); + assert_eq!(wide.sample_stored(0.5 / 3.0, 0.5, false)[0], 0.0); + assert_eq!(wide.sample_stored(1.5 / 3.0, 0.5, false)[0], 1.0); + assert!(wide.sample_stored(2.5 / 3.0, 0.5, false)[0].is_nan()); + assert_eq!(wide.sample_stored(0.5 / 3.0, 0.5, false)[3], 1.0); + } + + #[test] + fn integer_sampling_follows_clones_and_not_new_buffers() { + let img = Image::gray(4, 2, |x, y| (x + y) as f32 / 8.0).with_integer_sampling(true); + let plain = img.clone().with_integer_sampling(false); + assert_eq!(img.clone(), img); + assert_ne!(img, plain); + assert_eq!(img.data(), plain.data()); + assert!(img.clone().integer_sampling()); + assert!(img.to_gray().integer_sampling()); + assert!(img.resize(4, 2, false).integer_sampling()); + assert!(!img.resize(2, 1, false).integer_sampling()); + assert!(!img.map(Kind::Gray, |p| p).integer_sampling()); + assert!(!img.to_color().integer_sampling()); + assert!(img.try_to_kind(Kind::Gray, &Limits::default()).is_ok_and(|i| i.integer_sampling())); + assert!(Arc::ptr_eq(img.shared_data(), plain.shared_data())); + } + + #[test] + fn stored_sampling_keeps_finite_texels_finite() { + let l = Limits::default(); + let Ok(inf) = Image::from_data(2, 1, Kind::Gray, vec![1.0, f32::INFINITY], &l) else { panic!("valid 2 x 1 image") }; + // The center of texel 0 gives texel 0, though its neighbor is infinite. + assert_eq!(inf.sample_stored(0.25, 0.5, false)[0], 1.0); + assert_eq!(inf.sample_stored(0.5, 0.5, false)[0], f32::INFINITY); + let Ok(hdr) = Image::from_data(2, 1, Kind::Gray, vec![f32::MAX, -f32::MAX], &l) else { panic!("valid 2 x 1 image") }; + assert_eq!(hdr.sample_stored(0.5, 0.5, false)[0], 0.0); + assert_eq!(hdr.clone().with_integer_sampling(true).sample_stored(0.5, 0.5, false)[0], 0.0); + let max = Image::filled(2, 2, Kind::Gray, [f32::MAX; 4]); + for tile in [false, true] { + assert_eq!(max.sample_stored(0.3, 0.6, tile)[0], f32::MAX, "tile = {tile}"); + } + } + + #[test] + fn stored_sampling_keeps_the_repeat_period_at_large_coordinates() { + let img = stripe(); + let at_zero = img.sample_stored(0.0, 0.5, true)[0]; + assert_eq!(at_zero, 0.5); + for u in [1.0e9_f32, -1.0e9, 4.0e9, 3.0e12, f32::MAX] { + assert_eq!(img.sample_stored(u, 0.5, true)[0], at_zero, "u = {u}"); + let edge = if u > 0.0 { 1.0 } else { 0.0 }; + assert_eq!(img.sample_stored(u, 0.5, false)[0], edge, "u = {u}"); + } + } } From 5f453efa9ad7e803f65c3edc7e08992d43422c23 Mon Sep 17 00:00:00 2001 From: Diego Guerini Date: Sun, 15 Nov 2026 20:10:47 +0100 Subject: [PATCH 014/209] craft-docx: a relativeHeight of 0 stays the lowest stacking order Float::z kept relativeHeight as read, so a file's 0, the lowest order a file can give, read as 0, "none", which stacks above every object. z is now one more than the file's (as ODF's draw:z-index is read), and Float::stack gives the stacking key. The writer writes stored orders one less, numbers objects made in the app above every stored one, and numbers the stored ones again from Word's first order when u32::MAX leaves no room above them. Document::stacking_orders counts every story saved with the document, the glossary's building blocks included. Moved from storytold/wordcraft#2655 (wordcraft#2638), whose DOCX reader and writer now live here. --- crates/docx/src/read/story/drawing.rs | 5 +- crates/docx/src/write.rs | 24 +++++++ crates/docx/src/write/parts.rs | 4 +- crates/docx/src/write/story/drawing.rs | 13 ++-- crates/docx/tests/it/fixtures.rs | 99 +++++++++++++++++++++++++- crates/docx/tests/it/roundtrip.rs | 16 ++--- crates/wordmodel/src/document.rs | 31 ++++++++ crates/wordmodel/src/para/float.rs | 13 +++- 8 files changed, 184 insertions(+), 21 deletions(-) diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..ea48efcca 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -622,8 +622,9 @@ fn nv_alt(p: Option<&El>) -> (String, AltText) { fn anchor_float(c: &El) -> Float { // `relativeHeight`: the stacking order Word draws floating objects in (higher on top), which - // need not be their order in the file. - let z = c.attr("relativeHeight").and_then(|v| v.trim().parse::().ok()).unwrap_or(0); + // need not be their order in the file. Kept one more (see `Float::z`): 0, the lowest a file + // can give, isn't the 0 of an object without one, which stacks above them all. + let z = c.attr("relativeHeight").and_then(|v| v.trim().parse::().ok()).map_or(0, |z| z.saturating_add(1)); let mut f = Float { wrap: Wrap::Square, z, ..Float::default() }; let behind = on_off_attr(c, "behindDoc"); // Word wraps a position relative to the page's size (`wp14:pctPosVOffset`) in diff --git a/crates/docx/src/write.rs b/crates/docx/src/write.rs index 1d9eb0818..634f5f833 100644 --- a/crates/docx/src/write.rs +++ b/crates/docx/src/write.rs @@ -82,6 +82,26 @@ impl PartRels { } } +/// Where Word's stacking orders (`relativeHeight`) start for objects made in Word. +const FIRST_Z: u32 = 251_658_240; + +/// The `relativeHeight` the objects of `doc` that have none count up from, and the `relativeHeight` +/// each stored order is written as when it isn't its own (wordcraft#2638). A stored order +/// ([`craft_wordmodel::para::Float::z`]) is one more than the file's, and the new objects go above +/// every stored one. When the stored orders leave no room for them below `u32::MAX` (a file using +/// values that high; `u32::MAX` itself would read back as the order below it), the stored ones are +/// numbered again from [`FIRST_Z`] in their order, so no two objects share an order and none goes +/// out of range. +pub(crate) fn stacking(doc: &Document) -> (u32, BTreeMap) { + let (orders, unset) = doc.stacking_orders(); + let top = orders.last().map_or(0, |z| z.saturating_sub(1)).max(FIRST_Z); + if u32::try_from(unset).ok().and_then(|n| top.checked_add(n)).is_some_and(|last| last < u32::MAX) { + return (top, BTreeMap::new()); + } + let zmap: BTreeMap = orders.iter().zip(FIRST_Z..=u32::MAX).map(|(&z, to)| (z, to)).collect(); + (zmap.values().last().copied().unwrap_or(FIRST_Z), zmap) +} + pub(crate) struct Writer<'d> { doc: &'d Document, /// Numbered display equations written so far (automatic numbers are written as text). @@ -95,7 +115,11 @@ pub(crate) struct Writer<'d> { docpr: u32, /// Shapes' own drawing ids written so far (connectors refer to them). shape_ids: std::collections::HashSet, + /// The last stacking order (`relativeHeight`) given to an object that had none. z: u32, + /// Stored stacking orders written as other `relativeHeight`s (see [`stacking`]); empty: each + /// as its own. + zmap: BTreeMap, /// Note part id → file note id, in first-reference order. footnotes: Vec, endnotes: Vec, diff --git a/crates/docx/src/write/parts.rs b/crates/docx/src/write/parts.rs index 4cb847276..00826f4b5 100644 --- a/crates/docx/src/write/parts.rs +++ b/crates/docx/src/write/parts.rs @@ -31,7 +31,8 @@ pub(crate) fn write_parts(doc: &Document, flavor: Flavor) -> Result Result)> = Vec::new(); diff --git a/crates/docx/src/write/story/drawing.rs b/crates/docx/src/write/story/drawing.rs index 7c03cca6e..2c13e1e7d 100644 --- a/crates/docx/src/write/story/drawing.rs +++ b/crates/docx/src/write/story/drawing.rs @@ -252,9 +252,14 @@ impl Writer<'_> { } else { let d = emu(float.dist.clamp(0.0, 1584.0)); let (dt, db) = (emu(float.dist_top.clamp(0.0, 1584.0)), emu(float.dist_bottom.clamp(0.0, 1584.0))); - // The stacking order read from the file is kept; objects without one stack in - // document order above it. - self.z = if float.z > 0 { float.z } else { self.z.saturating_add(1) }; + // A stacking order read from a file is kept (one less: see `Float::z`); objects made + // here go above those. + let z = if float.z > 0 { + self.zmap.get(&float.z).copied().unwrap_or(float.z.saturating_sub(1)) + } else { + self.z = self.z.saturating_add(1); + self.z + }; let behind = if float.wrap == Wrap::BehindText { "1" } else { "0" }; w.open( "wp:anchor", @@ -264,7 +269,7 @@ impl Writer<'_> { ("distL", &d), ("distR", &d), ("simplePos", "0"), - ("relativeHeight", &self.z.to_string()), + ("relativeHeight", &z.to_string()), ("behindDoc", behind), ("locked", "0"), ("layoutInCell", if float.outside_cell { "0" } else { "1" }), diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 868e8cbb0..9c5fc1e56 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -1282,10 +1282,11 @@ fn anchor_stacking_order_is_read_and_kept() { paras(d)[0].objects.iter().filter_map(|o| if let InlineObject::Shape { float, .. } = o { Some(float.z) } else { None }).collect() }; let d = read_body(&body); - assert_eq!(zs(&d), [251660288, 251657214, 0]); + // Kept one more than the file's (`Float::z`), so a file's 0 isn't "none". + assert_eq!(zs(&d), [251660289, 251657215, 0]); let again = craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap(); let z = zs(&again); - assert_eq!(&z[..2], &[251660288, 251657214]); + assert_eq!(&z[..2], &[251660289, 251657215]); assert!(z[2] > 0, "an object without one gets an order on saving: {z:?}"); } @@ -2592,6 +2593,100 @@ fn anchored_wsp(name: &str, cx: u32, cy: u32, sppr: &str, style: &str, extra: &s ) } +/// wordcraft#2638: Word stacks floating objects by `relativeHeight`, not by their order in the text (a +/// shape sent to back, then copied with the shape over it, comes first in the text but stays +/// behind). The order is read and kept on save, and an object made here is saved above them. +#[test] +fn floating_objects_keep_their_stacking_order() { + let shape = |name: &str, z: u32| { + anchored_wsp(name, 127000, 127000, r#""#, "", "") + .replace(r#"relativeHeight="1""#, &format!(r#"relativeHeight="{z}""#)) + }; + let body = format!("{}{}", shape("Canopy", 251662336), shape("Trunk", 251661312)); + let mut d = read_body(&body); + let stacking = |d: &Document| paras(d)[0].objects.iter().map(|o| o.frame().map(|(_, _, f)| f.z)).collect::>(); + assert_eq!(stacking(&d), [Some(251662337), Some(251661313)]); + let mut made = paras(&d)[0].objects[0].clone(); + if let InlineObject::Shape { float, .. } = &mut made { + float.z = 0; + } + let Some(Block::Para(p)) = d.body.first_mut().map(std::sync::Arc::make_mut) else { panic!("a paragraph") }; + p.insert_object(0, made, &Default::default()).unwrap(); + let files = unzip(&craft_docx::write(&d).unwrap()); + let written = attr_values(&text(&files, "word/document.xml"), "wp:anchor", "relativeHeight"); + assert_eq!(written, ["251662337", "251662336", "251661312"]); + let back = craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap(); + assert_eq!(stacking(&back), [Some(251662338), Some(251662337), Some(251661313)]); +} + +/// wordcraft#2638: a file whose stacking orders reach `u32::MAX` leaves no room above them for an object +/// made here. The stored orders are then numbered again, in their order, so the new object still +/// goes over all of them and no two objects share an order (it was written as `u32::MAX` too). +#[test] +fn stacking_orders_at_the_top_of_the_range_are_numbered_again() { + let shape = |name: &str, z: u32| { + anchored_wsp(name, 127000, 127000, r#""#, "", "") + .replace(r#"relativeHeight="1""#, &format!(r#"relativeHeight="{z}""#)) + }; + let body = format!("{}{}", shape("Top", u32::MAX), shape("Low", 7)); + let mut d = read_body(&body); + let stacking = |d: &Document| paras(d)[0].objects.iter().map(|o| o.frame().map(|(_, _, f)| f.z)).collect::>(); + assert_eq!(stacking(&d), [Some(u32::MAX), Some(8)]); + let mut made = paras(&d)[0].objects[0].clone(); + if let InlineObject::Shape { float, .. } = &mut made { + float.z = 0; + } + let Some(Block::Para(p)) = d.body.first_mut().map(std::sync::Arc::make_mut) else { panic!("a paragraph") }; + p.insert_object(0, made, &Default::default()).unwrap(); + let back = craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap(); + let z: Vec = stacking(&back).into_iter().map(|z| z.expect("every anchor has an order")).collect(); + assert!(z[0] > z[1] && z[1] > z[2], "new over Top over Low, all different: {z:?}"); + // The note separators are saved with the document: an order there counts too. + let mut d = read_body(&format!("{}", shape("Low", 7))); + let mut sep = read_body(&format!("{}", shape("Separator", u32::MAX))); + d.footnote_separators.separator = Some(std::mem::take(&mut sep.body)); + let (orders, unset) = d.stacking_orders(); + assert_eq!((orders.last().copied(), unset), (Some(u32::MAX), 0), "the separator's order is seen"); + let mut made = paras(&d)[0].objects[0].clone(); + if let InlineObject::Shape { float, .. } = &mut made { + float.z = 0; + } + let Some(Block::Para(p)) = d.body.first_mut().map(std::sync::Arc::make_mut) else { panic!("a paragraph") }; + p.insert_object(0, made, &Default::default()).unwrap(); + let files = unzip(&craft_docx::write(&d).unwrap()); + let body: Vec = attr_values(&text(&files, "word/document.xml"), "wp:anchor", "relativeHeight").iter().map(|v| v.parse().unwrap()).collect(); + let seps: Vec = attr_values(&text(&files, "word/footnotes.xml"), "wp:anchor", "relativeHeight").iter().map(|v| v.parse().unwrap()).collect(); + assert_eq!(seps.len(), 1, "the separator's shape is saved"); + assert!(body[0] > seps[0] && seps[0] > body[1], "new over the separator's over Low: {body:?} {seps:?}"); + // So does one in a building block of the glossary, saved with the same numbering (wordcraft#2638 review). + let mut d = read_body(&format!("{}", shape("Low", 7))); + let entry = read_body(&format!("{}", shape("Entry", u32::MAX))); + d.glossary.push(craft_wordmodel::BuildingBlock { name: "Entry".into(), blocks: entry.body, ..Default::default() }); + assert_eq!(d.stacking_orders().0.last().copied(), Some(u32::MAX), "the glossary's order is seen"); + // With room above, the stored orders are written as they are. + let room = read_body(&format!("{}", shape("Top", u32::MAX - 5))); + let back = craft_docx::read(&craft_docx::write(&room).unwrap()).unwrap(); + assert_eq!(stacking(&back), [Some(u32::MAX - 4)]); +} + +/// wordcraft#2638 (review): `relativeHeight="0"`, the lowest a file can give, is a stacking order like +/// any other: under a shape with 1, not taken for an object without one (made here, which goes +/// over everything), and saved back as 0. +#[test] +fn a_stacking_order_of_zero_stays_the_lowest() { + let shape = |name: &str, z: u32| { + anchored_wsp(name, 127000, 127000, r#""#, "", "") + .replace(r#"relativeHeight="1""#, &format!(r#"relativeHeight="{z}""#)) + }; + let d = read_body(&format!("{}{}", shape("Bottom", 0), shape("Top", 1))); + let floats: Vec<_> = paras(&d)[0].objects.iter().filter_map(|o| o.frame().map(|(_, _, f)| f.clone())).collect(); + assert_eq!(floats.iter().map(|f| f.z).collect::>(), [1, 2]); + assert!(floats[0].stack() < floats[1].stack(), "0 stacks under 1"); + assert!(floats[1].stack() < craft_wordmodel::para::Float::default().stack(), "an object made here goes over both"); + let files = unzip(&craft_docx::write(&d).unwrap()); + assert_eq!(attr_values(&text(&files, "word/document.xml"), "wp:anchor", "relativeHeight"), ["0", "1"]); +} + /// #611: diagrams drawn with Word's shapes were mostly missing. Connectors and boxes take their /// outline and fill from the theme through their style references, or name preset and scheme /// colours; all of these are read, with line widths and arrowheads. diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 8f3bbdd0b..16f61be6b 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -23,18 +23,14 @@ fn paras(doc: &Document) -> Vec<&Paragraph> { doc.body.iter().filter_map(|b| b.as_para()).collect() } -/// `doc` with the stacking order of its body's floating objects cleared: saving gives objects -/// without one an order (`relativeHeight`), so tests about other properties compare without it. +/// `doc` without the stacking order a save gives objects made in the app (`relativeHeight`, +/// read back as `Float::z`), to compare them with the objects as made. fn unstacked(mut doc: Document) -> Document { - for b in &mut doc.body { - if let Block::Para(p) = std::sync::Arc::make_mut(b) { - for o in &mut p.objects { - if let Some(f) = o.float_mut() { - f.z = 0; - } - } + doc.objects_mut(&|_| true, &mut |o| { + if let Some(f) = o.float_mut() { + f.z = 0; } - } + }); doc } diff --git a/crates/wordmodel/src/document.rs b/crates/wordmodel/src/document.rs index 0a1cc43bd..5b0d1ae1a 100644 --- a/crates/wordmodel/src/document.rs +++ b/crates/wordmodel/src/document.rs @@ -721,6 +721,37 @@ impl Document { max.saturating_add(1).max(1) } + /// The stacking orders ([`para::Float::z`]) the floating objects of every story have, and how + /// many floating objects have none (objects made in the app). Every story saved with the document + /// counts: the body, the parts, the note separators and the glossary's building blocks (saved + /// with the same numbering, so none of their orders is left out when it is renumbered). + pub fn stacking_orders(&self) -> (std::collections::BTreeSet, usize) { + let (mut orders, mut unset) = (std::collections::BTreeSet::new(), 0usize); + let separators = [&self.footnote_separators, &self.endnote_separators] + .into_iter() + .flat_map(|s| [&s.separator, &s.continuation_separator, &s.continuation_notice]) + .flatten(); + let stories = std::iter::once(&self.body) + .chain(self.parts.values().map(|p| &p.blocks)) + .chain(separators) + .chain(self.glossary.iter().map(|g| &g.blocks)); + for blocks in stories { + for b in blocks { + crate::walk::each_para(b, 0, &mut |p| { + for (_, _, f) in p.objects.iter().filter_map(InlineObject::frame).filter(|(_, _, f)| f.wrap != para::Wrap::Inline) { + match f.z { + 0 => unset = unset.saturating_add(1), + z => { + orders.insert(z); + } + } + } + }); + } + } + (orders, unset) + } + /// Change every inline object in every story (paragraphs inside tables too), copying only /// the blocks whose paragraphs `wants` says have objects to change. pub fn objects_mut(&mut self, wants: &dyn Fn(&InlineObject) -> bool, f: &mut dyn FnMut(&mut InlineObject)) { diff --git a/crates/wordmodel/src/para/float.rs b/crates/wordmodel/src/para/float.rs index 315e0e042..b13db3533 100644 --- a/crates/wordmodel/src/para/float.rs +++ b/crates/wordmodel/src/para/float.rs @@ -182,8 +182,11 @@ pub struct Float { /// off, any VML shape in a header or footer): a picture watermark in a header that starts with a table. #[serde(skip_serializing_if = "is_false")] pub outside_cell: bool, - /// Stacking order among floating objects (`wp:anchor/@relativeHeight`): higher is drawn on - /// top. 0 = unknown, drawn in document order (objects inserted here, or from other formats). + /// Stacking order among the floating objects of its layer, one more than the file's + /// (`wp:anchor/@relativeHeight`, ODF's `draw:z-index`), so a file's lowest, 0, is 1: a higher + /// one is drawn over a lower one, whatever their order in the text, as Word does. 0 = none + /// (objects inserted by the app, or from other formats): stacks above every object that has + /// one, in text order. #[serde(skip_serializing_if = "is_zero_u32")] pub z: u32, /// The page a drawing anchored to a page is on, from 1 (ODF's `text:anchor-page-number`, @@ -242,6 +245,12 @@ fn is_zero_u32(v: &u32) -> bool { } impl Float { + /// Where it stacks among the floating objects of its layer: see [`Float::z`]. None (0) is + /// past every stored order, `u32::MAX` included. + pub fn stack(&self) -> u64 { + if self.z > 0 { u64::from(self.z) } else { u64::from(u32::MAX) + 1 } + } + /// The rotation and flips, the angle normalised to `0..360` (a NaN angle is no rotation). pub fn spin(&self) -> crate::geom::Spin { crate::geom::Spin::new(self.rot, self.flip_h, self.flip_v) From 2632c41eb4d4a0d07eda816b7a17eb436a791954 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 20:47:25 +0100 Subject: [PATCH 015/209] craft-drawingml: box ListStyle::default_level to keep ListStyle small Co-Authored-By: Claude Sonnet 5.5 --- crates/drawingml/src/part.rs | 2 +- crates/drawingml/src/read.rs | 2 +- crates/drawingml/src/text.rs | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/drawingml/src/part.rs b/crates/drawingml/src/part.rs index a251d4f3c..af2f6b736 100644 --- a/crates/drawingml/src/part.rs +++ b/crates/drawingml/src/part.rs @@ -354,7 +354,7 @@ mod tests { let xml = format!(r#"{}"#, w.finish()); let back = crate::read::text_body(&mut Part::new(), parse(xml.as_bytes()).unwrap().child(craft_xml::Local("txBody")).unwrap()); assert_eq!(back.list_style, ls); - let empty = ListStyle { default_level: Some(Default::default()), ..Default::default() }; + let empty = ListStyle { default_level: Some(Box::default()), ..Default::default() }; assert!(!empty.is_empty()); let mut w = craft_xml::Writer::new(); crate::write::list_style(&mut w, &mut Part::new(), "p:defaultTextStyle", &empty); diff --git a/crates/drawingml/src/read.rs b/crates/drawingml/src/read.rs index 012c20552..f26bf2073 100644 --- a/crates/drawingml/src/read.rs +++ b/crates/drawingml/src/read.rs @@ -625,7 +625,7 @@ pub fn list_style(src: &mut dyn Source, ls: &El) -> ListStyle { let n = e.local(); if n == "defPPr" { let (_, para, run) = ppr(src, e); - out.default_level = Some(LevelStyle { para, run: run.unwrap_or_default() }); + out.default_level = Some(Box::new(LevelStyle { para, run: run.unwrap_or_default() })); continue; } let Some(l) = n.strip_prefix("lvl").and_then(|r| r.strip_suffix("pPr")).and_then(|d| d.parse::().ok()) else { continue }; diff --git a/crates/drawingml/src/text.rs b/crates/drawingml/src/text.rs index 8fe0244e1..135d070f9 100644 --- a/crates/drawingml/src/text.rs +++ b/crates/drawingml/src/text.rs @@ -707,9 +707,11 @@ pub struct ListStyle { /// Levels 1–9 (index 0–8); `None` = not specified. pub levels: Vec>, /// The default paragraph style (`a:defPPr`) that sits under levels 1–9; `None` = the list - /// style has no `a:defPPr`. + /// style has no `a:defPPr`. Boxed: it sits inline in every `ListStyle`, hence in every text + /// body and shape, and the recursive group readers (pptx) would otherwise overflow the stack + /// of a debug build on Windows (1 MiB main thread) at the nesting depth the tests cut at. #[serde(skip_serializing_if = "Option::is_none")] - pub default_level: Option, + pub default_level: Option>, } impl ListStyle { @@ -763,6 +765,14 @@ mod tests { use super::*; use crate::color::Rgba; + /// `ListStyle` sits inline in every `TextBody`/`Shape`, which the pptx group reader holds on the + /// stack once per nesting level. Keep it small (the `defPPr` level is boxed) so a debug build + /// does not overflow the 1 MiB Windows main-thread stack. + #[test] + fn list_style_stays_small() { + assert!(std::mem::size_of::() <= 64, "{}", std::mem::size_of::()); + } + #[test] fn alternate_language_inherits_and_native_input_is_bounded() { let mut child = RunProps::default(); From d5b4dbef8827b7a5ef9c00f113ee2457359a9a14 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 20:51:33 +0100 Subject: [PATCH 016/209] craft-drawingml: tighten ListStyle size guard, add deep group stack test Co-Authored-By: Claude Sonnet 5.5 --- crates/drawingml/src/text.rs | 6 +++--- crates/pptx/tests/read.rs | 39 +++++++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/crates/drawingml/src/text.rs b/crates/drawingml/src/text.rs index 135d070f9..dc34cf10f 100644 --- a/crates/drawingml/src/text.rs +++ b/crates/drawingml/src/text.rs @@ -766,11 +766,11 @@ mod tests { use crate::color::Rgba; /// `ListStyle` sits inline in every `TextBody`/`Shape`, which the pptx group reader holds on the - /// stack once per nesting level. Keep it small (the `defPPr` level is boxed) so a debug build - /// does not overflow the 1 MiB Windows main-thread stack. + /// stack once per nesting level. Keep it at `Vec` (24) + `Option` (8) bytes (the `defPPr` + /// level is boxed) so a debug build does not overflow the 1 MiB Windows main-thread stack. #[test] fn list_style_stays_small() { - assert!(std::mem::size_of::() <= 64, "{}", std::mem::size_of::()); + assert!(std::mem::size_of::() <= 32, "{}", std::mem::size_of::()); } #[test] diff --git a/crates/pptx/tests/read.rs b/crates/pptx/tests/read.rs index 5fe263f02..fd03d2d98 100644 --- a/crates/pptx/tests/read.rs +++ b/crates/pptx/tests/read.rs @@ -217,23 +217,52 @@ fn other_files_are_refused() { assert!(craft_pptx::import_theme(&doc).is_err()); } -/// Groups nested past 32 levels are cut there; the rest of the slide is read. -#[test] -fn deep_group_nesting_is_cut() { +/// A slide whose group tree is 40 levels deep, with a shape after it. +fn deep_group_deck() -> Vec { let open = r#""#; let tree = format!("{}{}{}", open.repeat(40), sp(4, "Deep", "", XFRM, "x"), "".repeat(40)); let pres = format!(r#""#); let slide = format!(r#"{tree}{}"#, sp(5, "Top", "", XFRM, "y")); - let bytes = package( + package( &[ ("ppt/presentation.xml", CT_PRES, pres), ("ppt/slides/slide1.xml", "application/vnd.openxmlformats-officedocument.presentationml.slide+xml", slide), ], &[("", &[("officeDocument", "ppt/presentation.xml")]), ("ppt/presentation.xml", &[("slide", "slides/slide1.xml")])], - ); + ) +} + +/// Groups nested past 32 levels are cut there; the rest of the slide is read. +#[test] +fn deep_group_nesting_is_cut() { + let bytes = deep_group_deck(); let p = craft_pptx::import(&bytes, &Plain).unwrap(); let mut depth = 0; craft_pptx::walk(&p.slides[0].shapes, &mut |_, d| depth = depth.max(d)); assert_eq!(depth, 32, "33 levels of groups, the last one empty"); assert!(p.slides[0].shapes.iter().any(|s| s.name == "Top")); } + +/// The group reader recurses once per nesting level and keeps a `Shape` per level on the stack, so +/// anything that makes `Shape` (or `TextBody`/`ListStyle` inside it) bigger shows up here first. +/// A debug build is the worst case, so import the deepest accepted tree on a stack that is small +/// but ample for the current size: measured in a Linux debug build, this import needs about +/// 1.3 MiB, and about 2 MiB when `ListStyle` holds its `defPPr` level inline. Fails with a stack +/// overflow (the process aborts) when a type on the recursive path grows by value. Linux only: +/// frame sizes differ per platform, and the Windows CI runs `deep_group_nesting_is_cut` on the +/// default stack, where that growth first showed up. +#[test] +#[cfg(target_os = "linux")] +fn deep_group_import_fits_a_small_stack() { + const STACK: usize = 1664 * 1024; + std::thread::Builder::new() + .stack_size(STACK) + .spawn(|| { + let bytes = deep_group_deck(); + let p = craft_pptx::import(&bytes, &Plain).unwrap(); + assert!(p.slides[0].shapes.iter().any(|s| s.name == "Top")); + }) + .unwrap() + .join() + .unwrap(); +} From e7627d396c3ee4c10dd9ba0da6e6d4860c1959e8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 22:03:46 +0100 Subject: [PATCH 017/209] craft-drawingml: keep the default namespace on kept fragments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A part written with a default namespace (``, a slide with `xmlns="…/presentationml/2006/main"`) has unprefixed names. `keep_with` copied only prefixed declarations, so a kept fragment (theme custClrLst, extLst, objectDefaults, effect 3-D, slide/layout/master extLst) lost its namespace when written under a root that binds no default. A kept fragment now gets the part root's `xmlns` when one of its unprefixed names is not bound inside it; prefixed fragments are written as before. Co-Authored-By: Claude Opus 5.5 --- crates/drawingml/src/xmlx.rs | 50 +++++++++++++++++-- crates/pptx/tests/write.rs | 96 ++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/crates/drawingml/src/xmlx.rs b/crates/drawingml/src/xmlx.rs index d31616b39..27fa58dc7 100644 --- a/crates/drawingml/src/xmlx.rs +++ b/crates/drawingml/src/xmlx.rs @@ -156,7 +156,20 @@ fn used_prefixes(el: &El) -> HashSet<&str> { used } -/// Serialize `el`, declaring every prefix it uses that the source root declared. +/// Does `el` or a descendant have an unprefixed name that no `xmlns` inside `el` binds? Such a +/// name is in the default namespace of the part it was read from. Attributes never are. +fn uses_outer_default(el: &El) -> bool { + fn rec(e: &El, depth: usize) -> bool { + if depth > MAX_DEPTH || e.attrs.iter().any(|(k, _)| k == "xmlns") { + return false; + } + !e.name.contains(':') || e.elements().any(|c| rec(c, depth + 1)) + } + rec(el, 0) +} + +/// Serialize `el`, declaring every prefix it uses that the source root declared, and the +/// root's default namespace when an unprefixed name in `el` depends on it. pub fn keep_xml(el: &El, ns: &[(String, String)]) -> String { keep_with(el, &[ns], None) } @@ -166,11 +179,15 @@ pub fn keep_xml(el: &El, ns: &[(String, String)]) -> String { /// [`MAX_COPIED_NS_BYTES`]). pub fn keep_with(el: &El, decls: &[&[(String, String)]], budget: Option<&Cell>) -> String { let used = used_prefixes(el); + let default = uses_outer_default(el); let mut have: HashSet<&str> = el.attrs.iter().map(|(a, _)| a.as_str()).collect(); let mut extra = Vec::new(); for (k, v) in decls.iter().flat_map(|d| d.iter()) { - let p = k.strip_prefix("xmlns:").unwrap_or(""); - if !p.is_empty() && used.contains(p) && v.len() <= MAX_NS_VALUE && !have.contains(k.as_str()) { + let wanted = match k.strip_prefix("xmlns:") { + Some(p) => !p.is_empty() && used.contains(p), + None => k == "xmlns" && default, + }; + if wanted && v.len() <= MAX_NS_VALUE && !have.contains(k.as_str()) { let cost = k.len() + v.len(); if let Some(b) = budget { if b.get() < cost { @@ -324,6 +341,33 @@ mod tests { assert!(!has_rel_refs(root.child(Local("z")).unwrap())); } + /// A part written with a default namespace (``): an unprefixed + /// fragment, or one with an unprefixed descendant, carries that namespace; a prefixed one + /// and one that declares its own default are written as before. + #[test] + fn a_kept_fragment_carries_the_default_namespace_its_names_depend_on() { + let root = parse( + br#""#, + ) + .unwrap(); + let ns = ns_decls(&root); + let kid = |n: &str| root.child(Local(n)).unwrap(); + assert_eq!(keep_xml(kid("custClrLst"), &ns), r#""#); + assert_eq!(keep_xml(kid("y"), &ns), r#""#, "prefixed: as before"); + assert_eq!(keep_xml(kid("z"), &ns), r#""#, "an unprefixed descendant"); + assert_eq!(keep_xml(kid("own"), &ns), r#""#, "its own default wins"); + assert_eq!(keep_xml(kid("u"), &ns), r#""#, "an undeclared default stays"); + // A part with no default namespace: unprefixed names stay in no namespace. + let plain = parse(br#""#).unwrap(); + assert_eq!(keep_xml(plain.child(Local("x")).unwrap(), &ns_decls(&plain)), ""); + // The default declaration spends the budget like the others. + let budget = Cell::new(3); + assert_eq!( + keep_with(kid("custClrLst"), &[&ns], Some(&budget)), + r#""# + ); + } + #[test] fn inner_declarations_are_found_and_clashing_ones_dropped() { let root = parse(br#""#).unwrap(); diff --git a/crates/pptx/tests/write.rs b/crates/pptx/tests/write.rs index bec5fb239..b6315375a 100644 --- a/crates/pptx/tests/write.rs +++ b/crates/pptx/tests/write.rs @@ -1,5 +1,7 @@ //! The writer, read back with the reader. +mod common; + use std::sync::Arc; use craft_pptx::table::{Cell, Table}; @@ -193,6 +195,100 @@ fn reads_the_theme_of_a_pptx_and_of_a_thmx() { } } +const NS_A: &str = "http://schemas.openxmlformats.org/drawingml/2006/main"; +const NS_P: &str = "http://schemas.openxmlformats.org/presentationml/2006/main"; +const NS_THM15: &str = "http://schemas.microsoft.com/office/thememl/2012/main"; + +/// The namespace of every element named `local` in `xml`, resolved through the declarations in +/// scope (`""` for none). +fn namespaces_of(xml: &str, local: &str) -> Vec { + fn rec(e: &craft_xml::El, scope: &mut Vec<(String, String)>, local: &str, out: &mut Vec) { + let n = scope.len(); + scope.extend(e.ns_decls().map(|(p, u)| (p.to_string(), u.to_string()))); + if e.local() == local { + out.push(scope.iter().rev().find(|(p, _)| p == e.prefix()).map(|(_, u)| u.clone()).unwrap_or_default()); + } + for c in e.elements() { + rec(c, scope, local, out); + } + scope.truncate(n); + } + let root = craft_xml::Options::default().strict().parse(xml.as_bytes()).unwrap_or_else(|e| panic!("not well-formed: {e}\n{xml}")); + let mut out = vec![]; + rec(&root, &mut vec![], local, &mut out); + out +} + +/// A theme with a custom colour list and a theme family id, with the `a:` prefix. +const PREFIXED_THEME: &str = r#""#; + +/// The deck with its theme part replaced by `theme`. +fn with_theme(theme: &str) -> Vec { + let pptx = craft_pptx::export(&deck(), &Plain).unwrap(); + common::rewrite(&pptx, |name, data| Some(if name == "ppt/theme/theme1.xml" { theme.as_bytes().to_vec() } else { data.to_vec() }), &[]) +} + +/// A theme written with a default namespace instead of the `a:` prefix: the parts kept as +/// written (custom colours, theme family id) stay in the DrawingML namespace on save. +#[test] +fn a_default_namespace_theme_keeps_its_kept_parts_in_the_drawingml_namespace() { + let theme = PREFIXED_THEME.replace("").expect("themeElements")..].to_string(); + let twice = craft_pptx::export(&craft_pptx::import(&once, &Plain).unwrap(), &Plain).unwrap(); + assert_eq!(kept(&part(&twice, "ppt/theme/theme1.xml")), kept(&xml)); +} + +/// The same theme with the `a:` prefix: the kept parts are written as before (each declaring +/// the `a:` it uses), with no default namespace added. +#[test] +fn a_prefixed_theme_keeps_its_kept_parts_unchanged() { + let p = craft_pptx::import(&with_theme(PREFIXED_THEME), &Plain).unwrap(); + let xml = part(&craft_pptx::export(&p, &Plain).unwrap(), "ppt/theme/theme1.xml"); + assert!(!xml.contains(r#" xmlns=""#), "{xml}"); + let a = format!(r#"xmlns:a="{NS_A}""#); + let kept = format!( + r#""# + ); + assert!(xml.contains(&kept), "{xml}"); +} + +/// A slide written with a default PresentationML namespace: its kept extension list stays in +/// that namespace under the writer's `p:extLst`. +#[test] +fn a_default_namespace_slide_keeps_its_extensions_in_the_presentationml_namespace() { + let pptx = craft_pptx::export(&deck(), &Plain).unwrap(); + let ext = r#""#; + let doc = common::rewrite( + &pptx, + |name, data| { + if name != "ppt/slides/slide1.xml" { + return Some(data.to_vec()); + } + let xml = String::from_utf8(data.to_vec()).unwrap(); + assert!(xml.contains("") && !xml.contains("extLst"), "{xml}"); + Some(xml.replacen("", &format!("{ext}")).into_bytes()) + }, + &[], + ); + let p = craft_pptx::import(&doc, &Plain).unwrap(); + let xml = part(&craft_pptx::export(&p, &Plain).unwrap(), "ppt/slides/slide1.xml"); + assert!(xml.contains("{11111111-2222-3333-4444-555555555555}"), "{xml}"); + assert_eq!(namespaces_of(&xml, "ext"), [NS_P], "{xml}"); + assert_eq!(namespaces_of(&xml, "data"), ["urn:test"], "{xml}"); +} + #[test] fn templates_and_shows_differ_in_the_main_content_type() { let p = deck(); From 3b02f3ee6c9b3d7ce3bddbfd87c4a1436278fa4f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 22:11:54 +0100 Subject: [PATCH 018/209] craft-pptx: test kept 3-D and layout/master extensions in default-namespace parts Extends the default-namespace theme test with an effect style holding scene3d and sp3d, and adds a layout and master case whose unprefixed extLst/ext is written back through ext_children. To be precise about the previous commit: theme effect styles and shape spPr 3-D kept from a part with a default namespace now keep that namespace. Text bodyPr and group spPr 3-D without a prefix are still dropped on save, as before (valid_raw3d requires the a: prefix); that is out of scope here. Co-Authored-By: Claude Opus 5.5 --- crates/pptx/tests/write.rs | 49 +++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/crates/pptx/tests/write.rs b/crates/pptx/tests/write.rs index b6315375a..46f7513e7 100644 --- a/crates/pptx/tests/write.rs +++ b/crates/pptx/tests/write.rs @@ -220,7 +220,7 @@ fn namespaces_of(xml: &str, local: &str) -> Vec { } /// A theme with a custom colour list and a theme family id, with the `a:` prefix. -const PREFIXED_THEME: &str = r#""#; +const PREFIXED_THEME: &str = r#""#; /// The deck with its theme part replaced by `theme`. fn with_theme(theme: &str) -> Vec { @@ -244,6 +244,9 @@ fn a_default_namespace_theme_keeps_its_kept_parts_in_the_drawingml_namespace() { assert!(!ns.is_empty() && ns.iter().all(|n| n == NS_A), "{local}: {ns:?}\n{xml}"); } assert_eq!(namespaces_of(&xml, "themeFamily"), [NS_THM15], "{xml}"); + // The effect style's 3-D is kept as written too. + assert_eq!(namespaces_of(&xml, "scene3d"), [NS_A], "{xml}"); + assert_eq!(namespaces_of(&xml, "sp3d"), [NS_A], "{xml}"); // The second save writes the same kept parts. let kept = |x: &str| x[x.find("").expect("themeElements")..].to_string(); let twice = craft_pptx::export(&craft_pptx::import(&once, &Plain).unwrap(), &Plain).unwrap(); @@ -264,31 +267,55 @@ fn a_prefixed_theme_keeps_its_kept_parts_unchanged() { assert!(xml.contains(&kept), "{xml}"); } -/// A slide written with a default PresentationML namespace: its kept extension list stays in -/// that namespace under the writer's `p:extLst`. -#[test] -fn a_default_namespace_slide_keeps_its_extensions_in_the_presentationml_namespace() { +/// The deck with part `name` (root `root`) given a default PresentationML namespace and an +/// unprefixed extension list, saved again: the part as written. +fn default_namespace_extensions(name: &str, root: &str) -> String { let pptx = craft_pptx::export(&deck(), &Plain).unwrap(); let ext = r#""#; + let (open, close) = (format!("<{root} "), format!("")); let doc = common::rewrite( &pptx, - |name, data| { - if name != "ppt/slides/slide1.xml" { + |n, data| { + if n != name { return Some(data.to_vec()); } let xml = String::from_utf8(data.to_vec()).unwrap(); - assert!(xml.contains("") && !xml.contains("extLst"), "{xml}"); - Some(xml.replacen("", &format!("{ext}")).into_bytes()) + assert!(xml.contains(&open) && xml.contains(&close) && !xml.contains("extLst"), "{xml}"); + Some(xml.replacen(&open, &format!(r#"<{root} xmlns="{NS_P}" "#), 1).replace(&close, &format!("{ext}{close}")).into_bytes()) }, &[], ); let p = craft_pptx::import(&doc, &Plain).unwrap(); - let xml = part(&craft_pptx::export(&p, &Plain).unwrap(), "ppt/slides/slide1.xml"); + part(&craft_pptx::export(&p, &Plain).unwrap(), name) +} + +/// The namespaces of the `ext` elements in `xml` other than DrawingML's `a:ext` (an extent). +fn extension_namespaces(xml: &str) -> Vec { + namespaces_of(xml, "ext").into_iter().filter(|n| n != NS_A).collect() +} + +/// A slide written with a default PresentationML namespace: its kept extension list stays in +/// that namespace under the writer's `p:extLst`. +#[test] +fn a_default_namespace_slide_keeps_its_extensions_in_the_presentationml_namespace() { + let xml = default_namespace_extensions("ppt/slides/slide1.xml", "p:sld"); assert!(xml.contains("{11111111-2222-3333-4444-555555555555}"), "{xml}"); - assert_eq!(namespaces_of(&xml, "ext"), [NS_P], "{xml}"); + assert_eq!(extension_namespaces(&xml), [NS_P], "{xml}"); assert_eq!(namespaces_of(&xml, "data"), ["urn:test"], "{xml}"); } +/// The same for a layout and a master, whose kept extension list is written back through +/// `ext_children`. +#[test] +fn a_default_namespace_layout_and_master_keep_their_extensions_in_the_presentationml_namespace() { + for (name, root) in [("ppt/slideLayouts/slideLayout1.xml", "p:sldLayout"), ("ppt/slideMasters/slideMaster1.xml", "p:sldMaster")] { + let xml = default_namespace_extensions(name, root); + assert!(xml.contains("{11111111-2222-3333-4444-555555555555}"), "{name}: {xml}"); + assert_eq!(extension_namespaces(&xml), [NS_P], "{name}: {xml}"); + assert_eq!(namespaces_of(&xml, "data"), ["urn:test"], "{name}: {xml}"); + } +} + #[test] fn templates_and_shows_differ_in_the_main_content_type() { let p = deck(); From e70cceae3377e0fadf12ea3c35030da73a5091f2 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 23:23:49 +0100 Subject: [PATCH 019/209] DDS: pin the writer's and reader's output as SHA-256 digests Digests of uncompressed, BC1, BC2 and BC3 textures, with and without mip chains, from deterministic odd-sized pictures (ramps, noise, flat tiles, alpha at 0, 255 and in between) in RGBA, RGB, gray with alpha and gray, and of decoding those files and hand-built ones in the other stored layouts. The table is filled from the unchanged code; on a mismatch the test prints every digest it computed. --- crates/image-codecs/tests/it/dds_oracle.rs | 520 +++++++++++++++++++++ crates/image-codecs/tests/it/main.rs | 1 + 2 files changed, 521 insertions(+) create mode 100644 crates/image-codecs/tests/it/dds_oracle.rs diff --git a/crates/image-codecs/tests/it/dds_oracle.rs b/crates/image-codecs/tests/it/dds_oracle.rs new file mode 100644 index 000000000..05d000a73 --- /dev/null +++ b/crates/image-codecs/tests/it/dds_oracle.rs @@ -0,0 +1,520 @@ +//! Byte-for-byte oracle for the DDS reader and writer: SHA-256 digests of what the writer makes +//! for uncompressed, BC1, BC2 and BC3 textures, with and without mip chains, from deterministic +//! synthetic images, and of what the reader decodes from those files and from hand-built files +//! in the other stored layouts. The digests were taken from the code before the block encoder and +//! decoder were sped up; any change to them is a change of output. +//! +//! On a mismatch the test prints every digest it computed in the form of [`PINNED`], so a new +//! table can be pasted in once a change of output is intended. + +use craft_image_codecs::{ChannelLayout, DdsCompression, DdsOptions, EncodeOptions, Format, Image, decode, encode}; +use sha2::{Digest, Sha256}; + +/// `(case, SHA-256)`, taken from the code before the speed-ups. +const PINNED: &[(&str, &str)] = &[ + ("Rgba-517x389-none-mips0-encode", "e2aee5dcc86cc36e9735cb75e61852324210960092bfb2f2aec6cee9ba54ad16"), + ("Rgba-517x389-none-mips0-decode", "e0d9e319f260fa29481495313cfa8c3d224a27ad311e7aabd32f7cda8dfb81d2"), + ("Rgba-517x389-none-mips1-encode", "7611e8174d32962c9643e8f3208b8f30eba2cb85cfe16538c4321e419f3a98ce"), + ("Rgba-517x389-none-mips1-decode", "e0d9e319f260fa29481495313cfa8c3d224a27ad311e7aabd32f7cda8dfb81d2"), + ("Rgba-517x389-bc1-mips0-encode", "b6883d729a1eebbdc96176c9264754378d4f7f6bc19d355a12419c0ccfd3ac2a"), + ("Rgba-517x389-bc1-mips0-decode", "e23704d5f72ba4f2c63d815d5bff3d4c79433fa38122e1e21f1003fa1b12476a"), + ("Rgba-517x389-bc1-mips1-encode", "554ed42ba6b7c206f779324670629538d8ce12d85ef495ad36954aae2b952917"), + ("Rgba-517x389-bc1-mips1-decode", "e23704d5f72ba4f2c63d815d5bff3d4c79433fa38122e1e21f1003fa1b12476a"), + ("Rgba-517x389-bc2-mips0-encode", "473f389b464d2f74b05d44ab2d8c31d4d4cd64648d389e4fe27c27491003821b"), + ("Rgba-517x389-bc2-mips0-decode", "9d81c15d7f5f0dbc897063726fac6e2f3863818346e1c878326c22ed332ae328"), + ("Rgba-517x389-bc2-mips1-encode", "1eb8c1a73cb4e6965c60a5ca4322fb66f071cad7439d97d77dbea1a46231724a"), + ("Rgba-517x389-bc2-mips1-decode", "9d81c15d7f5f0dbc897063726fac6e2f3863818346e1c878326c22ed332ae328"), + ("Rgba-517x389-bc3-mips0-encode", "0b133915d5ac0376f8b4425cb2dd4a1ace609cf7c1d635be0ff46a37529363e2"), + ("Rgba-517x389-bc3-mips0-decode", "9c5aeecab34af4c04dd603cf5c379060e6615c07df05ae4e93058bca13a61bc9"), + ("Rgba-517x389-bc3-mips1-encode", "130e5657e870ee5c03bcfcc2c5e2befbc8a67fc0df1b7e7b71dbb67cc3f0b5e7"), + ("Rgba-517x389-bc3-mips1-decode", "9c5aeecab34af4c04dd603cf5c379060e6615c07df05ae4e93058bca13a61bc9"), + ("Rgb-517x389-none-mips0-encode", "6df9ec01a5ae0a8e0fe3b563a7ff670477446682cadd21c24ca5e0cff2b41ed8"), + ("Rgb-517x389-none-mips0-decode", "7f181edc81e2e38d56c2ed5c81edf480d52caa746e0bef90f0070e171ead89d1"), + ("Rgb-517x389-none-mips1-encode", "5a48af2cfa7d48455fe49105fc5d0173c6f4fd8187483af923f0af2fbf0af9ef"), + ("Rgb-517x389-none-mips1-decode", "7f181edc81e2e38d56c2ed5c81edf480d52caa746e0bef90f0070e171ead89d1"), + ("Rgb-517x389-bc1-mips0-encode", "1e8bf3d0ba485d0892ce771698868bf7029d2c4e63057f327ae3ab39bc57cd4a"), + ("Rgb-517x389-bc1-mips0-decode", "96feb4a7a3b7416ebc0d5e42271a9d6304131e6cbed25f745d3eef6a40279f8d"), + ("Rgb-517x389-bc1-mips1-encode", "76f866a8bbb0b3cad335c604cba97d173d01dcbfff4c8930a1d8af7c60924494"), + ("Rgb-517x389-bc1-mips1-decode", "96feb4a7a3b7416ebc0d5e42271a9d6304131e6cbed25f745d3eef6a40279f8d"), + ("Rgb-517x389-bc2-mips0-encode", "a8eaab96cf9c6f7926526069ed71f5df8c7f7d43f3ef40a85f1403639c09ac97"), + ("Rgb-517x389-bc2-mips0-decode", "8e00407a44922477b8d7c05b371f05c18ac3471095990b44aba438bf39db57b2"), + ("Rgb-517x389-bc2-mips1-encode", "d6fee09087b9a4c9a055051e7aa212a9a0a3b2660e1dfa49697c6fc9754a9aae"), + ("Rgb-517x389-bc2-mips1-decode", "8e00407a44922477b8d7c05b371f05c18ac3471095990b44aba438bf39db57b2"), + ("Rgb-517x389-bc3-mips0-encode", "2e177ad42c89f6662ebf2b834b691e10bfaea32a565b740e1bd48c7e4b39cb68"), + ("Rgb-517x389-bc3-mips0-decode", "8e00407a44922477b8d7c05b371f05c18ac3471095990b44aba438bf39db57b2"), + ("Rgb-517x389-bc3-mips1-encode", "0b53960adcf0844b47ed5927862901c95fb632af4e85cd94b5ceae59eccd78cd"), + ("Rgb-517x389-bc3-mips1-decode", "8e00407a44922477b8d7c05b371f05c18ac3471095990b44aba438bf39db57b2"), + ("GrayA-517x389-none-mips0-encode", "3f8e56201cb76990b3e25b62631666ac113a5340877acd57cdcb1c229127d3f4"), + ("GrayA-517x389-none-mips0-decode", "43503ba84a7d7579b1ebbe6ba522146f5e63f0116f8dd0eba095364cfba37c5d"), + ("GrayA-517x389-none-mips1-encode", "6282214743c82e3f74f2347c859ecfd79447cfbc87a2016a238e714f2d150b87"), + ("GrayA-517x389-none-mips1-decode", "43503ba84a7d7579b1ebbe6ba522146f5e63f0116f8dd0eba095364cfba37c5d"), + ("GrayA-517x389-bc1-mips0-encode", "32d7065b40986561349f0d76db148b1b62afedb1ce8be6a4e0b209c60c9d861f"), + ("GrayA-517x389-bc1-mips0-decode", "1cc1cbd355033b5674cc1000b879d4dc8cbb4b53b8dd38d90d8b439550e2be7c"), + ("GrayA-517x389-bc1-mips1-encode", "4b5e5c9db8ec9847b55e0c51b2e1c1cb41bb277517ef5a7bf59ace7581a7696a"), + ("GrayA-517x389-bc1-mips1-decode", "1cc1cbd355033b5674cc1000b879d4dc8cbb4b53b8dd38d90d8b439550e2be7c"), + ("GrayA-517x389-bc2-mips0-encode", "91fd37653f090668328d8feff0407c5998ef2ab96d220bd49e311faf7d768e00"), + ("GrayA-517x389-bc2-mips0-decode", "ac73339602e5062f78cf76694bde0a4ca538b355170eb424f0bd1cf7d2c12096"), + ("GrayA-517x389-bc2-mips1-encode", "05d77863968b0086ddf4a1c21ccbc8e1f2d9638ed4dffd114107a1d793ea3311"), + ("GrayA-517x389-bc2-mips1-decode", "ac73339602e5062f78cf76694bde0a4ca538b355170eb424f0bd1cf7d2c12096"), + ("GrayA-517x389-bc3-mips0-encode", "bb4057b77e0fa2959d6e79b77ff67090c9dd4d08f3c105b57ebbaea214ffc8c7"), + ("GrayA-517x389-bc3-mips0-decode", "fdcbd158c32404d09cf1ee38005b2870882e620bcd67c77332d492d07099d665"), + ("GrayA-517x389-bc3-mips1-encode", "65b2021ea2a385b349d2a6f3a44305e3655f35f3bee39fba5d177bb8c7a5d66a"), + ("GrayA-517x389-bc3-mips1-decode", "fdcbd158c32404d09cf1ee38005b2870882e620bcd67c77332d492d07099d665"), + ("Gray-517x389-none-mips0-encode", "fc6cf28e9220780d752ac3be0c9b9bf75412dbea4a54ab603577f78000f235d3"), + ("Gray-517x389-none-mips0-decode", "13df79103435da32999dff6deac57b02a872aef3fe7dafe580625a554258db30"), + ("Gray-517x389-none-mips1-encode", "e6f1bb2eb28353a1ead1b72e6e0c78f9490f9e933ee2fe464044ba2804d0a595"), + ("Gray-517x389-none-mips1-decode", "13df79103435da32999dff6deac57b02a872aef3fe7dafe580625a554258db30"), + ("Gray-517x389-bc1-mips0-encode", "ba3b1a2567ff1571dfb855b7c5ff8e2da276eb80f5f97db0fae661449bb51822"), + ("Gray-517x389-bc1-mips0-decode", "7df1ebd46f395e515af2763a0bf7189072a64f20ff811973098ea109df1ed832"), + ("Gray-517x389-bc1-mips1-encode", "e34896e52b393aa3035e641b43881df1a25ae4aa6261c9bdca34889ec42ea6ac"), + ("Gray-517x389-bc1-mips1-decode", "7df1ebd46f395e515af2763a0bf7189072a64f20ff811973098ea109df1ed832"), + ("Gray-517x389-bc2-mips0-encode", "2ebe06ad3ca05dc993a60c58dbae1d36c56516147176b4799648f534e9318b3a"), + ("Gray-517x389-bc2-mips0-decode", "2adde466f9a4c2c4905d50490bbed4cd5c4de0e3b951d63028702f7ab51d7927"), + ("Gray-517x389-bc2-mips1-encode", "8730c500b41e9b628cab1c789b2e2a2e4a4ad0350b6c3faaca5e3acd8b423ca7"), + ("Gray-517x389-bc2-mips1-decode", "2adde466f9a4c2c4905d50490bbed4cd5c4de0e3b951d63028702f7ab51d7927"), + ("Gray-517x389-bc3-mips0-encode", "e8b62246e29504bbbc92f1c5344846577232e2c12f5c6ab6fae68199bbcf5c3a"), + ("Gray-517x389-bc3-mips0-decode", "2adde466f9a4c2c4905d50490bbed4cd5c4de0e3b951d63028702f7ab51d7927"), + ("Gray-517x389-bc3-mips1-encode", "2b1122c5091c8a62c897fa4a133b2a21d19ec14a3ec2c5e0d4855570824e6211"), + ("Gray-517x389-bc3-mips1-decode", "2adde466f9a4c2c4905d50490bbed4cd5c4de0e3b951d63028702f7ab51d7927"), + ("Rgba-1x1-none-mips0-encode", "c059322936202ed846ae95243325c1756b3ca2734a310b9b58e957e7f6ecb65c"), + ("Rgba-1x1-none-mips0-decode", "ac6d8249b868a4df487ad3d4bc2bffa4ec32afed5bb89bcb916dc390872c00a4"), + ("Rgba-1x1-none-mips1-encode", "c059322936202ed846ae95243325c1756b3ca2734a310b9b58e957e7f6ecb65c"), + ("Rgba-1x1-none-mips1-decode", "ac6d8249b868a4df487ad3d4bc2bffa4ec32afed5bb89bcb916dc390872c00a4"), + ("Rgba-1x1-bc1-mips0-encode", "b128a04da98b6d855fd98c6908f9ba47d0acf2e79c1b39d0cf3e5787ee52b23c"), + ("Rgba-1x1-bc1-mips0-decode", "2c1567f9329e7d7046ebcba26c8b66a872a36d2140c47be6d8ee3fd1bdc7aa26"), + ("Rgba-1x1-bc1-mips1-encode", "b128a04da98b6d855fd98c6908f9ba47d0acf2e79c1b39d0cf3e5787ee52b23c"), + ("Rgba-1x1-bc1-mips1-decode", "2c1567f9329e7d7046ebcba26c8b66a872a36d2140c47be6d8ee3fd1bdc7aa26"), + ("Rgba-1x1-bc2-mips0-encode", "d53707f0da90291d459df619c204c09d0e5537b96055b7c5a7eebb164965a30c"), + ("Rgba-1x1-bc2-mips0-decode", "e41eb1f8866a8a844c5627ec9e119ea9b5b2c2afb62c2cc40d0b3463e191181f"), + ("Rgba-1x1-bc2-mips1-encode", "d53707f0da90291d459df619c204c09d0e5537b96055b7c5a7eebb164965a30c"), + ("Rgba-1x1-bc2-mips1-decode", "e41eb1f8866a8a844c5627ec9e119ea9b5b2c2afb62c2cc40d0b3463e191181f"), + ("Rgba-1x1-bc3-mips0-encode", "b7a5261aeaa590ee27d74e1ec40688cebfd95a8b32bad510dcf4889659bced99"), + ("Rgba-1x1-bc3-mips0-decode", "ac6d8249b868a4df487ad3d4bc2bffa4ec32afed5bb89bcb916dc390872c00a4"), + ("Rgba-1x1-bc3-mips1-encode", "b7a5261aeaa590ee27d74e1ec40688cebfd95a8b32bad510dcf4889659bced99"), + ("Rgba-1x1-bc3-mips1-decode", "ac6d8249b868a4df487ad3d4bc2bffa4ec32afed5bb89bcb916dc390872c00a4"), + ("Gray-1x1-none-mips0-encode", "9acce63d02641780f46268c15cb62bdf84b68edc091801d02f62d8840b90e76f"), + ("Gray-1x1-none-mips0-decode", "5ef973abd913579530066461f86da91115474c93b5a6cf1b73327feeb3650b83"), + ("Gray-1x1-none-mips1-encode", "9acce63d02641780f46268c15cb62bdf84b68edc091801d02f62d8840b90e76f"), + ("Gray-1x1-none-mips1-decode", "5ef973abd913579530066461f86da91115474c93b5a6cf1b73327feeb3650b83"), + ("Gray-1x1-bc1-mips0-encode", "ffec653603d347468446a15fe4101fda9cb34efb48dc69a8d0f403ce2e2fd9d7"), + ("Gray-1x1-bc1-mips0-decode", "bf04357a38e992a2513e373d00f664b687669a14e1405a2d1b95fb0de6e9a89b"), + ("Gray-1x1-bc1-mips1-encode", "ffec653603d347468446a15fe4101fda9cb34efb48dc69a8d0f403ce2e2fd9d7"), + ("Gray-1x1-bc1-mips1-decode", "bf04357a38e992a2513e373d00f664b687669a14e1405a2d1b95fb0de6e9a89b"), + ("Gray-1x1-bc2-mips0-encode", "3c36aac2c17a7aad99b2c1fa0f4e5b256a9b4421c38406f61961b09c64290178"), + ("Gray-1x1-bc2-mips0-decode", "bf04357a38e992a2513e373d00f664b687669a14e1405a2d1b95fb0de6e9a89b"), + ("Gray-1x1-bc2-mips1-encode", "3c36aac2c17a7aad99b2c1fa0f4e5b256a9b4421c38406f61961b09c64290178"), + ("Gray-1x1-bc2-mips1-decode", "bf04357a38e992a2513e373d00f664b687669a14e1405a2d1b95fb0de6e9a89b"), + ("Gray-1x1-bc3-mips0-encode", "73704fca801476ab72bf69b80a395de711e9a3fa4f476ce5481ae3858f844922"), + ("Gray-1x1-bc3-mips0-decode", "bf04357a38e992a2513e373d00f664b687669a14e1405a2d1b95fb0de6e9a89b"), + ("Gray-1x1-bc3-mips1-encode", "73704fca801476ab72bf69b80a395de711e9a3fa4f476ce5481ae3858f844922"), + ("Gray-1x1-bc3-mips1-decode", "bf04357a38e992a2513e373d00f664b687669a14e1405a2d1b95fb0de6e9a89b"), + ("Rgba-2x3-none-mips0-encode", "1f641b8e1cc01924b0d1a06d2d81ad22c759b44a4f640a83ba6339446e13fc6c"), + ("Rgba-2x3-none-mips0-decode", "af815e7ba5ce6369171335ae94e553524db701f94d152b903f220ee64b9373a6"), + ("Rgba-2x3-none-mips1-encode", "9be390b308b0961125828ca53894a87101983eafaf54e3c7e3da7b1f992fb518"), + ("Rgba-2x3-none-mips1-decode", "af815e7ba5ce6369171335ae94e553524db701f94d152b903f220ee64b9373a6"), + ("Rgba-2x3-bc1-mips0-encode", "743a3d25313924d68732ab399ed92c8e3f693ef26a415a12532918b80db085c4"), + ("Rgba-2x3-bc1-mips0-decode", "570f9feaf2d322a568b2a87ca652aa0606ff96cde281e24d247d071a9dc4e827"), + ("Rgba-2x3-bc1-mips1-encode", "711b0088c7c8083760921c3620d9d1adc02d47b5c3b0eb9e674d974fd1d6d641"), + ("Rgba-2x3-bc1-mips1-decode", "570f9feaf2d322a568b2a87ca652aa0606ff96cde281e24d247d071a9dc4e827"), + ("Rgba-2x3-bc2-mips0-encode", "b4facaf715dd8208fe02961fb99584b58c219117a99f2b1d15d10a556a9e5251"), + ("Rgba-2x3-bc2-mips0-decode", "66112c3066ef59936784fa938c26eb0e86772da586442237de5336c48558afbe"), + ("Rgba-2x3-bc2-mips1-encode", "cad117a2ddd6fe91876efd1dd0b80da0b393f3af21209efc3c1b0f165606a129"), + ("Rgba-2x3-bc2-mips1-decode", "66112c3066ef59936784fa938c26eb0e86772da586442237de5336c48558afbe"), + ("Rgba-2x3-bc3-mips0-encode", "b68ced41c6b33037f08b74c492da5688e9d80555c5843f9f323a74d7c86e114e"), + ("Rgba-2x3-bc3-mips0-decode", "051b60ea47094987d71180aeaa00b3f1b5427429e54812ea28b40f8e945044c8"), + ("Rgba-2x3-bc3-mips1-encode", "88e0fd6d04c896880d03731d6df1528f22290c1738b303eb2b09fe84d7882a64"), + ("Rgba-2x3-bc3-mips1-decode", "051b60ea47094987d71180aeaa00b3f1b5427429e54812ea28b40f8e945044c8"), + ("Gray-2x3-none-mips0-encode", "7e9d26894ca9a800b93f5dae0e5b58b18a6cbcaea56eacc657aba40a1933c476"), + ("Gray-2x3-none-mips0-decode", "d21c9cddccb970575eba36093410245f0a0fb15049fc96311874191f868de0d5"), + ("Gray-2x3-none-mips1-encode", "d2ee41ea49462e1d7be568cfcbf867c6395646710ba321c2dac9db319294bb5e"), + ("Gray-2x3-none-mips1-decode", "d21c9cddccb970575eba36093410245f0a0fb15049fc96311874191f868de0d5"), + ("Gray-2x3-bc1-mips0-encode", "000d5d9121042f7b88411d8e08a548a2c804b8b4b73b40bf317cff61899a5445"), + ("Gray-2x3-bc1-mips0-decode", "1a8f46dd535383398fbeebe5fdfca9bb256fa3b3a7b8552af34b67b09865e89d"), + ("Gray-2x3-bc1-mips1-encode", "07346ec09eb8d9b489e7dd50bd73851342205594ac985544bfe58b6bda73b21b"), + ("Gray-2x3-bc1-mips1-decode", "1a8f46dd535383398fbeebe5fdfca9bb256fa3b3a7b8552af34b67b09865e89d"), + ("Gray-2x3-bc2-mips0-encode", "5db43dcd4adec962a7a12c6f8da8bf05d3dc2e3d608ca3fa341e36b43c218c3d"), + ("Gray-2x3-bc2-mips0-decode", "afabd75898a95baac868e9e14905c2d89e90767750217e4c9ac84b1caeef41bc"), + ("Gray-2x3-bc2-mips1-encode", "3effae05c47854dfb5f20ed12e2c1588236464f17fbf5dd13e61958e5097a796"), + ("Gray-2x3-bc2-mips1-decode", "afabd75898a95baac868e9e14905c2d89e90767750217e4c9ac84b1caeef41bc"), + ("Gray-2x3-bc3-mips0-encode", "11c48dce5b7545cf3eafd1f01fc6b977b13cb7c7e8f2ef7ea1406a8f04b86cf9"), + ("Gray-2x3-bc3-mips0-decode", "afabd75898a95baac868e9e14905c2d89e90767750217e4c9ac84b1caeef41bc"), + ("Gray-2x3-bc3-mips1-encode", "6b7540afc583554e70f4131b4a0d8648eed65100293ab9b7060afcbd33751e64"), + ("Gray-2x3-bc3-mips1-decode", "afabd75898a95baac868e9e14905c2d89e90767750217e4c9ac84b1caeef41bc"), + ("Rgba-3x2-none-mips0-encode", "e821c1620e4e53588908268da45702924911ed756f3257cb0a21727748b736a6"), + ("Rgba-3x2-none-mips0-decode", "d4583449852070512fa3cb16c5132834ccee2f9dd3a6c0d6c8fa5bbbc2935e8d"), + ("Rgba-3x2-none-mips1-encode", "f79a699612ef8d8347a5e83af2fd9ee915eda35388bc86b4243b265f988fc3d8"), + ("Rgba-3x2-none-mips1-decode", "d4583449852070512fa3cb16c5132834ccee2f9dd3a6c0d6c8fa5bbbc2935e8d"), + ("Rgba-3x2-bc1-mips0-encode", "1c106b532f6afc132538b2bb2c1eb9e6bb075a448d19b6af2c67b4c31415e3b2"), + ("Rgba-3x2-bc1-mips0-decode", "f0e63ce55598b6c58c601a5cf3415522113181a08bc28f4d8cd499d160352cdd"), + ("Rgba-3x2-bc1-mips1-encode", "23d86eba138f7c4492f9955dd0a29eb1a675d69f1e612b54e783837d9414cb14"), + ("Rgba-3x2-bc1-mips1-decode", "f0e63ce55598b6c58c601a5cf3415522113181a08bc28f4d8cd499d160352cdd"), + ("Rgba-3x2-bc2-mips0-encode", "74a68d7b1cc0173f1e8a6e0f6101ffa010757df0434fcf5636db808ca20b341e"), + ("Rgba-3x2-bc2-mips0-decode", "bc63bba15c1bbe19cdef66535870c7dfeeae26a4686557e1530ae64a2b2253a5"), + ("Rgba-3x2-bc2-mips1-encode", "d28d7a4a7ee17d0f58f27ebef8d682a7fed42778a60e09186af523198a2f3562"), + ("Rgba-3x2-bc2-mips1-decode", "bc63bba15c1bbe19cdef66535870c7dfeeae26a4686557e1530ae64a2b2253a5"), + ("Rgba-3x2-bc3-mips0-encode", "fa0a02ba9822a007f1654d618b5225332f3934e5f07c51f8ca474e26658e2c45"), + ("Rgba-3x2-bc3-mips0-decode", "ee0221751e1d96bf76552df65a7e54d7711f834cf0431fd17a3a6542473b4ceb"), + ("Rgba-3x2-bc3-mips1-encode", "3a3c51f596a82ae666c6157da4266c3330e8e2e0744947e0ce663deb66e67ad6"), + ("Rgba-3x2-bc3-mips1-decode", "ee0221751e1d96bf76552df65a7e54d7711f834cf0431fd17a3a6542473b4ceb"), + ("Gray-3x2-none-mips0-encode", "6e1e49ee238e40108f1e934946b0d2498eb97f6824a82d4d71011759a58b83a4"), + ("Gray-3x2-none-mips0-decode", "e333fde639747f66aa4d74dbeae648566d6735ecc47de220e07b9da351e1bf37"), + ("Gray-3x2-none-mips1-encode", "906e16afa18db0290bec758a1a124de7a99255871344d462f797db54dbd30b4b"), + ("Gray-3x2-none-mips1-decode", "e333fde639747f66aa4d74dbeae648566d6735ecc47de220e07b9da351e1bf37"), + ("Gray-3x2-bc1-mips0-encode", "df204b170cd24f0375772d65b4f0548a824a4602c453cf891c9cc4aafc4b32ad"), + ("Gray-3x2-bc1-mips0-decode", "869a8202b3721a8d2bf232e75e9a91a271cf2ebad85c8df6a5260cb9c9b6b1ab"), + ("Gray-3x2-bc1-mips1-encode", "d06f38b5c40156888fe2ea04343f46d4753a80f83f63d80b3fd7ce82c5d7fe94"), + ("Gray-3x2-bc1-mips1-decode", "869a8202b3721a8d2bf232e75e9a91a271cf2ebad85c8df6a5260cb9c9b6b1ab"), + ("Gray-3x2-bc2-mips0-encode", "1832e5006137ae1583d34ba1bfab9c7b85f55791996f7cdf21ed799f3939c4f4"), + ("Gray-3x2-bc2-mips0-decode", "869a8202b3721a8d2bf232e75e9a91a271cf2ebad85c8df6a5260cb9c9b6b1ab"), + ("Gray-3x2-bc2-mips1-encode", "41d49c9cce6cbbdf1676033a9233ed774f339d92e8159d1d1b0cbc8503cfe72b"), + ("Gray-3x2-bc2-mips1-decode", "869a8202b3721a8d2bf232e75e9a91a271cf2ebad85c8df6a5260cb9c9b6b1ab"), + ("Gray-3x2-bc3-mips0-encode", "377e3d02919641d634bbe02cadb513237fcd90f540b8769c5e2a214d5ccf05e4"), + ("Gray-3x2-bc3-mips0-decode", "869a8202b3721a8d2bf232e75e9a91a271cf2ebad85c8df6a5260cb9c9b6b1ab"), + ("Gray-3x2-bc3-mips1-encode", "70b03c1901946a24abf34c60dd7bd9cc124df43a251a82622c15d56b0e2b2588"), + ("Gray-3x2-bc3-mips1-decode", "869a8202b3721a8d2bf232e75e9a91a271cf2ebad85c8df6a5260cb9c9b6b1ab"), + ("Rgba-5x5-none-mips0-encode", "be7677f4c86d5f3c0ba75ec04d36bcfd708bca44c9f6720c4b7494c290014491"), + ("Rgba-5x5-none-mips0-decode", "00dc5f1020a7de3a3bc5ee21e7a767d09dab9c9831e840290277af90d1e1e3be"), + ("Rgba-5x5-none-mips1-encode", "284c04f628c256ba546815e46df2b837ee997d90834b8a0f095f20e17ba36102"), + ("Rgba-5x5-none-mips1-decode", "00dc5f1020a7de3a3bc5ee21e7a767d09dab9c9831e840290277af90d1e1e3be"), + ("Rgba-5x5-bc1-mips0-encode", "a5350786ccbd004a88ca541c0d4d7cdf313f71fec9485a45acd380f318846435"), + ("Rgba-5x5-bc1-mips0-decode", "26559d0fc43534fad7318b66672ef9de924b6408b2528c00cb6be76a56a94654"), + ("Rgba-5x5-bc1-mips1-encode", "67e06000b2ee88a6e8e749b20ee6ec0016d294f2fba57e9773aea3a7addd1883"), + ("Rgba-5x5-bc1-mips1-decode", "26559d0fc43534fad7318b66672ef9de924b6408b2528c00cb6be76a56a94654"), + ("Rgba-5x5-bc2-mips0-encode", "6cf909177531b9ed88be4ffd031055a7b1cbcfab0c6082ca33913720d631f834"), + ("Rgba-5x5-bc2-mips0-decode", "060ad06944e973f57c866137f9bd8c903a4edd30af275f9ee92fadff54a0a4bb"), + ("Rgba-5x5-bc2-mips1-encode", "0590d1aa5b074abb89262432a3b6b5c3ff12edca0d454069dadfe20be690e727"), + ("Rgba-5x5-bc2-mips1-decode", "060ad06944e973f57c866137f9bd8c903a4edd30af275f9ee92fadff54a0a4bb"), + ("Rgba-5x5-bc3-mips0-encode", "8d24e90ef90f819353e1bbbb1ab1163463dbdf8d9aaeb89b3acbc8cf2d3dd625"), + ("Rgba-5x5-bc3-mips0-decode", "d3238486581875abcd37dfbe03e8fbea7aa4fcadb1fef866899deafb67ac38f9"), + ("Rgba-5x5-bc3-mips1-encode", "2c331cbc46a45536bc1ae50cff9a692bae969748aebfa13e93c0269055ad5031"), + ("Rgba-5x5-bc3-mips1-decode", "d3238486581875abcd37dfbe03e8fbea7aa4fcadb1fef866899deafb67ac38f9"), + ("Gray-5x5-none-mips0-encode", "8527392cacb17a19b3e1c0f7cbe05ea4abd7fba93a7952d30967047a2e317f3b"), + ("Gray-5x5-none-mips0-decode", "70193f4748cd97b7676693076afe2f1c20afa0dc37b4eb89ff3752114a0e0321"), + ("Gray-5x5-none-mips1-encode", "6c5bd6095bbed4d39c15a6b50dcb5da8d5da48aabf696d2fbfcaf9bd1954c69d"), + ("Gray-5x5-none-mips1-decode", "70193f4748cd97b7676693076afe2f1c20afa0dc37b4eb89ff3752114a0e0321"), + ("Gray-5x5-bc1-mips0-encode", "fc7740ff5a801fdcc82e645d25240b44899a64117e70d739c8cba1c17c5a0841"), + ("Gray-5x5-bc1-mips0-decode", "65a82c1c1048d612c3b5e947603206571f088d1348e6acc42b9e2c014121e227"), + ("Gray-5x5-bc1-mips1-encode", "3953cd92035a27902d5feaf61ba11b4696c8593fb139fbdb6fda0167ac6d7a58"), + ("Gray-5x5-bc1-mips1-decode", "65a82c1c1048d612c3b5e947603206571f088d1348e6acc42b9e2c014121e227"), + ("Gray-5x5-bc2-mips0-encode", "4b2c26f66303f83c35c4f0ca8e306afbde35cfac5e9986cb6dc2ab33696a21c1"), + ("Gray-5x5-bc2-mips0-decode", "65a82c1c1048d612c3b5e947603206571f088d1348e6acc42b9e2c014121e227"), + ("Gray-5x5-bc2-mips1-encode", "047e4d8df50b60f8936e8da938b7c62217da128aaf88a25a9384aa69a3701ce9"), + ("Gray-5x5-bc2-mips1-decode", "65a82c1c1048d612c3b5e947603206571f088d1348e6acc42b9e2c014121e227"), + ("Gray-5x5-bc3-mips0-encode", "9c31c80f1e254a0e590927faf77fe88007d01c93d7a99280b7d632c4d680a8c4"), + ("Gray-5x5-bc3-mips0-decode", "65a82c1c1048d612c3b5e947603206571f088d1348e6acc42b9e2c014121e227"), + ("Gray-5x5-bc3-mips1-encode", "e046981ea9208a3da3cabbb6dd3f3f439db09d8713e6594e099bd7a3ebcd50dc"), + ("Gray-5x5-bc3-mips1-decode", "65a82c1c1048d612c3b5e947603206571f088d1348e6acc42b9e2c014121e227"), + ("Rgba-1x9-none-mips0-encode", "94676df714a0e07fd91981b3f9795da76df174319d551573bf62a7324ceae4fb"), + ("Rgba-1x9-none-mips0-decode", "dbcf64981f64ab690faa081bf7a90405bc9710ca5789f2bbf63d643469d1dd78"), + ("Rgba-1x9-none-mips1-encode", "2614a6ecc2aa845cab81819cd8bcd8b84624d693df6979cba3df91e100f27361"), + ("Rgba-1x9-none-mips1-decode", "dbcf64981f64ab690faa081bf7a90405bc9710ca5789f2bbf63d643469d1dd78"), + ("Rgba-1x9-bc1-mips0-encode", "b7f1466fcebfb73bd6e3b6288d3a8adde258fd9801c694fd44d0f92fbde26c61"), + ("Rgba-1x9-bc1-mips0-decode", "7d3471a56c9ca5332bccb8943e9d5a01105e0ddfd432b403e77841137f927787"), + ("Rgba-1x9-bc1-mips1-encode", "4c7c22ba8e2653f31cd9114857d8ba63db6ac180507e40e8c567b61b9c6781d9"), + ("Rgba-1x9-bc1-mips1-decode", "7d3471a56c9ca5332bccb8943e9d5a01105e0ddfd432b403e77841137f927787"), + ("Rgba-1x9-bc2-mips0-encode", "9dc6e68ae27d0709f8859d6efa0dfc4028d9fbba2cc0aef106973c998417f5d1"), + ("Rgba-1x9-bc2-mips0-decode", "c9dd7bf6d44932b599eeb73b54c6f778880db038768110b005f1886f6993e1db"), + ("Rgba-1x9-bc2-mips1-encode", "5ca6d64da612af3b9018cefa50e98cb9097c2eb3dbda8f862571057ba02d4312"), + ("Rgba-1x9-bc2-mips1-decode", "c9dd7bf6d44932b599eeb73b54c6f778880db038768110b005f1886f6993e1db"), + ("Rgba-1x9-bc3-mips0-encode", "f1d5194773bea0b427702900c928d6829f35cbfd32ec0e76b3ee882670597a81"), + ("Rgba-1x9-bc3-mips0-decode", "feda0633d1e41776c56257893bbe2be5317f57bb771152e1a4336558c16c013c"), + ("Rgba-1x9-bc3-mips1-encode", "4faa40254411822689f7d6b356c447e58f89a00ea05de3f7c43253164d83823c"), + ("Rgba-1x9-bc3-mips1-decode", "feda0633d1e41776c56257893bbe2be5317f57bb771152e1a4336558c16c013c"), + ("Gray-1x9-none-mips0-encode", "deb41e75756b10819d44b9840171a088a5d0169f56962a055939131986b7ebd1"), + ("Gray-1x9-none-mips0-decode", "2713191d6e180f74fa7825564575abcf7cc40c34f09c2cb8ccd4d2b02a675bb0"), + ("Gray-1x9-none-mips1-encode", "2d0f36be14a693e7c0380d70841d56cc857e4b81d1a4acbd94eeb18fca0c9216"), + ("Gray-1x9-none-mips1-decode", "2713191d6e180f74fa7825564575abcf7cc40c34f09c2cb8ccd4d2b02a675bb0"), + ("Gray-1x9-bc1-mips0-encode", "0723a0ecd94984d4a24890abc427611135b5add4c8ced0cd570867abe49217df"), + ("Gray-1x9-bc1-mips0-decode", "dcac5ac2d759bc4166ec0d061863f6bf9a34a03a382ac1d88ebdec3e750ecf3e"), + ("Gray-1x9-bc1-mips1-encode", "b1c38854773fde2bf2fc6e0c13ea5d8f6c92a19c1cb7af5c2986851522f2c5c2"), + ("Gray-1x9-bc1-mips1-decode", "dcac5ac2d759bc4166ec0d061863f6bf9a34a03a382ac1d88ebdec3e750ecf3e"), + ("Gray-1x9-bc2-mips0-encode", "b55d775f6f6029fa6d58ea4bea1abeaed349478665874ae2d5a53be14389c595"), + ("Gray-1x9-bc2-mips0-decode", "dcac5ac2d759bc4166ec0d061863f6bf9a34a03a382ac1d88ebdec3e750ecf3e"), + ("Gray-1x9-bc2-mips1-encode", "2e90ad054e173a392042126fd8674516fd8dc928c098238f92b7d434181a5895"), + ("Gray-1x9-bc2-mips1-decode", "dcac5ac2d759bc4166ec0d061863f6bf9a34a03a382ac1d88ebdec3e750ecf3e"), + ("Gray-1x9-bc3-mips0-encode", "3316e2f5db9bccd337b9dfede5d8c7a4555a645939682f4fc323e746722ef850"), + ("Gray-1x9-bc3-mips0-decode", "dcac5ac2d759bc4166ec0d061863f6bf9a34a03a382ac1d88ebdec3e750ecf3e"), + ("Gray-1x9-bc3-mips1-encode", "0ce9b894539c7e6d22531884a6e352aef26fc899ccf97d8a86878f6f5a68d969"), + ("Gray-1x9-bc3-mips1-decode", "dcac5ac2d759bc4166ec0d061863f6bf9a34a03a382ac1d88ebdec3e750ecf3e"), + ("Rgba-9x1-none-mips0-encode", "29bc4ef0ce9120982b331bf34a460a75f7c086b630ea6710ddd907b01125cc30"), + ("Rgba-9x1-none-mips0-decode", "666aa0014607ab13b7141c423c1fb137bda8a5c92973b3252c0556247383d80b"), + ("Rgba-9x1-none-mips1-encode", "455e7e2173deb612b28ac2ea2cacaf66c740150b3702e2d027bf51dc1123217f"), + ("Rgba-9x1-none-mips1-decode", "666aa0014607ab13b7141c423c1fb137bda8a5c92973b3252c0556247383d80b"), + ("Rgba-9x1-bc1-mips0-encode", "8e21b90879f2768fe0341375245b42a1645bc776ef88f81d700cd7e4f538ece3"), + ("Rgba-9x1-bc1-mips0-decode", "f6b1a1af12998d9db067870ff988d9b527df09ebfbe6a06710474247155655a8"), + ("Rgba-9x1-bc1-mips1-encode", "034c32c5a3025a72161e02ea5422ead0253490df2abf37030a048c93a301b386"), + ("Rgba-9x1-bc1-mips1-decode", "f6b1a1af12998d9db067870ff988d9b527df09ebfbe6a06710474247155655a8"), + ("Rgba-9x1-bc2-mips0-encode", "7d9fbf241277124a62bd0bc466e687d915435cbc083cf0f228694546b612f75e"), + ("Rgba-9x1-bc2-mips0-decode", "5360be17f525cac699135d57837231fbeb993a906c0f7efbffea6c18da4bb9b4"), + ("Rgba-9x1-bc2-mips1-encode", "c7fb6cb488b40b0d08f9958d8d19e3e3f8ac19ce96cd67f99b4bf6d65fb7b0fa"), + ("Rgba-9x1-bc2-mips1-decode", "5360be17f525cac699135d57837231fbeb993a906c0f7efbffea6c18da4bb9b4"), + ("Rgba-9x1-bc3-mips0-encode", "1e9ad28a5057cd576ad5f47625ccbdd836e683fd321428ac30760a15753a3b0e"), + ("Rgba-9x1-bc3-mips0-decode", "a367950e128431e9c0728de32358905486a9f53efef56827cf8f7fa71c9382af"), + ("Rgba-9x1-bc3-mips1-encode", "0f36472c2a24a64e07893879b60a1fc15e492038a669a4fdf0f586942ff7d197"), + ("Rgba-9x1-bc3-mips1-decode", "a367950e128431e9c0728de32358905486a9f53efef56827cf8f7fa71c9382af"), + ("Gray-9x1-none-mips0-encode", "4d65e2aad9e331cfc4edf2c0aaa58673c7ba90f1ad37cfa9c6a31095976ebc0a"), + ("Gray-9x1-none-mips0-decode", "105c3ae0c54915506d98a823112f37182f15e2cbed571b5161c4dfa1007575b5"), + ("Gray-9x1-none-mips1-encode", "0943b2616b8800af4415b53893ea301f1020e062b20f37095f1be9653a31e9bd"), + ("Gray-9x1-none-mips1-decode", "105c3ae0c54915506d98a823112f37182f15e2cbed571b5161c4dfa1007575b5"), + ("Gray-9x1-bc1-mips0-encode", "2c41ef37fc366203a20e1970feecdc8b619659215412a916221ccaa2b273430e"), + ("Gray-9x1-bc1-mips0-decode", "afd18b30efff0e3eb4b59cf01dcbbb06691c3af6028596f8de9efdf370ea915a"), + ("Gray-9x1-bc1-mips1-encode", "d2de0189a1e7a6a70f9a5d4ea7c7a53e064b7d1511f9b183a8dc31268ddb6ac6"), + ("Gray-9x1-bc1-mips1-decode", "afd18b30efff0e3eb4b59cf01dcbbb06691c3af6028596f8de9efdf370ea915a"), + ("Gray-9x1-bc2-mips0-encode", "766180145e22838dc950a136791fb350d51821e39c58270be394357228462fc1"), + ("Gray-9x1-bc2-mips0-decode", "afd18b30efff0e3eb4b59cf01dcbbb06691c3af6028596f8de9efdf370ea915a"), + ("Gray-9x1-bc2-mips1-encode", "476abc493a613766ad73fdc6a89ab765a68677ef6623370472dfea2e3d531847"), + ("Gray-9x1-bc2-mips1-decode", "afd18b30efff0e3eb4b59cf01dcbbb06691c3af6028596f8de9efdf370ea915a"), + ("Gray-9x1-bc3-mips0-encode", "b2f6f7f90bd0530857b272a6b39ac60e20ff2c65f5c0f71de543276d48fc5a30"), + ("Gray-9x1-bc3-mips0-decode", "afd18b30efff0e3eb4b59cf01dcbbb06691c3af6028596f8de9efdf370ea915a"), + ("Gray-9x1-bc3-mips1-encode", "4376a76e8467759cd485d2241c6e1553b6d49e6eb2f83a466d71ca68754776cd"), + ("Gray-9x1-bc3-mips1-decode", "afd18b30efff0e3eb4b59cf01dcbbb06691c3af6028596f8de9efdf370ea915a"), + ("Rgba-6x13-none-mips0-encode", "f603fdcab8e987e1f4b023ec68a14d8c03808952fe429338669977f33131c5a8"), + ("Rgba-6x13-none-mips0-decode", "35c18e04c37154e00ba7a544476335c4e234d8390803eb8f370351551652e49c"), + ("Rgba-6x13-none-mips1-encode", "5254f124be9189174a400affb9192419d6350f6f3361d24cf417ea42c8d33411"), + ("Rgba-6x13-none-mips1-decode", "35c18e04c37154e00ba7a544476335c4e234d8390803eb8f370351551652e49c"), + ("Rgba-6x13-bc1-mips0-encode", "f1b5e39e9a7308be64aa3531cfd487116a4e7f3ea309b694292c9738721ac872"), + ("Rgba-6x13-bc1-mips0-decode", "0a63216b434cd9b24807ea58c6d63888d68e84fa4834a34a8068f7fbf054449d"), + ("Rgba-6x13-bc1-mips1-encode", "13e5b50aa32a32d3dc71d75426c773bb36b191bd97c16c6f2ee0d76f4ee4f54b"), + ("Rgba-6x13-bc1-mips1-decode", "0a63216b434cd9b24807ea58c6d63888d68e84fa4834a34a8068f7fbf054449d"), + ("Rgba-6x13-bc2-mips0-encode", "0e0f5815d03c6ec3b8d58715beacdbe67a3dde10bf3f3b635425dc93073dbb4d"), + ("Rgba-6x13-bc2-mips0-decode", "14d744c8aff8c596f409a6a5c7599bca6479e8e084aa63d99660a44bfe55be6e"), + ("Rgba-6x13-bc2-mips1-encode", "4d007cb53697f394feb540a4745818c1f8a9df4a47819eac38b2102039e9421c"), + ("Rgba-6x13-bc2-mips1-decode", "14d744c8aff8c596f409a6a5c7599bca6479e8e084aa63d99660a44bfe55be6e"), + ("Rgba-6x13-bc3-mips0-encode", "db7fc9607849f169b3e1a7a8e100a7515bdce53ff4f1db7a5fbb6cfc1d1b8162"), + ("Rgba-6x13-bc3-mips0-decode", "33ba59f2acc8ab6877b8474a4b2cf063a0ae54b4a27afc64fde51313ba4ca6bc"), + ("Rgba-6x13-bc3-mips1-encode", "c985feb0187e7ac21aa829e25c5e5fe08970c58e10aaf71b96416a7969c55e68"), + ("Rgba-6x13-bc3-mips1-decode", "33ba59f2acc8ab6877b8474a4b2cf063a0ae54b4a27afc64fde51313ba4ca6bc"), + ("Gray-6x13-none-mips0-encode", "f63bb759c82a74584022a92fa9dfb64f36f3709446e279157942bafef6edc9b9"), + ("Gray-6x13-none-mips0-decode", "71201dd954c691aff18dab0a886cb7f10c15dbb33fca26dcc9f1047d28382190"), + ("Gray-6x13-none-mips1-encode", "435bedffe3efccd86a63533671dd55c7cdc982ea176383adb7e56c4745d38d28"), + ("Gray-6x13-none-mips1-decode", "71201dd954c691aff18dab0a886cb7f10c15dbb33fca26dcc9f1047d28382190"), + ("Gray-6x13-bc1-mips0-encode", "f8cb01936384bbff861cbebfd7149753b31f185b52de437e03074102b1056abf"), + ("Gray-6x13-bc1-mips0-decode", "48669b49d5aec2ab69c120c938838ba206d4e2d589399d2bccf0df5580b48194"), + ("Gray-6x13-bc1-mips1-encode", "f09c7c8bc3def200c72ed32545de294bbcb5436dbc46e249e5d5d2b8ac987a64"), + ("Gray-6x13-bc1-mips1-decode", "48669b49d5aec2ab69c120c938838ba206d4e2d589399d2bccf0df5580b48194"), + ("Gray-6x13-bc2-mips0-encode", "a2b00acf35735c638e07c7468dd8d5ab3405a8d69c653cf95a369538168b8e85"), + ("Gray-6x13-bc2-mips0-decode", "48669b49d5aec2ab69c120c938838ba206d4e2d589399d2bccf0df5580b48194"), + ("Gray-6x13-bc2-mips1-encode", "40769615b4e3117ade50c33e69a51a3f2f1601018c49ea216971955bed74b2ea"), + ("Gray-6x13-bc2-mips1-decode", "48669b49d5aec2ab69c120c938838ba206d4e2d589399d2bccf0df5580b48194"), + ("Gray-6x13-bc3-mips0-encode", "a429bc2f9725fb23f84ac6bff8bd6a1faaf0ab2808912515069b90c002bbf460"), + ("Gray-6x13-bc3-mips0-decode", "48669b49d5aec2ab69c120c938838ba206d4e2d589399d2bccf0df5580b48194"), + ("Gray-6x13-bc3-mips1-encode", "df1d533175cc807a8474701fe4cbdecd694fadd03082d506be50d8716e3e138f"), + ("Gray-6x13-bc3-mips1-decode", "48669b49d5aec2ab69c120c938838ba206d4e2d589399d2bccf0df5580b48194"), + ("dx10-rgba8-decode", "154284a9d25eef84bb2ee0b8b1c2d3cd95149dbae9a61cbe0ddb0f4e59cd1482"), + ("dx10-bgra8-decode", "ec571c9b54af06a0e63fed4b8b00a878d634f62297e72a22372c38fec561b08f"), + ("dx10-bgrx8-decode", "1e9e7678fcd37dcb5c61047cadac102a2053c9d738fab17c55d9d4af573aea99"), + ("legacy-rgba8-decode", "387f6d2b46545aa0188614c6c407a4c7498f065f3124964b92992a304c4e997d"), + ("legacy-bgra8-padded-decode", "5d4a59bd952afd03eb206aa2e3eaeb652d1d6abd1bf2ada8d8bc67e153fd0b3b"), + ("legacy-rgb24-decode", "0d33d20540b8d8714fb2f50193505188a60a743edbca6729806210a990bcebb7"), + ("legacy-rgb565-decode", "e0529a28e87837cea810bee99263a4d3c1ec99bfa6547bff583fc5848b7319e9"), + ("legacy-argb4444-decode", "0cf215d8c21faeff39c8b4e37acb8a72c69a873b7d132746c66c2f4a6277d58a"), + ("legacy-l8-padded-decode", "0c72254df5d58d8a1b4c62d1da03548c10a058dab179d7486a1e1c394a4b1cee"), + ("legacy-l8a8-decode", "066ff8e0a187094a1feb4bdc57e1cb8a99bc6ed3d69647c87c9d54a093b6815a"), + ("legacy-dxt1-noise-decode", "78d4ec7cec69249f8b065667eb66a23b23e82a6b7ab167043539976ae874c433"), + ("legacy-dxt3-noise-decode", "2dae7a8abe345ca37af54ad56e65dcb8a6fc1837f233cc81b5693b0b547babc8"), + ("legacy-dxt5-noise-decode", "5a6edeb62a5d6043a474de25da0611a550cf81716a8960dbf57feaebddf7d950"), + ("dx10-bc1-noise-decode", "5c6ea27a01ccb2a1ab3badea75551a9d3209a0176335871186362cc7e1923ead"), + ("dx10-bc3-noise-decode", "9c5376a1d714471d575344920380755e40f5be6cd4cb54add5b982d833276823"), +]; + +/// The compressions the digests cover, listed here rather than taken from `DdsCompression::ALL` +/// so a compression added later doesn't change the set. +const COMPRESSIONS: [DdsCompression; 4] = [DdsCompression::None, DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3]; + +/// A 64-bit LCG (Knuth's MMIX constants) with a fixed seed: the same noise on every run. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u8 { + self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + (self.0 >> 33) as u8 + } +} + +/// An RGBA test picture in four vertical bands: smooth ramps with mid alpha, noise in every +/// channel, flat 8×8 tiles of a few colours (whole blocks of one colour, opaque), and a band +/// where alpha jumps between 0, 255 and in-between values over gentle colour. +fn picture(w: u32, h: u32, seed: u64) -> Vec { + let mut rng = Lcg(seed); + let tiles: [[u8; 4]; 5] = [[12, 200, 77, 255], [255, 255, 255, 255], [0, 0, 0, 255], [255, 0, 255, 255], [90, 91, 92, 255]]; + let band = w.div_ceil(4).max(1); + let mut data = Vec::with_capacity(w as usize * h as usize * 4); + for y in 0..h { + for x in 0..w { + let noise = [rng.next(), rng.next(), rng.next(), rng.next()]; + let px = match x / band { + 0 => [(x * 2) as u8, (y * 3) as u8, (x + y) as u8, (64 + y % 128) as u8], + 1 => noise, + 2 => tiles[((x / 8 + y / 8) % 5) as usize], + _ => { + let alpha = match (x / 3 + y / 5) % 4 { + 0 => 0, + 1 => 255, + 2 => 128, + _ => noise[3], + }; + [(y % 256) as u8, 128, (255 - y % 256) as u8, alpha] + } + }; + data.extend(px); + } + } + data +} + +/// The picture in `layout`, its channels picked from the RGBA picture without any conversion +/// formula: gray is red, and gray's alpha is alpha. +fn image(w: u32, h: u32, layout: ChannelLayout, seed: u64) -> Image { + let rgba = picture(w, h, seed); + let data = rgba + .as_chunks::<4>() + .0 + .iter() + .flat_map(|p| match layout { + ChannelLayout::Gray => vec![p[0]], + ChannelLayout::GrayA => vec![p[0], p[3]], + ChannelLayout::Rgb => vec![p[0], p[1], p[2]], + _ => p.to_vec(), + }) + .collect(); + Image::from_u8(w, h, layout, data).unwrap() +} + +fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect() +} + +/// A decoded image's digest: its size and layout, then its samples. +fn image_digest(img: &Image) -> String { + let mut h = Sha256::new(); + h.update(img.width().to_le_bytes()); + h.update(img.height().to_le_bytes()); + h.update(format!("{:?}", img.layout()).as_bytes()); + h.update(img.data()); + h.finalize().iter().map(|b| format!("{b:02x}")).collect() +} + +const DDPF_ALPHAPIXELS: u32 = 0x1; +const DDPF_FOURCC: u32 = 0x4; +const DDPF_RGB: u32 = 0x40; +const DDPF_LUMINANCE: u32 = 0x2_0000; +const DDSD_PITCH: u32 = 0x8; + +/// A DDS_HEADER for a single-level 2D texture, with `pitch` set (and flagged) when given. +fn header(width: u32, height: u32, pf_flags: u32, fourcc: &[u8; 4], bits: u32, masks: [u32; 4], pitch: Option) -> Vec { + let mut b = vec![0u8; 128]; + b[0..4].copy_from_slice(b"DDS "); + let mut put = |at: usize, v: u32| b[at..at + 4].copy_from_slice(&v.to_le_bytes()); + put(4, 124); + put(8, 0x1007 | if pitch.is_some() { DDSD_PITCH } else { 0 }); + put(12, height); + put(16, width); + put(20, pitch.unwrap_or(0)); + put(76, 32); + put(80, pf_flags); + put(88, bits); + for (i, m) in masks.into_iter().enumerate() { + put(92 + 4 * i, m); + } + put(108, 0x1000); + b[84..88].copy_from_slice(fourcc); + b +} + +/// A DX10 texture header for DXGI format `dxgi`. +fn dx10(width: u32, height: u32, dxgi: u32) -> Vec { + let mut b = header(width, height, DDPF_FOURCC, b"DX10", 0, [0; 4], None); + for v in [dxgi, 3, 0, 1, 0] { + b.extend(v.to_le_bytes()); + } + b +} + +/// `len` bytes of noise. +fn noise(len: usize, seed: u64) -> Vec { + let mut rng = Lcg(seed); + (0..len).map(|_| rng.next()).collect() +} + +/// Hand-built files in the layouts the writer never makes: RGBA and BGRA through the DX10 +/// header, padded rows, 24- and 16-bit masks, and block data of pure noise (which reaches BC1's +/// three-colour mode and BC3's six-value mode as often as their other modes). +fn foreign_files() -> Vec<(String, Vec)> { + let (w, h) = (37u32, 23u32); + let px = (w * h) as usize; + let blocks = (w.div_ceil(4) * h.div_ceil(4)) as usize; + let mut files = Vec::new(); + let mut add = |name: &str, mut head: Vec, body: Vec| { + head.extend(body); + files.push((name.to_string(), head)); + }; + add("dx10-rgba8", dx10(w, h, 28), noise(px * 4, 1)); + add("dx10-bgra8", dx10(w, h, 87), noise(px * 4, 2)); + add("dx10-bgrx8", dx10(w, h, 88), noise(px * 4, 3)); + add("legacy-rgba8", header(w, h, DDPF_RGB | DDPF_ALPHAPIXELS, &[0; 4], 32, [0xff, 0xff00, 0xff_0000, 0xff00_0000], None), noise(px * 4, 4)); + add( + "legacy-bgra8-padded", + header(w, h, DDPF_RGB | DDPF_ALPHAPIXELS, &[0; 4], 32, [0xff_0000, 0xff00, 0xff, 0xff00_0000], Some(w * 4 + 12)), + noise((w * 4 + 12) as usize * h as usize, 5), + ); + add("legacy-rgb24", header(w, h, DDPF_RGB, &[0; 4], 24, [0xff_0000, 0xff00, 0xff, 0], None), noise(px * 3, 6)); + add("legacy-rgb565", header(w, h, DDPF_RGB, &[0; 4], 16, [0xf800, 0x07e0, 0x001f, 0], None), noise(px * 2, 7)); + add("legacy-argb4444", header(w, h, DDPF_RGB | DDPF_ALPHAPIXELS, &[0; 4], 16, [0x0f00, 0x00f0, 0x000f, 0xf000], None), noise(px * 2, 8)); + add("legacy-l8-padded", header(w, h, DDPF_LUMINANCE, &[0; 4], 8, [0xff, 0, 0, 0], Some(w + 3)), noise((w + 3) as usize * h as usize, 9)); + add("legacy-l8a8", header(w, h, DDPF_LUMINANCE | DDPF_ALPHAPIXELS, &[0; 4], 16, [0xff, 0, 0, 0xff00], None), noise(px * 2, 10)); + add("legacy-dxt1-noise", header(w, h, DDPF_FOURCC, b"DXT1", 0, [0; 4], None), noise(blocks * 8, 11)); + add("legacy-dxt3-noise", header(w, h, DDPF_FOURCC, b"DXT3", 0, [0; 4], None), noise(blocks * 16, 12)); + add("legacy-dxt5-noise", header(w, h, DDPF_FOURCC, b"DXT5", 0, [0; 4], None), noise(blocks * 16, 13)); + add("dx10-bc1-noise", dx10(w, h, 71), noise(blocks * 8, 14)); + add("dx10-bc3-noise", dx10(w, h, 77), noise(blocks * 16, 15)); + files +} + +/// Every case and its digest, in a fixed order. +fn digests() -> Vec<(String, String)> { + let mut out = Vec::new(); + // Odd sizes, the sub-block sizes and the one-pixel strips whose mip chains reach the + // downsampler's edge spans in only one direction. + let sizes = [(517u32, 389u32), (1, 1), (2, 3), (3, 2), (5, 5), (1, 9), (9, 1), (6, 13)]; + let layouts = [ChannelLayout::Rgba, ChannelLayout::Rgb, ChannelLayout::GrayA, ChannelLayout::Gray]; + for (i, &(w, h)) in sizes.iter().enumerate() { + for layout in layouts { + // The large picture in every layout; the small ones in RGBA and gray. + if i > 0 && !matches!(layout, ChannelLayout::Rgba | ChannelLayout::Gray) { + continue; + } + let img = image(w, h, layout, 0x5eed + i as u64); + for compression in COMPRESSIONS { + for mipmaps in [false, true] { + let name = format!("{layout:?}-{w}x{h}-{}-mips{}", compression.name(), u8::from(mipmaps)); + let opts = EncodeOptions { dds: DdsOptions { compression, mipmaps }, ..Default::default() }; + let file = encode(&img, Format::Dds, &opts).unwrap_or_else(|e| panic!("{name}: {e}")); + out.push((format!("{name}-encode"), sha256_hex(&file))); + let back = decode(&file).unwrap_or_else(|e| panic!("{name}: {e}")); + out.push((format!("{name}-decode"), image_digest(&back))); + } + } + } + } + for (name, file) in foreign_files() { + let back = decode(&file).unwrap_or_else(|e| panic!("{name}: {e}")); + out.push((format!("{name}-decode"), image_digest(&back))); + } + out +} + +#[test] +fn dds_output_matches_the_pinned_digests() { + let got = digests(); + let pinned = |name: &str| PINNED.iter().find(|(n, _)| *n == name).map(|(_, d)| *d); + let mismatched: Vec<&str> = got.iter().filter(|(name, digest)| pinned(name) != Some(digest.as_str())).map(|(n, _)| n.as_str()).collect(); + let stale: Vec<&str> = PINNED.iter().filter(|(n, _)| !got.iter().any(|(g, _)| g == n)).map(|(n, _)| *n).collect(); + if !mismatched.is_empty() || !stale.is_empty() { + eprintln!("const PINNED: &[(&str, &str)] = &["); + for (name, digest) in &got { + eprintln!(" (\"{name}\", \"{digest}\"),"); + } + eprintln!("];"); + panic!("{} digests differ or are new: {mismatched:?}; {} pinned cases are gone: {stale:?}", mismatched.len(), stale.len()); + } +} diff --git a/crates/image-codecs/tests/it/main.rs b/crates/image-codecs/tests/it/main.rs index 4eda74a26..78762de5f 100644 --- a/crates/image-codecs/tests/it/main.rs +++ b/crates/image-codecs/tests/it/main.rs @@ -3,6 +3,7 @@ mod camera_raw; mod common; mod dds; +mod dds_oracle; mod dds_write; mod decode_warnings; mod deep_exr; From a98aa0b46218e24c5272b89b72e1226077b93b5a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 23:28:54 +0100 Subject: [PATCH 020/209] Move the DDS reader and writer into a craft-dds crate without dependencies craft-dds holds the header parsing, the uncompressed and BC1-BC3 readers and writers and the block encoder, with its own pixel layout and error types and rayon behind a `parallel` feature. It parses the header first (Texture::parse) so a caller can check the size against its limits before anything is allocated. craft-image-codecs reads and writes DDS through it, re-exports its DdsCompression and DdsOptions and maps its errors onto CodecError with the same messages, so its API and output stay the same. Apps that only need DDS can depend on craft-dds without the other codecs and their licences. --- Cargo.lock | 8 + Cargo.toml | 1 + README.md | 5 + crates/dds/Cargo.toml | 19 + crates/dds/README.md | 43 + .../src/codecs/dds_bc.rs => dds/src/bc.rs} | 0 crates/dds/src/lib.rs | 755 ++++++++++++++++++ crates/image-codecs/Cargo.toml | 3 + crates/image-codecs/README.md | 5 +- crates/image-codecs/src/codecs.rs | 1 - crates/image-codecs/src/codecs/dds.rs | 572 +------------ crates/image-codecs/src/options.rs | 56 +- 12 files changed, 875 insertions(+), 593 deletions(-) create mode 100644 crates/dds/Cargo.toml create mode 100644 crates/dds/README.md rename crates/{image-codecs/src/codecs/dds_bc.rs => dds/src/bc.rs} (100%) create mode 100644 crates/dds/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 7ed516463..406340734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1480,6 +1480,13 @@ dependencies = [ "proptest", ] +[[package]] +name = "craft-dds" +version = "0.1.0" +dependencies = [ + "rayon", +] + [[package]] name = "craft-deckcraft-format" version = "0.1.0" @@ -1925,6 +1932,7 @@ name = "craft-image-codecs" version = "0.1.0" dependencies = [ "avif-serialize", + "craft-dds", "craft-heif", "craft-jpeg", "craft-media-testkit", diff --git a/Cargo.toml b/Cargo.toml index 48a2edf1d..4d31e635d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ craft-print = { path = "crates/print" } craft-proof = { path = "crates/proof" } craft-raster-codecs = { path = "crates/raster-codecs" } craft-image-codecs = { path = "crates/image-codecs" } +craft-dds = { path = "crates/dds" } craft-psd = { path = "crates/psd" } craft-pst = { path = "crates/pst" } craft-quit = { path = "crates/quit" } diff --git a/README.md b/README.md index 0b4b9a415..af98f90e7 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,11 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit black and white levels and as-shot white balance. The complete derivation record (plan, sources log, scripts, verification) is in `crates/crw/clean-room/`. Colour stays in each app. - [`craft-csv`](crates/csv/README.md): delimited text (CSV, TSV) and SYLK: encoding detection (UTF-8, UTF-16, Windows-1252 or Windows-1256), delimiter sniffing, a record parser with a cap on fields per record, field quoting for writers, and a SYLK (`.slk`) reader with R1C1 formulas rewritten to A1. +- [`craft-dds`](crates/dds/README.md): DirectDraw Surface (DDS) textures, read and written: the + top mip level of uncompressed and BC1 to BC3 (DXT1/3/5) textures, legacy and DX10 headers, and + our own block encoder with optional mip chains. No dependencies (rayon behind `parallel`), so an + app that only exports textures needn't take the whole `craft-image-codecs` codec set and its + licences; `craft-image-codecs` reads and writes DDS through it. - [`craft-dialogs`](crates/dialogs/README.md): native file and message dialogs that never block the event loop: queued requests shown at the end of the frame and answered on a later frame, free-standing Open and Folder panels and Save sheets on macOS, case-insensitive filters. What diff --git a/crates/dds/Cargo.toml b/crates/dds/Cargo.toml new file mode 100644 index 000000000..e3cee5696 --- /dev/null +++ b/crates/dds/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "craft-dds" +description = "DirectDraw Surface (DDS) textures: uncompressed and BC1-BC3 block compression, read and write, without dependencies" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[features] +default = [] +# Encode rows of blocks and mip levels on several threads. +parallel = ["dep:rayon"] + +[dependencies] +rayon = { workspace = true, optional = true } diff --git a/crates/dds/README.md b/crates/dds/README.md new file mode 100644 index 000000000..465975ff2 --- /dev/null +++ b/crates/dds/README.md @@ -0,0 +1,43 @@ +# craft-dds + +DirectDraw Surface (DDS) textures, after Microsoft's public DDS and block-compression +documentation, with no dependencies. `craft-image-codecs` reads and writes DDS through this +crate and re-exports its `DdsCompression` and `DdsOptions`; an app that only needs DDS (a texture +exporter, say) can depend on it alone and leave out the other codecs and their licences. + +```toml +craft-dds = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +| | Read | Write | +|---|---|---| +| Uncompressed | 8, 16, 24 or 32 bits per pixel described by channel masks, RGB or luminance, with or without alpha; padded rows (`DDSD_PITCH`) | BGRA, BGRX, luminance, luminance with alpha | +| Block-compressed | BC1, BC2, BC3 (DXT1, DXT3, DXT5) | BC1, BC2, BC3 | +| Headers | legacy and DX10 | legacy | +| Mip levels | the top level only | optional full chain, box-filtered | + +Cubemaps, texture arrays, volume textures, premultiplied DXT2/DXT4 and the other DXGI formats +are refused with `DdsError::Unsupported` rather than read in part. + +Pixels are 8 bits per channel, interleaved and top row first, in one of four `Layout`s (`Gray`, +`GrayA`, `Rgb`, `Rgba`). Block compressions are written from RGBA. + +```rust +use craft_dds::{DdsCompression, DdsOptions, Layout, Texture}; + +let pixels = [10, 20, 30, 255].repeat(6 * 5); +let file = craft_dds::encode(6, 5, Layout::Rgba, &pixels, DdsOptions { compression: DdsCompression::Bc1, mipmaps: true })?; +// The header first: the size and layout are known before anything is allocated. +let texture = Texture::parse(&file)?; +assert_eq!((texture.width(), texture.height(), texture.layout()), (6, 5, Layout::Rgba)); +let decoded = texture.decode()?; +``` + +The block encoder is our own: RGB565 endpoints fitted along each block's principal axis and +refined by least squares, indices picked against the palette the reader decodes; BC1 turns texels +under half opacity into its transparent black, and BC3 picks whichever alpha mode fits. + +## Features + +- `parallel`: encodes rows of blocks and mip levels on several threads with rayon. + `craft-image-codecs` turns it on for native targets. diff --git a/crates/image-codecs/src/codecs/dds_bc.rs b/crates/dds/src/bc.rs similarity index 100% rename from crates/image-codecs/src/codecs/dds_bc.rs rename to crates/dds/src/bc.rs diff --git a/crates/dds/src/lib.rs b/crates/dds/src/lib.rs new file mode 100644 index 000000000..a98a08631 --- /dev/null +++ b/crates/dds/src/lib.rs @@ -0,0 +1,755 @@ +//! DirectDraw Surface (DDS) textures, after Microsoft's public DDS documentation ("DDS", +//! "DDS_HEADER", "DDS_PIXELFORMAT", "DDS_HEADER_DXT10"). +//! +//! Reading: the top mip level of a 2D texture, either uncompressed (8 to 32 bits per pixel +//! described by channel masks, RGB or luminance) or block-compressed BC1/BC2/BC3 +//! (DXT1/DXT3/DXT5), with or without the DX10 header. Cubemaps, texture arrays and volume +//! textures are refused rather than flattened to one face or slice. +//! +//! Writing: a 2D texture, uncompressed (BGRA or BGRX, luminance with or without alpha) or +//! BC1/BC2/BC3, optionally with its full mip chain. The legacy header describes all of these, +//! so no DX10 header is written; formats only DXGI can name (BC4–BC7, sRGB) aren't written. +//! +//! Pixels are 8 bits per channel, interleaved, top row first, in one of the [`Layout`]s. The +//! crate has no dependencies; the `parallel` feature encodes rows of blocks on several threads +//! with rayon. +//! +//! ``` +//! use craft_dds::{DdsCompression, DdsOptions, Layout}; +//! +//! let pixels = [10, 20, 30, 255].repeat(6 * 5); +//! let options = DdsOptions { compression: DdsCompression::Bc1, mipmaps: true }; +//! let file = craft_dds::encode(6, 5, Layout::Rgba, &pixels, options)?; +//! let texture = craft_dds::Texture::parse(&file)?; +//! assert_eq!((texture.width(), texture.height(), texture.layout()), (6, 5, Layout::Rgba)); +//! assert_eq!(texture.decode()?.len(), 6 * 5 * 4); +//! # Ok::<(), craft_dds::DdsError>(()) +//! ``` +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod bc; + +use std::borrow::Cow; +use std::fmt; + +use bc::{Block, Mask}; + +/// The channels of a pixel, 8 bits each, in this order. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Layout { + Gray, + /// Gray, then straight (not premultiplied) alpha. + GrayA, + Rgb, + /// Red, green, blue, then straight alpha. + Rgba, +} + +impl Layout { + /// Samples per pixel. + pub fn channels(self) -> usize { + match self { + Layout::Gray => 1, + Layout::GrayA => 2, + Layout::Rgb => 3, + Layout::Rgba => 4, + } + } +} + +/// Why a texture could not be read or written. More kinds may be added, so a `match` on it needs +/// a wildcard arm. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum DdsError { + /// The file is damaged or not a DDS file. + Malformed(String), + /// A valid texture in a form this crate doesn't read (a cubemap, an array, a compression it + /// doesn't decode). + Unsupported(String), + /// The pixels can't be written with the options given. + Encode(String), + /// An output buffer of this many bytes could not be allocated. + Alloc(usize), +} + +impl fmt::Display for DdsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DdsError::Malformed(reason) => write!(f, "malformed DDS: {reason}"), + DdsError::Unsupported(reason) => write!(f, "unsupported DDS: {reason}"), + DdsError::Encode(reason) => write!(f, "DDS encode failed: {reason}"), + DdsError::Alloc(len) => write!(f, "cannot allocate {len} bytes for the texture"), + } + } +} + +impl std::error::Error for DdsError {} + +/// How a DDS texture stores its pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DdsCompression { + /// Uncompressed, 8 bits per channel: BGRA (BGRX without alpha), or luminance with or + /// without alpha for grayscale. Lossless. + #[default] + None, + /// BC1 (DXT1): 4 bits per pixel. Alpha is on or off: texels under half opacity become + /// transparent black. + Bc1, + /// BC2 (DXT3): 8 bits per pixel, with 4 bits of explicit alpha per texel. + Bc2, + /// BC3 (DXT5): 8 bits per pixel, with smooth interpolated alpha. + Bc3, +} + +impl DdsCompression { + pub const ALL: [DdsCompression; 4] = [DdsCompression::None, DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3]; + + /// The name scripts and the CLI use: `none`, `bc1`, `bc2` or `bc3`. + pub fn name(self) -> &'static str { + match self { + DdsCompression::None => "none", + DdsCompression::Bc1 => "bc1", + DdsCompression::Bc2 => "bc2", + DdsCompression::Bc3 => "bc3", + } + } + + /// [`Self::name`] back to a compression, also taking the DXT names (`dxt1`, `dxt3`, + /// `dxt5`), in any case. Anything else is `None`. + pub fn from_name(name: &str) -> Option { + match name.to_ascii_lowercase().as_str() { + "none" => Some(DdsCompression::None), + "bc1" | "dxt1" => Some(DdsCompression::Bc1), + "bc2" | "dxt3" => Some(DdsCompression::Bc2), + "bc3" | "dxt5" => Some(DdsCompression::Bc3), + _ => None, + } + } + + /// A block compression (lossy), not uncompressed pixels. + pub fn is_block(self) -> bool { + self != DdsCompression::None + } +} + +/// What a DDS file is written with. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DdsOptions { + pub compression: DdsCompression, + /// Write the full mip chain down to 1×1, each level a box filter of the one above. + pub mipmaps: bool, +} + +/// `DDS ` plus the 124-byte DDS_HEADER. +const HEADER: usize = 128; +/// DDS_HEADER_DXT10, present when the FourCC is `DX10`. +const DX10_HEADER: usize = 20; + +const DDSD_DEPTH: u32 = 0x80_0000; +const DDPF_ALPHAPIXELS: u32 = 0x1; +const DDPF_FOURCC: u32 = 0x4; +const DDPF_RGB: u32 = 0x40; +const DDPF_LUMINANCE: u32 = 0x2_0000; +const DDSCAPS2_CUBEMAP: u32 = 0x200; +const DDSCAPS2_CUBEMAP_FACES: u32 = 0xfc00; +const DDSCAPS2_VOLUME: u32 = 0x20_0000; +const DIMENSION_TEXTURE2D: u32 = 3; +const MISC_TEXTURECUBE: u32 = 0x4; + +fn bad(reason: impl fmt::Display) -> DdsError { + DdsError::Malformed(reason.to_string()) +} +fn unsupported(reason: impl Into) -> DdsError { + DdsError::Unsupported(reason.into()) +} +fn le32(b: &[u8], at: usize) -> Result { + let bytes = b.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or_else(|| bad("truncated header"))?; + Ok(u32::from_le_bytes(bytes)) +} + +/// Whether the header declares more than the one image a reader takes: further mip levels, +/// cubemap faces, volume slices or array textures, even without their presence flags. +pub fn has_additional_images(bytes: &[u8]) -> Result { + let depth = le32(bytes, 24)?; + let mips = le32(bytes, 28)?; + let faces = (le32(bytes, 112)? & DDSCAPS2_CUBEMAP_FACES).count_ones(); + let dx10 = le32(bytes, 80)? & DDPF_FOURCC != 0 && bytes.get(84..88) == Some(b"DX10"); + let (cube, array) = if dx10 { (le32(bytes, 136)? & MISC_TEXTURECUBE != 0, le32(bytes, 140)?) } else { (false, 0) }; + Ok(depth > 1 || mips > 1 || faces > 1 || cube || array > 1) +} + +/// A FourCC as text when it is printable, else as the number legacy writers store there. +fn fourcc_name(fourcc: u32) -> String { + let bytes = fourcc.to_le_bytes(); + if bytes.iter().all(|c| c.is_ascii_graphic() || *c == b' ') { bytes.iter().map(|&c| char::from(c)).collect() } else { fourcc.to_string() } +} + +/// One channel of an uncompressed pixel: where it sits and its largest value. +#[derive(Clone, Copy)] +struct Field { + shift: u32, + max: u32, +} + +impl Field { + /// `None` for an empty mask; an error for one with gaps or wider than the pixel. + fn new(mask: u32, bits: u32) -> Result, DdsError> { + if mask == 0 { + return Ok(None); + } + let shift = mask.trailing_zeros(); + let max = mask >> shift; + if max & max.wrapping_add(1) != 0 || (bits < 32 && mask >> bits != 0) { + return Err(bad(format!("channel mask {mask:#x} is not a contiguous field of a {bits}-bit pixel"))); + } + Ok(Some(Field { shift, max })) + } + + fn get(self, pixel: u32) -> u8 { + let v = u64::from((pixel >> self.shift) & self.max); + let max = u64::from(self.max); + ((v * 255 + max / 2) / max) as u8 + } +} + +#[derive(Clone, Copy)] +enum Encoding { + /// Uncompressed: bytes per pixel, then red (or luminance), green, blue and alpha. + Masks { + bytes: usize, + gray: bool, + fields: [Option; 4], + }, + Bc1, + Bc2, + Bc3, +} + +impl Encoding { + fn masks(bits: u32, masks: [u32; 4], gray: bool) -> Result { + if !matches!(bits, 8 | 16 | 24 | 32) { + return Err(bad(format!("{bits} bits per pixel is not 8, 16, 24 or 32"))); + } + let mut fields = [None; 4]; + for (field, mask) in fields.iter_mut().zip(masks) { + *field = Field::new(mask, bits)?; + } + let colour = if gray { fields[0].is_some() } else { fields[..3].iter().any(Option::is_some) }; + if !colour { + return Err(bad("the pixel format has no colour channel masks")); + } + Ok(Encoding::Masks { bytes: bits as usize / 8, gray, fields }) + } + + fn layout(&self) -> Layout { + match self { + Encoding::Masks { gray: true, fields, .. } if fields[3].is_some() => Layout::GrayA, + Encoding::Masks { gray: true, .. } => Layout::Gray, + Encoding::Masks { fields, .. } if fields[3].is_none() => Layout::Rgb, + _ => Layout::Rgba, + } + } + + /// Stored row stride and byte size of the top mip level (block rows for BC). + fn level_layout(&self, width: u32, height: u32, pitch: Option) -> Result<(usize, usize), DdsError> { + let overflow = || bad("pixel data size overflow"); + let (w, h) = (usize::try_from(width).map_err(|_| overflow())?, usize::try_from(height).map_err(|_| overflow())?); + let (row_bytes, rows) = match self { + Encoding::Masks { bytes, .. } => { + let packed = w.checked_mul(*bytes).ok_or_else(overflow)?; + let row_bytes = match pitch { + Some(pitch) => usize::try_from(pitch).map_err(|_| overflow())?, + None => packed, + }; + if row_bytes < packed { + return Err(bad(format!("row pitch {row_bytes} is smaller than the {packed}-byte pixel row"))); + } + (row_bytes, h) + } + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { + let block = if matches!(self, Encoding::Bc1) { 8 } else { 16 }; + (w.div_ceil(4).checked_mul(block).ok_or_else(overflow)?, h.div_ceil(4)) + } + }; + let size = row_bytes.checked_mul(rows).ok_or_else(overflow)?; + Ok((row_bytes, size)) + } +} + +/// The legacy pixel format: a FourCC, or channel masks. +fn legacy_encoding(b: &[u8]) -> Result { + let (flags, fourcc, bits) = (le32(b, 80)?, le32(b, 84)?, le32(b, 88)?); + let alpha = if flags & DDPF_ALPHAPIXELS != 0 { le32(b, 104)? } else { 0 }; + if flags & DDPF_FOURCC != 0 { + return match &fourcc.to_le_bytes() { + b"DXT1" => Ok(Encoding::Bc1), + b"DXT3" => Ok(Encoding::Bc2), + b"DXT5" => Ok(Encoding::Bc3), + b"DXT2" | b"DXT4" => Err(unsupported("premultiplied-alpha DXT2/DXT4 textures are not supported")), + _ => Err(unsupported(format!("compression `{}` is not supported (DXT1, DXT3 and DXT5 are)", fourcc_name(fourcc)))), + }; + } + if flags & DDPF_RGB != 0 { + return Encoding::masks(bits, [le32(b, 92)?, le32(b, 96)?, le32(b, 100)?, alpha], false); + } + if flags & DDPF_LUMINANCE != 0 { + return Encoding::masks(bits, [le32(b, 92)?, 0, 0, alpha], true); + } + Err(unsupported(format!("pixel format flags {flags:#x} are not RGB, luminance or a supported compression"))) +} + +/// The DX10 header's DXGI format, refusing anything but a single 2D texture. +fn dx10_encoding(b: &[u8]) -> Result { + let (dxgi, dimension, misc, array) = (le32(b, HEADER)?, le32(b, HEADER + 4)?, le32(b, HEADER + 8)?, le32(b, HEADER + 12)?); + if misc & MISC_TEXTURECUBE != 0 { + return Err(unsupported("cubemaps are not supported")); + } + if dimension != DIMENSION_TEXTURE2D { + return Err(unsupported(format!("only 2D textures are supported (resource dimension {dimension})"))); + } + if array > 1 { + return Err(unsupported(format!("texture arrays are not supported ({array} textures)"))); + } + const RGBA: [u32; 4] = [0xff, 0xff00, 0xff_0000, 0xff00_0000]; + const BGRA: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0xff00_0000]; + const BGRX: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0]; + match dxgi { + 27..=29 => Encoding::masks(32, RGBA, false), + 87 | 90 | 91 => Encoding::masks(32, BGRA, false), + 88 | 92 | 93 => Encoding::masks(32, BGRX, false), + 70..=72 => Ok(Encoding::Bc1), + 73..=75 => Ok(Encoding::Bc2), + 76..=78 => Ok(Encoding::Bc3), + _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1, BC2, BC3 and 8-bit RGBA or BGRA are)"))), + } +} + +/// A DDS file whose header has been read: its size and pixel layout are known before +/// [`Texture::decode`] allocates anything, so a caller can check them against its limits. +pub struct Texture<'a> { + bytes: &'a [u8], + flags: u32, + width: u32, + height: u32, + encoding: Encoding, + /// Where the top level's pixel data starts. + start: usize, +} + +impl<'a> Texture<'a> { + /// Reads the header, refusing what [`Texture::decode`] can't read. + pub fn parse(b: &'a [u8]) -> Result, DdsError> { + if !b.starts_with(b"DDS ") { + return Err(bad("missing the `DDS ` signature")); + } + if le32(b, 4)? != 124 { + return Err(bad("header size is not 124")); + } + let (flags, height, width, depth, caps2) = (le32(b, 8)?, le32(b, 12)?, le32(b, 16)?, le32(b, 24)?, le32(b, 112)?); + if caps2 & DDSCAPS2_CUBEMAP != 0 { + return Err(unsupported("cubemaps are not supported")); + } + if caps2 & DDSCAPS2_VOLUME != 0 || (flags & DDSD_DEPTH != 0 && depth > 1) { + return Err(unsupported("volume textures are not supported")); + } + let dx10 = le32(b, 80)? & DDPF_FOURCC != 0 && &le32(b, 84)?.to_le_bytes() == b"DX10"; + let (encoding, start) = if dx10 { (dx10_encoding(b)?, HEADER + DX10_HEADER) } else { (legacy_encoding(b)?, HEADER) }; + Ok(Texture { bytes: b, flags, width, height, encoding, start }) + } + + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } + + /// The layout [`Texture::decode`] returns. + pub fn layout(&self) -> Layout { + self.encoding.layout() + } + + /// The top mip level's pixels in [`Texture::layout`]. + pub fn decode(&self) -> Result, DdsError> { + let (b, flags, encoding) = (self.bytes, self.flags, self.encoding); + let (width, height) = (self.width, self.height); + // The same header field is a total level size for BC, but a row stride when DDSD_PITCH is set. + let pitch = if flags & DDSD_PITCH != 0 { Some(le32(b, 20)?) } else { None }; + let (row_pitch, size) = encoding.level_layout(width, height, pitch)?; + let data = b.get(self.start..).and_then(|d| d.get(..size)).ok_or_else(|| bad(format!("truncated pixel data: {size} bytes needed")))?; + let (w, h) = (width as usize, height as usize); + let channels = encoding.layout().channels(); + let row_size = w.checked_mul(channels).ok_or_else(|| bad("pixel size overflow"))?; + let total = row_size.checked_mul(h).ok_or_else(|| bad("pixel size overflow"))?; + let mut pixels = vec![0u8; total]; + match encoding { + Encoding::Masks { bytes, gray, fields } => { + for (src_row, dst_row) in data.chunks_exact(row_pitch).zip(pixels.chunks_exact_mut(row_size)) { + // Stop at the output width, leaving any stored row padding unread. + for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { + let pixel = src.iter().rev().fold(0u32, |v, &byte| v << 8 | u32::from(byte)); + let channel = |i: usize| fields[i].map_or(0, |f| f.get(pixel)); + let values = if gray { [channel(0), channel(3), 0, 0] } else { [channel(0), channel(1), channel(2), channel(3)] }; + dst.copy_from_slice(&values[..channels]); + } + } + } + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { + let block_size = if matches!(encoding, Encoding::Bc1) { 8 } else { 16 }; + let blocks_x = w.div_ceil(4); + for (i, block) in data.chunks_exact(block_size).enumerate() { + let texels = match encoding { + Encoding::Bc1 => bc1(block, true), + Encoding::Bc2 => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc2_alpha(block)), + _ => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc3_alpha(block)), + }; + let (bx, by) = ((i % blocks_x) * 4, (i / blocks_x) * 4); + for (t, texel) in texels.iter().enumerate() { + let (x, y) = (bx + t % 4, by + t / 4); + if x < w && y < h { + let at = (y * w + x) * 4; + if let Some(dst) = pixels.get_mut(at..at + 4) { + dst.copy_from_slice(texel); + } + } + } + } + } + } + Ok(pixels) + } +} + +/// A BC1 colour block. BC2 and BC3 colour blocks always have four colours; only BC1's +/// `c0 <= c1` order selects three colours and transparent black. +fn bc1(block: &[u8], one_bit_alpha: bool) -> [[u8; 4]; 16] { + let byte = |i: usize| block.get(i).copied().unwrap_or(0); + let (c0, c1) = (u16::from_le_bytes([byte(0), byte(1)]), u16::from_le_bytes([byte(2), byte(3)])); + let palette = bc::colour_palette(c0, c1, c0 > c1 || !one_bit_alpha); + let indices = u32::from_le_bytes([byte(4), byte(5), byte(6), byte(7)]); + std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) +} + +/// BC2: four explicit bits of alpha per texel. +fn bc2_alpha(block: &[u8]) -> [u8; 16] { + let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0))); + std::array::from_fn(|t| ((bits >> (4 * t)) & 0xf) as u8 * 17) +} + +/// BC3: two alpha endpoints and a 3-bit index per texel into eight values, or into six plus +/// 0 and 255 when `a0 <= a1`. +fn bc3_alpha(block: &[u8]) -> [u8; 16] { + let byte = |i: usize| block.get(i).copied().unwrap_or(0); + let palette = bc::alpha_palette(byte(0), byte(1)); + let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]); + std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) +} + +fn with_alpha(mut texels: [[u8; 4]; 16], alpha: [u8; 16]) -> [[u8; 4]; 16] { + for (texel, a) in texels.iter_mut().zip(alpha) { + texel[3] = a; + } + texels +} + +const DDSD_CAPS: u32 = 0x1; +const DDSD_HEIGHT: u32 = 0x2; +const DDSD_WIDTH: u32 = 0x4; +const DDSD_PITCH: u32 = 0x8; +const DDSD_PIXELFORMAT: u32 = 0x1000; +const DDSD_MIPMAPCOUNT: u32 = 0x2_0000; +const DDSD_LINEARSIZE: u32 = 0x8_0000; +const DDSCAPS_COMPLEX: u32 = 0x8; +const DDSCAPS_TEXTURE: u32 = 0x1000; +const DDSCAPS_MIPMAP: u32 = 0x40_0000; + +fn encode_err(reason: impl fmt::Display) -> DdsError { + DdsError::Encode(reason.to_string()) +} + +/// A zeroed buffer of `len` bytes, or [`DdsError::Alloc`] when it can't be reserved. +fn alloc_zeroed(len: usize) -> Result, DdsError> { + let mut v = Vec::new(); + v.try_reserve_exact(len).map_err(|_| DdsError::Alloc(len))?; + v.resize(len, 0); + Ok(v) +} + +/// How a written texture stores its pixels. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Store { + /// 8-bit luminance. + Luminance, + /// 8-bit luminance, then 8-bit alpha. + LuminanceAlpha, + /// Blue, green, red and an unused byte (written as 255, so a reader that takes it for + /// alpha still sees an opaque texture). + Bgrx, + Bgra, + Bc1, + Bc2, + Bc3, +} + +impl Store { + /// The storage for pixels of `layout`. Block compressions take RGBA only. + fn new(compression: DdsCompression, layout: Layout) -> Result { + Ok(match (compression, layout) { + (DdsCompression::None, Layout::Gray) => Store::Luminance, + (DdsCompression::None, Layout::GrayA) => Store::LuminanceAlpha, + (DdsCompression::None, Layout::Rgb) => Store::Bgrx, + (DdsCompression::None, Layout::Rgba) => Store::Bgra, + (DdsCompression::Bc1, Layout::Rgba) => Store::Bc1, + (DdsCompression::Bc2, Layout::Rgba) => Store::Bc2, + (DdsCompression::Bc3, Layout::Rgba) => Store::Bc3, + (_, layout) => return Err(encode_err(format!("{layout:?} pixels can't be written as `{}`", compression.name()))), + }) + } + + /// Bytes per pixel, or per 4×4 block for a block compression. + fn unit(self) -> usize { + match self { + Store::Luminance => 1, + Store::LuminanceAlpha => 2, + Store::Bgrx | Store::Bgra => 4, + Store::Bc1 => 8, + Store::Bc2 | Store::Bc3 => 16, + } + } + + fn is_block(self) -> bool { + matches!(self, Store::Bc1 | Store::Bc2 | Store::Bc3) + } + + /// Bytes of one row of pixels (or of 4×4 blocks) of a level `width` wide. + fn row_bytes(self, width: usize) -> Option { + if self.is_block() { width.div_ceil(4).checked_mul(self.unit()) } else { width.checked_mul(self.unit()) } + } + + fn level_bytes(self, width: usize, height: usize) -> Option { + let rows = if self.is_block() { height.div_ceil(4) } else { height }; + self.row_bytes(width)?.checked_mul(rows) + } + + /// DDS_PIXELFORMAT: flags, FourCC, bits per pixel and the red, green, blue and alpha masks. + fn pixel_format(self) -> (u32, [u8; 4], u32, [u32; 4]) { + match self { + Store::Luminance => (DDPF_LUMINANCE, [0; 4], 8, [0xff, 0, 0, 0]), + Store::LuminanceAlpha => (DDPF_LUMINANCE | DDPF_ALPHAPIXELS, [0; 4], 16, [0xff, 0, 0, 0xff00]), + Store::Bgrx => (DDPF_RGB, [0; 4], 32, [0xff_0000, 0xff00, 0xff, 0]), + Store::Bgra => (DDPF_RGB | DDPF_ALPHAPIXELS, [0; 4], 32, [0xff_0000, 0xff00, 0xff, 0xff00_0000]), + Store::Bc1 => (DDPF_FOURCC, *b"DXT1", 0, [0; 4]), + Store::Bc2 => (DDPF_FOURCC, *b"DXT3", 0, [0; 4]), + Store::Bc3 => (DDPF_FOURCC, *b"DXT5", 0, [0; 4]), + } + } +} + +/// Levels in a full mip chain: the larger side halves down to 1. +fn mip_levels(width: u32, height: u32) -> u32 { + 32 - width.max(height).max(1).leading_zeros() +} + +fn half((w, h): (usize, usize)) -> (usize, usize) { + ((w / 2).max(1), (h / 2).max(1)) +} + +/// A DDS file of `width` × `height` `pixels` in `layout`, interleaved and top row first. +/// Block compressions take [`Layout::Rgba`] pixels; uncompressed textures take every layout. +pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: DdsOptions) -> Result, DdsError> { + let store = Store::new(options.compression, layout)?; + let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(layout.channels())); + if expected != Some(pixels.len()) { + return Err(encode_err(format!("{} bytes of pixels for a {width}x{height} {layout:?} texture", pixels.len()))); + } + let levels = if options.mipmaps { mip_levels(width, height) } else { 1 }; + let top = (width as usize, height as usize); + // Every level's size, checked before anything is allocated. + let mut sizes = Vec::new(); + let mut size = top; + for _ in 0..levels { + sizes.push(store.level_bytes(size.0, size.1).ok_or_else(|| encode_err("texture size overflow"))?); + size = half(size); + } + let total = sizes.iter().try_fold(HEADER, |total, &s| total.checked_add(s)).ok_or_else(|| encode_err("texture size overflow"))?; + let header = header(width, height, store, levels, sizes.first().copied().unwrap_or(0))?; + let mut out = alloc_zeroed(total)?; + out.get_mut(..HEADER).ok_or_else(|| encode_err("output buffer too small"))?.copy_from_slice(&header); + + let channels = layout.channels(); + let mut level = Cow::Borrowed(pixels); + let (mut size, mut at) = (top, HEADER); + for (i, &bytes) in sizes.iter().enumerate() { + let dst = out.get_mut(at..at + bytes).ok_or_else(|| encode_err("output buffer too small"))?; + write_level(store, &level, size, dst)?; + at += bytes; + if i + 1 < sizes.len() { + let next = half(size); + level = Cow::Owned(downsample(&level, size, next, channels)?); + size = next; + } + } + Ok(out) +} + +/// The `DDS ` signature and DDS_HEADER. `top` is the top level's size in bytes. +fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Result<[u8; HEADER], DdsError> { + let mut b = [0u8; HEADER]; + let mut put = |at: usize, v: u32| { + if let Some(d) = b.get_mut(at..at + 4) { + d.copy_from_slice(&v.to_le_bytes()); + } + }; + // Uncompressed: the pitch of one row. Compressed: the whole top level (its "linear size"). + let (size_flag, pitch_or_size) = if store.is_block() { (DDSD_LINEARSIZE, Some(top)) } else { (DDSD_PITCH, store.row_bytes(width as usize)) }; + let pitch_or_size = pitch_or_size + .and_then(|v| u32::try_from(v).ok()) + .ok_or_else(|| encode_err(format!("a {width}x{height} level is too large for a DDS header")))?; + let mipmapped = levels > 1; + put(4, 124); + put(8, DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT | size_flag | if mipmapped { DDSD_MIPMAPCOUNT } else { 0 }); + put(12, height); + put(16, width); + put(20, pitch_or_size); + put(28, levels); + let (flags, fourcc, bits, masks) = store.pixel_format(); + put(76, 32); + put(80, flags); + put(84, u32::from_le_bytes(fourcc)); + put(88, bits); + for (i, mask) in masks.into_iter().enumerate() { + put(92 + 4 * i, mask); + } + put(108, DDSCAPS_TEXTURE | if mipmapped { DDSCAPS_COMPLEX | DDSCAPS_MIPMAP } else { 0 }); + b[..4].copy_from_slice(b"DDS "); + Ok(b) +} + +/// Runs `f` on each `len`-byte row of `out` with its index, in parallel with the `parallel` +/// feature. +fn rows(out: &mut [u8], len: usize, f: impl Fn(usize, &mut [u8]) + Send + Sync) { + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + out.par_chunks_mut(len.max(1)).enumerate().for_each(|(i, row)| f(i, row)); + } + #[cfg(not(feature = "parallel"))] + for (i, row) in out.chunks_mut(len.max(1)).enumerate() { + f(i, row); + } +} + +/// Writes one level of `width` × `height` pixels (in the image's layout) into `dst`. +fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst: &mut [u8]) -> Result<(), DdsError> { + let row_bytes = store.row_bytes(width).ok_or_else(|| encode_err("texture size overflow"))?; + match store { + Store::Luminance | Store::LuminanceAlpha => { + // The image's gray (and alpha) samples, in the same order. + let src = pixels.get(..dst.len()).ok_or_else(|| encode_err("the level is smaller than its size"))?; + dst.copy_from_slice(src); + } + Store::Bgrx | Store::Bgra => { + let channels = if store == Store::Bgra { 4 } else { 3 }; + rows(dst, row_bytes, |y, row| { + let src = pixels.get(y * width * channels..).unwrap_or_default(); + for (d, s) in row.as_chunks_mut::<4>().0.iter_mut().zip(src.chunks_exact(channels)) { + let byte = |i: usize| s.get(i).copied().unwrap_or(255); + *d = [byte(2), byte(1), byte(0), byte(3)]; + } + }); + } + Store::Bc1 | Store::Bc2 | Store::Bc3 => { + let (texels, _) = pixels.as_chunks::<4>(); + rows(dst, row_bytes, |by, row| { + for (bx, d) in row.chunks_exact_mut(store.unit()).enumerate() { + let (block, inside) = gather(texels, (width, height), bx, by); + let encoded: &[u8] = match store { + Store::Bc1 => &bc::encode_bc1(&block, inside), + Store::Bc2 => &bc::encode_bc2(&block, inside), + _ => &bc::encode_bc3(&block, inside), + }; + for (d, s) in d.iter_mut().zip(encoded) { + *d = *s; + } + } + }); + } + } + Ok(()) +} + +/// Block (`bx`, `by`)'s texels, and which of them lie inside the image. +fn gather(texels: &[[u8; 4]], (width, height): (usize, usize), bx: usize, by: usize) -> (Block, Mask) { + let mut block = [[0u8; 4]; 16]; + let mut inside: Mask = 0; + for (t, texel) in block.iter_mut().enumerate() { + let (x, y) = (bx * 4 + t % 4, by * 4 + t / 4); + if x < width + && y < height + && let Some(p) = texels.get(y * width + x) + { + *texel = *p; + inside |= 1 << t; + } + } + (block, inside) +} + +/// The next mip level: each pixel the average of the 2×2 pixels it covers, or of 2×3, 3×2 or +/// 3×3 along the last column and row of an odd-sized level, so no pixel is dropped. +fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), channels: usize) -> Result, DdsError> { + let len = nw.checked_mul(nh).and_then(|n| n.checked_mul(channels)).ok_or_else(|| encode_err("mip level size overflow"))?; + let mut dst = alloc_zeroed(len)?; + let span = |i: usize, n: usize, size: usize| 2 * i..=if i + 1 == n { size.saturating_sub(1) } else { 2 * i + 1 }; + rows(&mut dst, nw * channels, |y, row| { + let ys = span(y, nh, h); + for (x, px) in row.chunks_exact_mut(channels.max(1)).enumerate() { + let xs = span(x, nw, w); + for (c, v) in px.iter_mut().enumerate() { + let (mut sum, mut n) = (0u32, 0u32); + for sy in ys.clone() { + for sx in xs.clone() { + if let Some(&s) = src.get((sy * w + sx) * channels + c) { + sum += u32::from(s); + n += 1; + } + } + } + *v = ((sum + n / 2) / n.max(1)) as u8; + } + } + }); + Ok(dst) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_pixel_buffer_of_the_wrong_size_is_refused() { + let options = DdsOptions::default(); + assert!(matches!(encode(2, 2, Layout::Rgb, &[0; 11], options), Err(DdsError::Encode(_)))); + assert!(matches!(encode(2, 2, Layout::Rgb, &[0; 13], options), Err(DdsError::Encode(_)))); + assert!(encode(2, 2, Layout::Rgb, &[0; 12], options).is_ok()); + } + + #[test] + fn block_compressions_take_rgba_only() { + let options = DdsOptions { compression: DdsCompression::Bc3, mipmaps: false }; + assert_eq!(encode(1, 1, Layout::Rgb, &[1, 2, 3], options), Err(DdsError::Encode("Rgb pixels can't be written as `bc3`".into()))); + } + + #[test] + fn the_header_is_read_before_the_pixels() { + // A header that promises more pixel data than the file holds: its size and layout are + // known, and only decoding fails. + let file = encode(8, 8, Layout::Gray, &[7; 64], DdsOptions::default()).unwrap(); + let texture = Texture::parse(&file[..HEADER + 10]).unwrap(); + assert_eq!((texture.width(), texture.height(), texture.layout()), (8, 8, Layout::Gray)); + assert!(matches!(texture.decode(), Err(DdsError::Malformed(_)))); + assert_eq!(Texture::parse(&file).and_then(|t| t.decode()), Ok(vec![7; 64])); + } +} diff --git a/crates/image-codecs/Cargo.toml b/crates/image-codecs/Cargo.toml index 208ff3ce7..ef8d7b76e 100644 --- a/crates/image-codecs/Cargo.toml +++ b/crates/image-codecs/Cargo.toml @@ -20,6 +20,7 @@ heif = ["dep:craft-heif"] avif-decode = ["dep:craft-video", "craft-video/av1-decode"] [dependencies] +craft-dds = { workspace = true } crc32fast = "1.5" thiserror = { workspace = true } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff", "gif", "bmp", "tga", "ico", "qoi", "hdr"] } @@ -43,6 +44,8 @@ avif-serialize = { version = "0.8", optional = true } hayro-jpeg2000 = { version = "=0.3.5", default-features = false, features = ["std", "simd"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# DDS encodes rows of blocks on several threads, as the other parallel codecs here do on native targets. +craft-dds = { workspace = true, features = ["parallel"] } rayon = { workspace = true } craft-video = { workspace = true, optional = true } exr = { version = "1.73", default-features = false, features = ["rayon"] } diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index a0e1afaa4..27ddddb69 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -46,7 +46,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "< | BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` | | TGA | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | no | `image` (read) / built in (RLE write, one scanline per packet) | | ICO | yes | yes | U8 | RGBA (at most 256×256) | yes | no | no | no | no | no | no | `image` | -| DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC3, when `EncodeOptions::dds` asks for one) | built in | +| DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC3, when `EncodeOptions::dds` asks for one) | [`craft-dds`](../dds/README.md) | | Netpbm (PBM/PGM/PPM/PAM/PFM) | yes | yes | U8, U16, F32 (float: PFM gray/RGB only) | all six (PAM `CMYK`/`CMYK_ALPHA`) | yes (PAM) | no | no | no | no | no | no | built in | | QOI | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` | | OpenEXR | yes | yes | F16, F32 | Y, YA, RGB, RGBA | yes | no | no | no | no | no | no | `exr` | @@ -92,7 +92,8 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them Reading remains in the separate #2868 PR. The `ASYMMETRIC_EXCEPTIONS` entry stays until import is merged. -* **DDS.** The top mip level of a 2D texture: uncompressed pixels described by channel masks +* **DDS** (through [`craft-dds`](../dds/README.md), whose `DdsCompression` and `DdsOptions` are + re-exported). The top mip level of a 2D texture: uncompressed pixels described by channel masks (8, 16, 24 or 32 bits, RGB or luminance, with or without alpha) or BC1/BC2/BC3 (DXT1/DXT3/DXT5), with the legacy header or the DX10 one. Uncompressed row padding follows the header's `DDSD_PITCH` stride; without that flag, rows are tightly packed. Everything opens as U8; the diff --git a/crates/image-codecs/src/codecs.rs b/crates/image-codecs/src/codecs.rs index 00b94b7ec..f647809af 100644 --- a/crates/image-codecs/src/codecs.rs +++ b/crates/image-codecs/src/codecs.rs @@ -2,7 +2,6 @@ pub(crate) mod avif; #[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] pub(crate) mod avif_decode; pub(crate) mod dds; -pub(crate) mod dds_bc; pub(crate) mod deep_exr; pub(crate) mod dicom; pub(crate) mod dpx; diff --git a/crates/image-codecs/src/codecs/dds.rs b/crates/image-codecs/src/codecs/dds.rs index 9ed4e3f35..9d682b03f 100644 --- a/crates/image-codecs/src/codecs/dds.rs +++ b/crates/image-codecs/src/codecs/dds.rs @@ -1,557 +1,57 @@ -//! DirectDraw Surface (DDS) textures, after Microsoft's public DDS documentation ("DDS", -//! "DDS_HEADER", "DDS_PIXELFORMAT", "DDS_HEADER_DXT10"). -//! -//! Reading: the top mip level of a 2D texture, either uncompressed (8 to 32 bits per pixel -//! described by channel masks, RGB or luminance) or block-compressed BC1/BC2/BC3 -//! (DXT1/DXT3/DXT5), with or without the DX10 header. Cubemaps, texture arrays and volume -//! textures are refused rather than flattened to one face or slice. -//! -//! Writing: a 2D texture, uncompressed (BGRA or BGRX, luminance with or without alpha) or -//! BC1/BC2/BC3, optionally with its full mip chain. The legacy header describes all of these, -//! so no DX10 header is written; formats only DXGI can name (BC4–BC7, sRGB) aren't written. +//! DirectDraw Surface (DDS) textures through [`craft_dds`]: its errors and pixel layouts mapped +//! onto this crate's, and the decoder's size checked against [`Limits`] before it allocates. -use std::borrow::Cow; - -use super::dds_bc::{self, Block, Mask}; use crate::SampleType; use crate::error::CodecError; use crate::fidelity::Plan; use crate::format::Format; use crate::image::{ChannelLayout, Image}; -use crate::options::{DdsCompression, EncodeOptions, Limits}; +use crate::options::{EncodeOptions, Limits}; +use craft_dds::{DdsError, Layout, Texture}; -/// `DDS ` plus the 124-byte DDS_HEADER. -const HEADER: usize = 128; -/// DDS_HEADER_DXT10, present when the FourCC is `DX10`. -const DX10_HEADER: usize = 20; - -const DDSD_DEPTH: u32 = 0x80_0000; -const DDPF_ALPHAPIXELS: u32 = 0x1; -const DDPF_FOURCC: u32 = 0x4; -const DDPF_RGB: u32 = 0x40; -const DDPF_LUMINANCE: u32 = 0x2_0000; -const DDSCAPS2_CUBEMAP: u32 = 0x200; -const DDSCAPS2_CUBEMAP_FACES: u32 = 0xfc00; -const DDSCAPS2_VOLUME: u32 = 0x20_0000; -const DIMENSION_TEXTURE2D: u32 = 3; -const MISC_TEXTURECUBE: u32 = 0x4; - -fn bad(reason: impl std::fmt::Display) -> CodecError { - CodecError::malformed(Format::Dds, reason) +fn error(e: DdsError) -> CodecError { + match e { + DdsError::Malformed(reason) => CodecError::malformed(Format::Dds, reason), + DdsError::Unsupported(reason) => CodecError::unsupported(Format::Dds, reason), + DdsError::Encode(reason) => CodecError::encode(Format::Dds, reason), + DdsError::Alloc(len) => CodecError::LimitExceeded(format!("cannot allocate {len} elements for the image")), + other => CodecError::malformed(Format::Dds, other), + } } -fn unsupported(reason: impl Into) -> CodecError { - CodecError::unsupported(Format::Dds, reason) -} -fn le32(b: &[u8], at: usize) -> Result { - let bytes = b.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or_else(|| bad("truncated header"))?; - Ok(u32::from_le_bytes(bytes)) + +fn channel_layout(layout: Layout) -> ChannelLayout { + match layout { + Layout::Gray => ChannelLayout::Gray, + Layout::GrayA => ChannelLayout::GrayA, + Layout::Rgb => ChannelLayout::Rgb, + Layout::Rgba => ChannelLayout::Rgba, + } } /// Header-declared extra subresources cannot survive a single-image rewrite, even without presence flags. pub(crate) fn has_additional_images(bytes: &[u8]) -> Result { - let depth = le32(bytes, 24)?; - let mips = le32(bytes, 28)?; - let faces = (le32(bytes, 112)? & DDSCAPS2_CUBEMAP_FACES).count_ones(); - let dx10 = le32(bytes, 80)? & DDPF_FOURCC != 0 && bytes.get(84..88) == Some(b"DX10"); - let (cube, array) = if dx10 { (le32(bytes, 136)? & MISC_TEXTURECUBE != 0, le32(bytes, 140)?) } else { (false, 0) }; - Ok(depth > 1 || mips > 1 || faces > 1 || cube || array > 1) -} - -/// A FourCC as text when it is printable, else as the number legacy writers store there. -fn fourcc_name(fourcc: u32) -> String { - let bytes = fourcc.to_le_bytes(); - if bytes.iter().all(|c| c.is_ascii_graphic() || *c == b' ') { bytes.iter().map(|&c| char::from(c)).collect() } else { fourcc.to_string() } -} - -/// One channel of an uncompressed pixel: where it sits and its largest value. -#[derive(Clone, Copy)] -struct Field { - shift: u32, - max: u32, -} - -impl Field { - /// `None` for an empty mask; an error for one with gaps or wider than the pixel. - fn new(mask: u32, bits: u32) -> Result, CodecError> { - if mask == 0 { - return Ok(None); - } - let shift = mask.trailing_zeros(); - let max = mask >> shift; - if max & max.wrapping_add(1) != 0 || (bits < 32 && mask >> bits != 0) { - return Err(bad(format!("channel mask {mask:#x} is not a contiguous field of a {bits}-bit pixel"))); - } - Ok(Some(Field { shift, max })) - } - - fn get(self, pixel: u32) -> u8 { - let v = u64::from((pixel >> self.shift) & self.max); - let max = u64::from(self.max); - ((v * 255 + max / 2) / max) as u8 - } -} - -enum Encoding { - /// Uncompressed: bytes per pixel, then red (or luminance), green, blue and alpha. - Masks { - bytes: usize, - gray: bool, - fields: [Option; 4], - }, - Bc1, - Bc2, - Bc3, -} - -impl Encoding { - fn masks(bits: u32, masks: [u32; 4], gray: bool) -> Result { - if !matches!(bits, 8 | 16 | 24 | 32) { - return Err(bad(format!("{bits} bits per pixel is not 8, 16, 24 or 32"))); - } - let mut fields = [None; 4]; - for (field, mask) in fields.iter_mut().zip(masks) { - *field = Field::new(mask, bits)?; - } - let colour = if gray { fields[0].is_some() } else { fields[..3].iter().any(Option::is_some) }; - if !colour { - return Err(bad("the pixel format has no colour channel masks")); - } - Ok(Encoding::Masks { bytes: bits as usize / 8, gray, fields }) - } - - fn layout(&self) -> ChannelLayout { - match self { - Encoding::Masks { gray: true, fields, .. } if fields[3].is_some() => ChannelLayout::GrayA, - Encoding::Masks { gray: true, .. } => ChannelLayout::Gray, - Encoding::Masks { fields, .. } if fields[3].is_none() => ChannelLayout::Rgb, - _ => ChannelLayout::Rgba, - } - } - - /// Stored row stride and byte size of the top mip level (block rows for BC). - fn level_layout(&self, width: u32, height: u32, pitch: Option) -> Result<(usize, usize), CodecError> { - let overflow = || bad("pixel data size overflow"); - let (w, h) = (usize::try_from(width).map_err(|_| overflow())?, usize::try_from(height).map_err(|_| overflow())?); - let (row_bytes, rows) = match self { - Encoding::Masks { bytes, .. } => { - let packed = w.checked_mul(*bytes).ok_or_else(overflow)?; - let row_bytes = match pitch { - Some(pitch) => usize::try_from(pitch).map_err(|_| overflow())?, - None => packed, - }; - if row_bytes < packed { - return Err(bad(format!("row pitch {row_bytes} is smaller than the {packed}-byte pixel row"))); - } - (row_bytes, h) - } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { - let block = if matches!(self, Encoding::Bc1) { 8 } else { 16 }; - (w.div_ceil(4).checked_mul(block).ok_or_else(overflow)?, h.div_ceil(4)) - } - }; - let size = row_bytes.checked_mul(rows).ok_or_else(overflow)?; - Ok((row_bytes, size)) - } -} - -/// The legacy pixel format: a FourCC, or channel masks. -fn legacy_encoding(b: &[u8]) -> Result { - let (flags, fourcc, bits) = (le32(b, 80)?, le32(b, 84)?, le32(b, 88)?); - let alpha = if flags & DDPF_ALPHAPIXELS != 0 { le32(b, 104)? } else { 0 }; - if flags & DDPF_FOURCC != 0 { - return match &fourcc.to_le_bytes() { - b"DXT1" => Ok(Encoding::Bc1), - b"DXT3" => Ok(Encoding::Bc2), - b"DXT5" => Ok(Encoding::Bc3), - b"DXT2" | b"DXT4" => Err(unsupported("premultiplied-alpha DXT2/DXT4 textures are not supported")), - _ => Err(unsupported(format!("compression `{}` is not supported (DXT1, DXT3 and DXT5 are)", fourcc_name(fourcc)))), - }; - } - if flags & DDPF_RGB != 0 { - return Encoding::masks(bits, [le32(b, 92)?, le32(b, 96)?, le32(b, 100)?, alpha], false); - } - if flags & DDPF_LUMINANCE != 0 { - return Encoding::masks(bits, [le32(b, 92)?, 0, 0, alpha], true); - } - Err(unsupported(format!("pixel format flags {flags:#x} are not RGB, luminance or a supported compression"))) -} - -/// The DX10 header's DXGI format, refusing anything but a single 2D texture. -fn dx10_encoding(b: &[u8]) -> Result { - let (dxgi, dimension, misc, array) = (le32(b, HEADER)?, le32(b, HEADER + 4)?, le32(b, HEADER + 8)?, le32(b, HEADER + 12)?); - if misc & MISC_TEXTURECUBE != 0 { - return Err(unsupported("cubemaps are not supported")); - } - if dimension != DIMENSION_TEXTURE2D { - return Err(unsupported(format!("only 2D textures are supported (resource dimension {dimension})"))); - } - if array > 1 { - return Err(unsupported(format!("texture arrays are not supported ({array} textures)"))); - } - const RGBA: [u32; 4] = [0xff, 0xff00, 0xff_0000, 0xff00_0000]; - const BGRA: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0xff00_0000]; - const BGRX: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0]; - match dxgi { - 27..=29 => Encoding::masks(32, RGBA, false), - 87 | 90 | 91 => Encoding::masks(32, BGRA, false), - 88 | 92 | 93 => Encoding::masks(32, BGRX, false), - 70..=72 => Ok(Encoding::Bc1), - 73..=75 => Ok(Encoding::Bc2), - 76..=78 => Ok(Encoding::Bc3), - _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1, BC2, BC3 and 8-bit RGBA or BGRA are)"))), - } + craft_dds::has_additional_images(bytes).map_err(error) } pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result { - if !b.starts_with(b"DDS ") { - return Err(bad("missing the `DDS ` signature")); - } - if le32(b, 4)? != 124 { - return Err(bad("header size is not 124")); - } - let (flags, height, width, depth, caps2) = (le32(b, 8)?, le32(b, 12)?, le32(b, 16)?, le32(b, 24)?, le32(b, 112)?); - if caps2 & DDSCAPS2_CUBEMAP != 0 { - return Err(unsupported("cubemaps are not supported")); - } - if caps2 & DDSCAPS2_VOLUME != 0 || (flags & DDSD_DEPTH != 0 && depth > 1) { - return Err(unsupported("volume textures are not supported")); - } - let dx10 = le32(b, 80)? & DDPF_FOURCC != 0 && &le32(b, 84)?.to_le_bytes() == b"DX10"; - let (encoding, start) = if dx10 { (dx10_encoding(b)?, HEADER + DX10_HEADER) } else { (legacy_encoding(b)?, HEADER) }; - let layout = encoding.layout(); - limits.check(width, height, layout, SampleType::U8)?; - // The same header field is a total level size for BC, but a row stride when DDSD_PITCH is set. - let pitch = if flags & DDSD_PITCH != 0 { Some(le32(b, 20)?) } else { None }; - let (row_pitch, size) = encoding.level_layout(width, height, pitch)?; - let data = b.get(start..).and_then(|d| d.get(..size)).ok_or_else(|| bad(format!("truncated pixel data: {size} bytes needed")))?; - let (w, h) = (width as usize, height as usize); - let channels = layout.channels(); - let row_size = w.checked_mul(channels).ok_or_else(|| bad("pixel size overflow"))?; - let total = row_size.checked_mul(h).ok_or_else(|| bad("pixel size overflow"))?; - let mut pixels = vec![0u8; total]; - match encoding { - Encoding::Masks { bytes, gray, fields } => { - for (src_row, dst_row) in data.chunks_exact(row_pitch).zip(pixels.chunks_exact_mut(row_size)) { - // Stop at the output width, leaving any stored row padding unread. - for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { - let pixel = src.iter().rev().fold(0u32, |v, &byte| v << 8 | u32::from(byte)); - let channel = |i: usize| fields[i].map_or(0, |f| f.get(pixel)); - let values = if gray { [channel(0), channel(3), 0, 0] } else { [channel(0), channel(1), channel(2), channel(3)] }; - dst.copy_from_slice(&values[..channels]); - } - } - } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { - let block_size = if matches!(encoding, Encoding::Bc1) { 8 } else { 16 }; - let blocks_x = w.div_ceil(4); - for (i, block) in data.chunks_exact(block_size).enumerate() { - let texels = match encoding { - Encoding::Bc1 => bc1(block, true), - Encoding::Bc2 => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc2_alpha(block)), - _ => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc3_alpha(block)), - }; - let (bx, by) = ((i % blocks_x) * 4, (i / blocks_x) * 4); - for (t, texel) in texels.iter().enumerate() { - let (x, y) = (bx + t % 4, by + t / 4); - if x < w && y < h { - let at = (y * w + x) * 4; - if let Some(dst) = pixels.get_mut(at..at + 4) { - dst.copy_from_slice(texel); - } - } - } - } - } - } - Image::from_u8(width, height, layout, pixels) -} - -/// A BC1 colour block. BC2 and BC3 colour blocks always have four colours; only BC1's -/// `c0 <= c1` order selects three colours and transparent black. -fn bc1(block: &[u8], one_bit_alpha: bool) -> [[u8; 4]; 16] { - let byte = |i: usize| block.get(i).copied().unwrap_or(0); - let (c0, c1) = (u16::from_le_bytes([byte(0), byte(1)]), u16::from_le_bytes([byte(2), byte(3)])); - let palette = dds_bc::colour_palette(c0, c1, c0 > c1 || !one_bit_alpha); - let indices = u32::from_le_bytes([byte(4), byte(5), byte(6), byte(7)]); - std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) -} - -/// BC2: four explicit bits of alpha per texel. -fn bc2_alpha(block: &[u8]) -> [u8; 16] { - let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0))); - std::array::from_fn(|t| ((bits >> (4 * t)) & 0xf) as u8 * 17) -} - -/// BC3: two alpha endpoints and a 3-bit index per texel into eight values, or into six plus -/// 0 and 255 when `a0 <= a1`. -fn bc3_alpha(block: &[u8]) -> [u8; 16] { - let byte = |i: usize| block.get(i).copied().unwrap_or(0); - let palette = dds_bc::alpha_palette(byte(0), byte(1)); - let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]); - std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) -} - -fn with_alpha(mut texels: [[u8; 4]; 16], alpha: [u8; 16]) -> [[u8; 4]; 16] { - for (texel, a) in texels.iter_mut().zip(alpha) { - texel[3] = a; - } - texels -} - -const DDSD_CAPS: u32 = 0x1; -const DDSD_HEIGHT: u32 = 0x2; -const DDSD_WIDTH: u32 = 0x4; -const DDSD_PITCH: u32 = 0x8; -const DDSD_PIXELFORMAT: u32 = 0x1000; -const DDSD_MIPMAPCOUNT: u32 = 0x2_0000; -const DDSD_LINEARSIZE: u32 = 0x8_0000; -const DDSCAPS_COMPLEX: u32 = 0x8; -const DDSCAPS_TEXTURE: u32 = 0x1000; -const DDSCAPS_MIPMAP: u32 = 0x40_0000; - -fn encode_err(reason: impl std::fmt::Display) -> CodecError { - CodecError::encode(Format::Dds, reason) -} - -/// How a written texture stores its pixels. -#[derive(Clone, Copy, PartialEq, Eq)] -enum Store { - /// 8-bit luminance. - Luminance, - /// 8-bit luminance, then 8-bit alpha. - LuminanceAlpha, - /// Blue, green, red and an unused byte (written as 255, so a reader that takes it for - /// alpha still sees an opaque texture). - Bgrx, - Bgra, - Bc1, - Bc2, - Bc3, -} - -impl Store { - /// The storage for an image of `layout`, as [`crate::fidelity::plan`] chose it. - fn new(compression: DdsCompression, layout: ChannelLayout) -> Result { - Ok(match (compression, layout) { - (DdsCompression::None, ChannelLayout::Gray) => Store::Luminance, - (DdsCompression::None, ChannelLayout::GrayA) => Store::LuminanceAlpha, - (DdsCompression::None, ChannelLayout::Rgb) => Store::Bgrx, - (DdsCompression::None, ChannelLayout::Rgba) => Store::Bgra, - (DdsCompression::Bc1, ChannelLayout::Rgba) => Store::Bc1, - (DdsCompression::Bc2, ChannelLayout::Rgba) => Store::Bc2, - (DdsCompression::Bc3, ChannelLayout::Rgba) => Store::Bc3, - (_, layout) => return Err(encode_err(format!("{layout:?} pixels can't be written as `{}`", compression.name()))), - }) - } - - /// Bytes per pixel, or per 4×4 block for a block compression. - fn unit(self) -> usize { - match self { - Store::Luminance => 1, - Store::LuminanceAlpha => 2, - Store::Bgrx | Store::Bgra => 4, - Store::Bc1 => 8, - Store::Bc2 | Store::Bc3 => 16, - } - } - - fn is_block(self) -> bool { - matches!(self, Store::Bc1 | Store::Bc2 | Store::Bc3) - } - - /// Bytes of one row of pixels (or of 4×4 blocks) of a level `width` wide. - fn row_bytes(self, width: usize) -> Option { - if self.is_block() { width.div_ceil(4).checked_mul(self.unit()) } else { width.checked_mul(self.unit()) } - } - - fn level_bytes(self, width: usize, height: usize) -> Option { - let rows = if self.is_block() { height.div_ceil(4) } else { height }; - self.row_bytes(width)?.checked_mul(rows) - } - - /// DDS_PIXELFORMAT: flags, FourCC, bits per pixel and the red, green, blue and alpha masks. - fn pixel_format(self) -> (u32, [u8; 4], u32, [u32; 4]) { - match self { - Store::Luminance => (DDPF_LUMINANCE, [0; 4], 8, [0xff, 0, 0, 0]), - Store::LuminanceAlpha => (DDPF_LUMINANCE | DDPF_ALPHAPIXELS, [0; 4], 16, [0xff, 0, 0, 0xff00]), - Store::Bgrx => (DDPF_RGB, [0; 4], 32, [0xff_0000, 0xff00, 0xff, 0]), - Store::Bgra => (DDPF_RGB | DDPF_ALPHAPIXELS, [0; 4], 32, [0xff_0000, 0xff00, 0xff, 0xff00_0000]), - Store::Bc1 => (DDPF_FOURCC, *b"DXT1", 0, [0; 4]), - Store::Bc2 => (DDPF_FOURCC, *b"DXT3", 0, [0; 4]), - Store::Bc3 => (DDPF_FOURCC, *b"DXT5", 0, [0; 4]), - } - } -} - -/// Levels in a full mip chain: the larger side halves down to 1. -fn mip_levels(width: u32, height: u32) -> u32 { - 32 - width.max(height).max(1).leading_zeros() -} - -fn half((w, h): (usize, usize)) -> (usize, usize) { - ((w / 2).max(1), (h / 2).max(1)) + let texture = Texture::parse(b).map_err(error)?; + let layout = channel_layout(texture.layout()); + limits.check(texture.width(), texture.height(), layout, SampleType::U8)?; + let pixels = texture.decode().map_err(error)?; + Image::from_u8(texture.width(), texture.height(), layout, pixels) } pub(crate) fn encode(image: &Image, plan: Plan, opts: &EncodeOptions) -> Result, CodecError> { let img = image.converted(plan.layout, SampleType::U8); - let pixels = img.as_u8().ok_or_else(|| encode_err("the image was not converted to 8 bits per channel"))?; - let store = Store::new(opts.dds.compression, img.layout())?; - let (width, height) = img.dimensions(); - let levels = if opts.dds.mipmaps { mip_levels(width, height) } else { 1 }; - let top = (width as usize, height as usize); - // Every level's size, checked before anything is allocated. - let mut sizes = Vec::new(); - let mut size = top; - for _ in 0..levels { - sizes.push(store.level_bytes(size.0, size.1).ok_or_else(|| encode_err("texture size overflow"))?); - size = half(size); - } - let total = sizes.iter().try_fold(HEADER, |total, &s| total.checked_add(s)).ok_or_else(|| encode_err("texture size overflow"))?; - let header = header(width, height, store, levels, sizes.first().copied().unwrap_or(0))?; - let mut out = crate::options::alloc_zeroed::(total)?; - out.get_mut(..HEADER).ok_or_else(|| encode_err("output buffer too small"))?.copy_from_slice(&header); - - let channels = img.layout().channels(); - let mut level = Cow::Borrowed(pixels); - let (mut size, mut at) = (top, HEADER); - for (i, &bytes) in sizes.iter().enumerate() { - let dst = out.get_mut(at..at + bytes).ok_or_else(|| encode_err("output buffer too small"))?; - write_level(store, &level, size, dst)?; - at += bytes; - if i + 1 < sizes.len() { - let next = half(size); - level = Cow::Owned(downsample(&level, size, next, channels)?); - size = next; - } - } - Ok(out) -} - -/// The `DDS ` signature and DDS_HEADER. `top` is the top level's size in bytes. -fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Result<[u8; HEADER], CodecError> { - let mut b = [0u8; HEADER]; - let mut put = |at: usize, v: u32| { - if let Some(d) = b.get_mut(at..at + 4) { - d.copy_from_slice(&v.to_le_bytes()); - } + let pixels = img.as_u8().ok_or_else(|| CodecError::encode(Format::Dds, "the image was not converted to 8 bits per channel"))?; + let layout = match img.layout() { + ChannelLayout::Gray => Layout::Gray, + ChannelLayout::GrayA => Layout::GrayA, + ChannelLayout::Rgb => Layout::Rgb, + ChannelLayout::Rgba => Layout::Rgba, + // `fidelity::plan` only picks the four layouts above for DDS. + other => return Err(CodecError::encode(Format::Dds, format!("{other:?} pixels can't be written as `{}`", opts.dds.compression.name()))), }; - // Uncompressed: the pitch of one row. Compressed: the whole top level (its "linear size"). - let (size_flag, pitch_or_size) = if store.is_block() { (DDSD_LINEARSIZE, Some(top)) } else { (DDSD_PITCH, store.row_bytes(width as usize)) }; - let pitch_or_size = pitch_or_size - .and_then(|v| u32::try_from(v).ok()) - .ok_or_else(|| encode_err(format!("a {width}x{height} level is too large for a DDS header")))?; - let mipmapped = levels > 1; - put(4, 124); - put(8, DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT | size_flag | if mipmapped { DDSD_MIPMAPCOUNT } else { 0 }); - put(12, height); - put(16, width); - put(20, pitch_or_size); - put(28, levels); - let (flags, fourcc, bits, masks) = store.pixel_format(); - put(76, 32); - put(80, flags); - put(84, u32::from_le_bytes(fourcc)); - put(88, bits); - for (i, mask) in masks.into_iter().enumerate() { - put(92 + 4 * i, mask); - } - put(108, DDSCAPS_TEXTURE | if mipmapped { DDSCAPS_COMPLEX | DDSCAPS_MIPMAP } else { 0 }); - b[..4].copy_from_slice(b"DDS "); - Ok(b) -} - -/// Runs `f` on each `len`-byte row of `out` with its index, in parallel on native targets. -fn rows(out: &mut [u8], len: usize, f: impl Fn(usize, &mut [u8]) + Send + Sync) { - #[cfg(not(target_arch = "wasm32"))] - { - use rayon::prelude::*; - out.par_chunks_mut(len.max(1)).enumerate().for_each(|(i, row)| f(i, row)); - } - #[cfg(target_arch = "wasm32")] - for (i, row) in out.chunks_mut(len.max(1)).enumerate() { - f(i, row); - } -} - -/// Writes one level of `width` × `height` pixels (in the image's layout) into `dst`. -fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst: &mut [u8]) -> Result<(), CodecError> { - let row_bytes = store.row_bytes(width).ok_or_else(|| encode_err("texture size overflow"))?; - match store { - Store::Luminance | Store::LuminanceAlpha => { - // The image's gray (and alpha) samples, in the same order. - let src = pixels.get(..dst.len()).ok_or_else(|| encode_err("the level is smaller than its size"))?; - dst.copy_from_slice(src); - } - Store::Bgrx | Store::Bgra => { - let channels = if store == Store::Bgra { 4 } else { 3 }; - rows(dst, row_bytes, |y, row| { - let src = pixels.get(y * width * channels..).unwrap_or_default(); - for (d, s) in row.as_chunks_mut::<4>().0.iter_mut().zip(src.chunks_exact(channels)) { - let byte = |i: usize| s.get(i).copied().unwrap_or(255); - *d = [byte(2), byte(1), byte(0), byte(3)]; - } - }); - } - Store::Bc1 | Store::Bc2 | Store::Bc3 => { - let (texels, _) = pixels.as_chunks::<4>(); - rows(dst, row_bytes, |by, row| { - for (bx, d) in row.chunks_exact_mut(store.unit()).enumerate() { - let (block, inside) = gather(texels, (width, height), bx, by); - let encoded: &[u8] = match store { - Store::Bc1 => &dds_bc::encode_bc1(&block, inside), - Store::Bc2 => &dds_bc::encode_bc2(&block, inside), - _ => &dds_bc::encode_bc3(&block, inside), - }; - for (d, s) in d.iter_mut().zip(encoded) { - *d = *s; - } - } - }); - } - } - Ok(()) -} - -/// Block (`bx`, `by`)'s texels, and which of them lie inside the image. -fn gather(texels: &[[u8; 4]], (width, height): (usize, usize), bx: usize, by: usize) -> (Block, Mask) { - let mut block = [[0u8; 4]; 16]; - let mut inside: Mask = 0; - for (t, texel) in block.iter_mut().enumerate() { - let (x, y) = (bx * 4 + t % 4, by * 4 + t / 4); - if x < width - && y < height - && let Some(p) = texels.get(y * width + x) - { - *texel = *p; - inside |= 1 << t; - } - } - (block, inside) -} - -/// The next mip level: each pixel the average of the 2×2 pixels it covers, or of 2×3, 3×2 or -/// 3×3 along the last column and row of an odd-sized level, so no pixel is dropped. -fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), channels: usize) -> Result, CodecError> { - let len = nw.checked_mul(nh).and_then(|n| n.checked_mul(channels)).ok_or_else(|| encode_err("mip level size overflow"))?; - let mut dst = crate::options::alloc_zeroed::(len)?; - let span = |i: usize, n: usize, size: usize| 2 * i..=if i + 1 == n { size.saturating_sub(1) } else { 2 * i + 1 }; - rows(&mut dst, nw * channels, |y, row| { - let ys = span(y, nh, h); - for (x, px) in row.chunks_exact_mut(channels.max(1)).enumerate() { - let xs = span(x, nw, w); - for (c, v) in px.iter_mut().enumerate() { - let (mut sum, mut n) = (0u32, 0u32); - for sy in ys.clone() { - for sx in xs.clone() { - if let Some(&s) = src.get((sy * w + sx) * channels + c) { - sum += u32::from(s); - n += 1; - } - } - } - *v = ((sum + n / 2) / n.max(1)) as u8; - } - } - }); - Ok(dst) + let (width, height) = img.dimensions(); + craft_dds::encode(width, height, layout, pixels, opts.dds).map_err(error) } diff --git a/crates/image-codecs/src/options.rs b/crates/image-codecs/src/options.rs index 71f6030b3..d0ef8ee9a 100644 --- a/crates/image-codecs/src/options.rs +++ b/crates/image-codecs/src/options.rs @@ -235,60 +235,8 @@ impl DpxBits { } } -/// How a DDS texture stores its pixels. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum DdsCompression { - /// Uncompressed, 8 bits per channel: BGRA (BGRX without alpha), or luminance with or - /// without alpha for grayscale. Lossless. - #[default] - None, - /// BC1 (DXT1): 4 bits per pixel. Alpha is on or off: texels under half opacity become - /// transparent black. - Bc1, - /// BC2 (DXT3): 8 bits per pixel, with 4 bits of explicit alpha per texel. - Bc2, - /// BC3 (DXT5): 8 bits per pixel, with smooth interpolated alpha. - Bc3, -} - -impl DdsCompression { - pub const ALL: [DdsCompression; 4] = [DdsCompression::None, DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3]; - - /// The name scripts and the CLI use: `none`, `bc1`, `bc2` or `bc3`. - pub fn name(self) -> &'static str { - match self { - DdsCompression::None => "none", - DdsCompression::Bc1 => "bc1", - DdsCompression::Bc2 => "bc2", - DdsCompression::Bc3 => "bc3", - } - } - - /// [`Self::name`] back to a compression, also taking the DXT names (`dxt1`, `dxt3`, - /// `dxt5`), in any case. Anything else is `None`. - pub fn from_name(name: &str) -> Option { - match name.to_ascii_lowercase().as_str() { - "none" => Some(DdsCompression::None), - "bc1" | "dxt1" => Some(DdsCompression::Bc1), - "bc2" | "dxt3" => Some(DdsCompression::Bc2), - "bc3" | "dxt5" => Some(DdsCompression::Bc3), - _ => None, - } - } - - /// A block compression (lossy), not uncompressed pixels. - pub fn is_block(self) -> bool { - self != DdsCompression::None - } -} - -/// What a DDS file is written with. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct DdsOptions { - pub compression: DdsCompression, - /// Write the full mip chain down to 1×1, each level a box filter of the one above. - pub mipmaps: bool, -} +/// How a DDS texture stores its pixels and whether it carries a mip chain, from [`craft_dds`]. +pub use craft_dds::{DdsCompression, DdsOptions}; /// The longer side, in pixels, at which a TIFF image pyramid stops: the last level written is /// the first one no longer than this ([`EncodeOptions::tiff_pyramid`]). From e22ccf3511d57115feaf5aa156d1118864f87537 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 23:32:31 +0100 Subject: [PATCH 021/209] DDS: encode and decode faster with the same bytes Encoding: - colour blocks keep their colours and endpoint candidates on the stack instead of in per-block vectors (the idea of storytold/photocraft#10669) - a refinement round whose endpoint pair was already fitted ends the search, which could not change the block any more - a block of one colour skips the principal-axis search, whose result for it is that colour at both ends - the alpha block gathers its 16 values once, finds both ranges in one pass and picks indices in a plain loop with the same tie-break - the nearest palette colour is found the same way - interior blocks are gathered as four rows of four texels, and interior mip pixels average their 2x2 source pixels directly Decoding: - block-compressed data is decoded one row of blocks per task, copying whole texel rows and checking bounds only on the right edge - uncompressed rows are decoded one per task; when every channel is a whole byte (RGBA, BGRA, BGRX, luminance), pixels are a byte shuffle The f32 arithmetic and its order are unchanged; the digest oracle pins the output. --- crates/dds/Cargo.toml | 2 +- crates/dds/README.md | 2 +- crates/dds/src/bc.rs | 131 ++++++++++++++++++++++++++-------- crates/dds/src/lib.rs | 158 +++++++++++++++++++++++++++++++++--------- 4 files changed, 229 insertions(+), 64 deletions(-) diff --git a/crates/dds/Cargo.toml b/crates/dds/Cargo.toml index e3cee5696..ca40bd69a 100644 --- a/crates/dds/Cargo.toml +++ b/crates/dds/Cargo.toml @@ -12,7 +12,7 @@ workspace = true [features] default = [] -# Encode rows of blocks and mip levels on several threads. +# Encode and decode rows of blocks (and mip levels) on several threads. parallel = ["dep:rayon"] [dependencies] diff --git a/crates/dds/README.md b/crates/dds/README.md index 465975ff2..3871452df 100644 --- a/crates/dds/README.md +++ b/crates/dds/README.md @@ -39,5 +39,5 @@ under half opacity into its transparent black, and BC3 picks whichever alpha mod ## Features -- `parallel`: encodes rows of blocks and mip levels on several threads with rayon. +- `parallel`: encodes and decodes rows of blocks (and mip levels) on several threads with rayon. `craft-image-codecs` turns it on for native targets. diff --git a/crates/dds/src/bc.rs b/crates/dds/src/bc.rs index df60e462c..1641a9e56 100644 --- a/crates/dds/src/bc.rs +++ b/crates/dds/src/bc.rs @@ -69,7 +69,8 @@ pub(crate) fn encode_bc2(block: &Block, inside: Mask) -> [u8; 16] { /// A BC3 block: an interpolated alpha block, then a four-colour block. pub(crate) fn encode_bc3(block: &Block, inside: Mask) -> [u8; 16] { - join(alpha_block(block, inside), colour_block(block, inside, 0, false)) + let alphas = std::array::from_fn(|t| block[t][3]); + join(value_block(&alphas, inside), colour_block(block, inside, 0, false)) } fn join(a: [u8; 8], b: [u8; 8]) -> [u8; 16] { @@ -92,21 +93,56 @@ fn to565(c: [f32; 3]) -> u16 { } fn distance(a: &[u8; 4], b: &[u8; 4]) -> u32 { - (0..3).map(|i| u32::from(a[i].abs_diff(b[i])).pow(2)).sum() + let d = |i: usize| u32::from(a[i].abs_diff(b[i])).pow(2); + d(0) + d(1) + d(2) +} + +/// The index of the first of `palette`'s entries nearest by `distance`, and that distance: the +/// entry `Iterator::min_by_key` picks, found without the iterator (a texel equal to an entry +/// stops the search, as no later entry can be nearer and an equal one doesn't win). +fn nearest(palette: &[T], distance: impl Fn(&T) -> u32) -> (usize, u32) { + let mut best = (0, u32::MAX); + for (i, p) in palette.iter().enumerate() { + let d = distance(p); + if d < best.1 { + best = (i, d); + if d == 0 { + break; + } + } + } + if palette.is_empty() { (0, 0) } else { best } } /// A colour block (endpoints, then 2-bit indices) for the `opaque` texels' colours, with the /// `transparent` ones on BC1's transparent index. `bc1` allows the three-colour mode; BC2 and /// BC3 always decode four colours. fn colour_block(block: &Block, opaque: Mask, transparent: Mask, bc1: bool) -> [u8; 8] { - let colours: Vec<[f32; 3]> = block.iter().enumerate().filter(|(t, _)| opaque >> t & 1 != 0).map(|(_, texel)| rgb_of(texel)).collect(); - if colours.is_empty() { + // The opaque texels' colours in texel order, on the stack (after storytold/photocraft#10669, + // which took the per-block allocations out of PhotoCraft's copy of this encoder). + let mut buffer = [[0f32; 3]; 16]; + let mut n = 0; + for (t, texel) in block.iter().enumerate() { + if opaque >> t & 1 != 0 + && let Some(slot) = buffer.get_mut(n) + { + *slot = rgb_of(texel); + n += 1; + } + } + let colours = buffer.get(..n).unwrap_or_default(); + let Some((first, rest)) = colours.split_first() else { // Nothing visible: every texel on the transparent index (or index 0 of black). let indices = if transparent != 0 { u32::MAX } else { 0 }; return pack(0, 0, indices); - } - let (lo, hi) = principal_endpoints(&colours); - let mut candidates = vec![(to565(lo), to565(hi))]; + }; + // One colour has no axis: `principal_endpoints` returns that colour (their mean) for both + // ends, so it is skipped. Everything after it runs as for any block. + let (lo, hi) = if rest.iter().all(|c| c == first) { (*first, *first) } else { principal_endpoints(colours) }; + let mut candidate = (to565(lo), to565(hi)); + // The endpoint pairs fitted so far: at most one per round. + let mut fitted = [candidate; 3]; + let mut count = 1; let modes: &[bool] = if transparent != 0 { &[false] } else if bc1 { @@ -118,22 +154,30 @@ fn colour_block(block: &Block, opaque: Mask, transparent: Mask, bc1: bool) -> [u // Each refinement starts from the best block so far: its indices give the least-squares // endpoints, which become one more candidate. for round in 0..3 { - for &(e0, e1) in &candidates { - for &four in modes { - let (bytes, error) = fit(block, opaque, transparent, e0, e1, four); - if best.is_none_or(|(e, _)| error < e) { - best = Some((error, bytes)); - } + let (e0, e1) = candidate; + for &four in modes { + let (bytes, error) = fit(block, opaque, transparent, e0, e1, four); + if best.is_none_or(|(e, _)| error < e) { + best = Some((error, bytes)); } } let Some((error, bytes)) = best else { break }; if error == 0 || round == 2 { break; } - candidates = least_squares(block, opaque, &bytes).into_iter().collect(); - if candidates.is_empty() { + let Some(next) = least_squares(block, opaque, &bytes) else { break }; + // A pair fitted before (in either order: `fit` orders the endpoints itself) fits the same + // again and can't beat the best block, which then gives the same pair again: the rounds + // left would change nothing. + let seen = fitted.get(..count).unwrap_or_default(); + if seen.iter().any(|&(a, b)| (a, b) == next || (b, a) == next) { break; } + if let Some(slot) = fitted.get_mut(count) { + *slot = next; + count += 1; + } + candidate = next; } best.map_or([0; 8], |(_, bytes)| bytes) } @@ -151,7 +195,7 @@ fn fit(block: &Block, opaque: Mask, transparent: Mask, e0: u16, e1: u16, four: b let index = if transparent >> t & 1 != 0 { 3 } else if opaque >> t & 1 != 0 { - let (i, d) = palette.iter().take(choices).enumerate().map(|(i, p)| (i, distance(texel, p))).min_by_key(|&(_, d)| d).unwrap_or((0, 0)); + let (i, d) = nearest(palette.get(..choices).unwrap_or_default(), |p| distance(texel, p)); error += d; i as u32 } else { @@ -248,29 +292,39 @@ fn least_squares(block: &Block, opaque: Mask, bytes: &[u8; 8]) -> Option<(u16, u Some((to565(e0), to565(e1))) } -/// A BC3 alpha block: whichever of the two modes (eight values between the extremes, or six -/// between the extremes other than 0 and 255, plus exact 0 and 255) fits better. -fn alpha_block(block: &Block, inside: Mask) -> [u8; 8] { - let alphas = || block.iter().enumerate().filter(move |(t, _)| inside >> t & 1 != 0).map(|(_, texel)| texel[3]); - let range = - |values: &mut dyn Iterator| values.fold(None, |r: Option<(u8, u8)>, a| Some(r.map_or((a, a), |(lo, hi)| (lo.min(a), hi.max(a))))); - let (lo, hi) = range(&mut alphas()).unwrap_or((0, 0)); +/// An interpolated block of 16 single-channel values (BC3's alpha): whichever of the two modes +/// (eight values between the extremes, or six between the extremes other than 0 and 255, plus +/// exact 0 and 255) fits better. Only the values of `inside` count. +pub(crate) fn value_block(values: &[u8; 16], inside: Mask) -> [u8; 8] { + // Both ranges in one pass: of all values, and of those other than 0 and 255. + let mut all: Option<(u8, u8)> = None; + let mut mid = all; + let widen = |r: Option<(u8, u8)>, v: u8| Some(r.map_or((v, v), |(lo, hi)| (lo.min(v), hi.max(v)))); + for (t, &v) in values.iter().enumerate() { + if inside >> t & 1 != 0 { + all = widen(all, v); + if !matches!(v, 0 | 255) { + mid = widen(mid, v); + } + } + } + let (lo, hi) = all.unwrap_or((0, 0)); // Eight values need a0 > a1; equal ends fall to the six-value mode, which starts with a0 too. - let eight = alpha_fit(block, inside, hi, lo); - let (mid_lo, mid_hi) = range(&mut alphas().filter(|a| !matches!(a, 0 | 255))).unwrap_or((0, 0)); - let six = alpha_fit(block, inside, mid_lo, mid_hi); + let eight = value_fit(values, inside, hi, lo); + let (mid_lo, mid_hi) = mid.unwrap_or((0, 0)); + let six = value_fit(values, inside, mid_lo, mid_hi); if six.1 < eight.1 { six.0 } else { eight.0 } } -fn alpha_fit(block: &Block, inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u32) { +fn value_fit(values: &[u8; 16], inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u32) { let palette = alpha_palette(a0, a1); let mut bits = 0u64; let mut error = 0u32; - for (t, texel) in block.iter().enumerate() { + for (t, &v) in values.iter().enumerate() { if inside >> t & 1 == 0 { continue; } - let (i, d) = palette.iter().enumerate().map(|(i, &p)| (i, u32::from(p.abs_diff(texel[3])).pow(2))).min_by_key(|&(_, d)| d).unwrap_or((0, 0)); + let (i, d) = nearest(&palette, |&p| u32::from(p.abs_diff(v)).pow(2)); error += d; bits |= (i as u64) << (3 * t); } @@ -333,7 +387,7 @@ mod tests { fn alpha_blocks_pick_the_mode_that_keeps_0_and_255() { // 0, 255 and a cluster in between: the six-value mode stores 0 and 255 exactly. let block: Block = std::array::from_fn(|t| [0, 0, 0, [0, 255, 100, 110][t % 4]]); - let bytes = alpha_block(&block, u16::MAX); + let bytes = value_block(&block.map(|t| t[3]), u16::MAX); assert!(bytes[0] <= bytes[1], "six-value mode"); let palette = alpha_palette(bytes[0], bytes[1]); let bits = u64::from_le_bytes([bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], 0, 0]); @@ -358,6 +412,23 @@ mod tests { assert!(ours < bounding_box, "{ours} vs the bounding box's {bounding_box}"); } + #[test] + fn nearest_picks_what_min_by_key_picks() { + // Ties (equal entries, entries equally far on both sides) and exact hits included. + let mut seed = 0x2545_f491u32; + for _ in 0..2000 { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let palette = [seed as u8 & 0xf0, (seed >> 8) as u8 & 0xf0, (seed >> 16) as u8 & 0xf0, (seed >> 24) as u8 & 0xf0]; + let v = (seed >> 4) as u8 & 0xf8; + let d = |p: &u8| u32::from(p.abs_diff(v)).pow(2); + let want = palette.iter().enumerate().map(|(i, p)| (i, d(p))).min_by_key(|&(_, d)| d).unwrap(); + assert_eq!(nearest(&palette, d), want, "{palette:?} {v}"); + } + assert_eq!(nearest::(&[], |_| 7), (0, 0)); + } + #[test] fn texels_outside_the_image_are_ignored() { // Only the top-left texel is inside: its colour alone decides the endpoints. diff --git a/crates/dds/src/lib.rs b/crates/dds/src/lib.rs index a98a08631..079b3e3b5 100644 --- a/crates/dds/src/lib.rs +++ b/crates/dds/src/lib.rs @@ -11,8 +11,8 @@ //! so no DX10 header is written; formats only DXGI can name (BC4–BC7, sRGB) aren't written. //! //! Pixels are 8 bits per channel, interleaved, top row first, in one of the [`Layout`]s. The -//! crate has no dependencies; the `parallel` feature encodes rows of blocks on several threads -//! with rayon. +//! crate has no dependencies; the `parallel` feature encodes and decodes rows of blocks (and mip +//! levels) on several threads with rayon. //! //! ``` //! use craft_dds::{DdsCompression, DdsOptions, Layout}; @@ -385,41 +385,104 @@ impl<'a> Texture<'a> { let total = row_size.checked_mul(h).ok_or_else(|| bad("pixel size overflow"))?; let mut pixels = vec![0u8; total]; match encoding { - Encoding::Masks { bytes, gray, fields } => { - for (src_row, dst_row) in data.chunks_exact(row_pitch).zip(pixels.chunks_exact_mut(row_size)) { - // Stop at the output width, leaving any stored row padding unread. - for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { - let pixel = src.iter().rev().fold(0u32, |v, &byte| v << 8 | u32::from(byte)); - let channel = |i: usize| fields[i].map_or(0, |f| f.get(pixel)); - let values = if gray { [channel(0), channel(3), 0, 0] } else { [channel(0), channel(1), channel(2), channel(3)] }; - dst.copy_from_slice(&values[..channels]); - } + Encoding::Masks { bytes, gray, fields } => decode_masks(Masks { bytes, gray, fields }, data, row_pitch, &mut pixels, row_size), + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => decode_blocks(encoding, data, (w, channels), &mut pixels), + } + Ok(pixels) + } +} + +/// An uncompressed pixel format: bytes per pixel, luminance or RGB, and the red (or +/// luminance), green, blue and alpha fields. +struct Masks { + bytes: usize, + gray: bool, + fields: [Option; 4], +} + +/// Uncompressed rows `row_pitch` apart into `pixels` (rows of `row_size` bytes), one row of +/// pixels per task. +fn decode_masks(m: Masks, data: &[u8], row_pitch: usize, pixels: &mut [u8], row_size: usize) { + let Masks { bytes, gray, fields } = m; + let channels = if gray { 1 + usize::from(fields[3].is_some()) } else { 3 + usize::from(fields[3].is_some()) }; + // The field each output channel comes from. + let picks = if gray { [0, 3, 0, 0] } else { [0, 1, 2, 3] }; + // An 8-bit field on a byte boundary scales to itself, so when every channel is one (or + // absent, and 0) the pixels are a byte shuffle: the byte each channel copies. + let mut at: [Option; 4] = [None; 4]; + let mut whole_bytes = true; + for (slot, &pick) in at.iter_mut().zip(&picks).take(channels) { + match fields.get(pick).copied().flatten() { + None => {} + Some(f) if f.max == 255 && f.shift % 8 == 0 => *slot = Some(f.shift as usize / 8), + Some(_) => whole_bytes = false, + } + } + // Four bytes to four channels, each present: RGBA, BGRA and the like. + let four = (whole_bytes && bytes == 4 && channels == 4 && at.iter().all(Option::is_some)).then(|| at.map(|i| i.unwrap_or(0) & 3)); + rows(pixels, row_size, |y, dst_row| { + // Stop at the output width, leaving any stored row padding unread. + let src_row = data.get(y * row_pitch..).unwrap_or_default(); + if let Some(order) = four { + let (src, dst) = (src_row.as_chunks::<4>().0, dst_row.as_chunks_mut::<4>().0); + if order == [0, 1, 2, 3] { + for (d, s) in dst.iter_mut().zip(src) { + *d = *s; + } + } else { + for (d, s) in dst.iter_mut().zip(src) { + *d = order.map(|i| s[i]); } } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { - let block_size = if matches!(encoding, Encoding::Bc1) { 8 } else { 16 }; - let blocks_x = w.div_ceil(4); - for (i, block) in data.chunks_exact(block_size).enumerate() { - let texels = match encoding { - Encoding::Bc1 => bc1(block, true), - Encoding::Bc2 => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc2_alpha(block)), - _ => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc3_alpha(block)), - }; - let (bx, by) = ((i % blocks_x) * 4, (i / blocks_x) * 4); - for (t, texel) in texels.iter().enumerate() { - let (x, y) = (bx + t % 4, by + t / 4); - if x < w && y < h { - let at = (y * w + x) * 4; - if let Some(dst) = pixels.get_mut(at..at + 4) { - dst.copy_from_slice(texel); - } - } + } else if whole_bytes { + for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { + for (v, i) in dst.iter_mut().zip(at) { + *v = i.and_then(|i| src.get(i)).copied().unwrap_or(0); + } + } + } else { + for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { + let pixel = src.iter().rev().fold(0u32, |v, &byte| v << 8 | u32::from(byte)); + let channel = |i: usize| fields[i].map_or(0, |f| f.get(pixel)); + let values = if gray { [channel(0), channel(3), 0, 0] } else { [channel(0), channel(1), channel(2), channel(3)] }; + dst.copy_from_slice(&values[..channels]); + } + } + }); +} + +/// Block-compressed data into `pixels` of `width` × `channels` bytes a row, one row of blocks +/// per task. Blocks wholly inside the image copy whole rows of texels; only those on the right +/// edge check each texel. +fn decode_blocks(encoding: Encoding, data: &[u8], (width, channels): (usize, usize), pixels: &mut [u8]) { + let block_size = if matches!(encoding, Encoding::Bc1) { 8 } else { 16 }; + let blocks_x = width.div_ceil(4); + let row_size = width * channels; + rows(pixels, row_size.saturating_mul(4), |by, band| { + let src = data.get(by * blocks_x * block_size..).unwrap_or_default(); + for (bx, block) in src.chunks_exact(block_size).take(blocks_x).enumerate() { + let texels = match encoding { + Encoding::Bc1 => bc1(block, true), + Encoding::Bc2 => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc2_alpha(block)), + _ => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc3_alpha(block)), + }; + let x0 = bx * 4; + // The band holds the block's rows that lie inside the image (fewer than four in the + // last band of a height that isn't a multiple of four). + for (line, row) in band.chunks_exact_mut(row_size).zip(texels.as_chunks::<4>().0) { + if x0 + 4 <= width && channels == 4 { + if let Some(dst) = line.get_mut(x0 * 4..x0 * 4 + 16) { + dst.copy_from_slice(row.as_flattened()); + } + } else { + let dst = line.get_mut(x0 * channels..).unwrap_or_default(); + for (d, texel) in dst.chunks_exact_mut(channels).zip(row) { + d.copy_from_slice(texel.get(..channels).unwrap_or_default()); } } } } - Ok(pixels) - } + }); } /// A BC1 colour block. BC2 and BC3 colour blocks always have four colours; only BC1's @@ -683,6 +746,20 @@ fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst /// Block (`bx`, `by`)'s texels, and which of them lie inside the image. fn gather(texels: &[[u8; 4]], (width, height): (usize, usize), bx: usize, by: usize) -> (Block, Mask) { let mut block = [[0u8; 4]; 16]; + // A block wholly inside the image: four rows of four texels. + let (x0, y0) = (bx * 4, by * 4); + if x0 + 4 <= width && y0 + 4 <= height { + let mut copied = 0; + for (ty, row) in block.as_chunks_mut::<4>().0.iter_mut().enumerate() { + if let Some(src) = texels.get((y0 + ty) * width + x0..).and_then(|s| s.first_chunk::<4>()) { + *row = *src; + copied += 1; + } + } + if copied == 4 { + return (block, Mask::MAX); + } + } let mut inside: Mask = 0; for (t, texel) in block.iter_mut().enumerate() { let (x, y) = (bx * 4 + t % 4, by * 4 + t / 4); @@ -703,9 +780,26 @@ fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), chan let len = nw.checked_mul(nh).and_then(|n| n.checked_mul(channels)).ok_or_else(|| encode_err("mip level size overflow"))?; let mut dst = alloc_zeroed(len)?; let span = |i: usize, n: usize, size: usize| 2 * i..=if i + 1 == n { size.saturating_sub(1) } else { 2 * i + 1 }; + let ch = channels.max(1); + let line = w * ch; rows(&mut dst, nw * channels, |y, row| { let ys = span(y, nh, h); - for (x, px) in row.chunks_exact_mut(channels.max(1)).enumerate() { + // Every pixel but those of the last column and row averages exactly 2×2: the same sum + // and rounding as below, from two source rows read in step. + let mut edge = 0; + if y + 1 < nh + && let (Some(r0), Some(r1)) = (src.get(2 * y * line..(2 * y + 1) * line), src.get((2 * y + 1) * line..(2 * y + 2) * line)) + { + let interior = nw.saturating_sub(1); + for ((px, a), b) in row.chunks_exact_mut(ch).zip(r0.chunks_exact(2 * ch)).zip(r1.chunks_exact(2 * ch)).take(interior) { + for (c, v) in px.iter_mut().enumerate() { + let sum = u32::from(a[c]) + u32::from(a[ch + c]) + u32::from(b[c]) + u32::from(b[ch + c]); + *v = ((sum + 2) / 4) as u8; + } + } + edge = interior; + } + for (x, px) in row.chunks_exact_mut(ch).enumerate().skip(edge) { let xs = span(x, nw, w); for (c, v) in px.iter_mut().enumerate() { let (mut sum, mut n) = (0u32, 0u32); From 203f225562114a0e227ba021b481857901e6f08f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 23:32:37 +0100 Subject: [PATCH 022/209] DDS: stop the principal-axis iteration once the axis is a fixed point A power-iteration step that gives back the same axis bit for bit gives it back on every later step, so stopping there keeps the output. Kept only if the timings show a gain. --- crates/dds/src/bc.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/dds/src/bc.rs b/crates/dds/src/bc.rs index 1641a9e56..12bf15179 100644 --- a/crates/dds/src/bc.rs +++ b/crates/dds/src/bc.rs @@ -235,7 +235,14 @@ fn principal_endpoints(colours: &[[f32; 3]]) -> ([f32; 3], [f32; 3]) { if !len.is_normal() { break; } - axis = next.map(|v| v / len); + let normalised = next.map(|v| v / len); + // A step that returns the same axis, bit for bit, returns it on every later step too: + // stopping there gives the very axis the remaining steps would. Kept only if timing + // shows the comparison pays for itself. + if normalised.map(f32::to_bits) == axis.map(f32::to_bits) { + break; + } + axis = normalised; } axis }; From f31c93cf59a7cc68947fd49f15a7b43d39aa8c53 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 23:38:08 +0100 Subject: [PATCH 023/209] DDS: read and write BC4 and BC5 (grayscale data and normal maps) BC4 is one block of 16 single-channel values and BC5 two of them, the block BC3 already stores its alpha in, so the alpha block encoder and decoder now serve all three (BC3's output stays the same). Reading: ATI1 and BC4U, and DXGI 79 (typeless) and 80, open as gray; ATI2 and BC5U, and DXGI 82 (typeless, as Pillow writes it) and 83, open as RGB with blue 0. ATI2 follows Microsoft's mapping, red in the first block. Signed BC4S/BC5S and DXGI 81/84 are refused, naming SNORM. Writing: DdsCompression::Bc4 (bc4, ati1) from gray and Bc5 (bc5, ati2) from RGB, with the DX10 header (BC4_UNORM, BC5_UNORM, one 2D texture), mip chains included. The encode plan converts other layouts to gray and RGB, so the fidelity warnings report colour to gray and dropped alpha; BC5's dropped blue is documented under LossyCompression. DdsCompression is now #[non_exhaustive] and ALL has six entries. --- README.md | 6 +- crates/dds/Cargo.toml | 2 +- crates/dds/README.md | 19 +- crates/dds/src/bc.rs | 70 +++++-- crates/dds/src/lib.rs | 216 ++++++++++++++++------ crates/image-codecs/README.md | 32 ++-- crates/image-codecs/src/codecs/dds.rs | 2 +- crates/image-codecs/src/fidelity.rs | 13 +- crates/image-codecs/src/format.rs | 2 +- crates/image-codecs/tests/it/dds.rs | 75 +++++++- crates/image-codecs/tests/it/dds_write.rs | 127 ++++++++++++- crates/raster-codecs/README.md | 2 +- crates/raster-codecs/src/lib.rs | 2 +- crates/raster-codecs/tests/dds.rs | 3 +- 14 files changed, 469 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index af98f90e7..7b0c88ded 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit log, scripts, verification) is in `crates/crw/clean-room/`. Colour stays in each app. - [`craft-csv`](crates/csv/README.md): delimited text (CSV, TSV) and SYLK: encoding detection (UTF-8, UTF-16, Windows-1252 or Windows-1256), delimiter sniffing, a record parser with a cap on fields per record, field quoting for writers, and a SYLK (`.slk`) reader with R1C1 formulas rewritten to A1. - [`craft-dds`](crates/dds/README.md): DirectDraw Surface (DDS) textures, read and written: the - top mip level of uncompressed and BC1 to BC3 (DXT1/3/5) textures, legacy and DX10 headers, and - our own block encoder with optional mip chains. No dependencies (rayon behind `parallel`), so an + top mip level of uncompressed, BC1 to BC3 (DXT1/3/5) and BC4/BC5 (grayscale data and normal + maps) textures, legacy and DX10 headers, and our own block encoder with optional mip chains. No dependencies (rayon behind `parallel`), so an app that only exports textures needn't take the whole `craft-image-codecs` codec set and its licences; `craft-image-codecs` reads and writes DDS through it. - [`craft-dialogs`](crates/dialogs/README.md): native file and message dialogs that never block @@ -245,7 +245,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit encoder, from PhotoCraft; includes a frame probe for routing. Metadata interpretation stays in each app. - [`craft-raster-codecs`](crates/raster-codecs/README.md): small raster formats, one module each, - from PhotoCraft: DDS (BC1 to BC3, read and write), DICOM, FITS (read and write), IFF/ILBM (read + from PhotoCraft: DDS (BC1 to BC3 read and write, BC4 and BC5 read), DICOM, FITS (read and write), IFF/ILBM (read and write), PCX, PNM/PAM/PFM (read and write) and XPM, into one flat image type. Bounded and panic-free on hostile input. Colour, orientation and the document mapping stay in each app. - [`craft-kiwi`](crates/kiwi/README.md): clean-room decoder and encoder for the kiwi binary schema diff --git a/crates/dds/Cargo.toml b/crates/dds/Cargo.toml index ca40bd69a..f93e8cbcc 100644 --- a/crates/dds/Cargo.toml +++ b/crates/dds/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "craft-dds" -description = "DirectDraw Surface (DDS) textures: uncompressed and BC1-BC3 block compression, read and write, without dependencies" +description = "DirectDraw Surface (DDS) textures: uncompressed and BC1-BC5 block compression, read and write, without dependencies" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/dds/README.md b/crates/dds/README.md index 3871452df..720ec9ba7 100644 --- a/crates/dds/README.md +++ b/crates/dds/README.md @@ -12,15 +12,20 @@ craft-dds = { git = "https://github.com/storytold/craft-libs", rev = "" | | Read | Write | |---|---|---| | Uncompressed | 8, 16, 24 or 32 bits per pixel described by channel masks, RGB or luminance, with or without alpha; padded rows (`DDSD_PITCH`) | BGRA, BGRX, luminance, luminance with alpha | -| Block-compressed | BC1, BC2, BC3 (DXT1, DXT3, DXT5) | BC1, BC2, BC3 | -| Headers | legacy and DX10 | legacy | +| Block-compressed | BC1, BC2, BC3 (DXT1, DXT3, DXT5); unsigned BC4 (ATI1, BC4U, DXGI 79 and 80) and BC5 (ATI2, BC5U, DXGI 82 and 83) | BC1, BC2, BC3, BC4, BC5 | +| Headers | legacy and DX10 | legacy; DX10 for BC4 and BC5 | | Mip levels | the top level only | optional full chain, box-filtered | -Cubemaps, texture arrays, volume textures, premultiplied DXT2/DXT4 and the other DXGI formats -are refused with `DdsError::Unsupported` rather than read in part. +Cubemaps, texture arrays, volume textures, premultiplied DXT2/DXT4, signed (SNORM) BC4/BC5 and +the other DXGI formats (BC6H, BC7, float) are refused with `DdsError::Unsupported` rather than read +in part. Pixels are 8 bits per channel, interleaved and top row first, in one of four `Layout`s (`Gray`, -`GrayA`, `Rgb`, `Rgba`). Block compressions are written from RGBA. +`GrayA`, `Rgb`, `Rgba`). BC1 to BC3 are written from RGBA, BC4 from gray and BC5 from RGB +(`DdsCompression::layout` names it). BC4 reads back as gray. BC5 stores red and green only, so it +reads back as RGB with blue 0: for a tangent-space normal map, X and Y, with Z left to the +renderer. ATI2 files are read as Microsoft's tools read them (red in the first block), not with +the two blocks swapped as FFmpeg does. ```rust use craft_dds::{DdsCompression, DdsOptions, Layout, Texture}; @@ -35,7 +40,9 @@ let decoded = texture.decode()?; The block encoder is our own: RGB565 endpoints fitted along each block's principal axis and refined by least squares, indices picked against the palette the reader decodes; BC1 turns texels -under half opacity into its transparent black, and BC3 picks whichever alpha mode fits. +under half opacity into its transparent black, and BC3, BC4 and BC5 pick whichever mode of their +value blocks fits. The block functions don't depend on the DDS header, so other containers of +the same blocks can share them. ## Features diff --git a/crates/dds/src/bc.rs b/crates/dds/src/bc.rs index 12bf15179..863aa3077 100644 --- a/crates/dds/src/bc.rs +++ b/crates/dds/src/bc.rs @@ -1,6 +1,10 @@ -//! BC1, BC2 and BC3 (DXT1, DXT3, DXT5) blocks, after Microsoft's public block compression +//! BC1 to BC5 (DXT1, DXT3, DXT5, ATI1, ATI2) blocks, after Microsoft's public block compression //! documentation ("Block Compression (Direct3D 10)" and "Texture Block Compression in Direct3D -//! 11"): the palettes the DDS reader decodes with, and the block encoder the DDS writer uses. +//! 11"): the block decoders and the block encoder the DDS reader and writer use. Nothing here +//! knows about DDS headers, so other containers of the same blocks can use it. +//! +//! BC4 is one block of 16 single-channel values, the same block BC3 stores its alpha in; BC5 is +//! two of them, red then green. [`value_block`] and [`decode_values`] serve all three. //! //! The encoder fits a block's colours with two RGB565 endpoints along their principal axis, //! refines them by least squares, and scores every candidate against the exact palette the @@ -35,7 +39,7 @@ pub(crate) fn colour_palette(c0: u16, c1: u16, four: bool) -> [[u8; 4]; 4] { } } -/// A BC3 alpha block's eight values: six interpolated at sevenths when `a0 > a1`, else four +/// A value block's eight values: six interpolated at sevenths when `a0 > a1`, else four /// interpolated at fifths followed by 0 and 255. pub(crate) fn alpha_palette(a0: u8, a1: u8) -> [u8; 8] { let (a0, a1) = (u32::from(a0), u32::from(a1)); @@ -54,6 +58,32 @@ pub(crate) fn alpha_palette(a0: u8, a1: u8) -> [u8; 8] { palette.map(|v| v as u8) } +/// A colour block's 16 texels as RGBA. BC2 and BC3 colour blocks always have four colours +/// (`one_bit_alpha` false); only BC1's `c0 <= c1` order selects three colours and transparent +/// black. Missing bytes read as 0. +pub(crate) fn decode_colour(block: &[u8], one_bit_alpha: bool) -> Block { + let byte = |i: usize| block.get(i).copied().unwrap_or(0); + let (c0, c1) = (u16::from_le_bytes([byte(0), byte(1)]), u16::from_le_bytes([byte(2), byte(3)])); + let palette = colour_palette(c0, c1, c0 > c1 || !one_bit_alpha); + let indices = u32::from_le_bytes([byte(4), byte(5), byte(6), byte(7)]); + std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) +} + +/// BC2's alpha: four explicit bits per texel. +pub(crate) fn decode_bc2_alpha(block: &[u8]) -> [u8; 16] { + let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0))); + std::array::from_fn(|t| ((bits >> (4 * t)) & 0xf) as u8 * 17) +} + +/// A value block (BC3's alpha, BC4, either half of BC5): two endpoints and a 3-bit index per +/// texel into eight values, or into six plus 0 and 255 when `a0 <= a1`. Missing bytes read as 0. +pub(crate) fn decode_values(block: &[u8]) -> [u8; 16] { + let byte = |i: usize| block.get(i).copied().unwrap_or(0); + let palette = alpha_palette(byte(0), byte(1)); + let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]); + std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) +} + /// A BC1 block. Texels under half opacity become BC1's transparent black, which needs the /// three-colour mode; an opaque block takes whichever mode fits its colours better. pub(crate) fn encode_bc1(block: &Block, inside: Mask) -> [u8; 8] { @@ -299,7 +329,7 @@ fn least_squares(block: &Block, opaque: Mask, bytes: &[u8; 8]) -> Option<(u16, u Some((to565(e0), to565(e1))) } -/// An interpolated block of 16 single-channel values (BC3's alpha): whichever of the two modes +/// A block of 16 single-channel values (BC3's alpha, BC4, either half of BC5): whichever of the two modes /// (eight values between the extremes, or six between the extremes other than 0 and 255, plus /// exact 0 and 255) fits better. Only the values of `inside` count. pub(crate) fn value_block(values: &[u8; 16], inside: Mask) -> [u8; 8] { @@ -343,13 +373,6 @@ fn value_fit(values: &[u8; 16], inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u32) mod tests { use super::*; - fn decode_colour(bytes: &[u8; 8], bc1: bool) -> Block { - let (c0, c1) = (u16::from_le_bytes([bytes[0], bytes[1]]), u16::from_le_bytes([bytes[2], bytes[3]])); - let palette = colour_palette(c0, c1, c0 > c1 || !bc1); - let indices = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); - std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) - } - #[test] fn solid_blocks_land_on_the_nearest_565_colour() { for colour in [[0, 0, 0, 255], [255, 255, 255, 255], [12, 200, 77, 255]] { @@ -419,6 +442,31 @@ mod tests { assert!(ours < bounding_box, "{ours} vs the bounding box's {bounding_box}"); } + #[test] + fn value_blocks_decode_the_frozen_vectors() { + // Eight-value mode (a0 = 200 > a1 = 50), every index 0. + assert_eq!(decode_values(&[0xc8, 0x32, 0, 0, 0, 0, 0, 0]), [200; 16]); + // Six-value mode (a0 = 10 <= a1 = 250), every index 1. + assert_eq!(decode_values(&[0x0a, 0xfa, 0x49, 0x92, 0x24, 0x49, 0x92, 0x24]), [250; 16]); + // Six-value mode's indices 6 and 7 are 0 and 255 whatever the endpoints. + let bits: u64 = (0..16).fold(0, |b, t| b | (6 + (t % 2)) << (3 * t)); + let b = bits.to_le_bytes(); + let got = decode_values(&[10, 250, b[0], b[1], b[2], b[3], b[4], b[5]]); + assert_eq!(got, std::array::from_fn(|t| if t % 2 == 0 { 0 } else { 255 })); + } + + #[test] + fn value_blocks_round_trip_ramps_and_solids() { + for values in [[200u8; 16], [0; 16], [255; 16], std::array::from_fn(|t| (t * 17) as u8), std::array::from_fn(|t| 100 + t as u8)] { + let back = decode_values(&value_block(&values, u16::MAX)); + let worst = values.iter().zip(back).map(|(a, b)| a.abs_diff(b)).max().unwrap_or(0); + // Within half a step of either mode: 255 / 7 apart in the eight-value mode, about + // 221 / 5 in the six-value one. + assert!(worst <= 23, "{values:?} -> {back:?}"); + } + assert_eq!(decode_values(&value_block(&[200; 16], u16::MAX)), [200; 16]); + } + #[test] fn nearest_picks_what_min_by_key_picks() { // Ties (equal entries, entries equally far on both sides) and exact hits included. diff --git a/crates/dds/src/lib.rs b/crates/dds/src/lib.rs index 079b3e3b5..5c088052a 100644 --- a/crates/dds/src/lib.rs +++ b/crates/dds/src/lib.rs @@ -3,12 +3,16 @@ //! //! Reading: the top mip level of a 2D texture, either uncompressed (8 to 32 bits per pixel //! described by channel masks, RGB or luminance) or block-compressed BC1/BC2/BC3 -//! (DXT1/DXT3/DXT5), with or without the DX10 header. Cubemaps, texture arrays and volume -//! textures are refused rather than flattened to one face or slice. +//! (DXT1/DXT3/DXT5) or unsigned BC4/BC5 (ATI1/ATI2), with or without the DX10 header. BC4 opens +//! as gray and BC5 as RGB with blue 0: BC5 stores only red and green, and a normal map's Z is +//! left for the caller to reconstruct. Cubemaps, texture arrays and volume textures are refused +//! rather than flattened to one face or slice, and signed (SNORM) BC4/BC5 is refused too. //! -//! Writing: a 2D texture, uncompressed (BGRA or BGRX, luminance with or without alpha) or -//! BC1/BC2/BC3, optionally with its full mip chain. The legacy header describes all of these, -//! so no DX10 header is written; formats only DXGI can name (BC4–BC7, sRGB) aren't written. +//! Writing: a 2D texture, uncompressed (BGRA or BGRX, luminance with or without alpha), +//! BC1/BC2/BC3 or BC4/BC5, optionally with its full mip chain. The legacy header describes all +//! but BC4 and BC5, which are written with the DX10 header (`BC4_UNORM`, `BC5_UNORM`): their +//! legacy FourCCs are vendor names that programs read differently (ATI2's two blocks are swapped +//! by some). BC6H, BC7 and sRGB formats aren't written. //! //! Pixels are 8 bits per channel, interleaved, top row first, in one of the [`Layout`]s. The //! crate has no dependencies; the `parallel` feature encodes and decodes rows of blocks (and mip @@ -86,8 +90,10 @@ impl fmt::Display for DdsError { impl std::error::Error for DdsError {} -/// How a DDS texture stores its pixels. +/// How a DDS texture stores its pixels. More compressions may be added, so a `match` on it needs +/// a wildcard arm. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] pub enum DdsCompression { /// Uncompressed, 8 bits per channel: BGRA (BGRX without alpha), or luminance with or /// without alpha for grayscale. Lossless. @@ -100,33 +106,55 @@ pub enum DdsCompression { Bc2, /// BC3 (DXT5): 8 bits per pixel, with smooth interpolated alpha. Bc3, + /// BC4 (ATI1): 4 bits per pixel, one channel: gray, for roughness, metallic, height or + /// ambient-occlusion maps. Written from gray pixels. + Bc4, + /// BC5 (ATI2): 8 bits per pixel, two channels: red and green, for tangent-space normal maps + /// (X and Y; Z is reconstructed by the renderer). Written from RGB pixels, blue dropped. + Bc5, } impl DdsCompression { - pub const ALL: [DdsCompression; 4] = [DdsCompression::None, DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3]; + pub const ALL: [DdsCompression; 6] = + [DdsCompression::None, DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3, DdsCompression::Bc4, DdsCompression::Bc5]; - /// The name scripts and the CLI use: `none`, `bc1`, `bc2` or `bc3`. + /// The name scripts and the CLI use: `none`, `bc1`, `bc2`, `bc3`, `bc4` or `bc5`. pub fn name(self) -> &'static str { match self { DdsCompression::None => "none", DdsCompression::Bc1 => "bc1", DdsCompression::Bc2 => "bc2", DdsCompression::Bc3 => "bc3", + DdsCompression::Bc4 => "bc4", + DdsCompression::Bc5 => "bc5", } } - /// [`Self::name`] back to a compression, also taking the DXT names (`dxt1`, `dxt3`, - /// `dxt5`), in any case. Anything else is `None`. + /// [`Self::name`] back to a compression, also taking the DXT and ATI names (`dxt1`, `dxt3`, + /// `dxt5`, `ati1`, `ati2`), in any case. Anything else is `None`. pub fn from_name(name: &str) -> Option { match name.to_ascii_lowercase().as_str() { "none" => Some(DdsCompression::None), "bc1" | "dxt1" => Some(DdsCompression::Bc1), "bc2" | "dxt3" => Some(DdsCompression::Bc2), "bc3" | "dxt5" => Some(DdsCompression::Bc3), + "bc4" | "ati1" => Some(DdsCompression::Bc4), + "bc5" | "ati2" => Some(DdsCompression::Bc5), _ => None, } } + /// The pixel layout the compression is written from: gray for BC4, RGB for BC5, RGBA for + /// the other block compressions. `None` for uncompressed textures, which take every layout. + pub fn layout(self) -> Option { + match self { + DdsCompression::None => None, + DdsCompression::Bc4 => Some(Layout::Gray), + DdsCompression::Bc5 => Some(Layout::Rgb), + DdsCompression::Bc1 | DdsCompression::Bc2 | DdsCompression::Bc3 => Some(Layout::Rgba), + } + } + /// A block compression (lossy), not uncompressed pixels. pub fn is_block(self) -> bool { self != DdsCompression::None @@ -224,6 +252,10 @@ enum Encoding { Bc1, Bc2, Bc3, + /// One value block: gray. + Bc4, + /// Two value blocks: red, then green. + Bc5, } impl Encoding { @@ -247,10 +279,21 @@ impl Encoding { Encoding::Masks { gray: true, fields, .. } if fields[3].is_some() => Layout::GrayA, Encoding::Masks { gray: true, .. } => Layout::Gray, Encoding::Masks { fields, .. } if fields[3].is_none() => Layout::Rgb, + Encoding::Bc4 => Layout::Gray, + Encoding::Bc5 => Layout::Rgb, _ => Layout::Rgba, } } + /// Bytes per 4×4 block; 0 for uncompressed pixels. + fn block_size(&self) -> usize { + match self { + Encoding::Masks { .. } => 0, + Encoding::Bc1 | Encoding::Bc4 => 8, + Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc5 => 16, + } + } + /// Stored row stride and byte size of the top mip level (block rows for BC). fn level_layout(&self, width: u32, height: u32, pitch: Option) -> Result<(usize, usize), DdsError> { let overflow = || bad("pixel data size overflow"); @@ -267,9 +310,8 @@ impl Encoding { } (row_bytes, h) } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { - let block = if matches!(self, Encoding::Bc1) { 8 } else { 16 }; - (w.div_ceil(4).checked_mul(block).ok_or_else(overflow)?, h.div_ceil(4)) + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc4 | Encoding::Bc5 => { + (w.div_ceil(4).checked_mul(self.block_size()).ok_or_else(overflow)?, h.div_ceil(4)) } }; let size = row_bytes.checked_mul(rows).ok_or_else(overflow)?; @@ -286,8 +328,14 @@ fn legacy_encoding(b: &[u8]) -> Result { b"DXT1" => Ok(Encoding::Bc1), b"DXT3" => Ok(Encoding::Bc2), b"DXT5" => Ok(Encoding::Bc3), + // Microsoft's mapping (DirectXTex reads ATI2 as BC5_UNORM, as Pillow does): red in + // the first block, green in the second. FFmpeg's reader swaps the two blocks of + // ATI2, after the old ATI 3Dc tools; this reader doesn't. + b"ATI1" | b"BC4U" => Ok(Encoding::Bc4), + b"ATI2" | b"BC5U" => Ok(Encoding::Bc5), + b"BC4S" | b"BC5S" => Err(unsupported(format!("signed (SNORM) `{}` textures are not supported", fourcc_name(fourcc)))), b"DXT2" | b"DXT4" => Err(unsupported("premultiplied-alpha DXT2/DXT4 textures are not supported")), - _ => Err(unsupported(format!("compression `{}` is not supported (DXT1, DXT3 and DXT5 are)", fourcc_name(fourcc)))), + _ => Err(unsupported(format!("compression `{}` is not supported (DXT1, DXT3, DXT5, ATI1/BC4U and ATI2/BC5U are)", fourcc_name(fourcc)))), }; } if flags & DDPF_RGB != 0 { @@ -321,7 +369,12 @@ fn dx10_encoding(b: &[u8]) -> Result { 70..=72 => Ok(Encoding::Bc1), 73..=75 => Ok(Encoding::Bc2), 76..=78 => Ok(Encoding::Bc3), - _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1, BC2, BC3 and 8-bit RGBA or BGRA are)"))), + // Typeless, then UNORM: a typeless texture's view is unsigned unless a program says + // otherwise, and Pillow writes BC5 as typeless. + 79 | 80 => Ok(Encoding::Bc4), + 82 | 83 => Ok(Encoding::Bc5), + 81 | 84 => Err(unsupported(format!("signed (SNORM) BC4 and BC5 textures are not supported (DXGI format {dxgi})"))), + _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1 to BC5 and 8-bit RGBA or BGRA are)"))), } } @@ -386,7 +439,9 @@ impl<'a> Texture<'a> { let mut pixels = vec![0u8; total]; match encoding { Encoding::Masks { bytes, gray, fields } => decode_masks(Masks { bytes, gray, fields }, data, row_pitch, &mut pixels, row_size), - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => decode_blocks(encoding, data, (w, channels), &mut pixels), + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc4 | Encoding::Bc5 => { + decode_blocks(encoding, data, (w, channels), &mut pixels) + } } Ok(pixels) } @@ -455,16 +510,19 @@ fn decode_masks(m: Masks, data: &[u8], row_pitch: usize, pixels: &mut [u8], row_ /// per task. Blocks wholly inside the image copy whole rows of texels; only those on the right /// edge check each texel. fn decode_blocks(encoding: Encoding, data: &[u8], (width, channels): (usize, usize), pixels: &mut [u8]) { - let block_size = if matches!(encoding, Encoding::Bc1) { 8 } else { 16 }; + let block_size = encoding.block_size().max(1); let blocks_x = width.div_ceil(4); let row_size = width * channels; rows(pixels, row_size.saturating_mul(4), |by, band| { let src = data.get(by * blocks_x * block_size..).unwrap_or_default(); for (bx, block) in src.chunks_exact(block_size).take(blocks_x).enumerate() { + let second = block.get(8..).unwrap_or_default(); let texels = match encoding { - Encoding::Bc1 => bc1(block, true), - Encoding::Bc2 => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc2_alpha(block)), - _ => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc3_alpha(block)), + Encoding::Bc1 => bc::decode_colour(block, true), + Encoding::Bc2 => with_alpha(bc::decode_colour(second, false), bc::decode_bc2_alpha(block)), + Encoding::Bc4 => values(&[bc::decode_values(block)]), + Encoding::Bc5 => values(&[bc::decode_values(block), bc::decode_values(second)]), + _ => with_alpha(bc::decode_colour(second, false), bc::decode_values(block)), }; let x0 = bx * 4; // The band holds the block's rows that lie inside the image (fewer than four in the @@ -485,29 +543,10 @@ fn decode_blocks(encoding: Encoding, data: &[u8], (width, channels): (usize, usi }); } -/// A BC1 colour block. BC2 and BC3 colour blocks always have four colours; only BC1's -/// `c0 <= c1` order selects three colours and transparent black. -fn bc1(block: &[u8], one_bit_alpha: bool) -> [[u8; 4]; 16] { - let byte = |i: usize| block.get(i).copied().unwrap_or(0); - let (c0, c1) = (u16::from_le_bytes([byte(0), byte(1)]), u16::from_le_bytes([byte(2), byte(3)])); - let palette = bc::colour_palette(c0, c1, c0 > c1 || !one_bit_alpha); - let indices = u32::from_le_bytes([byte(4), byte(5), byte(6), byte(7)]); - std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) -} - -/// BC2: four explicit bits of alpha per texel. -fn bc2_alpha(block: &[u8]) -> [u8; 16] { - let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0))); - std::array::from_fn(|t| ((bits >> (4 * t)) & 0xf) as u8 * 17) -} - -/// BC3: two alpha endpoints and a 3-bit index per texel into eight values, or into six plus -/// 0 and 255 when `a0 <= a1`. -fn bc3_alpha(block: &[u8]) -> [u8; 16] { - let byte = |i: usize| block.get(i).copied().unwrap_or(0); - let palette = bc::alpha_palette(byte(0), byte(1)); - let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]); - std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) +/// Texels whose first channels are `channels`' values, the rest 0: gray for BC4, red and green +/// (and blue 0) for BC5. +fn values(channels: &[[u8; 16]]) -> [[u8; 4]; 16] { + std::array::from_fn(|t| std::array::from_fn(|c| channels.get(c).map_or(0, |v| v[t]))) } fn with_alpha(mut texels: [[u8; 4]; 16], alpha: [u8; 16]) -> [[u8; 4]; 16] { @@ -527,6 +566,8 @@ const DDSD_LINEARSIZE: u32 = 0x8_0000; const DDSCAPS_COMPLEX: u32 = 0x8; const DDSCAPS_TEXTURE: u32 = 0x1000; const DDSCAPS_MIPMAP: u32 = 0x40_0000; +const DXGI_BC4_UNORM: u32 = 80; +const DXGI_BC5_UNORM: u32 = 83; fn encode_err(reason: impl fmt::Display) -> DdsError { DdsError::Encode(reason.to_string()) @@ -554,10 +595,13 @@ enum Store { Bc1, Bc2, Bc3, + Bc4, + Bc5, } impl Store { - /// The storage for pixels of `layout`. Block compressions take RGBA only. + /// The storage for pixels of `layout`. Block compressions take their + /// [`DdsCompression::layout`] only. fn new(compression: DdsCompression, layout: Layout) -> Result { Ok(match (compression, layout) { (DdsCompression::None, Layout::Gray) => Store::Luminance, @@ -567,6 +611,8 @@ impl Store { (DdsCompression::Bc1, Layout::Rgba) => Store::Bc1, (DdsCompression::Bc2, Layout::Rgba) => Store::Bc2, (DdsCompression::Bc3, Layout::Rgba) => Store::Bc3, + (DdsCompression::Bc4, Layout::Gray) => Store::Bc4, + (DdsCompression::Bc5, Layout::Rgb) => Store::Bc5, (_, layout) => return Err(encode_err(format!("{layout:?} pixels can't be written as `{}`", compression.name()))), }) } @@ -577,13 +623,22 @@ impl Store { Store::Luminance => 1, Store::LuminanceAlpha => 2, Store::Bgrx | Store::Bgra => 4, - Store::Bc1 => 8, - Store::Bc2 | Store::Bc3 => 16, + Store::Bc1 | Store::Bc4 => 8, + Store::Bc2 | Store::Bc3 | Store::Bc5 => 16, } } fn is_block(self) -> bool { - matches!(self, Store::Bc1 | Store::Bc2 | Store::Bc3) + matches!(self, Store::Bc1 | Store::Bc2 | Store::Bc3 | Store::Bc4 | Store::Bc5) + } + + /// The DXGI format of the DX10 header, for the formats the legacy header can't name. + fn dxgi(self) -> Option { + match self { + Store::Bc4 => Some(DXGI_BC4_UNORM), + Store::Bc5 => Some(DXGI_BC5_UNORM), + _ => None, + } } /// Bytes of one row of pixels (or of 4×4 blocks) of a level `width` wide. @@ -606,6 +661,7 @@ impl Store { Store::Bc1 => (DDPF_FOURCC, *b"DXT1", 0, [0; 4]), Store::Bc2 => (DDPF_FOURCC, *b"DXT3", 0, [0; 4]), Store::Bc3 => (DDPF_FOURCC, *b"DXT5", 0, [0; 4]), + Store::Bc4 | Store::Bc5 => (DDPF_FOURCC, *b"DX10", 0, [0; 4]), } } } @@ -620,7 +676,8 @@ fn half((w, h): (usize, usize)) -> (usize, usize) { } /// A DDS file of `width` × `height` `pixels` in `layout`, interleaved and top row first. -/// Block compressions take [`Layout::Rgba`] pixels; uncompressed textures take every layout. +/// Block compressions take the layout [`DdsCompression::layout`] names (RGBA for BC1 to BC3, +/// gray for BC4, RGB for BC5); uncompressed textures take every layout. pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: DdsOptions) -> Result, DdsError> { let store = Store::new(options.compression, layout)?; let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(layout.channels())); @@ -636,14 +693,18 @@ pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: D sizes.push(store.level_bytes(size.0, size.1).ok_or_else(|| encode_err("texture size overflow"))?); size = half(size); } - let total = sizes.iter().try_fold(HEADER, |total, &s| total.checked_add(s)).ok_or_else(|| encode_err("texture size overflow"))?; + let head = if store.dxgi().is_some() { HEADER + DX10_HEADER } else { HEADER }; + let total = sizes.iter().try_fold(head, |total, &s| total.checked_add(s)).ok_or_else(|| encode_err("texture size overflow"))?; let header = header(width, height, store, levels, sizes.first().copied().unwrap_or(0))?; let mut out = alloc_zeroed(total)?; - out.get_mut(..HEADER).ok_or_else(|| encode_err("output buffer too small"))?.copy_from_slice(&header); + let (Some(dst), Some(src)) = (out.get_mut(..head), header.get(..head)) else { + return Err(encode_err("output buffer too small")); + }; + dst.copy_from_slice(src); let channels = layout.channels(); let mut level = Cow::Borrowed(pixels); - let (mut size, mut at) = (top, HEADER); + let (mut size, mut at) = (top, head); for (i, &bytes) in sizes.iter().enumerate() { let dst = out.get_mut(at..at + bytes).ok_or_else(|| encode_err("output buffer too small"))?; write_level(store, &level, size, dst)?; @@ -657,9 +718,10 @@ pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: D Ok(out) } -/// The `DDS ` signature and DDS_HEADER. `top` is the top level's size in bytes. -fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Result<[u8; HEADER], DdsError> { - let mut b = [0u8; HEADER]; +/// The `DDS ` signature, DDS_HEADER and, for a format only DXGI names, DDS_HEADER_DXT10 (the +/// caller keeps the first [`HEADER`] bytes otherwise). `top` is the top level's size in bytes. +fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Result<[u8; HEADER + DX10_HEADER], DdsError> { + let mut b = [0u8; HEADER + DX10_HEADER]; let mut put = |at: usize, v: u32| { if let Some(d) = b.get_mut(at..at + 4) { d.copy_from_slice(&v.to_le_bytes()); @@ -686,6 +748,12 @@ fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Res put(92 + 4 * i, mask); } put(108, DDSCAPS_TEXTURE | if mipmapped { DDSCAPS_COMPLEX | DDSCAPS_MIPMAP } else { 0 }); + if let Some(dxgi) = store.dxgi() { + // One 2D texture, no misc flags, alpha mode unknown (the formats have no alpha). + put(HEADER, dxgi); + put(HEADER + 4, DIMENSION_TEXTURE2D); + put(HEADER + 12, 1); + } b[..4].copy_from_slice(b"DDS "); Ok(b) } @@ -739,10 +807,40 @@ fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst } }); } + Store::Bc4 | Store::Bc5 => { + // BC4: the gray channel; BC5: red, then green, of RGB pixels. + let channels = if store == Store::Bc4 { 1 } else { 3 }; + rows(dst, row_bytes, |by, row| { + for (bx, d) in row.chunks_exact_mut(store.unit()).enumerate() { + for (c, d) in d.as_chunks_mut::<8>().0.iter_mut().enumerate() { + let (values, inside) = gather_channel(pixels, (width, height), channels, c, (bx, by)); + *d = bc::value_block(&values, inside); + } + } + }); + } } Ok(()) } +/// Channel `c` of block (`bx`, `by`) of pixels with `channels` channels, and which of its texels +/// lie inside the image. +fn gather_channel(pixels: &[u8], (width, height): (usize, usize), channels: usize, c: usize, (bx, by): (usize, usize)) -> ([u8; 16], Mask) { + let mut values = [0u8; 16]; + let mut inside: Mask = 0; + for (t, v) in values.iter_mut().enumerate() { + let (x, y) = (bx * 4 + t % 4, by * 4 + t / 4); + if x < width + && y < height + && let Some(&p) = pixels.get((y * width + x) * channels + c) + { + *v = p; + inside |= 1 << t; + } + } + (values, inside) +} + /// Block (`bx`, `by`)'s texels, and which of them lie inside the image. fn gather(texels: &[[u8; 4]], (width, height): (usize, usize), bx: usize, by: usize) -> (Block, Mask) { let mut block = [[0u8; 4]; 16]; @@ -831,9 +929,17 @@ mod tests { } #[test] - fn block_compressions_take_rgba_only() { + fn block_compressions_take_their_layout_only() { let options = DdsOptions { compression: DdsCompression::Bc3, mipmaps: false }; assert_eq!(encode(1, 1, Layout::Rgb, &[1, 2, 3], options), Err(DdsError::Encode("Rgb pixels can't be written as `bc3`".into()))); + for compression in DdsCompression::ALL { + for layout in [Layout::Gray, Layout::GrayA, Layout::Rgb, Layout::Rgba] { + let options = DdsOptions { compression, mipmaps: false }; + let written = encode(1, 1, layout, &[9; 4][..layout.channels()], options); + let takes = compression.layout().is_none_or(|l| l == layout); + assert_eq!(written.is_ok(), takes, "{compression:?} from {layout:?}"); + } + } } #[test] diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index 27ddddb69..1bdf1418d 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -46,7 +46,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "< | BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` | | TGA | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | no | `image` (read) / built in (RLE write, one scanline per packet) | | ICO | yes | yes | U8 | RGBA (at most 256×256) | yes | no | no | no | no | no | no | `image` | -| DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC3, when `EncodeOptions::dds` asks for one) | [`craft-dds`](../dds/README.md) | +| DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC5, when `EncodeOptions::dds` asks for one) | [`craft-dds`](../dds/README.md) | | Netpbm (PBM/PGM/PPM/PAM/PFM) | yes | yes | U8, U16, F32 (float: PFM gray/RGB only) | all six (PAM `CMYK`/`CMYK_ALPHA`) | yes (PAM) | no | no | no | no | no | no | built in | | QOI | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` | | OpenEXR | yes | yes | F16, F32 | Y, YA, RGB, RGBA | yes | no | no | no | no | no | no | `exr` | @@ -94,19 +94,27 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them * **DDS** (through [`craft-dds`](../dds/README.md), whose `DdsCompression` and `DdsOptions` are re-exported). The top mip level of a 2D texture: uncompressed pixels described by channel masks - (8, 16, 24 or 32 bits, RGB or luminance, with or without alpha) or BC1/BC2/BC3 (DXT1/DXT3/DXT5), - with the legacy header or the DX10 one. Uncompressed row padding follows the header's - `DDSD_PITCH` stride; without that flag, rows are tightly packed. Everything opens as U8; the - other mip levels are ignored. Cubemaps, texture arrays, volume textures, premultiplied DXT2/DXT4 and other formats - (BC4–BC7, float) are refused with `CodecError::Unsupported`. Writing (`EncodeOptions::dds`) makes - a 2D texture with the legacy header: uncompressed by default (BGRA, BGRX, or luminance with or - without alpha, so it reads back exactly), or BC1/BC2/BC3. The block encoder is our own, from + (8, 16, 24 or 32 bits, RGB or luminance, with or without alpha), BC1/BC2/BC3 (DXT1/DXT3/DXT5) or + unsigned BC4/BC5 (ATI1/ATI2, BC4U/BC5U, DXGI 79, 80, 82 and 83), with the legacy header or the + DX10 one. BC4 opens as Gray and BC5 as RGB with blue 0 (BC5 stores only red and green; a normal + map's Z is not reconstructed). ATI2 is read as Microsoft reads it, red in the first block. + Uncompressed row padding follows the header's `DDSD_PITCH` stride; without that flag, rows are + tightly packed. Everything opens as U8; the other mip levels are ignored. Cubemaps, texture + arrays, volume textures, premultiplied DXT2/DXT4, signed (SNORM) BC4/BC5 and other formats + (BC6H, BC7, float) are refused with `CodecError::Unsupported`. Writing (`EncodeOptions::dds`) + makes a 2D texture: uncompressed by default (BGRA, BGRX, or luminance with or without alpha, so + it reads back exactly), BC1/BC2/BC3 (from RGBA) with the legacy header, or BC4 (from gray, for + roughness, height and other single-channel maps) and BC5 (from RGB, blue dropped, for normal + maps) with the DX10 header (`BC4_UNORM`, `BC5_UNORM`). The plan converts other layouts to what + the compression stores, so `fidelity_warnings` reports colour to gray for BC4 and dropped alpha + for both; BC5's dropped blue comes under `LossyCompression`. The block encoder is our own, from Microsoft's public block-compression documentation: RGB565 endpoints fitted along each block's principal axis and refined by least squares, indices picked against the palette the reader - decodes; BC1 turns texels under half opacity into its transparent black, and BC3 picks - whichever alpha mode fits. `mipmaps` adds the full chain down to 1×1, each level a box filter - of the one above (the last column and row of an odd-sized level average three pixels, so none - is dropped). Cubemaps, arrays, BC4–BC7 and sRGB (DX10-only) formats aren't written. + decodes; BC1 turns texels under half opacity into its transparent black, and BC3, BC4 and BC5 + pick whichever mode of their value blocks fits. `mipmaps` adds the full chain down to 1×1, each + level a box filter of the one above (the last column and row of an odd-sized level average + three pixels, so none is dropped). Cubemaps, arrays, BC6H, BC7 and sRGB (DX10-only) formats + aren't written. * **FITS.** The image in the primary HDU, or the first `IMAGE` extension, is read; `NAXIS3 = 3` opens as RGB and any other plane count opens the first plane with `DecodeWarning::MorePages`. diff --git a/crates/image-codecs/src/codecs/dds.rs b/crates/image-codecs/src/codecs/dds.rs index 9d682b03f..26a100fab 100644 --- a/crates/image-codecs/src/codecs/dds.rs +++ b/crates/image-codecs/src/codecs/dds.rs @@ -19,7 +19,7 @@ fn error(e: DdsError) -> CodecError { } } -fn channel_layout(layout: Layout) -> ChannelLayout { +pub(crate) fn channel_layout(layout: Layout) -> ChannelLayout { match layout { Layout::Gray => ChannelLayout::Gray, Layout::GrayA => ChannelLayout::GrayA, diff --git a/crates/image-codecs/src/fidelity.rs b/crates/image-codecs/src/fidelity.rs index 12c285d7b..52d6eaa23 100644 --- a/crates/image-codecs/src/fidelity.rs +++ b/crates/image-codecs/src/fidelity.rs @@ -4,7 +4,7 @@ use std::fmt; -use crate::codecs::dpx; +use crate::codecs::{dds, dpx}; use crate::format::{Format, caps}; use crate::image::{ChannelLayout, Image, SampleType}; use crate::options::{DdsCompression, EncodeOptions}; @@ -45,7 +45,8 @@ pub enum FidelityWarning { ColorToGray, /// The image will be quantized to a 256-colour palette. PaletteQuantized, - /// The format's encoder is lossy. + /// The format's encoder is lossy. DDS BC5 also keeps only red and green: blue is dropped and + /// reads back as 0 (for a normal map, the renderer rebuilds Z from X and Y). LossyCompression, IccDropped, ExifDropped, @@ -175,8 +176,12 @@ pub(crate) fn plan(image: &Image, format: Format, opts: &EncodeOptions) -> Plan }, // CIELab has no float form: float Lab is written with 16 bits. Format::Tiff if layout.is_lab() && sample.is_float() => Plan { layout: pick_layout(layout, c.layouts, c.alpha), sample: SampleType::U16 }, - // A block-compressed DDS stores RGBA whatever the source; uncompressed keeps gray and RGB. - Format::Dds if opts.dds.compression.is_block() => Plan { layout: ChannelLayout::Rgba, sample: SampleType::U8 }, + // A block-compressed DDS stores what its compression holds whatever the source: gray for + // BC4, red and green for BC5 (written from RGB, blue dropped), RGBA for BC1 to BC3. + // Uncompressed keeps gray and RGB. + Format::Dds if opts.dds.compression.is_block() => { + Plan { layout: opts.dds.compression.layout().map_or(ChannelLayout::Rgba, dds::channel_layout), sample: SampleType::U8 } + } _ => Plan { layout: pick_layout(layout, c.layouts, c.alpha), sample: pick_sample(sample, c.depths) }, } } diff --git a/crates/image-codecs/src/format.rs b/crates/image-codecs/src/format.rs index 26696ae2a..4b6ee5bb9 100644 --- a/crates/image-codecs/src/format.rs +++ b/crates/image-codecs/src/format.rs @@ -19,7 +19,7 @@ pub enum Format { Pcx, /// X PixMap icons (XPM2 plain text and XPM3 C-string arrays). Xpm, - /// DirectDraw Surface: a 2D texture, uncompressed or BC1–BC3. Reading takes the top mip + /// DirectDraw Surface: a 2D texture, uncompressed or BC1–BC5. Reading takes the top mip /// level; writing can add the full mip chain ([`crate::DdsOptions`]). Dds, Qoi, diff --git a/crates/image-codecs/tests/it/dds.rs b/crates/image-codecs/tests/it/dds.rs index 4f766a54c..ad927019a 100644 --- a/crates/image-codecs/tests/it/dds.rs +++ b/crates/image-codecs/tests/it/dds.rs @@ -318,10 +318,83 @@ fn refuses_cubemaps_arrays_and_volumes() { unsupported(&dx10(4, 4, 71, 3, 0, 6), "array"); unsupported(&dx10(4, 4, 71, 4, 0, 1), "2D"); unsupported(&dx10(4, 4, 98, 3, 0, 1), "98"); // BC7_UNORM - unsupported(&compressed(4, 4, b"ATI2"), "ATI2"); + unsupported(&compressed(4, 4, b"ATI3"), "ATI3"); unsupported(&compressed(4, 4, b"DXT2"), "premultiplied"); } +/// A BC4 block of 200 everywhere (eight-value mode, a0 = 200 > a1 = 50, every index 0). +const BC4_200: [u8; 8] = [0xc8, 0x32, 0, 0, 0, 0, 0, 0]; +/// A value block of 250 everywhere (six-value mode, a0 = 10 <= a1 = 250, every index 1). +const VALUES_250: [u8; 8] = [0x0a, 0xfa, 0x49, 0x92, 0x24, 0x49, 0x92, 0x24]; + +#[test] +fn decodes_bc4_as_gray_from_every_name() { + // 5×3: two blocks across, one down; the second block's texels outside the image are dropped. + let blocks = [BC4_200, BC4_200].concat(); + let mut files = vec![]; + for fourcc in [b"ATI1", b"BC4U"] { + let mut b = compressed(5, 3, fourcc); + b.extend(&blocks); + files.push((String::from_utf8_lossy(fourcc).into_owned(), b)); + } + for dxgi in [79, 80] { + let mut b = dx10(5, 3, dxgi, 3, 0, 1); // BC4_TYPELESS, BC4_UNORM + b.extend(&blocks); + files.push((format!("DXGI {dxgi}"), b)); + } + for (name, b) in files { + let im = decode(&b).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!((im.dimensions(), im.layout()), ((5, 3), ChannelLayout::Gray), "{name}"); + assert_eq!(im.data(), &[200; 15], "{name}"); + } +} + +#[test] +fn decodes_bc5_as_red_and_green_with_blue_zero() { + let block = [BC4_200, VALUES_250].concat(); + let mut files = vec![]; + // ATI2 is BC5 as Microsoft reads it: red in the first block, not swapped as FFmpeg does. + for fourcc in [b"ATI2", b"BC5U"] { + let mut b = compressed(3, 2, fourcc); + b.extend(&block); + files.push((String::from_utf8_lossy(fourcc).into_owned(), b)); + } + // BC5_TYPELESS (what Pillow writes) and BC5_UNORM. + for dxgi in [82, 83] { + let mut b = dx10(3, 2, dxgi, 3, 0, 1); + b.extend(&block); + files.push((format!("DXGI {dxgi}"), b)); + } + for (name, b) in files { + let im = decode(&b).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!((im.dimensions(), im.layout()), ((3, 2), ChannelLayout::Rgb), "{name}"); + assert!(im.data().chunks(3).all(|p| p == [200, 250, 0]), "{name}: {:?}", &im.data()[..3]); + } +} + +#[test] +fn refuses_signed_bc4_and_bc5() { + for mut b in [compressed(4, 4, b"BC4S"), compressed(4, 4, b"BC5S"), dx10(4, 4, 81, 3, 0, 1), dx10(4, 4, 84, 3, 0, 1)] { + b.extend([0; 16]); + match decode_as(Format::Dds, &b) { + Err(CodecError::Unsupported { reason, .. }) => assert!(reason.contains("SNORM"), "{reason}"), + other => panic!("{other:?}"), + } + } +} + +#[test] +fn truncated_bc4_and_bc5_data_is_refused() { + let mut b = dx10(5, 5, 83, 3, 0, 1); // four 16-byte blocks + b.extend([0x5a; 64]); + assert!(decode(&b).is_ok()); + assert!(matches!(decode_as(Format::Dds, &b[..b.len() - 1]), Err(CodecError::Malformed { .. }))); + let mut b = compressed(5, 5, b"ATI1"); // four 8-byte blocks + b.extend([0x5a; 32]); + assert!(decode(&b).is_ok()); + assert!(matches!(decode_as(Format::Dds, &b[..b.len() - 1]), Err(CodecError::Malformed { .. }))); +} + /// Face declarations matter even when the general cube flag is absent; padding does not. #[test] fn source_probe_distinguishes_extra_faces_from_single_images_and_padding() { diff --git a/crates/image-codecs/tests/it/dds_write.rs b/crates/image-codecs/tests/it/dds_write.rs index f9d9add8e..bc92c87b6 100644 --- a/crates/image-codecs/tests/it/dds_write.rs +++ b/crates/image-codecs/tests/it/dds_write.rs @@ -16,6 +16,8 @@ const DDPF_ALPHAPIXELS: u32 = 0x1; const DDPF_FOURCC: u32 = 0x4; const DDPF_RGB: u32 = 0x40; const DDPF_LUMINANCE: u32 = 0x2_0000; +const DXGI_BC4_UNORM: u32 = 80; +const DXGI_BC5_UNORM: u32 = 83; const BLOCKS: [DdsCompression; 3] = [DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3]; @@ -80,11 +82,21 @@ fn psnr(a: &Image, b: &Image) -> f64 { if mse == 0.0 { f64::INFINITY } else { 10.0 * (255.0f64.powi(2) / mse).log10() } } +/// Whether a written texture has the DX10 header after the legacy one. +fn has_dx10(file: &[u8]) -> bool { + le32(file, 80) & DDPF_FOURCC != 0 && &file[84..88] == b"DX10" +} + +/// Bytes of the headers before the first level. +fn head(file: &[u8]) -> usize { + if has_dx10(file) { 148 } else { 128 } +} + /// Bytes of one level as a reader computes them from the header's pixel format. fn level_bytes(file: &[u8], w: usize, h: usize) -> usize { if le32(file, 80) & DDPF_FOURCC != 0 { - let block = if &file[84..88] == b"DXT1" { 8 } else { 16 }; - w.div_ceil(4) * h.div_ceil(4) * block + let eight = &file[84..88] == b"DXT1" || (has_dx10(file) && le32(file, 128) == DXGI_BC4_UNORM); + w.div_ceil(4) * h.div_ceil(4) * if eight { 8 } else { 16 } } else { w * h * le32(file, 88) as usize / 8 } @@ -94,9 +106,9 @@ fn level_bytes(file: &[u8], w: usize, h: usize) -> usize { fn level(file: &[u8], i: u32) -> Image { let (w, h) = (le32(file, 16) as usize, le32(file, 12) as usize); let size = |i: u32| ((w >> i).max(1), (h >> i).max(1)); - let start = 128 + (0..i).map(|j| level_bytes(file, size(j).0, size(j).1)).sum::(); + let start = head(file) + (0..i).map(|j| level_bytes(file, size(j).0, size(j).1)).sum::(); let (lw, lh) = size(i); - let mut b = file[..128].to_vec(); + let mut b = file[..head(file)].to_vec(); b[12..16].copy_from_slice(&(lh as u32).to_le_bytes()); b[16..20].copy_from_slice(&(lw as u32).to_le_bytes()); b[28..32].copy_from_slice(&1u32.to_le_bytes()); @@ -338,15 +350,122 @@ fn fidelity_warnings_follow_the_compression() { } assert!(warns(DdsCompression::Bc1).contains(&FidelityWarning::AlphaBinarized)); assert!(!warns(DdsCompression::Bc3).contains(&FidelityWarning::AlphaBinarized)); + // BC4 is gray and BC5 has no alpha: translucent colour loses what they don't hold. + let bc4 = warns(DdsCompression::Bc4); + for w in [FidelityWarning::LossyCompression, FidelityWarning::ColorToGray, FidelityWarning::AlphaDiscarded] { + assert!(bc4.contains(&w), "BC4: {w:?} missing from {bc4:?}"); + } + let bc5 = warns(DdsCompression::Bc5); + assert!(bc5.contains(&FidelityWarning::LossyCompression) && bc5.contains(&FidelityWarning::AlphaDiscarded), "{bc5:?}"); + assert!(!bc5.contains(&FidelityWarning::ColorToGray) && !bc5.contains(&FidelityWarning::AlphaBinarized), "{bc5:?}"); + // A gray map written as BC4 loses only to the compression. + let gray = ramp(8, 8, ChannelLayout::Gray); + assert_eq!(fidelity_warnings_with(&gray, Format::Dds, &opts(DdsCompression::Bc4, false)), [FidelityWarning::LossyCompression]); +} + +/// The largest difference in channel `c` between two images of `channels` channels. +fn max_diff(a: &Image, b: &Image, channels: usize, c: usize) -> u8 { + a.data().chunks(channels).zip(b.data().chunks(channels)).map(|(x, y)| x[c].abs_diff(y[c])).max().unwrap_or(0) +} + +/// `gradient`'s red as gray, or its red, green and blue. +fn gradient_as(w: u32, h: u32, layout: ChannelLayout) -> Image { + let rgba = gradient(w, h); + let data = rgba.data().chunks(4).flat_map(|p| if layout == ChannelLayout::Gray { vec![p[0]] } else { p[..3].to_vec() }).collect(); + Image::from_u8(w, h, layout, data).unwrap() +} + +#[test] +fn bc4_and_bc5_are_written_with_the_dx10_header() { + for (compression, dxgi, block, layout) in + [(DdsCompression::Bc4, DXGI_BC4_UNORM, 8, ChannelLayout::Gray), (DdsCompression::Bc5, DXGI_BC5_UNORM, 16, ChannelLayout::Rgb)] + { + for (w, h) in [(1, 1), (2, 2), (5, 3), (33, 17)] { + let img = ramp(w, h, layout); + let b = encode(&img, Format::Dds, &opts(compression, false)).unwrap(); + let linear = w.div_ceil(4) * h.div_ceil(4) * block; + assert_eq!(&b[84..88], b"DX10"); + assert_eq!((le32(&b, 80), le32(&b, 88)), (DDPF_FOURCC, 0)); + assert_eq!(le32(&b, 8), REQUIRED | DDSD_LINEARSIZE, "linear size, not pitch"); + assert_eq!(le32(&b, 20), linear, "{compression:?} {w}x{h}: the top level's size"); + // DXGI format, a 2D texture, no misc flags, one texture, alpha mode unknown. + assert_eq!([le32(&b, 128), le32(&b, 132), le32(&b, 136), le32(&b, 140), le32(&b, 144)], [dxgi, 3, 0, 1, 0]); + assert_eq!(b.len(), 148 + linear as usize); + assert!(!craft_image_codecs::has_unretained_source_data(&b).unwrap()); + let back = decode(&b).unwrap(); + assert_eq!((back.dimensions(), back.layout()), ((w, h), layout), "{compression:?} {w}x{h}"); + } + } +} + +#[test] +fn bc4_and_bc5_error_on_gradients_is_bounded() { + // Each block of the gradient spans 12 levels in a channel: eight values (or six and the + // extremes) are under two levels apart. + let gray = gradient_as(64, 64, ChannelLayout::Gray); + let back = decode(&encode(&gray, Format::Dds, &opts(DdsCompression::Bc4, false)).unwrap()).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Gray); + let err = max_diff(&gray, &back, 1, 0); + assert!(err <= 2, "BC4 off by {err}"); + + let rgb = gradient_as(64, 64, ChannelLayout::Rgb); + let back = decode(&encode(&rgb, Format::Dds, &opts(DdsCompression::Bc5, false)).unwrap()).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Rgb); + let (red, green) = (max_diff(&rgb, &back, 3, 0), max_diff(&rgb, &back, 3, 1)); + assert!(red <= 2 && green <= 2, "BC5 red off by {red}, green by {green}"); + assert!(back.data().chunks(3).all(|p| p[2] == 0), "BC5 has no blue"); +} + +#[test] +fn bc4_and_bc5_take_any_source_layout() { + // RGBA to BC4 goes through gray; to BC5 through RGB, dropping alpha and then blue. + let img = gradient(9, 7); + for (compression, layout) in [(DdsCompression::Bc4, ChannelLayout::Gray), (DdsCompression::Bc5, ChannelLayout::Rgb)] { + let back = decode(&encode(&img, Format::Dds, &opts(compression, true)).unwrap()).unwrap(); + let want = img.convert(layout, img.sample_type()); + let ch = layout.channels(); + assert_eq!((back.dimensions(), back.layout()), ((9, 7), layout)); + for c in 0..ch.min(2) { + let err = max_diff(&want, &back, ch, c); + assert!(err <= 8, "{compression:?} channel {c} off by {err}"); + } + } +} + +#[test] +fn bc4_and_bc5_mip_chains_follow_the_box_filter() { + for (compression, layout, block) in [(DdsCompression::Bc4, ChannelLayout::Gray, 8), (DdsCompression::Bc5, ChannelLayout::Rgb, 16)] { + let img = gradient_as(12, 6, layout); + let b = encode(&img, Format::Dds, &opts(compression, true)).unwrap(); + // 12×6, 6×3, 3×1, 1×1: 3×2, 2×1, 1 and 1 blocks after both headers. + assert_eq!(le32(&b, 28), 4); + assert_eq!(le32(&b, 8), REQUIRED | DDSD_LINEARSIZE | DDSD_MIPMAPCOUNT); + assert_eq!(b.len(), 148 + (6 + 2 + 1 + 1) * block); + let mut want = img.clone(); + for i in 0..4 { + let got = level(&b, i); + assert_eq!((got.dimensions(), got.layout()), (want.dimensions(), layout), "{compression:?} level {i}"); + for c in 0..layout.channels().min(2) { + // Smaller levels are steeper (8, then 16 levels a pixel), so a block's values + // sit up to a few levels off its palette. + let err = max_diff(&want, &got, layout.channels(), c); + assert!(err <= 4, "{compression:?} level {i} channel {c} off by {err}"); + } + want = box_filter(&want); + } + } } #[test] fn compression_names_round_trip() { + assert_eq!(DdsCompression::ALL.len(), 6); for c in DdsCompression::ALL { assert_eq!(DdsCompression::from_name(c.name()), Some(c)); } assert_eq!(DdsCompression::from_name("DXT5"), Some(DdsCompression::Bc3)); assert_eq!(DdsCompression::from_name("dxt1"), Some(DdsCompression::Bc1)); + assert_eq!(DdsCompression::from_name("ATI1"), Some(DdsCompression::Bc4)); + assert_eq!(DdsCompression::from_name("ati2"), Some(DdsCompression::Bc5)); for bad in ["", "bc7", "dxt2", "uncompressed "] { assert_eq!(DdsCompression::from_name(bad), None, "{bad:?}"); } diff --git a/crates/raster-codecs/README.md b/crates/raster-codecs/README.md index 907e73763..eec7ac0d5 100644 --- a/crates/raster-codecs/README.md +++ b/crates/raster-codecs/README.md @@ -11,7 +11,7 @@ craft-raster-codecs = { git = "https://github.com/storytold/craft-libs", rev = " | Module | Format | Read | Write | |---|---|---|---| -| `dds` | DirectDraw Surface: uncompressed masks, luminance, BC1/BC2/BC3 (DXT1/3/5), legacy and DX10 headers; top mip level | yes | uncompressed and BC1-BC3, optional mip chain | +| `dds` | DirectDraw Surface: uncompressed masks, luminance, BC1/BC2/BC3 (DXT1/3/5), unsigned BC4/BC5 (ATI1/2, as gray and as RGB with blue 0), legacy and DX10 headers; top mip level | yes | uncompressed and BC1-BC3, optional mip chain | | `dicom` | DICOM Part 10, uncompressed transfer syntaxes: MONOCHROME1/2, RGB, YBR_FULL, 8 and 16 bits | yes | no | | `fits` | FITS primary HDU or first `IMAGE` extension, every `BITPIX`, `BSCALE`/`BZERO`, header cards as text | yes | gray or RGB; 8-bit, 16-bit, float | | `iff` | IFF ILBM (1 to 8 planes with a palette, 24 planes, mask plane or transparent colour) and PBM, stored or ByteRun1; HAM and EHB are refused | yes | 24-plane ILBM with an optional mask plane | diff --git a/crates/raster-codecs/src/lib.rs b/crates/raster-codecs/src/lib.rs index 7e705da99..a9aa7c4c6 100644 --- a/crates/raster-codecs/src/lib.rs +++ b/crates/raster-codecs/src/lib.rs @@ -3,7 +3,7 @@ //! //! | Module | Format | Read | Write | //! |---|---|---|---| -//! | [`dds`] | DirectDraw Surface: uncompressed masks, BC1-BC3 (DXT1/3/5), the DX10 header | yes | yes | +//! | [`dds`] | DirectDraw Surface: uncompressed masks, BC1-BC5 (DXT1/3/5, ATI1/2), the DX10 header | yes | yes (uncompressed, BC1-BC3) | //! | [`dicom`] | DICOM Part 10, uncompressed native pixel data | yes | no | //! | [`fits`] | FITS primary and image-extension HDUs | yes | yes | //! | [`iff`] | IFF ILBM (palette or 24-bit planes, masks) and PBM | yes | yes (24-plane ILBM) | diff --git a/crates/raster-codecs/tests/dds.rs b/crates/raster-codecs/tests/dds.rs index afb9066aa..d5c482aa7 100644 --- a/crates/raster-codecs/tests/dds.rs +++ b/crates/raster-codecs/tests/dds.rs @@ -320,7 +320,8 @@ fn refuses_cubemaps_arrays_and_volumes() { unsupported(&dx10(4, 4, 71, 3, 0, 6), "array"); unsupported(&dx10(4, 4, 71, 4, 0, 1), "2D"); unsupported(&dx10(4, 4, 98, 3, 0, 1), "98"); // BC7_UNORM - unsupported(&compressed(4, 4, b"ATI2"), "ATI2"); + unsupported(&compressed(4, 4, b"ATI3"), "ATI3"); + unsupported(&compressed(4, 4, b"BC5S"), "SNORM"); unsupported(&compressed(4, 4, b"DXT2"), "premultiplied"); } From c27333683a190ddd0aebd9e355b94a4538ba6692 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 23:38:25 +0100 Subject: [PATCH 024/209] DDS: time decoding a 30 MP texture too An ignored test beside the encode timings: the best of five decodes of the same texture written with each compression. It uses only the API the reader had before, so it runs on the old code for comparison. --- crates/image-codecs/tests/it/dds_write.rs | 36 ++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/crates/image-codecs/tests/it/dds_write.rs b/crates/image-codecs/tests/it/dds_write.rs index bc92c87b6..af6a2c7d2 100644 --- a/crates/image-codecs/tests/it/dds_write.rs +++ b/crates/image-codecs/tests/it/dds_write.rs @@ -479,6 +479,18 @@ fn compression_names_round_trip() { #[test] #[ignore] fn dds_encode_timings_on_30_mp() { + let img = texture_30_mp(); + for compression in DdsCompression::ALL { + for mipmaps in [false, true] { + let t = std::time::Instant::now(); + let b = encode(&img, Format::Dds, &opts(compression, mipmaps)).unwrap(); + eprintln!("{compression:?} mipmaps={mipmaps}: {:.0} ms, {} MB", t.elapsed().as_secs_f64() * 1e3, b.len() >> 20); + } + } +} + +/// A synthetic ~30 MP RGBA texture (6336×4752): gentle ramps with a little noise. +fn texture_30_mp() -> Image { let (w, h) = (6336u32, 4752u32); let mut s = 0x1234_5678u32; let mut data = Vec::with_capacity(w as usize * h as usize * 4); @@ -490,12 +502,28 @@ fn dds_encode_timings_on_30_mp() { data.extend([(x / 25 + y / 19) as u8, (x / 31) as u8 ^ (s as u8 & 3), (y / 23) as u8, (255 - x / 25) as u8]); } } - let img = Image::from_u8(w, h, ChannelLayout::Rgba, data).unwrap(); + Image::from_u8(w, h, ChannelLayout::Rgba, data).unwrap() +} + +/// Decode timings of the same texture written with each compression: the best of five runs, so +/// one slow run doesn't decide. Ignored by default; run in release: +/// +/// ```sh +/// cargo test --release -p craft-image-codecs --test it -- dds_write:: --ignored --nocapture +/// ``` +#[test] +#[ignore] +fn dds_decode_timings_on_30_mp() { + let img = texture_30_mp(); for compression in DdsCompression::ALL { - for mipmaps in [false, true] { + let b = encode(&img, Format::Dds, &opts(compression, false)).unwrap(); + let mut best = f64::INFINITY; + for _ in 0..5 { let t = std::time::Instant::now(); - let b = encode(&img, Format::Dds, &opts(compression, mipmaps)).unwrap(); - eprintln!("{compression:?} mipmaps={mipmaps}: {:.0} ms, {} MB", t.elapsed().as_secs_f64() * 1e3, b.len() >> 20); + let back = decode(&b).unwrap(); + best = best.min(t.elapsed().as_secs_f64()); + assert_eq!(back.dimensions(), img.dimensions()); } + eprintln!("{compression:?} decode: {:.0} ms, {} MB", best * 1e3, b.len() >> 20); } } From fc808abd499b624f06cd7ff6c1f7ea1c61dad95d Mon Sep 17 00:00:00 2001 From: vangel76 <271991477+vangel76@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:12:22 +0100 Subject: [PATCH 025/209] DOCX: read and write the "gutter at top" setting (w:gutterAtTop) The reader ignored w:gutterAtTop in settings.xml (it was only carried through on save), so an app using the model could not tell that Word adds the gutter to the top margin instead of the left one. Add Settings::gutter_at_top (document-wide, off by default), read it with the shared on/off helper, write it after w:mirrorMargins and before w:attachedTemplate, and make the settings merge treat it as a modelled child so turning it off removes it. Tests: round trip and default off, every on/off spelling, merge into a settings part from another writer (keep, remove, insert in schema order). Co-Authored-By: Claude Sonnet 5.5 --- crates/docx/src/read/meta.rs | 1 + crates/docx/src/write/settings.rs | 4 ++ crates/docx/src/write/settings_merge.rs | 17 +++++ crates/docx/tests/it/roundtrip.rs | 96 +++++++++++++++++++++++++ crates/wordmodel/src/settings.rs | 5 ++ 5 files changed, 123 insertions(+) diff --git a/crates/docx/src/read/meta.rs b/crates/docx/src/read/meta.rs index bb4804e0d..8cf4c99dd 100644 --- a/crates/docx/src/read/meta.rs +++ b/crates/docx/src/read/meta.rs @@ -56,6 +56,7 @@ impl Reader<'_> { } "m:mathPr" => s.math = Some(math::read_math_pr(k)), "w:mirrorMargins" => s.mirror_margins = on_off(k), + "w:gutterAtTop" => s.gutter_at_top = on_off(k), "w:autoHyphenation" => s.auto_hyphenation = on_off(k), "w:doNotHyphenateCaps" => s.hyphenate_caps = !on_off(k), "w:hyphenationZone" => { diff --git a/crates/docx/src/write/settings.rs b/crates/docx/src/write/settings.rs index 18f9f8239..199676e16 100644 --- a/crates/docx/src/write/settings.rs +++ b/crates/docx/src/write/settings.rs @@ -50,6 +50,10 @@ pub(super) fn settings_xml(doc: &Document, footnotes: bool, endnotes: bool, temp if s.mirror_margins { w.empty("w:mirrorMargins", &[]); } + // CT_Settings order: after `w:mirrorMargins`, before `w:attachedTemplate`. + if s.gutter_at_top { + w.empty("w:gutterAtTop", &[]); + } if let Some(id) = template_rel { w.empty("w:attachedTemplate", &[("r:id", id)]); } diff --git a/crates/docx/src/write/settings_merge.rs b/crates/docx/src/write/settings_merge.rs index 981b9c098..01b7d27c6 100644 --- a/crates/docx/src/write/settings_merge.rs +++ b/crates/docx/src/write/settings_merge.rs @@ -18,6 +18,7 @@ const OWNED: &[&str] = &[ "w:removeDateAndTime", "w:displayBackgroundShape", "w:mirrorMargins", + "w:gutterAtTop", "w:attachedTemplate", "w:linkStyles", "w:trackRevisions", @@ -386,4 +387,20 @@ mod tests { assert!(out.contains(""), "{out}"); assert!(out.contains(""), "{out}"); } + + #[test] + fn gutter_at_top_comes_from_the_generated_part() { + let orig = r#""#; + assert!(has_unowned(orig.as_bytes())); + // Turned off in the model: the original element goes, the foreign one stays. + let out = String::from_utf8(merge(orig.as_bytes(), GEN.as_bytes()).unwrap()).unwrap(); + assert!(!out.contains("gutterAtTop") && out.contains("").count(), 1, "{out}"); + assert!(out.find("") < out.find("") && out.find("") < out.find(""#)); + } } diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 8f3bbdd0b..3ba2f326a 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -2926,3 +2926,99 @@ fn drop_cap_options_round_trip() { (Some(4), Some(true), None, None) ); } + +/// `word/settings.xml` of a saved package. +fn settings_part(bytes: &[u8]) -> String { + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + std::io::Read::read_to_string(&mut zip.by_name("word/settings.xml").unwrap(), &mut xml).unwrap(); + xml +} + +/// Page Setup's "Gutter position: Top" (`w:gutterAtTop`) is a document setting: off by default +/// (no element), written when on after `w:mirrorMargins` and before `w:attachedTemplate`, and +/// read back; the section's own gutter width is untouched. +#[test] +fn gutter_at_top_round_trips_and_is_off_by_default() { + let plain = craft_docx::write(&Document::new()).unwrap(); + assert!(!settings_part(&plain).contains("gutterAtTop")); + assert!(!craft_docx::read(&plain).unwrap().settings.gutter_at_top); + + let mut d = Document::new(); + d.settings.gutter_at_top = true; + d.settings.mirror_margins = true; + d.settings.attached_template = Some("file:///C:/Templates/Report.dotx".into()); + d.last_section.gutter = 50.0; + let bytes = craft_docx::write(&d).unwrap(); + let xml = settings_part(&bytes); + assert_eq!(xml.matches("").count(), 1, "{xml}"); + let at = |tag: &str| xml.find(tag).unwrap_or_else(|| panic!("{tag} missing: {xml}")); + assert!(at(""#; + assert!(xml.contains(zoom), "{xml}"); + for (element, expected) in [ + ("", false), + ("", true), + (r#""#, true), + (r#""#, true), + (r#""#, true), + (r#""#, false), + (r#""#, false), + (r#""#, false), + ] { + let file = replace_part(&plain, "word/settings.xml", xml.replace(zoom, &format!("{zoom}{element}")).as_bytes()); + assert_eq!(craft_docx::read(&file).unwrap().settings.gutter_at_top, expected, "{element:?}"); + } +} + +/// A settings part from another writer (proofing state, rsids, document variables…) is merged on +/// save, not regenerated: `w:gutterAtTop` stays once where it was, goes when the setting is turned +/// off, and is added at its schema position (after `w:mirrorMargins`, before `w:proofState`) when +/// the file had none. +#[test] +fn gutter_at_top_is_merged_into_a_foreign_settings_part() { + let plain = craft_docx::write(&Document::new()).unwrap(); + let xml = settings_part(&plain); + let zoom = r#""#; + let foreign = |element: &str| { + let extra = format!( + r#"{zoom}{element}"# + ); + replace_part(&plain, "word/settings.xml", xml.replace(zoom, &extra).as_bytes()) + }; + + let with = craft_docx::read(&foreign("")).unwrap(); + assert!(with.settings.gutter_at_top && with.settings.mirror_margins); + let saved = settings_part(&craft_docx::write(&with).unwrap()); + assert_eq!(saved.matches("").count(), 1, "{saved}"); + assert!(saved.contains("").count(), 1, "{saved}"); + let at = |tag: &str| saved.find(tag).unwrap_or_else(|| panic!("{tag} missing: {saved}")); + assert!(at(" Date: Mon, 16 Nov 2026 00:17:26 +0100 Subject: [PATCH 026/209] DOCX: make the gutter-at-top tests check the order and False/FALSE The merge unit test compared Options, and its generated part had no w:mirrorMargins, so the order check proved nothing; put the element in and look the positions up. Add the False and FALSE spellings to the value test. Co-Authored-By: Claude Sonnet 5.5 --- crates/docx/src/write/settings_merge.rs | 8 +++++--- crates/docx/tests/it/roundtrip.rs | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/docx/src/write/settings_merge.rs b/crates/docx/src/write/settings_merge.rs index 01b7d27c6..a72046e83 100644 --- a/crates/docx/src/write/settings_merge.rs +++ b/crates/docx/src/write/settings_merge.rs @@ -390,16 +390,18 @@ mod tests { #[test] fn gutter_at_top_comes_from_the_generated_part() { - let orig = r#""#; + let orig = + r#""#; assert!(has_unowned(orig.as_bytes())); // Turned off in the model: the original element goes, the foreign one stays. let out = String::from_utf8(merge(orig.as_bytes(), GEN.as_bytes()).unwrap()).unwrap(); assert!(!out.contains("gutterAtTop") && out.contains("").count(), 1, "{out}"); - assert!(out.find("") < out.find("") && out.find("") < out.find("") < at("") && at("") < at(""#)); } diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 3ba2f326a..4bb6cab35 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -2981,6 +2981,8 @@ fn gutter_at_top_reads_every_value_spelling() { (r#""#, false), (r#""#, false), (r#""#, false), + (r#""#, false), + (r#""#, false), ] { let file = replace_part(&plain, "word/settings.xml", xml.replace(zoom, &format!("{zoom}{element}")).as_bytes()); assert_eq!(craft_docx::read(&file).unwrap().settings.gutter_at_top, expected, "{element:?}"); @@ -2997,9 +2999,8 @@ fn gutter_at_top_is_merged_into_a_foreign_settings_part() { let xml = settings_part(&plain); let zoom = r#""#; let foreign = |element: &str| { - let extra = format!( - r#"{zoom}{element}"# - ); + let extra = + format!(r#"{zoom}{element}"#); replace_part(&plain, "word/settings.xml", xml.replace(zoom, &extra).as_bytes()) }; From 39e1d41af7368926a16c979ec2bc61fefea02c64 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 01:34:20 +0100 Subject: [PATCH 027/209] DDS: refuse premultiplied-alpha DX10 textures The DX10 header's alpha mode (miscFlags2, low three bits) was never read, so RGBA, BGRA, BC2 and BC3 files marked premultiplied were read as straight alpha. They are refused as Unsupported, like the legacy DXT2/DXT4 FourCCs. Alpha modes 0, 1, 3 and 4 and the formats without alpha are unchanged. --- crates/dds/README.md | 2 +- crates/dds/src/lib.rs | 5 +++++ crates/image-codecs/README.md | 2 +- crates/image-codecs/tests/it/dds.rs | 29 +++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/dds/README.md b/crates/dds/README.md index 720ec9ba7..e3c3f4956 100644 --- a/crates/dds/README.md +++ b/crates/dds/README.md @@ -16,7 +16,7 @@ craft-dds = { git = "https://github.com/storytold/craft-libs", rev = "" | Headers | legacy and DX10 | legacy; DX10 for BC4 and BC5 | | Mip levels | the top level only | optional full chain, box-filtered | -Cubemaps, texture arrays, volume textures, premultiplied DXT2/DXT4, signed (SNORM) BC4/BC5 and +Cubemaps, texture arrays, volume textures, premultiplied-alpha textures (DXT2/DXT4, or DX10 alpha mode 2), signed (SNORM) BC4/BC5 and the other DXGI formats (BC6H, BC7, float) are refused with `DdsError::Unsupported` rather than read in part. diff --git a/crates/dds/src/lib.rs b/crates/dds/src/lib.rs index 5c088052a..2d026a660 100644 --- a/crates/dds/src/lib.rs +++ b/crates/dds/src/lib.rs @@ -359,6 +359,11 @@ fn dx10_encoding(b: &[u8]) -> Result { if array > 1 { return Err(unsupported(format!("texture arrays are not supported ({array} textures)"))); } + // miscFlags2 bits 0-2 are the alpha mode; 2 is premultiplied. Only the formats with a real + // alpha channel are affected, and the legacy path refuses their DXT2/DXT4 equivalents too. + if le32(b, HEADER + 16)? & 0x7 == 2 && matches!(dxgi, 27..=29 | 73..=78 | 87 | 90 | 91) { + return Err(unsupported(format!("premultiplied-alpha DX10 textures are not supported (DXGI format {dxgi})"))); + } const RGBA: [u32; 4] = [0xff, 0xff00, 0xff_0000, 0xff00_0000]; const BGRA: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0xff00_0000]; const BGRX: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0]; diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index 1bdf1418d..11ed6b40c 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -100,7 +100,7 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them map's Z is not reconstructed). ATI2 is read as Microsoft reads it, red in the first block. Uncompressed row padding follows the header's `DDSD_PITCH` stride; without that flag, rows are tightly packed. Everything opens as U8; the other mip levels are ignored. Cubemaps, texture - arrays, volume textures, premultiplied DXT2/DXT4, signed (SNORM) BC4/BC5 and other formats + arrays, volume textures, premultiplied-alpha textures (DXT2/DXT4, or DX10 alpha mode 2), signed (SNORM) BC4/BC5 and other formats (BC6H, BC7, float) are refused with `CodecError::Unsupported`. Writing (`EncodeOptions::dds`) makes a 2D texture: uncompressed by default (BGRA, BGRX, or luminance with or without alpha, so it reads back exactly), BC1/BC2/BC3 (from RGBA) with the legacy header, or BC4 (from gray, for diff --git a/crates/image-codecs/tests/it/dds.rs b/crates/image-codecs/tests/it/dds.rs index ad927019a..d174255b2 100644 --- a/crates/image-codecs/tests/it/dds.rs +++ b/crates/image-codecs/tests/it/dds.rs @@ -327,6 +327,35 @@ const BC4_200: [u8; 8] = [0xc8, 0x32, 0, 0, 0, 0, 0, 0]; /// A value block of 250 everywhere (six-value mode, a0 = 10 <= a1 = 250, every index 1). const VALUES_250: [u8; 8] = [0x0a, 0xfa, 0x49, 0x92, 0x24, 0x49, 0x92, 0x24]; +/// A DX10 file of one 1x1 level of `dxgi` with DDS_ALPHA_MODE `alpha_mode` (miscFlags2) and zeroed pixel data. +fn dx10_alpha_mode(dxgi: u32, alpha_mode: u32) -> Vec { + let mut b = dx10(1, 1, dxgi, 3, 0, 1); + let n = b.len(); + b[n - 4..].copy_from_slice(&alpha_mode.to_le_bytes()); + b.extend([0u8; 16]); + b +} + +#[test] +fn refuses_premultiplied_alpha_dx10_textures() { + // RGBA, BGRA (typeless, UNORM, sRGB) and BC2/BC3: alpha mode 2 is premultiplied, like DXT2/DXT4. + for dxgi in [27, 28, 29, 87, 90, 91, 73, 74, 75, 76, 77, 78] { + match decode_as(Format::Dds, &dx10_alpha_mode(dxgi, 2)) { + Err(CodecError::Unsupported { reason, .. }) => assert!(reason.contains("premultiplied"), "{dxgi}: {reason}"), + other => panic!("DXGI {dxgi}: {:?}", other.map(|_| ())), + } + } + // Straight, unknown, opaque and custom alpha modes, and formats without alpha, still decode. + for dxgi in [28, 87, 74, 77] { + for mode in [0, 1, 3, 4] { + assert!(decode_as(Format::Dds, &dx10_alpha_mode(dxgi, mode)).is_ok(), "{dxgi} mode {mode}"); + } + } + for dxgi in [88, 71, 80, 83] { + assert!(decode_as(Format::Dds, &dx10_alpha_mode(dxgi, 2)).is_ok(), "{dxgi}"); + } +} + #[test] fn decodes_bc4_as_gray_from_every_name() { // 5×3: two blocks across, one down; the second block's texels outside the image are dropped. From e76bf4bad443c5b74773175538ef7985b0356f53 Mon Sep 17 00:00:00 2001 From: vangel76 <271991477+vangel76@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:04:08 +0100 Subject: [PATCH 028/209] DOCX: an Office 2016 chart with a text-only fallback keeps its size as a placeholder read_chartex_alternate built the chart object only from a picture fallback. With a fallback that holds only text (for example "Chart not available."), it returned None and the reader used the fallback words, so the chart's 432 x 252 pt frame was never reserved and the pages collapsed. Now such a chart becomes a Chart graphic of the choice's wp:extent, drawn as the same kind of placeholder unsupported classic charts get (frame, title, chart type), with the choice and fallback kept so saving writes them back unchanged. A fallback that holds runs or paragraphs is still read as before, because the writer puts the fallback inside the object's own run. The frame reading at the top of read_drawing moves into drawing_frame so both paths share it. Part of storytold/wordcraft#6820 Co-Authored-By: Claude Sonnet 5.5 --- crates/docx/src/read/chart.rs | 50 ++++++++++++++++- crates/docx/src/read/chart/tests.rs | 24 ++++++++ crates/docx/src/read/story/drawing.rs | 79 ++++++++++++++++++++------- crates/docx/tests/it/fixtures.rs | 59 ++++++++++++++++++++ 4 files changed, 192 insertions(+), 20 deletions(-) diff --git a/crates/docx/src/read/chart.rs b/crates/docx/src/read/chart.rs index 5b02603d6..42c7a311c 100644 --- a/crates/docx/src/read/chart.rs +++ b/crates/docx/src/read/chart.rs @@ -37,7 +37,7 @@ use parse::{axis, category_text, labels, numbers}; use parse::{read_series, text_size, title_of}; #[cfg(test)] use plot::{Scale, narrow, nice_scale}; -use plot::{backdrop, cartesian}; +use plot::{backdrop, cartesian, label}; use polar::{pie, radar}; /// Most series read from one chart. @@ -141,6 +141,54 @@ fn placeholder(out: &mut Vec, space: &El, theme: &[Rgb], w: f32, h: text(out, line, &clip_text(&msg, area.w, size), size, TextAlign::Center, false); } +/// Items for an Office 2016 chart (`cx:chartSpace`: waterfall, funnel, treemap…), sized `w` x `h` +/// points: none of these types is drawn yet, so this is the same placeholder an unsupported +/// classic chart gets (frame, title, the type it is), at the size the chart takes in Word. +/// `space` is `None` when the part can't be read: the frame and "Chart" are still drawn. +pub(crate) fn chartex_items(space: Option<&El>, w: f32, h: f32) -> Vec { + let mut out = Vec::new(); + if !(w.is_finite() && h.is_finite() && w > 0.0 && h > 0.0) { + return out; + } + out.push(GraphicItem::Shape { rect: [0.0, 0.0, w, h], kind: ShapeKind::Rectangle, fill: Some(Rgb::WHITE), stroke: Some(AXIS), stroke_width: 0.75 }); + let mut area = Rect { x: PAD, y: PAD, w: (w - 2.0 * PAD).max(1.0), h: (h - 2.0 * PAD).max(1.0) }; + let chart = space.and_then(|s| s.child("cx:chart")); + let title = chart.and_then(|c| c.child("cx:title")).and_then(|t| { + let mut s = String::new(); + t.find_all("a:t").for_each(|r| s.push_str(&r.text())); + if s.is_empty() { + s = t.find("cx:v").map(|v| v.text().into_owned()).unwrap_or_default(); + } + let s = label(s.trim()); + (!s.is_empty()).then_some(s) + }); + if let Some(t) = title { + area = title_block(&mut out, &t, area); + } + let kind = chart.and_then(|c| c.find("cx:series")).and_then(|s| s.attr("layoutId")).unwrap_or(""); + let size = FONT; + let lh = size * LINE; + let msg = format!("{} (not supported yet)", chartex_name(kind)); + let line = Rect { x: area.x, y: area.y + ((area.h - lh) / 2.0).max(0.0), w: area.w, h: lh }; + text(&mut out, line, &clip_text(&msg, area.w, size), size, TextAlign::Center, false); + out +} + +/// What a placeholder calls an Office 2016 chart type (`cx:series` `layoutId`). +fn chartex_name(layout: &str) -> &'static str { + match layout { + "waterfall" => "Waterfall chart", + "funnel" => "Funnel chart", + "treemap" => "Treemap chart", + "sunburst" => "Sunburst chart", + "boxWhisker" => "Box and whisker chart", + "clusteredColumn" => "Histogram chart", + "paretoLine" => "Pareto chart", + "regionMap" => "Map chart", + _ => "Chart", + } +} + /// What a placeholder calls a chart type element WordCraft doesn't draw. fn unsupported_name(name: &str) -> &'static str { match name { diff --git a/crates/docx/src/read/chart/tests.rs b/crates/docx/src/read/chart/tests.rs index ae80c028e..d785d4400 100644 --- a/crates/docx/src/read/chart/tests.rs +++ b/crates/docx/src/read/chart/tests.rs @@ -899,3 +899,27 @@ fn source_linked_series_labels_do_not_inherit_the_chart_format() { let inherited = run(r#""#); assert!(inherited.contains(&"320%".to_string()), "{inherited:?}"); } + +#[test] +fn office_2016_chart_draws_a_placeholder_of_its_size() { + let cx = |series: &str, title: &str| { + let x = format!( + r#"{title}{series}"# + ); + crate::xml::parse(x.as_bytes()).unwrap() + }; + let title = r#"Quarterly steps"#; + let rich = r#"Sales funnel"#; + let waterfall = chartex_items(Some(&cx(r#""#, title)), 432.0, 252.0); + assert_eq!(texts(&waterfall), ["Quarterly steps", "Waterfall chart (not supported yet)"]); + let framed = waterfall.iter().any(|it| matches!(it, GraphicItem::Shape { rect, stroke: Some(_), .. } if *rect == [0.0, 0.0, 432.0, 252.0])); + assert!(framed, "the frame has the chart's size: {waterfall:?}"); + let funnel = chartex_items(Some(&cx(r#""#, rich)), 300.0, 200.0); + assert_eq!(texts(&funnel), ["Sales funnel", "Funnel chart (not supported yet)"]); + let other = chartex_items(Some(&cx(r#""#, "")), 300.0, 200.0); + assert_eq!(texts(&other), ["Chart (not supported yet)"]); + // An unreadable part still reserves and frames the chart's place. + let missing = chartex_items(None, 300.0, 200.0); + assert_eq!(texts(&missing), ["Chart (not supported yet)"]); + assert!(chartex_items(None, 0.0, 200.0).is_empty() && chartex_items(None, f32::NAN, 200.0).is_empty()); +} diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..ab820904d 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -12,23 +12,7 @@ impl Reader<'_> { /// text and drawing ids are carried into the object. Returns `None` when the frame or payload /// cannot be read. Ink written by WordCraft keeps its tool's caps rather than an outline cap. pub(super) fn read_drawing(&mut self, sc: &mut StoryCtx, d: &El, rels: &Rels) -> Option { - let (c, anchored) = match d.child("wp:inline") { - Some(c) => (c, false), - None => (d.child("wp:anchor")?, true), - }; - let ext = c.child("wp:extent"); - let dim = |n: &str| ext.and_then(|e| e.attr(n)).and_then(|v| measure(v, 12_700.0)).unwrap_or(0.0).clamp(0.0, crate::units::MAX_LEN_PT); - let (w, h) = (dim("cx"), dim("cy")); - let (alt, meta) = nv_alt(c.child("wp:docPr")); - let mut float = if anchored { anchor_float(c) } else { Float::default() }; - float.alt = meta; - float.hidden = c.child("wp:docPr").is_some_and(|p| on_off_attr(p, "hidden")); - if let Some(e) = c.child("wp:effectExtent") { - for (slot, n) in float.effect.iter_mut().zip(["l", "t", "r", "b"]) { - // Negative for a rotated object narrower than its frame; settled below. - *slot = e.attr(n).and_then(|v| measure(v, 12_700.0)).unwrap_or(0.0).clamp(-1584.0, 1584.0); - } - } + let Frame { c, anchored, w, h, alt, mut float } = drawing_frame(d)?; let gd = c.child("a:graphic").and_then(|g| g.child("a:graphicData"))?; // Checked by URI before the blip search: a diagram's data can hold a blip further down. let uri = gd.attr("uri").unwrap_or(""); @@ -188,14 +172,39 @@ impl Reader<'_> { let fallback = ac.child("mc:Fallback")?; let kept = El { name: "mc:AlternateContent".into(), attrs: Vec::new(), kids: vec![Node::El(choice.clone()), Node::El(fallback.clone())] }; let source = self.keep_embedded(&kept, rels, &[])?; - let Some(InlineObject::Image { media, w, h, alt, float, .. }) = self.read_drawing(sc, fallback.child("w:drawing")?, rels) else { - return None; + let picture = fallback.child("w:drawing").and_then(|d| self.read_drawing(sc, d, rels)); + let Some(InlineObject::Image { media, w, h, alt, float, .. }) = picture else { + return self.chartex_placeholder(choice, fallback, rels, source); }; let items = vec![GraphicItem::Image { rect: [0.0, 0.0, w, h], media }]; let graphic = Arc::new(Graphic { kind: GraphicKind::Chart, items, w, h, source: Some(source), ..Default::default() }); Some(InlineObject::Graphic { w, h, alt, float, graphic }) } + /// An Office 2016 chart whose fallback holds no picture (only text, say "Chart not + /// available"): a placeholder of the chart's own size (the `wp:extent` of its `mc:Choice`), + /// so the page breaks where they do with the chart drawn. The fallback is written back as it + /// was. `None` when the fallback has a picture or another drawing (read as before), or + /// holds runs or paragraphs, which a writer can't put inside the run that holds the object. + fn chartex_placeholder(&mut self, choice: &El, fallback: &El, rels: &Rels, source: Arc) -> Option { + const RUN_CONTENT: [&str; 6] = ["w:t", "w:tab", "w:br", "w:cr", "w:noBreakHyphen", "w:softHyphen"]; + if fallback.find("a:graphic").is_some() || !fallback.els().all(|e| RUN_CONTENT.contains(&e.name.as_str())) { + return None; + } + let Frame { c, w, h, alt, mut float, .. } = drawing_frame(choice.child("w:drawing")?)?; + let part = c + .child("a:graphic") + .and_then(|g| g.child("a:graphicData")) + .and_then(|gd| gd.child("cx:chart")) + .and_then(|cx| cx.attr("r:id")) + .and_then(|id| crate::read::part_of(rels, id, "chartEx")); + let space = part.and_then(|p| self.graphic_part(&p)); + let items = crate::read::chart::chartex_items(space.as_deref(), w, h); + float.effect = float.effect.map(|e| e.max(0.0)); + let graphic = Arc::new(Graphic { kind: GraphicKind::Chart, items, w, h, source: Some(source), ..Default::default() }); + Some(InlineObject::Graphic { w, h, alt, float, graphic }) + } + /// A group or canvas member that is native ink (`content`, a `w14:contentPart`, inside /// `alternate` when it has a picture fallback): what to write back (the whole alternate when /// it can be kept, else the ink alone) and its fallback picture (a `Document::media` key). @@ -598,6 +607,38 @@ impl Reader<'_> { } } +/// The frame of a `w:drawing`: its `wp:inline` or `wp:anchor` element, the size in points, +/// the alternative text, and the placement with its effect extent (rotation not yet settled). +struct Frame<'a> { + c: &'a El, + anchored: bool, + w: f32, + h: f32, + alt: String, + float: Float, +} + +fn drawing_frame(d: &El) -> Option> { + let (c, anchored) = match d.child("wp:inline") { + Some(c) => (c, false), + None => (d.child("wp:anchor")?, true), + }; + let ext = c.child("wp:extent"); + let dim = |n: &str| ext.and_then(|e| e.attr(n)).and_then(|v| measure(v, 12_700.0)).unwrap_or(0.0).clamp(0.0, crate::units::MAX_LEN_PT); + let (w, h) = (dim("cx"), dim("cy")); + let (alt, meta) = nv_alt(c.child("wp:docPr")); + let mut float = if anchored { anchor_float(c) } else { Float::default() }; + float.alt = meta; + float.hidden = c.child("wp:docPr").is_some_and(|p| on_off_attr(p, "hidden")); + if let Some(e) = c.child("wp:effectExtent") { + for (slot, n) in float.effect.iter_mut().zip(["l", "t", "r", "b"]) { + // Negative for a rotated object narrower than its frame; settled below. + *slot = e.attr(n).and_then(|v| measure(v, 12_700.0)).unwrap_or(0.0).clamp(-1584.0, 1584.0); + } + } + Some(Frame { c, anchored, w, h, alt, float }) +} + /// The `mc:Choice` holding an Office 2016 chart (`cx:chart`) in an `mc:AlternateContent`. fn chartex_choice(ac: &El) -> Option<&El> { ac.children("mc:Choice").find(|c| c.child("w:drawing").and_then(|d| d.find("a:graphicData")).and_then(|g| g.attr("uri")) == Some(CHARTEX_URI)) diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 348d6239e..295b788b9 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -1861,6 +1861,65 @@ fn a_link_on_an_office_2016_charts_drawing_is_kept() { } } +/// A package with one invented waterfall chart (432 x 252 pt) whose `mc:Fallback` is `fallback`. +fn chartex_text_fallback_docx(fallback: &str) -> Vec { + let chart = r#"Steps"#; + let frame = r#""#; + let body = format!( + r#"Before{frame}{fallback}"# + ); + let doc = format!(r#"{body}"#); + let doc_rels = r#""#; + let types = format!( + r#""# + ); + zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT_RELS.as_bytes()), + ("word/document.xml", doc.as_bytes()), + ("word/_rels/document.xml.rels", doc_rels.as_bytes()), + ("word/charts/chartEx1.xml", chart.as_bytes()), + ]) +} + +/// #6820: an Office 2016 chart whose fallback is only text keeps its 432 x 252 pt place (as a +/// placeholder) instead of becoming the fallback words, and is written back unchanged. +#[test] +fn office_2016_chart_with_a_text_fallback_keeps_its_size_through_saves() { + let mut d = craft_docx::read(&chartex_text_fallback_docx("Chart not available.")).unwrap(); + for _ in 0..2 { + let g = graphics(&d); + let [InlineObject::Graphic { w, h, graphic, .. }] = g.as_slice() else { panic!("the chart is an object: {g:?}") }; + assert_eq!((*w, *h), (432.0, 252.0)); + assert_eq!((graphic.w, graphic.h), (432.0, 252.0)); + assert!( + graphic.items.iter().any(|i| matches!(i, craft_wordmodel::graphic::GraphicItem::Text { text, .. } if text.starts_with("Waterfall chart"))), + "{:?}", + graphic.items + ); + assert!(!paras(&d).iter().any(|p| p.text.contains("not available")), "the fallback words are not body text"); + let files = unzip(&craft_docx::write(&d).unwrap()); + let body = text(&files, "word/document.xml"); + assert_eq!(attr_values(&body, "wp:extent", "cx"), ["5486400"], "the chart's frame; the text fallback has none: {body}"); + assert!(body.contains("Chart not available."), "the fallback is written back as it was: {body}"); + let [cx_id] = attr_values(&body, "cx:chart", "r:id").try_into().unwrap_or_else(|_| panic!("chart lost: {body}")); + let (kind, target) = rel(&text(&files, "word/_rels/document.xml.rels"), &cx_id); + assert!(kind.ends_with("/chartEx"), "{kind}"); + assert!(text(&files, &part_at("word", &target)).contains(r#"layoutId="waterfall""#)); + let entries: Vec<_> = files.iter().map(|(name, bytes)| (name.as_str(), bytes.as_slice())).collect(); + d = craft_docx::read(&zip(&entries)).unwrap(); + } +} + +/// A fallback that holds a run (not run content) can't be written back inside the object's own +/// run, so it is read as before: its text, no chart object. +#[test] +fn office_2016_chart_with_a_run_fallback_is_read_as_before() { + let d = craft_docx::read(&chartex_text_fallback_docx("Chart not available.")).unwrap(); + assert!(graphics(&d).is_empty()); + assert!(paras(&d).iter().any(|p| p.text.contains("Chart not available."))); +} + #[test] fn resized_office_2016_chart_resizes_its_fallback_too() { let mut d = craft_docx::read(&chartex_docx("cx1", "http://schemas.microsoft.com/office/drawing/2015/9/8/chartex")).unwrap(); From 5e3632533f8db0763cfcd5720b2462f82c7744ea Mon Sep 17 00:00:00 2001 From: vangel76 <271991477+vangel76@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:18:25 +0100 Subject: [PATCH 029/209] DOCX: read the placeholder chart title from its text, leave the frame unfilled Take the title of an Office 2016 chart from its rich text (all runs) or its text data, and draw the placeholder frame without a fill so it does not become a white box on a coloured page. The run-fallback test is a guard and says so. Part of storytold/wordcraft#6820 Co-Authored-By: Claude Sonnet 5.5 --- crates/docx/src/read/chart.rs | 9 +++------ crates/docx/tests/it/fixtures.rs | 14 ++++++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/docx/src/read/chart.rs b/crates/docx/src/read/chart.rs index 42c7a311c..4b7696eee 100644 --- a/crates/docx/src/read/chart.rs +++ b/crates/docx/src/read/chart.rs @@ -150,15 +150,12 @@ pub(crate) fn chartex_items(space: Option<&El>, w: f32, h: f32) -> Vec 0.0 && h > 0.0) { return out; } - out.push(GraphicItem::Shape { rect: [0.0, 0.0, w, h], kind: ShapeKind::Rectangle, fill: Some(Rgb::WHITE), stroke: Some(AXIS), stroke_width: 0.75 }); + out.push(GraphicItem::Shape { rect: [0.0, 0.0, w, h], kind: ShapeKind::Rectangle, fill: None, stroke: Some(AXIS), stroke_width: 0.75 }); let mut area = Rect { x: PAD, y: PAD, w: (w - 2.0 * PAD).max(1.0), h: (h - 2.0 * PAD).max(1.0) }; let chart = space.and_then(|s| s.child("cx:chart")); let title = chart.and_then(|c| c.child("cx:title")).and_then(|t| { - let mut s = String::new(); - t.find_all("a:t").for_each(|r| s.push_str(&r.text())); - if s.is_empty() { - s = t.find("cx:v").map(|v| v.text().into_owned()).unwrap_or_default(); - } + // Rich text keeps its runs together ("Sales " + "funnel"); a text-data title is one value. + let s = t.find("cx:rich").or_else(|| t.find("cx:v")).map(|e| e.deep_text()).unwrap_or_default(); let s = label(s.trim()); (!s.is_empty()).then_some(s) }); diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 295b788b9..5ddfb151f 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -1893,7 +1893,10 @@ fn office_2016_chart_with_a_text_fallback_keeps_its_size_through_saves() { assert_eq!((*w, *h), (432.0, 252.0)); assert_eq!((graphic.w, graphic.h), (432.0, 252.0)); assert!( - graphic.items.iter().any(|i| matches!(i, craft_wordmodel::graphic::GraphicItem::Text { text, .. } if text.starts_with("Waterfall chart"))), + graphic + .items + .iter() + .any(|i| matches!(i, craft_wordmodel::graphic::GraphicItem::Text { text, .. } if text.starts_with("Waterfall chart"))), "{:?}", graphic.items ); @@ -1911,13 +1914,12 @@ fn office_2016_chart_with_a_text_fallback_keeps_its_size_through_saves() { } } -/// A fallback that holds a run (not run content) can't be written back inside the object's own -/// run, so it is read as before: its text, no chart object. +/// Guard, not fix proof (it passes before the change too): a fallback that holds a run can't be +/// written back inside the object's own run, so it is not turned into a chart placeholder. #[test] -fn office_2016_chart_with_a_run_fallback_is_read_as_before() { +fn office_2016_chart_with_a_run_fallback_is_not_made_a_placeholder() { let d = craft_docx::read(&chartex_text_fallback_docx("Chart not available.")).unwrap(); - assert!(graphics(&d).is_empty()); - assert!(paras(&d).iter().any(|p| p.text.contains("Chart not available."))); + assert!(graphics(&d).is_empty(), "{:?}", graphics(&d)); } #[test] From 2e5e69091a1d41217f93f9f18678351c3ff8653f Mon Sep 17 00:00:00 2001 From: batistalucasv Date: Mon, 16 Nov 2026 00:41:54 -0300 Subject: [PATCH 030/209] craft-odf: preserve page background bitmap fill and legacy fallback (#1784) - Preserves page background images and bitmap fills (`draw:fill="bitmap"`) alongside legacy `` fallbacks on ODT export, preventing orphaned duplicate backgrounds and resolving naming collisions with header pictures. - Reuses kept picture files or fill images with identical bytes on export instead of duplicating background files and definitions on every save. - Allocates distinct fill names for replaced page pictures and maps colliding names in linear time. - Falls back to legacy `` when modern bitmap fill is missing or contains non-image bytes. - Adds comprehensive round-trip tests in `crates/odf/src/text/tests_more/page_setup.rs`. --- crates/odf/src/text/import.rs | 101 ++- crates/odf/src/text/kept.rs | 2 +- crates/odf/src/text/package.rs | 138 +++- crates/odf/src/text/tests_more/page_setup.rs | 659 +++++++++++++++++++ 4 files changed, 859 insertions(+), 41 deletions(-) diff --git a/crates/odf/src/text/import.rs b/crates/odf/src/text/import.rs index 486e67a26..fcd98b164 100644 --- a/crates/odf/src/text/import.rs +++ b/crates/odf/src/text/import.rs @@ -823,8 +823,8 @@ struct Parsed { kept: super::parts::Kept, /// Hidden form controls, kept for the next save. hidden_controls: Vec, - /// The page background picture's bytes and fill. - page_image: Option<(Vec, &'static str)>, + /// The page background picture's bytes, fill, and optional fill image name. + page_image: Option<(Vec, &'static str, Option)>, } /// Hyphenation is a paragraph's in ODF (`fo:hyphenate`) and the document's in WordCraft (as in @@ -911,8 +911,19 @@ fn parse_tracked(bytes: &[u8], paper: (f32, f32)) -> Result { // A frame may point at any entry, as often as it likes: `Pictures` keeps that bounded. pictures.get(&mut z, path) }; - // The page background picture's bytes. - let page_image = pages.doc.page_image.clone().and_then(|(href, fill)| fetch(&href).map(|b| (b, fill))); + // The page background picture's bytes: modern draw:fill="bitmap" preferred when valid, + // falling back to legacy style:background-image if modern is missing or not an image. + let page_image = { + let modern = pages.doc.page_image.clone().and_then(|(href, fill, name)| { + let bytes = fetch(&href)?; + if model::sniff_image(&bytes).is_some() { Some((bytes, fill, name)) } else { None } + }); + modern.or_else(|| { + let (href, fill) = pages.doc.legacy_image.clone()?; + let bytes = fetch(&href)?; + if model::sniff_image(&bytes).is_some() { Some((bytes, fill, None)) } else { None } + }) + }; let objects = (entries.as_slice(), &manifest); // One `text:s` budget for the headers and footers and then the body. let mut spaces_left = MAX_EXTRA_SPACES; @@ -1096,11 +1107,15 @@ pub fn import_on(bytes: &[u8], page: &SectionProps) -> Result doc.settings.odf = Some(craft_wordmodel::OdfLayout { tabs_from_indent }); doc.settings.sum_paragraph_spacing = sum_spacing; // The page background picture (Design › Page Color › Fill Effects › Picture). - if let Some((bytes, fill)) = page_image + if let Some((bytes, fill, name)) = page_image && let Some(ext) = model::sniff_image(&bytes) { let media = doc.add_media(bytes, ext); - doc.settings.page_picture = Some(craft_wordmodel::PagePicture { media, fill_type: fill.to_string(), title: String::new() }); + doc.settings.page_picture = Some(craft_wordmodel::PagePicture { + media, + fill_type: fill.to_string(), + title: name.map(|n| n.chars().take(255).collect()).unwrap_or_default(), + }); } if let Some((fmt, props)) = page_doc.notes[0] { (doc.settings.footnote_format, doc.settings.footnote_pr) = (fmt, props); @@ -1172,9 +1187,11 @@ pub(super) struct Pages { #[derive(Clone, Default)] pub(super) struct PageDoc { pub(super) page_color: Option, - /// The first page style's background picture (§17.3 `style:background-image`): its package - /// path and WordCraft's fill for its repeat (tiled, stretched, or centred at its size). - pub(super) page_image: Option<(String, &'static str)>, + /// The first page style's background picture (§17.3 `style:background-image` or §16.42 + /// `draw:fill-image` with `draw:fill="bitmap"`): package path, fill, and optional fill name. + pub(super) page_image: Option<(String, &'static str, Option)>, + /// Legacy fallback background picture from §17.3 `style:background-image`. + pub(super) legacy_image: Option<(String, &'static str)>, /// Footnotes', then endnotes', number format and options. pub(super) notes: [Option<(craft_wordmodel::section::NumFormat, craft_wordmodel::section::NoteProps)>; 2], } @@ -1248,12 +1265,43 @@ struct Layout { attrs: Attrs, /// Its background picture: package path and fill (see [`PageDoc::page_image`]). image: Option<(String, &'static str)>, + /// Its bitmap fill image name (`draw:fill-image-name`). + image_name: Option, + /// Its legacy background picture: package path and fill (§17.3 `style:background-image`). + legacy_image: Option<(String, &'static str)>, columns: Option, column_specs: Vec, header: Band, footer: Band, } +/// The background fill picture and its fill-image name from `page-layout-properties` +/// attributes `a` when modern `draw:fill="bitmap"` is declared (§16.42). +fn page_layout_fill(a: &super::xml::Attrs, fill_images: &HashMap) -> Option<((String, &'static str), String)> { + if get(a, "draw:fill") != Some("bitmap") { + return None; + } + let name = get(a, "draw:fill-image-name")?; + let href = fill_images.get(name)?; + let fill = match get(a, "style:repeat").map(str::trim) { + Some("no-repeat") => "center", + Some("stretch") => "frame", + _ => "tile", + }; + Some(((href.clone(), fill), name.to_string())) +} + +/// The page background picture and fill from legacy `background-image` attributes `a` (§17.3). +fn legacy_background_fill(a: &super::xml::Attrs) -> Option<(String, &'static str)> { + let href = get(a, "xlink:href").map(str::trim).filter(|h| !h.is_empty() && h.len() <= 1024)?; + let fill = match get(a, "style:repeat").map(str::trim) { + Some("no-repeat") => "center", + Some("stretch") => "frame", + _ => "tile", + }; + Some((href.trim_start_matches("./").to_string(), fill)) +} + /// A page stating nothing, `paper` (width, height) in size: no margins (§20.206–20.209 refer to /// XSL's margins, whose initial value is 0; whether it comes with the reference the standard /// doesn't say). @@ -1271,6 +1319,7 @@ fn unstated_page(paper: (f32, f32)) -> SectionProps { fn read_pages(xml: &[u8], paper: (f32, f32)) -> Pages { let mut layouts: HashMap = HashMap::new(); + let mut fill_images: HashMap = HashMap::new(); let mut cur: Option = None; // Inside a layout's footer style (else its header style). let mut in_footer = false; @@ -1289,6 +1338,12 @@ fn read_pages(xml: &[u8], paper: (f32, f32)) -> Pages { } } "linenumbering-configuration" if q.starts_with("text:") => lines = line_numbering(&a), + // Modern ODF 1.2/1.3 fill image definitions (§16.42 `draw:fill-image` in `office:styles`). + "fill-image" if q.starts_with("draw:") => { + if let (Some(name), Some(href)) = (get(&a, "draw:name"), get(&a, "xlink:href")) { + fill_images.insert(name.to_string(), href.trim_start_matches("./").to_string()); + } + } "page-layout" if q.starts_with("style:") => { cur = get(&a, "style:name").map(String::from); // A layout stating nothing is still one: the defaults' page. @@ -1300,16 +1355,14 @@ fn read_pages(xml: &[u8], paper: (f32, f32)) -> Pages { } // A page background picture (§17.3, in `style:page-layout-properties`): §20.249 // `style:repeat` (repeat is the default) as WordCraft's tile, stretch, or centre. + // Preserved as a fallback when modern `draw:fill="bitmap"` is missing or non-image. "background-image" if q.starts_with("style:") => { - if let (Some(l), Some(href)) = - (cur.as_ref().and_then(|n| layouts.get_mut(n)), get(&a, "xlink:href").map(str::trim).filter(|h| !h.is_empty() && h.len() <= 1024)) - { - let fill = match get(&a, "style:repeat").map(str::trim) { - Some("no-repeat") => "center", - Some("stretch") => "frame", - _ => "tile", - }; - l.image = Some((href.trim_start_matches("./").to_string(), fill)); + if let Some(l) = cur.as_ref().and_then(|n| layouts.get_mut(n)) { + let leg = legacy_background_fill(&a); + if l.image.is_none() { + l.image = leg.clone(); + } + l.legacy_image = leg; } } // A band whose height isn't stated is LibreOffice's default, 0.5 cm (measured; see @@ -1327,6 +1380,10 @@ fn read_pages(xml: &[u8], paper: (f32, f32)) -> Pages { } "page-layout-properties" => { if let Some(l) = cur.as_ref().and_then(|n| layouts.get_mut(n)) { + if let Some((img, name)) = page_layout_fill(&a, &fill_images) { + l.image = Some(img); + l.image_name = Some(name); + } l.attrs = a; } } @@ -1380,14 +1437,18 @@ fn read_pages(xml: &[u8], paper: (f32, f32)) -> Pages { }); let mut pages = Pages { doc, next: nexts, ..Default::default() }; for (name, layout, header, footer) in masters { - let Some(Layout { attrs: a, columns: cols, column_specs, header: hb, footer: fb, image }) = layouts.get(&layout) else { continue }; + let Some(Layout { attrs: a, columns: cols, column_specs, header: hb, footer: fb, image, image_name, legacy_image }) = layouts.get(&layout) + else { + continue; + }; // What a layout leaves unsaid: the default paper, and no margins. let mut sec = unstated_page(paper); apply_layout(a, &mut sec); sec.line_numbers.clone_from(&lines); if pages.first.is_none() { pages.doc.page_color = get(a, "fo:background-color").and_then(model::parse_color); - pages.doc.page_image.clone_from(image); + pages.doc.page_image = image.as_ref().map(|(href, fill)| (href.clone(), *fill, image_name.clone())); + pages.doc.legacy_image.clone_from(legacy_image); } // ODF starts a header at the page margin and the text after the header's band; Word // measures the header's distance and the text's margin from the edge. How the band's diff --git a/crates/odf/src/text/kept.rs b/crates/odf/src/text/kept.rs index e9efebdec..da49b3054 100644 --- a/crates/odf/src/text/kept.rs +++ b/crates/odf/src/text/kept.rs @@ -307,7 +307,7 @@ fn with_as_read( /// graphic style (the `Graphics` and `Frame` styles drawings derive from) is the file's: its /// drawings written as read derive from it, and those written from the model derived from it /// when read (what WordCraft models of it is in theirs). -fn drawing_def(xml: &str) -> bool { +pub(super) fn drawing_def(xml: &str) -> bool { xml.split('>').next().unwrap_or("").starts_with(", } /// The package files the file's own definitions refer to, and those the markup written as read -/// links to (`used`), written as read with them (added to `files` and `manifest` where no file -/// `taken` has the path). A picture only an edited paragraph showed isn't written. +/// links to (`w.kept_files_used`, `w.kept_used`), written as read with them (added to `files` and +/// `manifest` where no file `taken` has the path). A picture only an edited paragraph showed isn't written. fn with_kept_files( + doc: &Document, kept: Option<&craft_wordmodel::kept::KeptXml>, - used: &std::collections::BTreeSet, + styles: &str, + w: &Writer, taken: &dyn Fn(&str) -> bool, files: &mut Vec<(String, std::sync::Arc>)>, manifest: &mut String, ) { let Some(k) = kept else { return }; - let mut wanted = used.clone(); - for d in k.defs.values() { + let mut wanted = w.kept_files_used.clone(); + let pages_kept = k.pages_as_read(doc); + let styles_kept = k.styles_as_read(doc); + for (key, d) in &k.defs { + if !pages_kept && (key.starts_with("sauto:") || key.starts_with("master:")) { + continue; + } + if key.starts_with("common:") { + // The file's definition is written back whole (`styles_as_read`, non-`draw:` + // definitions win), or this one is restored for markup naming it. WordCraft's own + // definition takes a drawing definition's place: its href is no longer used. + let written = if styles_kept { + !(super::kept::drawing_def(d) && super::kept::defined(styles, d)) + } else { + key.strip_prefix("common:").is_some_and(|n| w.kept_used.1.contains_key(n)) && !super::kept::defined(styles, d) + }; + if !written { + continue; + } + } super::kept::hrefs(d, &mut wanted); } // A file linked, or one in a folder linked (an object's). @@ -1234,7 +1254,7 @@ fn export_as(doc: &Document, mime: &str) -> Result, String> { w.objects.iter().map(|(d, _, _)| d).chain(w.object_dirs.iter().map(|(d, _)| d)).map(|d| format!("{}/", d.trim_end_matches('/'))).collect(); let taken = |p: &str| w.pictures.iter().any(|(n, _, _)| n == p) || dirs.iter().any(|d| p.starts_with(d.as_str())); let mut kept_files: Vec<_> = kept_files.into_iter().filter(|(p, _)| !taken(p)).collect(); - with_kept_files(kept, &w.kept_files_used, &taken, &mut kept_files, &mut manifest); + with_kept_files(doc, kept, &styles, &w, &taken, &mut kept_files, &mut manifest); for (path, mime) in &kept_entries { if (path.ends_with('/') && kept_files.iter().any(|(p, _)| p.starts_with(path.as_str()))) || kept_files.iter().any(|(p, _)| p == path) { manifest.push_str(&format!("\n", x(path), x(mime))); @@ -1384,43 +1404,121 @@ fn variables(doc: &Document) -> std::collections::BTreeSet { out } +/// A `draw:fill-image` definition's name and picture path, when `xml` starts with one. +fn fill_image_of(xml: &str) -> Option<(String, String)> { + let tag = xml.split('>').next()?; + if !tag.trim_start_matches('<').starts_with("draw:fill-image") { + return None; + } + let name = super::kept::raw_attr(tag, "draw:name").map(super::kept::attr_text)?; + let href = super::kept::raw_attr(tag, "xlink:href").map(super::kept::attr_text)?; + Some((name.into_owned(), href.into_owned())) +} + +/// `name` as a `draw:name` (§16.42) may read: an NCName — letters, digits, `-`, `_` and `.`, not +/// starting with `-`, `.` or a digit. Characters outside those are turned into `_`. +fn fill_image_name(name: &str) -> String { + let mut out = String::with_capacity(name.len().min(255)); + for c in name.chars().take(255) { + let ok = c.is_alphanumeric() || c == '-' || c == '_' || c == '.'; + let start_ok = c.is_alphabetic() || c == '_'; + if out.is_empty() { + out.push(if start_ok { c } else { '_' }); + } else { + out.push(if ok { c } else { '_' }); + } + } + if out.is_empty() { "PageBackground".to_string() } else { out } +} + /// `styles` with the page background picture (Design › Page Color › Fill Effects › Picture) in -/// each page layout (§17.3 `style:background-image`, first in `style:page-layout-properties`), -/// the picture written into the package by `w`: tiled, stretched, or centred at its own size -/// (§20.249 `style:repeat`). -fn page_background(doc: &Document, w: &mut Writer, styles: String) -> String { +/// each page layout (§17.3 `style:background-image`, first in `style:page-layout-properties`, and +/// modern ODF 1.2/1.3 §16.42 `draw:fill="bitmap"` with `draw:fill-image`), the picture written +/// into the package by `w`: tiled, stretched, or centred at its own size (§20.249 `style:repeat`). +fn page_background(doc: &Document, w: &mut Writer, mut styles: String) -> String { let Some(pic) = doc.settings.page_picture.as_ref() else { return styles }; let Some(data) = doc.media.get(&pic.media) else { return styles }; let Some(ext) = model::sniff_image(data) else { return styles }; - let name = format!("Pictures/page-background.{ext}"); - if !w.pictures.iter().any(|(n, _, _)| *n == name) { - w.pictures.push((name.clone(), data.clone(), model::mime_of(ext).to_string())); + let kept = doc.kept.of("odt"); + // A kept package file holding these bytes is this picture already: its path is reused, so + // saving what was opened doesn't add a copy on every save. + let same_kept = |path: &str| kept.is_some_and(|k| k.files.get(path).is_some_and(|b| b.as_slice() == data.as_slice())); + let mut name = format!("Pictures/page-background.{ext}"); + let mut pic_n = 1; + while (w.reserved.contains(&name) && !same_kept(&name)) || w.pictures.iter().any(|(n, d, _)| n == &name && d.as_slice() != data.as_slice()) { + name = format!("Pictures/page-background-{pic_n}.{ext}"); + pic_n += 1; } let repeat = match pic.fill_type.to_ascii_lowercase().as_str() { "tile" => "repeat", "center" => "no-repeat", _ => "stretch", }; - let el = format!( + // The fill-image definitions already written, by name, once: renaming looks each up rather + // than scanning every definition for it (a hostile file names thousands alike). + let mut fill_images: std::collections::HashMap = std::collections::HashMap::new(); + for d in w.kept_defs.values() { + if let Some((n, h)) = fill_image_of(d) { + fill_images.insert(n, h); + } + } + for (i, _) in styles.match_indices("", x(&fill_name), x(&name)); + if !super::kept::defined(&styles, &fill_image_def) { + styles = styles.replacen("", &format!("{fill_image_def}"), 1); + } + let bg_el = format!( "", x(&name) ); + let fill_attrs = format!(" draw:fill=\"bitmap\" draw:fill-image-name=\"{}\" style:repeat=\"{repeat}\"", x(&fill_name)); + // After each layout's properties' opening tag. - let mut out = String::with_capacity(styles.len() + el.len() * 4); + let mut out = String::with_capacity(styles.len() + bg_el.len() * 4); let mut rest = styles.as_str(); while let Some(at) = rest.find("')).map(|e| at + e) else { break }; let self_closing = rest.get(end.saturating_sub(1)..end) == Some("/"); if self_closing { - out.push_str(rest.get(..end.saturating_sub(1)).unwrap_or_default()); + let before_slash = end.saturating_sub(1); + out.push_str(rest.get(..tag_start).unwrap_or_default()); + out.push_str(&fill_attrs); + out.push_str(rest.get(tag_start..before_slash).unwrap_or_default()); out.push('>'); - out.push_str(&el); + out.push_str(&bg_el); out.push_str(""); } else { - out.push_str(rest.get(..=end).unwrap_or_default()); - out.push_str(&el); + out.push_str(rest.get(..tag_start).unwrap_or_default()); + out.push_str(&fill_attrs); + out.push_str(rest.get(tag_start..=end).unwrap_or_default()); + out.push_str(&bg_el); } - rest = rest.get(end + 1..).unwrap_or_default(); + rest = rest.get(end.saturating_add(1)..).unwrap_or_default(); } out.push_str(rest); out diff --git a/crates/odf/src/text/tests_more/page_setup.rs b/crates/odf/src/text/tests_more/page_setup.rs index af687bd45..79b74f248 100644 --- a/crates/odf/src/text/tests_more/page_setup.rs +++ b/crates/odf/src/text/tests_more/page_setup.rs @@ -233,3 +233,662 @@ fn a_page_borders_padding_keeps_the_text_where_it_was() { } assert_eq!(s.page_borders.and_then(|b| b.left).map(|b| (b.width, b.space)), Some((3.0, 24.0))); } + +/// An ODT document with a page background and an active header containing a logo +/// preserves the page background (both modern draw:fill and legacy fallback) and the header logo (#5803). +#[test] +fn background_plus_small_header_logo_keeps_background() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let logo_png = { + let img = image::RgbImage::from_pixel(4, 4, image::Rgb([30, 200, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let bg_media = d.add_media(png, "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media: bg_media, fill_type: "frame".into(), title: "PageBackground".into() }); + let logo_media = d.add_media(logo_png, "png"); + let mut hp = craft_wordmodel::Paragraph::new(); + hp.text = format!("Company {}", craft_wordmodel::para::OBJ); + hp.objects.push(craft_wordmodel::InlineObject::Image { + media: logo_media, + w: 50.0, + h: 20.0, + alt: "Logo".into(), + float: Default::default(), + crop: [0.0; 4], + fx: Default::default(), + ole: None, + }); + hp.runs = vec![ + craft_wordmodel::para::Run { len: 8, props: Default::default() }, + craft_wordmodel::para::Run { len: craft_wordmodel::para::OBJ.len_utf8(), props: Default::default() }, + ]; + d.parts.insert( + 1, + craft_wordmodel::Part { kind: craft_wordmodel::PartKind::Header, blocks: vec![craft_wordmodel::para_block(hp)], body: Default::default() }, + ); + d.last_section.headers.default = Some(1); + + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + // Verify both draw:fill="bitmap" and style:background-image are emitted consistently + assert!(s.contains("draw:fill=\"bitmap\""), "page layout must declare draw:fill=\"bitmap\": {s}"); + assert!(s.contains("style:background-image"), "legacy background-image fallback must be present: {s}"); + assert!(s.contains(""# + ); + let content = format!( + r#"Cell test"# + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/cell-bg.png", &png)]); + let d = import(&pkg).unwrap(); + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains("style:background-image"), "cell style background-image must survive on kept path: {s}"); + assert!(s.contains("Pictures/cell-bg.png"), "cell background picture path must survive: {s}"); +} + +/// An ODT document with a text-only header retains its legacy page background image +/// and writes the image into the package archive. +#[test] +fn text_header_keeps_page_background_picture() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let bg_media = d.add_media(png, "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media: bg_media, fill_type: "frame".into(), title: String::new() }); + let mut hp = craft_wordmodel::Paragraph::new(); + hp.text = "Header text".into(); + hp.runs = vec![craft_wordmodel::para::Run { len: hp.text.len(), props: Default::default() }]; + d.parts.insert( + 1, + craft_wordmodel::Part { kind: craft_wordmodel::PartKind::Header, blocks: vec![craft_wordmodel::para_block(hp)], body: Default::default() }, + ); + d.last_section.headers.default = Some(1); + + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains("style:background-image"), "page background must be written when header has no picture: {s}"); + assert!(s.contains("Pictures/page-background.png")); +} + +/// Every picture written into the ODT zip archive must be referenced by styles.xml or content.xml (#5803). +#[test] +fn no_unreferenced_images_left_in_package() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let ns = crate::text::tests::NS_T; + let nsd = r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:xlink="http://www.w3.org/1999/xlink""#; + // An imported ODT with modern draw:fill="bitmap" and draw:fill-image + let styles = format!( + r#""# + ); + let content = format!( + r#"Body text."# + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/original.png", &png)]); + let mut d = import(&pkg).unwrap(); + + // Now insert a header picture into the document, changing page setup + let header_media = d.add_media(png.clone(), "png"); + let mut hp = craft_wordmodel::Paragraph::new(); + hp.text = format!("{}", craft_wordmodel::para::OBJ); + hp.objects.push(craft_wordmodel::InlineObject::Image { + media: header_media, + w: 100.0, + h: 50.0, + alt: "Header logo".into(), + float: Default::default(), + crop: [0.0; 4], + fx: Default::default(), + ole: None, + }); + hp.runs = vec![craft_wordmodel::para::Run { len: hp.text.len(), props: Default::default() }]; + d.parts.insert( + 1, + craft_wordmodel::Part { kind: craft_wordmodel::PartKind::Header, blocks: vec![craft_wordmodel::para_block(hp)], body: Default::default() }, + ); + d.last_section.headers.default = Some(1); + + let saved = export(&d).unwrap(); + let styles_xml = crate::text::tests::entry(&saved, "styles.xml"); + let content_xml = crate::text::tests::entry(&saved, "content.xml"); + + // Verify modern draw:fill bitmap and legacy style:background-image resolve to the same image target + let fill_name = + styles_xml.split("draw:fill-image-name=\"").nth(1).and_then(|s| s.split('"').next()).expect("styles.xml must contain draw:fill-image-name"); + let draw_fill_pattern = format!(" = (0..archive.len()).map(|i| archive.by_index(i).unwrap().name().unwrap().to_string()).collect(); + + let pic_files: Vec<&String> = file_names.iter().filter(|n| n.starts_with("Pictures/")).collect(); + assert!(!pic_files.is_empty(), "package should contain picture files"); + + for pic in pic_files { + let is_referenced = styles_xml.contains(pic.as_str()) || content_xml.contains(pic.as_str()); + assert!(is_referenced, "package image {pic} must be referenced in styles.xml or content.xml: styles={styles_xml}"); + } +} + +/// An inactive header (e.g. `first` when `title_page` is false, or `even` when `even_odd_headers` +/// is false) containing an image does NOT suppress the page background picture (#5803). +#[test] +fn inactive_header_picture_does_not_suppress_page_background() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let bg_media = d.add_media(png.clone(), "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media: bg_media, fill_type: "frame".into(), title: String::new() }); + + let header_media = d.add_media(png, "png"); + let mut hp = craft_wordmodel::Paragraph::new(); + hp.text = format!("{}", craft_wordmodel::para::OBJ); + hp.objects.push(craft_wordmodel::InlineObject::Image { + media: header_media, + w: 100.0, + h: 100.0, + alt: "Inactive header picture".into(), + float: Default::default(), + crop: [0.0; 4], + fx: Default::default(), + ole: None, + }); + hp.runs = vec![craft_wordmodel::para::Run { len: hp.text.len(), props: Default::default() }]; + d.parts.insert( + 100, + craft_wordmodel::Part { kind: craft_wordmodel::PartKind::Header, blocks: vec![craft_wordmodel::para_block(hp)], body: Default::default() }, + ); + + // Point headers.first to the header with picture, but title_page is FALSE. + d.last_section.headers.first = Some(100); + d.last_section.title_page = false; + + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains("style:background-image"), "background-image must be preserved when header picture is in an inactive header: {s}"); + assert!(s.contains("Pictures/page-background.png")); +} + +/// A noninitial continuous section header containing a picture does NOT produce a master page +/// and does not suppress the page background (#5803). +#[test] +fn continuous_section_header_picture_does_not_suppress_page_background() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let bg_media = d.add_media(png.clone(), "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media: bg_media, fill_type: "frame".into(), title: String::new() }); + + // Body with a first section ending paragraph + let mut p1 = craft_wordmodel::Paragraph::new(); + p1.text = "First paragraph in section 1".into(); + p1.runs = vec![craft_wordmodel::para::Run { len: p1.text.len(), props: Default::default() }]; + p1.section = Some(Box::new(craft_wordmodel::SectionProps::default())); + d.body.push(craft_wordmodel::para_block(p1)); + + // Second section (last_section) is continuous + let header_media = d.add_media(png, "png"); + let mut hp = craft_wordmodel::Paragraph::new(); + hp.text = format!("{}", craft_wordmodel::para::OBJ); + hp.objects.push(craft_wordmodel::InlineObject::Image { + media: header_media, + w: 100.0, + h: 100.0, + alt: "Continuous section header picture".into(), + float: Default::default(), + crop: [0.0; 4], + fx: Default::default(), + ole: None, + }); + hp.runs = vec![craft_wordmodel::para::Run { len: hp.text.len(), props: Default::default() }]; + d.parts.insert( + 200, + craft_wordmodel::Part { kind: craft_wordmodel::PartKind::Header, blocks: vec![craft_wordmodel::para_block(hp)], body: Default::default() }, + ); + + d.last_section.start = craft_wordmodel::section::SectionStart::Continuous; + d.last_section.headers.default = Some(200); + + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!( + s.contains("style:background-image"), + "page background must not be suppressed by continuous section header that emits no master page: {s}" + ); + assert!(s.contains("Pictures/page-background.png")); +} + +/// A non-continuous `last_section` without any body blocks following the section break +/// does not produce an exported page and does not suppress the page background (#5803). +#[test] +fn trailing_empty_last_section_header_picture_does_not_suppress_page_background() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let bg_media = d.add_media(png.clone(), "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media: bg_media, fill_type: "frame".into(), title: String::new() }); + + // Body with a single paragraph ending section 1 (nothing follows it) + let mut p1 = craft_wordmodel::Paragraph::new(); + p1.text = "Sole paragraph ending section 1".into(); + p1.runs = vec![craft_wordmodel::para::Run { len: p1.text.len(), props: Default::default() }]; + p1.section = Some(Box::new(craft_wordmodel::SectionProps::default())); + d.body.push(craft_wordmodel::para_block(p1)); + + // last_section is NextPage and has a header with a picture, but has no blocks following + let header_media = d.add_media(png, "png"); + let mut hp = craft_wordmodel::Paragraph::new(); + hp.text = format!("{}", craft_wordmodel::para::OBJ); + hp.objects.push(craft_wordmodel::InlineObject::Image { + media: header_media, + w: 100.0, + h: 100.0, + alt: "Unused trailing section header picture".into(), + float: Default::default(), + crop: [0.0; 4], + fx: Default::default(), + ole: None, + }); + hp.runs = vec![craft_wordmodel::para::Run { len: hp.text.len(), props: Default::default() }]; + d.parts.insert( + 300, + craft_wordmodel::Part { kind: craft_wordmodel::PartKind::Header, blocks: vec![craft_wordmodel::para_block(hp)], body: Default::default() }, + ); + + d.last_section.start = craft_wordmodel::section::SectionStart::NextPage; + d.last_section.headers.default = Some(300); + + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains("style:background-image"), "page background must not be suppressed by an unexported trailing last_section header: {s}"); + assert!(s.contains("Pictures/page-background.png")); +} + +/// When a page background picture is replaced with a new image while retaining an existing +/// fill-image name that belongs to a graphic style, export allocates a distinct fill name, +/// references it in page layouts, and preserves the retained definition and image (#5803). +#[test] +fn replaced_page_picture_allocates_distinct_fill_name_and_keeps_retained_def() { + let orig_png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([100, 200, 100])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let new_png = { + let img = image::RgbImage::from_pixel(4, 8, image::Rgb([200, 100, 200])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let ns = crate::text::tests::NS_T; + let nsd = "xmlns:draw=\"urn:oasis:names:tc:opendocument:xmlns:drawing:1.0\""; + let styles = format!( + r#""# + ); + let content = format!( + r#"Body text"# + ); + let pkg = + crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/original.png", &orig_png)]); + let mut d = import(&pkg).unwrap(); + let new_media = d.add_media(new_png.clone(), "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media: new_media, fill_type: "stretch".into(), title: "SharedBg".into() }); + + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains(r#" Vec { + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut out: Vec = + (0..archive.len()).map(|i| archive.by_index(i).unwrap().name().unwrap().to_string()).filter(|n| n.starts_with("Pictures/")).collect(); + out.sort(); + out +} + +/// Saving a page background picture and opening it again keeps one picture and one fill-image +/// definition: the kept file is reused while its bytes are the document's, not renamed (#5856). +#[test] +fn page_background_round_trips_do_not_grow_the_package() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let media = d.add_media(png, "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media, fill_type: "frame".into(), title: "PageBackground".into() }); + let mut current = d; + for generation in 1..=4 { + let saved = export(¤t).unwrap(); + let styles = crate::text::tests::entry(&saved, "styles.xml"); + assert_eq!(styles.matches(""# + ); + let content = format!( + r#"Body text."# + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/original.png", &png)]); + let d = import(&pkg).unwrap(); + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert_eq!(s.matches(""# + ); + let content = format!( + r#"Body text."# + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/para.png", &png)]); + let d = import(&pkg).unwrap(); + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains("Pictures/para.png"), "the style's background reference must stay: {s}"); + assert_eq!(picture_paths(&saved), ["Pictures/para.png"], "the referenced picture must be written"); +} + +/// A DOCX texture name (`o:title`, "Blue tissue paper") is not an XML NCName: the fill image it +/// names is written with one (#5856). +#[test] +fn a_texture_name_is_written_as_an_ncname() { + let png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([200, 30, 30])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let mut d = Document::new(); + let media = d.add_media(png, "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media, fill_type: "frame".into(), title: "Blue tissue paper".into() }); + let saved = export(&d).unwrap(); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains(r#""#)); + } + let styles = format!( + r#"{defs}"# + ); + let content = format!( + r#"Body text."# + ); + let pkg = crate::text::tests::zip_of(&[ + ("content.xml", content.as_bytes()), + ("styles.xml", styles.as_bytes()), + ("Pictures/bg.png", &bg), + ("Pictures/other.png", &other), + ]); + let mut d = import(&pkg).unwrap(); + // The user replaces the picture: its fill-image name now meets every kept name. + let media = d.add_media(replacement, "png"); + d.settings.page_picture = Some(craft_wordmodel::PagePicture { media, fill_type: "stretch".into(), title: "X".into() }); + let start = std::time::Instant::now(); + let saved = export(&d).unwrap(); + assert!(start.elapsed() < std::time::Duration::from_secs(5), "export took {:?}", start.elapsed()); + let s = crate::text::tests::entry(&saved, "styles.xml"); + assert!(s.contains(r#"` if present and valid (#1784). +#[test] +fn legacy_background_fallback_when_modern_bitmap_missing() { + let legacy_png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([10, 150, 20])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let ns = crate::text::tests::NS_T; + let nsd = r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:xlink="http://www.w3.org/1999/xlink""#; + let styles = format!( + r#""# + ); + let content = format!( + r#"Body text."# + ); + let pkg = + crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/legacy.png", &legacy_png)]); + let d = import(&pkg).unwrap(); + let pic = d.settings.page_picture.as_ref().expect("legacy fallback background picture must be imported"); + assert_eq!(d.media.get(&pic.media).map(|b| b.as_slice()), Some(legacy_png.as_slice())); + assert_eq!(pic.fill_type, "frame"); +} + +/// When modern `draw:fill="bitmap"` references a file with non-image bytes, +/// import falls back to legacy `` if present and valid (#1784). +#[test] +fn legacy_background_fallback_when_modern_bitmap_non_image() { + let legacy_png = { + let img = image::RgbImage::from_pixel(8, 4, image::Rgb([10, 150, 20])); + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + }; + let corrupt_data = b"This is plain text and definitely not a valid image format."; + let ns = crate::text::tests::NS_T; + let nsd = r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:xlink="http://www.w3.org/1999/xlink""#; + let styles = format!( + r#""# + ); + let content = format!( + r#"Body text."# + ); + let pkg = crate::text::tests::zip_of(&[ + ("content.xml", content.as_bytes()), + ("styles.xml", styles.as_bytes()), + ("Pictures/corrupt.bin", corrupt_data), + ("Pictures/legacy.png", &legacy_png), + ]); + let d = import(&pkg).unwrap(); + let pic = d.settings.page_picture.as_ref().expect("legacy fallback background picture must be imported when modern is non-image"); + assert_eq!(d.media.get(&pic.media).map(|b| b.as_slice()), Some(legacy_png.as_slice())); + assert_eq!(pic.fill_type, "frame"); +} From ce8ab5cc6d5ba4df57a29da6a43f3ccbd1c7205d Mon Sep 17 00:00:00 2001 From: Diego Guerini Date: Mon, 16 Nov 2026 08:52:44 +0100 Subject: [PATCH 031/209] craft-docx: a locked canvas's paragraph falls back on its level's alignment and the list style's defaults; bold or italic set off stays off --- crates/docx/src/read/story/locked_canvas.rs | 35 ++++++++++++++------- crates/docx/tests/it/locked_canvas.rs | 20 ++++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/crates/docx/src/read/story/locked_canvas.rs b/crates/docx/src/read/story/locked_canvas.rs index 01e8f2872..59a52cea2 100644 --- a/crates/docx/src/read/story/locked_canvas.rs +++ b/crates/docx/src/read/story/locked_canvas.rs @@ -102,17 +102,22 @@ fn picture(pic: &El) -> El { fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { body.children("a:p") .map(|p| { - // A run's properties default to its paragraph's own (`a:pPr/a:defRPr`), then to the - // list style's for the paragraph's level (`lvl`, 0–8), property by property. + // A paragraph's properties default to the list style's for its level (`lvl`, 0–8), + // then to the list style's defaults (`a:defPPr`), property by property; a run's to its + // paragraph's own (`a:pPr/a:defRPr`), then to those two's. let ppr_el = p.child("a:pPr"); let level = ppr_el.and_then(|pp| pp.attr("lvl")).and_then(|v| v.trim().parse::().ok()).filter(|l| *l <= 8).unwrap_or(0); - let level_style = body.child("a:lstStyle").and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))); - let defaults: Vec<&El> = - [ppr_el.and_then(|pp| pp.child("a:defRPr")), level_style.and_then(|l| l.child("a:defRPr"))].into_iter().flatten().collect(); + let list_style = body.child("a:lstStyle"); + let paragraph_styles: Vec<&El> = + [ppr_el, list_style.and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))), list_style.and_then(|l| l.child("a:defPPr"))] + .into_iter() + .flatten() + .collect(); + let defaults: Vec<&El> = paragraph_styles.iter().filter_map(|pp| pp.child("a:defRPr")).collect(); let defaults = defaults.as_slice(); let mut ppr = vec![Node::El(el("w:spacing", attrs(&[("w:before", "0"), ("w:after", "0"), ("w:line", "240"), ("w:lineRule", "auto")]), Vec::new()))]; - let jc = match ppr_el.and_then(|pp| pp.attr("algn")) { + let jc = match paragraph_styles.iter().find_map(|pp| pp.attr("algn")) { Some("ctr") => Some("center"), Some("r") => Some("right"), Some("just") => Some("both"), @@ -151,7 +156,12 @@ fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { let get = |n: &str| rpr.and_then(|r| r.attr(n)).or_else(|| defaults.iter().find_map(|d| d.attr(n))); let child = |n: &str| rpr.and_then(|r| r.child(n)).or_else(|| defaults.iter().find_map(|d| d.child(n))); - let on = |n: &str| get(n).is_some_and(|v| v == "1" || v == "true"); + // On, off (kept, so the document's style doesn't turn it back on), or not given. + let toggle = |n: &str| match get(n) { + Some("1" | "true") => Some(true), + Some("0" | "false") => Some(false), + _ => None, + }; let mut kids = Vec::new(); let mut push = |name: &str, a: &[(&str, &str)]| kids.push(Node::El(el(name, attrs(a), Vec::new()))); // Theme fonts (`+mn-lt`) are left to the document's. @@ -169,11 +179,12 @@ fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { if !fonts.is_empty() { push("w:rFonts", &fonts); } - if on("b") { - push("w:b", &[]); - } - if on("i") { - push("w:i", &[]); + for (drawingml, word) in [("b", "w:b"), ("i", "w:i")] { + match toggle(drawingml) { + Some(true) => push(word, &[]), + Some(false) => push(word, &[("w:val", "0")]), + None => {} + } } match get("cap") { Some("all") => push("w:caps", &[]), diff --git a/crates/docx/tests/it/locked_canvas.rs b/crates/docx/tests/it/locked_canvas.rs index 8fa7a0a20..fb01406a4 100644 --- a/crates/docx/tests/it/locked_canvas.rs +++ b/crates/docx/tests/it/locked_canvas.rs @@ -122,6 +122,26 @@ fn a_locked_canvas_text_shape_and_paragraph_defaults() { } } +/// A paragraph's alignment and its runs' properties fall back on its level's style, then on the +/// list style's defaults (`a:defPPr`); a run's bold set off stays off. +#[test] +fn a_locked_canvas_paragraph_takes_its_level_and_the_list_defaults() { + use craft_wordmodel::props::Align; + let text = r#"PlainCentredOwn"#; + let d = craft_docx::read(&docx(&drawing(text))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (_, _, children) = group(d); + let [shape] = &children[..] else { panic!("{children:?}") }; + let InlineObject::Shape { story: Some(id), .. } = &shape.obj else { panic!("{:?}", shape.obj) }; + let blocks = &d.parts.get(id).unwrap().blocks; + let para = |i: usize| blocks[i].as_para().unwrap(); + let aligns: Vec> = (0..3).map(|i| para(i).props.align).collect(); + assert_eq!(aligns, [Some(Align::Right), Some(Align::Center), Some(Align::Left)], "the list defaults', the level's, its own"); + let runs: Vec<(Option, Option)> = (0..3).map(|i| (para(i).runs[0].props.size, para(i).runs[0].props.bold)).collect(); + assert_eq!(runs, [(Some(16.0), Some(false)), (Some(16.0), Some(true)), (Some(16.0), Some(true))]); + } +} + /// Before wordcraft#5060 only the canvas's first picture showed: the shapes and their text were dropped. #[test] fn a_locked_canvas_is_not_read_as_its_picture_alone() { From f8388e2293a5441a93ef19efefb54ca6891abd77 Mon Sep 17 00:00:00 2001 From: Diego Guerini Date: Mon, 16 Nov 2026 08:59:48 +0100 Subject: [PATCH 032/209] craft-odf: an object without a stored stacking order is written above every stored one (wordcraft#2638) --- crates/odf/src/text/frames.rs | 17 ++++++++++--- crates/odf/src/text/package.rs | 2 ++ crates/odf/src/text/tests_more/floats.rs | 31 +++++++++++++++++++++++- crates/odf/src/text/writer.rs | 2 ++ crates/odf/src/text/writer/charts.rs | 2 +- crates/odf/src/text/writer/drawing.rs | 16 ++++++------ 6 files changed, 58 insertions(+), 12 deletions(-) diff --git a/crates/odf/src/text/frames.rs b/crates/odf/src/text/frames.rs index cb8766906..86e11b73c 100644 --- a/crates/odf/src/text/frames.rs +++ b/crates/odf/src/text/frames.rs @@ -443,9 +443,20 @@ pub(super) fn z_of(a: &Attrs) -> u32 { get(a, "draw:z-index").and_then(|v| v.trim().parse::().ok()).map_or(0, |z| z.saturating_add(1)) } -/// `float`'s place in the stacking order as `draw:z-index` (see [`z_of`]): 0 when unknown. -pub(crate) fn z_index(float: Option<&Float>) -> u32 { - float.map_or(0, |f| f.z.saturating_sub(1)) +/// `float`'s place in the stacking order as `draw:z-index` (see [`z_of`]). One without a stored +/// order (made in WordCraft, or nothing to place) takes `unset`, which the writer puts above every +/// stored order, where [`Float::z`] stacks it. +pub(crate) fn z_index(float: Option<&Float>, unset: u32) -> u32 { + match float.map(|f| f.z) { + Some(z) if z > 0 => z - 1, + _ => unset, + } +} + +/// The `draw:z-index` the objects of `doc` without a stored order take: past the highest stored +/// one (a stored order is one more than its `draw:z-index`), 0 when none has one. +pub(crate) fn unset_z_index(doc: &craft_wordmodel::Document) -> u32 { + doc.stacking_orders().0.last().copied().unwrap_or(0) } /// How far a frame `h` points tall in line with the text is raised from standing on the diff --git a/crates/odf/src/text/package.rs b/crates/odf/src/text/package.rs index 97520027c..c64306eeb 100644 --- a/crates/odf/src/text/package.rs +++ b/crates/odf/src/text/package.rs @@ -1038,6 +1038,7 @@ fn export_as(doc: &Document, mime: &str) -> Result, String> { // Paragraphs kept as read name their own automatic styles: WordCraft's take another prefix. let kept = doc.kept.of("odt"); let mut w = if kept.is_some() { Writer::with_prefix(KEPT_PREFIX) } else { Writer::default() }; + w.unset_z = super::frames::unset_z_index(doc); w.note_numbers = Some([false, true].map(|endnote| { let o = doc.note_options(&doc.last_section, endnote); (o.num_fmt, o.num_start) @@ -1166,6 +1167,7 @@ fn export_as(doc: &Document, mime: &str) -> Result, String> { let mut hw = Writer::with_prefix(if kept.is_some() { "wcM" } else { "M" }); // Headers' and footers' new pictures and objects skip the kept files' names too. hw.reserved = std::sync::Arc::clone(&w.reserved); + hw.unset_z = w.unset_z; hw.accents = accents; hw.variables.clone_from(&w.variables); let parts: Vec = pages.iter().map(|(_, sec)| page_parts(doc, sec, &mut hw)).collect(); diff --git a/crates/odf/src/text/tests_more/floats.rs b/crates/odf/src/text/tests_more/floats.rs index f8ac5fe99..ab0c0bf26 100644 --- a/crates/odf/src/text/tests_more/floats.rs +++ b/crates/odf/src/text/tests_more/floats.rs @@ -362,12 +362,41 @@ fn a_frame_keeps_its_place_in_the_stacking_order() { let saved = crate::text::tests::export_model(&d).unwrap(); assert!(crate::text::tests::entry(&saved, "content.xml").contains(r#"draw:z-index="3""#)); assert_eq!(float(&import(&saved).unwrap()).z, 4); - // Unknown (inserted in WordCraft): at the bottom, in document order. + // Unknown (inserted in WordCraft), with nothing else stored: 0. let d = open(r#"style:wrap="run-through""#, r#"text:anchor-type="paragraph""#); assert_eq!(float(&d).z, 0); assert!(crate::text::tests::entry(&crate::text::tests::export_model(&d).unwrap(), "content.xml").contains(r#"draw:z-index="0""#)); } +/// An object without a stored order (made in WordCraft) stacks above every object that has one +/// (`Float::z`), so it is written above them, not at the bottom. +#[test] +fn an_object_made_here_is_written_above_the_stored_orders() { + let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7u8; 64]].concat(); + let frame = |z: &str| { + format!( + r#""# + ) + }; + let content = format!( + r#"{}{}Text beside."#, + frame(r#" draw:z-index="3""#), + frame("") + ); + let d = import(&zip_of(&[("content.xml", content.as_bytes()), ("Pictures/a.png", &png)])).unwrap(); + let zs: Vec = d.body[0] + .as_para() + .unwrap() + .objects + .iter() + .filter_map(|o| if let InlineObject::Image { float, .. } = o { Some(float.z) } else { None }) + .collect(); + assert_eq!(zs, [4, 0], "one stored, one made here"); + let content = crate::text::tests::entry(&crate::text::tests::export_model(&d).unwrap(), "content.xml"); + let z: Vec<&str> = content.split(r#"draw:z-index=""#).skip(1).filter_map(|s| s.split('"').next()).collect(); + assert_eq!(z, ["3", "4"], "{content}"); +} + #[test] fn a_frame_anchored_to_a_page_stays_on_that_page() { // LibreOffice's page-anchored frames name their page. Written without it, a frame on page 3 diff --git a/crates/odf/src/text/writer.rs b/crates/odf/src/text/writer.rs index 2ad475739..220f5afab 100644 --- a/crates/odf/src/text/writer.rs +++ b/crates/odf/src/text/writer.rs @@ -176,6 +176,8 @@ pub(super) struct Writer { pub(super) named_text: std::collections::BTreeSet, /// Text boxes written so far (their frames' names). boxes: usize, + /// The `draw:z-index` of objects without a stored stacking order (see `frames::unset_z_index`). + pub(super) unset_z: u32, /// Notes written so far (their ids), and footnotes and endnotes among them (their numbers). notes: usize, footnotes: u32, diff --git a/crates/odf/src/text/writer/charts.rs b/crates/odf/src/text/writer/charts.rs index 1cc3e3e25..9c6c2a504 100644 --- a/crates/odf/src/text/writer/charts.rs +++ b/crates/odf/src/text/writer/charts.rs @@ -200,7 +200,7 @@ impl Writer { // Always placed by a style (see `image_frame`). let (anchor, props, xy) = placement(place, c.w, c.h); let (anchor, style) = (anchor, format!(" draw:style-name=\"{}\"{xy}", self.graphic_style(props))); - let z = place.z_index(); + let z = place.z_index(self.unset_z); out.push_str(&format!( "{inner}", self.prefix, diff --git a/crates/odf/src/text/writer/drawing.rs b/crates/odf/src/text/writer/drawing.rs index d49a07ea8..6a5f08feb 100644 --- a/crates/odf/src/text/writer/drawing.rs +++ b/crates/odf/src/text/writer/drawing.rs @@ -70,9 +70,10 @@ pub(in crate::text) enum Place<'a> { } impl<'a> Place<'a> { - /// Its place in the stacking order (§19.231 `draw:z-index`, see `frames::z_of`). - pub(in crate::text) fn z_index(self) -> u32 { - crate::text::frames::z_index(self.float()) + /// Its place in the stacking order (§19.231 `draw:z-index`, see `frames::z_of`), `unset` + /// without a stored one. + pub(in crate::text) fn z_index(self, unset: u32) -> u32 { + crate::text::frames::z_index(self.float(), unset) } /// What turns or mirrors the object, and places it in the text. @@ -323,7 +324,7 @@ impl Writer { let (anchor, props, xy) = placement(place, img.w, img.h); let (anchor, style) = (anchor, format!(" draw:style-name=\"{}\"{xy}", self.graphic_style(format!("{}{props}{clip}", super::GRAPHICS_PARENT)))); - let z = place.z_index(); + let z = place.z_index(self.unset_z); out.push_str(&format!( "{inner}", self.prefix, @@ -360,7 +361,8 @@ impl Writer { let (anchor, props, _) = placement(Place::Text(float.filter(|f| f.wrap != craft_wordmodel::para::Wrap::Inline)), size.0, size.1); let style = self.graphic_style(props); self.boxes += 1; - out.push_str(&format!("", self.prefix, self.boxes)); + let z = crate::text::frames::z_index(float, self.unset_z); + out.push_str(&format!("", self.prefix, self.boxes)); let at = Place::Text(float).origin(); self.members(members, at, 0, out); out.push_str(""); @@ -419,7 +421,7 @@ impl Writer { .and_then(|c| c.next.as_deref()) .map(|n| format!(" draw:chain-next-name=\"{}{}\"", self.prefix, x(n))) .unwrap_or_default(); - let z = place.z_index(); + let z = place.z_index(self.unset_z); out.push_str(&format!( "{body}", x(&name), @@ -522,7 +524,7 @@ impl Writer { }) .unwrap_or_default(); let conn = self.conn_attrs(kind, look); - let z = place.z_index(); + let z = place.z_index(self.unset_z); out.push_str(&format!( "<{element} draw:name=\"{}Shape{}\" draw:style-name=\"{style}\"{anchor}{at}{conn} draw:z-index=\"{z}\">{body}{geometry}", self.prefix, self.boxes, From cccb00b6b44be9ee0ee7007b29fa73969da0dffb Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:01:22 +0100 Subject: [PATCH 033/209] craft-odf: a group member's stored stacking order counts for the unset draw:z-index (wordcraft#2638) unset_z_index only looked at the floating objects' orders, so a group member without a stored order could be written below a member that has one. It now also walks the groups' members. --- crates/odf/src/text/frames.rs | 31 ++++++++++++++++++++++-- crates/odf/src/text/tests_more/floats.rs | 24 ++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/crates/odf/src/text/frames.rs b/crates/odf/src/text/frames.rs index 86e11b73c..44a31357a 100644 --- a/crates/odf/src/text/frames.rs +++ b/crates/odf/src/text/frames.rs @@ -454,9 +454,36 @@ pub(crate) fn z_index(float: Option<&Float>, unset: u32) -> u32 { } /// The `draw:z-index` the objects of `doc` without a stored order take: past the highest stored -/// one (a stored order is one more than its `draw:z-index`), 0 when none has one. +/// one (a stored order is one more than its `draw:z-index`), 0 when none has one. A group's +/// members are written with their own orders too, so theirs count as well as the floating +/// objects' ([`craft_wordmodel::Document::stacking_orders`]). pub(crate) fn unset_z_index(doc: &craft_wordmodel::Document) -> u32 { - doc.stacking_orders().0.last().copied().unwrap_or(0) + use craft_wordmodel::para::{GroupChild, InlineObject}; + fn members(children: &[GroupChild], depth: usize, top: &mut u32) { + for c in children { + if let Some((_, _, f)) = c.obj.frame() { + *top = (*top).max(f.z); + } + if let InlineObject::Group { children, .. } = &c.obj + && depth < craft_flow::MAX_GROUP_DEPTH + { + members(children, depth + 1, top); + } + } + } + let mut top = doc.stacking_orders().0.last().copied().unwrap_or(0); + for blocks in std::iter::once(&doc.body).chain(doc.parts.values().map(|p| &p.blocks)) { + for b in blocks { + craft_wordmodel::walk::each_para(b, 0, &mut |p| { + for o in &p.objects { + if let InlineObject::Group { children, .. } = o { + members(children, 0, &mut top); + } + } + }); + } + } + top } /// How far a frame `h` points tall in line with the text is raised from standing on the diff --git a/crates/odf/src/text/tests_more/floats.rs b/crates/odf/src/text/tests_more/floats.rs index ab0c0bf26..0fff3797a 100644 --- a/crates/odf/src/text/tests_more/floats.rs +++ b/crates/odf/src/text/tests_more/floats.rs @@ -397,6 +397,30 @@ fn an_object_made_here_is_written_above_the_stored_orders() { assert_eq!(z, ["3", "4"], "{content}"); } +/// A group's members are written with their own stacking orders, so a member without one (made +/// in WordCraft) goes above a member's stored order too, not just above the floating objects'. +#[test] +fn an_object_made_here_is_written_above_the_group_members_stored_orders() { + let content = format!( + r#"Text beside."# + ); + let mut d = import(&zip_of(&[("content.xml", content.as_bytes())])).unwrap(); + // The group stored at the bottom, its first member stored above it, its second made here. + d.objects_mut(&|o| matches!(o, InlineObject::Group { .. }), &mut |o| { + let InlineObject::Group { float, children, .. } = o else { return }; + float.z = 1; + let wrap = float.wrap; + for (c, z) in children.iter_mut().zip([4, 0]) { + if let Some(f) = c.obj.float_mut() { + (f.wrap, f.z) = (wrap, z); + } + } + }); + let content = crate::text::tests::entry(&crate::text::tests::export_model(&d).unwrap(), "content.xml"); + let z: Vec<&str> = content.split(r#"draw:z-index=""#).skip(1).filter_map(|s| s.split('"').next()).collect(); + assert_eq!(z, ["0", "3", "4"], "the group's, the stored member's, the member made here above it: {content}"); +} + #[test] fn a_frame_anchored_to_a_page_stays_on_that_page() { // LibreOffice's page-anchored frames name their page. Written without it, a frame on page 3 From ecc89eccffe8a3a99b9e611c55274469f4b2d35c Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:40:58 +0100 Subject: [PATCH 034/209] Add craft-unzip: Craft Anything's hardened ZIP reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves Craft Anything's crates/formats/src/zip.rs, zip_file.rs and the entry decoders from decompress.rs into a pure-Rust crate, keeping the R3.12 hardening: directory and extra-field bounds, ZIP64 end-record recovery after a prepended stub, multi-volume refusal, CRC-checked Info-ZIP Unicode Path names, decoded-size and CRC-32 checks. bzip2, LZMA/XZ and Zstandard entry methods sit behind default features; Serialize sits behind an optional serde feature. Part of #778. Co-authored-by: Afloat Co-authored-by: BFlat Co-authored-by: Brandon Thomas Co-authored-by: GGWA <65288322+rech4210@users.noreply.github.com> Co-authored-by: Max Mittelstädt <62167861+TheGalitube@users.noreply.github.com> Co-authored-by: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Co-authored-by: RRFZ <247457381+Plastic-time@users.noreply.github.com> Co-authored-by: Roman Serikoff --- Cargo.lock | 48 ++ Cargo.toml | 1 + README.md | 1 + crates/unzip/Cargo.toml | 34 ++ crates/unzip/src/bytes.rs | 41 ++ crates/unzip/src/decode.rs | 352 +++++++++++ crates/unzip/src/file.rs | 262 +++++++++ crates/unzip/src/lib.rs | 549 ++++++++++++++++++ crates/unzip/src/testkit.rs | 80 +++ crates/unzip/src/tests.rs | 393 +++++++++++++ crates/unzip/tests/common/mod.rs | 41 ++ .../tests/zip64_end_record_compatibility.rs | 169 ++++++ crates/unzip/tests/zip64_extra_boundaries.rs | 64 ++ crates/unzip/tests/zip_decoded_size.rs | 43 ++ crates/unzip/tests/zip_directory_bounds.rs | 74 +++ crates/unzip/tests/zip_entry_detection.rs | 43 ++ crates/unzip/tests/zip_split_volumes.rs | 165 ++++++ crates/unzip/tests/zip_stored_cap.rs | 21 + 18 files changed, 2381 insertions(+) create mode 100644 crates/unzip/Cargo.toml create mode 100644 crates/unzip/src/bytes.rs create mode 100644 crates/unzip/src/decode.rs create mode 100644 crates/unzip/src/file.rs create mode 100644 crates/unzip/src/lib.rs create mode 100644 crates/unzip/src/testkit.rs create mode 100644 crates/unzip/src/tests.rs create mode 100644 crates/unzip/tests/common/mod.rs create mode 100644 crates/unzip/tests/zip64_end_record_compatibility.rs create mode 100644 crates/unzip/tests/zip64_extra_boundaries.rs create mode 100644 crates/unzip/tests/zip_decoded_size.rs create mode 100644 crates/unzip/tests/zip_directory_bounds.rs create mode 100644 crates/unzip/tests/zip_entry_detection.rs create mode 100644 crates/unzip/tests/zip_split_volumes.rs create mode 100644 crates/unzip/tests/zip_stored_cap.rs diff --git a/Cargo.lock b/Cargo.lock index 480e000e5..7a00d57d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -921,6 +921,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bzip2-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beeb59e7e4c811ab37cc73680c798c7a5da77fc9989c62b09138e31ee740f735" +dependencies = [ + "crc32fast", + "tinyvec", +] + [[package]] name = "calloop" version = "0.13.0" @@ -2888,6 +2898,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "craft-unzip" +version = "0.1.0" +dependencies = [ + "bzip2-rs", + "lzma-rs", + "miniz_oxide 0.8.9", + "ruzstd", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-update" version = "0.1.0" @@ -3096,6 +3119,21 @@ dependencies = [ "zip", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc24" version = "0.1.6" @@ -6149,6 +6187,16 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "maybe-owned" version = "0.3.4" diff --git a/Cargo.toml b/Cargo.toml index bd08fa725..34ee6095e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,6 +174,7 @@ craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } +craft-unzip = { path = "crates/unzip" } craft-epub = { path = "crates/epub" } craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } diff --git a/README.md b/README.md index 8150dfea9..8a4c3d327 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit - [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. - [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. - [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. +- [`craft-unzip`](crates/unzip/src/lib.rs): ZIP reading over bytes already in memory (or one named entry from a seekable file), from Craft Anything: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind default features) checked against the declared size and CRC-32. Pure Rust with no `zip` crate underneath; for inspectors that must describe damaged archives rather than refuse them. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/crates/unzip/Cargo.toml b/crates/unzip/Cargo.toml new file mode 100644 index 000000000..8a414277b --- /dev/null +++ b/crates/unzip/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "craft-unzip" +description = "ZIP archives (APK, JAR, IPA, MSIX…) from hostile bytes, from Craft Anything: a central-directory listing with ZIP64, prepended-stub and multi-volume checks, and bounded entry reads checked against the declared size and CRC-32. Pure Rust, no I/O beyond an optional seekable reader." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +thiserror = { workspace = true } +miniz_oxide = { workspace = true } +# Entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. +bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 +lzma-rs = { version = "0.3", optional = true } # MIT +ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT +# `serde`: `Serialize` on the listing types, for apps that report listings as JSON. +serde = { workspace = true, optional = true } + +[features] +default = ["bzip2", "lzma", "zstd"] +# Method 12. +bzip2 = ["dep:bzip2-rs"] +# Methods 14 (ZIP LZMA) and 95 (XZ). +lzma = ["dep:lzma-rs"] +# Method 93. +zstd = ["dep:ruzstd"] +serde = ["dep:serde"] + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/crates/unzip/src/bytes.rs b/crates/unzip/src/bytes.rs new file mode 100644 index 000000000..be7aa3ebe --- /dev/null +++ b/crates/unzip/src/bytes.rs @@ -0,0 +1,41 @@ +//! Checked little-endian reads over untrusted bytes. Every read returns `None` when out of range. + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Reader<'a> { + data: &'a [u8], +} + +impl<'a> Reader<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Reader { data } + } + + pub(crate) fn data(&self) -> &'a [u8] { + self.data + } + + pub(crate) fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + let end = off.checked_add(len)?; + self.data.get(off..end) + } + + pub(crate) fn u8(&self, off: usize) -> Option { + self.data.get(off).copied() + } + + fn arr(&self, off: usize) -> Option<[u8; N]> { + self.slice(off, N)?.try_into().ok() + } + + pub(crate) fn u16_le(&self, off: usize) -> Option { + self.arr(off).map(u16::from_le_bytes) + } + + pub(crate) fn u32_le(&self, off: usize) -> Option { + self.arr(off).map(u32::from_le_bytes) + } + + pub(crate) fn u64_le(&self, off: usize) -> Option { + self.arr(off).map(u64::from_le_bytes) + } +} diff --git a/crates/unzip/src/decode.rs b/crates/unzip/src/decode.rs new file mode 100644 index 000000000..1d41d4a55 --- /dev/null +++ b/crates/unzip/src/decode.rs @@ -0,0 +1,352 @@ +//! Bounded decoders for the bzip2, ZIP LZMA, XZ and Zstandard entry methods, from Craft +//! Anything's `decompress.rs`. Each caps its output at `max_out` bytes (`TooLarge` rather than +//! allocating without limit) and never panics on a corrupt or truncated stream. + +use std::io; +#[cfg(any(feature = "bzip2", feature = "lzma"))] +use std::io::Read; +#[cfg(any(feature = "lzma", feature = "zstd"))] +use std::io::Write; + +/// Why a decode stopped. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub(crate) enum DecompressError { + #[error("the compressed stream is damaged")] + Corrupt, + #[error("the compressed stream ends early")] + #[cfg(any(feature = "lzma", feature = "zstd"))] + Truncated, + #[error("the output would exceed the size limit")] + TooLarge, +} + +/// A `Write` sink that collects at most `cap` bytes, then refuses the rest. The `exceeded` flag lets +/// the caller tell a size-limit stop from a genuine decode error. +#[cfg(any(feature = "lzma", feature = "zstd"))] +struct CapWrite { + buf: Vec, + cap: usize, + exceeded: bool, +} + +#[cfg(any(feature = "lzma", feature = "zstd"))] +impl CapWrite { + fn new(cap: usize) -> Self { + CapWrite { buf: Vec::new(), cap, exceeded: false } + } +} + +#[cfg(any(feature = "lzma", feature = "zstd"))] +impl Write for CapWrite { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.buf.len().saturating_add(data.len()) > self.cap { + self.exceeded = true; + return Err(io::Error::other("output size limit reached")); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn cap_of(max_out: u64) -> usize { + usize::try_from(max_out).unwrap_or(usize::MAX) +} + +/// Read a `Read` source into a `Vec`, capped at `cap` bytes. +#[cfg(feature = "bzip2")] +fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressError> { + let mut out = Vec::new(); + let mut chunk = [0u8; 64 * 1024]; + loop { + match src.read(&mut chunk) { + Ok(0) => return Ok(out), + Ok(n) => { + let slice = chunk.get(..n).unwrap_or(&[]); + if out.len().saturating_add(slice.len()) > cap { + return Err(DecompressError::TooLarge); + } + out.extend_from_slice(slice); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return Err(DecompressError::Corrupt), + } + } +} + +#[cfg(feature = "lzma")] +/// Decompress every xz stream, including standard stream padding, within one output budget. +pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> { + let ranges = xz_stream_ranges(input, max_out)?; + let mut out = CapWrite::new(cap_of(max_out)); + for range in ranges.into_iter().rev() { + let stream = input.get(range).ok_or(DecompressError::Truncated)?; + let mut cursor = io::Cursor::new(stream); + match lzma_rs::xz_decompress(&mut cursor, &mut out) { + Ok(()) => {} + Err(_) if out.exceeded => return Err(DecompressError::TooLarge), + Err(lzma_rs::error::Error::HeaderTooShort(_)) => return Err(DecompressError::Truncated), + Err(_) => return Err(DecompressError::Corrupt), + } + } + Ok(out.buf) +} + +#[cfg(feature = "lzma")] +/// Find stream boundaries backwards through their checked footer/index sizes, without +/// repeatedly trying the decoder at candidate magic bytes inside compressed data. +fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result>, DecompressError> { + let mut ranges = Vec::new(); + let mut end = input.len(); + let mut declared = 0u64; + while end > 0 { + let padded_end = end; + while end > 0 && input.get(end - 1) == Some(&0) { + end -= 1; + } + if end == 0 || !(padded_end - end).is_multiple_of(4) { + return Err(DecompressError::Corrupt); + } + let footer_at = end.checked_sub(12).ok_or(DecompressError::Truncated)?; + let footer = input.get(footer_at..end).ok_or(DecompressError::Truncated)?; + let read = crate::bytes::Reader::new(footer); + let footer_fields = footer.get(4..10).ok_or(DecompressError::Truncated)?; + if footer.get(10..) != Some(b"YZ") || read.u32_le(0) != Some(crate::crc32(footer_fields)) { + return Err(DecompressError::Corrupt); + } + let backward = u64::from(read.u32_le(4).ok_or(DecompressError::Truncated)?); + let index_len = usize::try_from((backward + 1) * 4).map_err(|_| DecompressError::Corrupt)?; + let index_at = footer_at.checked_sub(index_len).ok_or(DecompressError::Truncated)?; + let index = input.get(index_at..footer_at).ok_or(DecompressError::Truncated)?; + let crc_at = index.len().checked_sub(4).ok_or(DecompressError::Truncated)?; + let index_read = crate::bytes::Reader::new(index); + let index_fields = index.get(..crc_at).ok_or(DecompressError::Truncated)?; + if index.first() != Some(&0) || index_read.u32_le(crc_at) != Some(crate::crc32(index_fields)) { + return Err(DecompressError::Corrupt); + } + let mut records = index.get(1..crc_at).ok_or(DecompressError::Corrupt)?; + let count = xz_vli(&mut records)?; + if count > (records.len() / 2) as u64 { + return Err(DecompressError::Corrupt); + } + let mut block_bytes = 0usize; + for _ in 0..count { + let unpadded = xz_vli(&mut records)?; + if unpadded == 0 { + return Err(DecompressError::Corrupt); + } + let padded = usize::try_from(unpadded.checked_add(3).ok_or(DecompressError::Corrupt)? & !3).map_err(|_| DecompressError::Corrupt)?; + block_bytes = block_bytes.checked_add(padded).filter(|bytes| *bytes <= index_at).ok_or(DecompressError::Corrupt)?; + declared = declared.checked_add(xz_vli(&mut records)?).filter(|size| *size <= max_out).ok_or(DecompressError::TooLarge)?; + } + if records.len() > 3 || records.iter().any(|&byte| byte != 0) { + return Err(DecompressError::Corrupt); + } + let start = index_at.checked_sub(block_bytes).and_then(|at| at.checked_sub(12)).ok_or(DecompressError::Corrupt)?; + let magic_end = start.checked_add(6).ok_or(DecompressError::Corrupt)?; + if input.get(start..magic_end) != Some(b"\xfd7zXZ\0") { + return Err(DecompressError::Corrupt); + } + ranges.push(start..end); + end = start; + } + if ranges.is_empty() { + return Err(DecompressError::Corrupt); + } + Ok(ranges) +} + +#[cfg(feature = "lzma")] +/// XZ variable-length integers have at most nine bytes and no redundant final zero. +fn xz_vli(input: &mut &[u8]) -> Result { + let mut value = 0u64; + for shift in (0..63).step_by(7) { + let (&byte, rest) = input.split_first().ok_or(DecompressError::Truncated)?; + *input = rest; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return if shift != 0 && byte == 0 { Err(DecompressError::Corrupt) } else { Ok(value) }; + } + } + Err(DecompressError::Corrupt) +} + +#[cfg(feature = "lzma")] +/// Decode the property bytes and raw stream of a ZIP method-14 member. +pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, max_out: u64) -> Result, DecompressError> { + let cap = cap_of(max_out).min(isize::MAX as usize); + if expected_size > cap as u64 { + return Err(DecompressError::TooLarge); + } + let (properties, stream) = input.split_at_checked(5).ok_or(DecompressError::Truncated)?; + let [property, d0, d1, d2, d3] = <[u8; 5]>::try_from(properties).map_err(|_| DecompressError::Truncated)?; + let window_limit = u32::try_from(cap).unwrap_or(u32::MAX).max(4096); + let dictionary = u32::from_le_bytes([d0, d1, d2, d3]).max(4096).min(window_limit); + let [d0, d1, d2, d3] = dictionary.to_le_bytes(); + let properties = [property, d0, d1, d2, d3]; + let size = if has_end_marker { None } else { Some(expected_size) }; + let options = lzma_rs::decompress::Options { + unpacked_size: lzma_rs::decompress::UnpackedSize::UseProvided(size), + memlimit: Some(dictionary as usize), + ..Default::default() + }; + let mut out = CapWrite::new(cap); + let mut cursor = io::Cursor::new(properties).chain(io::Cursor::new(stream)); + match lzma_rs::lzma_decompress_with_options(&mut cursor, &mut out, &options) { + Ok(()) if cursor.get_ref().1.position() == stream.len() as u64 => Ok(out.buf), + Ok(()) => Err(DecompressError::Corrupt), + Err(_) if out.exceeded => Err(DecompressError::TooLarge), + Err(lzma_rs::error::Error::HeaderTooShort(_)) => Err(DecompressError::Truncated), + Err(_) => Err(DecompressError::Corrupt), + } +} + +#[cfg(feature = "bzip2")] +/// Decompress a bzip2 stream, producing at most `max_out` bytes. +pub(crate) fn bzip2(input: &[u8], max_out: u64) -> Result, DecompressError> { + read_capped(bzip2_rs::DecoderReader::new(input), cap_of(max_out)) +} + +#[cfg(feature = "zstd")] +/// Decompress a Zstandard stream, producing at most `max_out` bytes. +pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, DecompressError> { + if input.is_empty() { + return Err(DecompressError::Corrupt); + } + let mut out = CapWrite::new(cap_of(max_out)); + while !input.is_empty() { + if matches!(input.get(..4), Some([0x50..=0x5f, 0x2a, 0x4d, 0x18])) { + let header = input.get(4..8).and_then(|s| s.try_into().ok()).ok_or(DecompressError::Truncated)?; + let len = u32::from_le_bytes(header) as usize; + input = input.get(8..).and_then(|body| body.get(len..)).ok_or(DecompressError::Truncated)?; + continue; + } + let mut decoder = ruzstd::decoding::StreamingDecoder::new(&mut input).map_err(|_| DecompressError::Corrupt)?; + if io::copy(&mut decoder, &mut out).is_err() { + return Err(if out.exceeded { DecompressError::TooLarge } else { DecompressError::Corrupt }); + } + } + Ok(out.buf) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testkit as fx; + + #[cfg(feature = "lzma")] + #[test] + fn xz_round_trip() { + let c = fx::XZ_FIXTURE; + assert_eq!(xz(c, 1 << 20).unwrap(), fx::XZ_PLAIN); + // A tiny cap reports TooLarge, never a partial lie. + assert_eq!(xz(c, 4), Err(DecompressError::TooLarge)); + // Truncation never panics. + for len in 0..c.len() { + let _ = xz(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "lzma")] + #[test] + fn xz_concatenated_streams_and_padding_share_one_budget() { + let input = [fx::XZ_FIXTURE, &[0; 4], fx::XZ_FIXTURE, &[0; 8]].concat(); + let expected = [fx::XZ_PLAIN, fx::XZ_PLAIN].concat(); + assert_eq!(xz(&input, expected.len() as u64).unwrap(), expected); + assert_eq!(xz(&input, expected.len() as u64 - 1), Err(DecompressError::TooLarge)); + let mut empty = Vec::new(); + lzma_rs::xz_compress(&mut io::Cursor::new(b""), &mut empty).unwrap(); + let with_empty = [&empty[..], fx::XZ_FIXTURE, &empty[..]].concat(); + assert_eq!(xz(&with_empty, fx::XZ_PLAIN.len() as u64).unwrap(), fx::XZ_PLAIN); + assert!(xz(&empty, 0).unwrap().is_empty()); + } + + #[cfg(feature = "lzma")] + #[test] + fn xz_rejects_incomplete_members_and_nonstandard_padding() { + for padding in [1, 2, 3, 5] { + let input = [fx::XZ_FIXTURE, &vec![0; padding]].concat(); + assert!(xz(&input, 1000).is_err()); + } + assert!(xz(&[&[0; 4][..], fx::XZ_FIXTURE].concat(), 1000).is_err()); + assert!(xz(&[fx::XZ_FIXTURE, b"junk"].concat(), 1000).is_err()); + for missing in 1..fx::XZ_FIXTURE.len() { + let input = [fx::XZ_FIXTURE, &fx::XZ_FIXTURE[..fx::XZ_FIXTURE.len() - missing]].concat(); + assert!(xz(&input, 1000).is_err(), "missing {missing} bytes"); + } + for at in [fx::XZ_FIXTURE.len() - 13, fx::XZ_FIXTURE.len() - 12] { + let mut damaged = fx::XZ_FIXTURE.to_vec(); + damaged[at] ^= 1; + assert!(xz(&damaged, 1000).is_err()); + } + } + + #[cfg(feature = "lzma")] + #[test] + fn xz_index_sizes_and_integer_encodings_are_checked() { + assert_eq!(xz_vli(&mut &[0x80, 0][..]), Err(DecompressError::Corrupt)); + assert_eq!(xz_vli(&mut &[0xff; 10][..]), Err(DecompressError::Corrupt)); + assert_eq!(xz_vli(&mut &[0x81, 1][..]), Ok(129)); + let mut damaged = fx::XZ_FIXTURE.to_vec(); + let footer = damaged.len() - 12; + let backward = u32::from_le_bytes(damaged[footer + 4..footer + 8].try_into().unwrap()); + let index = footer - (backward as usize + 1) * 4; + damaged[index + 2] = 0; + let checksum = crate::crc32(&damaged[index..footer - 4]); + damaged[footer - 4..footer].copy_from_slice(&checksum.to_le_bytes()); + assert!(xz(&damaged, 1000).is_err()); + } + + #[cfg(feature = "bzip2")] + #[test] + fn bzip2_round_trip() { + let c = fx::BZIP2_FIXTURE; + assert_eq!(bzip2(c, 1 << 20).unwrap(), fx::BZIP2_PLAIN); + assert_eq!(bzip2(c, 2), Err(DecompressError::TooLarge)); + for len in 0..c.len() { + let _ = bzip2(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "zstd")] + #[test] + fn zstd_round_trip() { + let c = fx::ZSTD_FIXTURE; + assert_eq!(zstd(c, 1 << 20).unwrap(), fx::ZSTD_PLAIN); + assert_eq!(zstd(c, 2), Err(DecompressError::TooLarge)); + for len in 0..c.len() { + let _ = zstd(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "zstd")] + #[test] + fn zstandard_members_can_contain_multiple_frames() { + let frame = fx::ZSTD_FIXTURE; + let plain = fx::ZSTD_PLAIN; + let skip = [0x50, 0x2a, 0x4d, 0x18, 3, 0, 0, 0, 1, 2, 3]; + let input = [frame, &skip, frame].concat(); + let expected = [plain, plain].concat(); + assert_eq!(zstd(&input, expected.len() as u64).unwrap(), expected); + assert_eq!(zstd(&input, plain.len() as u64), Err(DecompressError::TooLarge)); + assert!(zstd(&[frame, &skip[..9]].concat(), u64::MAX).is_err()); + assert!(zstd(&[frame, &frame[..5]].concat(), u64::MAX).is_err()); + } + + #[test] + fn garbage_never_panics() { + for seed in 0..64u64 { + let n = fx::noise(seed, 400); + #[cfg(feature = "lzma")] + let _ = xz(&n, 1 << 20); + #[cfg(feature = "bzip2")] + let _ = bzip2(&n, 1 << 20); + #[cfg(feature = "zstd")] + let _ = zstd(&n, 1 << 20); + let _ = &n; + } + } +} diff --git a/crates/unzip/src/file.rs b/crates/unzip/src/file.rs new file mode 100644 index 000000000..1bb4a3662 --- /dev/null +++ b/crates/unzip/src/file.rs @@ -0,0 +1,262 @@ +//! One entry read straight from a ZIP file without loading the file: only its end records, its +//! central directory and that entry's bytes are read. For archives too big to hold in memory just +//! to look at one small entry (a JAR's manifest). + +use crate::bytes::Reader; +use crate::{self as zip, ZipError}; +use std::io::{Read, Seek, SeekFrom}; + +/// Most central directory read (some hundreds of thousands of entries); a bigger one is refused. +const MAX_DIRECTORY: u64 = 16 << 20; +/// The ZIP64 end record as `zip::parse` reads it, without any extensible data after it. +const ZIP64_RECORD: u64 = 56; + +/// The entry named `name` (not a folder), decompressed to at most `max_bytes` and checked against +/// its CRC-32. `None`: the archive has no such entry. +pub fn read_named_entry(file: &mut (impl Read + Seek), name: &str, max_bytes: u64) -> Result>, ZipError> { + let len = file.seek(SeekFrom::End(0)).map_err(io)?; + let tail_start = len.saturating_sub(zip::EOCD_SEARCH as u64); + let tail = read_at(file, tail_start, len - tail_start)?; + let eocd = zip::find_eocd(&tail).ok_or(ZipError::NotZip)?; + let mut end = tail.get(eocd..).ok_or(ZipError::Truncated("end record"))?.to_vec(); + let r = Reader::new(&end); + let field = |at| r.u16_le(at).ok_or(ZipError::Truncated("end record")); + let (size, classic_offset) = (r.u32_le(12).ok_or(ZipError::Truncated("end record"))?, r.u32_le(16).ok_or(ZipError::Truncated("end record"))?); + // Any of these saturated makes `zip::parse` read the ZIP64 records, as below. + let saturated = [field(4)?, field(6)?, field(8)?, field(10)?].contains(&0xffff) || size == 0xffff_ffff || classic_offset == 0xffff_ffff; + let end_at = tail_start + eocd as u64; + // A ZIP64 archive puts its own end record and a locator between the directory and the classic + // end record, whether or not the classic fields are saturated. + let locator_at = end_at.checked_sub(20); + let locator = match locator_at { + Some(at) => read_at(file, at, 20)?, + None => Vec::new(), + }; + // Its stated offset doesn't count a prepended stub; it then sits right before the locator. A + // locator signature with no record at either place is other data (an entry's comment). + let record_at = match (locator_at, locator.starts_with(b"PK\x06\x07")) { + (Some(at), true) => { + let declared = Reader::new(&locator).u64_le(8).ok_or(ZipError::Truncated("zip64 locator"))?; + [declared, at.saturating_sub(ZIP64_RECORD)].into_iter().find(|&at| read_at(file, at, 4).is_ok_and(|sig| sig == b"PK\x06\x06")) + } + _ => None, + }; + let zip64 = match record_at { + Some(record_at) => { + let record = read_at(file, record_at, ZIP64_RECORD)?; + let rr = Reader::new(&record); + let size = rr.u64_le(40).ok_or(ZipError::Truncated("zip64 record"))?; + let offset = rr.u64_le(48).ok_or(ZipError::Truncated("zip64 record"))?; + Some((record_at, size, offset, record)) + } + None => None, + }; + // Unsaturated classic fields must agree with a real ZIP64 record, as `zip::parse` requires; + // one that disagrees is other data shaped like it. + let zip64 = zip64.filter(|&(_, z_size, z_offset, _)| saturated || (z_size == u64::from(size) && z_offset == u64::from(classic_offset))); + let (directory_end, size, offset, zip64) = match zip64 { + Some((record_at, size, offset, record)) => (record_at, size, offset, saturated.then_some((record, locator))), + None => (end_at, u64::from(size), u64::from(classic_offset), None), + }; + if size > MAX_DIRECTORY { + return Err(ZipError::TooLarge(MAX_DIRECTORY)); + } + // The directory ends where the end records start; anything before its stated offset is a + // prepended stub, as `zip::parse` allows. + let start = directory_end.checked_sub(size).ok_or(ZipError::Corrupt("directory starts before the file".into()))?; + let prefix = start.checked_sub(offset).ok_or(ZipError::Corrupt("directory sits before its stated offset".into()))?; + // The directory and its end records alone, their offsets rewritten for that buffer. The + // ZIP64 records go along only when the classic fields need them, as `zip::parse` reads them. + let mut directory = read_at(file, start, size)?; + if let Some((mut record, mut locator)) = zip64 { + put(&mut record, 4, &(ZIP64_RECORD - 12).to_le_bytes())?; + put(&mut record, 48, &0u64.to_le_bytes())?; + put(&mut locator, 8, &size.to_le_bytes())?; + directory.extend_from_slice(&record); + directory.extend_from_slice(&locator); + } + // A saturated classic offset stays so: `zip::parse` then takes the rewritten ZIP64 one. + if classic_offset != 0xffff_ffff { + put(&mut end, 16, &[0; 4])?; + } + directory.extend_from_slice(&end); + // `MAX_DIRECTORY` already bounds how many entries there can be. + let archive = zip::parse(&directory, usize::MAX)?; + let Some(mut entry) = archive.entries.into_iter().find(|e| e.name == name && !e.is_dir) else { return Ok(None) }; + if entry.compressed_size > max_bytes.saturating_add(1 << 16) { + return Err(ZipError::TooLarge(max_bytes)); + } + let at = entry.local_header_offset.checked_add(prefix).ok_or(ZipError::Corrupt("local header offset".into()))?; + let header = read_at(file, at, 30)?; + let fields = Reader::new(&header); + let name_len = u64::from(fields.u16_le(26).ok_or(ZipError::Truncated("local header"))?); + let extra_len = u64::from(fields.u16_le(28).ok_or(ZipError::Truncated("local header"))?); + let local_len = (30 + name_len + extra_len).checked_add(entry.compressed_size).ok_or(ZipError::Corrupt("entry size".into()))?; + let local = read_at(file, at, local_len)?; + entry.local_header_offset = 0; + zip::read_entry(&local, &entry, max_bytes).map(Some) +} + +/// Overwrite `bytes` at `at` in a record read from the file. +fn put(record: &mut [u8], at: usize, bytes: &[u8]) -> Result<(), ZipError> { + record.get_mut(at..at + bytes.len()).ok_or(ZipError::Truncated("end record"))?.copy_from_slice(bytes); + Ok(()) +} + +/// Exactly `len` bytes from `at`; a file that ends sooner is truncated. +fn read_at(file: &mut (impl Read + Seek), at: u64, len: u64) -> Result, ZipError> { + file.seek(SeekFrom::Start(at)).map_err(io)?; + let mut out = Vec::new(); + file.take(len).read_to_end(&mut out).map_err(io)?; + if (out.len() as u64) < len { + return Err(ZipError::Truncated("file")); + } + Ok(out) +} + +fn io(e: std::io::Error) -> ZipError { + ZipError::Corrupt(format!("couldn't read it: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + const MANIFEST: &[u8] = b"Manifest-Version: 1.0\r\nMain-Class: app.Main\r\n\r\n"; + + fn read(bytes: &[u8], name: &str) -> Result>, ZipError> { + read_named_entry(&mut Cursor::new(bytes.to_vec()), name, 1 << 20) + } + + /// The same archive with a ZIP64 end record and locator before its classic end record, whose + /// fields at these offsets are saturated (entry counts and disk numbers 2 bytes, others 4). + fn as_zip64(archive: &[u8], saturate: &[usize]) -> Vec { + let end = archive.len() - 22; + let r = Reader::new(archive); + let (entries, size, offset) = + (u64::from(r.u16_le(end + 10).unwrap()), u64::from(r.u32_le(end + 12).unwrap()), u64::from(r.u32_le(end + 16).unwrap())); + let mut out = archive[..end].to_vec(); + let record_at = out.len() as u64; + out.extend_from_slice(b"PK\x06\x06"); + out.extend_from_slice(&44u64.to_le_bytes()); + out.extend_from_slice(&[45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + for v in [entries, entries, size, offset] { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(b"PK\x06\x07\0\0\0\0"); + out.extend_from_slice(&record_at.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + let mut classic = archive[end..].to_vec(); + for &at in saturate { + let width = if at < 12 { 2 } else { 4 }; + classic[at..at + width].fill(0xff); + } + out.extend_from_slice(&classic); + out + } + + #[test] + fn an_entry_is_read_from_the_directory_without_the_whole_file() { + let filler = vec![7u8; 200_000]; + let archive = crate::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + assert_eq!(read(&archive, "META-INF/MISSING").unwrap(), None); + assert_eq!(read(&archive, "META-INF/").unwrap(), None, "a folder isn't an entry to read"); + + let mut stubbed = b"#!/bin/sh\nexec java -jar \"$0\" \"$@\"\n".to_vec(); + stubbed.extend_from_slice(&archive); + assert_eq!(read(&stubbed, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "a launcher stub before the archive"); + + assert_eq!(read(b"not a zip", "META-INF/MANIFEST.MF"), Err(ZipError::NotZip)); + assert!(read(&archive[..archive.len() - 30], "META-INF/MANIFEST.MF").is_err(), "a cut-off file"); + assert_eq!(read_named_entry(&mut Cursor::new(archive), "app/Big.class", 1000), Err(ZipError::TooLarge(1000))); + } + + #[test] + fn a_directory_bigger_than_the_searched_tail_is_read() { + let names: Vec = (0..3000).map(|i| format!("com/example/generated/Class{i:05}.class")).collect(); + let mut files: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b"x"[..])).collect(); + files.push(("META-INF/MANIFEST.MF", MANIFEST)); + let archive = crate::testkit::zip_stored(&files); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// A one-entry archive whose entry is deflated, as `jar` writes a manifest. + fn zip_deflated(name: &str, body: &[u8]) -> Vec { + let packed = miniz_oxide::deflate::compress_to_vec(body, 6); + let fixed = |out: &mut Vec| { + out.extend_from_slice(&[20, 0, 0, 0, 8, 0, 0, 0, 0, 0]); + out.extend_from_slice(&zip::crc32(body).to_le_bytes()); + out.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&[0, 0]); + }; + let mut out = b"PK\x03\x04".to_vec(); + fixed(&mut out); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&packed); + let directory_at = out.len() as u32; + out.extend_from_slice(b"PK\x01\x02\x14\0"); + fixed(&mut out); + out.extend_from_slice(&[0; 14]); + out.extend_from_slice(name.as_bytes()); + let directory_len = out.len() as u32 - directory_at; + out.extend_from_slice(b"PK\x05\x06\0\0\0\0\x01\0\x01\0"); + out.extend_from_slice(&directory_len.to_le_bytes()); + out.extend_from_slice(&directory_at.to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out + } + + #[test] + fn a_deflated_entry_and_a_comment_that_looks_like_an_end_record() { + let deflated = zip_deflated("META-INF/MANIFEST.MF", MANIFEST); + assert_eq!(read(&deflated, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + + let mut commented = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let comment = b"built by PK\x05\x06 tools"; + let at = commented.len() - 2; + commented[at..].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + commented.extend_from_slice(comment); + assert_eq!(read(&commented, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// The last entry's comment ends like a ZIP64 record and locator, saying the directory is huge; + /// the classic fields disagree with it and win, as they do for `zip::parse`. + #[test] + fn a_comment_shaped_like_zip64_records_is_not_read_as_them() { + let mut archive = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let end = archive.len() - 22; + let last = archive.windows(4).rposition(|w| w == b"PK\x01\x02").unwrap(); + let mut fake = b"PK\x06\x06".to_vec(); + fake.extend_from_slice(&44u64.to_le_bytes()); + fake.extend_from_slice(&[0; 28]); + fake.extend_from_slice(&(MAX_DIRECTORY + 1).to_le_bytes()); + fake.extend_from_slice(&0u64.to_le_bytes()); + fake.extend_from_slice(b"PK\x06\x07\0\0\0\0"); + fake.extend_from_slice(&0u64.to_le_bytes()); + fake.extend_from_slice(&1u32.to_le_bytes()); + archive[last + 32..last + 34].copy_from_slice(&(fake.len() as u16).to_le_bytes()); + let size = Reader::new(&archive).u32_le(end + 12).unwrap() + fake.len() as u32; + archive[end + 12..end + 16].copy_from_slice(&size.to_le_bytes()); + archive.splice(end..end, fake); + assert!(zip::parse(&archive, 10).is_ok(), "precondition: a valid archive"); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// ZIP64 end records are read with any classic fields saturated or none (`zip -fz` saturates + /// only the offset), after a stub too. + #[test] + fn zip64_archives_are_read_in_parts() { + let archive = crate::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); + for saturate in [&[][..], &[8, 10, 12, 16], &[16], &[12], &[10], &[4]] { + let zip64 = as_zip64(&archive, saturate); + assert_eq!(read(&zip64, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}"); + let mut stubbed = b"#!/bin/sh\n".to_vec(); + stubbed.extend_from_slice(&zip64); + assert_eq!(read(&stubbed, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}, after a stub"); + } + } +} diff --git a/crates/unzip/src/lib.rs b/crates/unzip/src/lib.rs new file mode 100644 index 000000000..35dfbbfcf --- /dev/null +++ b/crates/unzip/src/lib.rs @@ -0,0 +1,549 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…): central directory listing and bounded entry reads. +//! +//! Only the central directory is trusted for the listing. Entry data is located through the local +//! header, decompressed with a hard output cap, and checked against the recorded CRC-32. Stored, +//! DEFLATE, bzip2, ZIP LZMA, XZ and Zstandard entries are supported; other methods and encrypted entries are listed. +//! +//! The bzip2, LZMA/XZ and Zstandard methods sit behind the default `bzip2`, `lzma` and `zstd` +//! features; without one, entries using it are listed and reading them is +//! [`ZipError::Unsupported`]. [`file::read_named_entry`] reads one entry from a seekable file +//! without loading the archive. With the `serde` feature the listing types are `Serialize`. +//! +//! From Craft Anything (`crates/formats/src/zip.rs` and `zip_file.rs`), with its hardening: +//! directory and extra-field bounds, ZIP64 end-record recovery after a prepended stub, refusal of +//! multi-volume archives, Info-ZIP Unicode Path names checked by CRC, decoded sizes checked +//! against the directory, and CRC-32 verification. + +mod bytes; +#[cfg(any(feature = "bzip2", feature = "lzma", feature = "zstd"))] +mod decode; +pub mod file; + +use crate::bytes::Reader; + +/// Biggest tail we search for the end-of-central-directory record (64 KiB comment + record). +pub const EOCD_SEARCH: usize = 22 + 0xffff; +const ZIP64_FIXED_SIZE: usize = 56; +/// Bound the search for a record shifted by a prepended stub, including extensible data. +const ZIP64_RECOVERY_SEARCH: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ZipError { + #[error("not a ZIP archive (no end-of-central-directory record)")] + NotZip, + #[error("ZIP is truncated: {0}")] + Truncated(&'static str), + #[error("ZIP is corrupt: {0}")] + Corrupt(String), + #[error("unsupported ZIP feature: {0}")] + Unsupported(String), + #[error("entry is larger than the {0}-byte limit")] + TooLarge(u64), + #[error("CRC-32 mismatch (expected {expected:08x}, found {found:08x})")] + CrcMismatch { expected: u32, found: u32 }, +} + +/// One central-directory entry. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct ZipEntry { + /// Entry name as stored (forward slashes). Bit 11 selects lossy UTF-8; otherwise + /// a valid Unicode Path extra field takes precedence over the UTF-8/CP437 fallback. + pub name: String, + pub method: u16, + pub flags: u16, + pub crc32: u32, + pub compressed_size: u64, + pub uncompressed_size: u64, + /// Offset of the local header, already adjusted for any prepended data. + pub local_header_offset: u64, + pub is_dir: bool, + pub encrypted: bool, + /// Unix mode from the external attributes (when stored by Unix or macOS). + #[cfg_attr(feature = "serde", serde(skip))] + pub unix_mode: Option, +} + +impl ZipEntry { + /// Why this entry is not a regular file or directory (symlink, device, FIFO, socket), if so. + pub fn special_kind(&self) -> Option<&'static str> { + match self.unix_mode? & 0o170000 { + 0o120000 => Some("symlink"), + 0o020000 => Some("character device"), + 0o060000 => Some("block device"), + 0o010000 => Some("FIFO"), + 0o140000 => Some("socket"), + _ => None, + } + } + + /// Human name of the compression method. + pub fn method_name(&self) -> &'static str { + match self.method { + 0 => "stored", + 8 => "deflate", + 9 => "deflate64", + 12 => "bzip2", + 14 => "lzma", + 93 => "zstd", + 95 => "xz", + 99 => "aes", + _ => "other", + } + } +} + +/// The parsed central directory. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct ZipArchive { + pub entries: Vec, + /// Entries the directory declares (may exceed `entries.len()` when truncated). + pub declared_entries: u64, + /// True when `max_entries` stopped the listing early. + pub truncated: bool, + pub zip64: bool, + /// Bytes before the archive proper (self-extracting stubs). + pub prefix_bytes: u64, + /// Inconsistent end-record metadata retained without hiding a readable directory. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Vec::is_empty"))] + pub warnings: Vec, +} + +/// True when the bytes start like a ZIP (local header or empty archive). +pub fn looks_like_zip(data: &[u8]) -> bool { + data.starts_with(b"PK\x03\x04") || data.starts_with(b"PK\x05\x06") || data.starts_with(b"PK\x07\x08") +} + +/// Recover a shifted end record only when its fixed fields still describe the classic +/// directory. Longer records must frame their extension exactly; the adjacent fixed +/// record can still report an inconsistent size through the normal warning path. +fn recover_zip64_record(data: &[u8], loc: usize, disk_entries: u64, total: u64, cd_size: u64, cd_offset: u64) -> Option { + let r = Reader::new(data); + let last = loc.checked_sub(ZIP64_FIXED_SIZE)?; + let first = loc.saturating_sub(ZIP64_RECOVERY_SEARCH); + let candidates = data.get(first..last.checked_add(4)?)?; + candidates.windows(4).enumerate().rev().find_map(|(relative, signature)| { + if signature != b"PK\x06\x06" { + return None; + } + let rec = first.checked_add(relative)?; + let fixed = Reader::new(r.slice(rec, ZIP64_FIXED_SIZE)?); + let payload_size = fixed.u64_le(4)?; + if payload_size < 44 { + return None; + } + let declared_end = (rec as u64).checked_add(12)?.checked_add(payload_size)?; + if rec != last && declared_end != loc as u64 { + return None; + } + let record_disk_entries = fixed.u64_le(24)?; + let record_total = fixed.u64_le(32)?; + let record_cd_size = fixed.u64_le(40)?; + let record_cd_offset = fixed.u64_le(48)?; + if fixed.u32_le(16)? != 0 + || fixed.u32_le(20)? != 0 + || record_disk_entries != record_total + || (disk_entries != 0xffff && disk_entries != record_disk_entries) + || (total != 0xffff && total != record_total) + || (cd_size != 0xffff_ffff && cd_size != record_cd_size) + || (cd_offset != 0xffff_ffff && cd_offset != record_cd_offset) + || record_cd_offset.checked_add(record_cd_size)? > rec as u64 + { + return None; + } + if record_total > 0 { + let directory = rec.checked_sub(usize::try_from(record_cd_size).ok()?)?; + if record_cd_size < 46 || r.slice(directory, 4) != Some(b"PK\x01\x02") { + return None; + } + } + Some(rec) + }) +} + +/// Parse the central directory, keeping at most `max_entries`. +pub fn parse(data: &[u8], max_entries: usize) -> Result { + parse_directory(data, max_entries, None) +} + +/// Search the validated directory while retaining only the requested entry. +pub fn has_entry(data: &[u8], name: &str) -> bool { + parse_directory(data, 1, Some(name)).is_ok_and(|archive| !archive.entries.is_empty()) +} + +/// List directory entries, optionally retaining only a matching decoded name. +fn parse_directory(data: &[u8], max_entries: usize, name_filter: Option<&str>) -> Result { + let r = Reader::new(data); + let eocd = find_eocd(data).ok_or(ZipError::NotZip)?; + let mut disk = u32::from(r.u16_le(eocd + 4).ok_or(ZipError::Truncated("end record"))?); + let mut directory_disk = u32::from(r.u16_le(eocd + 6).ok_or(ZipError::Truncated("end record"))?); + let mut disk_entries = u64::from(r.u16_le(eocd + 8).ok_or(ZipError::Truncated("end record"))?); + if !matches!(disk, 0 | 0xffff) || !matches!(directory_disk, 0 | 0xffff) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let mut total = u64::from(r.u16_le(eocd + 10).ok_or(ZipError::Truncated("end record"))?); + let mut cd_size = u64::from(r.u32_le(eocd + 12).ok_or(ZipError::Truncated("end record"))?); + let mut cd_offset = u64::from(r.u32_le(eocd + 16).ok_or(ZipError::Truncated("end record"))?); + let mut zip64 = false; + let mut warnings = Vec::new(); + let mut eocd_record_start = eocd as u64; + if total == 0xffff || disk_entries == 0xffff || disk == 0xffff || directory_disk == 0xffff || cd_size == 0xffff_ffff || cd_offset == 0xffff_ffff { + // ZIP64 locator sits right before the classic record. + if let Some(loc) = eocd.checked_sub(20) + && r.slice(loc, 4) == Some(b"PK\x06\x07") + { + if r.u32_le(loc + 4) != Some(0) || r.u32_le(loc + 16) != Some(1) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let rec = r.u64_le(loc + 8).ok_or(ZipError::Truncated("zip64 locator"))?; + let rec = usize::try_from(rec).ok().filter(|&rec| r.slice(rec, 4) == Some(b"PK\x06\x06")); + let rec = rec.or_else(|| { + let recovered = recover_zip64_record(data, loc, disk_entries, total, cd_size, cd_offset); + if recovered.is_some() { + warnings.push("ZIP64 locator offset does not identify its end record; recovered the record before the locator".into()); + } + recovered + }); + if let Some(rec) = rec { + let payload_size = r.u64_le(rec + 4).ok_or(ZipError::Truncated("zip64 record size"))?; + if payload_size < 44 { + return Err(ZipError::Corrupt("ZIP64 end record is smaller than its fixed fields".into())); + } + let declared_end = (rec as u64) + .checked_add(12) + .and_then(|start| start.checked_add(payload_size)) + .ok_or_else(|| ZipError::Corrupt("ZIP64 end record extent overflows".into()))?; + // Fixed fields must be present before the locator. A bad declared extension + // length cannot make the reader borrow locator or classic end-record bytes. + if rec.checked_add(ZIP64_FIXED_SIZE).is_none_or(|end| end > loc) { + return Err(ZipError::Truncated("zip64 record fixed fields")); + } + if declared_end != loc as u64 { + warnings.push("ZIP64 end record's declared extent does not end at its locator; using bounded fixed fields".into()); + } + disk = r.u32_le(rec + 16).ok_or(ZipError::Truncated("zip64 record"))?; + directory_disk = r.u32_le(rec + 20).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_disk_entries = r.u64_le(rec + 24).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_total = r.u64_le(rec + 32).ok_or(ZipError::Truncated("zip64 record"))?; + if (disk_entries != 0xffff && disk_entries != zip64_disk_entries) || (total != 0xffff && total != zip64_total) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + disk_entries = zip64_disk_entries; + total = zip64_total; + let zip64_cd_size = r.u64_le(rec + 40).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_cd_offset = r.u64_le(rec + 48).ok_or(ZipError::Truncated("zip64 record"))?; + if (cd_size != 0xffff_ffff && cd_size != zip64_cd_size) || (cd_offset != 0xffff_ffff && cd_offset != zip64_cd_offset) { + return Err(ZipError::Corrupt("ZIP64 and classic directory bounds disagree".into())); + } + cd_size = zip64_cd_size; + cd_offset = zip64_cd_offset; + zip64 = true; + eocd_record_start = rec as u64; + } else { + // A prepended stub can invalidate the locator's absolute offset. Keep the + // classic-directory fallback, including its existing prefix adjustment. + warnings.push("ZIP64 locator does not point to an end record; using the classic end record".into()); + } + } + } + if disk != 0 || directory_disk != 0 || disk_entries != total { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + // Self-extracting archives: the directory sits later than it claims by the stub length. + let expected_end = cd_offset.checked_add(cd_size).ok_or_else(|| ZipError::Corrupt("directory bounds overflow".into()))?; + let prefix = eocd_record_start.saturating_sub(expected_end); + let start = usize::try_from(cd_offset.saturating_add(prefix)).map_err(|_| ZipError::Corrupt("directory offset".into()))?; + let directory_end = (start as u64) + .checked_add(cd_size) + .filter(|&end| end <= eocd_record_start) + .and_then(|end| usize::try_from(end).ok()) + .ok_or_else(|| ZipError::Corrupt("directory extends into its end record".into()))?; + let mut entries = Vec::new(); + let mut pos = start; + let mut truncated = false; + let mut seen = 0u64; + while seen < total { + if entries.len() >= max_entries { + truncated = true; + break; + } + if pos.checked_add(46).is_none_or(|end| end > directory_end) { + return Err(ZipError::Truncated("directory entry")); + } + if r.slice(pos, 4) != Some(b"PK\x01\x02") { + if entries.is_empty() { + return Err(ZipError::Corrupt(format!("no central directory header at offset {pos}"))); + } + truncated = true; + break; + } + let flags = r.u16_le(pos + 8).ok_or(ZipError::Truncated("directory entry"))?; + let method = r.u16_le(pos + 10).ok_or(ZipError::Truncated("directory entry"))?; + let crc32 = r.u32_le(pos + 16).ok_or(ZipError::Truncated("directory entry"))?; + let mut csize = u64::from(r.u32_le(pos + 20).ok_or(ZipError::Truncated("directory entry"))?); + let mut usize_ = u64::from(r.u32_le(pos + 24).ok_or(ZipError::Truncated("directory entry"))?); + let name_len = usize::from(r.u16_le(pos + 28).ok_or(ZipError::Truncated("directory entry"))?); + let extra_len = usize::from(r.u16_le(pos + 30).ok_or(ZipError::Truncated("directory entry"))?); + let comment_len = usize::from(r.u16_le(pos + 32).ok_or(ZipError::Truncated("directory entry"))?); + let entry_end = pos + .checked_add(46 + name_len + extra_len + comment_len) + .filter(|&end| end <= directory_end) + .ok_or(ZipError::Truncated("directory entry fields"))?; + let host = r.slice(pos + 5, 1).and_then(|b| b.first().copied()).unwrap_or(0); + let attrs = r.u32_le(pos + 38).ok_or(ZipError::Truncated("entry attributes"))?; + let unix_mode = matches!(host, 3 | 19).then_some(attrs >> 16); + let mut local = u64::from(r.u32_le(pos + 42).ok_or(ZipError::Truncated("directory entry"))?); + let mut entry_disk = u32::from(r.u16_le(pos + 34).ok_or(ZipError::Truncated("directory entry"))?); + let name_bytes = r.slice(pos + 46, name_len).ok_or(ZipError::Truncated("entry name"))?; + let extra = r.slice(pos + 46 + name_len, extra_len).ok_or(ZipError::Truncated("entry extra fields"))?; + // Each extra field owns its payload: ZIP64 values cannot borrow a following field's bytes. + let needs_zip64 = usize_ == 0xffff_ffff || csize == 0xffff_ffff || local == 0xffff_ffff || entry_disk == 0xffff; + let mut found_zip64 = false; + let mut unicode_name = None; + let mut name_crc = None; + let mut x = 0usize; + let er = Reader::new(extra); + // Trailing bytes too short for a field header are padding, not truncation. + while x.saturating_add(4) <= extra.len() { + let id = er.u16_le(x).ok_or(ZipError::Truncated("extra field header"))?; + let len = usize::from(er.u16_le(x + 2).ok_or(ZipError::Truncated("extra field header"))?); + let body = x + 4; + let field = er.slice(body, len).ok_or(ZipError::Truncated("extra field payload"))?; + if id == 0x0001 && !found_zip64 { + let values = Reader::new(field); + let mut f = 0; + if usize_ == 0xffff_ffff { + usize_ = values.u64_le(f).ok_or(ZipError::Truncated("zip64 uncompressed size"))?; + f += 8; + } + if csize == 0xffff_ffff { + csize = values.u64_le(f).ok_or(ZipError::Truncated("zip64 compressed size"))?; + f += 8; + } + if local == 0xffff_ffff { + local = values.u64_le(f).ok_or(ZipError::Truncated("zip64 local header offset"))?; + f += 8; + } + if entry_disk == 0xffff { + entry_disk = values.u32_le(f).ok_or(ZipError::Truncated("zip64 entry disk"))?; + } + found_zip64 = true; + } else if id == 0x7075 && flags & (1 << 11) == 0 && unicode_name.is_none() { + let field = Reader::new(field); + if field.u8(0) == Some(1) && field.u32_le(1) == Some(*name_crc.get_or_insert_with(|| self::crc32(name_bytes))) { + unicode_name = field.data().get(5..).and_then(|bytes| std::str::from_utf8(bytes).ok()).filter(|s| !s.is_empty()); + } + } + x = body + len; + } + if needs_zip64 && !found_zip64 { + return Err(ZipError::Corrupt("missing ZIP64 extended information".into())); + } + if entry_disk != 0 { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let name = unicode_name.map(str::to_owned).unwrap_or_else(|| decode_name(name_bytes, flags)).replace('\\', "/"); + if name_filter.is_none_or(|wanted| wanted == name) { + entries.push(ZipEntry { + is_dir: name.ends_with('/') + || unix_mode.is_some_and(|m| m & 0o170000 == 0o040000) + || (matches!(host, 0 | 10 | 14) && attrs & 0x10 != 0), + name, + method, + flags, + crc32, + compressed_size: csize, + uncompressed_size: usize_, + local_header_offset: local.saturating_add(prefix), + encrypted: flags & 1 != 0, + unix_mode, + }); + } + seen += 1; + pos = entry_end; + } + Ok(ZipArchive { entries, declared_entries: total, truncated, zip64, prefix_bytes: prefix, warnings }) +} + +fn decode_name(bytes: &[u8], flags: u16) -> String { + if flags & 0x0800 != 0 || bytes.is_ascii() { + return String::from_utf8_lossy(bytes).into_owned(); + } + if let Ok(name) = std::str::from_utf8(bytes) { + return name.to_owned(); + } + const CP437: [char; 128] = [ + 'Ç', 'ü', 'é', 'â', 'ä', 'à', 'å', 'ç', 'ê', 'ë', 'è', 'ï', 'î', 'ì', 'Ä', 'Å', 'É', 'æ', 'Æ', 'ô', 'ö', 'ò', 'û', 'ù', 'ÿ', 'Ö', 'Ü', '¢', + '£', '¥', '₧', 'ƒ', 'á', 'í', 'ó', 'ú', 'ñ', 'Ñ', 'ª', 'º', '¿', '⌐', '¬', '½', '¼', '¡', '«', '»', '░', '▒', '▓', '│', '┤', '╡', '╢', '╖', + '╕', '╣', '║', '╗', '╝', '╜', '╛', '┐', '└', '┴', '┬', '├', '─', '┼', '╞', '╟', '╚', '╔', '╩', '╦', '╠', '═', '╬', '╧', '╨', '╤', '╥', '╙', + '╘', '╒', '╓', '╫', '╪', '┘', '┌', '█', '▄', '▌', '▐', '▀', 'α', 'ß', 'Γ', 'π', 'Σ', 'σ', 'µ', 'τ', 'Φ', 'Θ', 'Ω', 'δ', '∞', 'φ', 'ε', '∩', + '≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∙', '·', '√', 'ⁿ', '²', '■', '\u{00a0}', + ]; + bytes + .iter() + .map(|&byte| if byte.is_ascii() { char::from(byte) } else { CP437.get(usize::from(byte - 0x80)).copied().unwrap_or('\u{fffd}') }) + .collect() +} + +/// Where the end-of-central-directory record starts, searching backwards through the last +/// [`EOCD_SEARCH`] bytes. +pub fn find_eocd(data: &[u8]) -> Option { + if data.len() < 22 { + return None; + } + let from = data.len().saturating_sub(EOCD_SEARCH); + let tail = data.get(from..)?; + // Search backwards so a stray signature in the comment can't shadow the real record. + let mut i = tail.len().checked_sub(22)?; + let mut found = None; + let mut last_signature = None; + loop { + if tail.get(i..i + 4) == Some(b"PK\x05\x06") { + last_signature.get_or_insert(from + i); + if let Some(comment_len) = tail.get(i + 20..i + 22).and_then(|bytes| <[u8; 2]>::try_from(bytes).ok()).map(u16::from_le_bytes) + && i + 22 + usize::from(comment_len) == tail.len() + { + found = Some(from + i); + } + } + if i == 0 { + return found.or(last_signature); + } + i -= 1; + } +} + +/// The raw (possibly compressed) bytes of an entry. +pub fn raw_entry_data<'a>(data: &'a [u8], e: &ZipEntry) -> Result<&'a [u8], ZipError> { + let r = Reader::new(data); + let off = usize::try_from(e.local_header_offset).map_err(|_| ZipError::Corrupt("local header offset".into()))?; + if r.slice(off, 4) != Some(b"PK\x03\x04") { + return Err(ZipError::Corrupt(format!("no local header for {}", e.name))); + } + let n = usize::from(r.u16_le(off + 26).ok_or(ZipError::Truncated("local header"))?); + let x = usize::from(r.u16_le(off + 28).ok_or(ZipError::Truncated("local header"))?); + let start = off.checked_add(30 + n + x).ok_or(ZipError::Truncated("local header"))?; + let len = usize::try_from(e.compressed_size).map_err(|_| ZipError::TooLarge(e.compressed_size))?; + r.slice(start, len).ok_or(ZipError::Truncated("entry data")) +} + +/// Read and decompress one entry, refusing to produce more than `max_bytes`. +pub fn read_entry(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let out = read_entry_unverified(data, e, max_bytes)?; + let found = crc32(&out); + if found != e.crc32 { + return Err(ZipError::CrcMismatch { expected: e.crc32, found }); + } + Ok(out) +} + +/// Like [`read_entry`] but without comparing the stored CRC-32, so a caller that accepts a mismatch can still get the bytes. +pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + if e.encrypted { + return Err(ZipError::Unsupported(format!("{} is encrypted", e.name))); + } + if e.uncompressed_size > max_bytes { + return Err(ZipError::TooLarge(max_bytes)); + } + let raw = raw_entry_data(data, e)?; + let cap = usize::try_from(max_bytes).unwrap_or(usize::MAX); + let out = match e.method { + 0 if raw.len() as u64 > max_bytes => return Err(ZipError::TooLarge(max_bytes)), + 0 => raw.to_vec(), + 8 => miniz_oxide::inflate::decompress_to_vec_with_limit(raw, cap).map_err(|err| match err.status { + miniz_oxide::inflate::TINFLStatus::HasMoreOutput => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("deflate stream for {} is damaged", e.name)), + })?, + 12 | 95 => decode_bzip2_or_xz(raw, e, max_bytes)?, + 14 => decode_lzma(raw, e, max_bytes)?, + 93 => decode_zstd(raw, e, max_bytes)?, + m => return Err(ZipError::Unsupported(format!("compression method {m} ({}) for {}", e.method_name(), e.name))), + }; + if out.len() as u64 != e.uncompressed_size { + return Err(ZipError::Corrupt(format!( + "decoded size for {} is {}, but its directory entry declares {}", + e.name, + out.len(), + e.uncompressed_size + ))); + } + Ok(out) +} + +#[cfg(any(feature = "bzip2", feature = "lzma"))] +fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let decoded = match e.method { + #[cfg(feature = "bzip2")] + 12 => decode::bzip2(raw, max_bytes), + #[cfg(feature = "lzma")] + 95 => decode::xz(raw, max_bytes), + _ => return Err(not_built(e)), + }; + decoded.map_err(|error| match error { + decode::DecompressError::TooLarge => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("{} stream for {} is damaged: {error}", e.method_name(), e.name)), + }) +} + +#[cfg(not(any(feature = "bzip2", feature = "lzma")))] +fn decode_bzip2_or_xz(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +#[cfg(feature = "lzma")] +fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let header = Reader::new(raw); + let properties_size = header.u16_le(2).ok_or(ZipError::Truncated("LZMA properties header"))?; + if properties_size != 5 { + return Err(ZipError::Unsupported(format!("LZMA properties size {properties_size} for {}", e.name))); + } + let stream = raw.get(4..).ok_or(ZipError::Truncated("LZMA properties"))?; + decode::lzma_zip(stream, e.uncompressed_size, e.flags & 2 != 0, max_bytes).map_err(|error| match error { + decode::DecompressError::TooLarge => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("LZMA stream for {} is damaged: {error}", e.name)), + }) +} + +#[cfg(not(feature = "lzma"))] +fn decode_lzma(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +#[cfg(feature = "zstd")] +fn decode_zstd(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + decode::zstd(raw, max_bytes).map_err(|err| ZipError::Corrupt(format!("zstd stream for {} is damaged: {err}", e.name))) +} + +#[cfg(not(feature = "zstd"))] +fn decode_zstd(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +/// An entry whose method this build leaves out (see the crate features). +#[cfg_attr(all(feature = "bzip2", feature = "lzma", feature = "zstd"), allow(dead_code))] +fn not_built(e: &ZipEntry) -> ZipError { + ZipError::Unsupported(format!("compression method {} ({}) for {} is not built in", e.method, e.method_name(), e.name)) +} + +/// CRC-32 (IEEE, as used by ZIP and PNG). +pub fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + for &b in data { + crc ^= u32::from(b); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc +} + +#[cfg(test)] +mod testkit; + +#[cfg(test)] +mod tests; diff --git a/crates/unzip/src/testkit.rs b/crates/unzip/src/testkit.rs new file mode 100644 index 000000000..ae55cc052 --- /dev/null +++ b/crates/unzip/src/testkit.rs @@ -0,0 +1,80 @@ +//! Test fixtures from Craft Anything's testkit: known-good compressed streams built with the +//! reference encoders (`xz`, `bzip2`, `zstd`), a stored-only ZIP writer and a noise generator. + +#![allow(clippy::cast_possible_truncation, dead_code)] + +use crate::crc32; + +/// Plaintext of [`XZ_FIXTURE`] and [`LZMA_FIXTURE`]. +pub const XZ_PLAIN: &[u8] = b"Hello, xz compression world!\n"; +/// An xz stream of [`XZ_PLAIN`]. +pub const XZ_FIXTURE: &[u8] = &[ + 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, + 0x01, 0x00, 0x1c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x78, 0x7a, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x5f, 0x47, 0xab, 0x2a, 0x4c, 0xd2, 0xb2, 0x00, 0x01, 0x35, 0x1d, + 0x44, 0x1b, 0xb1, 0xe1, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x04, 0x59, 0x5a, +]; +/// Plaintext of [`BZIP2_FIXTURE`]. +pub const BZIP2_PLAIN: &[u8] = b"Hello, bzip2!\n"; +/// A bzip2 stream of [`BZIP2_PLAIN`]. +pub const BZIP2_FIXTURE: &[u8] = &[ + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0x04, 0xad, 0x5a, 0x6b, 0x00, 0x00, 0x02, 0xdd, 0x80, 0x00, 0x10, 0x60, 0x04, 0x10, + 0x00, 0x00, 0x40, 0x12, 0x24, 0xc0, 0x10, 0x20, 0x00, 0x22, 0x00, 0x03, 0x42, 0x01, 0xa0, 0x05, 0x67, 0xde, 0x89, 0xa1, 0xa0, 0x80, 0xf1, 0x77, + 0x24, 0x53, 0x85, 0x09, 0x00, 0x4a, 0xd5, 0xa6, 0xb0, +]; +/// Plaintext of [`ZSTD_FIXTURE`]. +pub const ZSTD_PLAIN: &[u8] = b"Hello, zstd!\n"; +/// A Zstandard frame of [`ZSTD_PLAIN`]. +pub const ZSTD_FIXTURE: &[u8] = &[ + 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x0d, 0x69, 0x00, 0x00, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x7a, 0x73, 0x74, 0x64, 0x21, 0x0a, 0x41, 0xb4, + 0x53, 0xb2, +]; + +/// A stored-only ZIP of `files`. +pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + let len = body.len() as u32; + out.extend_from_slice(b"PK\x03\x04\x14\0\0\0\0\0\0\0\0\0"); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02\x14\0\x14\0\0\0\0\0\0\0\0\0"); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops. +pub fn noise(seed: u64, len: usize) -> Vec { + let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() +} diff --git a/crates/unzip/src/tests.rs b/crates/unzip/src/tests.rs new file mode 100644 index 000000000..3ea92afe4 --- /dev/null +++ b/crates/unzip/src/tests.rs @@ -0,0 +1,393 @@ +//! The reader's tests, from Craft Anything's `zip.rs`. + +use super::*; + +/// Minimal stored-only ZIP writer for tests. +pub(crate) fn build(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + out.extend_from_slice(b"PK\x03\x04"); + out.extend_from_slice(&[20, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02"); + cd.extend_from_slice(&[20, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&(body.len() as u32).to_le_bytes()); + cd.extend_from_slice(&(body.len() as u32).to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +#[cfg(all(feature = "bzip2", feature = "lzma"))] +fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { + let mut file = build(&[("entry", compressed)]); + let central = 30 + "entry".len() + compressed.len(); + let version: u16 = if method == 12 { 46 } else { 63 }; + file[4..6].copy_from_slice(&version.to_le_bytes()); + file[8..10].copy_from_slice(&method.to_le_bytes()); + file[14..18].copy_from_slice(&crc32(plain).to_le_bytes()); + file[22..26].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + file[central + 6..central + 8].copy_from_slice(&version.to_le_bytes()); + file[central + 10..central + 12].copy_from_slice(&method.to_le_bytes()); + file[central + 16..central + 20].copy_from_slice(&crc32(plain).to_le_bytes()); + file[central + 24..central + 28].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + file +} + +#[cfg(all(feature = "bzip2", feature = "lzma"))] +#[test] +fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { + use crate::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; + for (method, compressed, plain) in [(12, BZIP2_FIXTURE, BZIP2_PLAIN), (95, XZ_FIXTURE, XZ_PLAIN)] { + let file = build_codec_entry(method, compressed, plain); + let original = parse(&file, 1).unwrap().entries.remove(0); + assert_eq!(read_entry(&file, &original, plain.len() as u64).unwrap(), plain); + let mut entry = original.clone(); + entry.crc32 ^= 1; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::CrcMismatch { .. }))); + assert_eq!(read_entry_unverified(&file, &entry, 100).unwrap(), plain); + entry = original.clone(); + entry.uncompressed_size = 0; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::Corrupt(_)))); + assert!(matches!(read_entry(&file, &entry, plain.len() as u64 - 1), Err(ZipError::TooLarge(_)))); + let mut damaged = file.clone(); + damaged[30 + "entry".len()] ^= 0xff; + assert!(matches!(read_entry(&damaged, &original, 100), Err(ZipError::Corrupt(_)))); + entry = original; + entry.encrypted = true; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::Unsupported(_)))); + } +} + +#[cfg(feature = "lzma")] +fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { + let mut encoded = Vec::new(); + let size = if end_marker { None } else { Some(plain.len() as u64) }; + let options = lzma_rs::compress::Options { unpacked_size: lzma_rs::compress::UnpackedSize::WriteToHeader(size) }; + lzma_rs::lzma_compress_with_options(&mut std::io::Cursor::new(plain), &mut encoded, &options).unwrap(); + let mut compressed = vec![9, 4, 5, 0]; + compressed.extend_from_slice(&encoded[..5]); + compressed.extend_from_slice(&encoded[13..]); + let mut file = build(&[("entry", &compressed)]); + let central = 30 + "entry".len() + compressed.len(); + let flags: u16 = if end_marker { 2 } else { 0 }; + for (version_at, flags_at, method_at, crc_at, size_at) in + [(4, 6, 8, 14, 22), (central + 6, central + 8, central + 10, central + 16, central + 24)] + { + file[version_at..version_at + 2].copy_from_slice(&63u16.to_le_bytes()); + file[flags_at..flags_at + 2].copy_from_slice(&flags.to_le_bytes()); + file[method_at..method_at + 2].copy_from_slice(&14u16.to_le_bytes()); + file[crc_at..crc_at + 4].copy_from_slice(&crc32(plain).to_le_bytes()); + file[size_at..size_at + 4].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + } + file +} + +#[cfg(feature = "lzma")] +#[test] +fn zip_lzma_reads_size_delimited_and_end_marked_streams() { + for end_marker in [false, true] { + for plain in [&b"ZIP LZMA contents"[..], &[0; 23][..]] { + let file = zip_lzma_member(plain, end_marker); + let original = parse(&file, 1).unwrap().entries.remove(0); + assert_eq!(read_entry(&file, &original, plain.len() as u64).unwrap(), plain); + assert!(matches!(read_entry(&file, &original, plain.len() as u64 - 1), Err(ZipError::TooLarge(_)))); + let mut entry = original.clone(); + entry.crc32 ^= 1; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::CrcMismatch { .. }))); + entry = original.clone(); + entry.uncompressed_size -= 1; + assert!(read_entry(&file, &entry, 100).is_err()); + entry = original.clone(); + entry.compressed_size -= 1; + assert!(read_entry(&file, &entry, 100).is_err()); + entry = original; + entry.compressed_size += 1; + assert!(read_entry(&file, &entry, 100).is_err()); + } + } +} + +#[cfg(feature = "lzma")] +#[test] +fn zip_lzma_rejects_bad_properties_without_unbounded_dictionaries() { + let mut file = zip_lzma_member(b"small", true); + let entry = parse(&file, 1).unwrap().entries.remove(0); + let body = 30 + "entry".len(); + file[body + 5..body + 9].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(read_entry(&file, &entry, 5).unwrap(), b"small"); + file[body + 4] = 255; + assert!(read_entry(&file, &entry, 5).is_err()); + file[body + 2..body + 4].copy_from_slice(&6u16.to_le_bytes()); + assert!(matches!(read_entry(&file, &entry, 5), Err(ZipError::Unsupported(_)))); +} + +fn build_named_entry(name: &[u8], flags: u16) -> Vec { + let mut zip = build(&[(&"x".repeat(name.len()), b"")]); + let central = 30 + name.len(); + zip[6..8].copy_from_slice(&flags.to_le_bytes()); + zip[30..central].copy_from_slice(name); + zip[central + 8..central + 10].copy_from_slice(&flags.to_le_bytes()); + zip[central + 46..central + 46 + name.len()].copy_from_slice(name); + zip +} + +#[test] +fn verified_unicode_path_overrides_a_legacy_zip_name() { + let raw = b"notes.txt"; + let unicode = "笔记.txt"; + for (flags, version, checksum, expected) in + [(0, 1, crc32(raw), unicode), (0, 1, crc32(b"old-name"), "notes.txt"), (0, 2, crc32(raw), "notes.txt"), (1 << 11, 1, crc32(raw), "notes.txt")] + { + let mut zip = build_named_entry(raw, flags); + let central = 30 + raw.len(); + let end = central + 46 + raw.len(); + let mut extra = 0x7075_u16.to_le_bytes().to_vec(); + extra.extend_from_slice(&((5 + unicode.len()) as u16).to_le_bytes()); + extra.push(version); + extra.extend_from_slice(&checksum.to_le_bytes()); + extra.extend_from_slice(unicode.as_bytes()); + zip[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + zip[end + 12..end + 16].copy_from_slice(&((end - central + extra.len()) as u32).to_le_bytes()); + zip.splice(end..end, extra); + assert_eq!(parse(&zip, 1).unwrap().entries[0].name, expected); + } +} + +#[test] +fn directory_markers_do_not_require_a_trailing_slash() { + let sample = build(&[("folder", b"")]); + let central = 30 + "folder".len(); + for (host, attrs, directory, special) in [ + (0, 0x10u32, true, None), + (10, 0x10, true, None), + (3, 0o040755 << 16, true, None), + (3, 0o120777 << 16, false, Some("symlink")), + (0, 0x20, false, None), + (14, 0x10, true, None), + (14, 0x20, false, None), + (19, 0o040755 << 16, true, None), + (19, 0o100644 << 16, false, None), + (19, 0o120777 << 16, false, Some("symlink")), + (42, 0x10, false, None), + ] { + let mut file = sample.clone(); + file[central + 5] = host; + file[central + 38..central + 42].copy_from_slice(&attrs.to_le_bytes()); + let archive = parse(&file, 1).unwrap(); + let entry = &archive.entries[0]; + assert_eq!((entry.is_dir, entry.special_kind()), (directory, special), "host {host}, attributes {attrs:#x}"); + } +} + +#[test] +fn a_cp437_name_without_the_utf8_flag_is_decoded() { + let zip = build_named_entry(b"caf\x82.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "café.txt"); +} + +#[test] +fn a_cp437_box_drawing_name_without_the_utf8_flag_is_decoded() { + let zip = build_named_entry(b"\xc9.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "╔.txt"); +} + +#[test] +fn a_utf8_name_with_the_utf8_flag_is_preserved() { + let zip = build_named_entry("한글/é.txt".as_bytes(), 0x0800); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "한글/é.txt"); +} + +#[test] +fn a_valid_utf8_name_without_the_utf8_flag_is_preserved() { + let zip = build_named_entry("é.txt".as_bytes(), 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "é.txt"); +} + +#[test] +fn an_ascii_name_is_preserved() { + let zip = build_named_entry(b"dir/file.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "dir/file.txt"); +} + +#[test] +fn valid_entry_survives_eocd_signature_in_comment() { + let mut zip = build(&[("a.txt", b"hello")]); + let comment = b"PK\x05\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + let eocd = zip.len() - 22; + zip[eocd + 20..eocd + 22].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + zip.extend_from_slice(comment); + + assert_eq!(parse(&zip, 10).unwrap().entries.len(), 1); +} + +#[test] +fn valid_entry_survives_bytes_after_eocd_comment() { + let mut zip = build(&[("a.txt", b"hello")]); + zip.extend_from_slice(b"tail"); + + assert_eq!(parse(&zip, 10).unwrap().entries.len(), 1); +} + +#[test] +fn checks_the_zip64_record_extent_including_extension_bytes() { + let mut record = vec![0; 56]; + record[..4].copy_from_slice(b"PK\x06\x06"); + record[4..12].copy_from_slice(&46u64.to_le_bytes()); + record.extend([1, 2]); // extensible data + record.extend(b"PK\x06\x07"); + record.extend([0; 12]); + record.extend(1u32.to_le_bytes()); + record.extend(b"PK\x05\x06"); + record.extend([0; 4]); + record.extend([0xff; 4]); + record.extend([0; 10]); + assert!(parse(&record, 1).unwrap().zip64); + for len in [44u64, 47] { + record[4..12].copy_from_slice(&len.to_le_bytes()); + let archive = parse(&record, 1).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.warnings.len(), 1); + } + for len in [43u64, u64::MAX] { + record[4..12].copy_from_slice(&len.to_le_bytes()); + assert!(parse(&record, 1).is_err(), "{len}"); + } +} + +#[test] +fn lists_and_reads_stored_entries() { + let z = build(&[("a.txt", b"hello"), ("dir/b.js", b"console.log(1)")]); + let a = parse(&z, 100).unwrap(); + assert_eq!(a.entries.len(), 2); + assert_eq!(a.entries[1].name, "dir/b.js"); + assert_eq!(read_entry(&z, &a.entries[0], 100).unwrap(), b"hello"); + assert!(matches!(read_entry(&z, &a.entries[1], 3), Err(ZipError::TooLarge(3)))); +} + +#[cfg(feature = "zstd")] +#[test] +fn zstandard_members_are_read_with_their_limits_and_checks() { + // A raw Zstandard frame holding "hello zip" (single segment, one raw block). + let body = b"hello zip"; + let mut frame = vec![0x28, 0xb5, 0x2f, 0xfd, 0x20, body.len() as u8]; + let block_header = ((body.len() as u32) << 3) | 1; + frame.extend(&block_header.to_le_bytes()[..3]); + frame.extend(body); + let mut file = vec![0u8; 30]; + file[..4].copy_from_slice(b"PK\x03\x04"); + file.extend(&frame); + let mut e = ZipEntry { + name: "x".into(), + method: 93, + flags: 0, + crc32: crc32(body), + compressed_size: frame.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + assert_eq!(read_entry(&file, &e, 100).unwrap(), body); + assert!(matches!(read_entry(&file, &e, 3), Err(ZipError::TooLarge(3)))); + e.crc32 ^= 1; + assert!(matches!(read_entry(&file, &e, 100), Err(ZipError::CrcMismatch { .. }))); +} + +#[test] +fn deflate_round_trip_and_crc_check() { + let body = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".repeat(20); + let comp = miniz_oxide::deflate::compress_to_vec(&body, 6); + let e = ZipEntry { + name: "x".into(), + method: 8, + flags: 0, + crc32: crc32(&body), + compressed_size: comp.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + let mut z = b"PK\x03\x04".to_vec(); + z.extend_from_slice(&[0; 26]); + z.extend_from_slice(&comp); + assert_eq!(read_entry(&z, &e, 1 << 20).unwrap(), body); + // A lying uncompressed size can't make us exceed the cap. + let liar = ZipEntry { uncompressed_size: 10, ..e.clone() }; + assert!(matches!(read_entry(&z, &liar, 10), Err(ZipError::TooLarge(10)))); + let bad = ZipEntry { crc32: 1, ..e }; + assert!(matches!(read_entry(&z, &bad, 1 << 20), Err(ZipError::CrcMismatch { .. }))); +} + +#[test] +fn hostile_inputs_do_not_panic() { + let z = build(&[("a", b"1"), ("b", b"22")]); + for cut in 0..z.len() { + if let Ok(a) = parse(&z[..cut], 10) { + for e in &a.entries { + let _ = read_entry(&z[..cut], e, 100); + } + } + } + for i in 0..z.len() { + let mut m = z.clone(); + m[i] ^= 0xff; + if let Ok(a) = parse(&m, 10) { + for e in &a.entries { + let _ = read_entry(&m, e, 100); + } + } + } + assert_eq!(parse(b"", 1), Err(ZipError::NotZip)); +} + +#[test] +fn crc_known_value() { + assert_eq!(crc32(b"123456789"), 0xcbf4_3926); +} + +#[cfg(feature = "serde")] +#[test] +fn listings_serialize_without_empty_warnings_or_unix_mode() { + let data = crate::testkit::zip_stored(&[("a.txt", b"hi")]); + let archive = parse(&data, 10).unwrap(); + let value = serde_json::to_value(&archive).unwrap(); + assert!(value.get("warnings").is_none()); + assert_eq!(value["entries"][0]["name"], "a.txt"); + assert!(value["entries"][0].get("unix_mode").is_none()); +} + +#[cfg(not(feature = "zstd"))] +#[test] +fn a_method_left_out_of_the_build_is_unsupported() { + let mut data = crate::testkit::zip_stored(&[("a", b"abc")]); + let archive = parse(&data, 10).unwrap(); + let mut entry = archive.entries[0].clone(); + entry.method = 93; + data[8..10].copy_from_slice(&93u16.to_le_bytes()); + assert!(matches!(read_entry_unverified(&data, &entry, 100), Err(ZipError::Unsupported(m)) if m.contains("not built in"))); +} diff --git a/crates/unzip/tests/common/mod.rs b/crates/unzip/tests/common/mod.rs new file mode 100644 index 000000000..ba0f3d778 --- /dev/null +++ b/crates/unzip/tests/common/mod.rs @@ -0,0 +1,41 @@ +//! The stored-only ZIP writer the integration tests share, as in Craft Anything's testkit. + +#![allow(dead_code, clippy::cast_possible_truncation)] + +use craft_unzip::crc32; + +/// A stored-only ZIP of `files`. +pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + let len = body.len() as u32; + out.extend_from_slice(b"PK\x03\x04\x14\0\0\0\0\0\0\0\0\0"); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02\x14\0\x14\0\0\0\0\0\0\0\0\0"); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} diff --git a/crates/unzip/tests/zip64_end_record_compatibility.rs b/crates/unzip/tests/zip64_end_record_compatibility.rs new file mode 100644 index 000000000..37f63cf76 --- /dev/null +++ b/crates/unzip/tests/zip64_end_record_compatibility.rs @@ -0,0 +1,169 @@ +// From Craft Anything crates/formats/tests/zip64_end_record_compatibility.rs. +use craft_unzip::{self as zip, ZipArchive, ZipError}; +mod common; +use common::zip_stored; + +fn warning_contains(archive: &ZipArchive, text: &str) -> bool { + archive.warnings.iter().any(|warning| warning.contains(text)) +} + +fn zip64_with_extension(extension: &[u8]) -> (Vec, usize) { + let mut data = zip_stored(&[("a.txt", b"retained payload")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let offset = u32::from_le_bytes(data[end + 16..end + 20].try_into().unwrap()); + let mut record = b"PK\x06\x06".to_vec(); + record.extend((44 + extension.len() as u64).to_le_bytes()); + record.extend([45, 0, 45, 0]); + record.extend([0; 8]); + record.extend(1u64.to_le_bytes()); + record.extend(1u64.to_le_bytes()); + record.extend(u64::from(size).to_le_bytes()); + record.extend(u64::from(offset).to_le_bytes()); + record.extend(extension); + record.extend(b"PK\x06\x07"); + record.extend(0u32.to_le_bytes()); + record.extend((end as u64).to_le_bytes()); + record.extend(1u32.to_le_bytes()); + data[end + 8..end + 12].fill(0xff); + data.splice(end..end, record); + (data, end) +} + +#[test] +fn readable_fixed_fields_survive_inconsistent_declared_extents() { + let (original, record) = zip64_with_extension(&[1, 2, 3, 4]); + for declared in [44u64, 47, 49, 4096] { + let mut data = original.clone(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.declared_entries, 1); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.prefix_bytes, 0); + assert!(warning_contains(&archive, "extent"), "declared payload {declared}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn correctly_sized_extensions_need_no_warning() { + for extension in [&[][..], &[1, 2, 3, 4][..]] { + let (data, _) = zip64_with_extension(extension); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert!(archive.warnings.is_empty()); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn too_small_and_overflowing_record_extents_remain_untrusted() { + let (original, record) = zip64_with_extension(&[]); + for declared in [0u64, 43, u64::MAX - 12, u64::MAX] { + let mut data = original.clone(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Corrupt(_))), "declared payload {declared}"); + } +} + +#[test] +fn fixed_fields_cannot_borrow_bytes_from_the_locator() { + let (mut data, record) = zip64_with_extension(&[]); + data.drain(record + 48..record + 56); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn prepended_stubs_keep_real_zip64_directories_and_entry_offsets() { + for extension in [&[][..], &[1, 2, 3, 4][..], &[0; 1024][..]] { + let (original, _) = zip64_with_extension(extension); + for prefix in [&b"MZ"[..], &b"MZ synthetic self-extracting prefix"[..]] { + for saturated_bounds in [false, true] { + let mut data = prefix.to_vec(); + data.extend(&original); + if saturated_bounds { + let end = data.len() - 22; + data[end + 12..end + 20].fill(0xff); + } + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.declared_entries, 1); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert_eq!(archive.entries[0].local_header_offset, prefix.len() as u64); + assert!(warning_contains(&archive, "recovered")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } + } + } +} + +#[test] +fn shifted_fixed_records_keep_extent_warnings() { + let (original, record) = zip64_with_extension(&[]); + let prefix = b"MZ synthetic self-extracting prefix"; + for declared in [45u64, 4096] { + let mut data = prefix.to_vec(); + data.extend(&original); + let record = record + prefix.len(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert!(warning_contains(&archive, "recovered")); + assert!(warning_contains(&archive, "extent")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn shifted_record_recovery_has_a_bounded_lookbehind() { + let (original, _) = zip64_with_extension(&[0; 65_536]); + let mut data = b"MZ synthetic self-extracting prefix".to_vec(); + data.extend(&original); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn an_unusable_locator_keeps_the_classic_directory_and_prefix_adjustment() { + // 65,535 is a valid classic count as well as a ZIP64 sentinel. A final entry comment + // can look like a locator without actually pointing to a ZIP64 end record. + let files = vec![("a.txt", &b"retained payload"[..]); 65_535]; + let original = zip_stored(&files); + let end = original.len() - 22; + let size = u32::from_le_bytes(original[end + 12..end + 16].try_into().unwrap()); + let last_entry = end - 46 - "a.txt".len(); + let mut locator = b"PK\x06\x07".to_vec(); + locator.extend(0u32.to_le_bytes()); + locator.extend(1u64.to_le_bytes()); + locator.extend(1u32.to_le_bytes()); + // A nearby record-shaped comment is still not a ZIP64 directory when its bounds + // disagree with the explicit classic fields. + let mut fake_record = [0; 56]; + fake_record[..4].copy_from_slice(b"PK\x06\x06"); + fake_record[4..12].copy_from_slice(&44u64.to_le_bytes()); + for record_shaped_comment in [&[][..], &fake_record[..]] { + let mut comment = record_shaped_comment.to_vec(); + comment.extend(&locator); + let mut archive_data = original.clone(); + archive_data[last_entry + 32..last_entry + 34].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + archive_data[end + 12..end + 16].copy_from_slice(&(size + comment.len() as u32).to_le_bytes()); + archive_data.splice(end..end, comment); + for prefix in [&[][..], &b"MZ synthetic self-extracting prefix"[..]] { + for offset in [1u64, u64::MAX] { + let mut data = prefix.to_vec(); + data.extend(&archive_data); + let locator = data.len() - 22 - 20; + data[locator + 8..locator + 16].copy_from_slice(&offset.to_le_bytes()); + let archive = zip::parse(&data, 2).unwrap(); + assert!(!archive.zip64); + assert_eq!(archive.declared_entries, 65_535); + assert_eq!(archive.entries.len(), 2); + assert!(archive.truncated); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert!(warning_contains(&archive, "classic end record")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } + } + } +} diff --git a/crates/unzip/tests/zip64_extra_boundaries.rs b/crates/unzip/tests/zip64_extra_boundaries.rs new file mode 100644 index 000000000..a8c673d08 --- /dev/null +++ b/crates/unzip/tests/zip64_extra_boundaries.rs @@ -0,0 +1,64 @@ +// From Craft Anything crates/formats/tests/zip64_extra_boundaries.rs. + +mod common; +use craft_unzip as zip; + +fn archive(extra: &[u8], sentinel: usize) -> Vec { + let mut data = common::zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + data[central + sentinel..central + sentinel + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + data[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + data.splice(end..end, extra.iter().copied()); + let end = data.len() - 22; + data[end + 12..end + 16].copy_from_slice(&((47 + extra.len()) as u32).to_le_bytes()); + data +} + +#[test] +fn a_short_zip64_field_cannot_borrow_values_from_its_neighbor() { + for field in [20, 24, 42] { + let mut extra = vec![1, 0, 0, 0, 2, 0, 8, 0]; + extra.extend_from_slice(&1u64.to_le_bytes()); + assert!(zip::parse(&archive(&extra, field), 10).is_err(), "field {field}"); + } +} + +#[test] +fn zip64_sentinels_require_complete_values() { + for field in [20, 24, 42] { + assert!(zip::parse(&archive(&[], field), 10).is_err()); + assert!(zip::parse(&archive(&[1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0], field), 10).is_err()); + } +} + +#[test] +fn a_complete_zip64_field_preserves_the_value() { + for field in [20, 24, 42] { + let value = if field == 42 { 0u64 } else { 1u64 }; + let mut extra = vec![1, 0, 8, 0]; + extra.extend_from_slice(&value.to_le_bytes()); + let data = archive(&extra, field); + let parsed = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &parsed.entries[0], 1).unwrap(), b"x"); + } +} + +#[test] +fn a_legitimate_64_bit_value_equal_to_the_32_bit_sentinel_is_preserved() { + let mut extra = vec![1, 0, 8, 0]; + extra.extend_from_slice(&u64::from(u32::MAX).to_le_bytes()); + let archive = zip::parse(&archive(&extra, 24), 10).unwrap(); + assert_eq!(archive.entries[0].uncompressed_size, u64::from(u32::MAX)); +} + +#[test] +fn trailing_bytes_too_short_for_a_field_header_are_padding() { + for stray in 1..=3usize { + let mut extra = vec![0xaa, 0xbb, 0, 0]; + extra.extend(std::iter::repeat_n(0u8, stray)); + let data = archive(&extra, 36); + let parsed = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &parsed.entries[0], 1).unwrap(), b"x", "stray {stray}"); + } +} diff --git a/crates/unzip/tests/zip_decoded_size.rs b/crates/unzip/tests/zip_decoded_size.rs new file mode 100644 index 000000000..5bf15ea94 --- /dev/null +++ b/crates/unzip/tests/zip_decoded_size.rs @@ -0,0 +1,43 @@ +// From Craft Anything crates/formats/tests/zip_decoded_size.rs. + +mod common; +use craft_unzip::{self as zip, ZipEntry, ZipError}; + +fn deflated(body: &[u8]) -> (Vec, ZipEntry) { + let compressed = miniz_oxide::deflate::compress_to_vec(body, 6); + let mut data = vec![0; 30]; + data[..4].copy_from_slice(b"PK\x03\x04"); + data.extend_from_slice(&compressed); + let entry = ZipEntry { + name: "payload".into(), + method: 8, + flags: 0, + crc32: zip::crc32(body), + compressed_size: compressed.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + (data, entry) +} + +#[test] +fn decoded_length_must_match_the_directory_even_when_the_crc_matches() { + let (data, good) = deflated(b"complete body"); + for declared in [1, good.uncompressed_size + 1] { + let entry = ZipEntry { uncompressed_size: declared, ..good.clone() }; + assert!(matches!(zip::read_entry(&data, &entry, 100), Err(ZipError::Corrupt(_)))); + assert!(matches!(zip::read_entry_unverified(&data, &entry, 100), Err(ZipError::Corrupt(_)))); + } + assert_eq!(zip::read_entry(&data, &good, 100).unwrap(), b"complete body"); +} + +#[test] +fn stored_length_must_also_match_the_directory() { + let data = common::zip_stored(&[("payload", b"hello")]); + let mut entry = zip::parse(&data, 10).unwrap().entries.remove(0); + entry.uncompressed_size = 4; + assert!(matches!(zip::read_entry(&data, &entry, 100), Err(ZipError::Corrupt(_)))); +} diff --git a/crates/unzip/tests/zip_directory_bounds.rs b/crates/unzip/tests/zip_directory_bounds.rs new file mode 100644 index 000000000..3f594dd74 --- /dev/null +++ b/crates/unzip/tests/zip_directory_bounds.rs @@ -0,0 +1,74 @@ +// From Craft Anything crates/formats/tests/zip_directory_bounds.rs. +use craft_unzip::{self as zip, ZipError}; +mod common; +use common::zip_stored; + +#[test] +fn a_short_declared_entry_is_rejected_before_its_signature_is_checked() { + for remaining in 0..46 { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + data[end + 8..end + 12].copy_from_slice(&[2, 0, 2, 0]); + data[end + 12..end + 16].copy_from_slice(&(size + remaining).to_le_bytes()); + data.splice(end..end, vec![0; remaining as usize]); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Truncated("directory entry"))), "remaining {remaining} bytes"); + } +} + +#[test] +fn directory_names_and_comments_cannot_borrow_end_record_bytes() { + for field in [28, 32] { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let length = if field == 28 { 2u16 } else { 1 }; + data[central + field..central + field + 2].copy_from_slice(&length.to_le_bytes()); + assert!(zip::parse(&data, 10).is_err(), "field {field}"); + } +} + +#[test] +fn directory_extra_fields_cannot_borrow_end_record_bytes() { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + data[central + 30..central + 32].copy_from_slice(&2u16.to_le_bytes()); + assert_eq!(zip::parse(&data, 10), Err(ZipError::Truncated("directory entry fields"))); +} + +#[test] +fn a_directory_cannot_extend_into_its_end_record() { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + data[end + 12..end + 16].copy_from_slice(&(size + 1).to_le_bytes()); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn self_extracting_archives_keep_both_offset_conventions() { + for adjusted in [false, true] { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + let stub = b"executable stub"; + if adjusted { + data[central + 42..central + 46].copy_from_slice(&(stub.len() as u32).to_le_bytes()); + data[end + 16..end + 20].copy_from_slice(&((central + stub.len()) as u32).to_le_bytes()); + } + let mut file = stub.to_vec(); + file.extend(data); + let archive = zip::parse(&file, 10).unwrap(); + assert_eq!(zip::read_entry(&file, &archive.entries[0], 10).unwrap(), b"x"); + } +} + +#[test] +fn a_directory_signature_after_the_entries_is_not_an_extra_file() { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let signature = b"PK\x05\x05\x01\x00x"; + data[end + 12..end + 16].copy_from_slice(&(size + signature.len() as u32).to_le_bytes()); + data.splice(end..end, signature.iter().copied()); + assert_eq!(zip::parse(&data, 10).unwrap().entries.len(), 1); +} diff --git a/crates/unzip/tests/zip_entry_detection.rs b/crates/unzip/tests/zip_entry_detection.rs new file mode 100644 index 000000000..8a74ecf63 --- /dev/null +++ b/crates/unzip/tests/zip_entry_detection.rs @@ -0,0 +1,43 @@ +// From Craft Anything crates/formats/tests/zip_entry_detection.rs. +use craft_unzip::has_entry as zip_has_entry; +mod common; +use common::zip_stored; + +/// A directory-shaped payload is ordinary file content, not an archive member. +#[test] +fn embedded_archive_does_not_supply_outer_entries() { + let inner = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + let outer = zip_stored(&[("backup.zip", &inner)]); + assert!(!zip_has_entry(&outer, "AndroidManifest.xml")); + assert!(zip_has_entry(&outer, "backup.zip")); +} + +/// A ZIP comment can contain another directory without changing the archive's entries. +#[test] +fn comment_does_not_supply_entries() { + let mut outer = zip_stored(&[("readme.txt", b"notes")]); + let inner = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + let len = outer.len(); + outer[len - 2..].copy_from_slice(&(inner.len() as u16).to_le_bytes()); + outer.extend(inner); + assert!(!zip_has_entry(&outer, "AndroidManifest.xml")); + assert!(zip_has_entry(&outer, "readme.txt")); +} + +/// Actual manifests are found regardless of member order, including prefixed archives. +#[test] +fn actual_entries_are_found() { + let data = zip_stored(&[("classes.dex", b"dex"), ("AndroidManifest.xml", b"manifest")]); + assert!(zip_has_entry(&data, "AndroidManifest.xml")); + let mut prefixed = b"stub".to_vec(); + prefixed.extend(data); + assert!(zip_has_entry(&prefixed, "AndroidManifest.xml")); + assert!(!zip_has_entry(&prefixed, "missing")); +} + +/// A stray central-directory signature without an archive is not an entry. +#[test] +fn incomplete_archive_is_not_a_manifest() { + let data = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + assert!(!zip_has_entry(&data[..data.len() - 22], "AndroidManifest.xml")); +} diff --git a/crates/unzip/tests/zip_split_volumes.rs b/crates/unzip/tests/zip_split_volumes.rs new file mode 100644 index 000000000..c5412b72e --- /dev/null +++ b/crates/unzip/tests/zip_split_volumes.rs @@ -0,0 +1,165 @@ +// From Craft Anything crates/formats/tests/zip_split_volumes.rs. +use craft_unzip::{self as zip, ZipError}; +mod common; +use common::zip_stored; + +fn assert_split(data: &[u8]) { + assert!(matches!(zip::parse(data, 10), Err(ZipError::Unsupported(_)))); +} + +#[test] +fn classic_end_records_must_describe_one_complete_volume() { + for field in [4, 6, 8] { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let value = if field == 8 { 0u16 } else { 1 }; + data[end + field..end + field + 2].copy_from_slice(&value.to_le_bytes()); + assert_split(&data); + } +} + +#[test] +fn central_entries_cannot_point_to_another_volume() { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + data[central + 34..central + 36].copy_from_slice(&1u16.to_le_bytes()); + assert_split(&data); +} + +fn zip64_entry_disk(disk: u32, with_sizes_and_offset: bool) -> Vec { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + data[central + 34..central + 36].copy_from_slice(&u16::MAX.to_le_bytes()); + let mut extra = vec![1, 0, 4, 0]; + if with_sizes_and_offset { + data[central + 20..central + 28].fill(0xff); + data[central + 42..central + 46].fill(0xff); + extra[2..4].copy_from_slice(&28u16.to_le_bytes()); + // Uncompressed size, compressed size, and local header offset precede the disk. + for value in [1u64, 1, 0] { + extra.extend(value.to_le_bytes()); + } + } + extra.extend(disk.to_le_bytes()); + data[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + let directory_size = (end - central + extra.len()) as u32; + data.splice(end..end, extra); + let end = data.len() - 22; + data[end + 12..end + 16].copy_from_slice(&directory_size.to_le_bytes()); + data +} + +#[test] +fn a_zip64_disk_sentinel_is_resolved_before_rejecting_the_entry() { + let data = zip64_entry_disk(0, false); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + assert_split(&zip64_entry_disk(1, false)); +} + +#[test] +fn a_zip64_disk_sentinel_follows_sizes_and_the_local_offset() { + let data = zip64_entry_disk(0, true); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.entries[0].uncompressed_size, 1); + assert_eq!(archive.entries[0].compressed_size, 1); + assert_eq!(archive.entries[0].local_header_offset, 0); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + assert_split(&zip64_entry_disk(1, true)); +} + +fn zip64_archive(record_disk: u32, central_disk: u32, locator_disk: u32, disks: u32, entries_on_disk: u64) -> Vec { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let central = u32::from_le_bytes(data[end + 16..end + 20].try_into().unwrap()); + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let mut record = b"PK\x06\x06".to_vec(); + record.extend(44u64.to_le_bytes()); + record.extend([45, 0, 45, 0]); + record.extend(record_disk.to_le_bytes()); + record.extend(central_disk.to_le_bytes()); + record.extend(entries_on_disk.to_le_bytes()); + record.extend(1u64.to_le_bytes()); + record.extend(u64::from(size).to_le_bytes()); + record.extend(u64::from(central).to_le_bytes()); + record.extend(b"PK\x06\x07"); + record.extend(locator_disk.to_le_bytes()); + record.extend((end as u64).to_le_bytes()); + record.extend(disks.to_le_bytes()); + data[end + 8..end + 12].fill(0xff); + data.splice(end..end, record); + data +} + +#[test] +fn zip64_end_records_and_locators_must_also_describe_one_volume() { + for fields in [(1, 0, 0, 1, 1), (0, 1, 0, 1, 1), (0, 0, 1, 1, 1), (0, 0, 0, 2, 1), (0, 0, 0, 1, 0)] { + assert_split(&zip64_archive(fields.0, fields.1, fields.2, fields.3, fields.4)); + } + assert_eq!(zip::parse(&zip64_archive(0, 0, 0, 1, 1), 10).unwrap().entries.len(), 1); +} + +#[test] +fn zip64_does_not_erase_explicit_classic_volume_markers() { + for field in [4, 6] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&1u16.to_le_bytes()); + assert_split(&data); + data[end + field..end + field + 2].fill(0xff); + assert_eq!(zip::parse(&data, 10).unwrap().entries.len(), 1); + } +} + +#[test] +fn zip64_does_not_erase_explicit_classic_entry_counts() { + for field in [8, 10] { + for count in [0u16, 2] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&count.to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Unsupported(_))), "field {field}, count {count}"); + } + } +} + +#[test] +fn mixed_classic_and_zip64_entry_counts_can_describe_one_volume() { + for field in [8, 10] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&1u16.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.entries.len(), 1, "field {field}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + } +} + +#[test] +fn zip64_cannot_replace_explicit_classic_directory_bounds() { + for field in [12, 16] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + let value = u32::from_le_bytes(data[end + field..end + field + 4].try_into().unwrap()); + data[end + field..end + field + 4].copy_from_slice(&(value + 1).to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Corrupt(_))), "field {field}"); + } +} + +#[test] +fn each_zip64_field_can_resolve_its_own_classic_sentinel() { + for (field, width) in [(4, 2), (6, 2), (8, 2), (10, 2), (12, 4), (16, 4)] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + 8..end + 10].copy_from_slice(&1u16.to_le_bytes()); + data[end + 10..end + 12].copy_from_slice(&1u16.to_le_bytes()); + data[end + field..end + field + width].fill(0xff); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64, "field {field}"); + assert_eq!(archive.declared_entries, 1, "field {field}"); + assert_eq!(archive.entries.len(), 1, "field {field}"); + assert_eq!(archive.prefix_bytes, 0, "field {field}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x", "field {field}"); + } +} diff --git a/crates/unzip/tests/zip_stored_cap.rs b/crates/unzip/tests/zip_stored_cap.rs new file mode 100644 index 000000000..61bb06d50 --- /dev/null +++ b/crates/unzip/tests/zip_stored_cap.rs @@ -0,0 +1,21 @@ +// From Craft Anything crates/formats/tests/zip_stored_cap.rs. + +mod common; +use craft_unzip::{self as zip, ZipError}; + +#[test] +fn a_stored_entry_cannot_understate_its_size_to_bypass_the_output_limit() { + let data = common::zip_stored(&[("payload", &[0x5a; 64])]); + let mut entry = zip::parse(&data, 10).unwrap().entries.remove(0); + entry.uncompressed_size = 1; + assert!(matches!(zip::read_entry(&data, &entry, 8), Err(ZipError::TooLarge(8)))); + assert!(matches!(zip::read_entry_unverified(&data, &entry, 8), Err(ZipError::TooLarge(8)))); +} + +#[test] +fn stored_entries_at_the_limit_and_empty_entries_still_read() { + let data = common::zip_stored(&[("payload", b"hello"), ("empty", b"")]); + let entries = zip::parse(&data, 10).unwrap().entries; + assert_eq!(zip::read_entry(&data, &entries[0], 5).unwrap(), b"hello"); + assert_eq!(zip::read_entry(&data, &entries[1], 0).unwrap(), b""); +} From a57f393d974be3c865ea20b20139f6cc4721b95a Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Mon, 16 Nov 2026 03:21:19 -0800 Subject: [PATCH 035/209] craft-video: the ffmpeg MJPEG round-trip test encodes the same picture every run we_decode_what_ffmpeg_encodes_to_what_ffmpeg_decodes built its input with ffmpeg's gradients source, which picks random colors on every run, so the agreement it measured changed from run to run. On one CI run the 4:2:0 case scored 30.5 dB against the 38 dB floor. The test now writes the file's own picture(67, 41, 3) as raw RGB and feeds it to ffmpeg through rawvideo, for both the plain and the default-tables encodes. --- crates/video/tests/mjpeg.rs | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/crates/video/tests/mjpeg.rs b/crates/video/tests/mjpeg.rs index d83b61438..fcd9cacd6 100644 --- a/crates/video/tests/mjpeg.rs +++ b/crates/video/tests/mjpeg.rs @@ -150,12 +150,14 @@ fn we_decode_what_ffmpeg_encodes_to_what_ffmpeg_decodes() { let Some(ff) = craft_media_testkit::ffmpeg() else { return }; let dir = craft_media_testkit::fixture_subdir("mjpeg"); let s = |p: &Path| p.to_str().expect("path").to_string(); + // the same input every run: ffmpeg's own test sources draw random colors + let input = dir.join("input.rgb"); + std::fs::write(&input, picture(67, 41, 3)).expect("write"); + let input = s(&input); + let source = ["-f", "rawvideo", "-pix_fmt", "rgb24", "-s", "67x41", "-i", input.as_str()]; for (name, pix_fmt, rgb_pix, channels) in [("420", "yuvj420p", "rgb24", 3), ("422", "yuvj422p", "rgb24", 3), ("444", "yuvj444p", "rgb24", 3)] { let jpg = dir.join(format!("{name}.jpg")); - ffmpeg( - &ff, - &["-f", "lavfi", "-i", "gradients=size=67x41:rate=1", "-frames:v", "1", "-c:v", "mjpeg", "-pix_fmt", pix_fmt, "-q:v", "2", &s(&jpg)], - ); + ffmpeg(&ff, &[&source[..], &["-frames:v", "1", "-c:v", "mjpeg", "-pix_fmt", pix_fmt, "-q:v", "2", &s(&jpg)]].concat()); let reference = dir.join(format!("{name}.raw")); ffmpeg(&ff, &["-i", &s(&jpg), "-f", "rawvideo", "-pix_fmt", rgb_pix, &s(&reference)]); let packet = std::fs::read(&jpg).expect("read"); @@ -170,23 +172,7 @@ fn we_decode_what_ffmpeg_encodes_to_what_ffmpeg_decodes() { let plain = dir.join(format!("{name}_default_tables.jpg")); ffmpeg( &ff, - &[ - "-f", - "lavfi", - "-i", - "gradients=size=67x41:rate=1", - "-frames:v", - "1", - "-c:v", - "mjpeg", - "-pix_fmt", - pix_fmt, - "-q:v", - "2", - "-huffman", - "default", - &s(&plain), - ], + &[&source[..], &["-frames:v", "1", "-c:v", "mjpeg", "-pix_fmt", pix_fmt, "-q:v", "2", "-huffman", "default", &s(&plain)]].concat(), ); let stripped = without_huffman_tables(&std::fs::read(&plain).expect("read")); let with_tables = Decoder::new().decode(&std::fs::read(&plain).expect("read")).expect("decode"); From 712feab58680652c22e28391ec2e610c17bb374b Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:25:00 +0100 Subject: [PATCH 036/209] craft-zip: the lenient reader as craft_zip::lenient instead of a second ZIP crate craft-unzip's raw-bytes reader moves unchanged into craft-zip as the `lenient` module, behind the `lenient` feature (DEFLATE) and `lenient-bzip2`, `lenient-lzma` and `lenient-zstd` for the other methods, with `serde` for the listing types. craft-zip's default build is unchanged; the module doesn't use the `zip` crate. Its unit and integration tests move with it and run in craft-zip's test build. --- Cargo.lock | 19 ++---- Cargo.toml | 1 - README.md | 3 +- crates/unzip/Cargo.toml | 34 ----------- crates/zip/Cargo.toml | 20 ++++++- crates/zip/README.md | 10 ++++ .../{unzip/src/lib.rs => zip/src/lenient.rs} | 33 ++++++----- .../{unzip/src => zip/src/lenient}/bytes.rs | 0 .../{unzip/src => zip/src/lenient}/decode.rs | 58 +++++++++---------- crates/{unzip/src => zip/src/lenient}/file.rs | 14 ++--- .../{unzip/src => zip/src/lenient}/testkit.rs | 2 +- .../{unzip/src => zip/src/lenient}/tests.rs | 20 +++---- crates/zip/src/lib.rs | 5 ++ .../tests/lenient_common}/mod.rs | 2 +- ...lenient_zip64_end_record_compatibility.rs} | 4 +- .../tests/lenient_zip64_extra_boundaries.rs} | 4 +- .../tests/lenient_zip_decoded_size.rs} | 4 +- .../tests/lenient_zip_directory_bounds.rs} | 4 +- .../tests/lenient_zip_entry_detection.rs} | 4 +- .../tests/lenient_zip_split_volumes.rs} | 4 +- .../tests/lenient_zip_stored_cap.rs} | 4 +- 21 files changed, 128 insertions(+), 121 deletions(-) delete mode 100644 crates/unzip/Cargo.toml rename crates/{unzip/src/lib.rs => zip/src/lenient.rs} (95%) rename crates/{unzip/src => zip/src/lenient}/bytes.rs (100%) rename crates/{unzip/src => zip/src/lenient}/decode.rs (91%) rename crates/{unzip/src => zip/src/lenient}/file.rs (95%) rename crates/{unzip/src => zip/src/lenient}/testkit.rs (99%) rename crates/{unzip/src => zip/src/lenient}/tests.rs (96%) rename crates/{unzip/tests/common => zip/tests/lenient_common}/mod.rs (98%) rename crates/{unzip/tests/zip64_end_record_compatibility.rs => zip/tests/lenient_zip64_end_record_compatibility.rs} (98%) rename crates/{unzip/tests/zip64_extra_boundaries.rs => zip/tests/lenient_zip64_extra_boundaries.rs} (96%) rename crates/{unzip/tests/zip_decoded_size.rs => zip/tests/lenient_zip_decoded_size.rs} (92%) rename crates/{unzip/tests/zip_directory_bounds.rs => zip/tests/lenient_zip_directory_bounds.rs} (96%) rename crates/{unzip/tests/zip_entry_detection.rs => zip/tests/lenient_zip_entry_detection.rs} (93%) rename crates/{unzip/tests/zip_split_volumes.rs => zip/tests/lenient_zip_split_volumes.rs} (98%) rename crates/{unzip/tests/zip_stored_cap.rs => zip/tests/lenient_zip_stored_cap.rs} (88%) diff --git a/Cargo.lock b/Cargo.lock index 7a00d57d7..27a34f299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2898,19 +2898,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "craft-unzip" -version = "0.1.0" -dependencies = [ - "bzip2-rs", - "lzma-rs", - "miniz_oxide 0.8.9", - "ruzstd", - "serde", - "serde_json", - "thiserror 2.0.21", -] - [[package]] name = "craft-update" version = "0.1.0" @@ -3113,8 +3100,14 @@ dependencies = [ name = "craft-zip" version = "0.1.0" dependencies = [ + "bzip2-rs", "craft-zip", "flate2", + "lzma-rs", + "miniz_oxide 0.8.9", + "ruzstd", + "serde", + "serde_json", "thiserror 2.0.21", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 34ee6095e..bd08fa725 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,7 +174,6 @@ craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } -craft-unzip = { path = "crates/unzip" } craft-epub = { path = "crates/epub" } craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } diff --git a/README.md b/README.md index 8a4c3d327..63303e3f0 100644 --- a/README.md +++ b/README.md @@ -392,8 +392,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit - [`craft-kra`](crates/kra/README.md): Krita documents (`.kra`, `.krz`), read: the ZIP archive, the `maindoc.xml` node tree as written, and the tiled pixel files with their LZF compression, a tile at a time. From PhotoCraft. - [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. - [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. -- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. -- [`craft-unzip`](crates/unzip/src/lib.rs): ZIP reading over bytes already in memory (or one named entry from a seekable file), from Craft Anything: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind default features) checked against the declared size and CRC-32. Pure Rust with no `zip` crate underneath; for inspectors that must describe damaged archives rather than refuse them. +- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. With the `lenient` feature, also a reader of the raw bytes without the `zip` crate, from Craft Anything, for inspectors that must describe damaged archives rather than refuse them: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind `lenient-*` features) checked against the declared size and CRC-32. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/crates/unzip/Cargo.toml b/crates/unzip/Cargo.toml deleted file mode 100644 index 8a414277b..000000000 --- a/crates/unzip/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "craft-unzip" -description = "ZIP archives (APK, JAR, IPA, MSIX…) from hostile bytes, from Craft Anything: a central-directory listing with ZIP64, prepended-stub and multi-volume checks, and bounded entry reads checked against the declared size and CRC-32. Pure Rust, no I/O beyond an optional seekable reader." -version.workspace = true -edition.workspace = true -license.workspace = true -rust-version.workspace = true -repository.workspace = true - -[lints] -workspace = true - -[dependencies] -thiserror = { workspace = true } -miniz_oxide = { workspace = true } -# Entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. -bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 -lzma-rs = { version = "0.3", optional = true } # MIT -ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT -# `serde`: `Serialize` on the listing types, for apps that report listings as JSON. -serde = { workspace = true, optional = true } - -[features] -default = ["bzip2", "lzma", "zstd"] -# Method 12. -bzip2 = ["dep:bzip2-rs"] -# Methods 14 (ZIP LZMA) and 95 (XZ). -lzma = ["dep:lzma-rs"] -# Method 93. -zstd = ["dep:ruzstd"] -serde = ["dep:serde"] - -[dev-dependencies] -serde_json = { workspace = true } diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 6d50b5352..89cb2d1a7 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -17,10 +17,28 @@ thiserror = { workspace = true } zip = { version = "9", default-features = false, features = ["deflate-flate2"] } # The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +# `lenient`: DEFLATE for the raw-bytes reader. +miniz_oxide = { workspace = true, optional = true } +# `lenient` entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. +bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 +lzma-rs = { version = "0.3", optional = true } # MIT +ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT +# `serde`: `Serialize` on the `lenient` listing types, for apps that report listings as JSON. +serde = { workspace = true, optional = true } [features] # `zip64_entry_count_sample`, the regression input for a reader's own entry-count test. test-support = [] +# `craft_zip::lenient`: the raw-bytes reader for inspectors (from Craft Anything), without `zip`. +lenient = ["dep:miniz_oxide"] +# Method 12. +lenient-bzip2 = ["lenient", "dep:bzip2-rs"] +# Methods 14 (ZIP LZMA) and 95 (XZ). +lenient-lzma = ["lenient", "dep:lzma-rs"] +# Method 93. +lenient-zstd = ["lenient", "dep:ruzstd"] +serde = ["dep:serde"] [dev-dependencies] -craft-zip = { path = ".", features = ["test-support"] } +craft-zip = { path = ".", features = ["test-support", "lenient-bzip2", "lenient-lzma", "lenient-zstd", "serde"] } +serde_json = { workspace = true } diff --git a/crates/zip/README.md b/crates/zip/README.md index 6cffa7786..071d2124d 100644 --- a/crates/zip/README.md +++ b/crates/zip/README.md @@ -79,3 +79,13 @@ bytes and charges repeated reads against twice that number. Call `Indexing::fini returned handle after the ZIP index has been parsed to permit normal streaming payload reads. The caller must still enforce entry-count, declared-size and overlap limits: the wrapper only bounds metadata reads and does not validate the archive. + +## Lenient reading (feature `lenient`) + +`craft_zip::lenient` reads an archive from its raw bytes without the `zip` crate, for inspectors +(Craft Anything) that must list and describe damaged, prepended, split or oddly encoded archives +instead of refusing them. Only the central directory is trusted for the listing; entry reads are +bounded, checked against the declared size and CRC-32, and decode stored and DEFLATE entries, plus +bzip2 (`lenient-bzip2`), ZIP LZMA and XZ (`lenient-lzma`) and Zstandard (`lenient-zstd`). +`lenient::file::read_named_entry` reads one entry from a seekable file without loading the +archive. With `serde`, the listing types are `Serialize`. diff --git a/crates/unzip/src/lib.rs b/crates/zip/src/lenient.rs similarity index 95% rename from crates/unzip/src/lib.rs rename to crates/zip/src/lenient.rs index 35dfbbfcf..c9bed3764 100644 --- a/crates/unzip/src/lib.rs +++ b/crates/zip/src/lenient.rs @@ -1,14 +1,17 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -#![forbid(unsafe_code)] -//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…): central directory listing and bounded entry reads. +//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…) read leniently from the raw bytes, for +//! inspectors that must list and describe damaged, prepended, split or oddly encoded archives +//! rather than refuse them (feature `lenient`). The rest of this crate is the strict side: it +//! refuses what fails [`crate::preflight`] and reads through the `zip` crate; nothing here uses +//! `zip`. //! //! Only the central directory is trusted for the listing. Entry data is located through the local //! header, decompressed with a hard output cap, and checked against the recorded CRC-32. Stored, //! DEFLATE, bzip2, ZIP LZMA, XZ and Zstandard entries are supported; other methods and encrypted entries are listed. //! -//! The bzip2, LZMA/XZ and Zstandard methods sit behind the default `bzip2`, `lzma` and `zstd` -//! features; without one, entries using it are listed and reading them is +//! The bzip2, LZMA/XZ and Zstandard methods sit behind the `lenient-bzip2`, `lenient-lzma` and +//! `lenient-zstd` features; without one, entries using it are listed and reading them is //! [`ZipError::Unsupported`]. [`file::read_named_entry`] reads one entry from a seekable file //! without loading the archive. With the `serde` feature the listing types are `Serialize`. //! @@ -18,11 +21,11 @@ //! against the directory, and CRC-32 verification. mod bytes; -#[cfg(any(feature = "bzip2", feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma", feature = "lenient-zstd"))] mod decode; pub mod file; -use crate::bytes::Reader; +use crate::lenient::bytes::Reader; /// Biggest tail we search for the end-of-central-directory record (64 KiB comment + record). pub const EOCD_SEARCH: usize = 22 + 0xffff; @@ -474,12 +477,12 @@ pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Resul Ok(out) } -#[cfg(any(feature = "bzip2", feature = "lzma"))] +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma"))] fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { let decoded = match e.method { - #[cfg(feature = "bzip2")] + #[cfg(feature = "lenient-bzip2")] 12 => decode::bzip2(raw, max_bytes), - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] 95 => decode::xz(raw, max_bytes), _ => return Err(not_built(e)), }; @@ -489,12 +492,12 @@ fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result Result, ZipError> { Err(not_built(e)) } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { let header = Reader::new(raw); let properties_size = header.u16_le(2).ok_or(ZipError::Truncated("LZMA properties header"))?; @@ -508,23 +511,23 @@ fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipE }) } -#[cfg(not(feature = "lzma"))] +#[cfg(not(feature = "lenient-lzma"))] fn decode_lzma(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { Err(not_built(e)) } -#[cfg(feature = "zstd")] +#[cfg(feature = "lenient-zstd")] fn decode_zstd(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { decode::zstd(raw, max_bytes).map_err(|err| ZipError::Corrupt(format!("zstd stream for {} is damaged: {err}", e.name))) } -#[cfg(not(feature = "zstd"))] +#[cfg(not(feature = "lenient-zstd"))] fn decode_zstd(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { Err(not_built(e)) } /// An entry whose method this build leaves out (see the crate features). -#[cfg_attr(all(feature = "bzip2", feature = "lzma", feature = "zstd"), allow(dead_code))] +#[cfg_attr(all(feature = "lenient-bzip2", feature = "lenient-lzma", feature = "lenient-zstd"), allow(dead_code))] fn not_built(e: &ZipEntry) -> ZipError { ZipError::Unsupported(format!("compression method {} ({}) for {} is not built in", e.method, e.method_name(), e.name)) } diff --git a/crates/unzip/src/bytes.rs b/crates/zip/src/lenient/bytes.rs similarity index 100% rename from crates/unzip/src/bytes.rs rename to crates/zip/src/lenient/bytes.rs diff --git a/crates/unzip/src/decode.rs b/crates/zip/src/lenient/decode.rs similarity index 91% rename from crates/unzip/src/decode.rs rename to crates/zip/src/lenient/decode.rs index 1d41d4a55..1189451b4 100644 --- a/crates/unzip/src/decode.rs +++ b/crates/zip/src/lenient/decode.rs @@ -3,9 +3,9 @@ //! allocating without limit) and never panics on a corrupt or truncated stream. use std::io; -#[cfg(any(feature = "bzip2", feature = "lzma"))] +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma"))] use std::io::Read; -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] use std::io::Write; /// Why a decode stopped. @@ -14,7 +14,7 @@ pub(crate) enum DecompressError { #[error("the compressed stream is damaged")] Corrupt, #[error("the compressed stream ends early")] - #[cfg(any(feature = "lzma", feature = "zstd"))] + #[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] Truncated, #[error("the output would exceed the size limit")] TooLarge, @@ -22,21 +22,21 @@ pub(crate) enum DecompressError { /// A `Write` sink that collects at most `cap` bytes, then refuses the rest. The `exceeded` flag lets /// the caller tell a size-limit stop from a genuine decode error. -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] struct CapWrite { buf: Vec, cap: usize, exceeded: bool, } -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] impl CapWrite { fn new(cap: usize) -> Self { CapWrite { buf: Vec::new(), cap, exceeded: false } } } -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] impl Write for CapWrite { fn write(&mut self, data: &[u8]) -> io::Result { if self.buf.len().saturating_add(data.len()) > self.cap { @@ -57,7 +57,7 @@ fn cap_of(max_out: u64) -> usize { } /// Read a `Read` source into a `Vec`, capped at `cap` bytes. -#[cfg(feature = "bzip2")] +#[cfg(feature = "lenient-bzip2")] fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressError> { let mut out = Vec::new(); let mut chunk = [0u8; 64 * 1024]; @@ -77,7 +77,7 @@ fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressErro } } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] /// Decompress every xz stream, including standard stream padding, within one output budget. pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> { let ranges = xz_stream_ranges(input, max_out)?; @@ -95,7 +95,7 @@ pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> Ok(out.buf) } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] /// Find stream boundaries backwards through their checked footer/index sizes, without /// repeatedly trying the decoder at candidate magic bytes inside compressed data. fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result>, DecompressError> { @@ -112,9 +112,9 @@ fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result Result Result Result { let mut value = 0u64; @@ -174,7 +174,7 @@ fn xz_vli(input: &mut &[u8]) -> Result { Err(DecompressError::Corrupt) } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] /// Decode the property bytes and raw stream of a ZIP method-14 member. pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, max_out: u64) -> Result, DecompressError> { let cap = cap_of(max_out).min(isize::MAX as usize); @@ -204,13 +204,13 @@ pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, m } } -#[cfg(feature = "bzip2")] +#[cfg(feature = "lenient-bzip2")] /// Decompress a bzip2 stream, producing at most `max_out` bytes. pub(crate) fn bzip2(input: &[u8], max_out: u64) -> Result, DecompressError> { read_capped(bzip2_rs::DecoderReader::new(input), cap_of(max_out)) } -#[cfg(feature = "zstd")] +#[cfg(feature = "lenient-zstd")] /// Decompress a Zstandard stream, producing at most `max_out` bytes. pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, DecompressError> { if input.is_empty() { @@ -235,9 +235,9 @@ pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, Decompress #[cfg(test)] mod tests { use super::*; - use crate::testkit as fx; + use crate::lenient::testkit as fx; - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_round_trip() { let c = fx::XZ_FIXTURE; @@ -250,7 +250,7 @@ mod tests { } } - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_concatenated_streams_and_padding_share_one_budget() { let input = [fx::XZ_FIXTURE, &[0; 4], fx::XZ_FIXTURE, &[0; 8]].concat(); @@ -264,7 +264,7 @@ mod tests { assert!(xz(&empty, 0).unwrap().is_empty()); } - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_rejects_incomplete_members_and_nonstandard_padding() { for padding in [1, 2, 3, 5] { @@ -284,7 +284,7 @@ mod tests { } } - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_index_sizes_and_integer_encodings_are_checked() { assert_eq!(xz_vli(&mut &[0x80, 0][..]), Err(DecompressError::Corrupt)); @@ -295,12 +295,12 @@ mod tests { let backward = u32::from_le_bytes(damaged[footer + 4..footer + 8].try_into().unwrap()); let index = footer - (backward as usize + 1) * 4; damaged[index + 2] = 0; - let checksum = crate::crc32(&damaged[index..footer - 4]); + let checksum = crate::lenient::crc32(&damaged[index..footer - 4]); damaged[footer - 4..footer].copy_from_slice(&checksum.to_le_bytes()); assert!(xz(&damaged, 1000).is_err()); } - #[cfg(feature = "bzip2")] + #[cfg(feature = "lenient-bzip2")] #[test] fn bzip2_round_trip() { let c = fx::BZIP2_FIXTURE; @@ -311,7 +311,7 @@ mod tests { } } - #[cfg(feature = "zstd")] + #[cfg(feature = "lenient-zstd")] #[test] fn zstd_round_trip() { let c = fx::ZSTD_FIXTURE; @@ -322,7 +322,7 @@ mod tests { } } - #[cfg(feature = "zstd")] + #[cfg(feature = "lenient-zstd")] #[test] fn zstandard_members_can_contain_multiple_frames() { let frame = fx::ZSTD_FIXTURE; @@ -340,11 +340,11 @@ mod tests { fn garbage_never_panics() { for seed in 0..64u64 { let n = fx::noise(seed, 400); - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] let _ = xz(&n, 1 << 20); - #[cfg(feature = "bzip2")] + #[cfg(feature = "lenient-bzip2")] let _ = bzip2(&n, 1 << 20); - #[cfg(feature = "zstd")] + #[cfg(feature = "lenient-zstd")] let _ = zstd(&n, 1 << 20); let _ = &n; } diff --git a/crates/unzip/src/file.rs b/crates/zip/src/lenient/file.rs similarity index 95% rename from crates/unzip/src/file.rs rename to crates/zip/src/lenient/file.rs index 1bb4a3662..fc50dfd26 100644 --- a/crates/unzip/src/file.rs +++ b/crates/zip/src/lenient/file.rs @@ -2,8 +2,8 @@ //! central directory and that entry's bytes are read. For archives too big to hold in memory just //! to look at one small entry (a JAR's manifest). -use crate::bytes::Reader; -use crate::{self as zip, ZipError}; +use crate::lenient::bytes::Reader; +use crate::lenient::{self as zip, ZipError}; use std::io::{Read, Seek, SeekFrom}; /// Most central directory read (some hundreds of thousands of entries); a bigger one is refused. @@ -159,7 +159,7 @@ mod tests { #[test] fn an_entry_is_read_from_the_directory_without_the_whole_file() { let filler = vec![7u8; 200_000]; - let archive = crate::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); + let archive = crate::lenient::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); assert_eq!(read(&archive, "META-INF/MISSING").unwrap(), None); assert_eq!(read(&archive, "META-INF/").unwrap(), None, "a folder isn't an entry to read"); @@ -178,7 +178,7 @@ mod tests { let names: Vec = (0..3000).map(|i| format!("com/example/generated/Class{i:05}.class")).collect(); let mut files: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b"x"[..])).collect(); files.push(("META-INF/MANIFEST.MF", MANIFEST)); - let archive = crate::testkit::zip_stored(&files); + let archive = crate::lenient::testkit::zip_stored(&files); assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); } @@ -215,7 +215,7 @@ mod tests { let deflated = zip_deflated("META-INF/MANIFEST.MF", MANIFEST); assert_eq!(read(&deflated, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); - let mut commented = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let mut commented = crate::lenient::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); let comment = b"built by PK\x05\x06 tools"; let at = commented.len() - 2; commented[at..].copy_from_slice(&(comment.len() as u16).to_le_bytes()); @@ -227,7 +227,7 @@ mod tests { /// the classic fields disagree with it and win, as they do for `zip::parse`. #[test] fn a_comment_shaped_like_zip64_records_is_not_read_as_them() { - let mut archive = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let mut archive = crate::lenient::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); let end = archive.len() - 22; let last = archive.windows(4).rposition(|w| w == b"PK\x01\x02").unwrap(); let mut fake = b"PK\x06\x06".to_vec(); @@ -250,7 +250,7 @@ mod tests { /// only the offset), after a stub too. #[test] fn zip64_archives_are_read_in_parts() { - let archive = crate::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); + let archive = crate::lenient::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); for saturate in [&[][..], &[8, 10, 12, 16], &[16], &[12], &[10], &[4]] { let zip64 = as_zip64(&archive, saturate); assert_eq!(read(&zip64, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}"); diff --git a/crates/unzip/src/testkit.rs b/crates/zip/src/lenient/testkit.rs similarity index 99% rename from crates/unzip/src/testkit.rs rename to crates/zip/src/lenient/testkit.rs index ae55cc052..ca87510e9 100644 --- a/crates/unzip/src/testkit.rs +++ b/crates/zip/src/lenient/testkit.rs @@ -3,7 +3,7 @@ #![allow(clippy::cast_possible_truncation, dead_code)] -use crate::crc32; +use crate::lenient::crc32; /// Plaintext of [`XZ_FIXTURE`] and [`LZMA_FIXTURE`]. pub const XZ_PLAIN: &[u8] = b"Hello, xz compression world!\n"; diff --git a/crates/unzip/src/tests.rs b/crates/zip/src/lenient/tests.rs similarity index 96% rename from crates/unzip/src/tests.rs rename to crates/zip/src/lenient/tests.rs index 3ea92afe4..c2ca97394 100644 --- a/crates/unzip/src/tests.rs +++ b/crates/zip/src/lenient/tests.rs @@ -39,7 +39,7 @@ pub(crate) fn build(files: &[(&str, &[u8])]) -> Vec { out } -#[cfg(all(feature = "bzip2", feature = "lzma"))] +#[cfg(all(feature = "lenient-bzip2", feature = "lenient-lzma"))] fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { let mut file = build(&[("entry", compressed)]); let central = 30 + "entry".len() + compressed.len(); @@ -55,10 +55,10 @@ fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { file } -#[cfg(all(feature = "bzip2", feature = "lzma"))] +#[cfg(all(feature = "lenient-bzip2", feature = "lenient-lzma"))] #[test] fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { - use crate::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; + use crate::lenient::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; for (method, compressed, plain) in [(12, BZIP2_FIXTURE, BZIP2_PLAIN), (95, XZ_FIXTURE, XZ_PLAIN)] { let file = build_codec_entry(method, compressed, plain); let original = parse(&file, 1).unwrap().entries.remove(0); @@ -80,7 +80,7 @@ fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { } } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { let mut encoded = Vec::new(); let size = if end_marker { None } else { Some(plain.len() as u64) }; @@ -104,7 +104,7 @@ fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { file } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] #[test] fn zip_lzma_reads_size_delimited_and_end_marked_streams() { for end_marker in [false, true] { @@ -129,7 +129,7 @@ fn zip_lzma_reads_size_delimited_and_end_marked_streams() { } } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] #[test] fn zip_lzma_rejects_bad_properties_without_unbounded_dictionaries() { let mut file = zip_lzma_member(b"small", true); @@ -286,7 +286,7 @@ fn lists_and_reads_stored_entries() { assert!(matches!(read_entry(&z, &a.entries[1], 3), Err(ZipError::TooLarge(3)))); } -#[cfg(feature = "zstd")] +#[cfg(feature = "lenient-zstd")] #[test] fn zstandard_members_are_read_with_their_limits_and_checks() { // A raw Zstandard frame holding "hello zip" (single segment, one raw block). @@ -373,7 +373,7 @@ fn crc_known_value() { #[cfg(feature = "serde")] #[test] fn listings_serialize_without_empty_warnings_or_unix_mode() { - let data = crate::testkit::zip_stored(&[("a.txt", b"hi")]); + let data = crate::lenient::testkit::zip_stored(&[("a.txt", b"hi")]); let archive = parse(&data, 10).unwrap(); let value = serde_json::to_value(&archive).unwrap(); assert!(value.get("warnings").is_none()); @@ -381,10 +381,10 @@ fn listings_serialize_without_empty_warnings_or_unix_mode() { assert!(value["entries"][0].get("unix_mode").is_none()); } -#[cfg(not(feature = "zstd"))] +#[cfg(not(feature = "lenient-zstd"))] #[test] fn a_method_left_out_of_the_build_is_unsupported() { - let mut data = crate::testkit::zip_stored(&[("a", b"abc")]); + let mut data = crate::lenient::testkit::zip_stored(&[("a", b"abc")]); let archive = parse(&data, 10).unwrap(); let mut entry = archive.entries[0].clone(); entry.method = 93; diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index 0a348dc1f..ff8ce17a6 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -13,6 +13,9 @@ //! timestamp on every entry so equal input gives equal bytes. //! - [`preflight`] is a stricter check of the whole end-record and central-directory structure //! on the raw bytes, for a caller that wants nothing ambiguous to reach a ZIP reader at all. +//! - `lenient` (feature `lenient`) is the other way round: a reader of the raw bytes without +//! `zip`, for inspectors that list and describe damaged or unusual archives instead of refusing +//! them, with bounded, CRC-checked entry reads. //! //! Every limit is the caller's number. Entry-name rules (which names are safe, which spellings //! are one part) stay with the format that knows them. @@ -23,6 +26,8 @@ struct ReadmeExamples; mod archive; mod guard; +#[cfg(feature = "lenient")] +pub mod lenient; mod metadata_reader; pub mod preflight; mod seekable; diff --git a/crates/unzip/tests/common/mod.rs b/crates/zip/tests/lenient_common/mod.rs similarity index 98% rename from crates/unzip/tests/common/mod.rs rename to crates/zip/tests/lenient_common/mod.rs index ba0f3d778..05d8c5fff 100644 --- a/crates/unzip/tests/common/mod.rs +++ b/crates/zip/tests/lenient_common/mod.rs @@ -2,7 +2,7 @@ #![allow(dead_code, clippy::cast_possible_truncation)] -use craft_unzip::crc32; +use craft_zip::lenient::crc32; /// A stored-only ZIP of `files`. pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { diff --git a/crates/unzip/tests/zip64_end_record_compatibility.rs b/crates/zip/tests/lenient_zip64_end_record_compatibility.rs similarity index 98% rename from crates/unzip/tests/zip64_end_record_compatibility.rs rename to crates/zip/tests/lenient_zip64_end_record_compatibility.rs index 37f63cf76..ed599fbda 100644 --- a/crates/unzip/tests/zip64_end_record_compatibility.rs +++ b/crates/zip/tests/lenient_zip64_end_record_compatibility.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip64_end_record_compatibility.rs. -use craft_unzip::{self as zip, ZipArchive, ZipError}; +use craft_zip::lenient::{self as zip, ZipArchive, ZipError}; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip64_extra_boundaries.rs b/crates/zip/tests/lenient_zip64_extra_boundaries.rs similarity index 96% rename from crates/unzip/tests/zip64_extra_boundaries.rs rename to crates/zip/tests/lenient_zip64_extra_boundaries.rs index a8c673d08..fae4fd5f9 100644 --- a/crates/unzip/tests/zip64_extra_boundaries.rs +++ b/crates/zip/tests/lenient_zip64_extra_boundaries.rs @@ -1,7 +1,9 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip64_extra_boundaries.rs. +#[path = "lenient_common/mod.rs"] mod common; -use craft_unzip as zip; +use craft_zip::lenient as zip; fn archive(extra: &[u8], sentinel: usize) -> Vec { let mut data = common::zip_stored(&[("a", b"x")]); diff --git a/crates/unzip/tests/zip_decoded_size.rs b/crates/zip/tests/lenient_zip_decoded_size.rs similarity index 92% rename from crates/unzip/tests/zip_decoded_size.rs rename to crates/zip/tests/lenient_zip_decoded_size.rs index 5bf15ea94..9db3ec80e 100644 --- a/crates/unzip/tests/zip_decoded_size.rs +++ b/crates/zip/tests/lenient_zip_decoded_size.rs @@ -1,7 +1,9 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_decoded_size.rs. +#[path = "lenient_common/mod.rs"] mod common; -use craft_unzip::{self as zip, ZipEntry, ZipError}; +use craft_zip::lenient::{self as zip, ZipEntry, ZipError}; fn deflated(body: &[u8]) -> (Vec, ZipEntry) { let compressed = miniz_oxide::deflate::compress_to_vec(body, 6); diff --git a/crates/unzip/tests/zip_directory_bounds.rs b/crates/zip/tests/lenient_zip_directory_bounds.rs similarity index 96% rename from crates/unzip/tests/zip_directory_bounds.rs rename to crates/zip/tests/lenient_zip_directory_bounds.rs index 3f594dd74..8e912fc3b 100644 --- a/crates/unzip/tests/zip_directory_bounds.rs +++ b/crates/zip/tests/lenient_zip_directory_bounds.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_directory_bounds.rs. -use craft_unzip::{self as zip, ZipError}; +use craft_zip::lenient::{self as zip, ZipError}; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip_entry_detection.rs b/crates/zip/tests/lenient_zip_entry_detection.rs similarity index 93% rename from crates/unzip/tests/zip_entry_detection.rs rename to crates/zip/tests/lenient_zip_entry_detection.rs index 8a74ecf63..8708b6400 100644 --- a/crates/unzip/tests/zip_entry_detection.rs +++ b/crates/zip/tests/lenient_zip_entry_detection.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_entry_detection.rs. -use craft_unzip::has_entry as zip_has_entry; +use craft_zip::lenient::has_entry as zip_has_entry; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip_split_volumes.rs b/crates/zip/tests/lenient_zip_split_volumes.rs similarity index 98% rename from crates/unzip/tests/zip_split_volumes.rs rename to crates/zip/tests/lenient_zip_split_volumes.rs index c5412b72e..9675177ce 100644 --- a/crates/unzip/tests/zip_split_volumes.rs +++ b/crates/zip/tests/lenient_zip_split_volumes.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_split_volumes.rs. -use craft_unzip::{self as zip, ZipError}; +use craft_zip::lenient::{self as zip, ZipError}; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip_stored_cap.rs b/crates/zip/tests/lenient_zip_stored_cap.rs similarity index 88% rename from crates/unzip/tests/zip_stored_cap.rs rename to crates/zip/tests/lenient_zip_stored_cap.rs index 61bb06d50..a5424605f 100644 --- a/crates/unzip/tests/zip_stored_cap.rs +++ b/crates/zip/tests/lenient_zip_stored_cap.rs @@ -1,7 +1,9 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_stored_cap.rs. +#[path = "lenient_common/mod.rs"] mod common; -use craft_unzip::{self as zip, ZipError}; +use craft_zip::lenient::{self as zip, ZipError}; #[test] fn a_stored_entry_cannot_understate_its_size_to_bypass_the_output_limit() { From b367c04c9deb2a1ac7dc391b270572ec4ab44b40 Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 12:25:17 +0100 Subject: [PATCH 037/209] Keep cached results for malformed empty XLSB names --- crates/xlsx/src/xlsb.rs | 16 ++++++++++++---- crates/xlsx/tests/xlsb.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index c1a2f8962..ff54ffcaf 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -449,11 +449,19 @@ fn defined_names( } }, }; - // Excel writes a workbook-level built-in with no formula (`Database`) beside a sheet's - // own; `.xlsx` leaves it out of `definedNames`, and formulas using it spell its name. + // Excel omits the formula for a workbook-level Database built-in and a VBA function + // (fFunc, fOB, fProc). They remain usable by formulas without a definedNames entry. + // Other empty names are malformed: omitting them must also omit their dependants. if p.rgce.is_empty() { - if let Some(n) = links.names.get_mut(p.index) { - n.read = true; + let database = p.flags & 0x20 != 0 && name.scope.is_none() && name.text == "_xlnm.Database"; + let vba_function = p.flags & 0x0E == 0x0E; + if database || vba_function { + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + } else { + warnings.warn("XLSB: some defined names were left out (empty formula)"); + omit(&mut omitted, p.index); } continue; } diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 27bcc18c1..6b403bfe8 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1737,3 +1737,34 @@ fn module_sheets_and_supporting_links_are_bounded() { assert_eq!(wb.sheets[0].formula(CellRef::new(0, 0)), None); assert!(wb.warnings.iter().any(|w| w.contains("sheet links or names could not be read")), "{:?}", wb.warnings); } + +#[test] +fn empty_ordinary_names_and_their_dependants_keep_cached_values() { + // Forward and transitive references to an empty ordinary name must not become dangling + // formulas. An empty sheet-scoped Database or another built-in is not the observed + // workbook-level Database exception either. + let mut list = names(); + let cell = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + list.push(brt_name(0, 0, WORKBOOK, "Early", &cell(16), None)); // 14 + list.push(brt_name(0, 0, WORKBOOK, "Empty", &[], None)); // 15 + list.push(brt_name(0, 0, WORKBOOK, "Alias", &cell(15), None)); // 16 + list.push(brt_name(0x20, 0, 0, "Database", &[], None)); // 17 + list.push(brt_name(0x20, 0, WORKBOOK, "Print_Area", &[], None)); // 18 + list.push(brt_name(0, 0, WORKBOOK, "Fine", &cell(9), None)); // 19 + let mut data = row(0); + for i in 14..=19 { + data.extend(formula_cell(i - 13, &cell(i), &[])); + } + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = NAMES_XML.replace("", r#"Rate"#); + assert_eq!(wb.defined_names, xml_names(&xml)); + let book = read(&bytes).unwrap(); + for col in 1..=5 { + assert_eq!(book.sheets[0].formula(CellRef::new(0, col)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, col)), Value::Number(0.0)); + } + assert_eq!(book.sheets[0].formula(CellRef::new(0, 6)), Some("Fine")); + assert!(book.warnings.iter().any(|w| w.contains("empty formula")), "{:?}", book.warnings); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} From 69d26616e0881d90ad5132b225e76d6faa86b35a Mon Sep 17 00:00:00 2001 From: tsw2k Date: Mon, 16 Nov 2026 11:34:11 +0000 Subject: [PATCH 038/209] craft-rtf: tables keep their column grid on odd rows Moves storytold/wordcraft#4595 into craft-rtf, where WordCraft's RTF reader now lives (#778). column_grid dropped the whole grid, so every spanning cell fell back to one of N equal columns, when a table had more than 63 distinct cell edges, a zero-width cell, a cell without \cellx, or two edges under a point apart (which the document model rejects as a column width). Forms exported from spreadsheets, with many narrow columns, hit this: a report title wrapped a few words per line in a quarter of the page and the document grew to hundreds of pages. Now the narrowest column merges into the next, one at a time, while there are over 63 columns, a column of a point or less, or a column under 6 pt with more columns left than the widest row has cells; each cell edge snaps to the nearest grid edge, and a zero-width or undefined cell takes a column in its own row only. --- crates/rtf/src/doc/parse.rs | 140 +++++++++++++++++++++++++++++++----- crates/rtf/src/doc/tests.rs | 135 ++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 19 deletions(-) diff --git a/crates/rtf/src/doc/parse.rs b/crates/rtf/src/doc/parse.rs index 60a65740a..f44d666ce 100644 --- a/crates/rtf/src/doc/parse.rs +++ b/crates/rtf/src/doc/parse.rs @@ -1,3 +1,6 @@ +use std::cmp::Reverse; +use std::collections::BinaryHeap; + use super::reader::Reader; use super::state::{CommentData, Dest, HeaderFooter, HfPages, MARK_CLOSE, MARK_OPEN, MAX_PICT, NoteData, PageGeom}; use super::{Arc, Block, Blocks, Cell, Document, FBlock, Flow, HashMap, InlineObject, Para, Paragraph, model}; @@ -262,43 +265,142 @@ fn finalize(mut r: Reader) -> Parsed { Parsed { flow: Flow { blocks, meta: r.meta }, page: r.page, last_section, default_font, headers_footers, notes: r.notes, comments: r.comments } } +/// Most grid columns a table gets (the document model's own cap). +const MAX_GRID_COLS: usize = 63; +/// Grid columns under this width, in twips (6 pt), merge while the rows leave room: import widens +/// any narrower column to 6 pt, which grows the table, and files exported from spreadsheets put +/// edges a few twips apart. +const MIN_GRID_GAP: i64 = 120; +/// Grid columns of this width or less, in twips (1 pt), always merge: the document model refuses +/// them, and would then give the table equal columns. +const NO_GRID_GAP: i64 = 20; + /// The column grid of a table from its rows' cell right edges: every distinct edge starts a grid /// column, and each cell spans the grid columns between its left and right edges, as Word lays /// out RTF rows whose cells don't line up. Returns the widths in points, or none (equal columns) -/// when the edges don't make a grid. +/// when no grid fits, which only a row of more cells than grid columns can cause. +/// +/// One odd row never costs the whole table its grid (a form exported from a spreadsheet, with +/// many narrow columns, otherwise had its wide title squeezed into one of four equal columns): +/// - Over [`MAX_GRID_COLS`] edges, or edges closer than [`MIN_GRID_GAP`], merge, leaving a column +/// for each cell of the widest row ([`coarse_edges`]); each cell edge moves to the nearest +/// grid edge. +/// - A cell that ends at or before the previous one (a zero-width cell, or a cell with no +/// `\cellx` of its own) still gets a grid column of its own; cells pushed past the table's +/// right edge move back. pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec { + if rights.len() != rows.len() { + return Vec::new(); + } let mut edges: Vec = rights.iter().flatten().copied().filter(|&x| x > 0).collect(); edges.sort_unstable(); edges.dedup(); - let ok = rights.len() == rows.len() - && rows - .iter() - .zip(rights) - .all(|(r, x)| r.len() == x.len() && x.windows(2).all(|w| w.first() < w.get(1)) && x.first().is_some_and(|&v| v > 0)) - && !edges.is_empty() - && edges.len() <= 63; - if !ok { + if edges.is_empty() { return Vec::new(); } - for (row, x) in rows.iter_mut().zip(rights) { + let grid = coarse_edges(&edges, rows.iter().map(Vec::len).max().unwrap_or(0)); + // Each cell's right edge as a grid edge number (1 = the first grid column's right edge). + let mut ends: Vec> = Vec::with_capacity(rows.len()); + for (row, x) in rows.iter().zip(rights) { + if row.len() > grid.len() { + return Vec::new(); + } + let mut at: Vec = Vec::with_capacity(row.len()); + let mut prev = 0; + for i in 0..row.len() { + let end = x.get(i).map_or(0, |&r| nearest_edge(&grid, r)).max(prev + 1); + at.push(end); + prev = end; + } + // Cells pushed past the right edge move back, each keeping a column of its own. + let mut limit = grid.len(); + for end in at.iter_mut().rev() { + *end = (*end).min(limit); + limit = end.saturating_sub(1); + } + ends.push(at); + } + for (row, at) in rows.iter_mut().zip(&ends) { let mut left = 0; - for (c, &right) in row.iter_mut().zip(x) { - let span = edges.iter().filter(|&&e| e > left && e <= right).count().max(1); - c.colspan = span as u32; - left = right; + for (c, &end) in row.iter_mut().zip(at) { + c.colspan = u32::try_from(end.saturating_sub(left)).unwrap_or(1).max(1); + left = end; } } - let mut prev = 0; - edges - .iter() + let mut prev = 0i64; + grid.iter() .map(|&e| { - let w = (e - prev) as f32 / 20.0; - prev = e; + let w = (i64::from(e) - prev) as f32 / 20.0; + prev = i64::from(e); w }) .collect() } +/// The grid edges kept from the sorted, distinct, positive `edges`. The narrowest column merges +/// into the next, one at a time, so an edge moves only when it has to: while there are over +/// [`MAX_GRID_COLS`] columns, or columns of [`NO_GRID_GAP`] or less, and while a column is under +/// [`MIN_GRID_GAP`] and more columns remain than the `widest_row`'s cells. The table's right edge +/// is always kept. +fn coarse_edges(edges: &[i32], widest_row: usize) -> Vec { + let n = edges.len(); + // Each edge's neighbours among the edges still kept. + let mut left: Vec> = (0..n).map(|i| i.checked_sub(1)).collect(); + let mut right: Vec> = (1..=n).map(|i| (i < n).then_some(i)).collect(); + let mut kept = vec![true; n]; + let at = |i: Option| i.and_then(|i| edges.get(i)).map_or(0, |&e| i64::from(e)); + let width = |i: usize, left: &[Option]| at(Some(i)) - at(left.get(i).copied().flatten()); + let mut narrowest: BinaryHeap> = (0..n).map(|i| Reverse((width(i, &left), i))).collect(); + let mut count = n; + while count > 1 { + let Some(Reverse((w, i))) = narrowest.pop() else { + break; + }; + // A column that has since grown, or gone, has a fresher entry or none. + if !kept.get(i).copied().unwrap_or(false) || w != width(i, &left) { + continue; + } + if count <= MAX_GRID_COLS && w > NO_GRID_GAP && (w >= MIN_GRID_GAP || count <= widest_row) { + break; + } + // The column's right edge goes, unless it is the table's: then its left edge does. + let last = right.get(i).copied().flatten().is_none(); + let Some(gone) = (if last { left.get(i).copied().flatten() } else { Some(i) }) else { + break; + }; + let before = left.get(gone).copied().flatten(); + let after = right.get(gone).copied().flatten(); + if let Some(slot) = before.and_then(|b| right.get_mut(b)) { + *slot = after; + } + if let Some(a) = after { + if let Some(slot) = left.get_mut(a) { + *slot = before; + } + narrowest.push(Reverse((width(a, &left), a))); + } + if let Some(slot) = kept.get_mut(gone) { + *slot = false; + } + count -= 1; + } + edges.iter().zip(&kept).filter(|&(_, &k)| k).map(|(&e, _)| e).collect() +} + +/// The number of the grid edge nearest `x` (1-based; 0 is the table's left edge). +fn nearest_edge(grid: &[i32], x: i32) -> usize { + if x <= 0 { + return 0; + } + let p = grid.partition_point(|&e| e < x); + let below = if p == 0 { 0 } else { grid.get(p - 1).copied().unwrap_or(0) }; + match grid.get(p) { + Some(&above) if i64::from(above) - i64::from(x) <= i64::from(x) - i64::from(below) => p + 1, + Some(_) => p, + None => grid.len(), + } +} + /// Parse RTF bytes into a document. pub fn import(bytes: &[u8]) -> Result { model::begin_import(); diff --git a/crates/rtf/src/doc/tests.rs b/crates/rtf/src/doc/tests.rs index 2a4103eb1..fda05f3b4 100644 --- a/crates/rtf/src/doc/tests.rs +++ b/crates/rtf/src/doc/tests.rs @@ -460,6 +460,141 @@ fn table_grid_from_cell_edges_and_borders() { assert_eq!(t.grid, [240.9, 241.0]); } +/// The table of `rtf`'s flow and every row's column spans; checks the spans fit the grid. +fn grid_of(rtf: &[u8]) -> (Vec, Vec>) { + let f = parse(rtf).unwrap(); + let FBlock::Table(t) = &f.blocks[0] else { panic!("no table") }; + let spans: Vec> = t.rows.iter().map(|r| r.iter().map(|c| c.colspan).collect()).collect(); + assert!(!t.widths.is_empty(), "the table kept no grid"); + assert_eq!(t.cols(), t.widths.len(), "spans {spans:?} don't fit {} grid columns", t.widths.len()); + assert!(t.widths.iter().all(|&w| w > 1.0), "a grid column under a point: {:?}", t.widths); + (t.widths.clone(), spans) +} + +/// Width in points of the first `n` grid columns. +fn first_cols(widths: &[f32], n: u32) -> f32 { + widths.iter().take(n as usize).sum() +} + +#[test] +fn table_grid_survives_more_than_63_edges() { + // A form exported from a spreadsheet: a wide title over rows whose middle edge moves by + // 100 twips, 80 distinct edges in all. The grid used to be dropped, giving equal columns. + let mut rtf = String::from(r"{\rtf1\ansi\trowd\cellx6000\cellx8000\pard\intbl Title\cell Codes\cell\row"); + for k in 1..80 { + rtf.push_str(&format!(r"\trowd\cellx{}\cellx8000\pard\intbl a\cell b\cell\row", k * 100)); + } + rtf.push_str(r"\pard\par}"); + let (widths, spans) = grid_of(rtf.as_bytes()); + assert!(widths.len() <= 63, "{} grid columns", widths.len()); + assert!((widths.iter().sum::() - 400.0).abs() < 0.01, "the table keeps its 400 pt"); + // The title keeps its 300 pt, within a grid column's merge. + let title = first_cols(&widths, spans[0][0]); + assert!((title - 300.0).abs() <= 5.0, "title {title} pt"); + // The staircase still climbs: the 60th row's first cell is about 300 pt wide too. + assert!((first_cols(&widths, spans[60][0]) - 300.0).abs() <= 5.0); + + // The same through import: the document table has the grid, not equal columns. + let d = import(rtf.as_bytes()).unwrap(); + let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); + assert_eq!(t.grid.len(), widths.len()); + let w = t.rows[0].cells[0].props.width.unwrap(); + assert!((w - 300.0).abs() <= 5.0, "title cell {w} pt"); +} + +#[test] +fn table_grid_survives_a_zero_width_cell_and_a_missing_cellx() { + // A zero-width cell (two edges at 3000) in one row: the other rows keep their grid. + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx6000\cellx9000 \pard\intbl Title\cell b\cell\row +\trowd\cellx2000\cellx2000\cellx9000 \pard\intbl a\cell\cell c\cell\row +\pard\par}", + ); + assert_eq!(first_cols(&widths, spans[0][0]), 300.0); + assert_eq!(spans[1].len(), 3, "the zero-width cell is kept: {spans:?}"); + assert_eq!(first_cols(&widths, spans[1][0]), 100.0, "the cell before it keeps its width"); + + // A row with a third cell but only two `\cellx`. + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx6000\cellx9000 \pard\intbl Title\cell b\cell\row +\trowd\cellx2000\cellx6000 \pard\intbl a\cell b\cell extra\cell\row +\pard\par}", + ); + assert_eq!(first_cols(&widths, spans[0][0]), 300.0); + assert_eq!(spans[1].len(), 3); + assert_eq!(first_cols(&widths, spans[1][0]), 100.0); +} + +#[test] +fn table_grid_merges_edges_a_few_twips_apart() { + // Two rows that mean the same 3000-twip edge, 5 twips apart: one grid column, not one of + // a quarter point (which the document model refuses, so the table got equal columns). + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx3000\cellx9000 \pard\intbl a\cell b\cell\row +\trowd\cellx3005\cellx9000 \pard\intbl c\cell d\cell\row +\pard\par}", + ); + assert_eq!(widths.len(), 2, "{widths:?}"); + assert_eq!(spans, [[1, 1], [1, 1]]); + assert_eq!(widths.iter().sum::(), 450.0); +} + +/// `edges` as one table row, a cell for each. +fn rtf_row(edges: &[usize]) -> String { + let cells: String = edges.iter().map(|e| format!(r"\cellx{e}")).collect(); + format!(r"\trowd{cells}\pard\intbl {}\row", r"x\cell ".repeat(edges.len())) +} + +#[test] +fn table_grid_keeps_a_column_for_each_cell_of_the_widest_row() { + // 64 evenly spaced edges: merging by a common gap halves them to 32 columns, too few for a + // row of 33 or more cells, and the table lost its grid. + let edges: Vec = (1..=64).map(|k| k * 100).collect(); + for cells in [33, 40, 50, 62] { + let wide: Vec = edges.iter().copied().take(cells).chain([6400]).collect(); + let rest: Vec = edges.iter().copied().skip(cells - 1).collect(); + let rtf = format!(r"{{\rtf1\ansi {}{}\pard\par}}", rtf_row(&wide), rtf_row(&rest)); + let (widths, spans) = grid_of(rtf.as_bytes()); + assert!((cells + 1..=63).contains(&widths.len()), "{cells} cells, {} grid columns", widths.len()); + assert_eq!(spans[0].len(), cells + 1, "every cell is kept"); + assert_eq!(widths.iter().sum::(), 320.0, "the table keeps its 320 pt"); + } +} + +#[test] +fn table_grid_merges_columns_import_would_widen() { + // Columns under 6 pt that no row needs: import raises each one to 6 pt, so this table of + // 3 pt columns doubled in width. Merged, it keeps its 60 pt. + let mut rtf = String::from(r"{\rtf1\ansi "); + for k in 1..=20 { + rtf.push_str(&rtf_row(&[k * 60, 1200])); + } + rtf.push_str(r"\pard\par}"); + let (widths, _) = grid_of(rtf.as_bytes()); + assert!(widths.iter().all(|&w| w >= 6.0), "{widths:?}"); + let d = import(rtf.as_bytes()).unwrap(); + let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); + assert_eq!(t.grid.iter().sum::(), 60.0, "{:?}", t.grid); + + // A row that needs them keeps them, narrow as they are. + let (widths, spans) = grid_of(format!(r"{{\rtf1\ansi {}\pard\par}}", rtf_row(&[60, 120, 180])).as_bytes()); + assert_eq!(widths, [3.0, 3.0, 3.0]); + assert_eq!(spans, [[1, 1, 1]]); +} + +#[test] +fn table_grid_never_keeps_a_column_the_document_model_refuses() { + // Three cells within a point of each other: too narrow to keep even for a row that needs + // them. The row can't fit, so the table has no grid, and nothing panics. + let f = parse(format!(r"{{\rtf1\ansi {}\pard\par}}", rtf_row(&[5, 10, 15])).as_bytes()).unwrap(); + let FBlock::Table(t) = &f.blocks[0] else { panic!("no table") }; + assert!(t.widths.is_empty(), "{:?}", t.widths); + assert_eq!(t.rows[0].len(), 3); +} + #[test] fn code_pages_from_ansicpg_and_font_charsets() { // The document's code page: Russian RTF as accounting systems write it. From fe68ce8c51d770a9586cfe6939ee2a28f2ecbc17 Mon Sep 17 00:00:00 2001 From: tsw2k Date: Mon, 16 Nov 2026 11:40:14 +0000 Subject: [PATCH 039/209] craft-flow: narrow table columns keep their width on import Moves the import part of storytold/wordcraft#4964 into craft-flow, where WordCraft's RTF and ODT import now builds its tables (#778). Import raised every column under 6 pt to 6 pt, so a form with many narrow columns came out wider than its source. Columns over 1 pt now keep the width the file gives; one of a point or less, none or not a number still gives the table equal columns, so nothing below a point, negative or NaN reaches layout. craft-rtf merged grid columns under 6 pt that no row needed, because import would widen them. With import keeping them, that merge only moved cell edges, so it goes: grid columns merge only while there are over 63 or one is a point or less. Stacked on the craft-rtf column grid change. --- crates/flow/src/to_doc.rs | 3 ++- crates/odf/src/text/tests.rs | 35 +++++++++++++++++++++++++++++++++++ crates/rtf/src/doc/parse.rs | 21 ++++++++------------- crates/rtf/src/doc/tests.rs | 28 ++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 18 deletions(-) diff --git a/crates/flow/src/to_doc.rs b/crates/flow/src/to_doc.rs index ce29a4448..0375c448c 100644 --- a/crates/flow/src/to_doc.rs +++ b/crates/flow/src/to_doc.rs @@ -841,7 +841,8 @@ impl<'a> Builder<'a> { tb.props.style = None; } if t.widths.len() == cols && t.widths.iter().all(|w| w.is_finite() && *w > 1.0) { - tb.grid = t.widths.iter().map(|w| w.clamp(6.0, 1584.0)).collect(); + // Narrow columns stay as they are: raising them would widen the table past its source. + tb.grid = t.widths.iter().map(|w| w.min(1584.0)).collect(); } tb.rows.clear(); if let Some(x) = &t.extra { diff --git a/crates/odf/src/text/tests.rs b/crates/odf/src/text/tests.rs index 8d7af18dc..da00b350c 100644 --- a/crates/odf/src/text/tests.rs +++ b/crates/odf/src/text/tests.rs @@ -442,6 +442,41 @@ fn table_column_widths_survive_a_round_trip() { assert_eq!(grid(&back), widths); } +#[test] +fn narrow_table_columns_keep_their_width() { + // Columns of a few points stay as the file gives them, so the table stays as wide as its source. + let content = format!( + r##"AB"## + ); + let doc = import(&package(&content, None)).unwrap(); + let grid_of = |doc: &craft_wordmodel::Document| { + doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }) + }; + assert_eq!(grid_of(&doc), Some(vec![3.0, 100.0])); + // Saved and opened again, the narrow column is still the file's width. + let back = import(&export(&doc).unwrap()).unwrap(); + let grid = grid_of(&back).unwrap(); + assert_eq!(grid.len(), 2, "{grid:?}"); + assert!((grid[0] - 3.0).abs() < 0.01 && (grid[1] - 100.0).abs() < 0.01, "{grid:?}"); +} + +#[test] +fn table_columns_of_a_point_or_less_give_equal_columns() { + // A column of a point or less, nothing or NaN (a damaged or hostile file) is no width to + // lay out: the whole grid falls back to equal columns, and nothing reaches layout below a + // point, negative or not a number. + for bad in ["1pt", "0.5pt", "0pt", "-3pt", "NaNpt"] { + let content = format!( + r##"AB"## + ); + let doc = import(&package(&content, None)).unwrap(); + let grid = doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }).unwrap(); + assert_eq!(grid.len(), 2, "{bad}: {grid:?}"); + assert!(grid.iter().all(|w| w.is_finite() && *w > 1.0), "{bad}: {grid:?}"); + assert_eq!(grid[0], grid[1], "{bad}: equal columns, {grid:?}"); + } +} + #[test] fn table_without_column_widths_keeps_the_equal_split() { let content = format!( diff --git a/crates/rtf/src/doc/parse.rs b/crates/rtf/src/doc/parse.rs index f44d666ce..01978da1e 100644 --- a/crates/rtf/src/doc/parse.rs +++ b/crates/rtf/src/doc/parse.rs @@ -267,10 +267,6 @@ fn finalize(mut r: Reader) -> Parsed { /// Most grid columns a table gets (the document model's own cap). const MAX_GRID_COLS: usize = 63; -/// Grid columns under this width, in twips (6 pt), merge while the rows leave room: import widens -/// any narrower column to 6 pt, which grows the table, and files exported from spreadsheets put -/// edges a few twips apart. -const MIN_GRID_GAP: i64 = 120; /// Grid columns of this width or less, in twips (1 pt), always merge: the document model refuses /// them, and would then give the table equal columns. const NO_GRID_GAP: i64 = 20; @@ -282,9 +278,9 @@ const NO_GRID_GAP: i64 = 20; /// /// One odd row never costs the whole table its grid (a form exported from a spreadsheet, with /// many narrow columns, otherwise had its wide title squeezed into one of four equal columns): -/// - Over [`MAX_GRID_COLS`] edges, or edges closer than [`MIN_GRID_GAP`], merge, leaving a column -/// for each cell of the widest row ([`coarse_edges`]); each cell edge moves to the nearest -/// grid edge. +/// - Over [`MAX_GRID_COLS`] edges, or edges [`NO_GRID_GAP`] or less apart, merge +/// ([`coarse_edges`]); each cell edge moves to the nearest grid edge. Narrow columns over a +/// point stay: import keeps them as narrow as the file gives them. /// - A cell that ends at or before the previous one (a zero-width cell, or a cell with no /// `\cellx` of its own) still gets a grid column of its own; cells pushed past the table's /// right edge move back. @@ -298,7 +294,7 @@ pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec> = Vec::with_capacity(rows.len()); for (row, x) in rows.iter().zip(rights) { @@ -339,10 +335,9 @@ pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec Vec { +/// [`MAX_GRID_COLS`] columns, or columns of [`NO_GRID_GAP`] or less. The table's right edge is +/// always kept. +fn coarse_edges(edges: &[i32]) -> Vec { let n = edges.len(); // Each edge's neighbours among the edges still kept. let mut left: Vec> = (0..n).map(|i| i.checked_sub(1)).collect(); @@ -360,7 +355,7 @@ fn coarse_edges(edges: &[i32], widest_row: usize) -> Vec { if !kept.get(i).copied().unwrap_or(false) || w != width(i, &left) { continue; } - if count <= MAX_GRID_COLS && w > NO_GRID_GAP && (w >= MIN_GRID_GAP || count <= widest_row) { + if count <= MAX_GRID_COLS && w > NO_GRID_GAP { break; } // The column's right edge goes, unless it is the table's: then its left edge does. diff --git a/crates/rtf/src/doc/tests.rs b/crates/rtf/src/doc/tests.rs index fda05f3b4..fddd198da 100644 --- a/crates/rtf/src/doc/tests.rs +++ b/crates/rtf/src/doc/tests.rs @@ -389,6 +389,26 @@ fn page_size_and_margins() { assert_eq!(d.last_section, craft_wordmodel::SectionProps::default()); } +#[test] +fn narrow_grid_columns_keep_their_width() { + // Rows' edges a few points apart, as in a form exported from a spreadsheet. The table stays as + // wide as its source, through an export and back too. + let rtf = br"{\rtf1\ansi +\trowd\cellx400\cellx800 \pard\intbl a\cell b\cell\row +\trowd\cellx80\cellx480\cellx800 \pard\intbl c\cell d\cell e\cell\row +\pard after\par}"; + let table = |doc: &Document| match doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.clone()) } else { None }) + { + Some(t) => t, + None => panic!("no table"), + }; + let doc = import(rtf).unwrap(); + let t = table(&doc); + assert_eq!(t.grid, [4.0, 16.0, 4.0, 16.0]); + assert_eq!(t.rows[0].cells.iter().map(|c| c.props.width).collect::>(), [Some(20.0), Some(20.0)]); + assert_eq!(table(&import(export(&doc).as_bytes()).unwrap()).grid, t.grid); +} + #[test] fn table_grid_from_cell_edges_and_borders() { // Rows whose cells don't line up share one grid: edges 2000, 3000, 9000. @@ -565,16 +585,16 @@ fn table_grid_keeps_a_column_for_each_cell_of_the_widest_row() { } #[test] -fn table_grid_merges_columns_import_would_widen() { - // Columns under 6 pt that no row needs: import raises each one to 6 pt, so this table of - // 3 pt columns doubled in width. Merged, it keeps its 60 pt. +fn table_grid_keeps_narrow_columns_at_their_width() { + // Columns under 6 pt that no row needs keep their 3 pt: import no longer widens them, so the + // table keeps its 60 pt. let mut rtf = String::from(r"{\rtf1\ansi "); for k in 1..=20 { rtf.push_str(&rtf_row(&[k * 60, 1200])); } rtf.push_str(r"\pard\par}"); let (widths, _) = grid_of(rtf.as_bytes()); - assert!(widths.iter().all(|&w| w >= 6.0), "{widths:?}"); + assert_eq!(widths, [3.0; 20], "{widths:?}"); let d = import(rtf.as_bytes()).unwrap(); let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); assert_eq!(t.grid.iter().sum::(), 60.0, "{:?}", t.grid); From 8b876291a1547ebe57b282b8f86b35644f731496 Mon Sep 17 00:00:00 2001 From: tsw2k Date: Mon, 16 Nov 2026 11:46:35 +0000 Subject: [PATCH 040/209] craft-docx: a frame preset shape is a border with a hole Moves the format part of storytold/wordcraft#6162 into craft-wordmodel, craft-docx, craft-flow and craft-odf, where WordCraft's DOCX and ODT I/O now live (#778). A DrawingML `prstGeom frame` (a letterhead's page border) was read as a rectangle, so its fill covered the whole page behind the text. It is now ShapeKind::Frame, its border width read from `adj1` (capped, as a rounded rectangle's corner is) and written back as the same preset. connector::frame_width gives the border in points. Formats with no frame shape (ODT and the other flow formats) get the freeform it draws: its outline and, wound the other way, its hole, so the hole stays empty under either fill rule. --- crates/docx/src/read/shape_style.rs | 1 + crates/docx/src/read/story/drawing.rs | 13 ++++--- crates/docx/src/write/story/drawing.rs | 12 +++++-- crates/docx/tests/it/fixtures.rs | 33 ++++++++++++++++++ crates/flow/src/from_doc.rs | 20 ++++++++++- crates/odf/src/text/shapes.rs | 3 +- crates/odf/src/text/tests_more/shapes.rs | 41 +++++++++++++++++++++++ crates/wordmodel/src/connector.rs | 12 +++++++ crates/wordmodel/src/para/inline_kinds.rs | 3 ++ crates/wordmodel/src/wrap.rs | 1 + 10 files changed, 130 insertions(+), 9 deletions(-) diff --git a/crates/docx/src/read/shape_style.rs b/crates/docx/src/read/shape_style.rs index 9f28ef2ff..983218f9e 100644 --- a/crates/docx/src/read/shape_style.rs +++ b/crates/docx/src/read/shape_style.rs @@ -47,6 +47,7 @@ pub(crate) fn preset_kind(prst: &str) -> ShapeKind { "downArrow" => ShapeKind::ArrowDown, "star4" | "star5" | "star6" | "star7" | "star8" | "star10" | "star12" | "star16" | "star24" | "star32" => ShapeKind::Star, "heart" => ShapeKind::Heart, + "frame" => ShapeKind::Frame, _ => ShapeKind::Rectangle, } } diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..6bc697a3b 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -526,12 +526,17 @@ impl Reader<'_> { outline_fill: paint.outline_fill, ..ShapeExtra::default() }; - // A rounded rectangle's own corner (`adj`, ECMA-376 §20.1.10.56); other presets drawn as - // rounded rectangles keep the default. - if prst == "roundRect" { + // A rounded rectangle's own corner (`adj`, ECMA-376 §20.1.10.56), a frame's own border + // width (`adj1`); other presets drawn as rounded rectangles keep the default. + let adj = match prst { + "roundRect" => Some("adj"), + "frame" => Some("adj1"), + _ => None, + }; + if let Some(adj) = adj { extra.corner = sppr .and_then(|s| s.child("a:prstGeom")) - .and_then(|g| preset_adj(g, "adj")) + .and_then(|g| preset_adj(g, adj)) .and_then(|v| u32::try_from(v.clamp(0, i64::from(craft_wordmodel::connector::MAX_CORNER))).ok()); } // Arrowheads: any end type but `none` (ECMA-376 §20.1.10.33) is drawn as a triangle of its diff --git a/crates/docx/src/write/story/drawing.rs b/crates/docx/src/write/story/drawing.rs index 7c03cca6e..1479d2bd8 100644 --- a/crates/docx/src/write/story/drawing.rs +++ b/crates/docx/src/write/story/drawing.rs @@ -117,16 +117,22 @@ impl Writer<'_> { ShapeKind::ArrowDown => "downArrow", ShapeKind::Star => "star5", ShapeKind::Heart => "heart", + ShapeKind::Frame => "frame", }; match geom { Some(f) => cust_geom(w, f, *sw, *sh), None => { w.open("a:prstGeom", &[("prst", prst)]); - match extra.corner.filter(|_| *kind == ShapeKind::RoundedRectangle) { - Some(c) => { + let adj = match kind { + ShapeKind::RoundedRectangle => Some("adj"), + ShapeKind::Frame => Some("adj1"), + _ => None, + }; + match adj.zip(extra.corner) { + Some((name, c)) => { w.open("a:avLst", &[]); let fmla = format!("val {}", c.min(craft_wordmodel::connector::MAX_CORNER)); - w.empty("a:gd", &[("name", "adj"), ("fmla", &fmla)]); + w.empty("a:gd", &[("name", name), ("fmla", &fmla)]); w.close("a:avLst"); } None => w.empty("a:avLst", &[]), diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 348d6239e..47663008d 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -2729,6 +2729,39 @@ fn rounded_rectangle_corners_are_read_and_saved() { assert_eq!(corners(&craft_docx::read(&bytes).unwrap()), expected); } +/// A `frame` preset (a letterhead's page border) is a frame, not a filled rectangle: its border +/// width is its `adj1`, read, capped and written back as the same preset. +#[test] +fn frame_presets_keep_their_hole_and_border_width() { + let geom = + |av: &str| format!(r#"{av}"#); + let gd = |fmla: &str| format!(r#""#); + let body = [geom(&gd("val 2604")), geom(""), geom(&gd("val 90000")), geom(r#""#)] + .iter() + .map(|sppr| anchored_wsp("Frame", 2540000, 1270000, sppr, "", "")) + .collect::(); + let frames = |d: &Document| -> Vec<(ShapeKind, Option)> { + paras(d)[0] + .objects + .iter() + .filter_map(|o| match o { + InlineObject::Shape { kind, extra, .. } => Some((*kind, extra.corner)), + _ => None, + }) + .collect() + }; + let f = ShapeKind::Frame; + let expected = vec![(f, Some(2604)), (f, None), (f, Some(50_000)), (f, None)]; + let d = read_body(&format!("{body}")); + assert_eq!(frames(&d), expected); + let bytes = craft_docx::write(&d).unwrap(); + let mut xml = String::new(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes.as_slice())).unwrap(); + std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap(); + assert!(xml.contains(r#""#), "{xml}"); + assert_eq!(frames(&craft_docx::read(&bytes).unwrap()), expected); +} + /// #611: legacy VML lines and rectangles (documents without DrawingML). #[test] fn vml_lines_and_rectangles() { diff --git a/crates/flow/src/from_doc.rs b/crates/flow/src/from_doc.rs index 810e67d55..b7204f1bf 100644 --- a/crates/flow/src/from_doc.rs +++ b/crates/flow/src/from_doc.rs @@ -342,6 +342,20 @@ fn rich_inlines(doc: &Document, o: &InlineObject, f: &Fmt, opts: FlowOpts) -> Op }) } +/// The freeform a `w` × `h` frame shape with border `corner` (see +/// [`craft_wordmodel::connector::frame_width`]) draws: its outline clockwise and its hole the +/// other way, so the hole stays empty under either fill rule. +fn frame_freeform(corner: Option, w: f32, h: f32) -> craft_wordmodel::freeform::Freeform { + use craft_wordmodel::freeform::{FreePath, Freeform}; + let (w, h) = (craft_wordmodel::geom::finite(w).max(0.0), craft_wordmodel::geom::finite(h).max(0.0)); + let t = craft_wordmodel::connector::frame_width(corner, w, h); + let mut paths = vec![FreePath { pts: vec![[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]], closed: true }]; + if w - 2.0 * t > 0.0 && h - 2.0 * t > 0.0 { + paths.push(FreePath { pts: vec![[t, t], [t, h - t], [w - t, h - t], [w - t, t]], closed: true }); + } + Freeform { w, h, paths, ..Default::default() } +} + /// Comment `c` (a reply to `parent`) of `doc` as a flow comment: its text holds no comments or /// notes of its own. /// The flow inline for picture, shape or group `o` of `doc` (a member of groups `depth` deep), @@ -358,8 +372,12 @@ fn drawing_inline(doc: &Document, o: &InlineObject, opts: FlowOpts, depth: usize { // A connector stays one in formats that glue them (ODT, with boxes); elsewhere it is a // straight line. + // A frame (a border with a hole), which the other formats have no shape for, is the + // freeform it draws: its outline and, wound the other way, its hole. + let frame = (*kind == ShapeKind::Frame).then(|| std::sync::Arc::new(frame_freeform(extra.corner, *w, *h))); let kind = match kind { ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector if !opts.boxes => ShapeKind::Line, + ShapeKind::Frame => ShapeKind::Freeform, k => *k, }; let inner = FlowOpts { notes: false, comments: false, boxes: false, ..opts }; @@ -383,7 +401,7 @@ fn drawing_inline(doc: &Document, o: &InlineObject, opts: FlowOpts, depth: usize Box::new(crate::Chain { name: format!("Frame{id}"), next }) }), }; - let path = freeform.clone().filter(|_| kind == ShapeKind::Freeform); + let path = frame.or_else(|| freeform.clone()).filter(|_| kind == ShapeKind::Freeform); Some(Inline::Shape { kind, w: w * sx, h: h * sy, float: kept_float(float), blocks, look, path }) } InlineObject::Image { media, w, h, alt, float, ole, crop, .. } => { diff --git a/crates/odf/src/text/shapes.rs b/crates/odf/src/text/shapes.rs index 7c08151aa..eb184f5c9 100644 --- a/crates/odf/src/text/shapes.rs +++ b/crates/odf/src/text/shapes.rs @@ -124,7 +124,8 @@ pub(super) fn element_of(kind: ShapeKind) -> (&'static str, Option<&'static str> ShapeKind::ArrowDown => custom("down-arrow"), ShapeKind::Star => custom("star5"), ShapeKind::Heart => custom("heart"), - ShapeKind::Freeform => ("draw:path", None), + // A frame reaches the writer as the freeform it draws (see `model::from_doc`). + ShapeKind::Freeform | ShapeKind::Frame => ("draw:path", None), ShapeKind::Rectangle | ShapeKind::TextBox => ("draw:rect", None), } } diff --git a/crates/odf/src/text/tests_more/shapes.rs b/crates/odf/src/text/tests_more/shapes.rs index f6bbe9ae8..8d1a9d5fd 100644 --- a/crates/odf/src/text/tests_more/shapes.rs +++ b/crates/odf/src/text/tests_more/shapes.rs @@ -188,3 +188,44 @@ fn a_shape_with_only_an_empty_paragraph_has_no_text() { let stories: Vec = d.body.iter().filter_map(|b| b.as_para()).flat_map(|p| p.objects.iter()).map(|o| o.text_box().is_some()).collect(); assert_eq!(stories, [false, false, true, false, false, false]); } + +/// A frame shape (a letterhead's page border, from DOCX `frame`), which ODT has no shape for, +/// is written as the outline it draws: its outer edge and its hole, so LibreOffice doesn't fill +/// the whole box; read back, it is that freeform, hole and all. +#[test] +fn a_frame_shape_is_written_as_a_border_with_a_hole() { + let mut p = Paragraph::new(); + p.text.push(OBJ); + p.objects.push(InlineObject::Shape { + kind: ShapeKind::Frame, + w: 200.0, + h: 100.0, + fill: Some(craft_wordmodel::fill::Fill::solid(Rgb(1, 2, 3))), + stroke: None, + stroke_width: 0.0, + float: Default::default(), + story: None, + freeform: None, + effects: Default::default(), + // A border a tenth of the shorter side: 10 pt. + extra: craft_wordmodel::connector::ShapeExtra { corner: Some(10_000), ..Default::default() }, + }); + p.runs = vec![craft_wordmodel::para::Run { len: p.text.len(), props: Default::default() }]; + let mut d = Document::new(); + d.body = vec![para_block(p)]; + let odt = export(&d).unwrap(); + let content = content_of(&odt); + assert!(content.contains("= 9.5 * sx && *x <= 190.5 * sx && *y >= 9.5 * sy && *y <= 90.5 * sy), "{hole:?}"); +} diff --git a/crates/wordmodel/src/connector.rs b/crates/wordmodel/src/connector.rs index 366679070..2233b67f6 100644 --- a/crates/wordmodel/src/connector.rs +++ b/crates/wordmodel/src/connector.rs @@ -201,6 +201,8 @@ pub struct ShapeExtra { pub end_style: ArrowStyle, /// A rounded rectangle's corner radius (`roundRect`'s `adj`, ECMA-376 §20.1.10.56): in /// 100000ths of the shorter side, at most [`MAX_CORNER`]; `None` draws [`DEFAULT_CORNER`]. + /// A [`ShapeKind::Frame`]'s border width (`frame`'s `adj1`), in the same units; `None` + /// draws [`DEFAULT_FRAME`]. #[serde(skip_serializing_if = "Option::is_none")] pub corner: Option, /// The outline dash pattern; absent in older saved documents means solid. @@ -273,6 +275,16 @@ pub const MAX_CORNER: u32 = 50_000; /// A rounded rectangle's corner radius without its own: 16% of the shorter side. pub const DEFAULT_CORNER: f32 = 0.16; +/// A frame's border width without its own: `frame`'s default `adj1`, 12.5% of the shorter side. +pub const DEFAULT_FRAME: f32 = 0.125; + +/// The border width, in points, of a `w` × `h` [`ShapeKind::Frame`] with `corner` (see +/// [`ShapeExtra::corner`]): at most half the shorter side, which leaves no hole. +pub fn frame_width(corner: Option, w: f32, h: f32) -> f32 { + let side = crate::geom::finite(w).min(crate::geom::finite(h)).max(0.0); + side * corner.map_or(DEFAULT_FRAME, |c| c.min(MAX_CORNER) as f32 / 100_000.0) +} + /// The corner radius, in points, of a `w` × `h` rounded rectangle with `corner` (see /// [`ShapeExtra::corner`]). pub fn corner_radius(corner: Option, w: f32, h: f32) -> f32 { diff --git a/crates/wordmodel/src/para/inline_kinds.rs b/crates/wordmodel/src/para/inline_kinds.rs index 686089f7f..c36bd8fe6 100644 --- a/crates/wordmodel/src/para/inline_kinds.rs +++ b/crates/wordmodel/src/para/inline_kinds.rs @@ -24,6 +24,9 @@ pub enum ShapeKind { ArrowDown, Star, Heart, + /// A picture frame (DrawingML's `frame`): a rectangle with a rectangular hole, its border as + /// wide as [`crate::connector::frame_width`] says. + Frame, /// A text box (rectangle with a text story). TextBox, /// A shape drawn point by point, such as ink (its geometry is the shape's `freeform`). diff --git a/crates/wordmodel/src/wrap.rs b/crates/wordmodel/src/wrap.rs index 1c235d73e..9b305fafe 100644 --- a/crates/wordmodel/src/wrap.rs +++ b/crates/wordmodel/src/wrap.rs @@ -70,6 +70,7 @@ pub fn shape_outline(kind: ShapeKind, w: f32, h: f32, corner: Option) -> Op | ShapeKind::TextBox | ShapeKind::Line | ShapeKind::Freeform + | ShapeKind::Frame | ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector => return None, From dd80bb5bc5f09afae4972bf4128d24647841e29e Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 13:21:01 +0100 Subject: [PATCH 041/209] Keep cached results for shadowed XLSB name references --- crates/xlsx/src/xlsb.rs | 22 ++++++++++++++++++++- crates/xlsx/tests/xlsb.rs | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index ff54ffcaf..02087ed4a 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -10,7 +10,7 @@ mod formula; mod ftab; mod styles; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use craft_opc::{ContentTypes, Rels}; @@ -433,6 +433,14 @@ fn defined_names( // out is left out too, whatever their order, so a cell using it keeps its saved result. let mut omitted = vec![false; links.names.len()]; let mut kept = vec![]; + // PtgName identifies a record, but a bare workbook name in a sheet-scoped definition + // would instead bind to a same-spelled local name. Without a workbook qualifier, keep + // the cached results rather than silently changing that identity. Index once so hostile + // files with many names and references cannot turn this check into a quadratic scan. + // Ordinary shadows still count if their own formulas are omitted: do not infer a new + // binding from a partially emitted name environment. Synthetic names are not definitions. + let local_names: HashSet<_> = + links.names.iter().filter(|n| !n.future && !formula::is_param(&n.text)).filter_map(|n| n.scope.map(|s| (s, n.text.to_lowercase()))).collect(); for p in pending { let Some(name) = links.names.get(p.index) else { continue }; if name.future || formula::is_param(&name.text) { @@ -474,6 +482,18 @@ fn defined_names( continue; } }; + if name.scope.is_some_and(|scope| { + used.iter().filter_map(|&i| links.names.get(i)).any(|target| { + target.scope.is_none() + && !target.future + && !formula::is_param(&target.text) + && local_names.contains(&(scope, target.text.to_lowercase())) + }) + }) { + warnings.warn("XLSB: some defined names were left out (workbook names shadowed by local names)"); + omit(&mut omitted, p.index); + continue; + } if !budget.try_formula_text(formula.len(), limits) { warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); omit(&mut omitted, p.index); diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 6b403bfe8..96c7cff5d 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1768,3 +1768,44 @@ fn empty_ordinary_names_and_their_dependants_keep_cached_values() { assert!(book.warnings.iter().any(|w| w.contains("empty formula")), "{:?}", book.warnings); assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); } + +#[test] +fn scoped_names_do_not_rebind_shadowed_workbook_names() { + let token = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + let list = vec![ + brt_name(0, 0, WORKBOOK, "Rate", &[0x1E, 1, 0], None), // 1 + brt_name(0, 0, 1, "rAtE", &[0x1E, 2, 0], None), // 2: Hidden shadows Rate case-insensitively. + brt_name(0, 0, 1, "Alias", &token(1), None), // 3: must not become Hidden!rAtE. + brt_name(0, 0, 1, "Forward", &token(5), None), // 4 + brt_name(0, 0, 1, "Dependent", &token(3), None), // 5 + brt_name(0, 0, 0, "Fine", &token(1), None), // 6: no shadow on Data. + brt_name(0, 0, 1, "LocalAlias", &token(2), None), // 7: intentionally uses Hidden!rAtE. + brt_name(0, 0, WORKBOOK, "Other", &[0x1E, 3, 0], None), // 8 + brt_name(0, 0, 1, "Other", &[], None), // 9: malformed, but still an ordinary source shadow. + brt_name(0, 0, 1, "Conservative", &token(8), None), // 10 + brt_name(0, 0, WORKBOOK, "Synthetic", &[0x1E, 4, 0], None), // 11 + brt_name(0x0002_0000, 0, 1, "Synthetic", &[0x1C, 0x1D], None), // 12: future-function metadata. + brt_name(0, 0, 1, "SyntheticControl", &token(11), None), // 13: synthetic name is not a local definition. + ]; + let mut data = row(0); + for (col, index) in [3u32, 4, 5, 6, 10, 13].into_iter().enumerate() { + // PtgNameX qualifies these names on their own sheets. + let formula = [&[0x39u8, 0, 0][..], &index.to_le_bytes()].concat(); + data.extend(formula_cell(col as u32 + 1, &formula, &[])); + } + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"12RaterAtE34Synthetic"#; + assert_eq!(wb.defined_names, xml_names(xml)); + let book = read(&bytes).unwrap(); + for col in 1..=3 { + assert_eq!(book.sheets[0].formula(CellRef::new(0, col)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, col)), Value::Number(0.0)); + } + assert_eq!(book.sheets[0].formula(CellRef::new(0, 4)), Some("Data!Fine")); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 5)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, 5)), Value::Number(0.0)); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 6)), Some("Hidden!SyntheticControl")); + assert!(book.warnings.iter().any(|w| w.contains("workbook names shadowed by local names")), "{:?}", book.warnings); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} From 21ceafea04a551781bd3124a823073b9ea310746 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 07:59:10 -0500 Subject: [PATCH 042/209] fix(svg): validate declaration encodings and internal subset boundaries --- crates/svg/src/write.rs | 70 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/crates/svg/src/write.rs b/crates/svg/src/write.rs index 658454ab2..af3eec992 100644 --- a/crates/svg/src/write.rs +++ b/crates/svg/src/write.rs @@ -258,6 +258,42 @@ pub fn path_data_fixed(path: &BezPath, decimals: usize) -> String { #[derive(Clone, Debug, Default)] pub struct SvgWriter(craft_xml::Writer); +// A subset ends with `] S? >`; quoted entity values, comments and PIs may contain that text. +// Scan once without allocations; malformed unterminated constructs remain the caller's error. +fn closes_internal_subset(internal: &str) -> bool { + let bytes = internal.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i..].starts_with(b"") } else { (2, b"?>") }; + let Some(offset) = bytes[i + start..].windows(end.len()).position(|part| part == end) else { + return false; + }; + i += start + offset + end.len(); + } else if matches!(bytes[i], b'\'' | b'"') { + let quote = bytes[i]; + i += 1; + while i < bytes.len() && bytes[i] != quote { + i += 1; + } + if i < bytes.len() { + i += 1; + } + } else if bytes[i] == b']' { + i += 1; + while i < bytes.len() && matches!(bytes[i], b' ' | b'\t' | b'\r' | b'\n') { + i += 1; + } + if bytes.get(i) == Some(&b'>') { + return true; + } + } else { + i += 1; + } + } + false +} + impl SvgWriter { /// A document: starts with the XML declaration. pub fn new() -> SvgWriter { @@ -270,25 +306,28 @@ impl SvgWriter { SvgWriter::default() } /// A document that starts with an XML declaration naming `encoding` (for example `UTF-8` or - /// `ISO-8859-1`) and a newline. Nothing is written if `encoding` is not an XML name. + /// `ISO-8859-1`) and a newline. Nothing is written for an invalid XML encoding name. pub fn with_encoding(encoding: &str) -> SvgWriter { let mut w = SvgWriter::fragment(); w.decl(encoding, "\n"); w } /// Writes `` followed by `newline`. Nothing is - /// written if `encoding` is not an XML name. + /// written unless `encoding` starts with an ASCII letter and continues with ASCII letters, + /// digits, `.`, `_`, or `-` (XML EncName). pub fn decl(&mut self, encoding: &str, newline: &str) { - if craft_xml::is_name(encoding) { + let mut bytes = encoding.bytes(); + if bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { self.0.raw(&format!("{newline}")); } } /// Writes `` + `newline`. When /// `internal` is empty, the inner newline is omitted, so a minified document can be /// `` with no extra breaks. Nothing is written if `name` is not an XML name - /// or `internal` contains `]]>`. + /// or `internal` closes the subset outside a quoted value, comment or processing instruction. + /// The caller still supplies valid DTD declarations; this is not a DTD validator. pub fn doctype_internal(&mut self, name: &str, internal: &str, newline: &str) { - if !craft_xml::is_name(name) || internal.contains("]]>") { + if !craft_xml::is_name(name) || closes_internal_subset(internal) { return; } let mut out = format!("\n"); assert!(SvgWriter::with_encoding("UTF-8\"").is_empty()); assert!(SvgWriter::with_encoding("").is_empty()); + for valid in ["A", "a0._-"] { + assert_eq!(SvgWriter::with_encoding(valid).finish(), format!("\n")); + } + for invalid in [":UTF-8", "_UTF8", "1UTF8", "é", "UTF:8", "UTFé", "UTF 8"] { + assert!(SvgWriter::with_encoding(invalid).is_empty(), "{invalid}"); + } let mut compact = SvgWriter::fragment(); compact.decl("ISO-8859-1", ""); compact.root().open(); @@ -479,7 +524,22 @@ mod tests { assert_eq!(empty.finish(), "\n"); let mut refused = SvgWriter::fragment(); refused.doctype_internal("svg", "]]>", "\n"); + refused.doctype_internal("svg", "]>", "?>"] { + let mut writer = SvgWriter::fragment(); + writer.doctype_internal("svg", internal, ""); + assert_eq!(writer.finish(), format!("")); + } + for internal in ["]>", "] >", "]>"] { + let mut writer = SvgWriter::fragment(); + writer.doctype_internal("svg", internal, ""); + assert!(writer.is_empty()); + } + } } From 0b0e5accd59f3347bcb4c19d0561f09932366050 Mon Sep 17 00:00:00 2001 From: StDenis1980 <288736242+StDenis1980@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:37:04 +0100 Subject: [PATCH 043/209] docx: set_application_name for apps that embed the writer An app that ships the writer under its own name names itself in docProps/app.xml (); "WordCraft" until set. Co-Authored-By: Claude Opus 5.5 --- crates/docx/src/lib.rs | 13 +++++++++++++ crates/docx/src/write/settings.rs | 2 +- crates/docx/tests/application_name.rs | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 crates/docx/tests/application_name.rs diff --git a/crates/docx/src/lib.rs b/crates/docx/src/lib.rs index 7f47d948b..c3c3e8df3 100644 --- a/crates/docx/src/lib.rs +++ b/crates/docx/src/lib.rs @@ -20,6 +20,19 @@ mod custom; mod flat; mod fonts; mod package; + +static APPLICATION_NAME: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Names the application written to `docProps/app.xml` (``), for an app that +/// embeds this writer under its own name; "WordCraft" until set. Only the first call counts. +pub fn set_application_name(name: &str) { + let _ = APPLICATION_NAME.set(name.to_owned()); +} + +/// The application `docProps/app.xml` names (see [`set_application_name`]). +pub fn application_name() -> &'static str { + APPLICATION_NAME.get().map_or("WordCraft", String::as_str) +} mod read; mod smart_art; #[cfg(test)] diff --git a/crates/docx/src/write/settings.rs b/crates/docx/src/write/settings.rs index 18f9f8239..3371986c4 100644 --- a/crates/docx/src/write/settings.rs +++ b/crates/docx/src/write/settings.rs @@ -349,7 +349,7 @@ pub(super) fn app_xml(doc: &Document) -> Vec { ("xmlns:vt", "http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes"), ], ); - w.leaf("Application", &[], "WordCraft"); + w.leaf("Application", &[], crate::application_name()); w.leaf("Words", &[], &doc.word_count().to_string()); w.leaf("Paragraphs", &[], &doc.paragraph_count().to_string()); w.leaf("DocSecurity", &[], "0"); diff --git a/crates/docx/tests/application_name.rs b/crates/docx/tests/application_name.rs new file mode 100644 index 000000000..1973e63ae --- /dev/null +++ b/crates/docx/tests/application_name.rs @@ -0,0 +1,19 @@ +//! An app that embeds the writer names itself in `docProps/app.xml` (its own test binary: the +//! name is set once per process). + +use std::io::Read; + +#[test] +fn the_embedding_app_names_itself_in_app_xml() { + assert_eq!(craft_docx::application_name(), "WordCraft", "the default"); + craft_docx::set_application_name("Example Writer"); + craft_docx::set_application_name("Other"); + assert_eq!(craft_docx::application_name(), "Example Writer", "only the first call counts"); + let mut d = craft_wordmodel::Document::new(); + d.body = vec![craft_wordmodel::para_block(craft_wordmodel::Paragraph::with_text("Text", Default::default()))]; + let bytes = craft_docx::write(&d).unwrap(); + let mut z = zip::ZipArchive::new(std::io::Cursor::new(&bytes)).unwrap(); + let mut app = String::new(); + z.by_name("docProps/app.xml").unwrap().read_to_string(&mut app).unwrap(); + assert!(app.contains("Example Writer"), "{app}"); +} From ab87149855cf6cf4c16b23dc9a207929d21fd2ea Mon Sep 17 00:00:00 2001 From: Hamza Date: Mon, 16 Nov 2026 18:51:40 +0500 Subject: [PATCH 044/209] craft-csv: decode Excel's character escapes in SYLK text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Excel writes a character outside ASCII in a SYLK text value as ESC N and one byte, the character's ISO 6937 code less 0x80; a diacritic's code is followed by the letter it goes on. The reader kept those bytes, so Größe für Köln opened as GrNHoN{e fNHur KNHoln (storytold/gridcraft#4321). Text values now decode them: a diacritic and letter become the precomposed letter (or the letter and its combining mark when Unicode has no single character), a spacing code its character; a code ISO 6937 doesn't define is kept as written. --- crates/csv/src/sylk.rs | 146 ++++++++++++++++++++++++++++++++++++++- crates/csv/tests/sylk.rs | 25 +++++++ 2 files changed, 168 insertions(+), 3 deletions(-) diff --git a/crates/csv/src/sylk.rs b/crates/csv/src/sylk.rs index f9a74405a..d8c0ebec5 100644 --- a/crates/csv/src/sylk.rs +++ b/crates/csv/src/sylk.rs @@ -219,12 +219,13 @@ fn position(arg: &str, max: u32) -> Option { arg.trim().parse::().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1) } -/// A `K` value: `"text"` (everything between the first and the last quote, as written), a -/// boolean, a number, or other text (an error value when it starts with `#`). +/// A `K` value: `"text"` (everything between the first and the last quote, with Excel's +/// character escapes decoded), a boolean, a number, or other text (an error value when it starts +/// with `#`). fn parse_value(arg: &str) -> Value { if let Some(text) = arg.strip_prefix('"') { let text = text.strip_suffix('"').unwrap_or(text); - return Value::Text(text.to_string()); + return Value::Text(decode_escapes(text)); } if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") { return Value::Bool(arg.eq_ignore_ascii_case("TRUE")); @@ -238,6 +239,145 @@ fn parse_value(arg: &str) -> Value { } } +/// ISO 6937's diacritics (codes 0xC1–0xCF): each code, its combining mark, and the letters it +/// composes with, as pairs of letter and precomposed letter (from Unicode's canonical composition). +const ISO6937_DIACRITICS: [(u8, char, &str); 13] = [ + (0xC1, '\u{0300}', "AÀaàEÈeèIÌiìNǸnǹOÒoòUÙuùWẀwẁYỲyỳ"), // grave + (0xC2, '\u{0301}', "AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute + (0xC3, '\u{0302}', "AÂaâCĈcĉEÊeêGĜgĝHĤhĥIÎiîJĴjĵOÔoôSŜsŝUÛuûWŴwŵYŶyŷZẐzẑ"), // circumflex + (0xC4, '\u{0303}', "AÃaãEẼeẽIĨiĩNÑnñOÕoõUŨuũVṼvṽYỸyỹ"), // tilde + (0xC5, '\u{0304}', "AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron + (0xC6, '\u{0306}', "AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve + (0xC7, '\u{0307}', "AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above + (0xC8, '\u{0308}', "AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis + (0xCA, '\u{030a}', "AÅaåUŮuůwẘyẙ"), // ring + (0xCB, '\u{0327}', "CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla + (0xCD, '\u{030b}', "OŐoőUŰuű"), // double acute + (0xCE, '\u{0328}', "AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek + (0xCF, '\u{030c}', "AǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron +]; + +/// ISO 6937's spacing characters above ASCII (codes 0xA1–0xFE that aren't diacritics). +const ISO6937_SPACING: [(u8, char); 71] = [ + (0xA1, '¡'), + (0xA2, '¢'), + (0xA3, '£'), + (0xA5, '¥'), + (0xA7, '§'), + (0xA8, '¤'), + (0xA9, '‘'), + (0xAA, '“'), + (0xAB, '«'), + (0xAC, '←'), + (0xAD, '↑'), + (0xAE, '→'), + (0xAF, '↓'), + (0xB0, '°'), + (0xB1, '±'), + (0xB2, '²'), + (0xB3, '³'), + (0xB4, '×'), + (0xB5, 'µ'), + (0xB6, '¶'), + (0xB7, '·'), + (0xB8, '÷'), + (0xB9, '’'), + (0xBA, '”'), + (0xBB, '»'), + (0xBC, '¼'), + (0xBD, '½'), + (0xBE, '¾'), + (0xBF, '¿'), + (0xD0, '―'), + (0xD1, '¹'), + (0xD2, '®'), + (0xD3, '©'), + (0xD4, '™'), + (0xD5, '♪'), + (0xD6, '¬'), + (0xD7, '¦'), + (0xDC, '⅛'), + (0xDD, '⅜'), + (0xDE, '⅝'), + (0xDF, '⅞'), + (0xE0, 'Ω'), + (0xE1, 'Æ'), + (0xE2, 'Đ'), + (0xE3, 'ª'), + (0xE4, 'Ħ'), + (0xE6, 'IJ'), + (0xE7, 'Ŀ'), + (0xE8, 'Ł'), + (0xE9, 'Ø'), + (0xEA, 'Œ'), + (0xEB, 'º'), + (0xEC, 'Þ'), + (0xED, 'Ŧ'), + (0xEE, 'Ŋ'), + (0xEF, 'ʼn'), + (0xF0, 'ĸ'), + (0xF1, 'æ'), + (0xF2, 'đ'), + (0xF3, 'ð'), + (0xF4, 'ħ'), + (0xF5, 'ı'), + (0xF6, 'ij'), + (0xF7, 'ŀ'), + (0xF8, 'ł'), + (0xF9, 'ø'), + (0xFA, 'œ'), + (0xFB, 'ß'), + (0xFC, 'þ'), + (0xFD, 'ŧ'), + (0xFE, 'ŋ'), +]; + +/// Text with Excel's SYLK character escapes decoded. Excel writes a character outside ASCII as +/// `ESC N` and one byte, the character's ISO 6937 code less 0x80. A diacritic's code is followed +/// by the letter it goes on (`ESC N H o` is `ö`, `ESC N {` is `ß`). A code ISO 6937 doesn't define, +/// or an `ESC` that isn't followed by `N` and a code, is kept as written. +fn decode_escapes(text: &str) -> String { + if !text.contains('\u{1b}') { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + let mut ahead = chars.clone(); + let code = (c == '\u{1b}' && ahead.next() == Some('N')) + .then(|| ahead.next()) + .flatten() + .and_then(|b| u8::try_from(b).ok()) + .filter(|b| (0x20..0x80).contains(b)); + let Some(code) = code.map(|b| b + 0x80) else { + out.push(c); + continue; + }; + if let Some(&(_, mark, pairs)) = ISO6937_DIACRITICS.iter().find(|(d, _, _)| *d == code) { + // The letter it goes on: precomposed where Unicode has the pair, else letter and mark. + let Some(base) = ahead.next() else { + out.push(c); + continue; + }; + let letters: Vec = pairs.chars().collect(); + match letters.chunks(2).find(|p| p.first() == Some(&base)).and_then(|p| p.get(1)) { + Some(&composed) => out.push(composed), + None => { + out.push(base); + out.push(mark); + } + } + } else if let Some(&(_, ch)) = ISO6937_SPACING.iter().find(|(s, _)| *s == code) { + out.push(ch); + } else { + out.push(c); + continue; + } + chars = ahead; + } + out +} + /// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or /// nothing (the cell's own row or column). #[derive(Clone, Copy, Debug, PartialEq)] diff --git a/crates/csv/tests/sylk.rs b/crates/csv/tests/sylk.rs index 763508983..b91d07427 100644 --- a/crates/csv/tests/sylk.rs +++ b/crates/csv/tests/sylk.rs @@ -115,3 +115,28 @@ fn survives_huge_relative_offsets_and_width_floods() { let _ = read(&b); } } + +#[test] +fn decodes_excels_character_escapes_in_text() { + // storytold/gridcraft#4321: Excel writes `Größe für Köln` with `ESC N` escapes (ISO 6937 codes + // less 0x80: `H` is the diaeresis, put on the next letter; `{` is ß). + let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\ + C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\ + C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\ + C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nE\r\n"; + let sh = read(file.as_bytes()).unwrap(); + let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default(); + let text = |s: &str| Value::Text(s.into()); + assert_eq!(value("A1"), text("Größe für Köln")); + assert_eq!(value("A2"), text("Plain ASCII 123"), "plain ASCII is unchanged"); + // Acute, grave, circumflex, cedilla, caron, double acute, ring, tilde. + assert_eq!(value("A3"), text("é à â ç š ő å ñ")); + // A letter the mark has no precomposed form with keeps the combining mark. + assert_eq!(value("A4"), text("à Z\u{300}")); + // Spacing characters, one code each: `q` æ, `y` ø, `{` ß, `a` Æ, `i` Ø. + assert_eq!(value("A5"), text("æ øStraße ÆØ")); + // A diacritic on a letter with no precomposed form, and a stray `ESC N` before an escape. + assert_eq!(value("A6"), text("q\u{308} \u{1b}Nö end\u{1b}N")); + // Codes ISO 6937 doesn't define (0xFF, 0xCC) and an `ESC` without `N` stay as written. + assert_eq!(value("A7"), text("\u{1b}N\u{7f} \u{1b}NL \u{1b}X")); +} From 16bf5f8578893dd318334010da6c8ff5e0fbdc37 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:52:55 +0100 Subject: [PATCH 045/209] media: add a pooled FileReader ByteSource FileReader reads a file in place and keeps at most MAX_OPEN_FILES descriptors open in a shared LRU pool, reopening by path when a file was closed. A project with hundreds of clips can stay under the process file-descriptor limit. A file truncated since it was opened is refused. Part of #778 and effectcraft#781. --- crates/media/src/file.rs | 206 +++++++++++++++++++++++++++++++++++++++ crates/media/src/lib.rs | 6 ++ 2 files changed, 212 insertions(+) create mode 100644 crates/media/src/file.rs diff --git a/crates/media/src/file.rs b/crates/media/src/file.rs new file mode 100644 index 000000000..93e36e648 --- /dev/null +++ b/crates/media/src/file.rs @@ -0,0 +1,206 @@ +//! Pooled file [`ByteSource`]s: many clips can be open at once without holding a descriptor each. +//! +//! Past [`MAX_OPEN_FILES`] the least recently read file is closed and reopened by path on demand, +//! so a project with hundreds of sources does not hit the process file-descriptor limit (often 256 +//! on macOS, 1024 on Linux). + +#![cfg(not(target_arch = "wasm32"))] + +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::source::ByteSource; + +fn eof() -> io::Error { + io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") +} + +/// Files [`FileReader`]s keep open at most, together. Past this many, the least recently read file +/// is closed and its reader reopens it on demand. +pub const MAX_OPEN_FILES: usize = 128; + +/// The open files of a set of [`FileReader`]s, least recently used first. +#[derive(Debug)] +pub(crate) struct FilePool { + cap: usize, + state: std::sync::Mutex, +} + +#[derive(Debug, Default)] +struct PoolState { + next_id: u64, + open: std::collections::VecDeque<(u64, Arc)>, +} + +impl FilePool { + pub(crate) fn new(cap: usize) -> Self { + Self { cap: cap.max(1), state: Default::default() } + } + + fn state(&self) -> std::sync::MutexGuard<'_, PoolState> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn next_id(&self) -> u64 { + let mut s = self.state(); + s.next_id = s.next_id.wrapping_add(1); + s.next_id + } + + /// The open file of reader `id`, now the most recently used. + fn get(&self, id: u64) -> Option> { + let mut s = self.state(); + let at = s.open.iter().position(|(i, _)| *i == id)?; + let entry = s.open.remove(at)?; + let file = entry.1.clone(); + s.open.push_back(entry); + Some(file) + } + + /// Keep `file` open for reader `id`, closing the least recently used files past the cap. A + /// file another thread is reading from stays open until that read ends. + fn insert(&self, id: u64, file: std::fs::File) -> Arc { + let mut s = self.state(); + if let Some((_, open)) = s.open.iter().find(|(i, _)| *i == id) { + return open.clone(); // another thread reopened it first + } + let file = Arc::new(file); + s.open.push_back((id, file.clone())); + while s.open.len() > self.cap { + s.open.pop_front(); + } + file + } + + fn remove(&self, id: u64) { + self.state().open.retain(|(i, _)| *i != id); + } + + #[cfg(test)] + fn open_count(&self) -> usize { + self.state().open.len() + } +} + +static FILE_POOL: std::sync::LazyLock> = std::sync::LazyLock::new(|| Arc::new(FilePool::new(MAX_OPEN_FILES))); + +/// A file on disk, read in place: only the requested ranges are read, so opening a clip costs its +/// index rather than its size. The file is kept open in a shared pool of at most +/// [`MAX_OPEN_FILES`] and reopened by path when the pool closed it. +#[derive(Debug)] +pub struct FileReader { + path: PathBuf, + len: u64, + id: u64, + pool: Arc, +} + +impl FileReader { + /// Open `path` in the process-wide pool. + pub fn open(path: &Path) -> io::Result { + Self::open_in(path, &FILE_POOL) + } + + pub(crate) fn open_in(path: &Path, pool: &Arc) -> io::Result { + let file = std::fs::File::open(path)?; + let m = file.metadata()?; + if !m.is_file() { + return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display()))); + } + let id = pool.next_id(); + pool.insert(id, file); + Ok(Self { path: path.to_path_buf(), len: m.len(), id, pool: pool.clone() }) + } + + /// The open file, reopened when the pool closed it. A file now shorter than when it was opened + /// (replaced or truncated) is refused rather than read as the old one; a longer one (still + /// being written) is read up to the opened length. + fn file(&self) -> io::Result> { + if let Some(file) = self.pool.get(self.id) { + return Ok(file); + } + let file = std::fs::File::open(&self.path)?; + let m = file.metadata()?; + if !m.is_file() || m.len() < self.len { + return Err(io::Error::other(format!("{} changed since it was opened", self.path.display()))); + } + Ok(self.pool.insert(self.id, file)) + } +} + +impl Drop for FileReader { + fn drop(&mut self) { + self.pool.remove(self.id); + } +} + +impl ByteSource for FileReader { + fn len(&self) -> u64 { + self.len + } + + /// Positional reads: threads decoding different parts of the file do not share a cursor. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) { + return Err(eof()); + } + let file = self.file()?; + file.read_at(offset, buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_reader_reads_ranges_in_place() { + let v: Vec = (0..=255).collect(); + let dir = craft_media_testkit::private_dir("file-reader"); + let path = dir.join("bytes.bin"); + std::fs::write(&path, &v).unwrap(); + let r = FileReader::open(&path).unwrap(); + assert_eq!(ByteSource::len(&r), 256); + let mut b = [0u8; 3]; + r.read_at(4, &mut b).unwrap(); + assert_eq!(&b, &v[4..7]); + // reads are positional: an earlier offset after a later one + r.read_at(0, &mut b).unwrap(); + assert_eq!(&b, &v[0..3]); + assert_eq!(r.read_at(254, &mut [0u8; 3]).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); + assert!(FileReader::open(&dir).is_err(), "a directory is not a media file"); + assert_eq!(FileReader::open(&dir.join("missing")).unwrap_err().kind(), io::ErrorKind::NotFound); + drop(r); + std::fs::remove_dir_all(&dir).ok(); + } + + /// More readers than the pool keeps open stay readable; past the cap the least recently read + /// file is closed and reopened on demand, and a file truncated meanwhile is refused. + #[test] + fn more_file_readers_than_the_pool_holds_stay_readable() { + let dir = craft_media_testkit::private_dir("file-pool"); + let pool = Arc::new(FilePool::new(2)); + let paths: Vec<_> = (0..5).map(|i| dir.join(format!("{i}.bin"))).collect(); + for (i, p) in paths.iter().enumerate() { + std::fs::write(p, format!("file{i}-0123456789")).unwrap(); + } + let readers: Vec<_> = paths.iter().map(|p| FileReader::open_in(p, &pool).unwrap()).collect(); + assert_eq!(pool.open_count(), 2); + for _ in 0..2 { + for (i, r) in readers.iter().enumerate() { + let mut b = [0u8; 5]; + r.read_at(0, &mut b).unwrap(); + assert_eq!(b, *format!("file{i}").as_bytes()); + assert!(pool.open_count() <= 2); + } + } + // reader 0 was closed by the later reads; its file is now shorter than when opened + std::fs::write(&paths[0], b"short").unwrap(); + let mut b = [0u8; 5]; + assert!(readers[0].read_at(0, &mut b).is_err()); + drop(readers); + assert_eq!(pool.open_count(), 0, "dropped readers close their files"); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index a9c812318..32bd583d8 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -14,6 +14,8 @@ //! program streams). //! - [`Container`] and [`CodecId`] (no feature): the identifiers the codec crates report a file's //! container and codec with. +//! - `FileReader` (native): a pooled file `ByteSource` so many clips can be open without holding a +//! descriptor each. //! //! `bitstream` and `isobmff` are on by default; `matroska`, `ogg`, `avi`, `mxf` and `mpegts` are opt-in. A crate that needs //! only bit I/O uses `default-features = false, features = ["bitstream"]`. @@ -22,6 +24,8 @@ pub mod avi; #[cfg(feature = "bitstream")] pub mod bitstream; +#[cfg(not(target_arch = "wasm32"))] +mod file; mod id; #[cfg(feature = "isobmff")] pub mod isobmff; @@ -34,6 +38,8 @@ pub mod mxf; #[cfg(feature = "ogg")] pub mod ogg; pub mod source; +#[cfg(not(target_arch = "wasm32"))] +pub use file::{FileReader, MAX_OPEN_FILES}; pub use id::{CodecId, CodecParams, Container, PcmFormat, xiph_lace}; pub use source::{ByteSource, ReadSeekSource}; From 43db2f024e8ae7f46c83556bb5fe5d1f50cc95ce Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:58:40 +0100 Subject: [PATCH 046/209] media: refuse FIFOs before opening a FileReader Opening a named pipe with no writer blocks. Check the path is a regular file first, then open with O_NONBLOCK (as filmcraft-media does), and check the handle again so a FIFO swapped in after the path check cannot hang the open or a reopen after eviction. --- crates/media/src/file.rs | 83 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/crates/media/src/file.rs b/crates/media/src/file.rs index 93e36e648..67ba07c32 100644 --- a/crates/media/src/file.rs +++ b/crates/media/src/file.rs @@ -16,6 +16,53 @@ fn eof() -> io::Error { io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source") } +/// Open `path` for reading without blocking on a FIFO that was swapped in after the metadata check +/// (`O_NONBLOCK`; it changes nothing for a regular file, and the callers check the handle's type). +/// `O_NONBLOCK` has no std constant (no libc dependency here): it is 0o4000 on most Linux targets, +/// 0x80 on Linux MIPS, 0x4000 on Linux SPARC and 4 on the BSD family, and the open stays a plain +/// one on any other target. +fn open_nonblocking(path: &Path) -> io::Result { + let mut o = std::fs::OpenOptions::new(); + o.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(all( + any(target_os = "linux", target_os = "android"), + any(target_arch = "mips", target_arch = "mips32r6", target_arch = "mips64", target_arch = "mips64r6") + ))] + o.custom_flags(0x80); + #[cfg(all(any(target_os = "linux", target_os = "android"), any(target_arch = "sparc", target_arch = "sparc64")))] + o.custom_flags(0x4000); + #[cfg(all( + any(target_os = "linux", target_os = "android"), + not(any( + target_arch = "mips", + target_arch = "mips32r6", + target_arch = "mips64", + target_arch = "mips64r6", + target_arch = "sparc", + target_arch = "sparc64" + )) + ))] + o.custom_flags(0o4000); + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + o.custom_flags(0x4); + } + o.open(path) +} + +fn not_a_file(path: &Path) -> io::Error { + io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())) +} + /// Files [`FileReader`]s keep open at most, together. Past this many, the least recently read file /// is closed and its reader reopens it on demand. pub const MAX_OPEN_FILES: usize = 128; @@ -103,10 +150,14 @@ impl FileReader { } pub(crate) fn open_in(path: &Path, pool: &Arc) -> io::Result { - let file = std::fs::File::open(path)?; + // checked before opening: opening a named pipe with no writer blocks the caller for good + if !std::fs::metadata(path)?.is_file() { + return Err(not_a_file(path)); + } + let file = open_nonblocking(path)?; let m = file.metadata()?; if !m.is_file() { - return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display()))); + return Err(not_a_file(path)); } let id = pool.next_id(); pool.insert(id, file); @@ -120,7 +171,7 @@ impl FileReader { if let Some(file) = self.pool.get(self.id) { return Ok(file); } - let file = std::fs::File::open(&self.path)?; + let file = open_nonblocking(&self.path)?; let m = file.metadata()?; if !m.is_file() || m.len() < self.len { return Err(io::Error::other(format!("{} changed since it was opened", self.path.display()))); @@ -203,4 +254,30 @@ mod tests { assert_eq!(pool.open_count(), 0, "dropped readers close their files"); std::fs::remove_dir_all(&dir).ok(); } + + /// A named pipe with no writer would block `open` for good: refuse it first. + #[cfg(unix)] + #[test] + fn a_fifo_is_refused_without_blocking() { + let dir = craft_media_testkit::private_dir("file-fifo"); + let fifo = dir.join("pipe"); + if !std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) { + return; // no mkfifo here + } + assert!(FileReader::open(&fifo).is_err()); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The open itself must not block on a FIFO swapped in after the path check (a FIFO with no writer). + #[cfg(target_os = "linux")] + #[test] + fn opening_a_fifo_with_no_writer_does_not_block() { + let dir = craft_media_testkit::private_dir("file-fifo-open"); + let fifo = dir.join("pipe"); + if std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) { + let f = super::open_nonblocking(&fifo).unwrap(); + assert!(!f.metadata().unwrap().is_file()); + } + std::fs::remove_dir_all(&dir).ok(); + } } From f71eeae5fabfcae2fe0c1c61c9463ba3f371cdaf Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:58:59 +0100 Subject: [PATCH 047/209] Add craft-color and craft-frame for compositor pictures craft-color holds ColorInfo, HdrMetadata, Y'CbCr matrices, transfer functions and the small helpers display conversion needs. It is not a colour-managed pipeline (no LUTs, log, grade or tone mapping). craft-frame holds VideoFrame, PixelData, Chroma and Region: the compositor and display picture type MediaSource will return. It is distinct from craft-video's decoder-output VideoFrame (#1231 / #1768). Came from the metadata and picture parts of filmcraft-color and filmcraft-frame. Part of #778 and effectcraft#781. --- Cargo.lock | 18 ++ Cargo.toml | 2 + crates/color/Cargo.toml | 17 ++ crates/color/src/lib.rs | 411 ++++++++++++++++++++++++++++++++++++++++ crates/frame/Cargo.toml | 17 ++ crates/frame/src/lib.rs | 376 ++++++++++++++++++++++++++++++++++++ 6 files changed, 841 insertions(+) create mode 100644 crates/color/Cargo.toml create mode 100644 crates/color/src/lib.rs create mode 100644 crates/frame/Cargo.toml create mode 100644 crates/frame/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1d5c223cf..c462980f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1451,6 +1451,14 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "craft-color" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", +] + [[package]] name = "craft-control-protocol" version = "0.1.0" @@ -1868,6 +1876,16 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "craft-frame" +version = "0.1.0" +dependencies = [ + "craft-color", + "craft-time", + "rayon", + "serde", +] + [[package]] name = "craft-gainmap" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f1a09e32a..beec53ea3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,8 @@ craft-captions = { path = "crates/captions" } craft-cfb = { path = "crates/cfb" } craft-codepage = { path = "crates/codepage" } craft-cms = { path = "crates/cms" } +craft-color = { path = "crates/color" } +craft-frame = { path = "crates/frame" } craft-crs = { path = "crates/crs" } craft-crw = { path = "crates/crw" } craft-csv = { path = "crates/csv" } diff --git a/crates/color/Cargo.toml b/crates/color/Cargo.toml new file mode 100644 index 000000000..44578baed --- /dev/null +++ b/crates/color/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-color" +description = "Colour tags, Y'CbCr matrices, transfer functions and HDR static metadata for compositor and display frames" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/color/src/lib.rs b/crates/color/src/lib.rs new file mode 100644 index 000000000..5bf6426b8 --- /dev/null +++ b/crates/color/src/lib.rs @@ -0,0 +1,411 @@ +//! Colour tags and the conversions compositor and display frames need. +//! +//! This is the slim colour-metadata layer used by craft-frame and by media +//! `MediaInfo`: [`ColorInfo`] (matrix, transfer, primaries, range), [`HdrMetadata`], Y'CbCr +//! matrices, transfer functions, and a few helpers (sRGB tables, HSL, hex). It is not a +//! colour-managed pipeline: LUTs, camera log, grade and tone mapping stay out. +//! +//! Came from FilmCraft's `filmcraft-color` (the metadata and transfer part of that crate). +//! Distinct from `craft-video`'s decoder colour tags (`ColorTags` in #1231 / #1768). + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +use serde::{Deserialize, Serialize}; +use std::sync::OnceLock; + +/// HDR reference white in cd/m² (ITU-R BT.2408). +pub const REFERENCE_WHITE_NITS: f64 = 203.0; + +/// YUV ↔ RGB matrix coefficients. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Matrix { + Bt601, + #[default] + Bt709, + Bt2020Ncl, +} + +impl Matrix { + /// (Kr, Kb) luma coefficients. + pub fn kr_kb(self) -> (f32, f32) { + match self { + Matrix::Bt601 => (0.299, 0.114), + Matrix::Bt709 => (0.2126, 0.0722), + Matrix::Bt2020Ncl => (0.2627, 0.0593), + } + } + /// From an ISO/IEC 23091-2 `matrix_coefficients` code. + pub fn from_code(c: u8) -> Option { + match c { + 1 => Some(Matrix::Bt709), + 5 | 6 => Some(Matrix::Bt601), + 9 | 10 => Some(Matrix::Bt2020Ncl), + _ => None, + } + } +} + +/// Transfer characteristics. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Transfer { + /// BT.709 / BT.1886 display (pure 2.4 gamma is what NLEs use for display-referred decode). + #[default] + Bt709, + Srgb, + Linear, + /// SMPTE ST 2084. + Pq, + /// ARIB STD-B67. + Hlg, +} + +impl Transfer { + /// From an ISO/IEC 23091-2 `transfer_characteristics` code. + pub fn from_code(c: u8) -> Option { + match c { + 1 | 6 | 14 | 15 => Some(Transfer::Bt709), + 13 => Some(Transfer::Srgb), + 8 => Some(Transfer::Linear), + 16 => Some(Transfer::Pq), + 18 => Some(Transfer::Hlg), + _ => None, + } + } +} + +/// Chromaticity primaries. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Primaries { + #[default] + Bt709, + Bt601_625, + Bt601_525, + Bt2020, + P3D65, +} + +/// Quantisation range of integer Y'CbCr samples. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Range { + /// 16–235 (8-bit) "video" range. + #[default] + Limited, + Full, +} + +/// Colour metadata attached to a compositor or display frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ColorInfo { + pub matrix: Matrix, + pub transfer: Transfer, + pub primaries: Primaries, + pub range: Range, +} + +impl ColorInfo { + pub const SRGB_FULL: ColorInfo = ColorInfo { matrix: Matrix::Bt709, transfer: Transfer::Srgb, primaries: Primaries::Bt709, range: Range::Full }; + pub const REC709: ColorInfo = ColorInfo { matrix: Matrix::Bt709, transfer: Transfer::Bt709, primaries: Primaries::Bt709, range: Range::Limited }; +} + +/// HDR static metadata of a stream: SMPTE ST 2086 mastering display luminance and the CTA-861.3 +/// content light levels (MaxCLL / MaxFALL), in cd/m². Read from `mdcv` / `clli` (MP4 / MOV) and +/// Matroska `MasteringMetadata` / `MaxCLL` / `MaxFALL`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HdrMetadata { + pub mastering_max_nits: Option, + pub mastering_min_nits: Option, + pub max_cll: Option, + pub max_fall: Option, +} + +impl HdrMetadata { + /// The content's peak for tone mapping: MaxCLL when signalled (the brightest pixel actually + /// in the content), else the mastering display's peak; `None` when neither is (0 means + /// unknown in both). Never below HDR reference white. + pub fn peak_nits(&self) -> Option { + let known = |v: Option| v.filter(|n| *n > 0.0); + let p = match (known(self.max_cll), known(self.mastering_max_nits)) { + (Some(c), Some(m)) => c.min(m), + (Some(c), None) => c, + (None, m) => m?, + }; + Some(p.clamp(REFERENCE_WHITE_NITS as f32, 10_000.0)) + } +} + +/// Convert normalised Y'CbCr (Y in 0..1, Cb/Cr in -0.5..0.5) to non-linear R'G'B'. +#[inline] +pub fn ycbcr_to_rgb(y: f32, cb: f32, cr: f32, m: Matrix) -> [f32; 3] { + let (kr, kb) = m.kr_kb(); + let kg = 1.0 - kr - kb; + let r = y + 2.0 * (1.0 - kr) * cr; + let b = y + 2.0 * (1.0 - kb) * cb; + let g = (y - kr * r - kb * b) / kg; + [r, g, b] +} + +/// Convert non-linear R'G'B' to normalised Y'CbCr. +#[inline] +pub fn rgb_to_ycbcr(r: f32, g: f32, b: f32, m: Matrix) -> [f32; 3] { + let (kr, kb) = m.kr_kb(); + let kg = 1.0 - kr - kb; + let y = kr * r + kg * g + kb * b; + [y, (b - y) / (2.0 * (1.0 - kb)), (r - y) / (2.0 * (1.0 - kr))] +} + +/// Normalise an integer luma code value to 0..1 given bit depth and range. +#[inline] +pub fn normalize_y(v: u32, bits: u32, range: Range) -> f32 { + let scale = (1u32 << (bits - 8)) as f32; + match range { + Range::Limited => (v as f32 - 16.0 * scale) / (219.0 * scale), + Range::Full => v as f32 / ((1u32 << bits) - 1) as f32, + } +} + +/// Normalise an integer chroma code value to −0.5..0.5 given bit depth and range. +#[inline] +pub fn normalize_c(v: u32, bits: u32, range: Range) -> f32 { + let scale = (1u32 << (bits - 8)) as f32; + match range { + Range::Limited => (v as f32 - 128.0 * scale) / (224.0 * scale), + Range::Full => (v as f32 - (1u32 << (bits - 1)) as f32) / ((1u32 << bits) - 1) as f32, + } +} + +/// sRGB electro-optical transfer (encoded → linear). +#[inline] +pub fn srgb_to_linear(v: f32) -> f32 { + if v <= 0.04045 { v / 12.92 } else { ((v + 0.055) / 1.055).powf(2.4) } +} + +/// sRGB inverse EOTF (linear → encoded). +#[inline] +pub fn linear_to_srgb(v: f32) -> f32 { + if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 } +} + +/// Encoded → linear for a transfer function (display-referred; PQ normalised to 100 nits = 1.0). +pub fn to_linear(v: f32, t: Transfer) -> f32 { + match t { + Transfer::Srgb => srgb_to_linear(v), + // Rec.709 is treated as display gamma 2.4; the sRGB curve is visually identical for + // previews and keeps round-trips exact. + Transfer::Bt709 => srgb_to_linear(v), + Transfer::Linear => v, + Transfer::Pq => pq_eotf(v) * 100.0, + Transfer::Hlg => hlg_inverse_oetf(v), + } +} + +/// Linear → encoded for a transfer function (the inverse of [`to_linear`]). +pub fn from_linear(v: f32, t: Transfer) -> f32 { + match t { + Transfer::Srgb | Transfer::Bt709 => linear_to_srgb(v), + Transfer::Linear => v, + Transfer::Pq => pq_inverse_eotf(v / 100.0), + Transfer::Hlg => hlg_oetf(v), + } +} + +/// SMPTE ST 2084 EOTF, output normalised to 10 000 nits = 1.0. +pub fn pq_eotf(e: f32) -> f32 { + let (m1, m2) = (0.159_301_76_f32, 78.84375_f32); + let (c1, c2, c3) = (0.835_937_5_f32, 18.851_563_f32, 18.6875_f32); + let p = e.max(0.0).powf(1.0 / m2); + ((p - c1).max(0.0) / (c2 - c3 * p)).powf(1.0 / m1) +} + +/// Inverse of [`pq_eotf`]. +pub fn pq_inverse_eotf(y: f32) -> f32 { + let (m1, m2) = (0.159_301_76_f32, 78.84375_f32); + let (c1, c2, c3) = (0.835_937_5_f32, 18.851_563_f32, 18.6875_f32); + let p = y.max(0.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p)).powf(m2) +} + +/// ARIB STD-B67 HLG OETF. +pub fn hlg_oetf(l: f32) -> f32 { + let (a, b, c) = (0.178_832_77_f32, 0.284_668_92_f32, 0.559_910_7_f32); + if l <= 1.0 / 12.0 { (3.0 * l.max(0.0)).sqrt() } else { a * (12.0 * l - b).ln() + c } +} + +/// Inverse of [`hlg_oetf`]. +pub fn hlg_inverse_oetf(e: f32) -> f32 { + let (a, b, c) = (0.178_832_77_f32, 0.284_668_92_f32, 0.559_910_7_f32); + if e <= 0.5 { e * e / 3.0 } else { (((e - c) / a).exp() + b) / 12.0 } +} + +/// 256-entry table: sRGB-encoded u8 → linear f32. +pub fn srgb_u8_to_linear_table() -> &'static [f32; 256] { + static T: OnceLock<[f32; 256]> = OnceLock::new(); + T.get_or_init(|| std::array::from_fn(|i| srgb_to_linear(i as f32 / 255.0))) +} + +/// 4096-entry table: linear (0..1, quantised to 12 bits) → sRGB-encoded u8. +pub fn linear_to_srgb_u8_table() -> &'static [u8; 4096] { + static T: OnceLock<[u8; 4096]> = OnceLock::new(); + T.get_or_init(|| std::array::from_fn(|i| (linear_to_srgb(i as f32 / 4095.0) * 255.0 + 0.5).clamp(0.0, 255.0) as u8)) +} + +/// Linear 0..1 to an sRGB-encoded u8, via [`linear_to_srgb_u8_table`]. +#[inline] +pub fn linear_to_srgb_u8(v: f32) -> u8 { + let i = (v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize; + linear_to_srgb_u8_table()[i] +} + +/// Rec.709 luma of a linear or encoded RGB triple. +#[inline] +pub fn luma709(r: f32, g: f32, b: f32) -> f32 { + 0.2126 * r + 0.7152 * g + 0.0722 * b +} + +/// RGB (0..1) → HSL (h in 0..1). +pub fn rgb_to_hsl(r: f32, g: f32, b: f32) -> [f32; 3] { + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let l = (max + min) / 2.0; + if (max - min).abs() < 1e-7 { + return [0.0, 0.0, l]; + } + let d = max - min; + let s = if l > 0.5 { d / (2.0 - max - min) } else { d / (max + min) }; + let h = if max == r { + (g - b) / d + if g < b { 6.0 } else { 0.0 } + } else if max == g { + (b - r) / d + 2.0 + } else { + (r - g) / d + 4.0 + }; + [h / 6.0, s, l] +} + +/// Inverse of [`rgb_to_hsl`]. +pub fn hsl_to_rgb(h: f32, s: f32, l: f32) -> [f32; 3] { + if s <= 0.0 { + return [l, l, l]; + } + let q = if l < 0.5 { l * (1.0 + s) } else { l + s - l * s }; + let p = 2.0 * l - q; + let f = |mut t: f32| { + t = t.rem_euclid(1.0); + if t < 1.0 / 6.0 { + p + (q - p) * 6.0 * t + } else if t < 0.5 { + q + } else if t < 2.0 / 3.0 { + p + (q - p) * (2.0 / 3.0 - t) * 6.0 + } else { + p + } + }; + [f(h + 1.0 / 3.0), f(h), f(h - 1.0 / 3.0)] +} + +/// Parse `#rrggbb` / `#rrggbbaa` into 0..1 floats. +pub fn parse_hex(s: &str) -> Option<[f32; 4]> { + let s = s.trim(); + let s = s.strip_prefix('#').unwrap_or(s); + if !s.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let b = |i: usize| u8::from_str_radix(s.get(i..i + 2)?, 16).ok().map(|v| v as f32 / 255.0); + match s.len() { + 6 => Some([b(0)?, b(2)?, b(4)?, 1.0]), + 8 => Some([b(0)?, b(2)?, b(4)?, b(6)?]), + _ => None, + } +} + +/// The inverse of [`parse_hex`], lowercase, omitting alpha when it is 1. +pub fn to_hex(c: [f32; 4]) -> String { + let q = |v: f32| (v.clamp(0.0, 1.0) * 255.0).round() as u8; + if c[3] >= 0.999 { + format!("#{:02x}{:02x}{:02x}", q(c[0]), q(c[1]), q(c[2])) + } else { + format!("#{:02x}{:02x}{:02x}{:02x}", q(c[0]), q(c[1]), q(c[2]), q(c[3])) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_hex_rejects_signs() { + assert_eq!(parse_hex("#+f+f+f"), None); + assert_eq!(parse_hex("#-f-f-f"), None); + assert_eq!(parse_hex("##0f0f0f"), None); + assert_eq!(parse_hex("#0f0f0"), None); + assert!(parse_hex("#0f0f0f").is_some()); + assert!(parse_hex("0f0f0f80").is_some()); + } + + #[test] + fn ycbcr_roundtrip() { + for m in [Matrix::Bt601, Matrix::Bt709, Matrix::Bt2020Ncl] { + let rgb = [0.8, 0.3, 0.1]; + let y = rgb_to_ycbcr(rgb[0], rgb[1], rgb[2], m); + let back = ycbcr_to_rgb(y[0], y[1], y[2], m); + for k in 0..3 { + assert!((rgb[k] - back[k]).abs() < 1e-5); + } + } + } + + #[test] + fn limited_range_levels() { + assert_eq!(normalize_y(16, 8, Range::Limited), 0.0); + assert_eq!(normalize_y(235, 8, Range::Limited), 1.0); + assert_eq!(normalize_y(940, 10, Range::Limited), 1.0); + assert_eq!(normalize_c(128, 8, Range::Limited), 0.0); + } + + #[test] + fn transfer_roundtrips() { + for t in [Transfer::Srgb, Transfer::Pq, Transfer::Hlg, Transfer::Linear] { + for v in [0.0, 0.1, 0.5, 0.9] { + let l = to_linear(v, t); + assert!((from_linear(l, t) - v).abs() < 1e-3, "{t:?} {v}"); + } + } + assert_eq!(linear_to_srgb_u8(1.0), 255); + assert_eq!(linear_to_srgb_u8(0.0), 0); + assert_eq!(linear_to_srgb_u8(srgb_u8_to_linear_table()[128]), 128); + } + + #[test] + fn hsl_roundtrip() { + let c = [0.2, 0.6, 0.9]; + let h = rgb_to_hsl(c[0], c[1], c[2]); + let b = hsl_to_rgb(h[0], h[1], h[2]); + for k in 0..3 { + assert!((c[k] - b[k]).abs() < 1e-5); + } + } + + #[test] + fn hex() { + assert_eq!(parse_hex("#ff8000"), Some([1.0, 128.0 / 255.0, 0.0, 1.0])); + assert_eq!(to_hex([1.0, 0.0, 0.0, 1.0]), "#ff0000"); + } + + #[test] + fn hdr_peak_nits() { + assert_eq!(HdrMetadata::default().peak_nits(), None); + assert_eq!(HdrMetadata { max_cll: Some(1000.0), mastering_max_nits: Some(4000.0), ..HdrMetadata::default() }.peak_nits(), Some(1000.0)); + assert_eq!(HdrMetadata { max_cll: Some(0.0), mastering_max_nits: Some(400.0), ..HdrMetadata::default() }.peak_nits(), Some(400.0)); + // never below reference white + assert_eq!(HdrMetadata { max_cll: Some(100.0), ..HdrMetadata::default() }.peak_nits(), Some(REFERENCE_WHITE_NITS as f32)); + } + + #[test] + fn codes() { + assert_eq!(Matrix::from_code(1), Some(Matrix::Bt709)); + assert_eq!(Matrix::from_code(6), Some(Matrix::Bt601)); + assert_eq!(Transfer::from_code(16), Some(Transfer::Pq)); + assert_eq!(Transfer::from_code(99), None); + } +} diff --git a/crates/frame/Cargo.toml b/crates/frame/Cargo.toml new file mode 100644 index 000000000..f74bf9365 --- /dev/null +++ b/crates/frame/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-frame" +description = "Compositor and display VideoFrame (RGBA and planar Y'CbCr) with ColorInfo, distinct from craft-video decoder pictures" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-color = { workspace = true } +craft-time = { workspace = true } +rayon = { workspace = true } +serde = { workspace = true } diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs new file mode 100644 index 000000000..2a7c82d1a --- /dev/null +++ b/crates/frame/src/lib.rs @@ -0,0 +1,376 @@ +//! Compositor and display video frames. +//! +//! [`VideoFrame`] is a picture the compositor, monitors and export share: planar Y'CbCr (8 or +//! 16-bit) from a decoder, sRGB-encoded RGBA8 from stills, or linear premultiplied RGBA f32 in the +//! compositor. Pixel data is `Arc`-shared so caches can hold frames for free. Colour is explicit +//! ([`craft_color::ColorInfo`]); time is a [`craft_time::Tick`]. +//! +//! This is **not** the decoder-output picture type in `craft-video` (`craft_video::frame::VideoFrame` +//! in #1231 / #1768), which is a planar codec frame (per-plane bit depth, field order, colour +//! tags as H.273 codes). This type is the one `MediaSource::video_frame` returns. +//! +//! Came from FilmCraft's `filmcraft-frame` (the picture type and display conversions; the frame +//! pool and audio buffer stay out — audio is [`craft_audio::AudioBuffer`]). + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +use std::sync::Arc; + +use craft_color::{ + ColorInfo, Matrix, Range, Transfer, linear_to_srgb_u8, normalize_c, normalize_y, srgb_u8_to_linear_table, to_linear, ycbcr_to_rgb, +}; +use craft_time::Tick; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +/// Chroma subsampling of planar Y'CbCr. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Chroma { + C420, + C422, + C444, +} + +impl Chroma { + /// (horizontal shift, vertical shift) + pub fn shifts(self) -> (u32, u32) { + match self { + Chroma::C420 => (1, 1), + Chroma::C422 => (1, 0), + Chroma::C444 => (0, 0), + } + } + /// A short 4:2:0 / 4:2:2 / 4:4:4 label. + pub fn label(self) -> &'static str { + match self { + Chroma::C420 => "4:2:0", + Chroma::C422 => "4:2:2", + Chroma::C444 => "4:4:4", + } + } +} + +/// Pixel storage of a [`VideoFrame`]. +#[derive(Clone, Debug)] +pub enum PixelData { + /// Straight-alpha, sRGB/709-encoded RGBA, 8 bits per channel. + Rgba8(Arc>), + /// Premultiplied, linear-light RGBA f32 (compositor working format). + RgbaF32(Arc>), + /// 8-bit planar Y'CbCr (optionally with an alpha plane). + Yuv8 { planes: [Arc>; 3], chroma: Chroma, alpha: Option>> }, + /// 9–16-bit planar Y'CbCr stored in u16 (`bits` significant bits). + Yuv16 { planes: [Arc>; 3], chroma: Chroma, bits: u32, alpha: Option>> }, +} + +/// A compositor or display picture. Pixel buffers are `Arc`-shared. +#[derive(Clone, Debug)] +pub struct VideoFrame { + pub width: u32, + pub height: u32, + pub data: PixelData, + pub color: ColorInfo, + /// Pixel aspect ratio (num, den). + pub par: (u32, u32), + /// Presentation time in media time (informational). + pub pts: Tick, +} + +/// A rectangle of pixels: `x`, `y` is its top-left corner. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Region { + pub x: usize, + pub y: usize, + pub w: usize, + pub h: usize, +} + +impl Region { + pub fn is_empty(&self) -> bool { + self.w == 0 || self.h == 0 + } + /// This rectangle cut to a `w`×`h` picture (empty when it lies outside). + pub fn clip(self, w: usize, h: usize) -> Region { + let x = self.x.min(w); + let y = self.y.min(h); + Region { x, y, w: self.w.min(w - x), h: self.h.min(h - y) } + } + /// Whether this rectangle is the whole `w`×`h` picture. + pub fn is_full(&self, w: usize, h: usize) -> bool { + (self.x, self.y, self.w, self.h) == (0, 0, w, h) + } +} + +impl VideoFrame { + /// An sRGB-full, 1:1, zero-pts RGBA8 frame. `data` is `width × height × 4` bytes. + pub fn rgba8(width: u32, height: u32, data: Vec) -> Self { + debug_assert_eq!(data.len(), (width * height * 4) as usize); + Self { width, height, data: PixelData::Rgba8(Arc::new(data)), color: ColorInfo::SRGB_FULL, par: (1, 1), pts: Tick::ZERO } + } + + /// A linear premultiplied RGBA f32 frame, sRGB-full tags, 1:1, zero pts. + pub fn rgba_f32(width: u32, height: u32, data: Vec) -> Self { + debug_assert_eq!(data.len(), (width * height * 4) as usize); + Self { width, height, data: PixelData::RgbaF32(Arc::new(data)), color: ColorInfo::SRGB_FULL, par: (1, 1), pts: Tick::ZERO } + } + + /// A transparent linear RGBA f32 frame of the given size. + pub fn transparent_f32(width: u32, height: u32) -> Self { + Self::rgba_f32(width, height, vec![0.0; (width * height * 4) as usize]) + } + + /// Sets the presentation timestamp and returns the frame. + pub fn with_pts(mut self, pts: Tick) -> Self { + self.pts = pts; + self + } + + /// A short format label for UI and logs. + pub fn format_label(&self) -> String { + match &self.data { + PixelData::Rgba8(_) => "RGBA 8-bit".into(), + PixelData::RgbaF32(_) => "RGBA 32-bit float".into(), + PixelData::Yuv8 { chroma, .. } => format!("YUV {} 8-bit", chroma.label()), + PixelData::Yuv16 { chroma, bits, .. } => format!("YUV {} {bits}-bit", chroma.label()), + } + } + + /// Approximate memory footprint in bytes (for caches). + pub fn byte_size(&self) -> usize { + match &self.data { + PixelData::Rgba8(d) => d.len(), + PixelData::RgbaF32(d) => d.len() * 4, + PixelData::Yuv8 { planes, alpha, .. } => planes.iter().map(|p| p.len()).sum::() + alpha.as_ref().map_or(0, |a| a.len()), + PixelData::Yuv16 { planes, alpha, .. } => (planes.iter().map(|p| p.len()).sum::() + alpha.as_ref().map_or(0, |a| a.len())) * 2, + } + } + + /// Premultiplied linear-light RGBA f32, one pixel per source pixel, row-major, 4 samples each. + pub fn to_linear_f32(&self) -> Vec { + let (w, h) = (self.width as usize, self.height as usize); + let mut out = vec![0.0; w.saturating_mul(h).saturating_mul(4)]; + if w == 0 || h == 0 { + return out; + } + match &self.data { + PixelData::RgbaF32(d) => { + let n = out.len().min(d.len()); + out[..n].copy_from_slice(&d[..n]); + } + PixelData::Rgba8(d) => { + let lut = srgb_u8_to_linear_table(); + for (o, s) in out.as_chunks_mut::<4>().0.iter_mut().zip(d.as_chunks::<4>().0) { + let a = s[3] as f32 / 255.0; + o[0] = lut[s[0] as usize] * a; + o[1] = lut[s[1] as usize] * a; + o[2] = lut[s[2] as usize] * a; + o[3] = a; + } + } + PixelData::Yuv8 { planes, chroma, alpha } => { + let (sx, sy) = chroma.shifts(); + let cw = w.div_ceil(1 << sx); + let info = self.color; + let lin: Vec = (0..4096).map(|i| to_linear(i as f32 / 4095.0, info.transfer)).collect(); + let q = |v: f32| lin[(v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize]; + let ytab: Vec = (0..256).map(|v| normalize_y(v, 8, info.range)).collect(); + let ctab: Vec = (0..256).map(|v| normalize_c(v, 8, info.range)).collect(); + let rgb_of = |y: usize, x: usize| { + let yy = ytab[planes[0][y * w + x] as usize]; + let u = ctab[planes[1][(y >> sy) * cw + (x >> sx)] as usize]; + let v = ctab[planes[2][(y >> sy) * cw + (x >> sx)] as usize]; + let rgb = ycbcr_to_rgb(yy, u, v, info.matrix); + let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / 255.0); + [q(rgb[0]) * a, q(rgb[1]) * a, q(rgb[2]) * a, a] + }; + for y in 0..h { + for x in 0..w { + let o = (y * w + x) * 4; + out[o..o + 4].copy_from_slice(&rgb_of(y, x)); + } + } + } + PixelData::Yuv16 { planes, chroma, bits, alpha } => { + let (sx, sy) = chroma.shifts(); + let cw = w.div_ceil(1 << sx); + let info = self.color; + let bits = *bits; + let amax = ((1u32 << bits) - 1) as f32; + let lin: Vec = (0..4096).map(|i| to_linear(i as f32 / 4095.0, info.transfer)).collect(); + let q = |v: f32| lin[(v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize]; + for y in 0..h { + for x in 0..w { + let yy = normalize_y(planes[0][y * w + x] as u32, bits, info.range); + let u = normalize_c(planes[1][(y >> sy) * cw + (x >> sx)] as u32, bits, info.range); + let v = normalize_c(planes[2][(y >> sy) * cw + (x >> sx)] as u32, bits, info.range); + let rgb = ycbcr_to_rgb(yy, u, v, info.matrix); + let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / amax); + let o = (y * w + x) * 4; + out[o] = q(rgb[0]) * a; + out[o + 1] = q(rgb[1]) * a; + out[o + 2] = q(rgb[2]) * a; + out[o + 3] = a; + } + } + } + } + out + } + + /// Convert to straight-alpha sRGB RGBA8 for display (fast path for 8-bit Rec.709/sRGB Y'CbCr + /// without alpha, and a copy for RGBA8). + pub fn to_rgba8(&self) -> Vec { + let (w, h) = (self.width as usize, self.height as usize); + match &self.data { + PixelData::Rgba8(d) => d.as_ref().clone(), + PixelData::Yuv8 { planes, chroma, alpha: None } if matches!(self.color.transfer, Transfer::Bt709 | Transfer::Srgb) => { + let (sx, sy) = chroma.shifts(); + let cw = w.div_ceil(1 << sx); + let (kr, kb) = self.color.matrix.kr_kb(); + let kg = 1.0 - kr - kb; + let (ys, yo, cs) = match self.color.range { + Range::Limited => (255.0 / 219.0, 16.0, 255.0 / 224.0), + Range::Full => (1.0, 0.0, 1.0), + }; + let (crr, cbb) = (2.0 * (1.0 - kr) * cs, 2.0 * (1.0 - kb) * cs); + let (cgr, cgb) = (crr * kr / kg, cbb * kb / kg); + let mut out = vec![0u8; w * h * 4]; + out.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| { + let cy = y >> sy; + let yrow = &planes[0][y * w..y * w + w]; + let urow = &planes[1][cy * cw..cy * cw + cw]; + let vrow = &planes[2][cy * cw..cy * cw + cw]; + for x in 0..w { + let yy = (yrow[x] as f32 - yo) * ys; + let u = urow[x >> sx] as f32 - 128.0; + let v = vrow[x >> sx] as f32 - 128.0; + let o = &mut row[x * 4..x * 4 + 4]; + o[0] = (yy + crr * v).round().clamp(0.0, 255.0) as u8; + o[1] = (yy - cgr * v - cgb * u).round().clamp(0.0, 255.0) as u8; + o[2] = (yy + cbb * u).round().clamp(0.0, 255.0) as u8; + o[3] = 255; + } + }); + out + } + _ => { + let lin = self.to_linear_f32(); + let mut out = vec![0u8; w * h * 4]; + out.par_chunks_mut(w * 4).zip(lin.par_chunks(w * 4)).for_each(|(o, s)| linear_premul_to_srgb8(s, o)); + out + } + } + } + + /// Luma plane (8-bit, for scopes and thumbnails). Y'CbCr 8-bit returns the Y plane; others go + /// through [`Self::to_rgba8`] and Rec.709 luma of the encoded RGB. + pub fn luma8(&self) -> Vec { + match &self.data { + PixelData::Yuv8 { planes, .. } => planes[0].as_ref().clone(), + _ => self + .to_rgba8() + .as_chunks::<4>() + .0 + .iter() + .map(|p| (0.2126 * p[0] as f32 + 0.7152 * p[1] as f32 + 0.0722 * p[2] as f32) as u8) + .collect(), + } + } +} + +/// Convert a row of premultiplied linear f32 RGBA into straight sRGB RGBA8. +pub fn linear_premul_to_srgb8(src: &[f32], dst: &mut [u8]) { + for (s, o) in src.as_chunks::<4>().0.iter().zip(dst.as_chunks_mut::<4>().0) { + let a = s[3].clamp(0.0, 1.0); + if a <= 0.0 { + o.copy_from_slice(&[0, 0, 0, 0]); + continue; + } + let inv = 1.0 / a; + o[0] = linear_to_srgb_u8(s[0] * inv); + o[1] = linear_to_srgb_u8(s[1] * inv); + o[2] = linear_to_srgb_u8(s[2] * inv); + o[3] = (a * 255.0 + 0.5) as u8; + } +} + +/// Matrix used when a decoder does not signal one: BT.601 for SD, BT.709 otherwise (common practice). +pub fn default_matrix(width: u32, height: u32) -> Matrix { + if width <= 1024 && height <= 576 { Matrix::Bt601 } else { Matrix::Bt709 } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rgba8_roundtrips_through_to_rgba8() { + let px = vec![10u8, 20, 30, 255, 0, 0, 0, 0]; + let f = VideoFrame::rgba8(2, 1, px.clone()).with_pts(Tick(42)); + assert_eq!(f.pts, Tick(42)); + assert_eq!(f.to_rgba8(), px); + assert_eq!(f.format_label(), "RGBA 8-bit"); + assert_eq!(f.byte_size(), 8); + assert_eq!(f.color, ColorInfo::SRGB_FULL); + } + + #[test] + fn rgba8_to_linear_premultiplies() { + // 50% red, 50% alpha, sRGB encoded. Linear of 128/255, times 0.5 alpha. + let f = VideoFrame::rgba8(1, 1, vec![128, 0, 0, 128]); + let lin = f.to_linear_f32(); + let a = 128.0 / 255.0; + let r = srgb_u8_to_linear_table()[128] * a; + assert!((lin[0] - r).abs() < 1e-6); + assert_eq!(lin[1], 0.0); + assert!((lin[3] - a).abs() < 1e-6); + let back = f.to_rgba8(); + assert_eq!(back[0], 128); + assert_eq!(back[3], 128); + } + + #[test] + fn limited_709_gray_yuv8_is_near_neutral() { + // Limited-range mid gray: Y=126 (~0.5), Cb=Cr=128. + let y = Arc::new(vec![126u8; 4]); + let c = Arc::new(vec![128u8; 4]); + let f = VideoFrame { + width: 2, + height: 2, + data: PixelData::Yuv8 { planes: [y, c.clone(), c], chroma: Chroma::C444, alpha: None }, + color: ColorInfo::REC709, + par: (1, 1), + pts: Tick::ZERO, + }; + let rgba = f.to_rgba8(); + for p in rgba.as_chunks::<4>().0 { + assert!((p[0] as i16 - p[1] as i16).abs() <= 1, "{p:?}"); + assert!((p[1] as i16 - p[2] as i16).abs() <= 1, "{p:?}"); + assert!(p[0] > 90 && p[0] < 160, "{p:?}"); + assert_eq!(p[3], 255); + } + assert_eq!(f.luma8(), vec![126, 126, 126, 126]); + assert_eq!(f.format_label(), "YUV 4:4:4 8-bit"); + } + + #[test] + fn default_matrix_is_601_for_sd_and_709_otherwise() { + assert_eq!(default_matrix(720, 576), Matrix::Bt601); + assert_eq!(default_matrix(1920, 1080), Matrix::Bt709); + } + + #[test] + fn region_clip() { + let r = Region { x: 8, y: 8, w: 16, h: 16 }.clip(10, 10); + assert_eq!(r, Region { x: 8, y: 8, w: 2, h: 2 }); + assert!(Region { x: 20, y: 0, w: 4, h: 4 }.clip(10, 10).is_empty()); + assert!(Region { x: 0, y: 0, w: 10, h: 10 }.is_full(10, 10)); + } + + #[test] + fn transparent_float_encodes_as_zero_rgba8() { + let f = VideoFrame::transparent_f32(2, 1); + assert_eq!(f.to_rgba8(), vec![0; 8]); + assert_eq!(f.byte_size(), 32); // 2 pixels × 4 f32 channels × 4 bytes + } +} From a1f8141307dbe3e3c93adfc1a55e263b7bead8e0 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:04:19 +0100 Subject: [PATCH 048/209] media: reopen FileReader from the path resolved at open A relative path was stored as given, so after the pool evicted the handle a later working-directory change could reopen a different same-length file. Resolve std::path::absolute once before the first open and reuse that path. The regression runs in a subprocess so it does not change other tests' cwd. --- crates/media/src/file.rs | 61 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/crates/media/src/file.rs b/crates/media/src/file.rs index 67ba07c32..d87b323c0 100644 --- a/crates/media/src/file.rs +++ b/crates/media/src/file.rs @@ -150,18 +150,21 @@ impl FileReader { } pub(crate) fn open_in(path: &Path, pool: &Arc) -> io::Result { + // Resolve once against this process's cwd, so a later cwd change cannot + // make a reopen after eviction open a different file of the same length. + let path = std::path::absolute(path)?; // checked before opening: opening a named pipe with no writer blocks the caller for good - if !std::fs::metadata(path)?.is_file() { - return Err(not_a_file(path)); + if !std::fs::metadata(&path)?.is_file() { + return Err(not_a_file(&path)); } - let file = open_nonblocking(path)?; + let file = open_nonblocking(&path)?; let m = file.metadata()?; if !m.is_file() { - return Err(not_a_file(path)); + return Err(not_a_file(&path)); } let id = pool.next_id(); pool.insert(id, file); - Ok(Self { path: path.to_path_buf(), len: m.len(), id, pool: pool.clone() }) + Ok(Self { path, len: m.len(), id, pool: pool.clone() }) } /// The open file, reopened when the pool closed it. A file now shorter than when it was opened @@ -280,4 +283,52 @@ mod tests { } std::fs::remove_dir_all(&dir).ok(); } + + /// After eviction, a reader opened with a relative path must still read the original file + /// even if the process working directory has changed (same-length substitute in the new cwd). + /// The harness runs in a subprocess so it cannot change this process's cwd. + #[test] + fn reopening_a_relative_path_stays_on_the_original_file() { + const FLAG: &str = "CRAFT_MEDIA_FILE_READER_CWD_TEST"; + const ROOT: &str = "CRAFT_MEDIA_FILE_READER_CWD_ROOT"; + if std::env::var_os(FLAG).is_some() { + let root = PathBuf::from(std::env::var_os(ROOT).expect("test root")); + let a = root.join("a"); + let b = root.join("b"); + std::env::set_current_dir(&a).unwrap(); + let pool = Arc::new(FilePool::new(1)); + let reader = FileReader::open_in(Path::new("clip.bin"), &pool).unwrap(); + let _evict = FileReader::open_in(Path::new("other.bin"), &pool).unwrap(); + std::env::set_current_dir(&b).unwrap(); + let mut buf = [0u8; 4]; + reader.read_at(0, &mut buf).unwrap(); + assert_eq!(&buf, b"AAAA", "reopen must not follow the new cwd to b/clip.bin"); + return; + } + + let dir = craft_media_testkit::private_dir("file-cwd"); + std::fs::create_dir_all(dir.join("a")).unwrap(); + std::fs::create_dir_all(dir.join("b")).unwrap(); + std::fs::write(dir.join("a").join("clip.bin"), b"AAAA").unwrap(); + std::fs::write(dir.join("a").join("other.bin"), b"XXXX").unwrap(); + std::fs::write(dir.join("b").join("clip.bin"), b"BBBB").unwrap(); + let exe = std::env::current_exe().unwrap(); + let out = std::process::Command::new(&exe) + .env(FLAG, "1") + .env(ROOT, &dir) + .env("RUST_TEST_THREADS", "1") + .args(["--exact", "file::tests::reopening_a_relative_path_stays_on_the_original_file"]) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + out.status.success() + && stdout.contains("test file::tests::reopening_a_relative_path_stays_on_the_original_file") + && stdout.contains("ok"), + "subprocess failed\nstatus: {:?}\nstdout:\n{stdout}\nstderr:\n{stderr}", + out.status + ); + std::fs::remove_dir_all(&dir).ok(); + } } From e096b5eb9b565596f06e083de7efd2ebc0db9b63 Mon Sep 17 00:00:00 2001 From: Hamza Date: Mon, 16 Nov 2026 19:07:27 +0500 Subject: [PATCH 049/209] craft-csv: a diacritic before another escape keeps it as written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An escape's diacritic took the next character as its letter even when that was the next escape (ESC N H ESC N H o gave ESC + U+0308 + NHo). A control character is no letter: the first escape is kept as written and the next one decodes on its own (ESC N H ö). --- crates/csv/src/sylk.rs | 4 +++- crates/csv/tests/sylk.rs | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/csv/src/sylk.rs b/crates/csv/src/sylk.rs index d8c0ebec5..576ceb8c2 100644 --- a/crates/csv/src/sylk.rs +++ b/crates/csv/src/sylk.rs @@ -355,7 +355,9 @@ fn decode_escapes(text: &str) -> String { }; if let Some(&(_, mark, pairs)) = ISO6937_DIACRITICS.iter().find(|(d, _, _)| *d == code) { // The letter it goes on: precomposed where Unicode has the pair, else letter and mark. - let Some(base) = ahead.next() else { + // The base is an ordinary character: another escape (or any control character) after a + // diacritic leaves this escape as written, and the next one is decoded on its own. + let Some(base) = ahead.next().filter(|b| !b.is_control()) else { out.push(c); continue; }; diff --git a/crates/csv/tests/sylk.rs b/crates/csv/tests/sylk.rs index b91d07427..9fa7cb7d3 100644 --- a/crates/csv/tests/sylk.rs +++ b/crates/csv/tests/sylk.rs @@ -123,7 +123,7 @@ fn decodes_excels_character_escapes_in_text() { let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\ C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\ C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\ - C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nE\r\n"; + C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nC;Y8;X1;K\"\u{1b}NH\u{1b}NHo \u{1b}NB\tx\"\r\nE\r\n"; let sh = read(file.as_bytes()).unwrap(); let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default(); let text = |s: &str| Value::Text(s.into()); @@ -139,4 +139,7 @@ fn decodes_excels_character_escapes_in_text() { assert_eq!(value("A6"), text("q\u{308} \u{1b}Nö end\u{1b}N")); // Codes ISO 6937 doesn't define (0xFF, 0xCC) and an `ESC` without `N` stay as written. assert_eq!(value("A7"), text("\u{1b}N\u{7f} \u{1b}NL \u{1b}X")); + // A diacritic followed by another escape (or a control character) is kept as written, and the + // next escape still decodes. + assert_eq!(value("A8"), text("\u{1b}NHö \u{1b}NB\tx")); } From 503caef9c4da1a03ce27dd121280aefd1f07c22f Mon Sep 17 00:00:00 2001 From: Hamza Date: Mon, 16 Nov 2026 19:20:31 +0500 Subject: [PATCH 050/209] craft-csv: a SYLK diacritic on a space is the spacing accent ISO 6937 writes a spacing accent as its diacritic followed by SPACE. ESC N B SPACE was read as a space and a combining acute; it is now the acute accent itself, and likewise for the other nine marks glibc's ISO_6937 map gives a spacing form. --- crates/csv/src/sylk.rs | 22 ++++++++++++---------- crates/csv/tests/sylk.rs | 7 ++++++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/csv/src/sylk.rs b/crates/csv/src/sylk.rs index 576ceb8c2..53283cb91 100644 --- a/crates/csv/src/sylk.rs +++ b/crates/csv/src/sylk.rs @@ -241,20 +241,22 @@ fn parse_value(arg: &str) -> Value { /// ISO 6937's diacritics (codes 0xC1–0xCF): each code, its combining mark, and the letters it /// composes with, as pairs of letter and precomposed letter (from Unicode's canonical composition). +/// A diacritic on a SPACE is the spacing accent (`´`, `¨`, `ˇ`, ...), as in glibc's ISO_6937 map; the +/// grave, circumflex and tilde have none, since `` ` ``, `^` and `~` are plain ASCII. const ISO6937_DIACRITICS: [(u8, char, &str); 13] = [ (0xC1, '\u{0300}', "AÀaàEÈeèIÌiìNǸnǹOÒoòUÙuùWẀwẁYỲyỳ"), // grave - (0xC2, '\u{0301}', "AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute + (0xC2, '\u{0301}', " ´AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute (0xC3, '\u{0302}', "AÂaâCĈcĉEÊeêGĜgĝHĤhĥIÎiîJĴjĵOÔoôSŜsŝUÛuûWŴwŵYŶyŷZẐzẑ"), // circumflex (0xC4, '\u{0303}', "AÃaãEẼeẽIĨiĩNÑnñOÕoõUŨuũVṼvṽYỸyỹ"), // tilde - (0xC5, '\u{0304}', "AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron - (0xC6, '\u{0306}', "AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve - (0xC7, '\u{0307}', "AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above - (0xC8, '\u{0308}', "AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis - (0xCA, '\u{030a}', "AÅaåUŮuůwẘyẙ"), // ring - (0xCB, '\u{0327}', "CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla - (0xCD, '\u{030b}', "OŐoőUŰuű"), // double acute - (0xCE, '\u{0328}', "AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek - (0xCF, '\u{030c}', "AǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron + (0xC5, '\u{0304}', " ¯AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron + (0xC6, '\u{0306}', " ˘AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve + (0xC7, '\u{0307}', " ˙AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above + (0xC8, '\u{0308}', " ¨AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis + (0xCA, '\u{030a}', " ˚AÅaåUŮuůwẘyẙ"), // ring + (0xCB, '\u{0327}', " ¸CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla + (0xCD, '\u{030b}', " ˝OŐoőUŰuű"), // double acute + (0xCE, '\u{0328}', " ˛AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek + (0xCF, '\u{030c}', " ˇAǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron ]; /// ISO 6937's spacing characters above ASCII (codes 0xA1–0xFE that aren't diacritics). diff --git a/crates/csv/tests/sylk.rs b/crates/csv/tests/sylk.rs index 9fa7cb7d3..de3c38d0b 100644 --- a/crates/csv/tests/sylk.rs +++ b/crates/csv/tests/sylk.rs @@ -123,7 +123,8 @@ fn decodes_excels_character_escapes_in_text() { let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\ C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\ C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\ - C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nC;Y8;X1;K\"\u{1b}NH\u{1b}NHo \u{1b}NB\tx\"\r\nE\r\n"; + C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nC;Y8;X1;K\"\u{1b}NH\u{1b}NHo \u{1b}NB\tx\"\r\n\ + C;Y9;X1;K\"\u{1b}NB |\u{1b}NE |\u{1b}NF |\u{1b}NG |\u{1b}NH |\u{1b}NJ |\u{1b}NK |\u{1b}NM |\u{1b}NN |\u{1b}NO |\u{1b}NA |\u{1b}NC \"\r\nE\r\n"; let sh = read(file.as_bytes()).unwrap(); let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default(); let text = |s: &str| Value::Text(s.into()); @@ -142,4 +143,8 @@ fn decodes_excels_character_escapes_in_text() { // A diacritic followed by another escape (or a control character) is kept as written, and the // next escape still decodes. assert_eq!(value("A8"), text("\u{1b}NHö \u{1b}NB\tx")); + // A diacritic on a SPACE is the spacing accent: acute, macron, breve, dot above, diaeresis, ring, + // cedilla, double acute, ogonek, caron. The grave and circumflex have none (`` ` `` and `^` are + // ASCII), so they stay a space with the combining mark. + assert_eq!(value("A9"), text("´|¯|˘|˙|¨|˚|¸|˝|˛|ˇ| \u{300}| \u{302}")); } From c864e1e8c9e0f7e2c9f24fca1faa6f298a068bca Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:46:17 -0500 Subject: [PATCH 051/209] psd: support caller-budgeted prediction scratch --- crates/psd/src/compression.rs | 40 ++++++++++++++++++++++++++--- crates/psd/src/compression/tests.rs | 36 ++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/crates/psd/src/compression.rs b/crates/psd/src/compression.rs index b41936a04..f9d4a744e 100644 --- a/crates/psd/src/compression.rs +++ b/crates/psd/src/compression.rs @@ -745,8 +745,42 @@ pub fn unpredict(buf: &mut [u8], layout: &PlaneLayout) -> Result<()> { if rb == 0 { return Ok(()); } + let mut scratch = if layout.depth == 32 && buf.len() >= rb { vec![0; rb] } else { Vec::new() }; + unpredict_rows(buf, layout, &mut scratch) +} + +/// Reverses prediction without allocating, using caller-accounted scratch memory. +/// +/// `buf` must contain exactly `planes * height` complete rows. For 32-bit samples, +/// `scratch` must hold at least one row (`width * 4` bytes); other depths need none. +/// Layout, input length and scratch capacity are checked before any bytes are changed. +/// Scratch contents are overwritten. Unlike [`unpredict`], partial or surplus rows +/// are rejected, making this suitable for callers with explicit decode budgets. +pub fn unpredict_with_scratch(buf: &mut [u8], layout: &PlaneLayout, scratch: &mut [u8]) -> Result<()> { + let rb = match layout.depth { + 1 => layout.width.div_ceil(8), + 8 => layout.width, + 16 => layout.width.checked_mul(2).ok_or(PsdError::LimitExceeded("row size overflow"))?, + 32 => layout.width.checked_mul(4).ok_or(PsdError::LimitExceeded("row size overflow"))?, + d => return Err(PsdError::Unsupported(format!("prediction for depth {d}"))), + }; + let expected = + rb.checked_mul(layout.height).and_then(|n| n.checked_mul(layout.planes)).ok_or(PsdError::LimitExceeded("decoded size overflow"))?; + if buf.len() != expected { + return Err(PsdError::Invalid("prediction buffer length does not match layout".into())); + } + if rb == 0 || buf.is_empty() { + return Ok(()); + } + if layout.depth == 32 && scratch.len() < rb { + return Err(PsdError::Invalid("prediction scratch is shorter than one row".into())); + } + unpredict_rows(buf, layout, scratch) +} + +fn unpredict_rows(buf: &mut [u8], layout: &PlaneLayout, scratch: &mut [u8]) -> Result<()> { + let rb = layout.row_bytes(); let w = layout.width; - let mut scratch = Vec::new(); for row in buf.chunks_exact_mut(rb) { match layout.depth { 1 | 8 => { @@ -763,8 +797,8 @@ pub fn unpredict(buf: &mut [u8], layout: &PlaneLayout) -> Result<()> { } } 32 => { - scratch.clear(); - scratch.extend_from_slice(row); + scratch[..rb].copy_from_slice(row); + let scratch = &scratch[..rb]; // The byte-wise running sum runs across the four planes, so // plane k starts at the wrapping total of planes 0..k. let (p0, rest) = scratch.split_at(w); diff --git a/crates/psd/src/compression/tests.rs b/crates/psd/src/compression/tests.rs index 6e0d86a7e..ed0fd2dbc 100644 --- a/crates/psd/src/compression/tests.rs +++ b/crates/psd/src/compression/tests.rs @@ -771,3 +771,39 @@ fn rle_chunks_use_the_callers_pool_and_report_panics() { assert!(map_chunks(2, vec![0, 1], |_| panic!("worker")).is_err()); }); } + +#[test] +fn caller_scratch_prediction_matches_existing_decoder_across_planes() { + for depth in [1u16, 8, 16, 32] { + let layout = PlaneLayout { planes: 3, width: 13, height: 7, depth, version: Version::Psb }; + let mut actual: Vec = (0..layout.decoded_len().unwrap()).map(|i| (i.wrapping_mul(31) % 251) as u8).collect(); + let mut expected = actual.clone(); + unpredict(&mut expected, &layout).unwrap(); + let needed = if depth == 32 { layout.row_bytes() } else { 0 }; + let mut scratch = vec![0xa5; needed + 11]; + unpredict_with_scratch(&mut actual, &layout, &mut scratch).unwrap(); + assert_eq!(actual, expected, "depth {depth}"); + assert_eq!(&scratch[needed..], &[0xa5; 11]); + } +} + +#[test] +fn caller_scratch_prediction_rejects_invalid_inputs_before_mutation() { + let layout = PlaneLayout { planes: 1, width: 3, height: 2, depth: 32, version: Version::Psd }; + let original = vec![0x73; 24]; + for (len, scratch_len) in [(23, 12), (25, 12), (24, 11)] { + let mut bytes = vec![0x73; len]; + let mut scratch = vec![0xa5; scratch_len]; + assert!(unpredict_with_scratch(&mut bytes, &layout, &mut scratch).is_err()); + assert_eq!(bytes, vec![0x73; len]); + assert_eq!(scratch, vec![0xa5; scratch_len]); + } + for invalid in [PlaneLayout { width: usize::MAX, ..layout }, PlaneLayout { planes: usize::MAX, ..layout }, PlaneLayout { depth: 12, ..layout }] { + let mut bytes = original.clone(); + assert!(unpredict_with_scratch(&mut bytes, &invalid, &mut []).is_err()); + assert_eq!(bytes, original); + } + for empty in [PlaneLayout { width: 0, ..layout }, PlaneLayout { height: 0, ..layout }] { + unpredict_with_scratch(&mut [], &empty, &mut []).unwrap(); + } +} From 9ac3245966eb2b88a37930b1d820523124e0a02f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:13 -0500 Subject: [PATCH 052/209] settings, atomic: give a replaced file's group bits only to its group (#558) * settings, atomic: give a replaced file's group bits only to its group write_backup copied the config's mode onto a new backup inode without giving it the config's group, and craft-atomic's stage() did the same for every replaced file: on a setgid directory or with another primary group, the group bits opened the contents to a group never granted them. Both now chown the new inode to the replaced file's group first and drop the group bits when that fails. * address review: skip fchown when the stage already has the replaced file's group * atomic: route the stage's group change through AtomicFs so the fallback is tested --- crates/atomic/src/lib.rs | 90 +++++++++++++++++++++++++++++++-- crates/settings/src/recovery.rs | 17 ++++--- crates/settings/tests/store.rs | 4 ++ 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/crates/atomic/src/lib.rs b/crates/atomic/src/lib.rs index 1590372ad..08b881e17 100644 --- a/crates/atomic/src/lib.rs +++ b/crates/atomic/src/lib.rs @@ -70,6 +70,9 @@ pub trait AtomicFs { fn sync_dir(&self, dir: &Path) -> io::Result<()>; /// Give the open temporary file its final permissions. fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()>; + /// Put the open temporary file in group `gid` (Unix only). + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()>; } /// The production file system implementation. @@ -126,6 +129,11 @@ impl AtomicFs for RealFs { fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> { file.set_permissions(perms) } + + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()> { + std::os::unix::fs::fchown(file, None, Some(gid)) + } } /// Flush `file` to stable storage (`sync_all`). @@ -342,10 +350,29 @@ fn split(target: &Path, path: &Path) -> io::Result<(PathBuf, String)> { Ok((dir, name.to_string_lossy().into_owned())) } -/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions of the file it -/// replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when the mode -/// already matches after `fill`. A leftover file with the same name is skipped. When any step -/// fails, the temporary file is removed. +/// Give `perms` the replaced file's group, or drop its group bits when the stage cannot join +/// that group. A stage already in the group needs no fchown, which some filesystems reject even +/// as a no-op, and dropping the bits there would narrow a mode the replaced file granted. +#[cfg(unix)] +fn adopt_group(fs: &dyn AtomicFs, file: &File, stage_gid: u32, replaced_gid: u32, perms: std::fs::Permissions) -> io::Result { + use std::os::unix::fs::PermissionsExt; + if perms.mode() & 0o070 == 0 || stage_gid == replaced_gid { + return Ok(perms); + } + if fs.set_group(file, replaced_gid).is_ok() { + return Ok(perms); + } + let mut narrowed = perms; + narrowed.set_mode(narrowed.mode() & !0o070); + Ok(narrowed) +} + +/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions and group of +/// the file it replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when +/// the mode already matches after `fill`; the group bits are dropped when the file cannot be put +/// in the replaced file's group, so they never open the file to a group that was not granted +/// them. A leftover file with the same name is skipped. When any step fails, the temporary file +/// is removed. fn stage( fs: &dyn AtomicFs, dir: &Path, @@ -383,6 +410,14 @@ fn stage( if (file.metadata()?.permissions().mode() & 0o7777) == (p.mode() & 0o7777) { return Ok(()); } + // The group bits belong to the replaced file's group, which a new + // inode need not be in (setgid directory, another primary group). + if let Some(replaced) = replaced { + let stage_gid = std::os::unix::fs::MetadataExt::gid(&file.metadata()?); + let replaced_gid = std::os::unix::fs::MetadataExt::gid(replaced); + let perms = adopt_group(fs, &file, stage_gid, replaced_gid, p)?; + return fs.set_permissions(&file, perms); + } } fs.set_permissions(&file, p) }) @@ -678,6 +713,9 @@ mod tests { fixed_mode: Option, fail_sync_dir: bool, fail_normal_create: bool, + /// Simulate a filesystem that refuses the stage's group change. + #[cfg(unix)] + fail_set_group: bool, normal_collisions: Cell, /// Another process creates the target just before the hard link. created_meanwhile: Option<&'static [u8]>, @@ -750,6 +788,15 @@ mod tests { } /// Record permission restoration and payload length, then inject a configured error /// or delegate the mode change to the real filesystem. + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()> { + self.events.borrow_mut().push("set_group"); + if self.fail_set_group { + return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected: set_group failed")); + } + RealFs.set_group(file, gid) + } + fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> { self.events.borrow_mut().push("set_permissions"); #[cfg(unix)] @@ -1090,6 +1137,36 @@ mod tests { Ok(()) } + #[cfg(unix)] + #[test] + fn group_bits_are_dropped_when_the_stage_cannot_join_the_replaced_group() -> io::Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let d = TempDir::new("group-fallback"); + let file = File::create(d.0.join("stage"))?; + let gid = file.metadata()?.gid(); + let elsewhere = gid.wrapping_add(1); + + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o640))?.mode(); + assert_eq!(mode & 0o777, 0o600, "group bits are dropped when the group cannot be joined"); + assert!(fs.events.borrow().contains(&"set_group")); + + // No group bits: the group change is never attempted. + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o600))?.mode(); + assert_eq!(mode & 0o777, 0o600); + assert!(!fs.events.borrow().contains(&"set_group")); + + // A stage already in the replaced file's group keeps its bits untouched. + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, gid, std::fs::Permissions::from_mode(0o640))?.mode(); + assert_eq!(mode & 0o777, 0o640); + assert!(!fs.events.borrow().contains(&"set_group")); + + assert_no_temp(&d.0); + Ok(()) + } + #[cfg(unix)] #[test] fn normal_policy_preserves_existing_modes_with_private_staging() -> io::Result<()> { @@ -1102,6 +1179,11 @@ mod tests { let fs = Faulty { fail_normal_create: true, ..Default::default() }; replace_with_options(&fs, NO_RETRY, &p, NewFilePermissions::PlatformDefault, |f| f.write_all(b"replacement"))?; assert_eq!(std::fs::metadata(&p)?.permissions().mode() & 0o777, mode); + assert_eq!( + std::os::unix::fs::MetadataExt::gid(&std::fs::metadata(&p)?), + std::os::unix::fs::MetadataExt::gid(&d.0.metadata()?), + "kept in the directory's group" + ); assert!(!fs.events.borrow().contains(&"create_new_normal")); assert_eq!(*fs.created_modes.borrow(), [0o600]); } diff --git a/crates/settings/src/recovery.rs b/crates/settings/src/recovery.rs index 074a1b4a8..1435797e0 100644 --- a/crates/settings/src/recovery.rs +++ b/crates/settings/src/recovery.rs @@ -37,23 +37,28 @@ pub fn backup_path(path: &Path) -> PathBuf { /// Writes `text`, the last version that parsed, to the backup file atomically. /// /// The caller decides what is safe to keep: a blank or unparsable file must never replace the -/// previous backup. On Unix the backup always gets the same mode as `path` (`0600` when there is -/// none), so it is never readable by more people than the settings themselves. +/// previous backup. On Unix the backup always gets the same mode and group as `path` (`0600` +/// when there is none), so it is never readable by more people than the settings themselves; +/// when it cannot be put in the config's group, it keeps the owner bits only. pub fn write_backup(path: &Path, text: &str) -> io::Result<()> { let backup = backup_path(path); #[cfg(unix)] { - use std::os::unix::fs::PermissionsExt; - let mode = std::fs::metadata(path).map(|m| m.permissions().mode() & 0o777).unwrap_or(0o600); + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let config = std::fs::metadata(path).ok(); + let mode = config.as_ref().map(|m| m.permissions().mode() & 0o777).unwrap_or(0o600); // An atomic replace keeps the existing file's mode, so narrow an old, broader backup // before new contents go into it, then match the config once written. if backup.exists() { - std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode))?; + std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode & 0o700))?; } craft_atomic::WriteOptions::default() .with_new_file_permissions(craft_atomic::NewFilePermissions::Private) .atomic_write(&backup, text.as_bytes())?; - std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode)) + // The group bits were granted to the config's group; the new file may have been created + // in another (a setgid directory, a different primary group). + let grouped = config.as_ref().is_some_and(|m| std::os::unix::fs::chown(&backup, None, Some(m.gid())).is_ok()); + std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(if grouped { mode } else { mode & 0o700 })) } #[cfg(not(unix))] { diff --git a/crates/settings/tests/store.rs b/crates/settings/tests/store.rs index bc55d97cb..a9c4b9553 100644 --- a/crates/settings/tests/store.rs +++ b/crates/settings/tests/store.rs @@ -289,4 +289,8 @@ fn backup_follows_a_mode_the_user_chose_for_the_config() { store.update(|p| p.general.language = "fr".into()); store.flush().unwrap(); assert_eq!(mode(&tmp.path().join("config.toml.bak")), 0o640); + // The group bits authorize the config's group, which the backup is in. + use std::os::unix::fs::MetadataExt; + let gid = |p: &std::path::Path| std::fs::metadata(p).unwrap().gid(); + assert_eq!(gid(&tmp.path().join("config.toml.bak")), gid(&config_path(tmp.path()))); } From eeae2bb7ca5bdc201c608c321d6bcf9330858fa3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:17 -0500 Subject: [PATCH 053/209] feat(ssip): extract bounded Speech Dispatcher wire helpers (#1500) Adapted from WordCraft SSIP implementation; preserves contributor framing behavior while rejecting overlong complete lines. Co-authored-by: Skaywalker <115165402+Skaywalker@users.noreply.github.com> --- Cargo.lock | 4 ++ Cargo.toml | 1 + crates/ssip/Cargo.toml | 11 ++++ crates/ssip/README.md | 19 ++++++ crates/ssip/src/lib.rs | 140 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+) create mode 100644 crates/ssip/Cargo.toml create mode 100644 crates/ssip/README.md create mode 100644 crates/ssip/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 83670ba58..3385eab1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2798,6 +2798,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-ssip" +version = "0.1.0" + [[package]] name = "craft-stencilcraft-format" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 07aebdc37..801dc9ba8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ craft-shx = { path = "crates/shx" } craft-http-url = { path = "crates/http-url" } craft-usd = { path = "crates/usd" } craft-smime = { path = "crates/smime" } +craft-ssip = { path = "crates/ssip" } craft-ber = { path = "crates/ber" } craft-proto = { path = "crates/proto" } craft-render-parity = { path = "crates/render-parity" } diff --git a/crates/ssip/Cargo.toml b/crates/ssip/Cargo.toml new file mode 100644 index 000000000..5ff35544f --- /dev/null +++ b/crates/ssip/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-ssip" +description = "Bounded Speech Dispatcher SSIP wire framing and message-data encoding" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true diff --git a/crates/ssip/README.md b/crates/ssip/README.md new file mode 100644 index 000000000..7fc98924e --- /dev/null +++ b/crates/ssip/README.md @@ -0,0 +1,19 @@ +# craft-ssip + +A focused Speech Dispatcher SSIP wire layer: sentence-as-data encoding, final response codes, +and a bounded incremental reply reader. No sockets, process launch, cancellation commands, +voice discovery, retry or playback scheduling: those policies remain with the application. + +Derived from Skaywalker's WordCraft SSIP implementation in +`crates/engine/src/speech/ssip.rs` at `e9750c5cb75d577a4669059d7bf6f9874e945839` +(MIT OR Apache-2.0). WordCraft #2937 (`65d7999eddebf120ca9fde523b01156b3e1f5fce`) +adds voice selection on this same core; this extraction does not duplicate those application changes. + +Unlike the original reader, a complete line that becomes overlong in the final read is refused. +The 4096-byte limit counts bytes before LF, including an optional CR. Reply buffering holds +at most that limit plus one 512-byte read. Invalid UTF-8 is replaced, preserving WordCraft behavior. +Message input is bounded by the caller; ASCII three-digit response codes are enforced. +Caller-provided `Read` implementations must obey Rust's Read contract; callers remain responsible +for transport timeouts, cancellation and an overall deadline between `next_line` calls. + +This is not a complete SSIP client and does not establish WordCraft consumer cutover. diff --git a/crates/ssip/src/lib.rs b/crates/ssip/src/lib.rs new file mode 100644 index 000000000..8bb4cce37 --- /dev/null +++ b/crates/ssip/src/lib.rs @@ -0,0 +1,140 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Speech Dispatcher SSIP wire helpers extracted from WordCraft's contributor implementation. +//! Socket discovery, deadlines, voice selection, retry and cancellation policy stay in the app. +use std::io::{self, ErrorKind, Read}; + +/// Maximum reply bytes before LF, including an optional CR. +pub const MAX_LINE: usize = 4096; + +/// Encode one sentence as data, never as protocol commands. CR/LF become spaces; a leading +/// dot is doubled and the message terminator is appended. The caller bounds input bytes. +pub fn message_data(text: &str, max_input_bytes: usize) -> io::Result { + if text.len() > max_input_bytes { + return Err(ErrorKind::InvalidInput.into()); + } + let size = text.len().checked_add(6).ok_or(ErrorKind::InvalidInput)?; + let mut out = String::new(); + out.try_reserve_exact(size).map_err(|_| io::Error::other("SSIP message allocation failed"))?; + if text.starts_with('.') { + out.push('.'); + } + out.extend(text.chars().map(|c| if c == '\r' || c == '\n' { ' ' } else { c })); + out.push_str("\r\n.\r\n"); + Ok(out) +} + +/// The three ASCII digits of a final response; continuation (`225-...`) is not final. +pub fn final_code(line: &str) -> Option { + let digits = line.as_bytes().get(..3)?; + if !digits.iter().all(u8::is_ascii_digit) || !(line.len() == 3 || line.as_bytes().get(3) == Some(&b' ')) { + return None; + } + line.get(..3)?.parse().ok() +} + +/// Incremental replies. At most one read per call lets the app enforce its own deadline. +/// Invalid UTF-8 retains WordCraft's replacement-character behavior. Overlong replies fail, +/// including those whose terminating newline arrives in the same read that exceeds the cap. +pub struct Lines { + reader: R, + pending: Vec, +} + +impl Lines { + pub fn new(reader: R) -> Self { + Self { reader, pending: Vec::new() } + } + + /// Complete next line without CR/LF, or None after a partial read or transient timeout. + /// The caller must set transport timeouts and enforce cancellation/deadlines. + pub fn next_line(&mut self) -> io::Result> { + if let Some(line) = self.complete_line()? { + return Ok(Some(line)); + } + let mut buf = [0u8; 512]; + match self.reader.read(&mut buf) { + Ok(0) => Err(ErrorKind::UnexpectedEof.into()), + Ok(n) => { + let bytes = buf.get(..n).ok_or(ErrorKind::InvalidData)?; + // Invariant: pending <= MAX_LINE before this read, so storage <= MAX_LINE+512. + self.pending.try_reserve(bytes.len()).map_err(|_| io::Error::other("SSIP reply allocation failed"))?; + self.pending.extend_from_slice(bytes); + self.complete_line() + } + Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted) => Ok(None), + Err(e) => Err(e), + } + } + + fn complete_line(&mut self) -> io::Result> { + let end = self.pending.iter().position(|&b| b == b'\n'); + if end.unwrap_or(self.pending.len()) > MAX_LINE { + return Err(ErrorKind::InvalidData.into()); + } + let Some(end) = end else { return Ok(None) }; + let bytes = self.pending.get(..end).ok_or(ErrorKind::InvalidData)?; + let line = String::from_utf8_lossy(bytes).trim_end_matches('\r').to_string(); + self.pending.drain(..=end); + Ok(Some(line)) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + use std::collections::VecDeque; + struct Chunks(VecDeque>>); + impl Read for Chunks { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let bytes = self.0.pop_front().unwrap_or(Ok(Vec::new()))?; + out[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + #[test] + fn message_text_cannot_inject_commands() { + assert_eq!(message_data(".\r\nCANCEL ALL\n!-!", 64).unwrap(), ".. CANCEL ALL !-!\r\n.\r\n"); + assert_eq!(message_data("你好", 6).unwrap(), "你好\r\n.\r\n"); + assert!(message_data("你好", 5).is_err()); + } + #[test] + fn final_codes_are_ascii_and_not_continuations() { + assert_eq!(final_code("225 OK"), Some(225)); + assert_eq!(final_code("701"), Some(701)); + for bad in ["225-12", "+22 OK", "22 OK", "225 OK", "225x", ""] { + assert_eq!(final_code(bad), None); + } + } + #[test] + fn fragmented_utf8_timeout_and_buffered_lines() { + let mut lines = Lines::new(Chunks(VecDeque::from([ + Ok(vec![b'2', b'2', b'5', b' ', 0xc3]), + Err(ErrorKind::TimedOut.into()), + Ok(vec![0xa9, b'\r', b'\n', b'7', b'0', b'2', b'\n']), + ]))); + assert!(lines.next_line().unwrap().is_none()); + assert!(lines.next_line().unwrap().is_none()); + assert_eq!(lines.next_line().unwrap().as_deref(), Some("225 é")); + assert_eq!(lines.next_line().unwrap().as_deref(), Some("702")); + assert_eq!(lines.next_line().unwrap_err().kind(), ErrorKind::UnexpectedEof); + } + #[test] + fn completed_overlong_line_is_rejected_at_boundary() { + for extra in [0, 1, 511] { + let mut chunks: VecDeque<_> = (0..8).map(|_| Ok(vec![b'x'; 512])).collect(); + let mut last = vec![b'x'; extra]; + last.push(b'\n'); + chunks.push_back(Ok(last)); + let mut lines = Lines::new(Chunks(chunks)); + for _ in 0..8 { + assert!(lines.next_line().unwrap().is_none()); + } + if extra == 0 { + assert_eq!(lines.next_line().unwrap().unwrap().len(), MAX_LINE); + } else { + assert_eq!(lines.next_line().unwrap_err().kind(), ErrorKind::InvalidData); + } + } + } +} From 094537374cc9878e141f829748fb91d329b472b7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:22 -0500 Subject: [PATCH 054/209] Integrate shared IPC with full executable ancestry validation (#1571) * craft-ipc: local-socket server, client and discovery for Craft apps Adds craft-ipc, the transport for a Craft app that talks to other Craft apps on one machine. It is native only: interprocess has no wasm32 build. - server and client: one JSON object per line, push events, a bulk connection for frames, token auth on every connection, connection and time limits. - discovery: one record per running app in a per-user runtime directory, with a lock file that shows whether the owner is alive. - launch, trust, cache: find an app binary, start it only after an executable check, keep caches in a private per-user directory. - activate: focus hand-off between apps. - moniker, record, payload, asset, refresh, fetch, watch: asset names, stored link records, open payloads, linked-file polling and a frame cache. - PROTOCOL.md describes the wire protocol. The record schemas and fixtures are in the crate. - The crate compiles and passes clippy -D warnings on x86_64-pc-windows-gnu and aarch64-apple-darwin. This is a compile and lint fix only. No one ran the Windows or macOS code paths, so this does not claim they work there. Part of #414. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: app.activate no longer claims a request it cannot make ActivationBackend gains can_apply_token (default false). On Linux a token only counts when the backend can apply it to a live window. Stock winit cannot, so the answer is false plus a request for user attention. Wire values are unchanged. PROTOCOL.md says what each value means. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: keep a partial reply line across client read timeouts pull_line dropped the bytes it had read when a read timed out, so next_event returned Ok(None) and the next read saw only the tail of the line. The client now keeps the partial line and resumes it on the next read. A test splits an event line across a timeout. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: scope the macOS fallback runtime directory to the user With TMPDIR unset or relative, the fallback was the shared path /tmp/craft. The first user to create it owned it, and every other user then failed the owner check. The fallback is now /tmp/craft-, as on Linux. The path choice is a pure function with a test. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: fetch rejects non-finite time, bounds its queue, keeps waited-for asks Three changes to FrameFetcher::fetch. A non-finite time_s was queued and failed in request_reply before any socket I/O. The worker read that as a dead connection and exited, so enough bad asks took every worker down. fetch now returns Fetched::Failed and queues nothing. The queue had no length limit, so stalled workers let it grow with the request rate. It now holds at most QUEUE_MAX asks (256, a new public constant). A miss past the cap drops the oldest queued ask and counts it in stats.skipped. next_job dropped any job older than max_age, even when the caller had waited longer than max_age and was still there. The caller then re-queued at the tail. Each job now records the latest deadline of its waiting callers and is skipped for age only when no caller is left. Tests cover each case. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: reject file:// media URLs that do not percent-decode local_path returned None for a file:// URL with a bad percent escape, and missing_media then skipped it, so the server accepted a local target without the media probe. Such a URL now gets payload_invalid. URLs that name no local file (remote host, other scheme, relative path) are still left to the app. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: open watched files without waiting on a FIFO Tracker::poll, Tracker::new and fingerprint opened the path with File::open before stat checked the file type. A FIFO with no writer made that open wait, and poll_entry held the entry lock for the whole call, so one such path stalled every link on the watcher and the watcher drop. The open now sets O_NONBLOCK (new crate fsopen), and stat rejects the FIFO on the open handle with LinkError::Invalid. A check before the open would not close the window between check and open. craft-ipc gains a unix-only dependency on libc for the open flags. libc 0.2.190 is already in Cargo.lock. The lock file gains one dependency line for craft-ipc. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: read_private checks and reads one open handle read_private checked the path with symlink_metadata and then read it with fs::read, so the entry could change between the two calls. On Unix it now opens once with O_NOFOLLOW and O_NONBLOCK, checks type, owner and mode on that handle, and reads from the same handle. A symlink is still Refused (found by looking at the path after a failed open, because the errno differs between Unix systems), a FIFO is Refused and cannot stall the open, and a missing file is still an I/O error. Other platforms keep the old checks. Tests cover the symlink and FIFO cases. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: Tracker reports an epoch it cannot raise instead of a false bump apply saturated the epoch at u64::MAX while poll and relink still returned bumped: true, so Change::bumped (documented as one added to the epoch) was false and a cached frame kept passing for changed content. poll and relink now return LinkError::Invalid and leave the tracker as it was when the bump would pass u64::MAX. A test covers both calls. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: document write_atomic errors that follow the rename The directory flush and the fingerprint run after the rename. An error from either can be returned while the path already holds the new bytes. The doc comment now says so. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: say that check_executable accepts group-writable paths The module and function docs described the check as protection from other local users. A group-writable file or directory passes, and the other members of its group can replace it. The docs now state that and keep the private-group reason. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: refuse an auth line that completes after the deadline authenticate returned a completed line before it looked at the deadline, so a line that finished after auth_deadline still authenticated. The overshoot was up to one 250 ms read poll. The deadline is now checked first, with the same auth failure reply as the other expiry paths. A test sends the line between the deadline and the next poll. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: do not append an error record after a partial frame write send_frame wrote an error record whenever write_frame failed. When the failure came from the stream (for example a send timeout in the middle of a record), the first record was cut short, and a peer that drained the buffer then read the error record as the rest of it. send_frame now checks that the record can be built before it writes. A record that cannot be built gets the error record as before. A write failure is returned alone and the bulk loop closes the connection. wire gains check_cfrm, the pre-write part of write_cfrm. write_cfrm behaves as before. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: LineReader applies the line limit to a last line without a newline At end of stream LineReader::read returned a buffered line with no length check, so a peer that sent one byte over the limit and closed its write side got that line dispatched, while the same bytes with a newline were refused. The end of stream path now returns TooLarge for a line over the limit. A test covers both paths. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: answer a control reply over the line limit with payload_too_large write_value passed every reply to write_line, which has no size check. A sink result larger than MAX_LINE went out as one line that the client refuses, and the rest of the line stayed unread on the connection. Replies now go through wire::write_reply_line. A reply over the limit is replaced by a payload_too_large error that keeps the request id. When that does not fit either, or the value has no id, nothing is written and the request loop ends. Event lines are unchanged. wire gains write_reply_line. write_line behaves as before. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: accept a shallow or deep poll in the native ticker test The test expected the first notice to come from a shallow poll. With a 50 ms deep interval the poll that first sees the edit can be a deep one, so the assertion failed on a slow or loaded machine. It now takes the deep flag from the notice and checks every other field. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: refuse a frame stride that cannot hold one row FrameLayout::check only tested len == stride * h, so w = 4, stride = 1, h = 3, len = 3 was a valid rgba8 layout. A reader that splits the payload by stride cannot decode it. frames.open, a rendered frame and the client reply parser now require stride >= w * bytes per pixel (4 for rgba8_premul, 8 for rgba16f_premul). open checks before it mints a ticket, so a refused layout leaves no ticket. The bulk test source used stride 16 for rgba16f with w = 4, which needs 32. It now uses a stride that fits the format. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: reject calendar days that do not exist in a record's created field valid_rfc3339 checked the day only against 1 to 31, so 2026-02-31T09:30:00Z passed LinkRecord::validate while the schema's date-time format refused it. The year, month and day now go through chrono::NaiveDate::from_ymd_opt, as the workspace rules require for calendar logic. The accepted syntax is unchanged. The schema agreement table gets rows for impossible days and for leap days. craft-ipc gains the workspace chrono dependency. chrono is already in Cargo.lock; the lock file gains one dependency line for craft-ipc. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: make the link record moniker pattern match Moniker::parse The asset.moniker pattern accepted any id without / ? #, so a record the schema passed could fail LinkRecord::validate (bad or truncated escape, raw space, escape that decodes to invalid UTF-8, id over 512 bytes). The pattern is now two patterns under allOf. One checks the shape: unreserved characters and two-digit escapes only, 1 to 512 of them, which is 1 to 512 decoded bytes. The other checks that the decoded bytes are valid UTF-8, written as the well-formed UTF-8 byte sequences. Neither uses lookahead, so any ECMA 262 engine can run them. The schema agreement test gets rows for each case in the list above and for the length limits, and a sweep of lead, second and tail bytes that compares the schema verdict with the Rust verdict. This tightens the published schema: documents it accepted that Rust refused are now refused. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: document that Assets methods must return in bounded time Server::shutdown and drop join every connection thread, and an Assets call runs on one with no deadline and no view of the stop flag. The trait docs now say so and ask implementations to bound their own I/O. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: name the writer type in the listener tests Clippy's type_complexity lint failed on the test helper's return type. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: run Assets and Activate calls on detached helper threads with a deadline Server::shutdown and drop join every connection thread, and an Assets or Activate call ran on one with no deadline and no view of the stop flag, so a provider that never returned made shutdown hang. These calls now run on a helper thread (server/provider.rs). The connection thread waits for the answer in 100 ms slices, for at most Limits::provider_deadline (default REPLY_DEADLINE_SECS) and not past the stop flag. On timeout the caller gets a timeout reply and the helper is detached, so shutdown does not join it. At most 32 helper threads are alive at once, stuck ones included, and a call past that gets a busy error without starting a thread. A panic in a provider becomes an io_error reply. Frames stays on the connection thread because it is one call per frame. Its trait docs now say that the stop flag is checked before each request and that a running call is waited for. The module docs describe the shutdown guarantee for each provider kind. Public API: new Limits::provider_deadline field. Tests use a provider that blocks until released, for Assets and for Activate. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: give a caller whose queued ask was dropped a Failed outcome When the fetch queue was full, the oldest ask was dropped but its waiting callers got no answer. They woke, found the key neither cached nor in flight, and queued it again, which dropped another ask and repeated the cycle. A dropped ask now records a failure for its key before the waiters are woken, so each waiter returns Fetched::Failed and does not queue the key again. The drop-oldest policy and stats.skipped are unchanged. Failure records are shared with the worker path and kept to a fixed number, oldest first. No type or variant changes. The Fetched::Failed doc now covers this case. A test overflows the queue while a caller waits on the oldest key, and checks that the caller returns at once and that no further asks are dropped. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: require two hex digits in a file URL percent escape percent_decode used u8::from_str_radix, which accepts a leading plus, so file:///tmp/a%+fb decoded to the byte 0x0f and went on to the media probe instead of getting payload_invalid. The two bytes after a percent sign must now both be ASCII hex digits. The decode test gets a plus, a minus, a space and a short escape. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: make the euid probe create only its own file, and gate the macOS path test On Unix systems other than Linux, current_euid read the owner of a file named .craft-uid- in the temp directory, made with create and truncate. A file or symlink planted at that name was opened and truncated. The probe now lives in probe_euid: random name, create_new (which refuses an existing path), mode 0600, and removal before it returns. The std library has no safe euid call and this crate forbids unsafe code, so the probe stays. A test plants a file and a symlink at the old and a similar name and checks that both are untouched and that the probe file is gone. macos_fallback_is_per_user compares Unix paths, so it and the macos_runtime_path helper it needs are now compiled for tests on Unix only. That removes the unused function warning in Windows test builds. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: an event larger than one line becomes an events.overflow marker Server::emit wrote the serialized event unchecked. A line over MAX_LINE made Client::next_event return TooLarge and left the stream out of sync. enqueue now checks the serialized size and queues the existing overflow marker instead. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * ipc: validate executable path ancestry * ipc: preserve queued replies after macOS peer shutdown * ipc: validate native transports and use portable platform fixtures * Make IPC integration fixtures portable and collision-free * Use native paths and unique directories across IPC fixtures * Bound Windows pipe I/O and stabilize IPC tests * Retry empty nonblocking Windows pipes * #433: Serialize hub tests sharing a Windows pipe * #433: Use native endpoint in schema capability tests * ipc: report a non-regular watch target as Invalid on every platform * ipc: format the new error classification --------- Co-authored-by: User Co-authored-by: Claude Sonnet 5.5 --- Cargo.lock | 286 ++++++ Cargo.toml | 1 + README.md | 1 + crates/ipc/Cargo.toml | 34 + crates/ipc/PROTOCOL.md | 297 ++++++ crates/ipc/README.md | 38 + .../ipc/schemas/instance-record.schema.json | 63 ++ crates/ipc/schemas/link-record.schema.json | 63 ++ crates/ipc/src/activate.rs | 362 ++++++++ crates/ipc/src/asset.rs | 70 ++ crates/ipc/src/auth.rs | 107 +++ crates/ipc/src/bulk.rs | 487 ++++++++++ crates/ipc/src/cache.rs | 278 ++++++ crates/ipc/src/client.rs | 853 ++++++++++++++++++ crates/ipc/src/discovery/dir.rs | 354 ++++++++ crates/ipc/src/discovery/mod.rs | 11 + crates/ipc/src/discovery/record.rs | 575 ++++++++++++ crates/ipc/src/error.rs | 134 +++ crates/ipc/src/fetch.rs | 370 ++++++++ crates/ipc/src/fsopen.rs | 25 + crates/ipc/src/ids.rs | 41 + crates/ipc/src/launch.rs | 777 ++++++++++++++++ crates/ipc/src/lib.rs | 39 + crates/ipc/src/moniker.rs | 313 +++++++ crates/ipc/src/payload.rs | 396 ++++++++ crates/ipc/src/record.rs | 237 +++++ crates/ipc/src/refresh.rs | 572 ++++++++++++ crates/ipc/src/server/dispatch.rs | 316 +++++++ crates/ipc/src/server/events.rs | 214 +++++ crates/ipc/src/server/listener.rs | 633 +++++++++++++ crates/ipc/src/server/mod.rs | 376 ++++++++ crates/ipc/src/server/provider.rs | 71 ++ crates/ipc/src/server/tcp.rs | 32 + crates/ipc/src/trust.rs | 215 +++++ crates/ipc/src/watch.rs | 397 ++++++++ crates/ipc/src/wire.rs | 561 ++++++++++++ crates/ipc/tests/bulk.rs | 334 +++++++ crates/ipc/tests/connection_limits.rs | 280 ++++++ crates/ipc/tests/discovery.rs | 97 ++ crates/ipc/tests/fetch.rs | 366 ++++++++ .../link-record-filmcraft-effectcraft.json | 13 + .../ipc/tests/fixtures/link-record-still.json | 15 + crates/ipc/tests/hub.rs | 334 +++++++ crates/ipc/tests/native_roundtrip.rs | 93 ++ crates/ipc/tests/open_payload.rs | 169 ++++ crates/ipc/tests/protocol.rs | 361 ++++++++ crates/ipc/tests/provider_stall.rs | 248 +++++ crates/ipc/tests/schemas.rs | 318 +++++++ crates/ipc/tests/security.rs | 290 ++++++ crates/ipc/tests/watch.rs | 361 ++++++++ 50 files changed, 12878 insertions(+) create mode 100644 crates/ipc/Cargo.toml create mode 100644 crates/ipc/PROTOCOL.md create mode 100644 crates/ipc/README.md create mode 100644 crates/ipc/schemas/instance-record.schema.json create mode 100644 crates/ipc/schemas/link-record.schema.json create mode 100644 crates/ipc/src/activate.rs create mode 100644 crates/ipc/src/asset.rs create mode 100644 crates/ipc/src/auth.rs create mode 100644 crates/ipc/src/bulk.rs create mode 100644 crates/ipc/src/cache.rs create mode 100644 crates/ipc/src/client.rs create mode 100644 crates/ipc/src/discovery/dir.rs create mode 100644 crates/ipc/src/discovery/mod.rs create mode 100644 crates/ipc/src/discovery/record.rs create mode 100644 crates/ipc/src/error.rs create mode 100644 crates/ipc/src/fetch.rs create mode 100644 crates/ipc/src/fsopen.rs create mode 100644 crates/ipc/src/ids.rs create mode 100644 crates/ipc/src/launch.rs create mode 100644 crates/ipc/src/lib.rs create mode 100644 crates/ipc/src/moniker.rs create mode 100644 crates/ipc/src/payload.rs create mode 100644 crates/ipc/src/record.rs create mode 100644 crates/ipc/src/refresh.rs create mode 100644 crates/ipc/src/server/dispatch.rs create mode 100644 crates/ipc/src/server/events.rs create mode 100644 crates/ipc/src/server/listener.rs create mode 100644 crates/ipc/src/server/mod.rs create mode 100644 crates/ipc/src/server/provider.rs create mode 100644 crates/ipc/src/server/tcp.rs create mode 100644 crates/ipc/src/trust.rs create mode 100644 crates/ipc/src/watch.rs create mode 100644 crates/ipc/src/wire.rs create mode 100644 crates/ipc/tests/bulk.rs create mode 100644 crates/ipc/tests/connection_limits.rs create mode 100644 crates/ipc/tests/discovery.rs create mode 100644 crates/ipc/tests/fetch.rs create mode 100644 crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json create mode 100644 crates/ipc/tests/fixtures/link-record-still.json create mode 100644 crates/ipc/tests/hub.rs create mode 100644 crates/ipc/tests/native_roundtrip.rs create mode 100644 crates/ipc/tests/open_payload.rs create mode 100644 crates/ipc/tests/protocol.rs create mode 100644 crates/ipc/tests/provider_stall.rs create mode 100644 crates/ipc/tests/schemas.rs create mode 100644 crates/ipc/tests/security.rs create mode 100644 crates/ipc/tests/watch.rs diff --git a/Cargo.lock b/Cargo.lock index 3385eab1f..5f28d1878 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,12 +116,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "serde", "version_check", "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned" version = "0.4.3" @@ -576,6 +586,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + [[package]] name = "bit-set" version = "0.10.0" @@ -594,6 +613,12 @@ dependencies = [ "bit-vec 0.10.1", ] +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -772,6 +797,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "borsh" version = "1.8.1" @@ -883,6 +914,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.2" @@ -2031,6 +2068,21 @@ dependencies = [ "indd", ] +[[package]] +name = "craft-ipc" +version = "0.1.0" +dependencies = [ + "chrono", + "getrandom 0.2.17", + "interprocess", + "jsonschema", + "libc", + "serde", + "serde_json", + "sha2 0.10.9", + "widestring", +] + [[package]] name = "craft-jmap" version = "0.1.0" @@ -3703,6 +3755,12 @@ dependencies = [ "libloading", ] +[[package]] +name = "doctest-file" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359" + [[package]] name = "document-features" version = "0.2.12" @@ -4051,6 +4109,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "emath" version = "0.37.0" @@ -4263,6 +4330,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -4376,6 +4454,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fmt-derive" version = "0.1.2" @@ -4528,6 +4617,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs-set-times" version = "0.20.3" @@ -4959,6 +5058,8 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -5607,6 +5708,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "interprocess" +version = "2.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb02ffbf25e8ef0f4a95bff054eef75f6d1bf16919474cf2caa63b21997478a" +dependencies = [ + "doctest-file", + "libc", + "recvmsg", + "widestring", + "windows-sys 0.61.2", +] + [[package]] name = "io-extras" version = "0.19.0" @@ -5784,6 +5898,58 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonschema" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "jsonschema-regex", + "jsonschema-value", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "strum", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "jsonschema-value" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" +dependencies = [ + "ahash", + "bytecount", + "fraction", + "num-cmp", + "num-traits", + "serde_json", +] + [[package]] name = "jxl-bitstream" version = "1.1.0" @@ -6308,6 +6474,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + [[package]] name = "mime" version = "0.3.17" @@ -6552,6 +6724,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -6579,6 +6765,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.6" @@ -7094,6 +7286,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -8219,6 +8417,12 @@ dependencies = [ "yoke", ] +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + [[package]] name = "redox_syscall" version = "0.4.1" @@ -8256,6 +8460,66 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "referencing" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown 0.17.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-lite" version = "0.1.9" @@ -10525,6 +10789,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "v_frame" version = "0.3.9" @@ -10847,6 +11121,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -11316,6 +11596,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi-util" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index 801dc9ba8..0ed80bda2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,7 @@ craft-heif = { path = "crates/heif" } craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } +craft-ipc = { path = "crates/ipc" } craft-idml = { path = "crates/idml" } craft-indd = { path = "crates/indd" } craft-jmap = { path = "crates/jmap" } diff --git a/README.md b/README.md index a64889992..47b223d48 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit primitives (`.abr`, `.asl`, `.aco`, `.act`, `.ase`, `.grd`, `.pat`, `.atn`, `.csh`, `.shc`) and the Photoshop blocks of a layered TIFF. Bounded and panic-free on hostile input. Compositing and the mapping onto a document stay in each app. +- [`craft-ipc`](crates/ipc/README.md): Craft apps talking on one machine: per-user local sockets (Unix socket, Windows named pipe), token auth, instance discovery, launch on demand, focus hand-off, frame transfer and linked-file watching. Native only: it does not build for `wasm32`. Protocol in [PROTOCOL.md](crates/ipc/PROTOCOL.md). - [`craft-jmap`](crates/jmap/README.md): the JMAP protocol core (RFC 8620), from MailCraft, without sockets: session, requests of method calls with back-references, responses, ids, address templates, credentials and the judgement of HTTP answers, over any HTTP implementation. Bounded diff --git a/crates/ipc/Cargo.toml b/crates/ipc/Cargo.toml new file mode 100644 index 000000000..5d4b0ff06 --- /dev/null +++ b/crates/ipc/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "craft-ipc" +description = "Local-socket control server, client and discovery for Craft apps on one machine: token auth, launch on demand, focus hand-off. Native only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +# Native only: `interprocess` has no wasm32 build, so this crate does not build for +# wasm32-unknown-unknown. Apps gate it by target. interprocess is pinned: the transport behavior is +# what the tests were run against. +[dependencies] +# Calendar checks only (AGENTS.md: all date logic goes through chrono). +chrono.workspace = true +getrandom = "0.2.15" +interprocess = "=2.4.4" +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10.9" + +[target.'cfg(unix)'.dependencies] +# O_NONBLOCK and O_NOFOLLOW for opens that must not wait on a FIFO or follow a symlink. Constants only. +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +widestring = "1" + +[dev-dependencies] +# Default features off: the tests read local schemas and need no network resolver. +jsonschema = { version = "0.49.9", default-features = false } + +[lints] +workspace = true diff --git a/crates/ipc/PROTOCOL.md b/crates/ipc/PROTOCOL.md new file mode 100644 index 000000000..27945f021 --- /dev/null +++ b/crates/ipc/PROTOCOL.md @@ -0,0 +1,297 @@ +# Craft Link protocol, version 1 + +Extends the existing Craft control channel (the control protocol document of each app). Every +existing method keeps its exact behaviour. New methods are additive. `craft-ipc` implements this +protocol. + +## 1. Transport and framing + +- **Endpoint**: from the instance record (`schemas/instance-record.schema.json`). Unix domain socket file + (mode 0600, directory 0700) on Linux and macOS; named pipe with an owner-only SDDL descriptor on + Windows; loopback TCP only when the user opts in (`--control `, legacy). +- **Control framing**: one UTF-8 JSON object per line, max 4 MiB per line. A request has `method` + (string), optional `id` (number or string), optional `params` (object). A reply has the request's + `id`, `ok` (bool), and `result` or `error`. Malformed input gets one error line and the connection is + closed (existing behaviour). +- **Push events** (new): lines with `event` and `seq` and no `id`: + `{"event": "", "seq": , "data": {...}}`. Sent only on connections that called + `events.subscribe`. Per-subscriber queue of 256; on overflow the server sends + `{"event":"events.overflow","seq":n,"data":{"since":}}` and drops events until the client + catches up. +- **Bulk connection** (new): a second connection to the same endpoint, used only for frames (section 6). +- **Connection limit**: 16 concurrent connections per app (existing). Bulk connections count. At most + 4 of the 16 may be unauthenticated at one time; the other 12 slots go only to peers that sent the + right token. A peer past either limit gets one line `{"ok":false,"error":"too many control + connections"}` or `"too many unauthenticated connections"` and a close. +- **Time limits** (server defaults): a connection must send a valid `auth` line within 2 s of the + accept; a request line that has started must end within 10 s; an authenticated control + connection with no input for 1 h, no event subscription and no bulk role is closed with + `{"id":null,"ok":false,"error":"idle connection closed"}`. A client that keeps a quiet connection + longer must call `events.subscribe` or reconnect. A request may arrive in any number of pieces + with pauses between them; the server keeps the bytes of an unfinished line. +- **Launch on demand** (library rule, not wire): `craft_ipc::launch` runs an app binary only if the + path is absolute and the file is regular and executable, and the file and its directory belong to + the user or root and are not writable by every user. Empty and relative `PATH` entries are + skipped. A fork that starts an app by another route must call `craft_ipc::trust::check_executable`. + Caches use `craft_ipc::cache::private_cache_dir` (`0700`, owner checked, no symlink, no + fallback to a shared temp directory); `CRAFT_LINK_CACHE_DIR` must be absolute. + +## 2. Authentication (required on every transport: sockets, pipes, loopback TCP) + +The first line on every connection must be: + +```json +{"id": 0, "method": "auth", "params": {"token": "<64 hex from the instance record>"}} +``` + +Reply `{"id":0,"ok":true,"result":{"authenticated":true}}`, or +`{"id":0,"ok":false,"error":"authentication required"}` followed by close. The server also checks +the peer's user id (Linux/macOS euid; Windows token user) and refuses other users. Any other first +line gets the same refusal. (Shape copied from PhotoCraft's existing handshake.) + +Loopback TCP (`--control `, legacy) runs the same handshake, the same unauthenticated cap and +the same 2 s auth deadline. It binds 127.0.0.1 only. There is no uid check on TCP, so the token is +the only proof. The instance record (0600) publishes the port as `tcp_port` next to the `token`; +a TCP client reads both from the record (`Client::connect_tcp`). Never print the token or put it +in a config file or on a command line. A TCP client without a token gets the auth refusal. + +## 3. Session methods (answered on the connection thread, no UI frame needed) + +### `link.hello` +```json +{"id":1,"method":"link.hello","params":{"client":{"app":"filmcraft","app_version":"0.9.1","pid":4242,"instance":""},"protocol":{"min":1,"max":1}}} +-> {"id":1,"ok":true,"result":{"app":"effectcraft","app_version":"0.8.3","pid":5151,"instance":"","protocol":1,"caps":["events","frames","activate","link.receive"]}} +``` +Errors: `protocol_unsupported{server_min, server_max}`. The client must check `pid` and `instance` +equal the instance record. Legacy clients that never call `link.hello` keep today's behaviour. + +`caps` is the fixed base set (`events`, `activate`, `link.receive`, `frames`) plus the `caps` array of +the instance record (`schemas/instance-record.schema.json`), without duplicates. Roles and optional verbs use +this array. Defined names: + +| Cap | Meaning | +|---|---| +| `hub` | The peer is a browser or launcher hub. It owns no document, it only reads and opens assets (section 9). | +| `assets` | The peer answers `link.resolve` and `asset.describe` itself, on the connection thread (section 4). | + +A client that does not know a cap ignores it. + +### `events.subscribe` / `events.unsubscribe` +```json +{"id":2,"method":"events.subscribe","params":{"events":["doc.changed","link.changed","app.closing"]}} +-> {"id":2,"ok":true,"result":{"seq":0}} +``` + +### `app.activate` +```json +{"id":3,"method":"app.activate","params":{"token":"","from_pid":4242,"open":{"link_id":""}}} +-> {"id":3,"ok":true,"result":{"activated":"requested"}} +``` +`activated` has three values: + +| Value | Meaning | +|---|---| +| `true` | The window has focus now. | +| `"requested"` | The app asked the OS to focus the window. The OS decides later. Nobody confirmed it, and the reply does not say it will happen. | +| `false` | Focus was not taken. The app asked for user attention instead (taskbar flash, dock bounce). | + +On Linux, `"requested"` needs a token that the app could apply to its existing window. An app whose +toolkit can mint a token but cannot apply one (stock winit 0.30 on Wayland) answers `false`. The +sender can still pass `token`. Old senders and receivers keep working: the three values did not change. +`activated` is distinct from the existing `ui.focus` method, which never steals focus by design. + +`open` is an object that names what to show. It carries `link_id` (open a link), `asset` (a moniker, +section 9: open that asset in its document) or both. A hub or launcher uses `asset` to send the user +to the app and document that owns the asset, with no link in between. The server rejects an `asset` +that is not a valid moniker with `bad_params`. An `open` with neither key is passed on unchanged +(existing behaviour). The receiving app decides what "show" means for the asset. + +### `link.bye` +Clean close. The server drops subscriptions and bulk tickets of this connection. + +## 4. Link methods (run on the UI thread through the app's existing request channel) + +### `link.open` (FilmCraft -> EffectCraft) +```json +{"id":10,"method":"link.open","params":{ + "link_id":"", + "payload":{"format":"otio-json","data":""}, + "place":{"seconds":12.48,"selection":[{"start_s":10.0,"end_s":14.0,"track_index":0}],"view":{"zoom":1.0}}, + "result_path":"/abs/path/Project Links/.mov", + "source":{"app":"filmcraft","doc_id":"","item":""}}} +-> {"id":10,"ok":true,"result":{"comp":"","project":"/abs/path/Project Links/.ecproj", + "format":{"w":1920,"h":1080,"rate_num":24,"rate_den":1,"duration_ticks":...,"alpha":true}, + "unsupported":[{"what":"effect","name":"Lumetri Color","kept_as":"none"}]}} +``` +Errors: `payload_invalid`, `media_missing`, `payload_too_large` (OTIO string > 4 MiB), `io_error`. +A request line longer than 4 MiB (`MAX_LINE`) is not an error reply: the server answers once in text +and closes the connection. `unsupported` lists parts that +did not survive the hand-off. + +**Payload formats.** `payload.format` selects the shape of the rest of `payload`: + +| `format` | Other fields | Use | +|---|---|---| +| `otio-json` | `data`: OTIO JSON string | Video: a nested sequence (above). | +| `image-doc` | `asset`: moniker (section 9), `path`: absolute path of the producer document or image file | Stills, vectors and other single images. `result_path` is the last-known flattened image. `place`, `source` and `format` in the reply stay as above. `rate_num`, `rate_den` and `duration_ticks` are absent from `format` in the reply. | + +An unknown `format` gets `payload_invalid`. A new `format` value is additive and needs no protocol +bump. The server checks the shape of `payload` with `craft_ipc::payload::OpenPayload` before it +forwards `link.open` to the app. A missing `payload`, a `payload` that is not an object, a missing or +unknown `format`, a missing field, a relative `path` or a bad `asset` moniker gets `payload_invalid` +and the app never sees the request. `params` that is not an object stays `bad_params`. The server +checks shape, and for `otio-json` it also opens every `ExternalReference` whose `target_url` is a +`file://` URL or an absolute path. A file that does not open gets `media_missing` with +`data.missing` (the paths) and the app never sees the request. A `file://` URL whose path has a bad +percent escape, or does not decode to UTF-8, gets `payload_invalid`. Relative targets are left to the app. + +Path rules. `result_path`, `payload.path` and every absolute media `target_url` must +be absolute, hold no NUL byte and no `..` component, or the request gets `payload_invalid`. The +server does not canonicalise and does not confine paths to a root: footage lives anywhere and +may be a symlink. The media check is the only file probe. It looks only at paths the payload +names, at most 4096 distinct ones, and reports a path as missing when it is not a regular file +(after symlinks) or does not open for reading. A missing file, a directory, a FIFO, a device and a +file without read access all look the same, and the probe never opens a FIFO. An OTIO document +with more than 4096 distinct files, more than 2^20 JSON values or nesting over 128 gets +`payload_invalid`. The app must apply its own checks to `result_path` before it writes there; +the server check is a first filter. +An app still answers `payload_invalid` when the content is bad (for example OTIO that does not +parse). + +### `link.attach` (FilmCraft -> EffectCraft, reopen an existing link) +`{"link_id","project","comp"}` -> `{"format", "epoch"}`. Errors: `comp_missing` (link becomes `Broken`). +The receiver makes the comp of the link the active one. It reads `project` (the sidecar written by +`link.open`) when it holds no such link and has no unsaved edits, otherwise `busy`. +FilmCraft sends this when `link.editIn` runs on a clip that plays a linked item already. It nests +nothing and creates no item. On `comp_missing` or `io_error` (no sidecar, damaged sidecar) it sends +`link.open` again with the same `link_id` and `result_path`, built from the nest in `nested_sequence`. + +### `link.focus` (either direction, used by Back) +`{"link_id","place"}` -> `{}`. The receiver moves to `place` and is then activated with `app.activate`. + +### `link.close` +`{"link_id"}` -> `{}`. EffectCraft drops the binding; the sidecar project stays on disk. + +### `link.refresh` (consumer side, any app that holds a link) +`{"link_id"}` -> `{"link_id","epoch","state":"current"|"modified"|"missing","result_path","hash"}`. +The receiver compares the last-known result file with the disk (rules in section 10), bumps the epoch +when the content changed, and answers the state. `hash` is `sha256:<64 hex>` and is absent when the +state is `missing`. A link that exists in the receiver but whose file is gone is not an error. The +answer is `state:"missing"`. Errors: `link_unknown`, `bad_params`, `io_error`. + +### `link.resolve` (producer or hub, any app that owns assets) +`{"asset":"craft://..."}` -> `{"asset","path","epoch","size","hash","mtime_ms"}`. `path` is an +absolute path of the last-known flattened result that a consumer can read now. `hash` is +`sha256:<64 hex>` of that file. `epoch` is the producer epoch. Errors: `link_unknown` (no such asset, +or the asset has no result file yet), `bad_params` (not a moniker), `io_error`. + +### `asset.describe` (producer or hub) +`{"asset":"craft://..."}` -> `{"asset","name","kind","epoch","w","h","hash","thumb_path","modified_ms"}`. +`kind` is `still`, `vector`, `sequence` or `comp`. `w`, `h`, `hash` and `thumb_path` are `null` when +unknown. `thumb_path` is an absolute path of a PNG, at most 512 px on the long side, that the +producer keeps in a per-document sidecar. A hub browses assets with this call and never opens the +document. Errors as `link.resolve`. + +`link.resolve` and `asset.describe` have no side effects and never need a UI frame. A server that was +started with an asset provider (`Server::start_with` and `Extensions`) answers both on the connection thread +and lists `assets` in `caps`. `link.refresh` is answered by the provider the same way. A server +started without a provider forwards all three to the app like any other link method (existing +behaviour). + +### Events +- `link.changed`: `{"link_id","epoch"}`. Sent by EffectCraft when the bound comp changes (any + `bump()` whose change touches the comp). +- `link.result`: `{"link_id","epoch","path"}`. A fresh last-known result was written. +- `doc.changed`: `{"doc_id","revision","cause"}`. Generic change feed (from `craft-command`). +- `app.closing`: `{}`. Sent before a clean exit. + +## 5. Command passthrough (existing, unchanged) + +`engine.execute {command, params}`, `engine.commands`, `ui.*`, `perf.stats` keep their current +contracts. `engine.commands` now also lists plugin commands and carries a `schema` per command +(`x-craft-source`: `doc` | `derived` | `opaque`). + +## 6. Bulk frames + +1. Control connection: `frames.open {"link_id","format":"rgba8_premul"|"rgba16f_premul","scale":1.0}` + -> `{"ticket":"<32 hex>","w","h","stride","len"}`. +2. New connection: `auth`, then `{"method":"bulk.attach","params":{"ticket":"..."}}` -> `{"ok":true}`. + The connection is now binary in both directions. +3. Client request record (binary): `'CFRQ' | u32 header_len | header JSON` with header + `{"req":,"time_s":,"epoch":,"region":[x,y,w,h]|null}`. +4. Server reply record: `'CFRM' | u32 header_len | u64 payload_len | header JSON | payload`, header + `{"req","epoch","w","h","stride","format","render_ms"}`. `epoch` is the epoch the frame was + rendered at; a client that asked for an older epoch may get a newer one. `payload_len` is + `stride * h`, and `stride` is at least one row: `w` times 4 bytes for `rgba8_premul`, `w` times 8 + bytes for `rgba16f_premul`. A layout that breaks this is refused at `frames.open`, and a reply + that breaks it is refused by the server and by the client. +5. Errors: header `{"req","error":"..."}` with `payload_len` 0. +All integers little endian. Tickets expire when the control connection closes. + +## 7. Error codes (in `error` objects for new methods) + +`{"code": "", "message": "", "data": {...}}`. Codes: `auth_required`, +`protocol_unsupported`, `unknown_method`, `bad_params`, `not_ready`, `payload_invalid`, +`media_missing`, `payload_too_large`, `comp_missing`, `link_unknown`, `io_error`, `busy`, `internal`. +`link_unknown` also covers an unknown asset moniker. +Existing methods keep their current string `error` field. + +## 8. Versioning + +- `protocol` is an integer. A server serves any client version in `[min, max]`. New methods and new + optional fields do not bump the version. Removing or changing the meaning of a field does. +- Method names are never reused with a different meaning. + +## 9. Asset identity: the moniker + +An asset is addressed by a moniker, a string that stays valid across restarts: + +``` +craft:///// +``` + +| Part | Rule | +|---|---| +| `app` | Same pattern as the instance record `app`: `^[a-z][a-z0-9-]{1,31}$`. The app that owns the document. | +| `doc` | The document id, a UUID in 8-4-4-4-12 hex form. Parsers accept either hex case and keep the text as written. | +| `kind` | `^[a-z][a-z0-9-]{0,31}$`. Known values: `comp`, `image`, `vector`, `sequence`, `layer`. An unknown value is valid and is kept. | +| `id` | Non-empty, at most 512 bytes after decoding, valid UTF-8. Characters other than `A-Z a-z 0-9 - . _ ~` are percent-encoded as `%XX` (uppercase hex) by the formatter. The parser accepts either hex case. A raw `/`, `?`, `#` or `%` not followed by two hex digits is invalid. | + +The `asset.moniker` pattern in `schemas/link-record.schema.json` states the same `id` rules, so a +record that the schema accepts also parses in Rust. + +A moniker has no query and no fragment. It names an asset, not a state: the state is the epoch. +A moniker can outlive its asset. Resolving a dead moniker gives `link_unknown`. Parsing then +formatting a moniker gives the canonical text (uppercase `%XX`). Formatting then parsing gives back +an equal moniker. `craft_ipc::moniker::Moniker` implements both. The module uses only `serde` and +`std`, so an app can copy it to a wasm build. The `craft-ipc` crate as a whole still does not build +for wasm32 (local sockets). + +## 10. Refresh and watch rules for linked files + +These rules apply to any linked asset with a `result_path` (a still, a rendered movie). + +1. **Producer writes are atomic.** The producer writes the file to a temporary name in the same + directory (`....tmp`), flushes it, renames it over `result_path`, and then flushes + the directory. A reader sees the old file or the new file, never a half file. A reader ignores + names that start with `.` and end with `.tmp`. +2. **Epoch.** `result_epoch` is a counter of result versions. The producer adds 1 on every saved + change and emits `link.changed` and `link.result`. A consumer that detects a change by polling + (rule 4) adds 1 to its own stored epoch. When the producer is reachable again, the consumer keeps + the larger of the two values. An epoch never decreases. +3. **Fingerprint.** `(size, mtime_ms, sha256)` of the file. The hash is of the full content. +4. **Polling when the producer is offline.** The consumer calls `poll` about once per second + (`craft_ipc::watch::Watcher` runs this schedule with a thread and an injectable ticker). A poll + reads size and mtime. It hashes the file only when size or mtime differ from the last fingerprint. + Some file systems have coarse mtime, so the consumer also calls a deep poll (always hash) about + once every 30 seconds. A touch that leaves the hash equal does not bump the epoch. +5. **States.** `current` (disk equals what the consumer loaded), `modified` (disk differs from what + the consumer loaded; the epoch was bumped), `missing` (the file does not exist). The consumer + moves `modified` to `current` when it has loaded the new content. A file that returns with the + hash it had when loaded goes back to `current` with no epoch bump. A file that returns with a + different hash is `modified` and bumps the epoch. +6. **Other read errors** (permission denied, I/O error) are errors, not `missing`. The state does not + change and the next poll tries again. +7. **Relink.** A consumer can point a link at a new path. The new path is polled at once. The state + is `current` when its hash equals what the consumer loaded and `modified` (epoch bump) otherwise. + A missing new path is rejected and the old path stays. diff --git a/crates/ipc/README.md b/crates/ipc/README.md new file mode 100644 index 000000000..8bcdbea66 --- /dev/null +++ b/crates/ipc/README.md @@ -0,0 +1,38 @@ +# craft-ipc + +`craft-ipc` is how a Craft app talks to another Craft app on the same machine. It has a server, a client, instance discovery, launch on demand and focus hand-off. It has no async runtime. The wire protocol is in [PROTOCOL.md](PROTOCOL.md). The crate depends on no app and on no other Craft crate. + +**Native only.** `craft-ipc` does not build for `wasm32-unknown-unknown`, because `interprocess` (local sockets and named pipes) has no wasm32 build. An app gates the dependency by target. The pure data modules (`moniker`, `record`, `payload`, `asset`, `ids`) use only `std` and `serde`. + +## Boundaries + +- Never crashes: no panics, no `unsafe`. +- Every connection must present a 64 hex token within 2 seconds. The server also checks the peer user id (Unix) or the token user (Windows). At most 16 connections are open, and at most 4 of them are unauthenticated. +- One UTF-8 JSON object per line, at most 4 MiB per line. A request can arrive in pieces with pauses between them. `wire::LineReader` keeps the partial line. Do not read control lines with a stateless `read_line` on a stream that has a receive timeout. +- On Unix, the socket file is mode `0600` inside a `0700` directory. The Windows code paths (named pipe, token user check) have not been run on Windows. + +## Modules + +| Module | Role | +|---|---| +| `server`, `client`, `wire`, `bulk`, `auth`, `launch`, `activate`, `discovery` | transport, session, frames, focus hand-off, instance records | +| `moniker`, `record`, `payload`, `asset`, `ids` | pure data types | +| `refresh` | `Tracker` (poll, epoch bump, relink) and `write_atomic` for linked files | +| `fetch` | `FrameFetcher`: ticket, a few `BulkClient` workers, a RAM cache keyed `FrameKey` with a byte budget and LRU eviction. `fetch` never waits past its `wait` argument. A late frame still lands in the cache | +| `watch` | `Watcher` thread that polls `Tracker`s: shallow every second, deep every 30 seconds. `IntervalTicker` runs on real time and `ManualTicker` is for tests | +| `trust`, `cache` | `check_executable` guards every launch (absolute path, regular file, executable, owner and mode checks). `private_cache_dir`, `write_private` and `read_private` make a `0700` cache with no `/tmp` fallback | + +- `server::Limits` holds the connection limits: auth deadline 2 s, started line 10 s, idle 1 h for unsubscribed control connections, 4 unauthenticated of 16. +- `payload::check_peer_path` is the one path filter: absolute, no NUL, no `..`. The media probe only touches regular files and never opens a FIFO. +- `Server::start_with(config, Extensions { assets, caps })` adds an asset provider and extra caps. `InstanceRecord` keeps `caps` in `extra`. Use `caps()` and `add_cap()`. +- Discovery: each app writes one JSON record, mode `0600`, into a per-user runtime directory. A lock file held for the life of the app shows whether the owner is alive. +- Launch: `launch::resolve` finds an app binary (environment, a read-only parse of the launcher's install record, `PATH`, usual install locations). `launch` starts it only when `trust::check_executable` accepts the file. +- Loopback TCP is available only when the app opts in. It binds 127.0.0.1 and runs the same handshake. A TCP client reads the port and token from the instance record (`Client::connect_tcp`). + +## Schemas and fixtures + +`schemas/instance-record.schema.json` and `schemas/link-record.schema.json` describe the stored records. `tests/schemas.rs` checks that each schema and its Rust type agree, and that the stored records in `tests/fixtures/` stay valid. + +## Tests + +A forked child holds a copy of every open file descriptor until exec, so a `flock` released by `drop` can look held. Unit tests that start a process take `crate::SPAWN_GUARD.write()`. Unit tests that publish or scan a discovery lock take `.read()`. diff --git a/crates/ipc/schemas/instance-record.schema.json b/crates/ipc/schemas/instance-record.schema.json new file mode 100644 index 000000000..6cc9b4f15 --- /dev/null +++ b/crates/ipc/schemas/instance-record.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "craft://schemas/link/instance-record/1", + "title": "Craft Link instance record", + "description": "Written by each running Craft app to /craft/-.json (mode 0600). See PROTOCOL.md.", + "type": "object", + "required": ["v", "app", "app_version", "pid", "instance", "endpoint", "token", "started", "protocol", "docs"], + "additionalProperties": true, + "properties": { + "v": { "const": 1 }, + "app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$" }, + "app_version": { "type": "string" }, + "pid": { "type": "integer", "minimum": 1 }, + "instance": { "type": "string", "format": "uuid" }, + "endpoint": { + "oneOf": [ + { + "type": "object", + "required": ["kind", "path"], + "properties": { "kind": { "const": "unix" }, "path": { "type": "string", "maxLength": 100 } } + }, + { + "type": "object", + "required": ["kind", "name"], + "properties": { "kind": { "const": "pipe" }, "name": { "type": "string", "pattern": "^craft-" } } + }, + { + "type": "object", + "required": ["kind", "port"], + "properties": { "kind": { "const": "tcp" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 } } + } + ] + }, + "token": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "started": { "type": "integer", "description": "unix seconds" }, + "protocol": { + "type": "object", + "required": ["link"], + "properties": { + "link": { "type": "integer", "minimum": 1 }, + "commands_schema": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } + } + }, + "caps": { + "type": "array", + "description": "Roles and optional verbs this app offers. Merged into the link.hello caps (link-protocol.md section 3). 'hub' marks a hub peer, 'assets' marks an app that answers link.resolve and asset.describe on the connection thread. Absent means none.", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z][a-z0-9._-]{0,63}$" } + }, + "docs": { + "type": "array", + "items": { + "type": "object", + "required": ["doc_id", "title"], + "properties": { + "doc_id": { "type": "string" }, + "title": { "type": "string" }, + "path": { "type": "string" } + } + } + } + } +} diff --git a/crates/ipc/schemas/link-record.schema.json b/crates/ipc/schemas/link-record.schema.json new file mode 100644 index 000000000..0a1183085 --- /dev/null +++ b/crates/ipc/schemas/link-record.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "craft://schemas/link/link-record/1", + "title": "craft.link project item metadata", + "description": "Stored in the consumer's project item metadata[\"craft.link\"] on the linked media item. Unknown fields must be preserved on save. Records written before the asset fields existed (FilmCraft to EffectCraft) stay valid.", + "type": "object", + "required": ["v", "link_id", "target_app", "target_project", "result_path", "result_epoch", "format", "created"], + "anyOf": [ + { "required": ["target_comp"] }, + { "required": ["asset"] } + ], + "additionalProperties": true, + "properties": { + "v": { "const": 1 }, + "link_id": { "type": "string", "format": "uuid" }, + "target_app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$", "description": "app id of the producer, same pattern as the instance record app" }, + "target_project": { "type": "string", "description": "path of the producer document, relative to the consumer project file when possible" }, + "target_comp": { "type": "string", "description": "comp id inside the producer document. Required unless asset is present." }, + "nested_sequence": { "type": "string", "description": "FilmCraft item id of the nest created by link.editIn. Absent for stills." }, + "asset": { + "type": "object", + "required": ["kind", "moniker"], + "additionalProperties": true, + "properties": { + "kind": { "enum": ["still", "vector", "sequence", "comp"] }, + "moniker": { + "type": "string", + "description": "craft://app/doc/kind/id. The id is 1 to 512 bytes. Each byte is an unreserved character (A-Z a-z 0-9 . _ ~ -) or a percent escape of two hex digits, and the bytes the escapes decode to must be valid UTF-8. The first pattern checks the shape and the byte count (one character or one escape is one byte). The second checks the UTF-8.", + "allOf": [ + { "pattern": "^craft://[a-z][a-z0-9-]{1,31}/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/[a-z][a-z0-9-]{0,31}/(?:[A-Za-z0-9._~-]|%[0-9A-Fa-f]{2}){1,512}$" }, + { "pattern": "^craft://[^/]+/[^/]+/[^/]+/(?:[A-Za-z0-9._~-]|%[0-7][0-9A-Fa-f]|%(?:[Cc][2-9A-Fa-f]|[Dd][0-9A-Fa-f])%[89ABab][0-9A-Fa-f]|%[Ee]0%[ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ee](?:[1-9A-Ca-c]|[EeFf])%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ee][Dd]%[89][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff]0%(?:9|[ABab])[0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff][1-3]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff]4%8[0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f])+$" } + ] + } + } + }, + "producer": { + "type": "object", + "required": ["app"], + "additionalProperties": true, + "properties": { + "app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$" }, + "app_version": { "type": "string" }, + "doc_id": { "type": "string" } + } + }, + "result_path": { "type": "string" }, + "result_epoch": { "type": "integer", "minimum": 0 }, + "format": { + "type": "object", + "required": ["w", "h", "alpha"], + "dependentRequired": { "rate_num": ["rate_den"], "rate_den": ["rate_num"] }, + "properties": { + "w": { "type": "integer", "minimum": 1 }, + "h": { "type": "integer", "minimum": 1 }, + "rate_num": { "type": "integer", "minimum": 1 }, + "rate_den": { "type": "integer", "minimum": 1 }, + "duration_ticks": { "type": "integer", "minimum": 0 }, + "alpha": { "type": "boolean" } + } + }, + "created": { "type": "string", "format": "date-time" } + } +} diff --git a/crates/ipc/src/activate.rs b/crates/ipc/src/activate.rs new file mode 100644 index 000000000..476634cdc --- /dev/null +++ b/crates/ipc/src/activate.rs @@ -0,0 +1,362 @@ +//! Focus handoff between Craft apps. +//! +//! The platform calls live behind [`ActivationBackend`]. An app adapter +//! implements it with the real winit and egui calls, so this crate depends on +//! neither. The decision logic here is plain Rust and runs against a fake +//! backend in tests. +//! +//! | Platform | Sender | Receiver | +//! |---|---|---| +//! | Linux | mint a token from the focused window | apply the token if the backend can, then `Focus`; else ask for attention | +//! | Windows | none | `Focus` (winit's foreground path) | +//! | macOS | yield activation to the target bundle | `activate`, then `Focus` | +//! +//! The three answers mean: +//! +//! - [`Activated::Yes`]: the window has focus. +//! - [`Activated::Requested`]: the app asked the OS to focus the window. The OS +//! decides later. Nobody confirmed it. +//! - [`Activated::No`]: focus was not taken. The app asked for user attention +//! instead (taskbar flash, dock bounce). +//! +//! On Linux, a token only counts when the backend can apply it to an existing +//! window ([`ActivationBackend::can_apply_token`]). Stock winit 0.30 cannot on +//! Wayland: the app can mint a token but has no call to use one. In that case +//! the answer is [`Activated::No`], never `Requested`. + +use crate::server::Activation; + +/// The answer to `app.activate`. Same three values as the wire reply. +pub type Activated = Activation; + +/// What a focus request did, as far as the adapter can tell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FocusResult { + /// The window has focus now. + Focused, + /// The request was sent. The compositor or window manager decides later. + Pending, + /// The platform refused or the window is gone. + Refused, +} + +/// Platform calls an app adapter provides. +/// +/// Every method is called on the thread that owns the UI. The adapter maps +/// them to winit and egui: +/// +/// - `mint_token`: egui's `Frame::request_activation_token` on an earlier +/// frame, then `Frame::take_activation_token`. Return the token or `None`. +/// - `can_apply_token`: true only if the winit in use can activate an existing window with a token. +/// - `set_pending_token`: the winit call that stores the token for that window. +/// - `focus`: `ViewportCommand::Focus`. +/// - `request_user_attention`: `ViewportCommand::RequestUserAttention`. +/// - `yield_activation_to` and `activate_application`: the `objc2-app-kit` +/// `NSApplication` and `NSRunningApplication` calls, macOS only. +pub trait ActivationBackend { + /// A fresh activation token, or `None` when the platform cannot mint one. + fn mint_token(&self) -> Option; + /// True when `set_pending_token` followed by `focus` really hands the token + /// to the compositor for an existing window. The default is false: no + /// default success. An adapter returns true only when its winit has a call + /// that applies a token to a live window. + fn can_apply_token(&self) -> bool { + false + } + /// Store `token` so the next `focus` call applies it. Called only when + /// `can_apply_token` is true. + fn set_pending_token(&self, token: &str); + /// Ask the window to take focus. + fn focus(&self) -> FocusResult; + /// Flash the taskbar or dock entry. Used when focus is refused. + fn request_user_attention(&self); + /// macOS sender. Hand activation to the app with `bundle_id`. Returns false on failure. + fn yield_activation_to(&self, bundle_id: &str) -> bool; + /// macOS receiver. Activate this app. Returns false on failure. + fn activate_application(&self) -> bool; +} + +/// The platform whose rules apply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Platform { + Linux, + Windows, + MacOs, + /// Anything else. No handoff is possible. + Other, +} + +impl Platform { + pub const fn current() -> Self { + if cfg!(target_os = "linux") { + Self::Linux + } else if cfg!(target_os = "windows") { + Self::Windows + } else if cfg!(target_os = "macos") { + Self::MacOs + } else { + Self::Other + } + } +} + +/// Sender side. A token to put in `app.activate`, or `None`. +/// +/// Only Linux has tokens. Windows and macOS return `None`. +pub fn mint_token(backend: &dyn ActivationBackend) -> Option { + mint_token_on(Platform::current(), backend) +} + +pub fn mint_token_on(platform: Platform, backend: &dyn ActivationBackend) -> Option { + match platform { + Platform::Linux => backend.mint_token().filter(|token| !token.is_empty()), + Platform::Windows | Platform::MacOs | Platform::Other => None, + } +} + +/// Sender side, before `app.activate`. macOS yields activation to `bundle_id`. +/// Other platforms do nothing and return true. A false return means the yield failed. +pub fn yield_to(backend: &dyn ActivationBackend, bundle_id: &str) -> bool { + yield_to_on(Platform::current(), backend, bundle_id) +} + +pub fn yield_to_on(platform: Platform, backend: &dyn ActivationBackend, bundle_id: &str) -> bool { + match platform { + Platform::MacOs => backend.yield_activation_to(bundle_id), + Platform::Linux | Platform::Windows | Platform::Other => true, + } +} + +/// Receiver side. Apply `app.activate`. `token` is the sender's token or `None`. +pub fn apply_activation(backend: &dyn ActivationBackend, token: Option<&str>) -> Activated { + apply_activation_on(Platform::current(), backend, token) +} + +pub fn apply_activation_on(platform: Platform, backend: &dyn ActivationBackend, token: Option<&str>) -> Activated { + match platform { + Platform::Linux => { + let token = token.filter(|text| !text.is_empty()); + let applied = match token { + Some(token) if backend.can_apply_token() => { + backend.set_pending_token(token); + true + } + _ => false, + }; + let result = backend.focus(); + // Pending means "the compositor may act". Without an applied token it will not. + let result = match (result, applied) { + (FocusResult::Pending, false) => FocusResult::Refused, + (other, _) => other, + }; + settle(backend, result) + } + Platform::Windows => settle(backend, backend.focus()), + Platform::MacOs => { + if !backend.activate_application() { + backend.request_user_attention(); + return Activated::No; + } + settle(backend, backend.focus()) + } + Platform::Other => { + backend.request_user_attention(); + Activated::No + } + } +} + +fn settle(backend: &dyn ActivationBackend, result: FocusResult) -> Activated { + match result { + FocusResult::Focused => Activated::Yes, + FocusResult::Pending => Activated::Requested, + FocusResult::Refused => { + backend.request_user_attention(); + Activated::No + } + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + + use super::*; + + #[derive(Default)] + struct Fake { + minted: Option, + focus: Option, + yield_ok: bool, + activate_ok: bool, + can_apply: bool, + calls: RefCell>, + } + + impl Fake { + fn calls(&self) -> Vec { + self.calls.borrow().clone() + } + fn log(&self, text: impl Into) { + self.calls.borrow_mut().push(text.into()); + } + } + + impl ActivationBackend for Fake { + fn mint_token(&self) -> Option { + self.log("mint"); + self.minted.clone() + } + fn can_apply_token(&self) -> bool { + self.can_apply + } + fn set_pending_token(&self, token: &str) { + self.log(format!("pending:{token}")); + } + fn focus(&self) -> FocusResult { + self.log("focus"); + self.focus.unwrap_or(FocusResult::Refused) + } + fn request_user_attention(&self) { + self.log("attention"); + } + fn yield_activation_to(&self, bundle_id: &str) -> bool { + self.log(format!("yield:{bundle_id}")); + self.yield_ok + } + fn activate_application(&self) -> bool { + self.log("activate"); + self.activate_ok + } + } + + #[test] + fn linux_sets_the_token_before_focus_and_reports_requested() { + let fake = Fake { focus: Some(FocusResult::Pending), can_apply: true, ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("tok")); + assert_eq!(got, Activated::Requested); + assert_eq!(fake.calls(), ["pending:tok", "focus"]); + } + + #[test] + fn linux_token_the_backend_cannot_apply_is_not_a_request() { + // Stock winit on Wayland: a token can be minted but not applied to an existing window. + let fake = Fake { focus: Some(FocusResult::Pending), can_apply: false, ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("tok")); + assert_eq!(got, Activated::No); + assert_eq!(fake.calls(), ["focus", "attention"]); + } + + #[test] + fn linux_token_the_backend_cannot_apply_still_reports_real_focus() { + let fake = Fake { focus: Some(FocusResult::Focused), can_apply: false, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Linux, &fake, Some("tok")), Activated::Yes); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn can_apply_token_defaults_to_false() { + struct Bare; + impl ActivationBackend for Bare { + fn mint_token(&self) -> Option { + None + } + fn set_pending_token(&self, _: &str) {} + fn focus(&self) -> FocusResult { + FocusResult::Pending + } + fn request_user_attention(&self) {} + fn yield_activation_to(&self, _: &str) -> bool { + true + } + fn activate_application(&self) -> bool { + true + } + } + assert!(!Bare.can_apply_token()); + } + + #[test] + fn linux_without_a_token_cannot_claim_requested() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, None); + assert_eq!(got, Activated::No); + assert_eq!(fake.calls(), ["focus", "attention"]); + } + + #[test] + fn linux_empty_token_counts_as_none() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("")); + assert_eq!(got, Activated::No); + assert!(!fake.calls().iter().any(|call| call.starts_with("pending"))); + } + + #[test] + fn linux_focused_window_is_yes_without_attention() { + let fake = Fake { focus: Some(FocusResult::Focused), ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Linux, &fake, None), Activated::Yes); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn refusal_requests_attention_on_every_platform() { + for platform in [Platform::Linux, Platform::Windows, Platform::MacOs] { + let fake = Fake { focus: Some(FocusResult::Refused), activate_ok: true, ..Fake::default() }; + assert_eq!(apply_activation_on(platform, &fake, Some("t")), Activated::No, "{platform:?}"); + assert_eq!(fake.calls().last().map(String::as_str), Some("attention"), "{platform:?}"); + } + } + + #[test] + fn windows_ignores_the_token() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Windows, &fake, Some("t")), Activated::Requested); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn macos_activates_then_focuses() { + let fake = Fake { focus: Some(FocusResult::Focused), activate_ok: true, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::MacOs, &fake, None), Activated::Yes); + assert_eq!(fake.calls(), ["activate", "focus"]); + } + + #[test] + fn macos_activation_failure_is_no_with_attention() { + let fake = Fake { focus: Some(FocusResult::Focused), activate_ok: false, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::MacOs, &fake, None), Activated::No); + assert_eq!(fake.calls(), ["activate", "attention"]); + } + + #[test] + fn other_platforms_only_ask_for_attention() { + let fake = Fake::default(); + assert_eq!(apply_activation_on(Platform::Other, &fake, Some("t")), Activated::No); + assert_eq!(fake.calls(), ["attention"]); + } + + #[test] + fn only_linux_mints_tokens() { + let fake = Fake { minted: Some("abc".into()), ..Fake::default() }; + assert_eq!(mint_token_on(Platform::Linux, &fake).as_deref(), Some("abc")); + assert_eq!(mint_token_on(Platform::Windows, &fake), None); + assert_eq!(mint_token_on(Platform::MacOs, &fake), None); + let empty = Fake { minted: Some(String::new()), ..Fake::default() }; + assert_eq!(mint_token_on(Platform::Linux, &empty), None); + } + + #[test] + fn only_macos_yields() { + let fake = Fake { yield_ok: false, ..Fake::default() }; + assert!(yield_to_on(Platform::Linux, &fake, "com.x")); + assert!(fake.calls().is_empty()); + assert!(!yield_to_on(Platform::MacOs, &fake, "com.x")); + assert_eq!(fake.calls(), ["yield:com.x"]); + } + + #[test] + fn current_platform_matches_the_build() { + assert_eq!(Platform::current() == Platform::Linux, cfg!(target_os = "linux")); + } +} diff --git a/crates/ipc/src/asset.rs b/crates/ipc/src/asset.rs new file mode 100644 index 000000000..9ffd6f429 --- /dev/null +++ b/crates/ipc/src/asset.rs @@ -0,0 +1,70 @@ +//! Reply bodies for `link.resolve`, `asset.describe` and `link.refresh` +//! (`PROTOCOL.md` section 4). +//! +//! `std`, `serde` and the sibling pure modules only. + +use serde::Serialize; + +use crate::moniker::Moniker; +use crate::record::AssetKind; +use crate::refresh::{FileState, Fingerprint}; + +/// Answer to `link.resolve`: where the last-known result is and which version it is. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Resolved { + pub asset: Moniker, + pub path: String, + pub epoch: u64, + pub size: u64, + pub hash: String, + pub mtime_ms: u64, +} + +impl Resolved { + /// Build the answer from a file fingerprint. + pub fn from_fingerprint(asset: Moniker, path: &str, epoch: u64, fp: &Fingerprint) -> Self { + Self { asset, path: path.to_string(), epoch, size: fp.size, hash: fp.hash.clone(), mtime_ms: fp.mtime_ms } + } +} + +/// Answer to `asset.describe`: enough for a hub to list an asset without opening its document. +/// Unknown values serialize as `null`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Described { + pub asset: Moniker, + pub name: String, + pub kind: AssetKind, + pub epoch: u64, + pub w: Option, + pub h: Option, + pub hash: Option, + pub thumb_path: Option, + pub modified_ms: Option, +} + +/// Answer to `link.refresh`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Refreshed { + pub link_id: String, + pub epoch: u64, + pub state: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub result_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hash: Option, +} + +impl Refreshed { + /// Build the answer from a tracker state. `hash` and `result_path` are + /// dropped when the file is missing. + pub fn new(link_id: &str, epoch: u64, state: FileState, result_path: &str, seen: Option<&Fingerprint>) -> Self { + let present = state != FileState::Missing; + Self { + link_id: link_id.to_string(), + epoch, + state: state.as_str(), + result_path: present.then(|| result_path.to_string()), + hash: seen.filter(|_| present).map(|fp| fp.hash.clone()), + } + } +} diff --git a/crates/ipc/src/auth.rs b/crates/ipc/src/auth.rs new file mode 100644 index 000000000..851ec7d8d --- /dev/null +++ b/crates/ipc/src/auth.rs @@ -0,0 +1,107 @@ +//! Session tokens and the peer user check. +//! +//! On Unix the peer check is the effective uid from `interprocess`. A missing +//! credential is a refusal. On Windows, `interprocess` 2.4.4 `PeerCreds` only +//! exposes a pid. The pipe SDDL is the user check there. A pid is not a user. + +use interprocess::local_socket::Stream; +#[cfg(unix)] +use interprocess::local_socket::traits::StreamCommon; + +#[cfg(unix)] +use crate::discovery::current_euid; +use crate::error::LinkError; + +const HEX: &[u8; 16] = b"0123456789abcdef"; + +pub fn encode_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} + +pub fn is_lower_hex(text: &str) -> bool { + text.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// 32 random bytes as 64 lowercase hex characters. +pub fn generate_token() -> Result { + let mut bytes = [0u8; 32]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + Ok(encode_hex(&bytes)) +} + +/// UUID version 4. The instance record stores this string. +pub fn random_uuid() -> Result { + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + let hex = encode_hex(&bytes); + Ok(format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32])) +} + +/// Compare two tokens without stopping at the first different byte. +/// Length is public (always 64). A different length is not equal. +pub fn tokens_equal(expected: &str, presented: &str) -> bool { + if expected.len() != 64 || presented.len() != 64 { + return false; + } + let mut diff = 0u8; + for (left, right) in expected.bytes().zip(presented.bytes()) { + diff |= left ^ right; + } + diff == 0 +} + +/// `Ok` when this Unix peer is the current effective user. +/// Missing credentials are `Err(Auth)`, not a pass. +/// +/// Windows returns `Ok`. The named-pipe SDDL admits only the owner and SYSTEM. +/// This function cannot see a Windows user id through interprocess 2.4.4. +pub fn ensure_peer_user(stream: &Stream) -> Result<(), LinkError> { + #[cfg(unix)] + { + let creds = stream.peer_creds().map_err(|_| LinkError::Auth)?; + let Some(peer) = creds.euid() else { + return Err(LinkError::Auth); + }; + let me = current_euid().map_err(|_| LinkError::Auth)?; + if peer == me { Ok(()) } else { Err(LinkError::Auth) } + } + #[cfg(not(unix))] + { + let _ = stream; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_is_64_lower_hex_and_compare_is_exact() { + let token = generate_token().unwrap(); + assert_eq!(token.len(), 64); + assert!(is_lower_hex(&token)); + assert!(tokens_equal(&token, &token)); + let mut other = token.clone(); + other.replace_range(0..1, if token.starts_with('a') { "b" } else { "a" }); + assert!(!tokens_equal(&token, &other)); + assert!(!tokens_equal(&token, "abcd")); + assert!(!tokens_equal(&token, &token.to_uppercase())); + } + + #[test] + fn uuid_v4_shape() { + let id = random_uuid().unwrap(); + assert_eq!(id.len(), 36); + assert_eq!(id.as_bytes()[14], b'4'); + let variant = id.as_bytes()[19]; + assert!(matches!(variant, b'8' | b'9' | b'a' | b'b')); + } +} diff --git a/crates/ipc/src/bulk.rs b/crates/ipc/src/bulk.rs new file mode 100644 index 000000000..488288778 --- /dev/null +++ b/crates/ipc/src/bulk.rs @@ -0,0 +1,487 @@ +//! Bulk frames (contract section 6). +//! +//! Server side. An app implements [`FrameSource`]. [`FrameTickets`] adapts it +//! to the server's [`Frames`] trait: it mints the 32 hex tickets, checks every +//! `CFRQ` header, and checks every reply before it reaches the wire. The server +//! drops a ticket when its control connection closes, which calls +//! [`Frames::close_ticket`] and ends the source's session for that ticket. +//! +//! Client side. [`Client::frames_open`] asks for a ticket. [`BulkClient`] opens +//! the second connection, attaches the ticket, and requests frames. + +use std::collections::HashMap; +use std::io::BufReader; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::{Map, Value, json}; + +use crate::InstanceRecord as Record; +use crate::auth::{encode_hex, is_lower_hex}; +use crate::client::{Client, Transport, map_reply, open_reader, read_json, write_json}; +use crate::error::LinkError; +use crate::server::{FrameOpen, Frames}; +use crate::wire; + +/// Largest `scale` a client may ask for in `frames.open`. +pub const SCALE_MAX: f64 = 16.0; + +/// Pixel format of a frame. Both are premultiplied alpha. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameFormat { + Rgba8Premul, + Rgba16fPremul, +} + +impl FrameFormat { + pub const fn as_str(self) -> &'static str { + match self { + Self::Rgba8Premul => "rgba8_premul", + Self::Rgba16fPremul => "rgba16f_premul", + } + } + + /// Bytes one pixel takes in a row. + pub(crate) const fn bytes_per_pixel(self) -> u32 { + match self { + Self::Rgba8Premul => 4, + Self::Rgba16fPremul => 8, + } + } + + pub fn parse(text: &str) -> Result { + match text { + "rgba8_premul" => Ok(Self::Rgba8Premul), + "rgba16f_premul" => Ok(Self::Rgba16fPremul), + other => Err(LinkError::BadParams(format!("unknown frame format {other}"))), + } + } +} + +/// What a client asked for in `frames.open`. +#[derive(Debug, Clone, PartialEq)] +pub struct FrameSpec { + pub link_id: String, + pub format: FrameFormat, + pub scale: f64, +} + +impl FrameSpec { + pub fn from_params(params: &Map) -> Result { + let link_id = params + .get("link_id") + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) + .ok_or_else(|| LinkError::BadParams("link_id must be a non-empty string".into()))? + .to_string(); + let format = + FrameFormat::parse(params.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::BadParams("format is required".into()))?)?; + let scale = match params.get("scale") { + None => 1.0, + Some(value) => value.as_f64().ok_or_else(|| LinkError::BadParams("scale must be a number".into()))?, + }; + if !scale.is_finite() || scale <= 0.0 || scale > SCALE_MAX { + return Err(LinkError::BadParams(format!("scale must be above 0 and at most {SCALE_MAX}"))); + } + Ok(Self { link_id, format, scale }) + } +} + +/// Size of the frames a ticket yields. `len` is `stride * h`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameLayout { + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +/// A row of `w` pixels in `format` must fit in `stride` bytes, or a reader that splits the payload +/// into rows of `stride` bytes cannot decode the frame. +fn check_row(w: u32, stride: u32, format: FrameFormat) -> Result<(), LinkError> { + if u64::from(stride) < u64::from(w) * u64::from(format.bytes_per_pixel()) { + return Err(LinkError::Invalid("frame layout: stride is smaller than one row of w pixels".into())); + } + Ok(()) +} + +impl FrameLayout { + fn check(&self) -> Result<(), LinkError> { + if u64::from(self.stride) * u64::from(self.h) != self.len { + return Err(LinkError::Invalid("frame layout: len is not stride * h".into())); + } + if self.len > wire::FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + Ok(()) + } +} + +/// One `CFRQ` header. +#[derive(Debug, Clone, PartialEq)] +pub struct FrameRequest { + pub req: u64, + pub time_s: f64, + /// The epoch the client last saw. The source may answer with a newer one. + pub epoch: u64, + /// `[x, y, w, h]` in pixels, or `None` for the whole frame. + pub region: Option<[u32; 4]>, +} + +impl FrameRequest { + pub fn from_header(header: &Value) -> Result { + let req = header.get("req").and_then(Value::as_u64).ok_or_else(|| LinkError::BadParams("req must be an unsigned integer".into()))?; + let time_s = header + .get("time_s") + .and_then(Value::as_f64) + .filter(|t| t.is_finite()) + .ok_or_else(|| LinkError::BadParams("time_s must be a finite number".into()))?; + let epoch = header.get("epoch").and_then(Value::as_u64).ok_or_else(|| LinkError::BadParams("epoch must be an unsigned integer".into()))?; + let region = match header.get("region") { + None | Some(Value::Null) => None, + Some(Value::Array(items)) if items.len() == 4 => { + let mut out = [0u32; 4]; + for (slot, item) in out.iter_mut().zip(items) { + let number = item.as_u64().ok_or_else(|| LinkError::BadParams("region entries must be unsigned integers".into()))?; + *slot = u32::try_from(number).map_err(|_| LinkError::BadParams("region entry is out of range".into()))?; + } + Some(out) + } + Some(_) => return Err(LinkError::BadParams("region must be null or [x, y, w, h]".into())), + }; + Ok(Self { req, time_s, epoch, region }) + } + + pub fn to_header(&self) -> Value { + json!({"req": self.req, "time_s": self.time_s, "epoch": self.epoch, "region": self.region}) + } +} + +/// A rendered frame from the source. +#[derive(Debug, Clone, PartialEq)] +pub struct RenderedFrame { + /// The epoch this frame was rendered at. + pub epoch: u64, + pub w: u32, + pub h: u32, + pub stride: u32, + pub format: FrameFormat, + pub render_ms: f64, + pub pixels: Vec, +} + +/// The app side of bulk frames. Calls may come from several connection threads. +pub trait FrameSource: Send + Sync { + /// Validate `spec` and report the frame layout. Errors become `frames.open` errors. + fn open(&self, spec: &FrameSpec) -> Result; + /// Render one frame. The returned epoch must be at least `request.epoch`. + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result; + /// The ticket for `spec` is gone. Release anything held for it. + fn close(&self, spec: &FrameSpec); +} + +/// [`Frames`] over a [`FrameSource`]. Pass it as `ServerConfig::frames`. +pub struct FrameTickets { + source: Arc, + open: Mutex>, +} + +impl FrameTickets { + pub fn new(source: Arc) -> Self { + Self { source, open: Mutex::new(HashMap::new()) } + } + + /// Count of tickets that are still valid. + pub fn open_tickets(&self) -> usize { + crate::server::lock(&self.open).len() + } + + fn spec(&self, ticket: &str) -> Result { + crate::server::lock(&self.open).get(ticket).cloned().ok_or_else(|| LinkError::BadParams("unknown or closed ticket".into())) + } +} + +fn new_ticket() -> Result { + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + Ok(encode_hex(&bytes)) +} + +impl Frames for FrameTickets { + fn open(&self, params: &Map) -> Result { + let spec = FrameSpec::from_params(params)?; + let layout = self.source.open(&spec)?; + layout.check()?; + check_row(layout.w, layout.stride, spec.format)?; + let ticket = new_ticket()?; + crate::server::lock(&self.open).insert(ticket.clone(), spec); + Ok(FrameOpen { ticket, w: layout.w, h: layout.h, stride: layout.stride, len: layout.len }) + } + + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError> { + let spec = self.spec(ticket)?; + let request = FrameRequest::from_header(header)?; + let frame = self.source.render(&spec, &request)?; + if frame.epoch < request.epoch { + return Err(LinkError::NotReady(format!("source is at epoch {}, request wants {}", frame.epoch, request.epoch))); + } + check_row(frame.w, frame.stride, frame.format)?; + let expect = u64::from(frame.stride) * u64::from(frame.h); + if u64::try_from(frame.pixels.len()).ok() != Some(expect) { + return Err(LinkError::Invalid("rendered payload is not stride * h bytes".into())); + } + let reply = json!({ + "req": request.req, + "epoch": frame.epoch, + "w": frame.w, + "h": frame.h, + "stride": frame.stride, + "format": frame.format.as_str(), + "render_ms": frame.render_ms, + }); + Ok((reply, frame.pixels)) + } + + fn close_ticket(&self, ticket: &str) { + let removed = crate::server::lock(&self.open).remove(ticket); + if let Some(spec) = removed { + self.source.close(&spec); + } + } +} + +/// The `frames.open` reply. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameTicket { + pub ticket: String, + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +impl Client { + /// `frames.open`. The ticket lives until this connection closes. + pub fn frames_open(&mut self, link_id: &str, format: FrameFormat, scale: f64) -> Result { + let result = self.call("frames.open", json!({"link_id": link_id, "format": format.as_str(), "scale": scale}))?; + let ticket = result + .get("ticket") + .and_then(Value::as_str) + .filter(|t| t.len() == 32 && is_lower_hex(t)) + .ok_or_else(|| LinkError::Invalid("frames.open ticket is not 32 hex characters".into()))?; + Ok(FrameTicket { + ticket: ticket.to_string(), + w: result_u32(&result, "w")?, + h: result_u32(&result, "h")?, + stride: result_u32(&result, "stride")?, + len: result.get("len").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("frames.open result has no len".into()))?, + }) + } +} + +fn result_u32(result: &Value, key: &str) -> Result { + result + .get(key) + .and_then(Value::as_u64) + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| LinkError::Invalid(format!("frames.open result has no {key}"))) +} + +/// One `CFRM` reply that carried pixels. +#[derive(Debug, Clone, PartialEq)] +pub struct Frame { + pub req: u64, + /// The epoch the frame was rendered at. It can be newer than the epoch asked for. + pub epoch: u64, + pub w: u32, + pub h: u32, + pub stride: u32, + pub format: FrameFormat, + pub render_ms: f64, + pub pixels: Vec, +} + +/// The second connection of contract section 6. It is binary after the attach. +pub struct BulkClient { + reader: BufReader, + next_req: u64, + io_timeout: Duration, +} + +impl BulkClient { + /// Connect, authenticate, and `bulk.attach` with `ticket`. + pub fn attach(record: &Record, ticket: &str) -> Result { + let mut reader = open_reader(record)?; + write_json(&mut reader, &json!({"id": 1, "method": "bulk.attach", "params": {"ticket": ticket}}))?; + let deadline = std::time::Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + let reply = read_json(&mut reader, &mut Vec::new(), deadline)?.ok_or(LinkError::Closed)?; + map_reply(&reply)?; + Ok(Self { reader, next_req: 1, io_timeout: Duration::from_secs(wire::REPLY_DEADLINE_SECS) }) + } + + /// How long one request may wait for the socket to take the request or give the reply. + /// The default is [`wire::REPLY_DEADLINE_SECS`]. A request that runs out of time is + /// `Err(Timeout)`, and the connection must then be dropped, because a reply that arrives + /// late cannot be told apart from the next one. + pub fn set_io_timeout(&mut self, timeout: Duration) { + self.io_timeout = timeout; + } + + /// Ask for one frame and wait for its reply. + /// + /// A reply with an `error` header is `Err(Invalid(message))`. The + /// connection stays usable after it. A short or malformed record is an + /// error and the connection must be dropped. [`BulkClient::request_reply`] + /// tells the two apart. + pub fn request(&mut self, time_s: f64, epoch: u64, region: Option<[u32; 4]>) -> Result { + self.request_reply(time_s, epoch, region)?.map_err(LinkError::Invalid) + } + + /// Like [`BulkClient::request`], with the two failures apart. The outer `Err` is the + /// connection: drop it. The inner `Err` is the text of an `error` header from the server: + /// the connection is still good. + pub fn request_reply(&mut self, time_s: f64, epoch: u64, region: Option<[u32; 4]>) -> Result, LinkError> { + if !time_s.is_finite() { + return Err(LinkError::BadParams("time_s must be finite".into())); + } + let req = self.next_req; + self.next_req = req.checked_add(1).ok_or_else(|| LinkError::Invalid("request id overflow".into()))?; + let header = FrameRequest { req, time_s, epoch, region }.to_header(); + let transport = self.reader.get_ref(); + transport.arm_send_timeout(Some(self.io_timeout))?; + transport.arm_recv_timeout(Some(self.io_timeout))?; + wire::write_cfrq(self.reader.get_mut(), &header)?; + let (reply, pixels) = wire::read_cfrm(&mut self.reader)?; + if reply.get("req").and_then(Value::as_u64) != Some(req) { + return Err(LinkError::Invalid("reply req does not match the request".into())); + } + if let Some(message) = reply.get("error") { + return Ok(Err(message.as_str().unwrap_or("frame error").to_string())); + } + parse_reply(req, &reply, pixels).map(Ok) + } +} + +fn parse_reply(req: u64, reply: &Value, pixels: Vec) -> Result { + let number = |key: &str| reply.get(key).and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid(format!("frame header has no {key}"))); + let small = |key: &str| number(key).and_then(|n| u32::try_from(n).map_err(|_| LinkError::Invalid(format!("frame header {key} is out of range")))); + let (w, h, stride) = (small("w")?, small("h")?, small("stride")?); + let format = + FrameFormat::parse(reply.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::Invalid("frame header has no format".into()))?)?; + check_row(w, stride, format)?; + if u64::try_from(pixels.len()).ok() != Some(u64::from(stride) * u64::from(h)) { + return Err(LinkError::Invalid("payload is not stride * h bytes".into())); + } + let render_ms = reply.get("render_ms").and_then(Value::as_f64).ok_or_else(|| LinkError::Invalid("frame header has no render_ms".into()))?; + Ok(Frame { req, epoch: number("epoch")?, w, h, stride, format, render_ms, pixels }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_header_round_trips() { + let request = FrameRequest { req: 9, time_s: 2.5, epoch: 4, region: Some([1, 2, 3, 4]) }; + assert_eq!(FrameRequest::from_header(&request.to_header()).unwrap(), request); + let whole = FrameRequest { req: 1, time_s: 0.0, epoch: 0, region: None }; + assert_eq!(FrameRequest::from_header(&whole.to_header()).unwrap(), whole); + } + + #[test] + fn bad_request_headers_are_rejected() { + for header in [ + json!({"time_s": 1.0, "epoch": 0}), + json!({"req": 1, "epoch": 0}), + json!({"req": 1, "time_s": 1.0}), + json!({"req": 1, "time_s": 1.0, "epoch": 0, "region": [1, 2, 3]}), + json!({"req": 1, "time_s": 1.0, "epoch": 0, "region": [1, 2, 3, -4]}), + json!({"req": -1, "time_s": 1.0, "epoch": 0}), + ] { + assert!(FrameRequest::from_header(&header).is_err(), "{header}"); + } + } + + #[test] + fn spec_rejects_bad_scale_and_format() { + let ok = json!({"link_id": "a", "format": "rgba8_premul", "scale": 2.0}); + assert!(FrameSpec::from_params(ok.as_object().unwrap()).is_ok()); + for bad in [ + json!({"link_id": "a", "format": "rgba8_premul", "scale": 0.0}), + json!({"link_id": "a", "format": "rgba8_premul", "scale": 17.0}), + json!({"link_id": "a", "format": "bgra"}), + json!({"format": "rgba8_premul"}), + ] { + assert!(FrameSpec::from_params(bad.as_object().unwrap()).is_err(), "{bad}"); + } + } + + struct OneLayout { + w: u32, + stride: u32, + h: u32, + } + + impl FrameSource for OneLayout { + fn open(&self, _spec: &FrameSpec) -> Result { + Ok(FrameLayout { w: self.w, h: self.h, stride: self.stride, len: u64::from(self.stride) * u64::from(self.h) }) + } + + fn render(&self, spec: &FrameSpec, _request: &FrameRequest) -> Result { + let len = usize::try_from(u64::from(self.stride) * u64::from(self.h)).unwrap(); + Ok(RenderedFrame { epoch: 1, w: self.w, h: self.h, stride: self.stride, format: spec.format, render_ms: 0.0, pixels: vec![0; len] }) + } + + fn close(&self, _spec: &FrameSpec) {} + } + + fn open_params(format: &str) -> Map { + json!({"link_id": "a", "format": format}).as_object().unwrap().clone() + } + + #[test] + fn a_stride_that_cannot_hold_one_row_is_refused_before_a_ticket_exists() { + // (w, stride, format, accepted): one row is w * 4 bytes for rgba8 and w * 8 for rgba16f. + for (w, stride, format, accepted) in [ + (4, 16, "rgba8_premul", true), + (4, 17, "rgba8_premul", true), + (4, 15, "rgba8_premul", false), + (4, 1, "rgba8_premul", false), + (4, 32, "rgba16f_premul", true), + (4, 16, "rgba16f_premul", false), + (0, 0, "rgba8_premul", true), + (u32::MAX, u32::MAX, "rgba16f_premul", false), + ] { + let tickets = FrameTickets::new(Arc::new(OneLayout { w, stride, h: 3 })); + let got = tickets.open(&open_params(format)); + assert_eq!(got.is_ok(), accepted, "w {w} stride {stride} {format}: {got:?}"); + assert_eq!(tickets.open_tickets(), usize::from(accepted), "a refused layout leaves no ticket"); + } + } + + #[test] + fn a_rendered_frame_with_a_short_stride_is_refused_and_the_client_refuses_it_too() { + let render = json!({"req": 1, "time_s": 0.0, "epoch": 0}); + let tickets = FrameTickets::new(Arc::new(OneLayout { w: 4, stride: 16, h: 3 })); + let ticket = tickets.open(&open_params("rgba8_premul")).unwrap().ticket; + assert!(tickets.render(&ticket, &render).is_ok()); + // A source that reports a narrower stride than it promised at open. + let lying = FrameTickets::new(Arc::new(OneLayout { w: 4, stride: 2, h: 3 })); + lying.open.lock().unwrap().insert(ticket.clone(), FrameSpec { link_id: "a".into(), format: FrameFormat::Rgba8Premul, scale: 1.0 }); + assert!(matches!(lying.render(&ticket, &render), Err(LinkError::Invalid(_)))); + + let reply = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 1, "format": "rgba8_premul", "render_ms": 1.0}); + assert!(matches!(parse_reply(1, &reply, vec![0; 3]), Err(LinkError::Invalid(_)))); + let fine = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + assert!(parse_reply(1, &fine, vec![0; 48]).is_ok()); + } + + #[test] + fn tickets_are_32_lower_hex_and_distinct() { + let a = new_ticket().unwrap(); + let b = new_ticket().unwrap(); + assert_eq!(a.len(), 32); + assert!(is_lower_hex(&a)); + assert_ne!(a, b); + } +} diff --git a/crates/ipc/src/cache.rs b/crates/ipc/src/cache.rs new file mode 100644 index 000000000..6cc77545d --- /dev/null +++ b/crates/ipc/src/cache.rs @@ -0,0 +1,278 @@ +//! A private per-user cache directory for Craft Link. +//! +//! Prepared media and other link caches must not sit in a directory that another user can create +//! first. This module picks the directory, creates it `0700`, refuses a symlink, a wrong owner or a +//! loose mode, and writes files `0600` through a temp name and a rename. It never falls back to +//! `$TMPDIR` or `/tmp`: with no home and no override it returns an error. +//! +//! Order: `CRAFT_LINK_CACHE_DIR` (absolute), `$XDG_CACHE_HOME/artcraft/link`, then +//! `$HOME/.cache/artcraft/link`. A leaf name `sub` is added under the chosen directory. + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::error::LinkError; + +/// Environment variable that overrides the cache root. +pub const CACHE_DIR_ENV: &str = "CRAFT_LINK_CACHE_DIR"; + +/// [`cache_dir_with`] for the process environment. +pub fn private_cache_dir(sub: &str) -> Result { + let env = std::env::vars().collect::>(); + cache_dir_with(&env, sub) +} + +/// Pick, create and check the cache directory `sub` for `env`. +pub fn cache_dir_with(env: &BTreeMap, sub: &str) -> Result { + if sub.is_empty() || sub.starts_with('.') || !sub.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { + return Err(refused(Path::new(sub), "cache name must be letters, digits, dot, dash or underscore")); + } + let (base, rest) = base_and_rest(env)?; + let mut path = base.clone(); + create_base(&path)?; + for part in rest.iter().map(String::as_str).chain([sub]) { + path.push(part); + crate::discovery::ensure_private_dir(&path)?; + } + Ok(path) +} + +fn base_and_rest(env: &BTreeMap) -> Result<(PathBuf, Vec), LinkError> { + let absolute = |key: &str| env.get(key).filter(|value| !value.is_empty()).map(PathBuf::from); + if let Some(root) = absolute(CACHE_DIR_ENV) { + if !root.is_absolute() { + return Err(refused(&root, "CRAFT_LINK_CACHE_DIR is not absolute")); + } + let (Some(parent), Some(leaf)) = (root.parent(), root.file_name().and_then(|name| name.to_str())) else { + return Err(refused(&root, "CRAFT_LINK_CACHE_DIR has no parent directory")); + }; + return Ok((parent.to_path_buf(), vec![leaf.to_string()])); + } + if let Some(xdg) = absolute("XDG_CACHE_HOME").filter(|path| path.is_absolute()) { + return Ok((xdg, vec!["artcraft".into(), "link".into()])); + } + if let Some(home) = absolute("HOME").filter(|path| path.is_absolute()) { + return Ok((home.join(".cache"), vec!["artcraft".into(), "link".into()])); + } + Err(refused(Path::new(""), "no cache directory: set CRAFT_LINK_CACHE_DIR, XDG_CACHE_HOME or HOME (a shared temp directory is not used)")) +} + +/// Create `path` and missing parents `0700`. An existing directory is left as it is: it is the +/// user's own `~/.cache` or the parent of the override. +fn create_base(path: &Path) -> Result<(), LinkError> { + let mut builder = std::fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path)?; + Ok(()) +} + +/// Write `bytes` to `/` with mode `0600`. The file is made under a temp name with +/// `create_new` (which does not follow a symlink or reuse a file) and renamed into place, so a +/// reader never sees half a file and a planted symlink at `name` is replaced, not followed. +pub fn write_private(dir: &Path, name: &str, bytes: &[u8]) -> Result<(), LinkError> { + if name.is_empty() || name.contains(['/', '\\', '\0']) || name.starts_with('.') { + return Err(refused(&dir.join(name), "file name must be one plain component")); + } + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let tmp = dir.join(format!(".{name}.{}.tmp", crate::auth::encode_hex(&rand))); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&tmp)?; + let written = file.write_all(bytes).and_then(|()| file.sync_all()); + drop(file); + if let Err(err) = written.and_then(|()| std::fs::rename(&tmp, dir.join(name))) { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(()) +} + +/// Read `/` when it is a regular file (not a symlink) that this user owns and no one +/// else can access. Anything else is `Refused`. A missing file is an I/O error. +/// +/// On Unix the open refuses a symlink and does not wait on a FIFO, and the checks and the read use +/// that one handle, so the entry cannot change between the check and the read. +pub fn read_private(dir: &Path, name: &str) -> Result, LinkError> { + let path = dir.join(name); + #[cfg(unix)] + { + use std::io::Read; + use std::os::unix::fs::MetadataExt; + let mut file = match crate::fsopen::open_read(&path, false) { + Ok(file) => file, + // The errno for a symlink in the last component differs between Unix systems. + Err(_) if std::fs::symlink_metadata(&path).is_ok_and(|meta| meta.file_type().is_symlink()) => { + return Err(refused(&path, "not a regular file (a symlink is not followed)")); + } + Err(err) => return Err(err.into()), + }; + let meta = file.metadata()?; + if !meta.file_type().is_file() { + return Err(refused(&path, "not a regular file")); + } + let euid = crate::discovery::current_euid()?; + if meta.uid() != euid { + return Err(refused(&path, &format!("owner is {}, euid is {euid}", meta.uid()))); + } + if meta.mode() & 0o077 != 0 { + return Err(refused(&path, &format!("mode is {:o}, group and others must have no access", meta.mode() & 0o777))); + } + let mut bytes = Vec::with_capacity(usize::try_from(meta.len()).unwrap_or(0)); + file.read_to_end(&mut bytes)?; + Ok(bytes) + } + #[cfg(not(unix))] + { + let meta = std::fs::symlink_metadata(&path)?; + if !meta.file_type().is_file() { + return Err(refused(&path, "not a regular file (a symlink is not followed)")); + } + Ok(std::fs::read(&path)?) + } +} + +fn refused(path: &Path, reason: &str) -> LinkError { + LinkError::Refused { path: path.display().to_string(), reason: reason.to_string() } +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; + + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-cache-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + path + } + + fn env(pairs: &[(&str, &Path)]) -> BTreeMap { + pairs.iter().map(|(key, value)| ((*key).to_string(), value.display().to_string())).collect() + } + + fn mode(path: &Path) -> u32 { + std::fs::symlink_metadata(path).unwrap().mode() & 0o777 + } + + #[test] + fn home_and_xdg_give_a_0700_chain() { + let home = scratch("home"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + assert_eq!(dir, home.join(".cache/artcraft/link/prepared")); + for part in [".cache/artcraft", ".cache/artcraft/link", ".cache/artcraft/link/prepared"] { + assert_eq!(mode(&home.join(part)), 0o700, "{part}"); + } + let xdg = scratch("xdg"); + let dir = cache_dir_with(&env(&[("HOME", &home), ("XDG_CACHE_HOME", &xdg)]), "prepared").unwrap(); + assert_eq!(dir, xdg.join("artcraft/link/prepared")); + assert_eq!(mode(&dir), 0o700); + // Same call twice is fine. + assert_eq!(cache_dir_with(&env(&[("XDG_CACHE_HOME", &xdg)]), "prepared").unwrap(), dir); + std::fs::remove_dir_all(&home).unwrap(); + std::fs::remove_dir_all(&xdg).unwrap(); + } + + #[test] + fn no_home_means_an_error_not_tmp() { + let err = cache_dir_with(&BTreeMap::new(), "prepared").unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err:?}"); + let relative: BTreeMap<_, _> = + [("HOME".to_string(), "relative".to_string()), ("XDG_CACHE_HOME".to_string(), "also/relative".to_string())].into(); + assert!(cache_dir_with(&relative, "prepared").is_err()); + let rel_override: BTreeMap<_, _> = [(CACHE_DIR_ENV.to_string(), "cache".to_string())].into(); + assert!(cache_dir_with(&rel_override, "prepared").is_err()); + } + + #[test] + fn the_override_is_used_and_checked() { + let root = scratch("override"); + let chosen = root.join("mine"); + let dir = cache_dir_with(&env(&[(CACHE_DIR_ENV, &chosen), ("HOME", Path::new("/nonexistent"))]), "prepared").unwrap(); + assert_eq!(dir, chosen.join("prepared")); + assert_eq!(mode(&chosen), 0o700); + // A loose existing directory is refused, not tightened behind the user's back. + std::fs::set_permissions(&chosen, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(cache_dir_with(&env(&[(CACHE_DIR_ENV, &chosen)]), "prepared").is_err()); + // A symlink is refused. + let real = root.join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = root.join("link"); + symlink(&real, &link).unwrap(); + assert!(cache_dir_with(&env(&[(CACHE_DIR_ENV, &link)]), "prepared").is_err()); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn a_fifo_or_a_missing_file_is_not_read_and_does_not_block() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let home = scratch("fifo"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + let made = std::process::Command::new("mkfifo").arg("-m").arg("600").arg(dir.join("pipe.bin")).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + let (tx, rx) = std::sync::mpsc::channel(); + let reading = dir.clone(); + std::thread::spawn(move || { + let _ = tx.send((read_private(&reading, "pipe.bin"), read_private(&reading, "absent.bin"))); + }); + let (fifo, absent) = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("a FIFO blocked the read"); + assert!(matches!(fifo, Err(LinkError::Refused { .. })), "{fifo:?}"); + assert!(matches!(absent, Err(LinkError::Io(_))), "{absent:?}"); + std::fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn bad_names_are_refused() { + let home = scratch("names"); + std::fs::create_dir_all(&home).unwrap(); + for bad in ["", ".hidden", "a/b", "..", "a b", "x\0y"] { + assert!(cache_dir_with(&env(&[("HOME", &home)]), bad).is_err(), "{bad:?}"); + } + std::fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn files_are_0600_and_symlinks_are_replaced_not_followed() { + let home = scratch("files"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + write_private(&dir, "a.bin", b"one").unwrap(); + assert_eq!(mode(&dir.join("a.bin")), 0o600); + assert_eq!(read_private(&dir, "a.bin").unwrap(), b"one"); + // A symlink planted at the name is replaced. The target keeps its content. + let victim = home.join("victim"); + std::fs::write(&victim, b"keep").unwrap(); + let planted = dir.join("b.bin"); + symlink(&victim, &planted).unwrap(); + assert!(matches!(read_private(&dir, "b.bin"), Err(LinkError::Refused { .. })), "a symlink is refused, not read"); + write_private(&dir, "b.bin", b"two").unwrap(); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep"); + assert_eq!(read_private(&dir, "b.bin").unwrap(), b"two"); + // A loose file is refused on read. + std::fs::set_permissions(dir.join("a.bin"), std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(read_private(&dir, "a.bin").is_err()); + for bad in ["", "../x", "a/b", ".x"] { + assert!(write_private(&dir, bad, b"x").is_err(), "{bad:?}"); + } + std::fs::remove_dir_all(&home).unwrap(); + } +} diff --git a/crates/ipc/src/client.rs b/crates/ipc/src/client.rs new file mode 100644 index 000000000..8090be1d3 --- /dev/null +++ b/crates/ipc/src/client.rs @@ -0,0 +1,853 @@ +//! Craft Link client. +//! +//! Connects to one instance record, authenticates on local sockets and pipes, +//! and exchanges JSON lines. Push events that arrive before a reply are kept +//! until [`Client::next_event`]. + +use std::collections::VecDeque; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +use interprocess::local_socket::prelude::*; +use interprocess::local_socket::{ConnectOptions, GenericFilePath, Stream}; +use serde_json::{Map, Value, json}; + +use crate::error::LinkError; +use crate::wire; +use crate::{Endpoint, InstanceRecord}; + +/// Identity sent in `link.hello`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HelloClient { + pub app: String, + pub app_version: String, + pub pid: u32, + pub instance: String, + pub protocol_min: u32, + pub protocol_max: u32, +} + +/// One push line: `event`, `seq`, and `data`. +#[derive(Debug, Clone, PartialEq)] +pub struct Event { + pub event: String, + pub seq: u64, + pub data: Value, +} + +/// A connected Craft Link session. +pub struct Client { + record: InstanceRecord, + reader: BufReader, + next_id: u64, + events: VecDeque, + hello: Option, + /// Bytes of a reply line read before a timeout, kept so the next read resumes the line. + partial: Vec, +} + +impl Client { + /// Connect to `record.endpoint`. Every transport sends auth id 0 with the record token first. + pub fn connect(record: &InstanceRecord) -> Result { + let reader = open_reader(record)?; + Ok(Self { record: record.clone(), reader, next_id: 1, events: VecDeque::new(), hello: None, partial: Vec::new() }) + } + + /// Connect to the loopback TCP control port published in `record` (`record.tcp_port()`), + /// with the same token handshake. `Err(Invalid)` when the app opened no TCP port. + pub fn connect_tcp(record: &InstanceRecord) -> Result { + let Some(port) = record.tcp_port() else { + return Err(LinkError::Invalid("the instance record has no tcp_port; start the app with --control".into())); + }; + let mut tcp = record.clone(); + tcp.endpoint = Endpoint::Tcp { port }; + Self::connect(&tcp) + } + + /// Send one request. `id` starts at 1 and increases by one each call. + /// Lines with `event` and a different id are buffered. + pub fn call(&mut self, method: &str, params: Value) -> Result { + let id = self.next_id; + self.next_id = self.next_id.checked_add(1).ok_or_else(|| LinkError::Invalid("request id overflow".into()))?; + let msg = json!({"id": id, "method": method, "params": params}); + write_json(&mut self.reader, &msg)?; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + loop { + let value = match read_json(&mut self.reader, &mut self.partial, deadline)? { + Some(value) => value, + None => return Err(LinkError::Closed), + }; + if !value.is_object() { + return Err(LinkError::Invalid("reply is not an object".into())); + } + if id_is(&value, id) { + return map_reply(&value); + } + if value.get("event").is_some() { + self.events.push_back(parse_event(&value)?); + continue; + } + return Err(LinkError::Invalid("reply id does not match the request".into())); + } + } + + /// `link.hello`. Checks `pid` and `instance` against the connected record. + /// A successful hello is stored and sent again by [`Client::reconnect`]. + pub fn hello(&mut self, identity: &HelloClient) -> Result { + let result = self.call("link.hello", hello_params(identity))?; + check_identity(&result, &self.record)?; + self.hello = Some(identity.clone()); + Ok(result) + } + + /// `events.subscribe`. Returns `result.seq`. + pub fn subscribe(&mut self, events: &[&str]) -> Result { + let result = self.call("events.subscribe", json!({"events": events}))?; + result.get("seq").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("events.subscribe result has no seq".into())) + } + + /// `events.unsubscribe`. + pub fn unsubscribe(&mut self) -> Result<(), LinkError> { + let _result = self.call("events.unsubscribe", json!({}))?; + Ok(()) + } + + /// The next buffered event, or the next event read before `timeout`. + /// `None` waits up to [`wire::REPLY_DEADLINE_SECS`]. A zero duration does not read. + pub fn next_event(&mut self, timeout: Option) -> Result, LinkError> { + if let Some(event) = self.events.pop_front() { + return Ok(Some(event)); + } + if timeout.is_some_and(|wait| wait.is_zero()) { + return Ok(None); + } + let wait = timeout.unwrap_or(Duration::from_secs(wire::REPLY_DEADLINE_SECS)); + let deadline = Instant::now() + wait; + if Instant::now() >= deadline { + return Ok(None); + } + match read_json(&mut self.reader, &mut self.partial, deadline) { + Ok(None) => Err(LinkError::Closed), + Ok(Some(value)) => { + if value.get("event").is_some() { + Ok(Some(parse_event(&value)?)) + } else { + Err(LinkError::Invalid("expected an event line".into())) + } + } + Err(LinkError::Timeout) => Ok(None), + Err(err) => Err(err), + } + } + + /// Connect to the stored record again. Does not scan discovery. + /// Sends auth again. Sends the stored hello when one succeeded. + pub fn reconnect(&mut self) -> Result<(), LinkError> { + let reader = open_reader(&self.record)?; + self.reader = reader; + self.partial.clear(); + self.events.clear(); + self.next_id = 1; + if let Some(identity) = self.hello.clone() { + self.hello(&identity)?; + } + Ok(()) + } +} + +pub(crate) enum Transport { + Local(Stream), + Tcp(TcpStream), + #[cfg(windows)] + Pipe(PipeTransport), +} + +/// Windows named pipes have no socket-style timeouts. Poll their nonblocking I/O so +/// authentication and frame requests retain the same bounded waits as other transports. +#[cfg(windows)] +pub(crate) struct PipeTransport { + stream: Stream, + recv_timeout: std::cell::Cell>, + send_timeout: std::cell::Cell>, +} + +#[cfg(windows)] +impl PipeTransport { + pub(crate) fn new(stream: Stream) -> std::io::Result { + stream.set_nonblocking(true)?; + Ok(Self { stream, recv_timeout: std::cell::Cell::new(None), send_timeout: std::cell::Cell::new(None) }) + } + + pub(crate) fn set_timeouts(&self, recv: Duration, send: Duration) { + self.recv_timeout.set(Some(recv)); + self.send_timeout.set(Some(send)); + } + + fn poll(timeout: Option, mut operation: impl FnMut() -> std::io::Result) -> std::io::Result { + let start = Instant::now(); + loop { + match operation() { + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + if timeout.is_some_and(|limit| start.elapsed() >= limit) { + return Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "named pipe I/O timed out")); + } + std::thread::sleep(Duration::from_millis(1)); + } + result => return result, + } + } + } +} + +#[cfg(windows)] +impl Read for PipeTransport { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + Self::poll(self.recv_timeout.get(), || match self.stream.read(buf) { + // interprocess converts Windows ERROR_NO_DATA on an empty nonblocking pipe + // to EOF. A disconnected pipe has no peer; otherwise retry until the deadline. + Ok(0) if !buf.is_empty() && self.stream.peer_creds().is_ok() => Err(std::io::ErrorKind::WouldBlock.into()), + result => result, + }) + } +} + +#[cfg(windows)] +impl Write for PipeTransport { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Self::poll(self.send_timeout.get(), || match self.stream.write(buf) { + Ok(0) if !buf.is_empty() => Err(std::io::ErrorKind::WouldBlock.into()), + result => result, + }) + } + fn flush(&mut self) -> std::io::Result<()> { + self.stream.flush() + } +} + +impl Transport { + pub(crate) fn arm_recv_timeout(&self, timeout: Option) -> std::io::Result<()> { + match self { + Self::Local(stream) => { + let result = stream.set_recv_timeout(timeout); + // Darwin rejects socket options after full shutdown (sosetoptlock), even when + // unread reply bytes remain. Reading those bytes or EOF cannot block, so let + // the read report the actual protocol result instead of this timeout error. + #[cfg(target_os = "macos")] + if result.as_ref().is_err_and(|err| err.raw_os_error() == Some(libc::EINVAL)) { + return Ok(()); + } + result + } + Self::Tcp(stream) => stream.set_read_timeout(timeout), + #[cfg(windows)] + Self::Pipe(stream) => { + stream.recv_timeout.set(timeout); + Ok(()) + } + } + } + + pub(crate) fn arm_send_timeout(&self, timeout: Option) -> std::io::Result<()> { + match self { + Self::Local(stream) => stream.set_send_timeout(timeout), + Self::Tcp(stream) => stream.set_write_timeout(timeout), + #[cfg(windows)] + Self::Pipe(stream) => { + stream.send_timeout.set(timeout); + Ok(()) + } + } + } +} + +impl Read for Transport { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + Self::Local(stream) => stream.read(buf), + Self::Tcp(stream) => stream.read(buf), + #[cfg(windows)] + Self::Pipe(stream) => stream.read(buf), + } + } +} + +impl Write for Transport { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + Self::Local(stream) => stream.write(buf), + Self::Tcp(stream) => stream.write(buf), + #[cfg(windows)] + Self::Pipe(stream) => stream.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Local(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + #[cfg(windows)] + Self::Pipe(stream) => stream.flush(), + } + } +} + +pub(crate) fn open_reader(record: &InstanceRecord) -> Result, LinkError> { + let (transport, needs_auth) = open_transport(&record.endpoint)?; + let mut reader = BufReader::new(transport); + if needs_auth { + authenticate(&mut reader, &record.token)?; + } + Ok(reader) +} + +fn open_transport(endpoint: &Endpoint) -> Result<(Transport, bool), LinkError> { + match endpoint { + Endpoint::Unix { path } => Ok((Transport::Local(connect_unix(path)?), true)), + Endpoint::Pipe { name } => { + #[cfg(windows)] + { + Ok((Transport::Pipe(PipeTransport::new(connect_pipe(name)?)?), true)) + } + #[cfg(not(windows))] + { + Ok((Transport::Local(connect_pipe(name)?), true)) + } + } + Endpoint::Tcp { port } => { + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, *port)); + let wait = Duration::from_secs(wire::REPLY_DEADLINE_SECS); + match TcpStream::connect_timeout(&addr, wait) { + Ok(stream) => Ok((Transport::Tcp(stream), true)), + Err(err) if is_timeout(&err) => Err(LinkError::Timeout), + Err(err) => Err(err.into()), + } + } + } +} + +fn connect_unix(path: &str) -> Result { + let name = path.to_fs_name::().map_err(LinkError::io)?; + ConnectOptions::new().name(name).connect_sync().map_err(LinkError::io) +} + +/// Named pipes are the Windows endpoint. Other platforms cannot open one. +/// This returns [`LinkError::Invalid`] instead of pretending the pipe connected. +#[cfg(not(windows))] +fn connect_pipe(name: &str) -> Result { + let _ = name; + Err(LinkError::Invalid("named pipes are a Windows endpoint and are not connected on this platform".into())) +} + +#[cfg(windows)] +fn connect_pipe(name: &str) -> Result { + use interprocess::local_socket::GenericNamespaced; + let socket_name = name.to_ns_name::().map_err(LinkError::io)?; + ConnectOptions::new().name(socket_name).connect_sync().map_err(LinkError::io) +} + +fn authenticate(reader: &mut BufReader, token: &str) -> Result<(), LinkError> { + let request = json!({"id": 0, "method": "auth", "params": {"token": token}}); + write_json(reader, &request)?; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + let expected = wire::auth_success(&json!(0)); + let mut partial = Vec::new(); + loop { + let line = match pull_line(reader, &mut partial, deadline) { + Ok(Some(bytes)) => bytes, + Ok(None) => return Err(LinkError::Auth), + Err(LinkError::Timeout) => return Err(LinkError::Timeout), + Err(LinkError::TooLarge) => return Err(LinkError::TooLarge), + Err(LinkError::Io(text)) => return Err(LinkError::Io(text)), + Err(_) => return Err(LinkError::Auth), + }; + let Ok(text) = std::str::from_utf8(&line) else { + return Err(LinkError::Auth); + }; + let text = text.trim(); + if text.is_empty() { + continue; + } + let Ok(value) = serde_json::from_str::(text) else { + return Err(LinkError::Auth); + }; + if value == expected { + return Ok(()); + } + return Err(LinkError::Auth); + } +} + +pub(crate) fn write_json(reader: &mut BufReader, value: &Value) -> Result<(), LinkError> { + reader.get_ref().arm_send_timeout(Some(Duration::from_secs(wire::REPLY_DEADLINE_SECS)))?; + wire::write_line(reader.get_mut(), value) +} + +/// Reads one JSON line. `partial` holds the bytes of a line that a timeout cut short, so a caller +/// that keeps the connection after `Err(Timeout)` passes the same buffer again and gets the whole line. +pub(crate) fn read_json(reader: &mut BufReader, partial: &mut Vec, deadline: Instant) -> Result, LinkError> { + loop { + let Some(bytes) = pull_line(reader, partial, deadline)? else { + return Ok(None); + }; + let text = std::str::from_utf8(&bytes).map_err(|err| LinkError::Invalid(err.to_string()))?; + let text = text.trim(); + if text.is_empty() { + continue; + } + return Ok(Some(serde_json::from_str(text)?)); + } +} + +/// Same line limit as [`wire::read_line`]. The io error kind is kept so a +/// socket timeout becomes [`LinkError::Timeout`] instead of a generic io string. +/// +/// Bytes read before a timeout stay in `partial` and the next call continues that line. +fn pull_line(reader: &mut BufReader, partial: &mut Vec, deadline: Instant) -> Result>, LinkError> { + let max = wire::MAX_LINE; + let limit = u64::try_from(max).map_err(|_| LinkError::TooLarge)?.saturating_add(1); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(LinkError::Timeout); + } + reader.get_ref().arm_recv_timeout(Some(remaining))?; + let left = limit.saturating_sub(u64::try_from(partial.len()).unwrap_or(u64::MAX)); + match reader.take(left).read_until(b'\n', partial) { + Ok(0) if partial.is_empty() => return Ok(None), + Ok(_) => return finish_line(std::mem::take(partial), max), + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) if is_timeout(&err) => return Err(LinkError::Timeout), + Err(err) => return Err(err.into()), + } + } +} + +fn finish_line(mut buf: Vec, max: usize) -> Result>, LinkError> { + if buf.last() == Some(&b'\n') { + buf.pop(); + if buf.last() == Some(&b'\r') { + buf.pop(); + } + } else if buf.len() > max { + return Err(LinkError::TooLarge); + } + if buf.len() > max { + return Err(LinkError::TooLarge); + } + Ok(Some(buf)) +} + +fn is_timeout(err: &std::io::Error) -> bool { + if matches!(err.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) { + return true; + } + // Linux EAGAIN 11, ETIMEDOUT 110. macOS EAGAIN 35, ETIMEDOUT 60. + // Windows ERROR_SEM_TIMEOUT 121, WAIT_TIMEOUT 258. + matches!(err.raw_os_error(), Some(11 | 35 | 60 | 110 | 121 | 258)) +} + +fn hello_params(identity: &HelloClient) -> Value { + json!({ + "client": { + "app": identity.app, + "app_version": identity.app_version, + "pid": identity.pid, + "instance": identity.instance, + }, + "protocol": {"min": identity.protocol_min, "max": identity.protocol_max}, + }) +} + +fn check_identity(result: &Value, record: &InstanceRecord) -> Result<(), LinkError> { + let got_pid = result.get("pid"); + if got_pid.and_then(Value::as_u64) != Some(u64::from(record.pid)) { + return Err(LinkError::Mismatch { + field: "pid".into(), + expected: record.pid.to_string(), + got: got_pid.map(json_text).unwrap_or_else(|| "missing".into()), + }); + } + let got_instance = result.get("instance"); + if got_instance.and_then(Value::as_str) != Some(record.instance.as_str()) { + return Err(LinkError::Mismatch { + field: "instance".into(), + expected: record.instance.clone(), + got: got_instance.map(json_text).unwrap_or_else(|| "missing".into()), + }); + } + Ok(()) +} + +fn json_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Number(number) => number.to_string(), + Value::Bool(flag) => flag.to_string(), + Value::Null => "null".into(), + other => other.to_string(), + } +} + +fn id_is(value: &Value, id: u64) -> bool { + value.get("id").and_then(Value::as_u64) == Some(id) +} + +fn parse_event(value: &Value) -> Result { + let event = value.get("event").and_then(Value::as_str).ok_or_else(|| LinkError::Invalid("event line has no event name".into()))?; + let seq = value.get("seq").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("event line has no seq".into()))?; + let data = value.get("data").cloned().unwrap_or(Value::Null); + Ok(Event { event: event.to_string(), seq, data }) +} + +pub(crate) fn map_reply(value: &Value) -> Result { + let Some(ok) = value.get("ok").and_then(Value::as_bool) else { + return Err(LinkError::Invalid("reply has no ok flag".into())); + }; + if ok { + return value.get("result").cloned().ok_or_else(|| LinkError::Invalid("reply has no result".into())); + } + match value.get("error") { + Some(Value::String(text)) => Err(LinkError::Invalid(text.clone())), + Some(Value::Object(map)) => map_error_object(map), + Some(_) => Err(LinkError::Invalid("error is not a string or an object".into())), + None => Err(LinkError::Invalid("reply has no error".into())), + } +} + +fn map_error_object(map: &Map) -> Result { + let code = map.get("code").and_then(Value::as_str).unwrap_or(""); + let message = map.get("message").and_then(Value::as_str).filter(|text| !text.is_empty()).unwrap_or(code).to_string(); + match code { + "protocol_unsupported" => { + let data = map.get("data").and_then(Value::as_object); + let server_min = protocol_bound(data, "server_min")?; + let server_max = protocol_bound(data, "server_max")?; + Err(LinkError::Protocol { server_min, server_max }) + } + "auth_required" => Err(LinkError::Auth), + "bad_params" => Err(LinkError::BadParams(message)), + "payload_too_large" => Err(LinkError::TooLarge), + "payload_invalid" => Err(LinkError::PayloadInvalid(message)), + "media_missing" => { + let paths = map + .get("data") + .and_then(Value::as_object) + .and_then(|data| data.get("missing")) + .and_then(Value::as_array) + .map(|list| list.iter().filter_map(Value::as_str).map(str::to_string).collect()) + .unwrap_or_default(); + Err(LinkError::MediaMissing { paths }) + } + "not_ready" => Err(LinkError::NotReady(message)), + "link_unknown" => Err(LinkError::Unknown(message)), + "busy" => Err(LinkError::Busy(message)), + _ => Err(LinkError::Invalid(if message.is_empty() { "unknown error".into() } else { message })), + } +} + +fn protocol_bound(data: Option<&Map>, key: &str) -> Result { + let Some(value) = data.and_then(|map| map.get(key)).and_then(Value::as_u64) else { + return Err(LinkError::Invalid(format!("protocol_unsupported is missing {key}"))); + }; + u32::try_from(value).map_err(|_| LinkError::Invalid(format!("{key} does not fit in u32"))) +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::io::{BufReader, Write}; + use std::os::unix::net::{UnixListener, UnixStream}; + use std::path::PathBuf; + use std::thread::JoinHandle; + use std::time::{Duration, Instant}; + + use serde_json::{Value, json}; + + use super::*; + + struct Peer { + path: PathBuf, + record: InstanceRecord, + join: Option>, + } + + impl Drop for Peer { + fn drop(&mut self) { + if let Some(join) = self.join.take() { + let _ = join.join(); + } + let _ = std::fs::remove_file(&self.path); + } + } + + fn sock_path(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))) + } + + fn peer(label: &str, script: F) -> Peer + where + F: FnOnce(UnixStream, InstanceRecord) + Send + 'static, + { + let path = sock_path(label); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let endpoint = Endpoint::Unix { path: path.to_str().unwrap().to_string() }; + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let moved = record.clone(); + let join = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(10))).unwrap(); + script(stream, moved); + }); + Peer { path, record, join: Some(join) } + } + + fn read_req(reader: &mut BufReader) -> Value { + let bytes = crate::wire::read_line(reader, crate::wire::MAX_LINE).unwrap().expect("peer eof"); + serde_json::from_slice(&bytes).unwrap() + } + + fn write_val(reader: &mut BufReader, value: &Value) { + crate::wire::write_line(reader.get_mut(), value).unwrap(); + } + + fn accept_auth(reader: &mut BufReader, token: &str) { + let req = read_req(reader); + assert_eq!(req["method"], "auth"); + assert_eq!(req["id"], json!(0)); + assert_eq!(req["params"]["token"], token); + // Key order differs from serde_json's BTreeMap on purpose. + reader.get_mut().write_all(b"{\"result\":{\"authenticated\":true},\"ok\":true,\"id\":0}\n").unwrap(); + reader.get_mut().flush().unwrap(); + } + + #[test] + fn auth_then_call_parses_the_result() { + let server = peer("call", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + assert_eq!(req["method"], "engine.execute"); + assert_eq!(req["id"], json!(1)); + assert_eq!(req["params"]["command"], "ping"); + let id = req["id"].clone(); + write_val(&mut reader, &crate::wire::reply_ok(&id, json!({"pong": true}))); + }); + let mut client = Client::connect(&server.record).unwrap(); + let result = client.call("engine.execute", json!({"command": "ping"})).unwrap(); + assert_eq!(result, json!({"pong": true})); + } + + #[test] + fn event_line_before_the_reply_is_returned_by_next_event() { + let server = peer("ev", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + let id = req["id"].clone(); + write_val(&mut reader, &json!({"event": "doc.changed", "seq": 4, "data": {"doc_id": "abc"}})); + write_val(&mut reader, &crate::wire::reply_ok(&id, json!({"n": 1}))); + }); + let mut client = Client::connect(&server.record).unwrap(); + assert!(client.next_event(Some(Duration::ZERO)).unwrap().is_none()); + let result = client.call("link.focus", json!({})).unwrap(); + assert_eq!(result, json!({"n": 1})); + let event = client.next_event(Some(Duration::ZERO)).unwrap().expect("buffered event"); + assert_eq!(event.event, "doc.changed"); + assert_eq!(event.seq, 4); + assert_eq!(event.data, json!({"doc_id": "abc"})); + assert!(client.next_event(Some(Duration::ZERO)).unwrap().is_none()); + } + + #[test] + fn queued_events_are_read_after_the_peer_has_closed() { + let mut server = peer("closed", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + for seq in [1, 2] { + write_val(&mut reader, &json!({"event": "doc.changed", "seq": seq, "data": {}})); + } + }); + let mut client = Client::connect(&server.record).unwrap(); + server.join.take().unwrap().join().unwrap(); + for seq in [1, 2] { + let event = client.next_event(Some(Duration::from_secs(1))).unwrap().expect("queued event after peer close"); + assert_eq!(event.seq, seq); + } + assert!(matches!(client.next_event(Some(Duration::from_secs(1))), Err(LinkError::Closed))); + } + + #[test] + fn reconnect_sends_auth_again() { + let path = sock_path("re"); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let endpoint = Endpoint::Unix { path: path.to_str().unwrap().to_string() }; + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let token = record.token.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let join = std::thread::spawn(move || { + for _ in 0..2 { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(stream); + let req = read_req(&mut reader); + tx.send(req).unwrap(); + write_val(&mut reader, &crate::wire::auth_success(&json!(0))); + } + }); + let mut client = Client::connect(&record).unwrap(); + client.reconnect().unwrap(); + drop(client); + let first = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let second = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(first["method"], "auth"); + assert_eq!(second["method"], "auth"); + assert_eq!(first["params"]["token"], token); + assert_eq!(second["params"]["token"], token); + join.join().unwrap(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn failed_auth_reply_is_auth() { + let server = peer("bad", |stream, _record| { + let mut reader = BufReader::new(stream); + let _req = read_req(&mut reader); + write_val(&mut reader, &crate::wire::auth_failure()); + }); + let err = match Client::connect(&server.record) { + Err(err) => err, + Ok(_client) => panic!("connect should fail auth"), + }; + assert!(matches!(err, LinkError::Auth), "{err}"); + } + + #[test] + fn closed_before_auth_reply_is_auth() { + let server = peer("eof", |stream, _record| { + let mut reader = BufReader::new(stream); + let _req = read_req(&mut reader); + drop(reader); + }); + let err = match Client::connect(&server.record) { + Err(err) => err, + Ok(_client) => panic!("connect should fail when the peer closes"), + }; + assert!(matches!(err, LinkError::Auth), "{err}"); + } + + #[test] + fn hello_pid_mismatch_is_mismatch() { + let server = peer("hi", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + assert_eq!(req["method"], "link.hello"); + let id = req["id"].clone(); + let result = json!({ + "app": "effectcraft", + "app_version": "0.8.3", + "pid": 4243, + "instance": record.instance, + "protocol": 1, + }); + write_val(&mut reader, &crate::wire::reply_ok(&id, result)); + }); + let mut client = Client::connect(&server.record).unwrap(); + let identity = HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "11111111-1111-4111-8111-111111111111".into(), + protocol_min: wire::PROTOCOL_MIN, + protocol_max: wire::PROTOCOL_MAX, + }; + let err = client.hello(&identity).unwrap_err(); + match err { + LinkError::Mismatch { field, expected, got } => { + assert_eq!(field, "pid"); + assert_eq!(expected, "4242"); + assert_eq!(got, "4243"); + } + other => panic!("expected mismatch, got {other}"), + } + } + + #[test] + fn error_object_and_string_map_onto_link_error() { + let server = peer("err", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + for _ in 0..3 { + let req = read_req(&mut reader); + let id = req["id"].clone(); + let method = req["method"].as_str().unwrap().to_string(); + let error = match method.as_str() { + "proto" => json!({"code": "protocol_unsupported", "message": "no", "data": {"server_min": 2, "server_max": 5}}), + "text" => json!("no such method"), + "params" => json!({"code": "bad_params", "message": "missing command", "data": {}}), + other => panic!("unexpected method {other}"), + }; + write_val(&mut reader, &json!({"id": id, "ok": false, "error": error})); + } + }); + let mut client = Client::connect(&server.record).unwrap(); + match client.call("proto", json!({})).unwrap_err() { + LinkError::Protocol { server_min, server_max } => { + assert_eq!(server_min, 2); + assert_eq!(server_max, 5); + } + other => panic!("{other}"), + } + match client.call("text", json!({})).unwrap_err() { + LinkError::Invalid(text) => assert_eq!(text, "no such method"), + other => panic!("{other}"), + } + match client.call("params", json!({})).unwrap_err() { + LinkError::BadParams(text) => assert_eq!(text, "missing command"), + other => panic!("{other}"), + } + } + + #[test] + fn next_event_resumes_a_line_cut_short_by_a_timeout() { + let server = peer("split", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let out = reader.get_mut(); + out.write_all(b"{\"event\":\"doc.changed\",\"seq\":9,").unwrap(); + out.flush().unwrap(); + std::thread::sleep(Duration::from_millis(600)); + out.write_all(b"\"data\":{\"doc_id\":\"abc\"}}\n").unwrap(); + out.flush().unwrap(); + std::thread::sleep(Duration::from_millis(300)); + }); + let mut client = Client::connect(&server.record).unwrap(); + assert!(client.next_event(Some(Duration::from_millis(200))).unwrap().is_none()); + let event = client.next_event(Some(Duration::from_secs(5))).unwrap().expect("the whole event"); + assert_eq!(event.event, "doc.changed"); + assert_eq!(event.seq, 9); + assert_eq!(event.data, json!({"doc_id": "abc"})); + } + + #[test] + fn next_event_times_out_when_the_peer_is_quiet() { + let server = peer("wait", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + std::thread::sleep(Duration::from_millis(800)); + }); + let mut client = Client::connect(&server.record).unwrap(); + let started = Instant::now(); + let got = client.next_event(Some(Duration::from_millis(300))).unwrap(); + let elapsed = started.elapsed(); + assert!(got.is_none()); + assert!(elapsed >= Duration::from_millis(200), "returned too fast: {elapsed:?}"); + assert!(elapsed < Duration::from_secs(5), "waited too long: {elapsed:?}"); + } +} diff --git a/crates/ipc/src/discovery/dir.rs b/crates/ipc/src/discovery/dir.rs new file mode 100644 index 000000000..976c316bb --- /dev/null +++ b/crates/ipc/src/discovery/dir.rs @@ -0,0 +1,354 @@ +//! Per-user runtime directory. Hand-rolled environment lookups. No `dirs` crate. + +use std::path::{Path, PathBuf}; + +use crate::error::LinkError; + +/// The directory that holds instance records, locks, and socket files. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeDir { + path: PathBuf, +} + +impl RuntimeDir { + /// Resolve the directory for this process and create it when it is missing. + pub fn resolve() -> Result { + Self::from_resolved_path(runtime_path()?) + } + + fn from_resolved_path(path: PathBuf) -> Result { + // LOCALAPPDATA exists on a fresh Windows install, but its Craft namespace does not. + // Create and validate that single namespace first; do not recursively follow arbitrary + // missing ancestors. RuntimeDir::at keeps its existing caller-provided-path contract. + #[cfg(windows)] + if let Some(parent) = path.parent() { + ensure_private_dir(parent)?; + } + Self::at(path) + } + + /// Use `path` as the runtime directory. Tests use this so they do not touch + /// the real `$XDG_RUNTIME_DIR/craft`. + pub fn at(path: impl Into) -> Result { + let path = path.into(); + ensure_private_dir(&path)?; + Ok(Self { path }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn record_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.json")) + } + + pub fn lock_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.lock")) + } + + pub fn socket_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.sock")) + } + + /// Unix socket endpoint. The path must fit the instance-record limit of 100 bytes. + pub fn unix_endpoint(&self, app: &str, pid: u32) -> Result { + let path = self.socket_file(app, pid); + let text = path.to_str().ok_or_else(|| LinkError::Invalid("socket path is not utf-8".into()))?.to_string(); + if text.len() > 100 { + return Err(LinkError::Invalid(format!("socket path is {} bytes, the record allows 100", text.len()))); + } + Ok(super::Endpoint::Unix { path: text }) + } +} + +/// Set `path` to `mode` (for example `0o600`) and refuse a different result. +/// Unix socket mode cannot be set through `ListenerOptions::set_mode` (it is +/// `pub(crate)` in interprocess 2.4.4). Call this after the socket file exists. +#[cfg(unix)] +pub fn enforce_mode(path: &Path, mode: u32) -> Result<(), LinkError> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode))?; + let got = std::fs::symlink_metadata(path)?.mode() & 0o777; + if got != mode { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("mode is {got:o}, want {mode:o}") }); + } + Ok(()) +} + +#[cfg(unix)] +pub(crate) fn current_euid() -> Result { + #[cfg(target_os = "linux")] + { + let text = std::fs::read_to_string("/proc/self/status")?; + for line in text.lines() { + let Some(rest) = line.strip_prefix("Uid:") else { continue }; + let mut fields = rest.split_whitespace(); + let _real = fields.next(); + let Some(eff) = fields.next() else { + return Err(LinkError::Io("Uid line is short".into())); + }; + return eff.parse().map_err(LinkError::io); + } + Err(LinkError::Io("no Uid line in /proc/self/status".into())) + } + #[cfg(all(unix, not(target_os = "linux")))] + { + probe_euid(&std::env::temp_dir()) + } +} + +/// The effective uid, read from the owner of a file this process creates in `dir`. The std library +/// has no safe call for the euid and this crate forbids `unsafe`. The file has a random name and is +/// made with `create_new`, which fails on an existing path (a planted file or symlink) instead of +/// opening it, so the probe only ever touches a file it made. The file is removed before the +/// function returns. +#[cfg(any(all(unix, not(target_os = "linux")), all(test, unix)))] +fn probe_euid(dir: &Path) -> Result { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let path = dir.join(format!(".craft-uid-{}-{}", std::process::id(), crate::auth::encode_hex(&rand))); + let file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&path)?; + let uid = file.metadata().map(|meta| meta.uid()); + drop(file); + let _ = std::fs::remove_file(&path); + Ok(uid?) +} + +fn runtime_path() -> Result { + #[cfg(target_os = "linux")] + { + let xdg = std::env::var_os("XDG_RUNTIME_DIR"); + Ok(linux_runtime_path(xdg.as_deref(), current_euid()?)) + } + #[cfg(target_os = "macos")] + { + let tmpdir = std::env::var_os("TMPDIR"); + Ok(macos_runtime_path(tmpdir.as_deref(), current_euid()?)) + } + #[cfg(windows)] + { + let value = std::env::var_os("LOCALAPPDATA").filter(|v| !v.is_empty()); + let Some(value) = value else { + return Err(LinkError::Refused { path: "%LOCALAPPDATA%".into(), reason: "LOCALAPPDATA is unset".into() }); + }; + let base = PathBuf::from(value); + if !base.is_absolute() { + return Err(LinkError::Refused { path: base.display().to_string(), reason: "LOCALAPPDATA is not absolute".into() }); + } + Ok(base.join("Craft").join("run")) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + Err(LinkError::Refused { path: String::new(), reason: "no runtime directory for this platform".into() }) + } +} + +#[cfg(target_os = "linux")] +fn linux_runtime_path(xdg: Option<&std::ffi::OsStr>, euid: u32) -> PathBuf { + if let Some(xdg) = xdg { + let path = PathBuf::from(xdg); + if !xdg.is_empty() && path.is_absolute() { + return path.join("craft"); + } + } + PathBuf::from(format!("/tmp/craft-{euid}")) +} + +/// `$TMPDIR/craft` when `TMPDIR` is absolute. Otherwise `/tmp/craft-`, because a shared +/// `/tmp/craft` belongs to the first user who creates it and refuses everyone else. +#[cfg(any(target_os = "macos", all(test, unix)))] +fn macos_runtime_path(tmpdir: Option<&std::ffi::OsStr>, euid: u32) -> PathBuf { + if let Some(tmpdir) = tmpdir { + let path = PathBuf::from(tmpdir); + if !tmpdir.is_empty() && path.is_absolute() { + return path.join("craft"); + } + } + PathBuf::from(format!("/tmp/craft-{euid}")) +} + +pub(crate) fn ensure_private_dir(path: &Path) -> Result<(), LinkError> { + if is_symlink(path) { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "runtime directory is a symlink".into() }); + } + let created = match create_dir(path) { + Ok(()) => true, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => false, + Err(err) => return Err(LinkError::io(err)), + }; + if is_symlink(path) { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "runtime directory is a symlink".into() }); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if created { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))?; + } + verify_unix_dir(path)?; + } + #[cfg(not(unix))] + { + let _ = created; + let meta = std::fs::metadata(path)?; + if !meta.is_dir() { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "not a directory".into() }); + } + } + Ok(()) +} + +fn create_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path) + } + #[cfg(not(unix))] + { + std::fs::create_dir(path) + } +} + +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) +} + +#[cfg(unix)] +fn verify_unix_dir(path: &Path) -> Result<(), LinkError> { + use std::os::unix::fs::MetadataExt; + let meta = std::fs::symlink_metadata(path)?; + if !meta.is_dir() { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "not a directory".into() }); + } + let uid = current_euid()?; + if meta.uid() != uid { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("owner is {}, euid is {uid}", meta.uid()) }); + } + let mode = meta.mode() & 0o777; + if mode != 0o700 { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("mode is {mode:o}, want 700") }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + path + } + + #[test] + fn creates_0700_and_refuses_a_loose_dir() { + let path = scratch("dir"); + let dir = RuntimeDir::at(&path).unwrap(); + assert_eq!(dir.path(), path.as_path()); + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + assert_eq!(std::fs::symlink_metadata(&path).unwrap().mode() & 0o777, 0o700); + let loose = scratch("loose"); + std::fs::create_dir(&loose).unwrap(); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o755)).unwrap(); + let err = RuntimeDir::at(&loose).unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err}"); + let _ = std::fs::remove_dir_all(&loose); + } + let _ = std::fs::remove_dir_all(&path); + } + + #[cfg(unix)] + #[test] + fn refuses_a_symlink() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let real = scratch("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = scratch("link"); + symlink(&real, &link).unwrap(); + assert!(RuntimeDir::at(&link).is_err()); + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_dir_all(&real); + } + + #[cfg(target_os = "linux")] + #[test] + fn xdg_fallback_is_pure() { + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("/run/user/1000")), 5), PathBuf::from("/run/user/1000/craft")); + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("relative")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(linux_runtime_path(None, 9), PathBuf::from("/tmp/craft-9")); + } + + #[cfg(unix)] + #[test] + fn macos_fallback_is_per_user() { + use std::ffi::OsStr; + assert_eq!(macos_runtime_path(Some(OsStr::new("/var/folders/ab/T")), 5), PathBuf::from("/var/folders/ab/T/craft")); + assert_eq!(macos_runtime_path(Some(OsStr::new("")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(macos_runtime_path(Some(OsStr::new("relative")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(macos_runtime_path(None, 9), PathBuf::from("/tmp/craft-9")); + assert_ne!(macos_runtime_path(None, 5), macos_runtime_path(None, 6)); + } + + #[cfg(unix)] + #[test] + fn the_uid_probe_leaves_a_planted_file_alone_and_cleans_up() { + use std::os::unix::fs::symlink; + let dir = scratch("probe"); + std::fs::create_dir_all(&dir).unwrap(); + // The names the probe used to pick: a planted file and a planted symlink must not be opened. + let planted = dir.join(format!(".craft-uid-{}", std::process::id())); + std::fs::write(&planted, b"keep me").unwrap(); + let victim = dir.join("victim"); + std::fs::write(&victim, b"keep me too").unwrap(); + let link = dir.join(format!(".craft-uid-{}-0000000000000000", std::process::id())); + symlink(&victim, &link).unwrap(); + let uid = probe_euid(&dir).unwrap(); + assert_eq!(uid, current_euid().unwrap()); + assert_eq!(std::fs::read(&planted).unwrap(), b"keep me"); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep me too"); + let mut names: Vec = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + names.sort(); + assert_eq!(names.len(), 3, "the probe file is gone: {names:?}"); + // Two probes do not collide. + assert_eq!(probe_euid(&dir).unwrap(), uid); + // A directory that cannot be written is an error, not a guess. + assert!(probe_euid(&dir.join("missing")).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(windows)] + #[test] + fn a_fresh_windows_runtime_creates_its_namespace() { + let base = scratch("fresh-runtime"); + std::fs::create_dir_all(&base).unwrap(); + let path = base.join("Craft").join("run"); + let dir = RuntimeDir::from_resolved_path(path.clone()).unwrap(); + assert_eq!(dir.path(), path); + assert!(base.join("Craft").is_dir()); + std::fs::remove_dir_all(base).unwrap(); + } + + #[test] + fn resolve_points_at_a_private_craft_dir() { + let dir = RuntimeDir::resolve().unwrap(); + #[cfg(windows)] + assert!(dir.path().ends_with(Path::new("Craft").join("run")), "{}", dir.path().display()); + #[cfg(unix)] + assert!(dir.path().ends_with("craft") || dir.path().ends_with(format!("craft-{}", current_euid().unwrap())), "{}", dir.path().display()); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!(std::fs::symlink_metadata(dir.path()).unwrap().mode() & 0o777, 0o700); + } + } +} diff --git a/crates/ipc/src/discovery/mod.rs b/crates/ipc/src/discovery/mod.rs new file mode 100644 index 000000000..f81731970 --- /dev/null +++ b/crates/ipc/src/discovery/mod.rs @@ -0,0 +1,11 @@ +//! Discovery records for Craft apps on this machine. + +mod dir; +mod record; + +pub use dir::RuntimeDir; +pub use record::{Discovery, DocInfo, Endpoint, InstanceRecord, Protocol, valid_app_name}; + +pub(crate) use dir::ensure_private_dir; +#[cfg(unix)] +pub(crate) use dir::{current_euid, enforce_mode}; diff --git a/crates/ipc/src/discovery/record.rs b/crates/ipc/src/discovery/record.rs new file mode 100644 index 000000000..290ac0bb5 --- /dev/null +++ b/crates/ipc/src/discovery/record.rs @@ -0,0 +1,575 @@ +//! Instance records. A live process holds `-.lock`. `scan` deletes +//! the record, the socket, and the lock when `try_lock` acquires that file. + +use std::collections::BTreeSet; +use std::fs::{File, OpenOptions, TryLockError}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Deserialize; +use serde::Serialize; +use serde_json::{Map, Value}; + +use crate::auth::{self, is_lower_hex}; +use crate::error::LinkError; +use crate::wire::PROTOCOL_MAX; + +use super::RuntimeDir; + +/// Where clients connect. Serialized with an internal `kind` tag. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Endpoint { + Unix { path: String }, + Pipe { name: String }, + Tcp { port: u16 }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Protocol { + pub link: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commands_schema: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DocInfo { + pub doc_id: String, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// One running app. Unknown JSON fields are kept in `extra`. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct InstanceRecord { + pub v: u32, + pub app: String, + pub app_version: String, + pub pid: u32, + pub instance: String, + pub endpoint: Endpoint, + pub token: String, + pub started: u64, + pub protocol: Protocol, + pub docs: Vec, + #[serde(default, flatten)] + pub extra: Map, +} + +impl std::fmt::Debug for InstanceRecord { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InstanceRecord") + .field("v", &self.v) + .field("app", &self.app) + .field("app_version", &self.app_version) + .field("pid", &self.pid) + .field("instance", &self.instance) + .field("endpoint", &self.endpoint) + .field("token", &"redacted") + .field("started", &self.started) + .field("protocol", &self.protocol) + .field("docs", &self.docs) + .finish_non_exhaustive() + } +} + +impl InstanceRecord { + pub fn new(app: &str, app_version: &str, pid: u32, endpoint: Endpoint) -> Result { + let record = Self { + v: 1, + app: app.to_string(), + app_version: app_version.to_string(), + pid, + instance: auth::random_uuid()?, + endpoint, + token: auth::generate_token()?, + started: unix_now()?, + protocol: Protocol { link: PROTOCOL_MAX, commands_schema: None, extra: Map::new() }, + docs: Vec::new(), + extra: Map::new(), + }; + record.validate()?; + Ok(record) + } + + pub fn validate(&self) -> Result<(), LinkError> { + if self.v != 1 { + return Err(LinkError::Invalid(format!("record v is {}, want 1", self.v))); + } + if !valid_app_name(&self.app) { + return Err(LinkError::Invalid(format!("app {:?} does not match the record pattern", self.app))); + } + if self.app_version.is_empty() { + return Err(LinkError::Invalid("app_version is empty".into())); + } + if self.pid < 1 { + return Err(LinkError::Invalid("pid must be >= 1".into())); + } + if !valid_uuid(&self.instance) { + return Err(LinkError::Invalid("instance is not a uuid".into())); + } + if self.token.len() != 64 || !is_lower_hex(&self.token) { + return Err(LinkError::Invalid("token must be 64 lowercase hex characters".into())); + } + if self.protocol.link < 1 { + return Err(LinkError::Invalid("protocol.link must be >= 1".into())); + } + if let Some(schema) = &self.protocol.commands_schema { + let Some(hex) = schema.strip_prefix("sha256:") else { + return Err(LinkError::Invalid("commands_schema must start with sha256:".into())); + }; + if hex.len() != 64 || !is_lower_hex(hex) { + return Err(LinkError::Invalid("commands_schema digest is not 64 lowercase hex characters".into())); + } + } + match &self.endpoint { + Endpoint::Unix { path } => { + if path.len() > 100 || path.is_empty() || path.contains('\0') || !Path::new(path).is_absolute() { + return Err(LinkError::Invalid("unix endpoint path must be absolute and at most 100 bytes".into())); + } + } + Endpoint::Pipe { name } => { + if !name.starts_with("craft-") || name.contains('/') || name.contains('\\') { + return Err(LinkError::Invalid("pipe name must start with craft-".into())); + } + } + Endpoint::Tcp { port } => { + if *port < 1 { + return Err(LinkError::Invalid("tcp port must be >= 1".into())); + } + } + } + for doc in &self.docs { + if doc.doc_id.is_empty() || doc.title.is_empty() { + return Err(LinkError::Invalid("doc_id and title are required".into())); + } + } + self.check_caps()?; + self.check_tcp_port()?; + Ok(()) + } + + /// Roles and optional verbs from the optional `caps` field (stored in + /// `extra`, so the struct shape and old struct literals stay valid). + /// Empty when the field is absent or malformed. [`Self::validate`] rejects + /// a malformed field. + pub fn caps(&self) -> Vec { + self.extra + .get("caps") + .and_then(Value::as_array) + .map(|list| list.iter().filter_map(|c| c.as_str().map(str::to_string)).collect()) + .unwrap_or_default() + } + + /// Add `cap` to `caps` when it is not there yet. + pub fn add_cap(&mut self, cap: &str) -> Result<(), LinkError> { + if !valid_cap(cap) { + return Err(LinkError::Invalid(format!("cap {cap:?} must match [a-z][a-z0-9._-]{{0,63}}"))); + } + let mut caps = self.caps(); + if !caps.iter().any(|c| c == cap) { + caps.push(cap.to_string()); + } + self.extra.insert("caps".to_string(), Value::from(caps)); + Ok(()) + } + + /// Loopback TCP control port, when the app opened one (`--control `). Stored in + /// `extra`. The record file is 0600, so only the owner can read this port and the token + /// that the TCP listener requires. + pub fn tcp_port(&self) -> Option { + self.extra.get("tcp_port").and_then(Value::as_u64).and_then(|port| u16::try_from(port).ok()).filter(|port| *port >= 1) + } + + /// Publish the TCP control port in the record. + pub fn set_tcp_port(&mut self, port: u16) { + self.extra.insert("tcp_port".to_string(), Value::from(port)); + } + + fn check_tcp_port(&self) -> Result<(), LinkError> { + match self.extra.get("tcp_port") { + None => Ok(()), + Some(value) if value.as_u64().is_some_and(|port| (1..=65535).contains(&port)) => Ok(()), + Some(_) => Err(LinkError::Invalid("tcp_port must be an integer from 1 to 65535".into())), + } + } + + fn check_caps(&self) -> Result<(), LinkError> { + let Some(value) = self.extra.get("caps") else { + return Ok(()); + }; + let Some(list) = value.as_array() else { + return Err(LinkError::Invalid("caps must be an array of strings".into())); + }; + let mut seen = BTreeSet::new(); + for item in list { + let Some(cap) = item.as_str().filter(|c| valid_cap(c)) else { + return Err(LinkError::Invalid("each cap must match [a-z][a-z0-9._-]{0,63}".into())); + }; + if !seen.insert(cap) { + return Err(LinkError::Invalid(format!("cap {cap:?} is listed twice"))); + } + } + Ok(()) + } +} + +fn valid_cap(cap: &str) -> bool { + let bytes = cap.as_bytes(); + (1..=64).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_lowercase) + && bytes.iter().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-')) +} + +/// Holds the lock file for this process. Drop releases the lock and leaves the +/// files in place. [`Discovery::remove`] deletes them. [`Discovery::scan`] +/// deletes them when the holder is gone. +pub struct Discovery { + dir: PathBuf, + stem: String, + record: InstanceRecord, + lock: File, +} + +impl std::fmt::Debug for Discovery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Discovery").field("stem", &self.stem).field("record", &self.record).finish_non_exhaustive() + } +} + +impl Discovery { + /// Write the record and hold `-.lock` until drop. + /// + /// `try_lock` is the acquire. A successful `try_lock` is the same exclusive + /// lock as `File::lock`, and it returns [`LinkError::Busy`] when a live + /// process already holds the file instead of waiting. + pub fn publish(dir: &RuntimeDir, record: InstanceRecord) -> Result { + record.validate()?; + let stem = format!("{}-{}", record.app, record.pid); + let lock_path = dir.path().join(format!("{stem}.lock")); + let mut opts = OpenOptions::new(); + opts.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let lock = opts.open(&lock_path)?; + #[cfg(unix)] + crate::discovery::enforce_mode(&lock_path, 0o600)?; + match lock.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(LinkError::Busy(format!("{stem} is already live"))); + } + Err(TryLockError::Error(err)) => return Err(err.into()), + } + let keep = unix_path(&record); + clear_payload(dir.path(), &stem, keep.as_deref())?; + let bytes = serde_json::to_vec(&record)?; + write_atomic(dir.path(), &stem, &bytes)?; + Ok(Self { dir: dir.path().to_path_buf(), stem, record, lock }) + } + + pub fn record(&self) -> &InstanceRecord { + &self.record + } + + /// Delete this process's record, socket, and lock. + pub fn remove(self) -> Result<(), LinkError> { + reap_dead(&self.dir, &self.stem, &self.lock) + } + + /// Live records in `dir`. A lock that `try_lock` acquires belongs to a dead + /// process: this deletes its record, socket, and lock. + pub fn scan(dir: &RuntimeDir) -> Result, LinkError> { + let mut stems = BTreeSet::new(); + let entries = std::fs::read_dir(dir.path())?; + for entry in entries { + let entry = entry?; + let Some(name) = entry.file_name().into_string().ok() else { continue }; + if let Some(stem) = stem_from_entry(&name) { + stems.insert(stem); + } + } + let mut live = Vec::new(); + for stem in stems { + if let Some(record) = settle(dir.path(), &stem)? { + live.push(record); + } + } + Ok(live) + } +} + +pub fn valid_app_name(app: &str) -> bool { + crate::ids::valid_app_id(app) +} + +fn valid_uuid(text: &str) -> bool { + crate::ids::valid_uuid(text) +} + +fn unix_now() -> Result { + Ok(SystemTime::now().duration_since(UNIX_EPOCH).map_err(LinkError::io)?.as_secs()) +} + +fn stem_from_entry(name: &str) -> Option { + if let Some(stem) = name.strip_suffix(".json").or_else(|| name.strip_suffix(".lock")).or_else(|| name.strip_suffix(".sock")) { + return accept_stem(stem); + } + let rest = name.strip_prefix('.')?.strip_suffix(".tmp")?; + let (stem, hex) = rest.rsplit_once('.')?; + if hex.len() < 2 || !is_lower_hex(hex) { + return None; + } + accept_stem(stem) +} + +fn accept_stem(stem: &str) -> Option { + let (app, pid) = stem.rsplit_once('-')?; + let pid: u32 = pid.parse().ok()?; + if pid < 1 || !valid_app_name(app) { + return None; + } + Some(stem.to_string()) +} + +fn settle(dir: &Path, stem: &str) -> Result, LinkError> { + let lock_path = dir.join(format!("{stem}.lock")); + let json_path = dir.join(format!("{stem}.json")); + if lock_path.is_file() { + let lock = OpenOptions::new().read(true).write(true).open(&lock_path)?; + match lock.try_lock() { + Err(TryLockError::WouldBlock) => return read_live(&json_path), + Err(TryLockError::Error(err)) => return Err(err.into()), + Ok(()) => { + reap_dead(dir, stem, &lock)?; + return Ok(None); + } + } + } + if json_path.exists() || dir.join(format!("{stem}.sock")).exists() { + clear_payload(dir, stem, None)?; + } else { + remove_tmps(dir, stem)?; + } + Ok(None) +} + +fn read_live(json_path: &Path) -> Result, LinkError> { + if !json_path.is_file() { + return Ok(None); + } + let bytes = std::fs::read(json_path)?; + let record: InstanceRecord = match serde_json::from_slice(&bytes) { + Ok(record) => record, + Err(_) => return Ok(None), + }; + if record.validate().is_err() { + return Ok(None); + } + Ok(Some(record)) +} + +fn reap_dead(dir: &Path, stem: &str, lock: &File) -> Result<(), LinkError> { + clear_payload(dir, stem, None)?; + let lock_path = dir.join(format!("{stem}.lock")); + match std::fs::remove_file(&lock_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + lock.unlock()?; + Ok(()) +} + +fn unix_path(record: &InstanceRecord) -> Option { + match &record.endpoint { + Endpoint::Unix { path } => Some(PathBuf::from(path)), + Endpoint::Pipe { .. } | Endpoint::Tcp { .. } => None, + } +} + +/// Remove the json file, temp files, and socket files. `keep` is the socket the +/// caller just bound and is about to publish. Deleting that name would unlist +/// a live listener. +fn clear_payload(dir: &Path, stem: &str, keep: Option<&Path>) -> Result<(), LinkError> { + let json_path = dir.join(format!("{stem}.json")); + if let Ok(bytes) = std::fs::read(&json_path) + && let Ok(record) = serde_json::from_slice::(&bytes) + && let Some(old) = unix_path(&record) + && keep.is_none_or(|path| path != old.as_path()) + { + remove_socket_inside(dir, &old); + } + let conventional = dir.join(format!("{stem}.sock")); + if keep.is_none_or(|path| path != conventional.as_path()) { + remove_socket_inside(dir, &conventional); + } + match std::fs::remove_file(&json_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + remove_tmps(dir, stem) +} + +fn remove_tmps(dir: &Path, stem: &str) -> Result<(), LinkError> { + let prefix = format!(".{stem}."); + let entries = std::fs::read_dir(dir)?; + for entry in entries { + let entry = entry?; + let Some(name) = entry.file_name().into_string().ok() else { continue }; + if name.starts_with(&prefix) && name.ends_with(".tmp") { + std::fs::remove_file(entry.path())?; + } + } + Ok(()) +} + +#[cfg(unix)] +fn remove_socket_inside(runtime: &Path, candidate: &Path) { + use std::os::unix::fs::FileTypeExt; + let Ok(meta) = std::fs::symlink_metadata(candidate) else { return }; + if !meta.file_type().is_socket() { + return; + } + let Some(parent) = candidate.parent() else { return }; + let Ok(parent_real) = std::fs::canonicalize(parent) else { return }; + let Ok(runtime_real) = std::fs::canonicalize(runtime) else { return }; + if parent_real == runtime_real { + let _ = std::fs::remove_file(candidate); + } +} + +/// Only Unix endpoints leave socket files behind. A named pipe has no file to remove. +#[cfg(not(unix))] +fn remove_socket_inside(runtime: &Path, candidate: &Path) { + let _ = (runtime, candidate); +} + +/// `.-..tmp`, then rename onto `-.json`. +fn write_atomic(dir: &Path, stem: &str, bytes: &[u8]) -> Result<(), LinkError> { + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let tmp = dir.join(format!(".{stem}.{}.tmp", auth::encode_hex(&rand))); + let final_path = dir.join(format!("{stem}.json")); + let mut opts = OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut file = opts.open(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + if let Err(err) = std::fs::rename(&tmp, &final_path) { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(label: &str) -> (PathBuf, RuntimeDir) { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + let dir = RuntimeDir::at(&path).unwrap(); + (path, dir) + } + + #[test] + fn schema_accepts_a_record_and_rejects_a_bad_token() { + let (_path, dir) = scratch("schema"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let mut value = serde_json::to_value(&record).unwrap(); + value["note"] = Value::String("kept".into()); + let restored: InstanceRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(restored.extra.get("note").and_then(Value::as_str), Some("kept")); + let schema_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schemas/instance-record.schema.json"); + let schema: Value = serde_json::from_slice(&std::fs::read(&schema_path).unwrap()).unwrap(); + let validator = jsonschema::options().should_validate_formats(true).build(&schema).unwrap(); + let errors: Vec<_> = validator.iter_errors(&value).map(|err| err.to_string()).collect(); + assert!(errors.is_empty(), "{errors:?}"); + value["token"] = Value::String("nope".into()); + assert!(!validator.is_valid(&value)); + let _ = std::fs::remove_dir_all(dir.path()); + } + + #[test] + fn scan_keeps_a_held_record_and_drops_it_after_unlock() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let (path, dir) = scratch("scan"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let decoy = dir.socket_file("effectcraft", 4242); + std::fs::write(&decoy, b"not a socket").unwrap(); + let held = Discovery::publish(&dir, record.clone()).unwrap(); + let live = Discovery::scan(&dir).unwrap(); + assert_eq!(live.len(), 1); + assert_eq!(live[0].instance, record.instance); + assert_eq!(live[0].token, record.token); + drop(held); + assert!(Discovery::scan(&dir).unwrap().is_empty()); + assert!(!dir.record_file("effectcraft", 4242).exists()); + assert!(!dir.lock_file("effectcraft", 4242).exists()); + assert!(decoy.exists(), "scan must not delete a regular file"); + let _ = std::fs::remove_dir_all(&path); + } + + #[cfg(unix)] + #[test] + fn scan_unlinks_a_dead_socket() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + use interprocess::local_socket::prelude::*; + use interprocess::local_socket::{GenericFilePath, ListenerOptions}; + + let (path, dir) = scratch("sock"); + let endpoint = dir.unix_endpoint("filmcraft", 5151).unwrap(); + let sock = match &endpoint { + Endpoint::Unix { path } => path.clone(), + Endpoint::Pipe { .. } | Endpoint::Tcp { .. } => panic!("unix endpoint"), + }; + let name = Path::new(&sock).to_fs_name::().unwrap(); + let listener = ListenerOptions::new().name(name).reclaim_name(false).create_sync().unwrap(); + crate::discovery::enforce_mode(Path::new(&sock), 0o600).unwrap(); + let record = InstanceRecord::new("filmcraft", "0.9.1", 5151, endpoint).unwrap(); + let held = Discovery::publish(&dir, record).unwrap(); + assert!(Path::new(&sock).exists(), "publish must leave the bound socket in place"); + drop(held); + assert!(Path::new(&sock).exists(), "dropping the lock must not unlink the socket"); + assert!(Discovery::scan(&dir).unwrap().is_empty()); + assert!(!Path::new(&sock).exists(), "scan removes the socket once the lock is free"); + drop(listener); + let _ = std::fs::remove_dir_all(&path); + } + + #[test] + fn second_publish_of_a_live_stem_is_busy() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let (path, dir) = scratch("busy"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let first = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint.clone()).unwrap(); + let held = Discovery::publish(&dir, first).unwrap(); + let second = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let err = Discovery::publish(&dir, second).unwrap_err(); + assert!(matches!(err, LinkError::Busy(_)), "{err}"); + drop(held); + let _ = std::fs::remove_dir_all(&path); + } +} diff --git a/crates/ipc/src/error.rs b/crates/ipc/src/error.rs new file mode 100644 index 000000000..65259fd58 --- /dev/null +++ b/crates/ipc/src/error.rs @@ -0,0 +1,134 @@ +//! Failures a Craft Link caller can show or turn into a protocol error object. + +use serde::Serialize; +use serde_json::{Map, Value}; + +/// Wire error object for new methods. Existing methods keep a string `error`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ErrorObject { + pub code: String, + pub message: String, + pub data: Map, +} + +/// A link failure. The first-line auth refusal is still the string +/// `authentication required`. [`LinkError::to_object`] is for new methods. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LinkError { + Io(String), + /// The runtime directory or socket mode is not acceptable. + Refused { + path: String, + reason: String, + }, + Invalid(String), + Busy(String), + Auth, + Protocol { + server_min: u32, + server_max: u32, + }, + BadParams(String), + NotInstalled { + app: String, + }, + Mismatch { + field: String, + expected: String, + got: String, + }, + Closed, + Timeout, + TooLarge, + NotReady(String), + /// An asset moniker, link id or file that the peer does not know. + Unknown(String), + /// A `link.open` payload with a bad shape or an unknown `format`. + PayloadInvalid(String), + /// A `link.open` payload names media files that do not exist or cannot be opened. The + /// sender renders or relinks them and sends the call again. + MediaMissing { + paths: Vec, + }, +} + +impl LinkError { + pub fn io(err: impl std::fmt::Display) -> Self { + Self::Io(err.to_string()) + } + + /// Contract section 7 code. + pub fn code(&self) -> &'static str { + match self { + Self::Auth => "auth_required", + Self::Protocol { .. } => "protocol_unsupported", + Self::BadParams(_) | Self::Invalid(_) => "bad_params", + Self::NotReady(_) => "not_ready", + Self::Unknown(_) => "link_unknown", + Self::PayloadInvalid(_) => "payload_invalid", + Self::MediaMissing { .. } => "media_missing", + Self::TooLarge => "payload_too_large", + Self::Io(_) => "io_error", + Self::Busy(_) => "busy", + Self::NotInstalled { .. } => "not_ready", + Self::Refused { .. } | Self::Mismatch { .. } | Self::Closed | Self::Timeout => "internal", + } + } + + pub fn to_object(&self) -> ErrorObject { + let mut data = Map::new(); + if let Self::Protocol { server_min, server_max } = self { + data.insert("server_min".into(), Value::from(*server_min)); + data.insert("server_max".into(), Value::from(*server_max)); + } + if let Self::MediaMissing { paths } = self { + data.insert("missing".into(), Value::from(paths.clone())); + } + ErrorObject { code: self.code().to_string(), message: self.to_string(), data } + } +} + +impl std::fmt::Display for LinkError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(msg) => write!(f, "{msg}"), + Self::Refused { path, reason } => write!(f, "{path}: {reason}"), + Self::Invalid(msg) => write!(f, "{msg}"), + Self::Busy(msg) => write!(f, "{msg}"), + Self::Auth => write!(f, "authentication required"), + Self::Protocol { server_min, server_max } => { + write!(f, "protocol not supported (server {server_min} to {server_max})") + } + Self::BadParams(msg) => write!(f, "{msg}"), + Self::NotInstalled { app } => write!(f, "{app} is not installed"), + Self::Mismatch { field, expected, got } => { + write!(f, "{field} is {got}, record says {expected}") + } + Self::Closed => write!(f, "connection closed"), + Self::Timeout => write!(f, "timeout"), + Self::TooLarge => write!(f, "payload too large"), + Self::NotReady(msg) => write!(f, "{msg}"), + Self::Unknown(msg) | Self::PayloadInvalid(msg) => write!(f, "{msg}"), + Self::MediaMissing { paths } => write!(f, "the payload names media that cannot be opened: {}", paths.join(", ")), + } + } +} + +impl std::error::Error for LinkError {} + +impl From for LinkError { + /// A socket read timeout is `WouldBlock` on Linux ("Resource temporarily unavailable") and + /// `TimedOut` elsewhere. Both mean the peer was quiet, not that the connection broke. + fn from(err: std::io::Error) -> Self { + match err.kind() { + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut => Self::Timeout, + _ => Self::Io(err.to_string()), + } + } +} + +impl From for LinkError { + fn from(err: serde_json::Error) -> Self { + Self::Invalid(err.to_string()) + } +} diff --git a/crates/ipc/src/fetch.rs b/crates/ipc/src/fetch.rs new file mode 100644 index 000000000..ca3f4d194 --- /dev/null +++ b/crates/ipc/src/fetch.rs @@ -0,0 +1,370 @@ +//! Pull frames from a peer without blocking the caller past a deadline. +//! +//! [`FrameFetcher`] owns the control connection that holds the `frames.open` ticket and a few +//! bulk connections ([`crate::bulk::BulkClient`]), each on its own worker thread. A caller asks +//! with [`FrameFetcher::fetch`] and a time limit. +//! +//! - A frame in the RAM cache answers at once. The cache keeps the newest epoch of each +//! [`FrameKey`] inside a byte budget and drops the least recently used frame first. +//! - A miss goes to a worker. Asks for one key share one render. +//! - A caller that runs out of time gets [`Fetched::TimedOut`]. The render goes on, and its frame +//! lands in the cache for the next ask. The caller never waits for the socket. +//! - A queued ask older than [`FetchOptions::max_age`] is dropped when a worker reaches it, unless +//! a caller is still waiting for it. The queue holds at most [`QUEUE_MAX`] asks: a new miss past +//! that drops the oldest. +//! - A connection error ends that worker. When the last worker ends the fetcher is down and every +//! ask that misses the cache returns [`Fetched::Down`] at once. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use crate::InstanceRecord; +use crate::bulk::{BulkClient, Frame, FrameFormat, FrameTicket}; +use crate::client::Client; +use crate::error::LinkError; + +/// Most bulk connections one fetcher opens. +pub const CONNECTIONS_MAX: usize = 8; + +/// Most asks that wait for a worker at once. A new miss past this drops the oldest queued ask. +pub const QUEUE_MAX: usize = 256; + +/// What identifies a frame for the cache: the caller's frame number and scale (in thousandths). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrameKey { + pub frame: i64, + pub scale_milli: u32, +} + +/// Tuning of a [`FrameFetcher`]. +#[derive(Debug, Clone)] +pub struct FetchOptions { + /// Bulk connections, and so renders in flight. Clamped to `1..=`[`CONNECTIONS_MAX`]. + pub connections: usize, + /// How long one socket read or write may take before the connection is dropped. + pub io_timeout: Duration, + /// Cache size in bytes of pixel data. + pub cache_bytes: usize, + /// A queued ask older than this is skipped, unless a caller is still waiting for it. + pub max_age: Duration, +} + +impl Default for FetchOptions { + fn default() -> Self { + Self { connections: 3, io_timeout: Duration::from_secs(5), cache_bytes: 256 * 1024 * 1024, max_age: Duration::from_secs(1) } + } +} + +/// The answer to one [`FrameFetcher::fetch`]. +#[derive(Debug, Clone)] +pub enum Fetched { + /// A frame whose epoch is at least the one asked for. + Frame(Arc), + /// The time limit passed. The render may still finish and fill the cache. + TimedOut, + /// The peer answered this frame with an error. The fetcher still works. + /// + /// Also the answer to a caller whose ask was dropped from a full queue (see [`QUEUE_MAX`]). That + /// caller is not re-queued: it asks again if it still wants the frame. + Failed(String), + /// The connections are gone and the cache has no answer. + Down, +} + +/// Counters since [`FrameFetcher::connect`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FetchStats { + /// Calls to [`FrameFetcher::fetch`]. + pub requests: u64, + /// Calls answered by the cache on the first look. + pub hits: u64, + /// Frames a worker received. + pub renders: u64, + pub timeouts: u64, + pub failures: u64, + /// Queued asks dropped for age or for a full queue. + pub skipped: u64, +} + +struct Job { + key: FrameKey, + time_s: f64, + epoch: u64, + queued: Instant, + /// The latest deadline of a caller that is waiting for this job. + wanted_until: Instant, +} + +struct Cached { + frame: Arc, + used: u64, +} + +#[derive(Default)] +struct Cache { + frames: HashMap, + bytes: usize, + tick: u64, +} + +impl Cache { + fn get(&mut self, key: FrameKey, min_epoch: u64) -> Option> { + self.tick += 1; + let tick = self.tick; + let entry = self.frames.get_mut(&key).filter(|entry| entry.frame.epoch >= min_epoch)?; + entry.used = tick; + Some(Arc::clone(&entry.frame)) + } + + fn insert(&mut self, key: FrameKey, frame: Arc, budget: usize) { + if self.frames.get(&key).is_some_and(|old| old.frame.epoch > frame.epoch) { + return; + } + self.tick += 1; + self.bytes += frame.pixels.len(); + if let Some(old) = self.frames.insert(key, Cached { frame, used: self.tick }) { + self.bytes -= old.frame.pixels.len(); + } + while self.bytes > budget && self.frames.len() > 1 { + let Some(oldest) = self.frames.iter().min_by_key(|(_, entry)| entry.used).map(|(key, _)| *key) else { break }; + if let Some(gone) = self.frames.remove(&oldest) { + self.bytes -= gone.frame.pixels.len(); + } + } + } +} + +#[derive(Default)] +struct State { + queue: VecDeque, + inflight: HashSet, + cache: Cache, + failed: HashMap, + fail_seq: u64, + stats: FetchStats, + stop: bool, +} + +/// Most failure records kept. Callers only look for a record newer than their own ask, so old +/// ones go first. +const FAILED_MAX: usize = 4 * QUEUE_MAX; + +impl State { + /// Record a failure of `key` that callers waiting for it will see. + fn note_failed(&mut self, key: FrameKey, message: String) { + self.fail_seq += 1; + self.failed.insert(key, (self.fail_seq, message)); + if self.failed.len() > FAILED_MAX { + let keep_from = self.fail_seq.saturating_sub(u64::try_from(FAILED_MAX).unwrap_or(u64::MAX)); + self.failed.retain(|_, (seq, _)| *seq > keep_from); + } + } +} + +struct Shared { + state: Mutex, + /// Workers wait here for a job. + work: Condvar, + /// Callers wait here for a frame, a failure or the end. + done: Condvar, + alive: AtomicUsize, + options: FetchOptions, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +/// Frames pulled from one link of a peer. See the module docs. +pub struct FrameFetcher { + shared: Arc, + ticket: FrameTicket, + /// Holds the ticket: the peer drops it when this connection closes. + control: Mutex, +} + +impl FrameFetcher { + /// Open the ticket and the bulk connections. Any failure is returned here, so a caller knows + /// the peer can serve frames before it relies on them. + pub fn connect(record: &InstanceRecord, link_id: &str, format: FrameFormat, scale: f64, options: FetchOptions) -> Result { + let mut control = Client::connect(record)?; + let ticket = control.frames_open(link_id, format, scale)?; + let connections = options.connections.clamp(1, CONNECTIONS_MAX); + let mut clients = Vec::with_capacity(connections); + for _ in 0..connections { + let mut client = BulkClient::attach(record, &ticket.ticket)?; + client.set_io_timeout(options.io_timeout); + clients.push(client); + } + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + work: Condvar::new(), + done: Condvar::new(), + alive: AtomicUsize::new(clients.len()), + options, + }); + for client in clients { + let worker = Arc::clone(&shared); + // Workers are not joined: one may sit in a socket read for up to `io_timeout`, and a + // drop on a UI thread must not wait for that. A worker sees `stop` and leaves. + if let Err(err) = std::thread::Builder::new().name("craft-ipc-fetch".into()).spawn(move || run_worker(&worker, client)) { + shared_stop(&shared); + return Err(err.into()); + } + } + Ok(Self { shared, ticket, control: Mutex::new(control) }) + } + + /// Size of the frames this fetcher yields. + pub fn layout(&self) -> FrameTicket { + self.ticket.clone() + } + + /// The cached frame for `key` if its epoch is at least `min_epoch`. Never asks the peer. + pub fn cached(&self, key: FrameKey, min_epoch: u64) -> Option> { + self.shared.lock().cache.get(key, min_epoch) + } + + /// The frame for `key` at an epoch of at least `min_epoch`, waiting at most `wait`. + /// `time_s` is the comp time the peer renders. + /// + /// A `time_s` that is not finite is answered with [`Fetched::Failed`] and is never queued: the + /// request would fail before it reaches the socket, and a worker reads that as a dead connection. + pub fn fetch(&self, key: FrameKey, time_s: f64, min_epoch: u64, wait: Duration) -> Fetched { + if !time_s.is_finite() { + return Fetched::Failed("time_s must be finite".into()); + } + let deadline = Instant::now() + wait; + let shared = &self.shared; + let mut state = shared.lock(); + state.stats.requests += 1; + let started_at = state.fail_seq; + let mut first = true; + loop { + if let Some(frame) = state.cache.get(key, min_epoch) { + if first { + state.stats.hits += 1; + } + return Fetched::Frame(frame); + } + first = false; + if let Some((seq, message)) = state.failed.get(&key) + && *seq > started_at + { + return Fetched::Failed(message.clone()); + } + if shared.alive.load(Ordering::SeqCst) == 0 || state.stop { + return Fetched::Down; + } + if state.inflight.contains(&key) { + // Still waiting for a worker: this caller keeps the job alive until its own deadline. + if let Some(job) = state.queue.iter_mut().find(|job| job.key == key) { + job.wanted_until = job.wanted_until.max(deadline); + } + } else { + if state.queue.len() >= QUEUE_MAX + && let Some(oldest) = state.queue.pop_front() + { + state.inflight.remove(&oldest.key); + state.stats.skipped += 1; + // Callers waiting for the dropped key see this before they could queue it again. + state.note_failed(oldest.key, "dropped: the fetch queue is full".into()); + shared.done.notify_all(); + } + state.inflight.insert(key); + state.queue.push_back(Job { key, time_s, epoch: min_epoch, queued: Instant::now(), wanted_until: deadline }); + shared.work.notify_one(); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + state.stats.timeouts += 1; + return Fetched::TimedOut; + } + state = shared.done.wait_timeout(state, left).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } + + /// True once every connection has ended. + pub fn is_down(&self) -> bool { + self.shared.alive.load(Ordering::SeqCst) == 0 + } + + pub fn stats(&self) -> FetchStats { + self.shared.lock().stats + } + + /// Ask the peer to close the ticket now instead of at drop. + pub fn close(&self) { + shared_stop(&self.shared); + let mut control = self.control.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _ = control.call("link.bye", serde_json::json!({})); + } +} + +impl Drop for FrameFetcher { + fn drop(&mut self) { + shared_stop(&self.shared); + } +} + +fn shared_stop(shared: &Shared) { + shared.lock().stop = true; + shared.work.notify_all(); + shared.done.notify_all(); +} + +fn run_worker(shared: &Shared, mut client: BulkClient) { + while let Some(job) = next_job(shared) { + let reply = client.request_reply(job.time_s, job.epoch, None); + let mut state = shared.lock(); + state.inflight.remove(&job.key); + match reply { + Ok(Ok(frame)) => { + state.stats.renders += 1; + state.failed.remove(&job.key); + let budget = shared.options.cache_bytes; + state.cache.insert(job.key, Arc::new(frame), budget); + } + Ok(Err(message)) => { + state.stats.failures += 1; + state.note_failed(job.key, message); + } + Err(_) => { + drop(state); + break; + } + } + drop(state); + shared.done.notify_all(); + } + if shared.alive.fetch_sub(1, Ordering::SeqCst) == 1 { + let mut state = shared.lock(); + state.queue.clear(); + state.inflight.clear(); + } + shared.done.notify_all(); +} + +/// The oldest queued job that is still worth rendering. `None` when the fetcher stops. +fn next_job(shared: &Shared) -> Option { + let mut state = shared.lock(); + loop { + if state.stop { + return None; + } + while let Some(job) = state.queue.pop_front() { + if job.queued.elapsed() > shared.options.max_age && job.wanted_until <= Instant::now() { + state.inflight.remove(&job.key); + state.stats.skipped += 1; + shared.done.notify_all(); + continue; + } + return Some(job); + } + state = shared.work.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } +} diff --git a/crates/ipc/src/fsopen.rs b/crates/ipc/src/fsopen.rs new file mode 100644 index 000000000..ec1316f68 --- /dev/null +++ b/crates/ipc/src/fsopen.rs @@ -0,0 +1,25 @@ +//! Read-only opens that no file type at the path can stall. +//! +//! `File::open` on a FIFO with no writer blocks until a writer appears. A path that was checked +//! and then replaced gives the same wait, so a type check before the open does not help. These +//! opens ask the kernel not to wait, and the caller checks the type on the open handle. + +use std::fs::File; +use std::path::Path; + +/// Open `path` for reading without waiting on a FIFO. On Unix `follow` false also refuses a +/// symlink in the last component (the error is not specific, so a caller that must tell a symlink +/// apart looks at `symlink_metadata` after a failed open). Other platforms open as `File::open`. +pub(crate) fn open_read(path: &Path, follow: bool) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + let flags = if follow { libc::O_NONBLOCK } else { libc::O_NONBLOCK | libc::O_NOFOLLOW }; + std::fs::OpenOptions::new().read(true).custom_flags(flags).open(path) + } + #[cfg(not(unix))] + { + let _ = follow; + File::open(path) + } +} diff --git a/crates/ipc/src/ids.rs b/crates/ipc/src/ids.rs new file mode 100644 index 000000000..c2ed1abd4 --- /dev/null +++ b/crates/ipc/src/ids.rs @@ -0,0 +1,41 @@ +//! Id checks shared by the instance record, the moniker and the link record. +//! `std` only, so the pure modules can use it. + +/// App id: `^[a-z][a-z0-9-]{1,31}$`. +pub fn valid_app_id(app: &str) -> bool { + let bytes = app.as_bytes(); + (2..=32).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_lowercase) + && bytes.iter().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') +} + +/// UUID text in 8-4-4-4-12 form. Either hex case. +pub fn valid_uuid(text: &str) -> bool { + let bytes = text.as_bytes(); + bytes.len() == 36 + && bytes.iter().enumerate().all(|(index, byte)| if matches!(index, 8 | 13 | 18 | 23) { *byte == b'-' } else { byte.is_ascii_hexdigit() }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn app_ids() { + for ok in ["ec", "filmcraft", "a-b-1", &"a".repeat(32)] { + assert!(valid_app_id(ok), "{ok}"); + } + for bad in ["", "a", "A1", "1a", "a_b", "Film", &"a".repeat(33)] { + assert!(!valid_app_id(bad), "{bad}"); + } + } + + #[test] + fn uuids() { + assert!(valid_uuid("550e8400-e29b-41d4-a716-446655440000")); + assert!(valid_uuid("550E8400-E29B-41D4-A716-446655440000")); + assert!(!valid_uuid("550e8400e29b41d4a716446655440000")); + assert!(!valid_uuid("550e8400-e29b-41d4-a716-44665544000g")); + assert!(!valid_uuid("")); + } +} diff --git a/crates/ipc/src/launch.rs b/crates/ipc/src/launch.rs new file mode 100644 index 000000000..73365d30f --- /dev/null +++ b/crates/ipc/src/launch.rs @@ -0,0 +1,777 @@ +//! Find and start a Craft app. +//! +//! `resolve` reads the process environment. The decision itself is +//! `resolve_with`, which takes an owned env map, a home directory, and the +//! candidate directories. Tests call that function and do not change the +//! process environment. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use crate::client::{Client, HelloClient}; +use crate::error::LinkError; +use crate::trust; +use crate::{Discovery, InstanceRecord, RuntimeDir}; + +const RECORD_WAIT: Duration = Duration::from_secs(15); +const RECORD_POLL: Duration = Duration::from_millis(50); + +/// Progress reported by [`launch_at`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + Spawned, + RecordSeen, + HelloOk, +} + +/// Find the binary for `app` from the process environment. +pub fn resolve(app: &str) -> Result { + let env = std::env::vars().collect::>(); + let home = std::env::var_os("HOME").map(PathBuf::from); + let exe_dir = std::env::current_exe().ok().and_then(|path| path.parent().map(Path::to_path_buf)); + let search = default_search_dirs(home.as_deref()); + resolve_with(app, &env, home.as_deref(), exe_dir.as_deref(), &search) +} + +/// [`resolve`] then [`launch_at`] with no args and the real runtime directory. +pub fn launch(app: &str, extra_env: &[(&str, &str)]) -> Result { + let bin = resolve(app)?; + launch_at(app, &bin, &[], extra_env, None, &mut |_| {}) +} + +/// Spawn `bin` and wait until its discovery record answers `link.hello`. +/// +/// `extra_env` is applied with [`Command::env`]. This function does not mint +/// an activation token. `XDG_ACTIVATION_TOKEN` and `DESKTOP_STARTUP_ID` are +/// passed through when the caller puts them in `extra_env`. +/// +/// `runtime` `None` scans [`RuntimeDir::resolve`]. The returned record is one +/// this function observed for the spawned pid. +pub fn launch_at( + app: &str, + bin: &Path, + args: &[OsString], + extra_env: &[(&str, &str)], + runtime: Option<&RuntimeDir>, + on_phase: &mut dyn FnMut(Phase), +) -> Result { + // `resolve` already checked `bin`. The caller may pass any path, and time has passed. + trust::check_executable(bin)?; + let mut cmd = Command::new(bin); + cmd.args(args); + for (key, value) in extra_env { + cmd.env(key, value); + } + // A new process group. The child is not in this process's group. + // The waiter thread reaps it. kill_on_drop stays off. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let mut child = cmd.spawn().map_err(LinkError::io)?; + let pid = child.id(); + std::thread::Builder::new() + .name("craft-ipc-wait".into()) + .spawn(move || { + let _ = child.wait(); + }) + .map_err(LinkError::io)?; + on_phase(Phase::Spawned); + if let Some(dir) = runtime { + finish_launch(app, pid, dir, on_phase) + } else { + let owned = RuntimeDir::resolve()?; + finish_launch(app, pid, &owned, on_phase) + } +} + +fn finish_launch(app: &str, pid: u32, dir: &RuntimeDir, on_phase: &mut dyn FnMut(Phase)) -> Result { + let record = wait_for_record(dir, app, pid)?; + on_phase(Phase::RecordSeen); + greet(&record, app)?; + on_phase(Phase::HelloOk); + Ok(record) +} + +fn wait_for_record(dir: &RuntimeDir, app: &str, pid: u32) -> Result { + let deadline = Instant::now() + RECORD_WAIT; + loop { + let records = Discovery::scan(dir)?; + if let Some(record) = records.into_iter().find(|record| record.app == app && record.pid == pid) { + return Ok(record); + } + let now = Instant::now(); + if now >= deadline { + return Err(LinkError::Timeout); + } + std::thread::sleep(RECORD_POLL.min(deadline.saturating_duration_since(now))); + } +} + +fn greet(record: &InstanceRecord, app: &str) -> Result<(), LinkError> { + let mut client = Client::connect(record).map_err(as_not_ready)?; + let identity = HelloClient { + app: app.to_string(), + app_version: "launch".into(), + pid: std::process::id(), + instance: crate::random_uuid().map_err(as_not_ready)?, + protocol_min: crate::wire::PROTOCOL_MIN, + protocol_max: crate::wire::PROTOCOL_MAX, + }; + client.hello(&identity).map_err(as_not_ready)?; + Ok(()) +} + +fn as_not_ready(err: LinkError) -> LinkError { + match err { + LinkError::Timeout => LinkError::Timeout, + LinkError::NotReady(_) => err, + other => LinkError::NotReady(other.to_string()), + } +} + +fn resolve_with( + app: &str, + env: &BTreeMap, + home: Option<&Path>, + exe_dir: Option<&Path>, + search_dirs: &[PathBuf], +) -> Result { + if let Some(found) = env_override(app, env) { + return found; + } + if let Some(manifest) = manifest_path(env, home, exe_dir) + && let Some(bin) = from_manifest(app, &manifest) + { + return Ok(bin); + } + if let Some(bin) = from_path_env(app, env) { + return Ok(bin); + } + for dir in search_dirs { + if let Some(bin) = search_dir(app, dir) { + return Ok(bin); + } + } + if let Some(bin) = platform_bundle(app, home) { + return Ok(bin); + } + Err(LinkError::NotInstalled { app: app.to_string() }) +} + +/// `CRAFT__BIN`. Set means authoritative, including a missing file. +fn env_override(app: &str, env: &BTreeMap) -> Option> { + let value = env.get(&bin_env_key(app))?; + let path = PathBuf::from(value); + if !path.is_file() { + return Some(Err(LinkError::NotInstalled { app: app.to_string() })); + } + Some(trust::check_executable(&path).map(|()| path)) +} + +fn bin_env_key(app: &str) -> String { + let mut key = String::from("CRAFT_"); + for byte in app.bytes() { + if byte.is_ascii_alphanumeric() { + key.push(byte.to_ascii_uppercase() as char); + } else { + key.push('_'); + } + } + key.push_str("_BIN"); + key +} + +fn manifest_path(env: &BTreeMap, home: Option<&Path>, exe_dir: Option<&Path>) -> Option { + if let Some(dir) = exe_dir + && dir.join("portable.txt").is_file() + { + return Some(dir.join("ArtCraftLauncherData").join("data").join("installed.json")); + } + Some(data_dir(env, home)?.join(launcher_dir_name()).join("installed.json")) +} + +fn launcher_dir_name() -> &'static str { + if cfg!(any(target_os = "macos", windows)) { "ArtCraft Launcher" } else { "artcraft-launcher" } +} + +fn data_dir(env: &BTreeMap, home: Option<&Path>) -> Option { + #[cfg(target_os = "linux")] + { + linux_data_dir(env, home) + } + #[cfg(target_os = "macos")] + { + let _ = env; + home.map(|dir| dir.join("Library").join("Application Support")) + } + #[cfg(windows)] + { + let _ = home; + env.get("LOCALAPPDATA").map(PathBuf::from).filter(|path| path.is_absolute()) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + let _ = (env, home); + None + } +} + +#[cfg(target_os = "linux")] +fn linux_data_dir(env: &BTreeMap, home: Option<&Path>) -> Option { + if let Some(xdg) = env.get("XDG_DATA_HOME") { + let path = PathBuf::from(xdg); + if path.is_absolute() { + return Some(path); + } + } + home.map(|dir| dir.join(".local").join("share")) +} + +/// Only `path` is required. An unknown `kind` is ignored. A path that is not a file is skipped. +fn from_manifest(app: &str, path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + let text = value.get("apps")?.get(app)?.get("path")?.as_str()?; + let bin = PathBuf::from(text); + trust::check_executable(&bin).is_ok().then_some(bin) +} + +/// A `PATH` entry that is empty or relative names the current directory, so it is skipped. +fn from_path_env(app: &str, env: &BTreeMap) -> Option { + let path = env.get("PATH")?; + for dir in std::env::split_paths(path) { + if !dir.is_absolute() { + continue; + } + let candidate = dir.join(app); + if trust::check_executable(&candidate).is_ok() { + return Some(candidate); + } + } + None +} + +fn default_search_dirs(home: Option<&Path>) -> Vec { + #[cfg(target_os = "linux")] + { + let mut dirs = Vec::new(); + if let Some(home) = home { + dirs.push(home.join("Applications")); + dirs.push(home.join("AppImages")); + dirs.push(home.join(".local").join("bin")); + } + dirs.push(PathBuf::from("/usr/bin")); + dirs.push(PathBuf::from("/usr/local/bin")); + dirs + } + #[cfg(not(target_os = "linux"))] + { + let _ = home; + Vec::new() + } +} + +fn search_dir(app: &str, dir: &Path) -> Option { + let exact = dir.join(app); + if trust::check_executable(&exact).is_ok() { + return Some(exact); + } + let entries = std::fs::read_dir(dir).ok()?; + let mut names = Vec::new(); + for entry in entries { + let Ok(entry) = entry else { continue }; + let Ok(name) = entry.file_name().into_string() else { continue }; + if is_appimage(app, &name) { + names.push(name); + } + } + names.sort(); + for name in names { + let path = dir.join(name); + if trust::check_executable(&path).is_ok() { + return Some(path); + } + } + None +} + +/// `{app}.appimage` in any ASCII case, or a name that starts with `{app}-` and ends with `.AppImage`. +fn is_appimage(app: &str, name: &str) -> bool { + const EXT: &str = ".appimage"; + let Some(split) = name.len().checked_sub(EXT.len()) else { + return false; + }; + let Some(ext) = name.get(split..) else { + return false; + }; + let Some(base) = name.get(..split) else { + return false; + }; + if !ext.eq_ignore_ascii_case(EXT) { + return false; + } + if base.eq_ignore_ascii_case(app) { + return true; + } + let mut prefix = String::with_capacity(app.len() + 1); + prefix.push_str(app); + prefix.push('-'); + name.starts_with(&prefix) +} + +/// An id that ends with `craft` becomes `PrefixCraft` (`filmcraft` to `FilmCraft`). +/// Used by the macOS bundle search. Linux resolves AppImages without this name. +#[cfg_attr(not(any(test, target_os = "macos")), allow(dead_code))] +fn display_name(app: &str) -> String { + if let Some(prefix) = app.strip_suffix("craft") { + let mut name = String::new(); + let mut chars = prefix.chars(); + if let Some(first) = chars.next() { + name.push(first.to_ascii_uppercase()); + name.push_str(chars.as_str()); + } + name.push_str("Craft"); + return name; + } + let mut chars = app.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + let mut name = String::new(); + name.push(first.to_ascii_uppercase()); + name.push_str(chars.as_str()); + name +} + +/// macOS bundles. Windows uninstall-registry lookup is not implemented (no winreg). +/// PATH, `CRAFT__BIN`, and `installed.json` still apply on Windows. +#[cfg(target_os = "macos")] +fn platform_bundle(app: &str, home: Option<&Path>) -> Option { + let bundle = format!("{}.app", display_name(app)); + let mut candidates = vec![PathBuf::from("/Applications").join(&bundle)]; + if let Some(home) = home { + candidates.push(home.join("Applications").join(&bundle)); + } + for bundle_path in candidates { + if !bundle_path.is_dir() { + continue; + } + let exe = bundle_path.join("Contents").join("MacOS").join(app); + if trust::check_executable(&exe).is_ok() { + return Some(exe); + } + } + None +} + +#[cfg(not(target_os = "macos"))] +fn platform_bundle(_app: &str, _home: Option<&Path>) -> Option { + None +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::collections::BTreeMap; + use std::path::{Path, PathBuf}; + + use super::*; + + struct Tmp { + path: PathBuf, + } + + impl Tmp { + fn new(label: &str) -> Self { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } + } + + fn touch(path: &Path) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, b"bin").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + + /// Mode of the directory that holds `path`, for the trust tests. + #[cfg(unix)] + fn chmod(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + #[test] + fn relative_and_empty_path_entries_are_skipped() { + let root = Tmp::new("relpath"); + let cwd = std::env::current_dir().unwrap(); + // A program in the current directory must not be found through "" or ".". + let here = cwd.join(format!("effectcraft-relpath-{}", std::process::id())); + touch(&here); + let name = here.file_name().unwrap().to_str().unwrap().to_string(); + let good_dir = root.path().join("bin"); + for path in ["", ".", "./", "bin:", ":bin"] { + let env = env_map(&[("PATH", path)]); + let err = super::resolve_with(&name, &env, None, None, &[]); + assert!(err.is_err(), "PATH={path:?} found {err:?}"); + } + // An absolute entry later in the list still works. + touch(&good_dir.join(&name)); + let path = format!(":.:{}", good_dir.display()); + let env = env_map(&[("PATH", path.as_str())]); + assert_eq!(super::resolve_with(&name, &env, None, None, &[]).unwrap(), good_dir.join(&name)); + std::fs::remove_file(&here).unwrap(); + } + + #[cfg(unix)] + #[test] + fn loose_programs_and_directories_are_not_run() { + let root = Tmp::new("loose"); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let env = env_map(&[("PATH", bin_dir.to_str().unwrap())]); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_ok()); + chmod(&bin_dir, 0o777); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "world-writable directory"); + // The same holds for the override, the manifest and the fixed search directories. + let by_env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", bin.to_str().unwrap())]); + assert!(matches!(super::resolve_with("effectcraft", &by_env, None, None, &[]), Err(LinkError::Refused { .. }))); + assert!(super::resolve_with("effectcraft", &env_map(&[]), None, None, std::slice::from_ref(&bin_dir)).is_err()); + let data = root.path().join("xdg"); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&data)), "effectcraft", &bin); + let by_manifest = data_env(&data); + chmod(&bin_dir, 0o755); + assert_eq!(super::resolve_with("effectcraft", &by_manifest, Some(&home), None, &[]).unwrap(), bin); + chmod(&bin_dir, 0o777); + assert!(super::resolve_with("effectcraft", &by_manifest, Some(&home), None, &[]).is_err()); + chmod(&bin_dir, 0o755); + chmod(&bin, 0o666); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "not executable and world-writable"); + chmod(&bin, 0o644); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "not executable"); + // A relative override is refused, not run from the current directory. + let relative = env_map(&[("CRAFT_EFFECTCRAFT_BIN", "Cargo.toml")]); + assert!(super::resolve_with("effectcraft", &relative, None, None, &[]).is_err()); + // launch_at checks the path too. + let err = launch_at("effectcraft", &bin, &[], &[], None, &mut |_| {}).unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err:?}"); + } + + fn env_map(pairs: &[(&str, &str)]) -> BTreeMap { + pairs.iter().map(|(key, value)| ((*key).to_string(), (*value).to_string())).collect() + } + + // Independent fixture paths for the launcher's documented native install record. + fn installed_manifest(home: &Path, xdg: Option<&Path>) -> PathBuf { + if cfg!(target_os = "macos") { + home.join("Library/Application Support/ArtCraft Launcher/installed.json") + } else if cfg!(windows) { + xdg.unwrap_or(home).join("ArtCraft Launcher/installed.json") + } else { + xdg.map_or_else(|| home.join(".local/share"), Path::to_path_buf).join("artcraft-launcher/installed.json") + } + } + + fn data_env(data: &Path) -> BTreeMap { + let key = if cfg!(windows) { "LOCALAPPDATA" } else { "XDG_DATA_HOME" }; + env_map(&[(key, data.to_str().unwrap())]) + } + + fn write_manifest(path: &Path, app: &str, bin: &Path) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + let body = serde_json::json!({ + "apps": { + app: {"kind": "mystery", "path": bin} + } + }); + std::fs::write(path, serde_json::to_vec(&body).unwrap()).unwrap(); + } + + #[test] + fn bin_env_key_uppercases_and_replaces_other_bytes() { + assert_eq!(super::bin_env_key("effectcraft"), "CRAFT_EFFECTCRAFT_BIN"); + assert_eq!(super::bin_env_key("film-craft"), "CRAFT_FILM_CRAFT_BIN"); + } + + #[test] + fn display_name_for_craft_apps() { + assert_eq!(super::display_name("filmcraft"), "FilmCraft"); + assert_eq!(super::display_name("effectcraft"), "EffectCraft"); + assert_eq!(super::display_name("pdfcraft"), "PdfCraft"); + } + + #[test] + fn appimage_names() { + assert!(super::is_appimage("effectcraft", "effectcraft.AppImage")); + assert!(super::is_appimage("effectcraft", "EffectCraft.appimage")); + assert!(super::is_appimage("effectcraft", "effectcraft-0.8.3-linux-x86_64.AppImage")); + assert!(!super::is_appimage("effectcraft", "effectcraft-0.8.3.deb")); + assert!(!super::is_appimage("effectcraft", "not-effectcraft.AppImage")); + assert!(!super::is_appimage("effectcraft", "effectcraft.AppImage.zsync")); + } + + #[test] + fn env_override_missing_file_does_not_fall_through() { + let root = Tmp::new("miss"); + let missing = root.path().join("nope"); + let real_dir = root.path().join("bin"); + let real = real_dir.join("effectcraft"); + touch(&real); + let env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", missing.to_str().unwrap()), ("PATH", real_dir.to_str().unwrap())]); + let err = super::resolve_with("effectcraft", &env, None, None, &[real_dir]).unwrap_err(); + match err { + LinkError::NotInstalled { app } => assert_eq!(app, "effectcraft"), + other => panic!("expected NotInstalled, got {other}"), + } + } + + #[test] + fn env_override_existing_file_is_that_path() { + let root = Tmp::new("envok"); + let bin = root.path().join("custom"); + touch(&bin); + let env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", bin.to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn installed_json_supplies_the_path() { + let root = Tmp::new("man"); + let data = root.path().join("xdg"); + let bin = root.path().join("from-manifest"); + touch(&bin); + let home = root.path().join("home"); + let manifest = installed_manifest(&home, Some(&data)); + write_manifest(&manifest, "effectcraft", &bin); + let env = data_env(&data); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[cfg(target_os = "linux")] + #[test] + fn relative_xdg_data_home_uses_home() { + let root = Tmp::new("rel"); + let home = root.path().join("home"); + let bin = root.path().join("home-bin"); + touch(&bin); + let manifest = installed_manifest(&home, None); + write_manifest(&manifest, "effectcraft", &bin); + let env = env_map(&[("XDG_DATA_HOME", "relative/data")]); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[cfg(windows)] + #[test] + fn relative_localappdata_is_not_used() { + let env = env_map(&[("LOCALAPPDATA", "relative/data")]); + assert!(matches!(super::resolve_with("effectcraft", &env, None, None, &[]), Err(LinkError::NotInstalled { .. }))); + } + + #[test] + fn portable_manifest_wins_over_native_data_directory() { + let root = Tmp::new("port"); + let exe = root.path().join("exe-dir"); + std::fs::create_dir_all(&exe).unwrap(); + std::fs::write(exe.join("portable.txt"), b"1").unwrap(); + let bin = root.path().join("portable-bin"); + touch(&bin); + write_manifest(&exe.join("ArtCraftLauncherData").join("data").join("installed.json"), "effectcraft", &bin); + let xdg = root.path().join("xdg"); + let decoy = root.path().join("decoy-bin"); + touch(&decoy); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&xdg)), "effectcraft", &decoy); + let env = data_env(&xdg); + let got = super::resolve_with("effectcraft", &env, Some(&home), Some(&exe), &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn manifest_directory_is_skipped_for_path() { + let root = Tmp::new("skip"); + let data = root.path().join("xdg"); + let folder = root.path().join("not-a-file"); + std::fs::create_dir_all(&folder).unwrap(); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&data)), "effectcraft", &folder); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let mut env = data_env(&data); + env.insert("PATH".into(), bin_dir.to_str().unwrap().into()); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn path_supplies_the_path() { + let root = Tmp::new("path"); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let env = env_map(&[("PATH", bin_dir.to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn linux_location_supplies_the_path() { + let root = Tmp::new("fs"); + let apps = root.path().join("Applications"); + let image = apps.join("effectcraft-1.0.0.AppImage"); + touch(&image); + std::fs::write(apps.join("not-effectcraft.AppImage"), b"no").unwrap(); + let later = root.path().join("usr-bin"); + let exact = later.join("effectcraft"); + touch(&exact); + let env = env_map(&[("PATH", root.path().join("empty").to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[apps.clone(), later]).unwrap(); + assert_eq!(got, image); + } + + #[cfg(unix)] + #[test] + fn launch_at_reaches_hello() { + // The child branch returns at once when CRAFT_LAUNCH_DIR is unset. + // A normal cargo test takes the parent path and does not park. + if let Ok(dir) = std::env::var("CRAFT_LAUNCH_DIR") { + child_launch(&dir); + } + parent_launch(); + } + + #[cfg(unix)] + fn child_launch(dir: &str) -> ! { + use std::io::BufReader; + use std::os::unix::net::UnixListener; + + let runtime = RuntimeDir::at(dir).unwrap(); + let app = "effectcraft"; + let pid = std::process::id(); + std::fs::write(runtime.path().join("child.pid"), pid.to_string()).unwrap(); + let sock = runtime.socket_file(app, pid); + let _ = std::fs::remove_file(&sock); + let listener = UnixListener::bind(&sock).unwrap(); + crate::discovery::enforce_mode(&sock, 0o600).unwrap(); + let endpoint = runtime.unix_endpoint(app, pid).unwrap(); + let record = InstanceRecord::new(app, "0.8.3", pid, endpoint).unwrap(); + let published = Discovery::publish(&runtime, record.clone()).unwrap(); + std::fs::write(runtime.path().join("ready"), pid.to_string()).unwrap(); + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(20))).unwrap(); + let mut reader = BufReader::new(stream); + let auth = crate::wire::read_line(&mut reader, crate::wire::MAX_LINE).unwrap().unwrap(); + let auth_value: Value = serde_json::from_slice(&auth).unwrap(); + assert_eq!(auth_value["method"], "auth"); + assert_eq!(auth_value["params"]["token"], record.token); + crate::wire::write_line(reader.get_mut(), &crate::wire::auth_success(&serde_json::json!(0))).unwrap(); + let hello = crate::wire::read_line(&mut reader, crate::wire::MAX_LINE).unwrap().unwrap(); + let hello_value: Value = serde_json::from_slice(&hello).unwrap(); + assert_eq!(hello_value["method"], "link.hello"); + let id = hello_value.get("id").cloned().unwrap_or(serde_json::json!(1)); + let result = serde_json::json!({ + "app": app, + "app_version": record.app_version, + "pid": record.pid, + "instance": record.instance, + "protocol": 1, + }); + crate::wire::write_line(reader.get_mut(), &crate::wire::reply_ok(&id, result)).unwrap(); + let _keep = (published, listener, reader); + // park can wake without an unpark. Loop so Drop does not run until SIGKILL. + loop { + std::thread::park(); + } + } + + #[cfg(unix)] + fn parent_launch() { + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-launch-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let _cleanup = LaunchCleanup { dir: path.clone() }; + let exe = std::env::current_exe().unwrap(); + let args = vec![OsString::from("--exact"), OsString::from("launch::tests::launch_at_reaches_hello"), OsString::from("--test-threads=1")]; + let dir_text = path.to_str().unwrap(); + let mut phases = Vec::new(); + // The test binary can sit on a file system that marks every file 0777, and `launch_at` + // refuses such a program. A private launcher script that execs it keeps the pid. + let script = path.join("launcher"); + std::fs::write(&script, format!("#!/bin/sh\nexec '{}' \"$@\"\n", exe.display())).unwrap(); + chmod(&script, 0o755); + let result = launch_at("effectcraft", &script, &args, &[("CRAFT_LAUNCH_DIR", dir_text)], Some(&runtime), &mut |phase| phases.push(phase)); + let record = result.unwrap_or_else(|err| panic!("launch_at failed: {err}; phases={phases:?}")); + assert_eq!(phases, vec![Phase::Spawned, Phase::RecordSeen, Phase::HelloOk]); + let ready = std::fs::read_to_string(path.join("ready")).unwrap(); + let child_pid: u32 = ready.trim().parse().unwrap(); + assert_eq!(record.pid, child_pid); + assert_eq!(record.app, "effectcraft"); + } + + #[cfg(unix)] + struct LaunchCleanup { + dir: PathBuf, + } + + #[cfg(unix)] + impl Drop for LaunchCleanup { + fn drop(&mut self) { + kill_recorded(&self.dir); + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + #[cfg(unix)] + fn kill_recorded(dir: &Path) { + for name in ["child.pid", "ready"] { + let Ok(text) = std::fs::read_to_string(dir.join(name)) else { continue }; + let Ok(pid) = text.trim().parse::() else { continue }; + let _ = Command::new("kill").args(["-KILL", &pid.to_string()]).status(); + let proc = PathBuf::from(format!("/proc/{pid}")); + let deadline = Instant::now() + Duration::from_secs(2); + while proc.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + return; + } + } +} diff --git a/crates/ipc/src/lib.rs b/crates/ipc/src/lib.rs new file mode 100644 index 000000000..53b071fa6 --- /dev/null +++ b/crates/ipc/src/lib.rs @@ -0,0 +1,39 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +//! Craft Link transport. +//! +//! Local sockets, discovery files, and the JSON-lines session. The server and +//! the client are separate modules. There is no tokio runtime in this crate. + +pub mod activate; +pub mod asset; +pub mod auth; +pub mod bulk; +pub mod cache; +pub mod client; +pub mod discovery; +pub mod error; +pub mod fetch; +mod fsopen; +pub mod ids; +pub mod launch; +pub mod moniker; +pub mod payload; +pub mod record; +pub mod refresh; +pub mod server; +pub mod trust; +pub mod watch; +pub mod wire; + +/// Unit tests that hold a discovery lock take `read`. Unit tests that start a +/// child process take `write`. A forked child owns a copy of every open file +/// until it calls exec, and a `flock` lock lives until the last copy closes. +/// Without this guard a lock released by `drop` can look held for a few +/// milliseconds, and `scan` or `publish` in another test then fails at random. +#[cfg(test)] +pub(crate) static SPAWN_GUARD: std::sync::RwLock<()> = std::sync::RwLock::new(()); + +pub use auth::{ensure_peer_user, generate_token, random_uuid, tokens_equal}; +pub use discovery::{Discovery, DocInfo, Endpoint, InstanceRecord, Protocol, RuntimeDir}; +pub use error::{ErrorObject, LinkError}; diff --git a/crates/ipc/src/moniker.rs b/crates/ipc/src/moniker.rs new file mode 100644 index 000000000..5bee3e77c --- /dev/null +++ b/crates/ipc/src/moniker.rs @@ -0,0 +1,313 @@ +//! Asset monikers: `craft://///`. +//! +//! The grammar is in `PROTOCOL.md` section 9. This module uses +//! only `std` and `serde`, so an app can copy it to a wasm build. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::error::LinkError; +use crate::ids::{valid_app_id, valid_uuid}; + +const SCHEME: &str = "craft://"; +/// Longest decoded `id`, in bytes. +pub const MAX_ID_BYTES: usize = 512; + +/// What an asset is. Unknown words are valid and kept in [`MonikerKind::Other`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum MonikerKind { + Comp, + Image, + Vector, + Sequence, + Layer, + Other(String), +} + +impl MonikerKind { + pub fn as_str(&self) -> &str { + match self { + Self::Comp => "comp", + Self::Image => "image", + Self::Vector => "vector", + Self::Sequence => "sequence", + Self::Layer => "layer", + Self::Other(word) => word, + } + } + + fn parse(word: &str) -> Result { + if !valid_word(word, 32) { + return Err(bad(format!("kind {word:?} must match [a-z][a-z0-9-]{{0,31}}"))); + } + Ok(match word { + "comp" => Self::Comp, + "image" => Self::Image, + "vector" => Self::Vector, + "sequence" => Self::Sequence, + "layer" => Self::Layer, + other => Self::Other(other.to_string()), + }) + } +} + +/// A parsed moniker. `doc` keeps the hex case it was written with. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Moniker { + app: String, + doc: String, + kind: MonikerKind, + id: String, +} + +impl Moniker { + /// Build a moniker from parts. Every part is checked, so a value of this + /// type always formats to text that parses back to an equal value. + pub fn new(app: &str, doc: &str, kind: MonikerKind, id: &str) -> Result { + if !valid_app_id(app) { + return Err(bad(format!("app {app:?} must match [a-z][a-z0-9-]{{1,31}}"))); + } + if !valid_uuid(doc) { + return Err(bad(format!("doc {doc:?} is not a uuid"))); + } + if let MonikerKind::Other(word) = &kind + && (!valid_word(word, 32) || is_known_word(word)) + { + return Err(bad(format!("kind {word:?} is not a valid unknown kind"))); + } + if id.is_empty() || id.len() > MAX_ID_BYTES { + return Err(bad(format!("id must be 1 to {MAX_ID_BYTES} bytes"))); + } + Ok(Self { app: app.to_string(), doc: doc.to_string(), kind, id: id.to_string() }) + } + + pub fn parse(text: &str) -> Result { + let Some(rest) = text.strip_prefix(SCHEME) else { + return Err(bad(format!("moniker must start with {SCHEME}"))); + }; + let mut parts = rest.split('/'); + let (Some(app), Some(doc), Some(kind), Some(id), None) = (parts.next(), parts.next(), parts.next(), parts.next(), parts.next()) else { + return Err(bad("moniker must have exactly app, doc, kind and id".to_string())); + }; + if id.contains(['?', '#']) { + return Err(bad("moniker has no query or fragment".to_string())); + } + let kind = MonikerKind::parse(kind)?; + let id = decode_id(id)?; + Self::new(app, doc, kind, &id) + } + + pub fn app(&self) -> &str { + &self.app + } + + pub fn doc(&self) -> &str { + &self.doc + } + + pub fn kind(&self) -> &MonikerKind { + &self.kind + } + + /// The decoded id. + pub fn id(&self) -> &str { + &self.id + } +} + +impl fmt::Display for Moniker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{SCHEME}{}/{}/{}/", self.app, self.doc, self.kind.as_str())?; + for byte in self.id.bytes() { + if is_unreserved(byte) { + write!(f, "{}", char::from(byte))?; + } else { + write!(f, "%{byte:02X}")?; + } + } + Ok(()) + } +} + +impl FromStr for Moniker { + type Err = LinkError; + + fn from_str(text: &str) -> Result { + Self::parse(text) + } +} + +impl Serialize for Moniker { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Moniker { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + Self::parse(&text).map_err(serde::de::Error::custom) + } +} + +fn bad(message: String) -> LinkError { + LinkError::BadParams(message) +} + +fn is_known_word(word: &str) -> bool { + matches!(word, "comp" | "image" | "vector" | "sequence" | "layer") +} + +fn is_unreserved(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') +} + +fn valid_word(word: &str, max: usize) -> bool { + let mut bytes = word.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + word.len() <= max && first.is_ascii_lowercase() && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +fn decode_id(text: &str) -> Result { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + let pair = bytes.get(index + 1..index + 3).ok_or_else(|| bad("id has a truncated %XX escape".to_string()))?; + let hi = hex_value(pair.first().copied()); + let lo = hex_value(pair.get(1).copied()); + let (Some(hi), Some(lo)) = (hi, lo) else { + return Err(bad("id has a bad %XX escape".to_string())); + }; + out.push(hi * 16 + lo); + index += 3; + } else if is_unreserved(byte) { + out.push(byte); + index += 1; + } else { + return Err(bad(format!("id has the raw byte 0x{byte:02X}, which must be percent-encoded"))); + } + } + String::from_utf8(out).map_err(|_| bad("id is not valid UTF-8".to_string())) +} + +fn hex_value(byte: Option) -> Option { + match byte? { + b @ b'0'..=b'9' => Some(b - b'0'), + b @ b'a'..=b'f' => Some(b - b'a' + 10), + b @ b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + const DOC: &str = "550e8400-e29b-41d4-a716-446655440000"; + + fn sample(id: &str) -> Moniker { + Moniker::new("photocraft", DOC, MonikerKind::Image, id).unwrap() + } + + #[test] + fn plain_id_formats_and_parses() { + let text = format!("craft://photocraft/{DOC}/image/layer-3"); + assert_eq!(sample("layer-3").to_string(), text); + assert_eq!(Moniker::parse(&text).unwrap(), sample("layer-3")); + } + + #[test] + fn round_trip_keeps_awkward_ids() { + for id in ["a b", "a/b", "100%", "x?y#z", "caf\u{e9}", "\u{1F600}", "~._-", "%2F", "a%20b", &"x".repeat(MAX_ID_BYTES)] { + let m = sample(id); + let text = m.to_string(); + let back = Moniker::parse(&text).unwrap(); + assert_eq!(back, m, "{id:?} via {text}"); + assert_eq!(back.id(), id); + assert_eq!(back.to_string(), text); + } + } + + #[test] + fn encoded_slash_stays_inside_the_id() { + let m = sample("a/b"); + assert_eq!(m.to_string(), format!("craft://photocraft/{DOC}/image/a%2Fb")); + assert_eq!(Moniker::parse(&m.to_string()).unwrap().id(), "a/b"); + } + + #[test] + fn lowercase_escapes_parse_and_format_uppercase() { + let m = Moniker::parse(&format!("craft://photocraft/{DOC}/image/a%2fb")).unwrap(); + assert_eq!(m.id(), "a/b"); + assert_eq!(m.to_string(), format!("craft://photocraft/{DOC}/image/a%2Fb")); + } + + #[test] + fn doc_case_is_kept() { + let upper = DOC.to_uppercase(); + let m = Moniker::parse(&format!("craft://photocraft/{upper}/comp/c1")).unwrap(); + assert_eq!(m.doc(), upper); + assert_eq!(m.to_string(), format!("craft://photocraft/{upper}/comp/c1")); + } + + #[test] + fn kinds_known_and_unknown() { + for (word, kind) in [ + ("comp", MonikerKind::Comp), + ("image", MonikerKind::Image), + ("vector", MonikerKind::Vector), + ("sequence", MonikerKind::Sequence), + ("layer", MonikerKind::Layer), + ("palette", MonikerKind::Other("palette".into())), + ] { + let m = Moniker::parse(&format!("craft://designcraft/{DOC}/{word}/x")).unwrap(); + assert_eq!(m.kind(), &kind); + assert_eq!(m.to_string(), format!("craft://designcraft/{DOC}/{word}/x")); + } + assert!(Moniker::new("designcraft", DOC, MonikerKind::Other("comp".into()), "x").is_err(), "known word must use its variant"); + } + + #[test] + fn rejects_bad_text() { + let bad_texts = [ + "".to_string(), + "craft:/photocraft".to_string(), + "http://photocraft/x/image/a".to_string(), + format!("craft://photocraft/{DOC}/image"), + format!("craft://photocraft/{DOC}/image/a/b"), + format!("craft://photocraft/{DOC}/image/"), + format!("craft://photocraft/{DOC}/image/a?x=1"), + format!("craft://photocraft/{DOC}/image/a#frag"), + format!("craft://photocraft/{DOC}/image/a b"), + format!("craft://photocraft/{DOC}/image/%"), + format!("craft://photocraft/{DOC}/image/%4"), + format!("craft://photocraft/{DOC}/image/%GG"), + format!("craft://photocraft/{DOC}/image/%FF"), + format!("craft://Photo/{DOC}/image/a"), + format!("craft://p/{DOC}/image/a"), + "craft://photocraft/not-a-uuid/image/a".to_string(), + format!("craft://photocraft/{DOC}/Image/a"), + format!("craft://photocraft/{DOC}/1image/a"), + format!("craft://photocraft/{DOC}/image/{}", "x".repeat(MAX_ID_BYTES + 1)), + ]; + for text in bad_texts { + assert!(Moniker::parse(&text).is_err(), "{text:?} should not parse"); + } + } + + #[test] + fn serde_uses_the_string_form() { + let m = sample("a b"); + let value = serde_json::to_value(&m).unwrap(); + assert_eq!(value, serde_json::json!(format!("craft://photocraft/{DOC}/image/a%20b"))); + assert_eq!(serde_json::from_value::(value).unwrap(), m); + assert!(serde_json::from_value::(serde_json::json!("craft://nope")).is_err()); + } +} diff --git a/crates/ipc/src/payload.rs b/crates/ipc/src/payload.rs new file mode 100644 index 000000000..ddbd7d2dd --- /dev/null +++ b/crates/ipc/src/payload.rs @@ -0,0 +1,396 @@ +//! The `payload` object of `link.open`. +//! +//! Formats are listed in `PROTOCOL.md` section 4. The server +//! calls [`OpenPayload::from_value`] on every `link.open` before it forwards the +//! request, and answers `payload_invalid` on an error. An app may call it again. +//! +//! This module uses only `std`, `serde_json`, [`crate::moniker`] and the size constants in +//! [`crate::wire`]. + +use serde_json::Value; + +use crate::error::LinkError; +use crate::moniker::Moniker; +use crate::wire::MAX_LINK_PAYLOAD; + +/// A parsed `link.open` payload. +#[derive(Debug, Clone, PartialEq)] +pub enum OpenPayload { + /// Video: an OTIO JSON string. + OtioJson { data: String }, + /// A still or vector: the producer document path and the asset moniker. + ImageDoc { path: String, asset: Moniker }, +} + +impl OpenPayload { + pub fn from_value(payload: &Value) -> Result { + let obj = payload.as_object().ok_or_else(|| LinkError::PayloadInvalid("payload must be an object".into()))?; + let format = obj.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::PayloadInvalid("payload.format must be a string".into()))?; + match format { + "otio-json" => { + let data = text_field(obj, "data")?; + if data.len() as u64 > MAX_LINK_PAYLOAD { + return Err(LinkError::TooLarge); + } + let missing = missing_media(&data)?; + if !missing.is_empty() { + return Err(LinkError::MediaMissing { paths: missing }); + } + Ok(Self::OtioJson { data }) + } + "image-doc" => { + let path = text_field(obj, "path")?; + check_peer_path(&path, "payload.path")?; + let asset = Moniker::parse(&text_field(obj, "asset")?).map_err(|err| LinkError::PayloadInvalid(format!("payload.asset: {err}")))?; + Ok(Self::ImageDoc { path, asset }) + } + other => Err(LinkError::PayloadInvalid(format!("payload.format {other:?} is not supported"))), + } + } +} + +/// The most distinct files one OTIO document may name. A timeline with more is not an editor's +/// timeline, and each one costs a file system call. +pub const MAX_MEDIA_REFS: usize = 4096; +/// The most JSON values one OTIO document may hold. The line cap (4 MiB) allows about two million. +pub const MAX_NODES: usize = 1 << 20; +/// Deepest nesting walked. `serde_json` stops parsing at 128, so a deeper document never gets here. +pub const MAX_DEPTH: usize = 128; + +/// Check a path that a peer names (`payload.path`, a media `target_url`). It must be absolute, hold +/// no NUL byte and no `..` component. The server does not canonicalise it: the app opens it, and a +/// symlink to a file elsewhere is a normal way to keep footage. This check only refuses a path +/// that is written to climb out of the directory it starts in. +pub fn check_peer_path(path: &str, what: &str) -> Result<(), LinkError> { + let parsed = std::path::Path::new(path); + if !parsed.is_absolute() { + return Err(LinkError::PayloadInvalid(format!("{what} must be absolute"))); + } + if path.contains('\0') { + return Err(LinkError::PayloadInvalid(format!("{what} must not hold a NUL byte"))); + } + if parsed.components().any(|part| matches!(part, std::path::Component::ParentDir)) { + return Err(LinkError::PayloadInvalid(format!("{what} must not hold a `..` component"))); + } + Ok(()) +} + +/// The files that `ExternalReference`s of an OTIO document name and that cannot be used. Each path +/// appears once, in document order. +/// +/// A file is usable when it is a regular file (after symlinks) and opens for reading. A missing +/// file, a directory, a FIFO, a device and a file without read access all give the same answer, so +/// the reply says nothing beyond "this path from your payload does not work". The probe never opens +/// anything that is not a regular file, so a FIFO cannot hold the connection thread. Only paths the +/// payload itself names are probed, and at most [`MAX_MEDIA_REFS`] of them. +/// +/// Empty when every file works, when `data` is not JSON (the app reports that), and for references +/// that are not an absolute path or a `file://` URL: they have no base here to resolve against. +/// `Err` when the document is too deep or too big, or names an absolute path with `..` or NUL. +pub fn missing_media(data: &str) -> Result, LinkError> { + let Ok(document) = serde_json::from_str::(data) else { return Ok(Vec::new()) }; + let mut walk = Walk { seen: Vec::new(), nodes: 0 }; + walk.collect(&document, 0)?; + walk.seen.retain(|path| !media_usable(path)); + Ok(walk.seen) +} + +fn media_usable(path: &str) -> bool { + let path = std::path::Path::new(path); + std::fs::metadata(path).is_ok_and(|meta| meta.is_file()) && std::fs::File::open(path).is_ok() +} + +struct Walk { + seen: Vec, + nodes: usize, +} + +impl Walk { + fn collect(&mut self, value: &Value, depth: usize) -> Result<(), LinkError> { + if depth > MAX_DEPTH { + return Err(LinkError::PayloadInvalid(format!("the OTIO document nests deeper than {MAX_DEPTH}"))); + } + self.nodes += 1; + if self.nodes > MAX_NODES { + return Err(LinkError::PayloadInvalid(format!("the OTIO document holds more than {MAX_NODES} values"))); + } + match value { + Value::Object(map) => { + let external = map.get("OTIO_SCHEMA").and_then(Value::as_str).is_some_and(|schema| schema.starts_with("ExternalReference")); + let local = match map.get("target_url").and_then(Value::as_str) { + Some(url) if external => local_path(url)?, + _ => None, + }; + if let Some(path) = local { + check_peer_path(&path, "target_url")?; + if !self.seen.contains(&path) { + if self.seen.len() >= MAX_MEDIA_REFS { + return Err(LinkError::PayloadInvalid(format!("the OTIO document names more than {MAX_MEDIA_REFS} media files"))); + } + self.seen.push(path); + } + } + for child in map.values() { + self.collect(child, depth + 1)?; + } + } + Value::Array(items) => { + for child in items { + self.collect(child, depth + 1)?; + } + } + _ => {} + } + Ok(()) + } +} + +/// The local path a `target_url` names: a `file://` URL (percent-decoded, host empty or +/// `localhost`) or an absolute path. `Ok(None)` for anything else. `Err` for a `file://` URL +/// whose path does not percent-decode to UTF-8: it names a local file, so it cannot be skipped. +fn local_path(url: &str) -> Result, LinkError> { + let path = match url.strip_prefix("file://") { + Some(rest) => { + let rest = rest.strip_prefix("localhost").unwrap_or(rest); + if !rest.starts_with('/') { + return Ok(None); + } + percent_decode(rest).ok_or_else(|| LinkError::PayloadInvalid("target_url is a file:// URL with a bad percent escape".into()))? + } + None => url.to_string(), + }; + // `/C:/dir/a.mov` is how a Windows path sits in a URL. + let path = match path.as_bytes() { + [b'/', drive, b':', ..] if drive.is_ascii_alphabetic() && cfg!(windows) => path[1..].to_string(), + _ => path, + }; + Ok(std::path::Path::new(&path).is_absolute().then_some(path)) +} + +fn percent_decode(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + let pair = bytes.get(index + 1..index + 3)?; + // `from_str_radix` also takes a leading `+`, so the two digits are checked first. + if !pair.iter().all(u8::is_ascii_hexdigit) { + return None; + } + out.push(u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok()?); + index += 3; + } else { + out.push(byte); + index += 1; + } + } + String::from_utf8(out).ok() +} + +fn text_field(obj: &serde_json::Map, key: &str) -> Result { + match obj.get(key) { + Some(Value::String(text)) if !text.is_empty() => Ok(text.clone()), + _ => Err(LinkError::PayloadInvalid(format!("payload.{key} must be a non-empty string"))), + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use serde_json::json; + + use super::*; + + const ASSET: &str = "craft://photocraft/550e8400-e29b-41d4-a716-446655440000/image/flat"; + + fn absolute(path: &str) -> String { + if cfg!(windows) { format!("C:{path}") } else { path.to_owned() } + } + + fn file_url(path: &str) -> String { + let path = path.replace('\\', "/").replace(' ', "%20"); + format!("file://{}{path}", if path.starts_with('/') { "" } else { "/" }) + } + + #[test] + fn parses_otio() { + let got = OpenPayload::from_value(&json!({"format": "otio-json", "data": "{}"})).unwrap(); + assert_eq!(got, OpenPayload::OtioJson { data: "{}".into() }); + } + + fn otio_with(url: &str) -> String { + json!({ + "OTIO_SCHEMA": "Timeline.1", + "tracks": {"OTIO_SCHEMA": "Stack.1", "children": [{"OTIO_SCHEMA": "Track.1", "children": [{ + "OTIO_SCHEMA": "Clip.2", + "media_references": {"DEFAULT_MEDIA": {"OTIO_SCHEMA": "ExternalReference.1", "target_url": url}}, + "active_media_reference_key": "DEFAULT_MEDIA" + }]}]} + }) + .to_string() + } + + #[test] + fn rejects_media_that_does_not_exist_with_the_paths() { + let missing = absolute("/nowhere/My Clip.mov"); + let payload = json!({"format": "otio-json", "data": otio_with(&file_url(&missing))}); + let err = OpenPayload::from_value(&payload).unwrap_err(); + assert_eq!(err, LinkError::MediaMissing { paths: vec![missing.clone()] }); + assert_eq!(err.code(), "media_missing"); + assert_eq!(err.to_object().data.get("missing"), Some(&json!([missing]))); + assert!(err.to_string().contains(&missing), "{err}"); + // A bare absolute path counts the same. + let bare = json!({"format": "otio-json", "data": otio_with(&absolute("/nowhere/clip.mov"))}); + assert!(matches!(OpenPayload::from_value(&bare), Err(LinkError::MediaMissing { .. }))); + } + + #[test] + fn accepts_media_that_opens_and_references_it_cannot_resolve() { + let dir = std::env::temp_dir().join(format!("craft-payload-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("My Clip.mov"); + std::fs::write(&file, b"x").unwrap(); + let url = file_url(file.to_str().unwrap()); + assert_eq!(std::path::PathBuf::from(local_path(&url).unwrap().unwrap()), file); + let good = json!({"format": "otio-json", "data": otio_with(&url)}); + assert!(OpenPayload::from_value(&good).is_ok()); + // A relative target has no base here. The app resolves it. + let relative = json!({"format": "otio-json", "data": otio_with("clips/a.mov")}); + assert!(OpenPayload::from_value(&relative).is_ok()); + // A generator or a missing reference names no file. + let none = json!({"format": "otio-json", "data": "{\"OTIO_SCHEMA\":\"GeneratorReference.1\"}"}); + assert!(OpenPayload::from_value(&none).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn parses_image_doc() { + let expected = absolute("/tmp/a.photocraft"); + let got = OpenPayload::from_value(&json!({"format": "image-doc", "path": expected, "asset": ASSET})).unwrap(); + let OpenPayload::ImageDoc { path, asset } = got else { + panic!("wrong variant"); + }; + assert_eq!(path, expected); + assert_eq!(asset.to_string(), ASSET); + } + + fn payload_with(url: &str) -> Value { + json!({"format": "otio-json", "data": otio_with(url)}) + } + + #[test] + fn traversal_in_a_named_path_is_payload_invalid() { + for bad in [ + absolute("/tmp/../etc/passwd"), + file_url(&absolute("/tmp/a/../../etc/passwd")), + file_url(&absolute("/tmp/%2e%2e/etc/passwd")), + absolute("/tmp/a\u{0}b"), + file_url(&absolute("/tmp/a%00b")), + ] { + let err = OpenPayload::from_value(&payload_with(&bad)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad:?}: {err:?}"); + } + let err = OpenPayload::from_value(&json!({"format": "image-doc", "path": absolute("/tmp/../etc/x.png"), "asset": ASSET})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // A relative target is the app's business, `..` or not. + assert!(OpenPayload::from_value(&payload_with("clips/../a.mov")).is_ok()); + } + + #[test] + fn a_file_url_that_does_not_decode_is_payload_invalid() { + for bad in [ + "file:///tmp/clip%GG.mov", + "file:///tmp/clip%4", + "file:///tmp/clip%", + "file://localhost/tmp/clip%zz.mov", + "file:///tmp/clip%FF.mov", + // A sign or a space is not a hex digit, although `from_str_radix` takes `+`. + "file:///tmp/a%+fb", + "file:///tmp/a%-fb", + "file:///tmp/a% fb", + "file:///tmp/a%f", + ] { + let err = OpenPayload::from_value(&payload_with(bad)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad:?}: {err:?}"); + assert!(matches!(missing_media(&otio_with(bad)), Err(LinkError::PayloadInvalid(_))), "{bad:?}"); + } + // A reference that names no local file stays the app's business. + for skipped in ["file://remote-host/tmp/clip%GG.mov", "http://example.com/clip%GG.mov", "clips/clip%GG.mov"] { + assert!(OpenPayload::from_value(&payload_with(skipped)).is_ok(), "{skipped:?}"); + } + } + + #[cfg(unix)] + #[test] + fn only_regular_files_count_as_present_and_a_fifo_does_not_block() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("craft-payload-fifo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("pipe.mov"); + let made = std::process::Command::new("mkfifo").arg(&fifo).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + let real = dir.join("real.mov"); + std::fs::write(&real, b"x").unwrap(); + let link = dir.join("link.mov"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let start = std::time::Instant::now(); + let data = json!({"OTIO_SCHEMA": "Timeline.1", "refs": [ + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": fifo.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": dir.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": link.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": real.to_str().unwrap()}, + ]}) + .to_string(); + let missing = missing_media(&data).unwrap(); + assert!(start.elapsed() < std::time::Duration::from_secs(2), "the probe waited on a FIFO"); + assert_eq!(missing, vec![fifo.to_str().unwrap().to_string(), dir.to_str().unwrap().to_string()]); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn size_depth_and_count_are_bounded() { + // Too many distinct media files. + let refs: Vec = (0..=MAX_MEDIA_REFS) + .map(|n| json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": absolute(&format!("/nowhere/{n}.mov"))})) + .collect(); + let many = json!({"OTIO_SCHEMA": "Timeline.1", "refs": refs}).to_string(); + let err = OpenPayload::from_value(&json!({"format": "otio-json", "data": many})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // The same file named many times counts once. + let same: Vec = + (0..MAX_MEDIA_REFS * 2).map(|_| json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": absolute("/nowhere/a.mov")})).collect(); + let doc = json!({"OTIO_SCHEMA": "Timeline.1", "refs": same}).to_string(); + assert_eq!(missing_media(&doc).unwrap(), vec![absolute("/nowhere/a.mov")]); + // Too deep. serde_json refuses at 128, so the document reads as "not JSON" and the app reports it. + let deep = format!("{}{}", "[".repeat(300), "]".repeat(300)); + assert_eq!(missing_media(&deep).unwrap(), Vec::::new()); + // Just inside serde_json's limit the walker still finishes. + let nested = format!("{}{}", "[".repeat(MAX_DEPTH - 1), "]".repeat(MAX_DEPTH - 1)); + assert_eq!(missing_media(&nested).unwrap(), Vec::::new()); + // Too big. + let big = format!("{{\"pad\":\"{}\"}}", "a".repeat(usize::try_from(MAX_LINK_PAYLOAD).unwrap())); + let err = OpenPayload::from_value(&json!({"format": "otio-json", "data": big})).unwrap_err(); + assert_eq!(err, LinkError::TooLarge); + assert_eq!(err.code(), "payload_too_large"); + } + + #[test] + fn rejects_bad_payloads() { + for bad in [ + json!("x"), + json!({}), + json!({"format": "otio-json"}), + json!({"format": "otio-json", "data": ""}), + json!({"format": "image-doc", "path": "rel.png", "asset": ASSET}), + json!({"format": "image-doc", "path": "/a.png", "asset": "craft://bad"}), + json!({"format": "image-doc", "asset": ASSET}), + json!({"format": "svg", "data": "x"}), + ] { + let err = OpenPayload::from_value(&bad).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad}: {err:?}"); + assert_eq!(err.code(), "payload_invalid"); + } + } +} diff --git a/crates/ipc/src/record.rs b/crates/ipc/src/record.rs new file mode 100644 index 000000000..6b8b9f09f --- /dev/null +++ b/crates/ipc/src/record.rs @@ -0,0 +1,237 @@ +//! The `craft.link` project item record. +//! +//! Schema: `schemas/link-record.schema.json`. A consumer stores one of these +//! on the linked media item. Records written before the asset fields existed +//! (FilmCraft to EffectCraft) parse and validate unchanged. Unknown JSON fields +//! stay in the `extra` maps and come back out on save. +//! +//! This module uses only `std` and `serde`, so an app can copy it to a wasm build. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::error::LinkError; +use crate::ids::{valid_app_id, valid_uuid}; +use crate::moniker::Moniker; + +/// What the linked asset is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AssetKind { + Still, + Vector, + Sequence, + Comp, +} + +impl AssetKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Still => "still", + Self::Vector => "vector", + Self::Sequence => "sequence", + Self::Comp => "comp", + } + } +} + +/// The linked asset: its kind and its durable id. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AssetRef { + pub kind: AssetKind, + /// A moniker string. [`LinkRecord::validate`] parses it. + pub moniker: String, + #[serde(default, flatten)] + pub extra: Map, +} + +/// Who wrote the result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Producer { + pub app: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub doc_id: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// Size of the linked result. The video fields are absent for stills. +/// +/// `rate_num` and `rate_den` come as a pair. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Format { + pub w: u64, + pub h: u64, + pub alpha: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_num: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_den: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ticks: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// One link from a consumer media item to a producer asset. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LinkRecord { + pub v: u32, + pub link_id: String, + pub target_app: String, + pub target_project: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_comp: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nested_sequence: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub producer: Option, + pub result_path: String, + pub result_epoch: u64, + pub format: Format, + pub created: String, + #[serde(default, flatten)] + pub extra: Map, +} + +impl LinkRecord { + /// Check the record rules. Deserialization can accept a record this rejects. + pub fn validate(&self) -> Result<(), LinkError> { + if self.v != 1 { + return Err(invalid(format!("v is {}, want 1", self.v))); + } + if !valid_uuid(&self.link_id) { + return Err(invalid("link_id is not a uuid".into())); + } + validate_app("target_app", &self.target_app)?; + require_nonempty("target_project", &self.target_project)?; + require_nonempty("result_path", &self.result_path)?; + if let Some(comp) = &self.target_comp { + require_nonempty("target_comp", comp)?; + } + if let Some(nest) = &self.nested_sequence { + require_nonempty("nested_sequence", nest)?; + } + if self.target_comp.is_none() && self.asset.is_none() { + return Err(invalid("a record needs target_comp or asset".into())); + } + if let Some(asset) = &self.asset { + Moniker::parse(&asset.moniker).map_err(|err| invalid(format!("asset.moniker: {err}")))?; + } + if let Some(producer) = &self.producer { + validate_app("producer.app", &producer.app)?; + } + self.format.validate()?; + if !valid_rfc3339(&self.created) { + return Err(invalid("created is not an RFC 3339 date-time".into())); + } + Ok(()) + } +} + +impl Format { + fn validate(&self) -> Result<(), LinkError> { + require_at_least_one("format.w", self.w)?; + require_at_least_one("format.h", self.h)?; + match (self.rate_num, self.rate_den) { + (Some(num), Some(den)) => { + require_at_least_one("format.rate_num", num)?; + require_at_least_one("format.rate_den", den) + } + (None, None) => Ok(()), + _ => Err(invalid("format.rate_num and format.rate_den come as a pair".into())), + } + } +} + +fn invalid(message: String) -> LinkError { + LinkError::Invalid(message) +} + +fn validate_app(name: &str, app: &str) -> Result<(), LinkError> { + if valid_app_id(app) { Ok(()) } else { Err(invalid(format!("{name} {app:?} must match [a-z][a-z0-9-]{{1,31}}"))) } +} + +fn require_nonempty(name: &str, value: &str) -> Result<(), LinkError> { + if value.is_empty() { Err(invalid(format!("{name} is empty"))) } else { Ok(()) } +} + +fn require_at_least_one(name: &str, value: u64) -> Result<(), LinkError> { + if value >= 1 { Ok(()) } else { Err(invalid(format!("{name} must be >= 1"))) } +} + +/// `YYYY-MM-DDTHH:MM:SS` with an optional fraction and a `Z` or numeric offset. +fn valid_rfc3339(text: &str) -> bool { + let bytes = text.as_bytes(); + if bytes.len() < 20 { + return false; + } + let (Some(month), Some(day), Some(hour), Some(minute), Some(second)) = + (two_digits(bytes, 5), two_digits(bytes, 8), two_digits(bytes, 11), two_digits(bytes, 14), two_digits(bytes, 17)) + else { + return false; + }; + if !four_digits(bytes, 0) + || byte_at(bytes, 4) != Some(b'-') + || byte_at(bytes, 7) != Some(b'-') + || !matches!(byte_at(bytes, 10), Some(b'T' | b't')) + || byte_at(bytes, 13) != Some(b':') + || byte_at(bytes, 16) != Some(b':') + { + return false; + } + if !(1..=12).contains(&month) || !(1..=31).contains(&day) || hour > 23 || minute > 59 || second > 60 { + return false; + } + let (Some(century), Some(year_in_century)) = (two_digits(bytes, 0), two_digits(bytes, 2)) else { + return false; + }; + if chrono::NaiveDate::from_ymd_opt(i32::from(century * 100 + year_in_century), u32::from(month), u32::from(day)).is_none() { + return false; + } + let mut index = 19; + if byte_at(bytes, index) == Some(b'.') { + index += 1; + let start = index; + while byte_at(bytes, index).is_some_and(|byte| byte.is_ascii_digit()) { + index += 1; + } + if index == start { + return false; + } + } + match byte_at(bytes, index) { + Some(b'Z' | b'z') => index + 1 == bytes.len(), + Some(b'+' | b'-') => numeric_offset(bytes, index + 1), + _ => false, + } +} + +fn numeric_offset(bytes: &[u8], index: usize) -> bool { + if bytes.len() != index + 5 || byte_at(bytes, index + 2) != Some(b':') { + return false; + } + matches!((two_digits(bytes, index), two_digits(bytes, index + 3)), (Some(hour), Some(minute)) if hour <= 23 && minute <= 59) +} + +fn four_digits(bytes: &[u8], index: usize) -> bool { + bytes.get(index..index + 4).is_some_and(|slice| slice.iter().all(u8::is_ascii_digit)) +} + +fn two_digits(bytes: &[u8], index: usize) -> Option { + let slice = bytes.get(index..index + 2)?; + let hi = slice.first().copied()?; + let lo = slice.get(1).copied()?; + if !hi.is_ascii_digit() || !lo.is_ascii_digit() { + return None; + } + Some(u16::from(hi - b'0') * 10 + u16::from(lo - b'0')) +} + +fn byte_at(bytes: &[u8], index: usize) -> Option { + bytes.get(index).copied() +} diff --git a/crates/ipc/src/refresh.rs b/crates/ipc/src/refresh.rs new file mode 100644 index 000000000..f4b48a72e --- /dev/null +++ b/crates/ipc/src/refresh.rs @@ -0,0 +1,572 @@ +//! Watch and refresh rules for linked files (`PROTOCOL.md` section 10). +//! +//! A producer writes `result_path` with [`write_atomic`]. A consumer keeps one +//! [`Tracker`] per linked file and calls [`Tracker::poll`] on a timer. The +//! tracker needs no connection to the producer, so it also works while the +//! producer is offline. `std` only. +//! +//! | State | Meaning | +//! |---|---| +//! | `Current` | the disk has the content the consumer loaded | +//! | `Modified` | the disk differs from what the consumer loaded; the epoch went up | +//! | `Missing` | the file does not exist | + +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::UNIX_EPOCH; + +use sha2::{Digest, Sha256}; + +use crate::error::LinkError; + +/// Size, modification time and content hash of a file at one moment. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fingerprint { + pub size: u64, + pub mtime_ms: u64, + /// `sha256:` followed by 64 lowercase hex characters. + pub hash: String, +} + +/// What a consumer shows in its Links panel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileState { + Current, + Modified, + Missing, +} + +impl FileState { + /// The wire word used by `link.refresh`. + pub fn as_str(self) -> &'static str { + match self { + Self::Current => "current", + Self::Modified => "modified", + Self::Missing => "missing", + } + } +} + +/// The result of one [`Tracker::poll`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Change { + pub state: FileState, + pub epoch: u64, + /// True when this poll added 1 to the epoch. + pub bumped: bool, +} + +/// Watches one file for a consumer. +#[derive(Debug)] +pub struct Tracker { + path: PathBuf, + epoch: u64, + /// What the consumer has loaded. + loaded: Option, + /// What the last poll saw on disk. + seen: Option, + state: FileState, +} + +impl Tracker { + /// Start watching `path` at `epoch`. A file that exists now counts as loaded. + pub fn new(path: impl Into, epoch: u64) -> Result { + let path = path.into(); + let seen = fingerprint_if_present(&path)?; + let state = if seen.is_some() { FileState::Current } else { FileState::Missing }; + Ok(Self { path, epoch, loaded: seen.clone(), seen, state }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn epoch(&self) -> u64 { + self.epoch + } + + pub fn state(&self) -> FileState { + self.state + } + + /// The fingerprint from the last poll, or `None` while the file is missing. + pub fn seen(&self) -> Option<&Fingerprint> { + self.seen.as_ref() + } + + /// Raise the epoch to `epoch` when it is larger. Used when the producer is + /// reachable again and reports its own epoch. An epoch never decreases. + pub fn adopt_epoch(&mut self, epoch: u64) { + self.epoch = self.epoch.max(epoch); + } + + /// Look at the disk once. + /// + /// A shallow poll (`deep` false) reads size and mtime and hashes the file + /// only when either differs from the last poll. A deep poll always hashes. + /// An edit that keeps size and mtime is invisible to a shallow poll. + /// + /// A missing file gives `Missing` and no epoch bump. Any other read error + /// is returned and leaves the state as it was. So does a change that needs + /// a bump at epoch `u64::MAX`: that is `Err(Invalid)`, because an epoch that + /// cannot go up would let a cached frame pass for the new content. + pub fn poll(&mut self, deep: bool) -> Result { + let Some(mut file) = open_if_present(&self.path)? else { + self.seen = None; + self.state = FileState::Missing; + return Ok(self.change(false)); + }; + let (size, mtime_ms) = stat(&file, &self.path)?; + if !deep + && let Some(prev) = &self.seen + && prev.size == size + && prev.mtime_ms == mtime_ms + { + return Ok(self.change(false)); + } + let now = Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? }; + let changed = self.seen.as_ref().is_none_or(|prev| prev.hash != now.hash); + let bumped = changed && self.differs_from_loaded_or_was_seen(&now); + self.apply(now, bumped)?; + Ok(self.change(bumped)) + } + + /// The consumer loaded the content it last saw. `Modified` becomes `Current`. + pub fn accept(&mut self) { + self.loaded = self.seen.clone(); + self.state = if self.seen.is_some() { FileState::Current } else { FileState::Missing }; + } + + /// Point the link at `new_path`. The new file is read at once. Its hash + /// equal to what the consumer loaded gives `Current` with no bump. A + /// different hash gives `Modified` and one bump. A missing `new_path` is an + /// error and the old path stays. So it does when the bump would pass `u64::MAX`. + pub fn relink(&mut self, new_path: impl Into) -> Result { + let new_path = new_path.into(); + let Some(now) = fingerprint_if_present(&new_path)? else { + return Err(LinkError::Unknown(format!("{} does not exist", new_path.display()))); + }; + let same = self.loaded.as_ref().is_some_and(|loaded| loaded.hash == now.hash); + self.apply(now, !same)?; + self.path = new_path; + Ok(self.change(!same)) + } + + /// Whether a changed hash counts as a new version. The first sighting after + /// `Missing` that equals the loaded content is not a new version. + fn differs_from_loaded_or_was_seen(&self, now: &Fingerprint) -> bool { + match (&self.seen, &self.loaded) { + (Some(_), _) => true, + (None, Some(loaded)) => loaded.hash != now.hash, + (None, None) => true, + } + } + + /// Record `now`. A bump adds 1 to the epoch, or fails with the tracker untouched. + fn apply(&mut self, now: Fingerprint, bumped: bool) -> Result<(), LinkError> { + if bumped { + self.epoch = self.epoch.checked_add(1).ok_or_else(|| LinkError::Invalid("the epoch is at its maximum and cannot go up".into()))?; + } + let current = self.loaded.as_ref().is_some_and(|loaded| loaded.hash == now.hash); + self.seen = Some(now); + self.state = if current { FileState::Current } else { FileState::Modified }; + Ok(()) + } + + fn change(&self, bumped: bool) -> Change { + Change { state: self.state, epoch: self.epoch, bumped } + } +} + +/// Fingerprint `path`. A missing file is an error. +pub fn fingerprint(path: &Path) -> Result { + let mut file = crate::fsopen::open_read(path, true)?; + let (size, mtime_ms) = stat(&file, path)?; + Ok(Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? }) +} + +/// `None` when the file does not exist. +pub fn fingerprint_if_present(path: &Path) -> Result, LinkError> { + match open_if_present(path)? { + None => Ok(None), + Some(mut file) => { + let (size, mtime_ms) = stat(&file, path)?; + Ok(Some(Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? })) + } + } +} + +fn open_if_present(path: &Path) -> Result, LinkError> { + // `stat` checks the type on this handle. An open that waits for a FIFO writer would stall the caller. + match crate::fsopen::open_read(path, true) { + Ok(file) => Ok(Some(file)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + // Windows cannot open a directory for reading while Unix can, so `stat` would never + // run there and callers would see an OS-specific Io instead. Classify a path that + // exists but is not a regular file here, so both platforms report Invalid. + Err(err) => match std::fs::metadata(path) { + Ok(meta) if !meta.is_file() => Err(LinkError::Invalid(format!("{} is not a regular file", path.display()))), + _ => Err(LinkError::Io(format!("{}: {err}", path.display()))), + }, + } +} + +fn stat(file: &File, path: &Path) -> Result<(u64, u64), LinkError> { + let meta = file.metadata().map_err(|err| LinkError::Io(format!("{}: {err}", path.display())))?; + if !meta.is_file() { + return Err(LinkError::Invalid(format!("{} is not a regular file", path.display()))); + } + let modified = meta.modified().map_err(|err| LinkError::Io(format!("{}: {err}", path.display())))?; + let since = modified.duration_since(UNIX_EPOCH).map_err(|_| LinkError::Io(format!("{}: mtime is before 1970", path.display())))?; + let ms = u64::try_from(since.as_millis()).map_err(|_| LinkError::Io(format!("{}: mtime is out of range", path.display())))?; + Ok((meta.len(), ms)) +} + +fn hash_reader(reader: &mut impl Read) -> Result { + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(buf.get(..n).unwrap_or_default()); + } + Ok(format!("sha256:{}", crate::auth::encode_hex(&hasher.finalize()))) +} + +/// True for the temporary names [`write_atomic`] uses. Readers skip them. +pub fn is_temp_name(name: &str) -> bool { + name.starts_with('.') && name.ends_with(".tmp") +} + +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Write `bytes` to `path` so a reader sees the old file or the new file. +/// +/// The bytes go to `....tmp` in the same directory, are flushed, +/// and are renamed over `path`. The directory is flushed on Unix. +/// +/// An error before the rename removes the temporary file and leaves `path` as it was. The +/// directory flush and the final fingerprint run after the rename, so an error from either can +/// come back while `path` already holds the new bytes. Such an error means the write is not +/// confirmed durable or not fingerprinted, not that `path` is unchanged. +pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result { + let name = path.file_name().and_then(|n| n.to_str()).ok_or_else(|| LinkError::Invalid(format!("{} has no file name", path.display())))?; + let dir = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let tmp = dir.join(format!(".{name}.{}.{}.tmp", std::process::id(), TEMP_COUNTER.fetch_add(1, Ordering::Relaxed))); + let written = write_and_rename(&tmp, path, bytes); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written?; + #[cfg(unix)] + File::open(dir)?.sync_all()?; + fingerprint(path) +} + +fn write_and_rename(tmp: &Path, path: &Path, bytes: &[u8]) -> Result<(), LinkError> { + let mut file = OpenOptions::new().write(true).create_new(true).open(tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(tmp, path)?; + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::time::Duration; + + use super::*; + + struct Tmp(PathBuf); + + impl Tmp { + fn new(label: &str) -> Self { + let nanos = std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let dir = std::env::temp_dir().join(format!("cl-refresh-{label}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + Self(dir) + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn sha_of(bytes: &[u8]) -> String { + format!("sha256:{}", crate::auth::encode_hex(&Sha256::digest(bytes))) + } + + #[test] + fn hash_is_sha256_of_the_content() { + let tmp = Tmp::new("hash"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"abc").unwrap(); + let fp = fingerprint(&path).unwrap(); + assert_eq!(fp.hash, "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert_eq!(fp.size, 3); + assert!(fp.mtime_ms > 0); + } + + #[test] + fn unchanged_file_stays_current_with_no_bump() { + let tmp = Tmp::new("same"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 4).unwrap(); + assert_eq!(tracker.state(), FileState::Current); + for deep in [false, true] { + let change = tracker.poll(deep).unwrap(); + assert_eq!(change, Change { state: FileState::Current, epoch: 4, bumped: false }); + } + } + + #[test] + fn changed_content_is_modified_and_bumps_once() { + let tmp = Tmp::new("mod"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 4).unwrap(); + write_atomic(&path, b"two!").unwrap(); + let change = tracker.poll(false).unwrap(); + assert_eq!(change, Change { state: FileState::Modified, epoch: 5, bumped: true }); + // Polling again with no new edit does not bump again. + let again = tracker.poll(true).unwrap(); + assert_eq!(again, Change { state: FileState::Modified, epoch: 5, bumped: false }); + // A second edit bumps again. + write_atomic(&path, b"three").unwrap(); + assert_eq!(tracker.poll(false).unwrap().epoch, 6); + // Loading the new content clears the state. + tracker.accept(); + assert_eq!(tracker.state(), FileState::Current); + assert_eq!(tracker.poll(true).unwrap(), Change { state: FileState::Current, epoch: 6, bumped: false }); + } + + #[test] + fn touch_without_content_change_does_not_bump() { + let tmp = Tmp::new("touch"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let file = OpenOptions::new().write(true).open(&path).unwrap(); + file.set_modified(std::time::SystemTime::now() + Duration::from_secs(60)).unwrap(); + drop(file); + let change = tracker.poll(false).unwrap(); + assert_eq!(change, Change { state: FileState::Current, epoch: 0, bumped: false }); + assert!(tracker.seen().unwrap().mtime_ms > 0); + } + + #[test] + fn shallow_poll_catches_a_same_size_edit_when_mtime_moves() { + let tmp = Tmp::new("mtime"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"aaaa").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let before = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"bbbb").unwrap(); + OpenOptions::new().write(true).open(&path).unwrap().set_modified(before + Duration::from_secs(10)).unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn shallow_poll_misses_an_edit_that_keeps_size_and_mtime_and_deep_poll_finds_it() { + let tmp = Tmp::new("deep"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"aaaa").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"bbbb").unwrap(); + OpenOptions::new().write(true).open(&path).unwrap().set_modified(mtime).unwrap(); + assert_eq!(tracker.poll(false).unwrap().state, FileState::Current, "documented limit of a shallow poll"); + let change = tracker.poll(true).unwrap(); + assert_eq!(change, Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn missing_then_same_content_returns_to_current_without_bump() { + let tmp = Tmp::new("missing"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 2).unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Missing, epoch: 2, bumped: false }); + assert!(tracker.seen().is_none()); + std::fs::write(&path, b"one").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Current, epoch: 2, bumped: false }); + } + + #[test] + fn missing_then_new_content_is_modified_with_a_bump() { + let tmp = Tmp::new("missing2"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 2).unwrap(); + std::fs::remove_file(&path).unwrap(); + tracker.poll(false).unwrap(); + std::fs::write(&path, b"different").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 3, bumped: true }); + } + + #[test] + fn tracker_for_a_file_that_does_not_exist_yet() { + let tmp = Tmp::new("pending"); + let path = tmp.0.join("later.png"); + let mut tracker = Tracker::new(&path, 0).unwrap(); + assert_eq!(tracker.state(), FileState::Missing); + write_atomic(&path, b"first").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn unreadable_path_is_an_error_not_missing() { + let tmp = Tmp::new("dir"); + let path = tmp.0.join("a.png"); + std::fs::create_dir(&path).unwrap(); + assert!(Tracker::new(&path, 0).is_err(), "a directory is not a result file"); + } + + #[cfg(unix)] + #[test] + fn a_fifo_at_the_path_is_an_error_and_never_blocks() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let tmp = Tmp::new("fifo"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 3).unwrap(); + std::fs::remove_file(&path).unwrap(); + let made = std::process::Command::new("mkfifo").arg(&path).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + // The open of a FIFO with no writer waits for one. Run each call where a hang fails the test instead of the run. + let (tx, rx) = std::sync::mpsc::channel(); + let watched = path.clone(); + std::thread::spawn(move || { + let polled = tracker.poll(true); + let kept = (tracker.state(), tracker.epoch()); + let _ = tx.send((polled, kept, Tracker::new(&watched, 0).map(|_| ()), fingerprint(&watched).map(|_| ()))); + }); + let (polled, kept, created, printed) = rx.recv_timeout(Duration::from_secs(5)).expect("a FIFO blocked the open"); + assert!(matches!(polled, Err(LinkError::Invalid(_))), "{polled:?}"); + assert_eq!(kept, (FileState::Current, 3), "a failed poll leaves the state as it was"); + assert!(matches!(created, Err(LinkError::Invalid(_))), "{created:?}"); + assert!(matches!(printed, Err(LinkError::Invalid(_))), "{printed:?}"); + } + + #[test] + fn relink_compares_hashes_and_rejects_a_missing_target() { + let tmp = Tmp::new("relink"); + let old = tmp.0.join("old.png"); + let same = tmp.0.join("same.png"); + let other = tmp.0.join("other.png"); + std::fs::write(&old, b"one").unwrap(); + std::fs::write(&same, b"one").unwrap(); + std::fs::write(&other, b"two").unwrap(); + let mut tracker = Tracker::new(&old, 7).unwrap(); + + let err = tracker.relink(tmp.0.join("nope.png")).unwrap_err(); + assert!(matches!(err, LinkError::Unknown(_)), "{err:?}"); + assert_eq!(tracker.path(), old.as_path(), "old path stays"); + + assert_eq!(tracker.relink(&same).unwrap(), Change { state: FileState::Current, epoch: 7, bumped: false }); + assert_eq!(tracker.path(), same.as_path()); + assert_eq!(tracker.relink(&other).unwrap(), Change { state: FileState::Modified, epoch: 8, bumped: true }); + } + + #[test] + fn a_bump_at_the_largest_epoch_is_an_error_and_changes_nothing() { + let tmp = Tmp::new("maxepoch"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let other = tmp.0.join("other.png"); + std::fs::write(&other, b"two").unwrap(); + let mut tracker = Tracker::new(&path, u64::MAX).unwrap(); + // No bump needed, no error. + assert_eq!(tracker.poll(true).unwrap(), Change { state: FileState::Current, epoch: u64::MAX, bumped: false }); + write_atomic(&path, b"changed").unwrap(); + for _ in 0..2 { + assert!(matches!(tracker.poll(true), Err(LinkError::Invalid(_)))); + assert_eq!((tracker.state(), tracker.epoch()), (FileState::Current, u64::MAX)); + } + assert!(matches!(tracker.relink(&other), Err(LinkError::Invalid(_)))); + assert_eq!(tracker.path(), path.as_path(), "the old path stays"); + assert_eq!((tracker.state(), tracker.epoch()), (FileState::Current, u64::MAX)); + } + + #[test] + fn adopt_epoch_never_decreases() { + let tmp = Tmp::new("adopt"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"x").unwrap(); + let mut tracker = Tracker::new(&path, 5).unwrap(); + tracker.adopt_epoch(3); + assert_eq!(tracker.epoch(), 5); + tracker.adopt_epoch(9); + assert_eq!(tracker.epoch(), 9); + } + + #[test] + fn write_atomic_replaces_whole_and_leaves_no_temp() { + let tmp = Tmp::new("atomic"); + let path = tmp.0.join("out.png"); + let first = write_atomic(&path, b"first").unwrap(); + assert_eq!(first.hash, sha_of(b"first")); + let second = write_atomic(&path, b"second version").unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"second version"); + assert_eq!(second.hash, sha_of(b"second version")); + let names: Vec = std::fs::read_dir(&tmp.0).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert_eq!(names, vec!["out.png".to_string()], "no temp file left"); + } + + #[test] + fn write_atomic_failure_keeps_the_old_file_and_removes_the_temp() { + let tmp = Tmp::new("atomic-fail"); + let path = tmp.0.join("out.png"); + write_atomic(&path, b"keep").unwrap(); + // A directory at the destination makes rename fail after the temp file is written. + let blocked = tmp.0.join("blocked"); + std::fs::create_dir(&blocked).unwrap(); + std::fs::write(blocked.join("child"), b"x").unwrap(); + assert!(write_atomic(&blocked, b"nope").is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"keep"); + let temps = std::fs::read_dir(&tmp.0).unwrap().filter(|e| is_temp_name(e.as_ref().unwrap().file_name().to_str().unwrap())).count(); + assert_eq!(temps, 0); + } + + #[test] + fn reader_never_sees_a_half_file_while_a_writer_replaces_it() { + let tmp = Tmp::new("race"); + let path = tmp.0.join("out.bin"); + let a = vec![b'a'; 512 * 1024]; + let b = vec![b'b'; 768 * 1024]; + write_atomic(&path, &a).unwrap(); + let writer_path = path.clone(); + let (wa, wb) = (a.clone(), b.clone()); + let writer = std::thread::spawn(move || { + for i in 0..60 { + write_atomic(&writer_path, if i % 2 == 0 { &wb } else { &wa }).unwrap(); + } + }); + while !writer.is_finished() { + let got = std::fs::read(&path).unwrap(); + assert!(got == a || got == b, "saw a torn file of {} bytes", got.len()); + } + writer.join().unwrap(); + } + + #[test] + fn temp_names() { + assert!(is_temp_name(".out.png.12.0.tmp")); + assert!(!is_temp_name("out.png")); + assert!(!is_temp_name("notes.tmp")); + } +} diff --git a/crates/ipc/src/server/dispatch.rs b/crates/ipc/src/server/dispatch.rs new file mode 100644 index 000000000..5c7e24d33 --- /dev/null +++ b/crates/ipc/src/server/dispatch.rs @@ -0,0 +1,316 @@ +//! Methods answered on the connection thread, plus the sink forward. + +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use super::events::{self, Subscription}; +use super::provider; +use super::{Activation, Assets, Hub, lock}; +use crate::error::LinkError; +use crate::moniker::Moniker; +use crate::payload::OpenPayload; +use crate::wire::{self, Request}; + +pub(super) enum Outcome { + Continue(Value), + Close(Option), + Bulk { reply: Value, ticket: String }, +} + +pub(super) fn dispatch(hub: &Hub, conn_id: u64, sub: &Subscription, first: bool, req: &Request) -> Outcome { + match req.method.as_str() { + "link.hello" => continue_result(&req.id, hello(hub, &req.params)), + "events.subscribe" => continue_result(&req.id, subscribe(sub, &req.params)), + "events.unsubscribe" => { + events::unsubscribe(sub); + Outcome::Continue(wire::reply_ok(&req.id, json!({}))) + } + "link.bye" => { + events::unsubscribe(sub); + release_tickets(hub, conn_id); + Outcome::Close(Some(wire::reply_ok(&req.id, json!({})))) + } + "app.activate" => continue_result(&req.id, activate(hub, &req.params)), + "link.resolve" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "link.resolve", &req.params, resolve)), + "asset.describe" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "asset.describe", &req.params, describe)), + "link.refresh" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "link.refresh", &req.params, refresh)), + "link.open" => match validate_open(&req.params) { + Ok(()) => forward(hub, &req.method, &req.id, req.params.clone()), + Err(err) => Outcome::Continue(wire::reply_err(&req.id, &err)), + }, + "frames.open" => continue_result(&req.id, frames_open(hub, conn_id, &req.params)), + "bulk.attach" => bulk_attach(hub, first, req), + method => forward(hub, method, &req.id, req.params.clone()), + } +} + +pub(super) fn release_tickets(hub: &Hub, conn_id: u64) { + let owned = { + let mut tickets = lock(&hub.tickets); + let owned: Vec = tickets.iter().filter(|(_, id)| **id == conn_id).map(|(ticket, _)| ticket.clone()).collect(); + for ticket in &owned { + tickets.remove(ticket); + } + owned + }; + for ticket in owned { + hub.frames.close_ticket(&ticket); + } +} + +fn continue_result(id: &Value, result: Result) -> Outcome { + match result { + Ok(result) => Outcome::Continue(wire::reply_ok(id, result)), + Err(err) => Outcome::Continue(wire::reply_err(id, &err)), + } +} + +fn hello(hub: &Hub, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let protocol = obj.get("protocol").and_then(Value::as_object).ok_or_else(|| LinkError::BadParams("protocol required".into()))?; + let min = required_u32(protocol.get("min"), "protocol.min")?; + let max = required_u32(protocol.get("max"), "protocol.max")?; + if !wire::protocols_overlap(min, max) { + return Err(LinkError::Protocol { server_min: wire::PROTOCOL_MIN, server_max: wire::PROTOCOL_MAX }); + } + let record = &hub.record; + let mut caps: Vec = ["events", "activate", "link.receive", "frames"].iter().map(|c| (*c).to_string()).collect(); + for cap in record.caps() { + if !caps.contains(&cap) { + caps.push(cap); + } + } + Ok(json!({ + "app": record.app, + "app_version": record.app_version, + "pid": record.pid, + "instance": record.instance, + "protocol": wire::PROTOCOL_MAX, + "caps": caps, + })) +} + +fn subscribe(sub: &Subscription, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let list = obj.get("events").and_then(Value::as_array).ok_or_else(|| LinkError::BadParams("events must be an array".into()))?; + let mut names = Vec::with_capacity(list.len()); + for item in list { + let Some(name) = item.as_str() else { + return Err(LinkError::BadParams("event name must be a string".into())); + }; + names.push(name.to_string()); + } + events::subscribe(sub, names); + Ok(json!({"seq": 0})) +} + +fn activate(hub: &Hub, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let token = optional_string(obj, "token")?; + let from_pid = optional_u32(obj.get("from_pid"), "from_pid")?; + let open = match obj.get("open") { + None => None, + Some(Value::Object(open)) => { + if let Some(asset) = open.get("asset") { + let text = asset.as_str().ok_or_else(|| LinkError::BadParams("open.asset must be a moniker string".into()))?; + Moniker::parse(text).map_err(|err| LinkError::BadParams(format!("open.asset: {err}")))?; + } + Some(Value::Object(open.clone())) + } + Some(_) => return Err(LinkError::BadParams("open must be an object".into())), + }; + let provider = std::sync::Arc::clone(&hub.activate); + let activated = match provider::call(hub, "app.activate", move || provider.activate(token, from_pid, open))? { + Activation::Yes => Value::Bool(true), + Activation::No => Value::Bool(false), + Activation::Requested => Value::String("requested".into()), + }; + Ok(json!({"activated": activated})) +} + +/// Run an asset method on a helper thread, with the deadline and the stop check of [`provider::call`]. +fn with_assets(hub: &Hub, what: &str, params: &Value, run: fn(&dyn Assets, &Value) -> Result) -> Result { + let Some(assets) = &hub.assets else { + return Err(LinkError::NotReady("this app has no asset provider".into())); + }; + let assets = std::sync::Arc::clone(assets); + let params = params.clone(); + provider::call(hub, what, move || run(assets.as_ref(), ¶ms))? +} + +fn asset_param(params: &Value) -> Result { + let obj = wire::params_object(params)?; + let text = obj.get("asset").and_then(Value::as_str).ok_or_else(|| LinkError::BadParams("asset must be a moniker string".into()))?; + Moniker::parse(text).map_err(|err| LinkError::BadParams(format!("asset: {err}"))) +} + +fn resolve(assets: &dyn Assets, params: &Value) -> Result { + let asset = asset_param(params)?; + Ok(serde_json::to_value(assets.resolve(&asset)?)?) +} + +fn describe(assets: &dyn Assets, params: &Value) -> Result { + let asset = asset_param(params)?; + Ok(serde_json::to_value(assets.describe(&asset)?)?) +} + +fn refresh(assets: &dyn Assets, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let link_id = obj + .get("link_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .ok_or_else(|| LinkError::BadParams("link_id must be a non-empty string".into()))?; + Ok(serde_json::to_value(assets.refresh(link_id)?)?) +} + +/// `link.open` carries a `payload` whose shape depends on `format`. A bad one +/// never reaches the app. Params that are not an object stay `bad_params`. +fn validate_open(params: &Value) -> Result<(), LinkError> { + let obj = wire::params_object(params)?; + let payload = obj.get("payload").ok_or_else(|| LinkError::PayloadInvalid("payload is required".into()))?; + OpenPayload::from_value(payload)?; + match obj.get("result_path") { + None | Some(Value::Null) => Ok(()), + Some(Value::String(path)) => crate::payload::check_peer_path(path, "result_path"), + Some(_) => Err(LinkError::PayloadInvalid("result_path must be a string".into())), + } +} + +fn frames_open(hub: &Hub, conn_id: u64, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let opened = hub.frames.open(obj)?; + if opened.ticket.len() != 32 || !crate::auth::is_lower_hex(&opened.ticket) { + hub.frames.close_ticket(&opened.ticket); + return Err(LinkError::BadParams("ticket must be 32 lowercase hex characters".into())); + } + let mut tickets = lock(&hub.tickets); + if tickets.contains_key(&opened.ticket) { + return Err(LinkError::BadParams("ticket already registered".into())); + } + tickets.insert(opened.ticket.clone(), conn_id); + drop(tickets); + Ok(json!({ + "ticket": opened.ticket, + "w": opened.w, + "h": opened.h, + "stride": opened.stride, + "len": opened.len, + })) +} + +fn bulk_attach(hub: &Hub, first: bool, req: &Request) -> Outcome { + if !first { + let err = LinkError::BadParams("bulk.attach must be the first method".into()); + return Outcome::Close(Some(wire::reply_err(&req.id, &err))); + } + let ticket = match ticket_param(&req.params) { + Ok(ticket) => ticket, + Err(err) => return Outcome::Close(Some(wire::reply_err(&req.id, &err))), + }; + let known = lock(&hub.tickets).contains_key(&ticket); + if !known { + let err = LinkError::BadParams("unknown ticket".into()); + return Outcome::Close(Some(wire::reply_err(&req.id, &err))); + } + Outcome::Bulk { reply: wire::reply_ok(&req.id, json!({})), ticket } +} + +fn ticket_param(params: &Value) -> Result { + let obj = wire::params_object(params)?; + let Some(ticket) = obj.get("ticket").and_then(Value::as_str) else { + return Err(LinkError::BadParams("ticket required".into())); + }; + Ok(ticket.to_string()) +} + +fn forward(hub: &Hub, method: &str, id: &Value, params: Value) -> Outcome { + let Some(rx) = hub.sink.submit(method.to_string(), params) else { + return Outcome::Close(None); + }; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + loop { + if hub.stop.load(std::sync::atomic::Ordering::SeqCst) { + return Outcome::Close(None); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Outcome::Continue(timeout_reply(method, id)); + } + let slice = left.min(Duration::from_millis(200)); + match rx.recv_timeout(slice) { + Ok(result) => return Outcome::Continue(adopt_sink_reply(id, result)), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Outcome::Close(None), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +/// FilmCraft's UI sends a finished reply object (`ok` plus `result` or `error`). +/// A value with `ok` is that object. Insert `id` and do not wrap it again. +/// Any other value is a result payload and gets [`wire::reply_ok`]. +fn adopt_sink_reply(id: &Value, value: Value) -> Value { + let Some(obj) = value.as_object() else { + return wire::reply_ok(id, value); + }; + if !obj.contains_key("ok") { + return wire::reply_ok(id, value); + } + let mut reply = value; + if let Some(map) = reply.as_object_mut() + && !map.contains_key("id") + { + map.insert("id".to_string(), id.clone()); + } + reply +} + +fn timeout_reply(method: &str, id: &Value) -> Value { + if is_new_method(method) { wire::reply_err(id, &LinkError::Timeout) } else { wire::reply_err_text(id, "timeout") } +} + +/// New methods use an error object. Legacy names keep the string `"timeout"`. +fn is_new_method(method: &str) -> bool { + matches!( + method, + "link.open" + | "link.attach" + | "link.focus" + | "link.close" + | "link.hello" + | "events.subscribe" + | "events.unsubscribe" + | "link.bye" + | "app.activate" + | "frames.open" + | "bulk.attach" + | "link.resolve" + | "asset.describe" + ) +} + +fn optional_string(obj: &Map, key: &str) -> Result, LinkError> { + match obj.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) => Ok(Some(text.clone())), + Some(_) => Err(LinkError::BadParams(format!("{key} must be a string or null"))), + } +} + +fn optional_u32(value: Option<&Value>, key: &str) -> Result, LinkError> { + match value { + None => Ok(None), + Some(value) => required_u32(Some(value), key).map(Some), + } +} + +fn required_u32(value: Option<&Value>, key: &str) -> Result { + let Some(value) = value else { + return Err(LinkError::BadParams(format!("{key} is required"))); + }; + let Some(number) = value.as_u64() else { + return Err(LinkError::BadParams(format!("{key} must be a number"))); + }; + u32::try_from(number).map_err(|_| LinkError::BadParams(format!("{key} is out of range"))) +} diff --git a/crates/ipc/src/server/events.rs b/crates/ipc/src/server/events.rs new file mode 100644 index 000000000..6de840964 --- /dev/null +++ b/crates/ipc/src/server/events.rs @@ -0,0 +1,214 @@ +//! Per-connection event queue. `emit` only enqueues. +//! +//! The writer sleeps a short grace before it drains so a burst fills the +//! queue of [`crate::wire::EVENT_QUEUE`] instead of racing one write at a time +//! into the socket buffer. Overflow occupies one of those slots. + +use std::collections::{HashSet, VecDeque}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use super::{Hub, lock}; +use crate::wire::{self, EVENT_QUEUE}; + +/// How long a burst may land in the queue before the writer drains it. +const BATCH_GRACE: Duration = Duration::from_millis(30); +const WAIT_SLICE: Duration = Duration::from_millis(200); + +pub(crate) struct Subscription { + pub id: u64, + filter: Mutex, + outbox: Mutex, + cv: Condvar, + closed: AtomicBool, +} + +struct Filter { + names: Option>, +} + +struct Outbox { + items: VecDeque, + latched: bool, + next_seq: u64, +} + +struct Queued { + line: Value, + overflow: bool, +} + +impl Subscription { + pub(super) fn new(id: u64) -> Arc { + Arc::new(Self { + id, + filter: Mutex::new(Filter { names: None }), + outbox: Mutex::new(Outbox { items: VecDeque::new(), latched: false, next_seq: 1 }), + cv: Condvar::new(), + closed: AtomicBool::new(false), + }) + } +} + +pub(super) fn subscribe(sub: &Subscription, names: Vec) { + let mut filter = lock(&sub.filter); + filter.names = Some(names.into_iter().collect()); +} + +pub(super) fn is_subscribed(sub: &Subscription) -> bool { + lock(&sub.filter).names.is_some() +} + +pub(super) fn unsubscribe(sub: &Subscription) { + let mut filter = lock(&sub.filter); + filter.names = None; +} + +pub(super) fn enqueue(sub: &Subscription, event: &str, data: &Value) { + if sub.closed.load(Ordering::SeqCst) { + return; + } + let interested = { + let filter = lock(&sub.filter); + filter.names.as_ref().is_some_and(|names| names.contains(event)) + }; + if !interested { + return; + } + { + let mut outbox = lock(&sub.outbox); + if sub.closed.load(Ordering::SeqCst) || outbox.latched { + return; + } + let line = wire::push_event(event, outbox.next_seq, data.clone()); + let oversize = serde_json::to_vec(&line).map(|bytes| bytes.len() > wire::MAX_LINE).unwrap_or(true); + if oversize || outbox.items.len() >= EVENT_QUEUE { + // A full queue gives up its newest slot. An event that is too large for one line + // is dropped alone: everything queued before it is still delivered. + let since = if outbox.items.len() >= EVENT_QUEUE { + outbox.items.pop_back().map(|item| item_seq(&item)).unwrap_or(outbox.next_seq) + } else { + outbox.items.back().map(item_seq).unwrap_or(outbox.next_seq.saturating_sub(1)) + }; + let seq = outbox.next_seq; + outbox.next_seq = outbox.next_seq.saturating_add(1); + outbox.items.push_back(Queued { overflow: true, line: wire::push_event("events.overflow", seq, json!({"since": since})) }); + outbox.latched = true; + } else { + outbox.next_seq = outbox.next_seq.saturating_add(1); + outbox.items.push_back(Queued { overflow: false, line }); + } + } + sub.cv.notify_one(); +} + +fn item_seq(item: &Queued) -> u64 { + item.line.get("seq").and_then(Value::as_u64).unwrap_or(0) +} + +pub(super) fn close_sub(sub: &Subscription) { + sub.closed.store(true, Ordering::SeqCst); + // The outbox lock pairs with the check in `write_loop`, so the notify + // cannot land before the writer has entered `wait`. + drop(lock(&sub.outbox)); + sub.cv.notify_one(); +} + +pub(super) fn wake_all(subs: &Mutex>>) { + let guard = lock(subs); + for sub in guard.iter() { + drop(lock(&sub.outbox)); + sub.cv.notify_one(); + } +} + +pub(super) fn remove_sub(hub: &Hub, id: u64) { + lock(&hub.subs).retain(|sub| sub.id != id); +} + +pub(super) fn write_loop(sub: Arc, writer: Arc>>, stop: &AtomicBool) { + loop { + { + let guard = lock(&sub.outbox); + if sub.closed.load(Ordering::SeqCst) || stop.load(Ordering::SeqCst) { + return; + } + if guard.items.is_empty() { + let waited = sub.cv.wait_timeout(guard, WAIT_SLICE); + let (guard, _) = match waited { + Ok(pair) => pair, + Err(err) => err.into_inner(), + }; + drop(guard); + continue; + } + } + std::thread::sleep(BATCH_GRACE); + if sub.closed.load(Ordering::SeqCst) || stop.load(Ordering::SeqCst) { + return; + } + let batch = { + let mut guard = lock(&sub.outbox); + std::mem::take(&mut guard.items) + }; + for item in batch { + let wrote = { + let mut guard = lock(&writer); + wire::write_line(&mut *guard, &item.line).is_ok() + }; + if item.overflow { + lock(&sub.outbox).latched = false; + } + if !wrote { + sub.closed.store(true, Ordering::SeqCst); + return; + } + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn an_event_over_the_line_limit_becomes_an_overflow_marker() { + let sub = Subscription::new(8); + subscribe(&sub, vec!["doc.changed".into()]); + enqueue(&sub, "doc.changed", &json!({"n": 1})); + enqueue(&sub, "doc.changed", &json!({"blob": "x".repeat(wire::MAX_LINE)})); + let outbox = lock(&sub.outbox); + assert_eq!(outbox.items.len(), 2, "the earlier event stays queued"); + assert_eq!(outbox.items[0].line["event"], "doc.changed"); + assert_eq!(outbox.items[1].line["event"], "events.overflow"); + assert_eq!(outbox.items[1].line["data"]["since"], 1); + assert!(outbox.items.iter().all(|item| serde_json::to_vec(&item.line).unwrap().len() <= wire::MAX_LINE)); + } + + #[test] + fn overflow_occupies_one_queue_slot() { + let sub = Subscription::new(7); + subscribe(&sub, vec!["doc.changed".into()]); + enqueue(&sub, "other", &json!({"n": 0})); + assert!(lock(&sub.outbox).items.is_empty()); + for n in 0..EVENT_QUEUE { + enqueue(&sub, "doc.changed", &json!({"n": n})); + } + enqueue(&sub, "doc.changed", &json!({"n": EVENT_QUEUE})); + enqueue(&sub, "doc.changed", &json!({"n": EVENT_QUEUE + 1})); + let outbox = lock(&sub.outbox); + assert!(outbox.latched); + assert_eq!(outbox.items.len(), EVENT_QUEUE); + let overflow: Vec<_> = outbox.items.iter().filter(|item| item.overflow).collect(); + assert_eq!(overflow.len(), 1); + assert_eq!(outbox.items.iter().filter(|item| !item.overflow).count(), EVENT_QUEUE - 1); + let last = overflow[0]; + assert_eq!(last.line["event"], json!("events.overflow")); + assert!(last.line["data"]["since"].is_number()); + assert_eq!(outbox.items.front().expect("queued").line["seq"], json!(1)); + } +} diff --git a/crates/ipc/src/server/listener.rs b/crates/ipc/src/server/listener.rs new file mode 100644 index 000000000..eec2402e7 --- /dev/null +++ b/crates/ipc/src/server/listener.rs @@ -0,0 +1,633 @@ +//! Local socket accept loop, auth, and the per-connection threads. +//! +//! The 17th connection receives one refusal line and does not take a slot. +//! Bulk connections count toward the same cap. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use interprocess::TryClone; +#[cfg(unix)] +use interprocess::local_socket::GenericFilePath; +use interprocess::local_socket::prelude::*; +use interprocess::local_socket::{Listener, ListenerNonblockingMode, ListenerOptions, Stream}; +use serde_json::Value; + +use super::dispatch::{self, Outcome}; +use super::events::{self, Subscription}; +use super::tcp::BoundTcp; +use super::{Hub, lock}; +use crate::discovery::{Endpoint, RuntimeDir}; +use crate::error::LinkError; +use crate::wire::{self, LineReader, MAX_LINE}; + +const READ_POLL: Duration = Duration::from_millis(250); +const REFUSE_WRITE: Duration = Duration::from_secs(2); + +pub(super) fn bind(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + #[cfg(unix)] + { + bind_unix(runtime, app, pid) + } + #[cfg(windows)] + { + let _ = runtime; + bind_pipe(app, pid) + } +} + +pub(super) fn endpoint(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + #[cfg(unix)] + { + runtime.unix_endpoint(app, pid) + } + #[cfg(windows)] + { + let _ = runtime; + Ok(Endpoint::Pipe { name: pipe_name(app, pid) }) + } +} + +#[cfg(unix)] +fn bind_unix(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + let path = runtime.socket_file(app, pid); + let name = path.to_fs_name::()?; + let listener = ListenerOptions::new().name(name).nonblocking(ListenerNonblockingMode::Accept).reclaim_name(false).create_sync()?; + Ok(listener) +} + +/// Owner and SYSTEM only. A peer pid is not an allow; Windows `PeerCreds` has no user id. +#[cfg(windows)] +fn bind_pipe(app: &str, pid: u32) -> Result { + use interprocess::local_socket::GenericNamespaced; + use interprocess::os::windows::local_socket::ListenerOptionsExt; + use interprocess::os::windows::security_descriptor::SecurityDescriptor; + use widestring::U16CString; + + let wide = U16CString::from_str("D:P(A;;GA;;;OW)(A;;GA;;;SY)").map_err(LinkError::io)?; + let descriptor = SecurityDescriptor::deserialize(wide.as_ucstr())?; + let name = pipe_name(app, pid).to_ns_name::()?; + let listener = ListenerOptions::new() + .name(name) + .nonblocking(ListenerNonblockingMode::Accept) + .reclaim_name(false) + .security_descriptor(descriptor) + .create_sync()?; + Ok(listener) +} + +#[cfg(windows)] +fn pipe_name(app: &str, pid: u32) -> String { + match windows_user() { + Some(user) => format!("craft-{user}-{app}-{pid}"), + None => format!("craft-{app}-{pid}"), + } +} + +/// `USERNAME` with non-alphanumeric bytes replaced. Reading the env is enough; +/// there is no dependency-free Windows user API in this crate. +#[cfg(windows)] +fn windows_user() -> Option { + let raw = std::env::var("USERNAME").ok()?; + if raw.is_empty() { + return None; + } + let mut cleaned = String::with_capacity(raw.len()); + for ch in raw.chars() { + if ch.is_ascii_alphanumeric() { + cleaned.push(ch); + } else { + cleaned.push('_'); + } + } + let trimmed = cleaned.trim_matches('_'); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } +} + +pub(super) fn accept_loop(listener: Listener, tcp: Option, hub: Arc) { + while !hub.stop.load(Ordering::SeqCst) { + reap(&hub); + let mut busy = false; + match listener.accept() { + Ok(stream) => { + busy = true; + accept_local(stream, &hub); + } + Err(err) if idle_accept(&err) => {} + Err(_) => { + std::thread::sleep(Duration::from_millis(10)); + continue; + } + } + if let Some(bound) = tcp.as_ref() + && let Ok(Some(stream)) = super::tcp::poll_accept(&bound.listener) + { + busy = true; + accept_tcp(stream, &hub); + } + if !busy { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +fn idle_accept(err: &std::io::Error) -> bool { + matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted) +} + +fn reap(hub: &Hub) { + let mut threads = lock(&hub.threads); + let pending = std::mem::take(&mut *threads); + for handle in pending { + if handle.is_finished() { + let _ = handle.join(); + } else { + threads.push(handle); + } + } +} + +/// One connection slot. A connection that has to authenticate also holds a share of the small +/// unauthenticated pool until it does. The slots outside that pool are only for peers that proved +/// the token, so silent peers cannot fill the server. +struct Slot { + hub: Arc, + pending: bool, +} + +/// Why [`Slot::admit`] said no. +enum Refusal { + Full, + Unauthenticated, +} + +impl Refusal { + fn text(&self) -> &'static str { + match self { + Self::Full => "too many control connections", + Self::Unauthenticated => "too many unauthenticated connections", + } + } +} + +/// Add one to `counter` unless it is at `cap`. +fn take_one(counter: &AtomicUsize, cap: usize) -> bool { + loop { + let cur = counter.load(Ordering::SeqCst); + if cur >= cap { + return false; + } + if counter.compare_exchange(cur, cur + 1, Ordering::SeqCst, Ordering::SeqCst).is_ok() { + return true; + } + } +} + +impl Slot { + fn admit(hub: &Arc, needs_auth: bool) -> Result { + if !take_one(&hub.open, wire::MAX_CONNECTIONS) { + return Err(Refusal::Full); + } + if needs_auth && !take_one(&hub.pending, hub.limits.max_unauthenticated.min(wire::MAX_CONNECTIONS)) { + hub.open.fetch_sub(1, Ordering::SeqCst); + return Err(Refusal::Unauthenticated); + } + Ok(Slot { hub: Arc::clone(hub), pending: needs_auth }) + } + + /// The peer sent the right token. It leaves the unauthenticated pool and keeps its slot. + fn authenticated(&mut self) { + if self.pending { + self.pending = false; + self.hub.pending.fetch_sub(1, Ordering::SeqCst); + } + } +} + +impl Drop for Slot { + fn drop(&mut self) { + self.authenticated(); + self.hub.open.fetch_sub(1, Ordering::SeqCst); + } +} + +fn accept_local(mut stream: Stream, hub: &Arc) { + let _ = stream.set_send_timeout(Some(REFUSE_WRITE)); + let slot = match Slot::admit(hub, true) { + Ok(slot) => slot, + Err(refusal) => { + refuse(&mut stream, &refusal); + return; + } + }; + if crate::ensure_peer_user(&stream).is_err() { + let _ = wire::write_line(&mut stream, &wire::auth_failure()); + drop(slot); + return; + } + let _ = stream.set_recv_timeout(Some(READ_POLL)); + let Ok(writer_stream) = stream.try_clone() else { + return; + }; + #[cfg(windows)] + let (reader, writer): (Box, Box) = { + let (Ok(reader), Ok(writer)) = (crate::client::PipeTransport::new(stream), crate::client::PipeTransport::new(writer_stream)) else { + return; + }; + reader.set_timeouts(READ_POLL, REFUSE_WRITE); + writer.set_timeouts(READ_POLL, REFUSE_WRITE); + (Box::new(reader), Box::new(writer)) + }; + #[cfg(not(windows))] + let (reader, writer): (Box, Box) = (Box::new(stream), Box::new(writer_stream)); + spawn(slot, Arc::clone(hub), BufReader::new(reader), Arc::new(std::sync::Mutex::new(writer)), true); +} + +fn accept_tcp(mut stream: std::net::TcpStream, hub: &Arc) { + let _ = stream.set_write_timeout(Some(REFUSE_WRITE)); + let slot = match Slot::admit(hub, true) { + Ok(slot) => slot, + Err(refusal) => { + refuse(&mut stream, &refusal); + return; + } + }; + // The listener is bound to 127.0.0.1. Refuse anything else anyway. + if !stream.peer_addr().is_ok_and(|addr| addr.ip().is_loopback()) { + let _ = wire::write_line(&mut stream, &wire::auth_failure()); + return; + } + let _ = stream.set_read_timeout(Some(READ_POLL)); + let Ok(writer_stream) = stream.try_clone() else { + return; + }; + let reader: Box = Box::new(stream); + let writer: Box = Box::new(writer_stream); + spawn(slot, Arc::clone(hub), BufReader::new(reader), Arc::new(std::sync::Mutex::new(writer)), true); +} + +fn refuse(stream: &mut impl Write, refusal: &Refusal) { + let line = serde_json::json!({"ok": false, "error": refusal.text()}); + let _ = wire::write_line(stream, &line); +} + +fn spawn(slot: Slot, hub: Arc, reader: BufReader>, writer: Arc>>, auth: bool) { + let thread_hub = Arc::clone(&hub); + let started = std::thread::Builder::new().name("craft-ipc-conn".into()).spawn(move || { + let mut slot = slot; + connection(thread_hub, reader, writer, auth, &mut slot); + }); + if let Ok(handle) = started { + lock(&hub.threads).push(handle); + } +} + +struct Cleanup { + hub: Arc, + conn_id: u64, + sub: Arc, + writer: Option>, +} + +impl Drop for Cleanup { + fn drop(&mut self) { + events::close_sub(&self.sub); + if let Some(handle) = self.writer.take() { + let _ = handle.join(); + } + events::remove_sub(&self.hub, self.conn_id); + dispatch::release_tickets(&self.hub, self.conn_id); + } +} + +fn connection( + hub: Arc, + mut reader: BufReader>, + writer: Arc>>, + auth: bool, + slot: &mut Slot, +) { + let first_patience = if auth { hub.limits.partial_line.min(hub.limits.auth_deadline) } else { hub.limits.partial_line }; + let mut lines = LineReader::new(MAX_LINE).with_patience(first_patience); + if auth && authenticate(&hub, &mut reader, &mut lines, &writer).is_err() { + return; + } + slot.authenticated(); + lines.set_patience(hub.limits.partial_line); + let conn_id = hub.next_id.fetch_add(1, Ordering::SeqCst); + let sub = Subscription::new(conn_id); + lock(&hub.subs).push(Arc::clone(&sub)); + let write_sub = Arc::clone(&sub); + let write_io = Arc::clone(&writer); + let write_stop = Arc::clone(&hub); + let writer_thread = std::thread::Builder::new().name("craft-ipc-write".into()).spawn(move || { + events::write_loop(write_sub, write_io, &write_stop.stop); + }); + let Ok(writer_thread) = writer_thread else { + events::remove_sub(&hub, conn_id); + return; + }; + let mut cleanup = Cleanup { hub: Arc::clone(&hub), conn_id, sub: Arc::clone(&sub), writer: Some(writer_thread) }; + let mut first = true; + let mut last_input = Instant::now(); + loop { + if hub.stop.load(Ordering::SeqCst) { + break; + } + let line = match next_line(&hub, &mut reader, &mut lines) { + Incoming::Again => { + if lines.partial_age().is_some_and(|age| age >= hub.limits.partial_line) { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, "request line not finished in time; closing the connection")); + break; + } + if lines.partial_age().is_none() && last_input.elapsed() >= hub.limits.idle && !events::is_subscribed(&sub) { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, "idle connection closed")); + break; + } + continue; + } + Incoming::Stop => break, + Incoming::TooLarge => { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, &overlong())); + break; + } + Incoming::Line(bytes) => bytes, + }; + last_input = Instant::now(); + let req = match wire::parse_request(&line) { + Ok(None) => continue, + Ok(Some(req)) => req, + Err(text) => { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, &text)); + break; + } + }; + let outcome = dispatch::dispatch(&hub, conn_id, &sub, first, &req); + first = false; + match outcome { + Outcome::Continue(reply) => { + if write_value(&writer, &reply).is_err() { + break; + } + } + Outcome::Close(reply) => { + if let Some(reply) = reply { + let _ = write_value(&writer, &reply); + } + break; + } + Outcome::Bulk { reply, ticket } => { + events::close_sub(&sub); + if let Some(handle) = cleanup.writer.take() { + let _ = handle.join(); + } + if write_value(&writer, &reply).is_err() { + break; + } + bulk_loop(&mut reader, &writer, &hub, &ticket); + break; + } + } + } + drop(cleanup); +} + +fn authenticate( + hub: &Hub, + reader: &mut BufReader>, + lines: &mut LineReader, + writer: &Arc>>, +) -> Result<(), ()> { + let deadline = Instant::now() + hub.limits.auth_deadline; + loop { + if hub.stop.load(Ordering::SeqCst) { + return Err(()); + } + let line = match next_line(hub, reader, lines) { + Incoming::Again if Instant::now() >= deadline => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Incoming::Again => continue, + Incoming::Stop | Incoming::TooLarge => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + // A line that completes after the deadline is too late, even when the read poll has not run since. + Incoming::Line(_) if Instant::now() >= deadline => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Incoming::Line(bytes) => bytes, + }; + match wire::parse_request(&line) { + Ok(None) if Instant::now() < deadline => continue, + Ok(None) => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Ok(Some(req)) if req.method == "auth" && token_ok(&req.params, &hub.record.token) => { + return write_value(writer, &wire::auth_success(&req.id)).map_err(|_| ()); + } + Ok(Some(_)) | Err(_) => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + } + } +} + +fn token_ok(params: &Value, expected: &str) -> bool { + params.get("token").and_then(Value::as_str).is_some_and(|token| crate::tokens_equal(expected, token)) +} + +enum Incoming { + Line(Vec), + Again, + TooLarge, + Stop, +} + +fn next_line(hub: &Hub, reader: &mut BufReader>, lines: &mut LineReader) -> Incoming { + match lines.read(reader) { + Ok(Some(bytes)) => Incoming::Line(bytes), + Ok(None) => Incoming::Stop, + Err(LinkError::TooLarge) => Incoming::TooLarge, + Err(err) if is_timeout(&err) => { + if hub.stop.load(Ordering::SeqCst) { + Incoming::Stop + } else { + Incoming::Again + } + } + Err(_) => Incoming::Stop, + } +} + +fn is_timeout(err: &LinkError) -> bool { + if matches!(err, LinkError::Timeout) { + return true; + } + let LinkError::Io(msg) = err else { + return false; + }; + let lower = msg.to_ascii_lowercase(); + lower.contains("timed out") || lower.contains("timeout") +} + +fn overlong() -> String { + format!("request line longer than {MAX_LINE} bytes; closing the connection") +} + +fn write_value(writer: &Arc>>, value: &Value) -> Result<(), LinkError> { + let mut guard = lock(writer); + wire::write_reply_line(&mut *guard, value) +} + +/// Retries reads that time out until `deadline`, so a record split across the read poll is whole. +struct Patient<'a, R> { + inner: &'a mut R, + deadline: Instant, + stop: &'a std::sync::atomic::AtomicBool, +} + +impl Read for Patient<'_, R> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + loop { + match self.inner.read(out) { + Err(err) if idle_accept(&err) && Instant::now() < self.deadline && !self.stop.load(Ordering::SeqCst) => {} + other => return other, + } + } + } +} + +fn bulk_loop(reader: &mut BufReader>, writer: &Arc>>, hub: &Hub, ticket: &str) { + loop { + if hub.stop.load(Ordering::SeqCst) { + return; + } + // Wait for the first byte of a request. A timeout here loses nothing. + match reader.fill_buf() { + Ok([]) => return, + Ok(_) => {} + Err(err) if idle_accept(&err) => continue, + Err(_) => return, + } + // The record has started. A pause inside it is not a new request, so keep + // reading until it ends, the server stops, or the peer takes too long. + let mut patient = Patient { inner: &mut *reader, deadline: Instant::now() + hub.limits.partial_line, stop: &hub.stop }; + let header = match wire::read_cfrq(&mut patient) { + Ok(header) => header, + Err(_) => return, + }; + match hub.frames.render(ticket, &header) { + Ok((hdr, payload)) => { + if send_frame(writer, &header, &hdr, &payload).is_err() { + return; + } + } + Err(err) => { + let fail = error_header(&header, &err.to_string()); + if write_frame(writer, &fail, &[]).is_err() { + return; + } + } + } + } +} + +/// Send one frame record. A record that cannot be built (bad header, payload over the limit) has +/// written nothing, so an error record takes its place. A failure while writing may leave part of a +/// record on the stream, and a second record would read as its tail: that error is returned alone +/// and the caller closes the connection. +fn send_frame(writer: &Arc>>, request: &Value, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + if let Err(err) = wire::check_cfrm(header, payload) { + let fail = error_header(request, &err.to_string()); + write_frame(writer, &fail, &[])?; + return Err(err); + } + write_frame(writer, header, payload) +} + +fn error_header(request: &Value, message: &str) -> Value { + let req = request.get("req").cloned().unwrap_or(Value::Null); + serde_json::json!({"req": req, "error": message}) +} + +fn write_frame(writer: &Arc>>, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + let mut guard = lock(writer); + wire::write_cfrm(&mut *guard, header, payload) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::sync::Mutex; + + use serde_json::json; + + use super::*; + + /// Takes `room` bytes, fails one write, then takes everything: a socket whose send timeout + /// fired mid-record and whose peer then drained the buffer. + struct Choked { + seen: Arc>>, + room: usize, + failed: bool, + } + + impl Write for Choked { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.room == 0 && !self.failed { + self.failed = true; + self.room = usize::MAX; + return Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "choked")); + } + let take = buf.len().min(self.room); + self.seen.lock().unwrap().extend_from_slice(&buf[..take]); + self.room = self.room.saturating_sub(take); + Ok(take) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + type Writer = Arc>>; + + fn rig(room: usize) -> (Writer, Arc>>) { + let seen = Arc::new(Mutex::new(Vec::new())); + let writer: Box = Box::new(Choked { seen: Arc::clone(&seen), room, failed: false }); + (Arc::new(Mutex::new(writer)), seen) + } + + #[test] + fn a_failed_write_is_not_followed_by_an_error_record() { + let (writer, seen) = rig(10); + let request = json!({"req": 7}); + let header = json!({"req": 7, "w": 1}); + let payload = vec![1u8; 64]; + assert!(send_frame(&writer, &request, &header, &payload).is_err()); + let mut whole = Vec::new(); + wire::write_cfrm(&mut whole, &header, &payload).unwrap(); + assert_eq!(*seen.lock().unwrap(), whole[..10].to_vec(), "only the bytes of the first record, and no error record after them"); + } + + #[test] + fn a_record_that_cannot_be_built_gets_an_error_record() { + let (writer, seen) = rig(usize::MAX); + let request = json!({"req": 7}); + // A header that is not an object is refused before any byte is written. + assert!(matches!(send_frame(&writer, &request, &json!("no"), b"x"), Err(LinkError::BadParams(_)))); + let written = seen.lock().unwrap().clone(); + let (header, payload) = wire::read_cfrm(&mut std::io::Cursor::new(written)).unwrap(); + assert_eq!(header["req"], json!(7)); + assert!(header["error"].as_str().is_some_and(|text| text.contains("object")), "{header}"); + assert!(payload.is_empty()); + } +} diff --git a/crates/ipc/src/server/mod.rs b/crates/ipc/src/server/mod.rs new file mode 100644 index 000000000..7b28543a0 --- /dev/null +++ b/crates/ipc/src/server/mod.rs @@ -0,0 +1,376 @@ +//! Craft Link server. +//! +//! One accept thread, then a read thread and a write thread per connection. +//! There is no tokio runtime here. +//! +//! Shutdown. [`Server::shutdown`] and drop set the stop flag, then join the accept thread and every +//! connection thread. A connection thread leaves its loop within one read poll (250 ms) of the +//! flag. Calls into app code differ in what they can delay that: +//! +//! - [`ControlSink`], [`Assets`] and [`Activate`] calls never delay it for longer than a stop poll +//! (at most 200 ms). The server waits for a sink reply, or for an `Assets` or `Activate` answer, in +//! short slices and gives up when the flag is set. An `Assets` or `Activate` call runs on a helper +//! thread that the server detaches when it gives up, so a provider that never returns leaves one +//! idle thread behind and delays nothing. See [`Limits::provider_deadline`]. +//! - [`Frames`] calls run on the connection thread. The flag is checked before each request, but a +//! call that is already running is waited for. A `Frames` method that never returns blocks shutdown. + +mod dispatch; +mod events; +mod listener; +mod provider; +mod tcp; + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::Value; + +use crate::asset::{Described, Refreshed, Resolved}; +use crate::discovery::{Discovery, InstanceRecord, RuntimeDir}; +use crate::error::LinkError; +use crate::moniker::Moniker; + +/// App request channel. +/// +/// `submit` returns `None` when the app is gone. The connection then closes. +/// The `Receiver` yields either a result payload or a finished reply object. +/// A payload is wrapped with [`crate::wire::reply_ok`]. An object that already +/// contains `ok` is passed through, and `id` is inserted when it is missing. +/// This signature cannot carry a [`LinkError`]. +/// A sink that never replies hits [`crate::wire::REPLY_DEADLINE_SECS`]. +pub trait ControlSink: Send + Sync { + /// None means the app is gone. The connection must close. + fn submit(&self, method: String, params: Value) -> Option>; +} + +/// What [`Activate::activate`] decided. The server serializes this as JSON +/// `true`, `false`, or the string `"requested"`. It does not rewrite the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Activation { + /// The window has focus. JSON `true`. + Yes, + /// Focus was not taken. The app asked for user attention instead. JSON `false`. + No, + /// The app asked the OS to focus the window. The OS decides later and + /// nobody confirmed it. JSON `"requested"`. + Requested, +} + +/// Raises or focuses the app. There is no default success. +/// +/// A call runs on a helper thread with a time limit of [`Limits::provider_deadline`]. On timeout the +/// caller gets a timeout reply and the server detaches the thread. The thread keeps running until +/// `activate` returns, possibly after the server is gone, so an implementation must not rely on the +/// server being alive when it returns. +pub trait Activate: Send + Sync { + fn activate(&self, token: Option, from_pid: Option, open: Option) -> Activation; +} + +/// One bulk ticket from [`Frames::open`]. `ticket` is 32 lowercase hex characters. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameOpen { + pub ticket: String, + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +/// Frame tickets and rendered payloads. The server registers the ticket on the +/// control connection that called `frames.open`. +/// +/// Every method runs on a connection thread, one call per frame, with no helper thread and no +/// time limit. The server checks its stop flag before it reads each request, so shutdown takes +/// effect between frames. It cannot interrupt a call that is running: [`Server::shutdown`] and +/// drop wait for it, and `close_ticket` runs when the connection ends. Each method must return in +/// bounded time, for example by putting a timeout on any I/O it does. +pub trait Frames: Send + Sync { + fn open(&self, params: &serde_json::Map) -> Result; + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError>; + fn close_ticket(&self, ticket: &str); +} + +/// Answers asset questions without a UI frame (`link.resolve`, `asset.describe`, +/// `link.refresh`). Pass one in [`Extensions::assets`]. Without it these three +/// methods go to the [`ControlSink`] like any other link method. +/// +/// Methods do not run on the UI thread, so an implementation must be thread safe +/// and must not wait on the UI thread. +/// +/// A call runs on a helper thread with a time limit of [`Limits::provider_deadline`]. On timeout the +/// caller gets a timeout reply and the server detaches the thread. A call that never returns +/// leaves one thread behind and does not hold up [`Server::shutdown`]. The server keeps at most +/// 32 such threads, and a call past that gets a `busy` error. The thread keeps running until the +/// method returns, possibly after the server is gone, so an implementation must not rely on the +/// server being alive when it returns. +pub trait Assets: Send + Sync { + /// Where the last-known result of `asset` is. `Err(LinkError::Unknown)` when + /// the asset or its result file does not exist. + fn resolve(&self, asset: &Moniker) -> Result; + /// Name, kind and thumbnail of `asset`. + fn describe(&self, asset: &Moniker) -> Result; + /// Re-check the file behind `link_id` and report its state. + fn refresh(&self, link_id: &str) -> Result; +} + +/// Optional parts for [`Server::start_with`]. The default adds nothing. +#[derive(Default)] +pub struct Extensions { + /// Answers asset methods. Adds the `assets` cap. + pub assets: Option>, + /// Extra caps for the instance record and `link.hello`, such as `hub`. + pub caps: Vec, +} + +/// What [`Server::start`] needs. `tcp_port: None` does not bind a TCP socket. +pub struct ServerConfig { + pub app: String, + pub app_version: String, + pub instance: Option, + pub runtime: Option, + pub tcp_port: Option, + pub sink: Arc, + pub activate: Arc, + pub frames: Arc, +} + +/// Time and count limits on connections. `Limits::default()` is what the apps run. +/// Tests shorten the times through [`Server::start_with_limits`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// A new connection must send a valid `auth` line within this time, or it is closed. + pub auth_deadline: Duration, + /// A request line, once started, must end within this time, or the connection is closed. + pub partial_line: Duration, + /// An authenticated control connection that sends nothing for this long is closed. A + /// connection with an event subscription and a bulk connection are exempt. + pub idle: Duration, + /// Connections that have not authenticated yet. At most `MAX_CONNECTIONS` minus this many + /// slots can be held by them. + pub max_unauthenticated: usize, + /// How long an `Assets` or `Activate` call may run before the caller gets a timeout reply and + /// the server detaches the helper thread that runs it. + pub provider_deadline: Duration, +} + +impl Default for Limits { + fn default() -> Self { + Self { + auth_deadline: Duration::from_secs(2), + partial_line: Duration::from_secs(10), + idle: Duration::from_secs(60 * 60), + max_unauthenticated: crate::wire::MAX_UNAUTHENTICATED, + provider_deadline: Duration::from_secs(crate::wire::REPLY_DEADLINE_SECS), + } + } +} + +pub(super) struct Hub { + pub(super) limits: Limits, + pub(super) pending: AtomicUsize, + pub(super) record: InstanceRecord, + pub(super) sink: Arc, + pub(super) activate: Arc, + pub(super) frames: Arc, + pub(super) assets: Option>, + pub(super) stop: AtomicBool, + pub(super) open: AtomicUsize, + /// Live `Assets` and `Activate` helper threads, stuck ones included. See `provider`. + pub(super) providers: Arc, + pub(super) next_id: AtomicU64, + pub(super) tickets: Mutex>, + pub(in crate::server) subs: Mutex>>, + pub(super) threads: Mutex>>, +} + +/// A running link endpoint. [`Server::shutdown`] removes the instance record. +/// Drop only joins the accept thread and the connection threads. A helper thread that runs a +/// stuck `Assets` or `Activate` call is detached and not joined. +pub struct Server { + discovery: Option, + hub: Arc, + accept: Option>, + tcp_port: Option, + record: InstanceRecord, +} + +impl Server { + /// Bind, publish, then accept. Scan runs before the bind so a dead socket + /// from an earlier process is unlinked. A lock that is still held is left + /// alone. The instance record keeps the local socket as its endpoint. When `tcp_port` is set the + /// record also carries the bound port (`record.tcp_port()`), and the TCP listener requires the + /// same `auth` token line as the socket. + pub fn start(config: ServerConfig) -> Result { + Self::start_inner(config, Extensions::default(), Limits::default()) + } + + /// Like [`Server::start`], plus the optional parts in [`Extensions`]. + pub fn start_with(config: ServerConfig, extensions: Extensions) -> Result { + Self::start_inner(config, extensions, Limits::default()) + } + + /// Like [`Server::start_with`], with connection limits other than the defaults. + pub fn start_with_limits(config: ServerConfig, extensions: Extensions, limits: Limits) -> Result { + Self::start_inner(config, extensions, limits) + } + + fn start_inner(config: ServerConfig, extensions: Extensions, limits: Limits) -> Result { + let runtime = match config.runtime { + Some(dir) => dir, + None => RuntimeDir::resolve()?, + }; + let _existing = Discovery::scan(&runtime)?; + let pid = std::process::id(); + #[cfg(unix)] + let socket_path = runtime.socket_file(&config.app, pid); + let listener = listener::bind(&runtime, &config.app, pid)?; + #[cfg(unix)] + if let Err(err) = crate::discovery::enforce_mode(&socket_path, 0o600) { + drop(listener); + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + let endpoint = listener::endpoint(&runtime, &config.app, pid)?; + let mut record = match InstanceRecord::new(&config.app, &config.app_version, pid, endpoint) { + Ok(record) => record, + Err(err) => { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + }; + let Extensions { assets, caps } = extensions; + let mut wanted = caps; + if assets.is_some() { + wanted.push("assets".to_string()); + } + for cap in &wanted { + if let Err(err) = record.add_cap(cap) { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + } + if let Some(instance) = config.instance { + record.instance = instance; + if let Err(err) = record.validate() { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + } + // Bind TCP before publishing so the record carries the real port (port 0 picks one). + let tcp = match config.tcp_port { + Some(port) => match tcp::bind_loopback(port) { + Ok(bound) => Some(bound), + Err(err) => { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + }, + None => None, + }; + let tcp_port = tcp.as_ref().map(|bound| bound.port); + if let Some(port) = tcp_port { + record.set_tcp_port(port); + } + let discovery = match Discovery::publish(&runtime, record.clone()) { + Ok(discovery) => discovery, + Err(err) => { + drop(listener); + return Err(err); + } + }; + let hub = Arc::new(Hub { + limits, + pending: AtomicUsize::new(0), + record: record.clone(), + sink: config.sink, + activate: config.activate, + frames: config.frames, + assets, + stop: AtomicBool::new(false), + open: AtomicUsize::new(0), + providers: Arc::new(AtomicUsize::new(0)), + next_id: AtomicU64::new(1), + tickets: Mutex::new(HashMap::new()), + subs: Mutex::new(Vec::new()), + threads: Mutex::new(Vec::new()), + }); + let accept_hub = Arc::clone(&hub); + let accept = match std::thread::Builder::new().name("craft-ipc-accept".into()).spawn(move || { + listener::accept_loop(listener, tcp, accept_hub); + }) { + Ok(handle) => handle, + Err(err) => { + let _ = discovery.remove(); + return Err(err.into()); + } + }; + Ok(Self { discovery: Some(discovery), hub, accept: Some(accept), tcp_port, record }) + } + + pub fn record(&self) -> &InstanceRecord { + &self.record + } + + /// Queue `event` on subscribed connections. This returns after the enqueue, + /// not after the socket write. A full per-connection queue records one + /// `events.overflow` and then drops until that overflow line is written. + pub fn emit(&self, event: &str, data: Value) -> Result<(), LinkError> { + let subs = lock(&self.hub.subs); + for sub in subs.iter() { + events::enqueue(sub, event, &data); + } + Ok(()) + } + + /// Stop accept, join connection threads, and delete the instance record. A stuck `Assets` or + /// `Activate` call does not delay this. A running `Frames` call does. + pub fn shutdown(mut self) -> Result<(), LinkError> { + self.stop_threads(); + match self.discovery.take() { + Some(discovery) => discovery.remove(), + None => Ok(()), + } + } + + /// Bound loopback port, or `None` when TCP was not requested. + pub fn tcp_port(&self) -> Option { + self.tcp_port + } + + fn stop_threads(&mut self) { + self.hub.stop.store(true, Ordering::SeqCst); + events::wake_all(&self.hub.subs); + if let Some(accept) = self.accept.take() { + let _ = accept.join(); + } + let handles = std::mem::take(&mut *lock(&self.hub.threads)); + for handle in handles { + let _ = handle.join(); + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop_threads(); + } +} + +pub(super) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(err) => err.into_inner(), + } +} diff --git a/crates/ipc/src/server/provider.rs b/crates/ipc/src/server/provider.rs new file mode 100644 index 000000000..f74a62923 --- /dev/null +++ b/crates/ipc/src/server/provider.rs @@ -0,0 +1,71 @@ +//! Calls into app code that must not hold up shutdown. +//! +//! `Assets` and `Activate` are control plane calls, rare and slow enough to pay for a thread. Each +//! call runs on a helper thread while the connection thread waits for the answer, for at most +//! [`super::Limits::provider_deadline`] and not past the stop flag. A call that does not finish in +//! time gets a timeout reply, and the helper thread is detached: nobody joins it, so +//! [`super::Server::shutdown`] and drop do not wait for it. At most [`MAX_HELPERS`] helper +//! threads are alive at once. A new call past that gets a `busy` error until a stuck one returns. +//! +//! `Frames` is not run here. It is the hot path with one call per frame, so it stays on the +//! connection thread (see [`super::Frames`]). + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use super::Hub; +use crate::error::LinkError; +use crate::wire::MAX_CONNECTIONS; + +/// Most provider helper threads alive at once, running or stuck. Every connection can have one +/// call in flight, so this leaves room for the same number of stuck ones. +pub(super) const MAX_HELPERS: usize = 2 * MAX_CONNECTIONS; + +/// How often a waiting connection thread looks at the stop flag. +const STOP_POLL: Duration = Duration::from_millis(100); + +/// Counts one live helper thread. Dropping it, on any path, frees the slot. +struct Slot(Arc); + +impl Drop for Slot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Run `run` on a helper thread and wait for its result. +/// +/// `Err(Timeout)` when it takes longer than the deadline, `Err(Closed)` when the server stops +/// first, `Err(Busy)` when [`MAX_HELPERS`] helpers are already alive, and `Err(Io)` when the +/// helper could not start or panicked. +pub(super) fn call(hub: &Hub, what: &str, run: impl FnOnce() -> T + Send + 'static) -> Result { + if hub.providers.fetch_add(1, Ordering::SeqCst) >= MAX_HELPERS { + hub.providers.fetch_sub(1, Ordering::SeqCst); + return Err(LinkError::Busy(format!("{what}: too many provider calls are stuck"))); + } + let slot = Slot(Arc::clone(&hub.providers)); + let (tx, rx) = mpsc::channel(); + let started = std::thread::Builder::new().name("craft-ipc-provider".into()).spawn(move || { + let _slot = slot; + let _ = tx.send(run()); + }); + if let Err(err) = started { + return Err(LinkError::Io(format!("{what}: cannot start a helper thread: {err}"))); + } + let deadline = Instant::now() + hub.limits.provider_deadline; + loop { + if hub.stop.load(Ordering::SeqCst) { + return Err(LinkError::Closed); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(LinkError::Timeout); + } + match rx.recv_timeout(left.min(STOP_POLL)) { + Ok(value) => return Ok(value), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => return Err(LinkError::Io(format!("{what}: the call ended without an answer"))), + } + } +} diff --git a/crates/ipc/src/server/tcp.rs b/crates/ipc/src/server/tcp.rs new file mode 100644 index 000000000..6c1ae386b --- /dev/null +++ b/crates/ipc/src/server/tcp.rs @@ -0,0 +1,32 @@ +//! Optional loopback TCP. Off unless [`crate::server::ServerConfig::tcp_port`] is set. +//! It binds 127.0.0.1 only and runs the same token handshake, unauthenticated cap and +//! deadlines as the local socket. The 16-connection cap is shared +//! with the local listener. The instance record (0600) carries the port and the token. + +use std::net::{TcpListener, TcpStream}; + +use crate::error::LinkError; + +pub(super) struct BoundTcp { + pub listener: TcpListener, + pub port: u16, +} + +pub(super) fn bind_loopback(port: u16) -> Result { + let listener = TcpListener::bind(("127.0.0.1", port))?; + listener.set_nonblocking(true)?; + let port = listener.local_addr()?.port(); + Ok(BoundTcp { listener, port }) +} + +pub(super) fn poll_accept(listener: &TcpListener) -> std::io::Result> { + match listener.accept() { + Ok((stream, _addr)) => Ok(Some(stream)), + Err(err) if is_idle(&err) => Ok(None), + Err(err) => Err(err), + } +} + +fn is_idle(err: &std::io::Error) -> bool { + matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted) +} diff --git a/crates/ipc/src/trust.rs b/crates/ipc/src/trust.rs new file mode 100644 index 000000000..7f57f6d5b --- /dev/null +++ b/crates/ipc/src/trust.rs @@ -0,0 +1,215 @@ +//! Checks before this crate runs a program. +//! +//! The launcher finds an app binary through `CRAFT__BIN`, `installed.json`, `PATH` and a few +//! fixed directories. A same-user process can write most of those places, so a name that resolves +//! is not proof that the file is the app. These checks do not stop a process of the same user, +//! which can do anything the user can. They stop two mistakes: running a file that any local +//! user can replace, and running a file that a relative path or an empty `PATH` entry picked from +//! the current directory. +//! +//! Group write is not checked. A file or directory that is group-writable passes, so every other +//! member of its group can replace the program. The check accepts that because most Linux +//! distributions give each user a private group with no other members, and the check cannot see +//! who is in a group. + +use std::path::Path; + +use crate::error::LinkError; + +/// `Ok` when `path` is an absolute path to a regular, executable file, owned by this user or root +/// and not writable by every user, in a directory with the same owner and mode rules. +/// +/// Symlinks are followed, and the ancestors of both the supplied and resolved paths are checked. +/// A sticky, trusted-owner directory may appear above the immediate directory: it cannot let +/// another user replace the next trusted-owner path component (as with a private folder in /tmp). +/// A group-writable file or directory passes, and the other members of its group can replace it. +/// Most Linux distributions give each user a private group, and the check cannot see group +/// members. A file or directory that every user can write fails. +/// +/// Non-Unix targets only check that the path is an absolute regular file. The Windows ACL on the +/// install directory is the user's check there. +pub fn check_executable(path: &Path) -> Result<(), LinkError> { + if !path.is_absolute() { + return Err(refused(path, "path is not absolute")); + } + #[cfg(unix)] + { + unix::check(path) + } + #[cfg(not(unix))] + { + if std::fs::metadata(path).is_ok_and(|meta| meta.is_file()) { Ok(()) } else { Err(refused(path, "not a file")) } + } +} + +fn refused(path: &Path, reason: &str) -> LinkError { + LinkError::Refused { path: path.display().to_string(), reason: reason.to_string() } +} + +#[cfg(unix)] +mod unix { + use std::os::unix::fs::MetadataExt; + use std::path::Path; + + use super::refused; + use crate::error::LinkError; + + const WORLD_WRITE: u32 = 0o002; + const ANY_EXEC: u32 = 0o111; + const STICKY: u32 = 0o1000; + + pub(super) fn check(path: &Path) -> Result<(), LinkError> { + let euid = crate::discovery::current_euid()?; + let meta = std::fs::metadata(path).map_err(|err| refused(path, &format!("cannot stat: {err}")))?; + if !meta.is_file() { + return Err(refused(path, "not a regular file")); + } + if meta.mode() & ANY_EXEC == 0 { + return Err(refused(path, "not executable")); + } + owner_and_mode(path, &meta, euid, "file")?; + let real = std::fs::canonicalize(path).map_err(|err| refused(path, &format!("cannot resolve: {err}")))?; + for candidate in [path, real.as_path()] { + // Check each named component as well as its resolved directory. In particular a + // foreign-owned symlink in a sticky ancestor can be replaced by that symlink's owner. + for component in candidate.ancestors() { + let named = std::fs::symlink_metadata(component).map_err(|err| refused(component, &format!("cannot stat: {err}")))?; + if named.file_type().is_symlink() && named.uid() != euid && named.uid() != 0 { + return Err(refused(component, "symlink is owned by another user")); + } + } + for (depth, parent) in candidate.ancestors().skip(1).enumerate() { + let dir = std::fs::metadata(parent).map_err(|err| refused(parent, &format!("cannot stat: {err}")))?; + // Sticky ancestors protect the trusted-owned child checked on the preceding + // iteration. The immediate directory still must not be world-writable. + if depth > 0 && dir.mode() & STICKY != 0 { + if dir.uid() != euid && dir.uid() != 0 { + return Err(refused(parent, "sticky ancestor is owned by another user")); + } + } else { + owner_and_mode(parent, &dir, euid, "directory")?; + } + } + } + Ok(()) + } + + fn owner_and_mode(path: &Path, meta: &std::fs::Metadata, euid: u32, what: &str) -> Result<(), LinkError> { + if meta.uid() != euid && meta.uid() != 0 { + return Err(refused(path, &format!("{what} is owned by uid {}, not uid {euid} or root", meta.uid()))); + } + if meta.mode() & WORLD_WRITE != 0 { + return Err(refused(path, &format!("{what} is writable by every user"))); + } + Ok(()) + } +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::os::unix::fs::PermissionsExt; + use std::path::PathBuf; + + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-trust-{label}-{}", crate::auth::encode_hex(&bytes))); + std::fs::create_dir_all(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + path + } + + fn program(dir: &Path, mode: u32) -> PathBuf { + let file = dir.join("app"); + std::fs::write(&file, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(mode)).unwrap(); + file + } + + #[test] + fn a_private_executable_passes() { + let dir = scratch("ok"); + assert!(check_executable(&program(&dir, 0o755)).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn relative_and_empty_paths_fail() { + assert!(matches!(check_executable(Path::new("app")), Err(LinkError::Refused { .. }))); + assert!(matches!(check_executable(Path::new("")), Err(LinkError::Refused { .. }))); + assert!(matches!(check_executable(Path::new("./app")), Err(LinkError::Refused { .. }))); + } + + #[test] + fn loose_modes_and_non_programs_fail() { + let dir = scratch("modes"); + let plain = program(&dir, 0o644); + assert!(check_executable(&plain).is_err(), "not executable"); + let open = program(&dir, 0o777); + assert!(check_executable(&open).is_err(), "world-writable file"); + std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&open).is_ok()); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&open).is_err(), "world-writable directory"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o1777)).unwrap(); + assert!(check_executable(&open).is_err(), "sticky does not excuse a world-writable directory"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&dir).is_err(), "a directory is not a program"); + assert!(check_executable(&dir.join("missing")).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_symlink_into_a_loose_directory_fails() { + let good = scratch("linkgood"); + let loose = scratch("linkloose"); + let target = program(&loose, 0o755); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o777)).unwrap(); + let link = good.join("app"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + assert!(check_executable(&link).is_err()); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&link).is_ok()); + std::fs::remove_dir_all(&good).unwrap(); + std::fs::remove_dir_all(&loose).unwrap(); + } + + #[test] + fn system_programs_pass() { + assert!(check_executable(Path::new("/usr/bin/env")).is_ok() || check_executable(Path::new("/bin/sh")).is_ok()); + } + + #[test] + fn writable_ancestor_cannot_replace_a_private_program_directory() { + let ancestor = scratch("ancestor"); + let bin = ancestor.join("install/bin"); + std::fs::create_dir_all(&bin).unwrap(); + let app = program(&bin, 0o755); + assert!(check_executable(&app).is_ok()); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&app).is_err(), "a safe immediate parent does not protect its ancestors"); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o1777)).unwrap(); + assert!(check_executable(&app).is_ok(), "sticky ancestor protects the owned install directory"); + std::fs::remove_dir_all(&ancestor).unwrap(); + } + + #[test] + fn symlink_target_ancestors_are_checked_too() { + let good = scratch("ancestor-link"); + let ancestor = scratch("ancestor-target"); + let bin = ancestor.join("install/bin"); + std::fs::create_dir_all(&bin).unwrap(); + let target = program(&bin, 0o755); + let link = good.join("app"); + std::os::unix::fs::symlink(target, &link).unwrap(); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&link).is_err()); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&link).is_ok()); + std::fs::remove_dir_all(good).unwrap(); + std::fs::remove_dir_all(ancestor).unwrap(); + } +} diff --git a/crates/ipc/src/watch.rs b/crates/ipc/src/watch.rs new file mode 100644 index 000000000..cc765314c --- /dev/null +++ b/crates/ipc/src/watch.rs @@ -0,0 +1,397 @@ +//! Scheduled polling of linked files (`PROTOCOL.md` section 10, rule 4). +//! +//! [`crate::refresh::Tracker`] looks at the disk once per call. A [`Watcher`] +//! owns trackers and calls them on a timer: a shallow poll every +//! [`SHALLOW_INTERVAL`] (1 s) and a deep poll every [`DEEP_INTERVAL`] (30 s). +//! A deep poll replaces the shallow poll that falls on the same tick. Nothing +//! here needs the producer, so it works while the producer is offline. +//! +//! # Time +//! +//! The watcher never reads a clock. A [`Ticker`] blocks until the next tick and +//! returns the time elapsed since it started. Two tickers exist: +//! +//! - [`IntervalTicker`] waits on a condition variable with a timeout. It is the +//! native ticker. It uses no CPU between ticks and wakes at once on close. +//! - [`ManualTicker`] moves only when [`ManualTicker::advance`] is called. Tests +//! use it, so they never sleep. +//! +//! # Threads and shutdown +//! +//! [`Watcher::new`] starts one thread. Dropping the watcher closes the ticker +//! and joins the thread. A poll that is running finishes first. A panic in the +//! notice callback ends the thread, and the panic message goes to stderr. +//! +//! # Notices +//! +//! The callback runs on the watcher thread with no lock held, so it may call +//! back into the watcher. It receives [`Notice::Changed`] when a poll bumped the +//! epoch or moved the state (`current`, `modified`, `missing`), and +//! [`Notice::PollFailed`] when a poll hit a read error (rule 6). A repeated +//! identical error is reported once until a poll succeeds. An app turns +//! `Changed` into a `link.changed` event with the new epoch. +//! +//! `std` only. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use crate::error::LinkError; +use crate::refresh::{Change, FileState, Tracker}; + +/// How often a shallow poll runs (size and mtime, hash only on a difference). +pub const SHALLOW_INTERVAL: Duration = Duration::from_secs(1); +/// How often a deep poll runs (always hash). Covers coarse file system mtime. +pub const DEEP_INTERVAL: Duration = Duration::from_secs(30); + +/// Poll periods. [`Intervals::default`] is the contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Intervals { + pub shallow: Duration, + pub deep: Duration, +} + +impl Default for Intervals { + fn default() -> Self { + Self { shallow: SHALLOW_INTERVAL, deep: DEEP_INTERVAL } + } +} + +impl Intervals { + fn validate(self) -> Result<(), LinkError> { + if self.shallow.is_zero() { + return Err(LinkError::Invalid("the shallow interval must be above zero".into())); + } + if self.deep < self.shallow { + return Err(LinkError::Invalid("the deep interval must not be shorter than the shallow interval".into())); + } + Ok(()) + } +} + +/// A source of ticks for a [`Watcher`]. +pub trait Ticker: Send + Sync { + /// Block until the next tick, then return the time since the ticker started. + /// The value never decreases. Return `None` after [`Ticker::close`]. + fn wait(&self) -> Option; + /// Wake a blocked [`Ticker::wait`] and make every later call return `None`. + fn close(&self); +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// The native ticker: one tick per period on the wall clock. +/// +/// A tick that comes late skips the ticks it missed. It does not fire them in a burst. +#[derive(Debug)] +pub struct IntervalTicker { + start: Instant, + period: Duration, + state: Mutex, + wake: Condvar, +} + +#[derive(Debug)] +struct IntervalState { + /// The number of the next tick, counted from 1. + next: u32, + closed: bool, +} + +impl IntervalTicker { + /// `period` must be above zero. + pub fn new(period: Duration) -> Result { + if period.is_zero() { + return Err(LinkError::Invalid("the tick period must be above zero".into())); + } + Ok(Self { start: Instant::now(), period, state: Mutex::new(IntervalState { next: 1, closed: false }), wake: Condvar::new() }) + } +} + +impl Ticker for IntervalTicker { + fn wait(&self) -> Option { + let mut state = lock(&self.state); + loop { + if state.closed { + return None; + } + let deadline = self.start + self.period.saturating_mul(state.next); + let now = Instant::now(); + if now >= deadline { + let elapsed = now.duration_since(self.start); + let passed = elapsed.as_nanos() / self.period.as_nanos().max(1); + state.next = u32::try_from(passed.saturating_add(1)).unwrap_or(u32::MAX); + return Some(elapsed); + } + state = self.wake.wait_timeout(state, deadline - now).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } + + fn close(&self) { + lock(&self.state).closed = true; + self.wake.notify_all(); + } +} + +/// A ticker that moves only when told. For tests and for hosts with their own clock. +#[derive(Debug, Default)] +pub struct ManualTicker { + state: Mutex, + wake: Condvar, +} + +#[derive(Debug, Default)] +struct ManualState { + now: Duration, + delivered: Duration, + /// The watcher thread is blocked in `wait` and has handled every earlier tick. + idle: bool, + closed: bool, +} + +impl ManualTicker { + pub fn new() -> Self { + Self::default() + } + + /// Move time forward by `by` and wake the watcher. The watcher sees one tick + /// at the new time, however large `by` is. + pub fn advance(&self, by: Duration) { + let mut state = lock(&self.state); + let moved = state.now.saturating_add(by); + if moved > state.now { + state.now = moved; + state.idle = false; + } + drop(state); + self.wake.notify_all(); + } + + /// Block until the watcher has handled every tick so far and waits again. + /// Returns `false` when `limit` passes first or the ticker is closed. The + /// limit only guards a test from hanging. A healthy watcher settles at once. + pub fn settle(&self, limit: Duration) -> bool { + let deadline = Instant::now() + limit; + let mut state = lock(&self.state); + loop { + if state.closed { + return false; + } + if state.idle && state.delivered == state.now { + return true; + } + let now = Instant::now(); + if now >= deadline { + return false; + } + state = self.wake.wait_timeout(state, deadline - now).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } +} + +impl Ticker for ManualTicker { + fn wait(&self) -> Option { + let mut state = lock(&self.state); + state.idle = state.delivered == state.now; + self.wake.notify_all(); + loop { + if state.closed { + return None; + } + if state.now > state.delivered { + state.delivered = state.now; + state.idle = false; + return Some(state.now); + } + state = self.wake.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + + fn close(&self) { + lock(&self.state).closed = true; + self.wake.notify_all(); + } +} + +/// What a [`Watcher`] tells its callback. +#[derive(Debug, Clone, PartialEq)] +pub enum Notice { + /// A poll bumped the epoch or moved the state of `link_id`. + Changed { link_id: String, change: Change, deep: bool }, + /// A poll could not read the file. The state did not change. + PollFailed { link_id: String, error: LinkError }, +} + +/// How many polls the watcher thread has run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PollCounts { + pub shallow: u64, + pub deep: u64, +} + +type Callback = Box; + +struct Entry { + id: String, + state: Mutex, +} + +struct EntryState { + /// `None` after `unwatch`, for a poll that was already under way. + tracker: Option, + /// The state the callback last heard about. + reported: FileState, + last_error: Option, +} + +struct Shared { + entries: Mutex>>, + notify: Callback, + shallow_polls: AtomicU64, + deep_polls: AtomicU64, +} + +/// Polls a set of trackers on a schedule. See the module docs. +pub struct Watcher { + shared: Arc, + ticker: Arc, + thread: Option>, +} + +impl Watcher { + /// Watch on the wall clock with the contract intervals. + pub fn start(notify: impl Fn(Notice) + Send + Sync + 'static) -> Result { + let ticker = Arc::new(IntervalTicker::new(SHALLOW_INTERVAL)?); + Self::new(ticker, Intervals::default(), notify) + } + + /// Watch with a chosen ticker. The ticker should tick at least as often as + /// `intervals.shallow`, or polls run at the ticker rate. + pub fn new(ticker: Arc, intervals: Intervals, notify: impl Fn(Notice) + Send + Sync + 'static) -> Result { + intervals.validate()?; + let shared = Arc::new(Shared { + entries: Mutex::new(Vec::new()), + notify: Box::new(notify), + shallow_polls: AtomicU64::new(0), + deep_polls: AtomicU64::new(0), + }); + let thread = { + let shared = Arc::clone(&shared); + let ticker = Arc::clone(&ticker); + std::thread::Builder::new().name("craft-ipc-watch".into()).spawn(move || run(&shared, ticker.as_ref(), intervals))? + }; + Ok(Self { shared, ticker, thread: Some(thread) }) + } + + /// Start polling `tracker` under `link_id`. A second tracker for the same id is an error. + pub fn watch(&self, link_id: impl Into, tracker: Tracker) -> Result<(), LinkError> { + let id = link_id.into(); + let mut entries = lock(&self.shared.entries); + if entries.iter().any(|entry| entry.id == id) { + return Err(LinkError::Invalid(format!("{id} is already watched"))); + } + let reported = tracker.state(); + entries.push(Arc::new(Entry { id, state: Mutex::new(EntryState { tracker: Some(tracker), reported, last_error: None }) })); + Ok(()) + } + + /// Stop polling `link_id` and hand its tracker back. + pub fn unwatch(&self, link_id: &str) -> Option { + let entry = { + let mut entries = lock(&self.shared.entries); + let at = entries.iter().position(|entry| entry.id == link_id)?; + entries.remove(at) + }; + // The thread may hold a clone of the entry for one more poll, which skips it now. + lock(&entry.state).tracker.take() + } + + /// Run `act` on the tracker of `link_id`: `accept`, `relink`, `adopt_epoch` + /// or a read. `None` when the id is not watched. The callback does not hear + /// about the state that `act` leaves behind. + pub fn with_tracker(&self, link_id: &str, act: impl FnOnce(&mut Tracker) -> R) -> Option { + let entry = lock(&self.shared.entries).iter().find(|entry| entry.id == link_id).cloned()?; + let mut state = lock(&entry.state); + let tracker = state.tracker.as_mut()?; + let out = act(tracker); + let now = tracker.state(); + state.reported = now; + Some(out) + } + + pub fn poll_counts(&self) -> PollCounts { + PollCounts { shallow: self.shared.shallow_polls.load(Ordering::SeqCst), deep: self.shared.deep_polls.load(Ordering::SeqCst) } + } +} + +impl Drop for Watcher { + fn drop(&mut self) { + self.ticker.close(); + if let Some(thread) = self.thread.take() { + // An Err means the callback panicked. The panic message is already on stderr. + let _ = thread.join(); + } + } +} + +/// Next due time on the grid `first, first + step, ...` that is later than `now`. +fn next_after(due: u128, step: u128, now: u128) -> u128 { + if due > now { due } else { due + ((now - due) / step + 1) * step } +} + +fn run(shared: &Shared, ticker: &dyn Ticker, intervals: Intervals) { + let shallow = intervals.shallow.as_nanos(); + let deep = intervals.deep.as_nanos(); + let mut due_shallow = shallow; + let mut due_deep = deep; + while let Some(elapsed) = ticker.wait() { + let now = elapsed.as_nanos(); + if now >= due_deep { + due_deep = next_after(due_deep, deep, now); + due_shallow = next_after(due_shallow, shallow, now); + shared.deep_polls.fetch_add(1, Ordering::SeqCst); + poll_all(shared, true); + } else if now >= due_shallow { + due_shallow = next_after(due_shallow, shallow, now); + shared.shallow_polls.fetch_add(1, Ordering::SeqCst); + poll_all(shared, false); + } + } +} + +fn poll_all(shared: &Shared, deep: bool) { + let entries: Vec> = lock(&shared.entries).clone(); + for entry in entries { + let notice = poll_entry(&entry, deep); + if let Some(notice) = notice { + (shared.notify)(notice); + } + } +} + +fn poll_entry(entry: &Entry, deep: bool) -> Option { + let mut state = lock(&entry.state); + let polled = state.tracker.as_mut()?.poll(deep); + match polled { + Ok(change) => { + state.last_error = None; + if change.bumped || change.state != state.reported { + state.reported = change.state; + Some(Notice::Changed { link_id: entry.id.clone(), change, deep }) + } else { + None + } + } + Err(error) => { + if state.last_error.as_ref() == Some(&error) { + return None; + } + state.last_error = Some(error.clone()); + Some(Notice::PollFailed { link_id: entry.id.clone(), error }) + } + } +} diff --git a/crates/ipc/src/wire.rs b/crates/ipc/src/wire.rs new file mode 100644 index 000000000..f3dc76621 --- /dev/null +++ b/crates/ipc/src/wire.rs @@ -0,0 +1,561 @@ +//! JSON lines and bulk frame records. No sockets live here. + +use std::io::{BufRead, Read, Write}; +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use crate::error::LinkError; + +/// Longest accepted control line, without the newline. Matches the app control servers. +pub const MAX_LINE: usize = 4 * 1024 * 1024; +/// Open connections per server, authenticated or not. Bulk connections count. +pub const MAX_CONNECTIONS: usize = 16; +/// Connections that have not authenticated yet. The other `MAX_CONNECTIONS - this` slots are +/// reserved for authenticated peers. +pub const MAX_UNAUTHENTICATED: usize = 4; +pub const EVENT_QUEUE: usize = 256; +pub const REPLY_DEADLINE_SECS: u64 = 60; +/// `link.open` payload ceiling. The control line cap is the real limit: a longer request line +/// closes the connection. This number is the same, so `OpenPayload` gives `payload_too_large` +/// to a caller that does not come through the line reader. +pub const MAX_LINK_PAYLOAD: u64 = MAX_LINE as u64; +pub const BULK_HEADER_MAX: u32 = 1024 * 1024; +pub const FRAME_PAYLOAD_MAX: u64 = 256 * 1024 * 1024; +pub const PROTOCOL_MIN: u32 = 1; +pub const PROTOCOL_MAX: u32 = 1; +pub const AUTH_ERROR: &str = "authentication required"; + +const CFRQ: &[u8; 4] = b"CFRQ"; +const CFRM: &[u8; 4] = b"CFRM"; + +/// One control request. `id` is JSON null when the caller omitted it. +#[derive(Debug, Clone, PartialEq)] +pub struct Request { + pub id: Value, + pub method: String, + pub params: Value, +} + +/// Read one line of at most `max` bytes. `Ok(None)` is a clean end of stream. +/// +/// This keeps no state between calls. A read timeout drops the bytes read so far, so use it only +/// on a stream that cannot time out. A server loop that polls with a receive timeout uses +/// [`LineReader`]. +pub fn read_line(reader: &mut impl BufRead, max: usize) -> Result>, LinkError> { + LineReader::new(max).read(reader) +} + +/// Reads lines from a stream whose reads can time out. Bytes read before a timeout stay in the +/// reader, so a line that arrives in pieces more than one poll apart is not lost. +#[derive(Debug)] +pub struct LineReader { + buf: Vec, + max: usize, + started: Option, + patience: Option, +} + +impl LineReader { + pub fn new(max: usize) -> Self { + Self { buf: Vec::new(), max, started: None, patience: None } + } + + /// How long the current unfinished line has been open. `None` when no line is in progress. + pub fn partial_age(&self) -> Option { + self.started.map(|start| start.elapsed()) + } + + /// Return `Err(Timeout)` once a line has been open this long, even when bytes keep arriving. + /// Without it a peer that sends one byte per poll holds the connection for ever. + pub fn with_patience(mut self, patience: Duration) -> Self { + self.patience = Some(patience); + self + } + + /// Change the limit that [`LineReader::with_patience`] sets. + pub fn set_patience(&mut self, patience: Duration) { + self.patience = Some(patience); + } + + /// Read the next line. On `Err(Timeout)` the partial line stays here and the next call + /// continues it. `Ok(None)` is a clean end of stream with no partial line. + pub fn read(&mut self, reader: &mut impl BufRead) -> Result>, LinkError> { + loop { + let available = match reader.fill_buf() { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) => return Err(err.into()), + }; + if available.is_empty() { + // End of stream. A final line without a newline is still a line. + if self.buf.is_empty() { + return Ok(None); + } + let line = self.finish(); + if line.len() > self.max { + return Err(LinkError::TooLarge); + } + return Ok(Some(line)); + } + let newline = available.iter().position(|byte| *byte == b'\n'); + let take = newline.map_or(available.len(), |at| at + 1); + if self.buf.len().saturating_add(take) > self.max.saturating_add(1) { + return Err(LinkError::TooLarge); + } + self.buf.extend_from_slice(&available[..take]); + reader.consume(take); + if self.started.is_none() { + self.started = Some(Instant::now()); + } + if newline.is_some() { + let mut line = self.finish(); + line.pop(); + if line.last() == Some(&b'\r') { + line.pop(); + } + if line.len() > self.max { + return Err(LinkError::TooLarge); + } + return Ok(Some(line)); + } + if self.patience.zip(self.partial_age()).is_some_and(|(limit, age)| age >= limit) { + return Err(LinkError::Timeout); + } + } + } + + fn finish(&mut self) -> Vec { + self.started = None; + std::mem::take(&mut self.buf) + } +} + +/// Parse one control line. +/// +/// `Ok(None)` is a blank line (the caller skips it). `Err` is the existing +/// control-server reject text: one reply, then close. +pub fn parse_request(bytes: &[u8]) -> Result, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "not a control request (invalid UTF-8); closing the connection".to_string())?; + let text = text.trim(); + if text.is_empty() { + return Ok(None); + } + let msg: Value = + serde_json::from_str(text).map_err(|err| format!("bad JSON ({err}); expected one JSON request object per line; closing the connection"))?; + let Some(obj) = msg.as_object() else { + return Err("not a control request (expected a JSON object); closing the connection".into()); + }; + let Some(method) = obj.get("method").and_then(Value::as_str) else { + return Err("not a control request (missing string `method`); closing the connection".into()); + }; + if method.is_empty() { + return Err("not a control request (missing string `method`); closing the connection".into()); + } + let id = obj.get("id").cloned().unwrap_or(Value::Null); + if !(id.is_null() || id.is_number() || id.is_string()) { + return Err("not a control request (id must be a number, string, or null); closing the connection".into()); + } + let params = obj.get("params").cloned().unwrap_or_else(|| json!({})); + Ok(Some(Request { id, method: method.to_string(), params })) +} + +pub fn params_object(params: &Value) -> Result<&Map, LinkError> { + params.as_object().ok_or_else(|| LinkError::BadParams("params must be an object".into())) +} + +pub fn auth_failure() -> Value { + json!({"id": 0, "ok": false, "error": AUTH_ERROR}) +} + +pub fn auth_success(id: &Value) -> Value { + let id = if id.is_null() { json!(0) } else { id.clone() }; + json!({"id": id, "ok": true, "result": {"authenticated": true}}) +} + +pub fn reply_ok(id: &Value, result: Value) -> Value { + json!({"id": id, "ok": true, "result": result}) +} + +pub fn reply_err_text(id: &Value, error: &str) -> Value { + json!({"id": id, "ok": false, "error": error}) +} + +pub fn reply_err(id: &Value, err: &LinkError) -> Value { + json!({"id": id, "ok": false, "error": err.to_object()}) +} + +pub fn push_event(event: &str, seq: u64, data: Value) -> Value { + json!({"event": event, "seq": seq, "data": data}) +} + +/// Write a reply line that a peer will read with the [`MAX_LINE`] limit. A value longer than the +/// limit is replaced by a `payload_too_large` reply that keeps the request `id`. When even that +/// does not fit, or the value has no `id` to answer, nothing is written and the result is +/// `Err(TooLarge)`. The limit counts the content without the newline, as the readers do. +pub fn write_reply_line(writer: &mut impl Write, value: &Value) -> Result<(), LinkError> { + let mut bytes = serde_json::to_vec(value)?; + if bytes.len() > MAX_LINE { + let id = value.get("id").ok_or(LinkError::TooLarge)?; + bytes = serde_json::to_vec(&reply_err(id, &LinkError::TooLarge))?; + if bytes.len() > MAX_LINE { + return Err(LinkError::TooLarge); + } + } + bytes.push(b'\n'); + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +pub fn write_line(writer: &mut impl Write, value: &Value) -> Result<(), LinkError> { + let mut bytes = serde_json::to_vec(value)?; + bytes.push(b'\n'); + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +pub fn protocols_overlap(min: u32, max: u32) -> bool { + min <= max && min <= PROTOCOL_MAX && max >= PROTOCOL_MIN +} + +pub fn write_cfrq(writer: &mut impl Write, header: &Value) -> Result<(), LinkError> { + write_magic(writer, CFRQ, header)?; + Ok(()) +} + +pub fn read_cfrq(reader: &mut impl Read) -> Result { + read_magic(reader, CFRQ) +} + +/// The checks that [`write_cfrm`] makes before it writes a byte: the header is an object that fits +/// `BULK_HEADER_MAX`, and the payload fits `FRAME_PAYLOAD_MAX`. A record that fails them has put +/// nothing on the wire, so a caller may still send a different record. An error from `write_cfrm` +/// after this check passed came from the stream and may have left part of a record on it. +pub fn check_cfrm(header: &Value, payload: &[u8]) -> Result<(), LinkError> { + cfrm_parts(header, payload).map(|_| ()) +} + +fn cfrm_parts(header: &Value, payload: &[u8]) -> Result<(Vec, u64), LinkError> { + let header_bytes = header_bytes(header)?; + let payload_len = u64::try_from(payload.len()).map_err(|_| LinkError::TooLarge)?; + if payload_len > FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + Ok((header_bytes, payload_len)) +} + +pub fn write_cfrm(writer: &mut impl Write, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + let (header_bytes, payload_len) = cfrm_parts(header, payload)?; + writer.write_all(CFRM)?; + writer.write_all(&u32_len(&header_bytes)?.to_le_bytes())?; + writer.write_all(&payload_len.to_le_bytes())?; + writer.write_all(&header_bytes)?; + writer.write_all(payload)?; + writer.flush()?; + Ok(()) +} + +pub fn read_cfrm(reader: &mut impl Read) -> Result<(Value, Vec), LinkError> { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if &magic != CFRM { + return Err(LinkError::Invalid("bulk reply magic".into())); + } + let header_len = read_u32(reader)?; + if header_len > BULK_HEADER_MAX { + return Err(LinkError::TooLarge); + } + let payload_len = read_u64(reader)?; + if payload_len > FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + let header = read_header(reader, header_len)?; + let mut payload = vec![0u8; usize::try_from(payload_len).map_err(|_| LinkError::TooLarge)?]; + reader.read_exact(&mut payload)?; + Ok((header, payload)) +} + +fn write_magic(writer: &mut impl Write, magic: &[u8; 4], header: &Value) -> Result<(), LinkError> { + let bytes = header_bytes(header)?; + writer.write_all(magic)?; + writer.write_all(&u32_len(&bytes)?.to_le_bytes())?; + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +fn read_magic(reader: &mut impl Read, expect: &[u8; 4]) -> Result { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if &magic != expect { + return Err(LinkError::Invalid("bulk request magic".into())); + } + let len = read_u32(reader)?; + if len > BULK_HEADER_MAX { + return Err(LinkError::TooLarge); + } + read_header(reader, len) +} + +fn header_bytes(header: &Value) -> Result, LinkError> { + if !header.is_object() { + return Err(LinkError::BadParams("bulk header must be an object".into())); + } + let bytes = serde_json::to_vec(header)?; + if bytes.len() > usize::try_from(BULK_HEADER_MAX).unwrap_or(usize::MAX) { + return Err(LinkError::TooLarge); + } + Ok(bytes) +} + +fn u32_len(bytes: &[u8]) -> Result { + u32::try_from(bytes.len()).map_err(|_| LinkError::TooLarge) +} + +fn read_u32(reader: &mut impl Read) -> Result { + let mut buf = [0u8; 4]; + reader.read_exact(&mut buf)?; + Ok(u32::from_le_bytes(buf)) +} + +fn read_u64(reader: &mut impl Read) -> Result { + let mut buf = [0u8; 8]; + reader.read_exact(&mut buf)?; + Ok(u64::from_le_bytes(buf)) +} + +fn read_header(reader: &mut impl Read, len: u32) -> Result { + let mut buf = vec![0u8; len as usize]; + reader.read_exact(&mut buf)?; + let value: Value = serde_json::from_slice(&buf)?; + if !value.is_object() { + return Err(LinkError::BadParams("bulk header must be an object".into())); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::*; + + #[test] + fn blank_and_request_round_trip() { + assert!(parse_request(b" \n").unwrap().is_none()); + let req = parse_request(br#"{"id":1,"method":"engine.execute","params":{"command":"x"}}"#).unwrap().unwrap(); + assert_eq!(req.method, "engine.execute"); + assert_eq!(req.id, json!(1)); + assert!(parse_request(b"[]").is_err()); + assert!(parse_request(b"{").is_err()); + } + + #[test] + fn a_reply_over_the_line_limit_becomes_a_too_large_error_with_the_same_id() { + let mut out = Vec::new(); + let small = reply_ok(&json!(3), json!({"a": 1})); + write_reply_line(&mut out, &small).unwrap(); + assert_eq!(out, [serde_json::to_vec(&small).unwrap(), b"\n".to_vec()].concat(), "a reply inside the limit is written as it is"); + + // The content is exactly MAX_LINE bytes: still inside. + let overhead = serde_json::to_vec(&reply_ok(&json!(3), json!(""))).unwrap().len(); + let exact = reply_ok(&json!(3), json!("x".repeat(MAX_LINE - overhead))); + assert_eq!(serde_json::to_vec(&exact).unwrap().len(), MAX_LINE); + let mut out = Vec::new(); + write_reply_line(&mut out, &exact).unwrap(); + assert_eq!(out.len(), MAX_LINE + 1); + + let big = reply_ok(&json!(3), json!("x".repeat(MAX_LINE - overhead + 1))); + let mut out = Vec::new(); + write_reply_line(&mut out, &big).unwrap(); + let line = read_line(&mut Cursor::new(out), MAX_LINE).unwrap().unwrap(); + let reply: Value = serde_json::from_slice(&line).unwrap(); + assert_eq!(reply["id"], json!(3)); + assert_eq!(reply["ok"], json!(false)); + assert_eq!(reply["error"]["code"], json!("payload_too_large")); + + // No id to answer: nothing is written. + let mut out = Vec::new(); + let anonymous = json!({"ok": true, "result": "x".repeat(MAX_LINE)}); + assert!(matches!(write_reply_line(&mut out, &anonymous), Err(LinkError::TooLarge))); + assert!(out.is_empty()); + // An id so long that the error reply does not fit either. + let huge_id = json!({"id": "i".repeat(MAX_LINE + 1), "ok": true}); + assert!(matches!(write_reply_line(&mut out, &huge_id), Err(LinkError::TooLarge))); + assert!(out.is_empty()); + } + + #[test] + fn a_last_line_without_a_newline_obeys_the_limit() { + let at_limit = vec![b'a'; 8]; + let line = read_line(&mut Cursor::new(at_limit.clone()), 8).unwrap().unwrap(); + assert_eq!(line, at_limit); + let over = vec![b'a'; 9]; + assert!(matches!(read_line(&mut Cursor::new(over), 8), Err(LinkError::TooLarge))); + // The same bytes with a newline are refused too, so the two paths agree. + let mut with_newline = vec![b'a'; 9]; + with_newline.push(b'\n'); + assert!(matches!(read_line(&mut Cursor::new(with_newline), 8), Err(LinkError::TooLarge))); + } + + #[test] + fn line_limit_and_crlf() { + let mut cur = Cursor::new(b"abc\r\n".to_vec()); + let line = read_line(&mut cur, 8).unwrap().unwrap(); + assert_eq!(line, b"abc"); + let mut huge = vec![b'a'; 20]; + huge.push(b'\n'); + let mut cur = Cursor::new(huge); + assert!(matches!(read_line(&mut cur, 8), Err(LinkError::TooLarge))); + } + + /// A reader that hands out `chunks` and fails with a timeout after each one, like a socket + /// with a receive timeout and a slow peer. + struct Stuttering { + chunks: std::collections::VecDeque>, + pending_timeout: bool, + current: Vec, + pos: usize, + } + + impl Stuttering { + fn new(chunks: Vec>) -> Self { + Self { chunks: chunks.into(), pending_timeout: false, current: Vec::new(), pos: 0 } + } + } + + impl Read for Stuttering { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.pos >= self.current.len() { + if self.pending_timeout { + self.pending_timeout = false; + return Err(std::io::ErrorKind::WouldBlock.into()); + } + let Some(next) = self.chunks.pop_front() else { return Ok(0) }; + self.current = next; + self.pos = 0; + if self.current.is_empty() { + return Err(std::io::ErrorKind::WouldBlock.into()); + } + self.pending_timeout = true; + } + let n = out.len().min(self.current.len() - self.pos); + out[..n].copy_from_slice(&self.current[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } + + fn collect_lines(chunks: Vec>, max: usize) -> Vec> { + let mut reader = std::io::BufReader::with_capacity(4, Stuttering::new(chunks)); + let mut lines = LineReader::new(max); + let mut got = Vec::new(); + loop { + match lines.read(&mut reader) { + Ok(Some(line)) => got.push(line), + Ok(None) => return got, + Err(LinkError::Timeout) => {} + Err(other) => panic!("{other:?}"), + } + } + } + + #[test] + fn a_line_split_at_every_offset_survives_a_timeout() { + let text = b"{\"id\":1,\"method\":\"link.hello\"}\r\n{\"id\":2}\n"; + let want = vec![b"{\"id\":1,\"method\":\"link.hello\"}".to_vec(), b"{\"id\":2}".to_vec()]; + for cut in 0..=text.len() { + let chunks = vec![text[..cut].to_vec(), text[cut..].to_vec()]; + assert_eq!(collect_lines(chunks, 1024), want, "cut {cut}"); + } + // Two cuts, and one byte at a time. + for a in 0..text.len() { + for b in a..text.len() { + let chunks = vec![text[..a].to_vec(), text[a..b].to_vec(), text[b..].to_vec()]; + assert_eq!(collect_lines(chunks, 1024), want, "cuts {a} {b}"); + } + } + let bytes: Vec> = text.iter().map(|byte| vec![*byte]).collect(); + assert_eq!(collect_lines(bytes, 1024), want); + } + + #[test] + fn the_line_cap_holds_across_timeouts() { + let mut reader = std::io::BufReader::new(Stuttering::new(vec![vec![b'a'; 6], vec![b'b'; 6], b"\n".to_vec()])); + let mut lines = LineReader::new(8); + let mut result = None; + for _ in 0..8 { + match lines.read(&mut reader) { + Err(LinkError::Timeout) => {} + other => { + result = Some(other); + break; + } + } + } + assert!(matches!(result, Some(Err(LinkError::TooLarge))), "{result:?}"); + } + + /// One byte per read, a short pause each, and never a newline: a slow-loris peer whose reads + /// never time out. + struct Trickle; + + impl Read for Trickle { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + std::thread::sleep(Duration::from_millis(10)); + out[0] = b'a'; + Ok(1) + } + } + + #[test] + fn a_trickling_line_is_cut_by_patience() { + let mut reader = std::io::BufReader::with_capacity(1, Trickle); + let mut lines = LineReader::new(MAX_LINE).with_patience(Duration::from_millis(100)); + let start = Instant::now(); + assert!(matches!(lines.read(&mut reader), Err(LinkError::Timeout))); + assert!(start.elapsed() < Duration::from_secs(2), "{:?}", start.elapsed()); + assert!(lines.partial_age().is_some_and(|age| age >= Duration::from_millis(100))); + } + + #[test] + fn partial_age_tracks_an_open_line() { + let mut reader = std::io::BufReader::new(Stuttering::new(vec![b"abc".to_vec(), b"d\n".to_vec()])); + let mut lines = LineReader::new(64); + assert!(lines.partial_age().is_none()); + assert!(matches!(lines.read(&mut reader), Err(LinkError::Timeout))); + assert!(lines.partial_age().is_some()); + assert_eq!(lines.read(&mut reader).unwrap().unwrap(), b"abcd"); + assert!(lines.partial_age().is_none()); + } + + #[test] + fn bulk_round_trip() { + let header = json!({"req": 7, "time_s": 1.5, "epoch": 3, "region": null}); + let mut buf = Vec::new(); + write_cfrq(&mut buf, &header).unwrap(); + let got = read_cfrq(&mut Cursor::new(&buf)).unwrap(); + assert_eq!(got["req"], json!(7)); + + let mut buf = Vec::new(); + let reply = json!({"req": 7, "error": "no"}); + write_cfrm(&mut buf, &reply, b"").unwrap(); + let (h, payload) = read_cfrm(&mut Cursor::new(&buf)).unwrap(); + assert_eq!(h["error"], json!("no")); + assert!(payload.is_empty()); + + let mut buf = Vec::new(); + write_cfrm(&mut buf, &json!({"req": 1, "w": 2}), &[9, 8, 7]).unwrap(); + let (_h, payload) = read_cfrm(&mut Cursor::new(buf)).unwrap(); + assert_eq!(payload, vec![9, 8, 7]); + } + + #[test] + fn protocol_window() { + assert!(protocols_overlap(1, 1)); + assert!(!protocols_overlap(2, 2)); + assert!(!protocols_overlap(2, 1)); + } +} diff --git a/crates/ipc/tests/bulk.rs b/crates/ipc/tests/bulk.rs new file mode 100644 index 000000000..8cd8295a2 --- /dev/null +++ b/crates/ipc/tests/bulk.rs @@ -0,0 +1,334 @@ +//! Bulk frames over a real local socket with a fake `FrameSource`. +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use craft_ipc::bulk::{BulkClient, FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; +use craft_ipc::client::Client; +use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; +use craft_ipc::wire; +use craft_ipc::{Endpoint, InstanceRecord, LinkError, RuntimeDir}; + +const W: u32 = 4; +const H: u32 = 3; +const STRIDE: u32 = 16; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +/// Pixels encode the request so a wrong frame cannot match by accident. +struct FakeSource { + epoch: AtomicU64, + closed: Mutex>, +} + +/// One row holds `W` pixels: 4 bytes each in rgba8, 8 bytes each in rgba16f. +fn stride_for(format: FrameFormat) -> u32 { + match format { + FrameFormat::Rgba8Premul => STRIDE, + FrameFormat::Rgba16fPremul => W * 8, + } +} + +fn pixels_for(link_id: &str, time_s: f64, region: Option<[u32; 4]>) -> Vec { + let seed = + link_id.bytes().fold(0u8, |acc, b| acc.wrapping_add(b)).wrapping_add((time_s * 10.0) as u8).wrapping_add(region.map_or(0, |r| r[0] as u8)); + (0..(STRIDE * H) as usize).map(|i| seed.wrapping_add((i as u8).wrapping_mul(7))).collect() +} + +impl FrameSource for FakeSource { + fn open(&self, spec: &FrameSpec) -> Result { + if spec.link_id == "missing" { + return Err(LinkError::NotReady("link_unknown".into())); + } + let stride = stride_for(spec.format); + Ok(FrameLayout { w: W, h: H, stride, len: u64::from(stride * H) }) + } + + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { + let stride = stride_for(spec.format); + let mut pixels = pixels_for(&spec.link_id, request.time_s, request.region); + pixels.resize((stride * H) as usize, 0); + Ok(RenderedFrame { epoch: self.epoch.load(Ordering::SeqCst), w: W, h: H, stride, format: spec.format, render_ms: 0.25, pixels }) + } + + fn close(&self, spec: &FrameSpec) { + self.closed.lock().unwrap().push(spec.link_id.clone()); + } +} + +struct Rig { + _tmp: Tmp, + server: Server, + record: InstanceRecord, + source: Arc, + tickets: Arc, +} + +fn boot(epoch: u64) -> Rig { + // Parallel tests can observe the same clock tick on macOS. + static NEXT_RIG: AtomicU64 = AtomicU64::new(0); + let serial = NEXT_RIG.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-blk-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let source = Arc::new(FakeSource { epoch: AtomicU64::new(epoch), closed: Mutex::new(Vec::new()) }); + let tickets = Arc::new(FrameTickets::new(Arc::clone(&source) as Arc)); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(RuntimeDir::at(&path).unwrap()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: Arc::clone(&tickets) as Arc, + }) + .unwrap(); + let record = server.record().clone(); + Rig { _tmp: Tmp(path), server, record, source, tickets } +} + +fn wait_for(what: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while !done() { + assert!(Instant::now() < deadline, "timed out waiting for {what}"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn socket_of(record: &InstanceRecord) -> PathBuf { + match &record.endpoint { + Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + } +} + +#[test] +fn round_trip_returns_the_exact_bytes() { + let rig = boot(3); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + assert_eq!((ticket.w, ticket.h, ticket.stride, ticket.len), (W, H, STRIDE, u64::from(STRIDE * H))); + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + let whole = bulk.request(1.5, 3, None).unwrap(); + assert_eq!(whole.pixels, pixels_for("link-a", 1.5, None)); + assert_eq!((whole.req, whole.epoch, whole.w, whole.h, whole.stride), (1, 3, W, H, STRIDE)); + assert_eq!(whole.format, FrameFormat::Rgba8Premul); + assert!((whole.render_ms - 0.25).abs() < f64::EPSILON); + let part = bulk.request(2.0, 3, Some([5, 0, 2, 2])).unwrap(); + assert_eq!(part.pixels, pixels_for("link-a", 2.0, Some([5, 0, 2, 2]))); + assert_ne!(part.pixels, whole.pixels); + assert_eq!(part.req, 2); + drop(bulk); + drop(control); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_newer_epoch_is_returned_and_an_older_source_is_an_error() { + let rig = boot(7); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba16fPremul, 2.0).unwrap(); + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + let frame = bulk.request(0.0, 2, None).unwrap(); + assert_eq!(frame.epoch, 7, "asked for epoch 2, source is at 7"); + assert_eq!(frame.format, FrameFormat::Rgba16fPremul); + let err = bulk.request(0.0, 9, None).unwrap_err(); + assert!(err.to_string().contains("epoch 7"), "{err}"); + // The error did not poison the connection. + rig.source.epoch.store(9, Ordering::SeqCst); + assert_eq!(bulk.request(0.0, 9, None).unwrap().epoch, 9); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_ticket_is_refused_after_its_control_connection_closes() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut live = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + live.request(0.0, 1, None).unwrap(); + assert_eq!(rig.tickets.open_tickets(), 1); + drop(control); + wait_for("source.close", || !rig.source.closed.lock().unwrap().is_empty()); + assert_eq!(rig.tickets.open_tickets(), 0); + assert!(live.request(0.0, 1, None).is_err(), "an attached connection must lose its frames"); + let again = BulkClient::attach(&rig.record, &ticket.ticket); + assert!(again.is_err(), "reuse after control close must be refused"); + rig.server.shutdown().unwrap(); +} + +#[test] +fn bulk_attach_must_be_first_and_the_ticket_must_exist() { + let rig = boot(1); + let bogus = "0123456789abcdef0123456789abcdef"; + assert!(BulkClient::attach(&rig.record, bogus).is_err()); + let mut control = Client::connect(&rig.record).unwrap(); + assert!(control.frames_open("missing", FrameFormat::Rgba8Premul, 1.0).is_err()); + assert!(control.frames_open("a", FrameFormat::Rgba8Premul, 0.0).is_err()); + assert_eq!(rig.tickets.open_tickets(), 0); + rig.server.shutdown().unwrap(); +} + +fn raw_attach(rig: &Rig, ticket: &str) -> UnixStream { + let mut stream = UnixStream::connect(socket_of(&rig.record)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + wire::write_line(&mut stream, &json!({"id": 0, "method": "auth", "params": {"token": rig.record.token}})).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &json!({"id": 1, "method": "bulk.attach", "params": {"ticket": ticket}})).unwrap(); + let reply: Value = serde_json::from_slice(&read_line(&mut stream)).unwrap(); + assert_eq!(reply["ok"], json!(true), "{reply}"); + stream +} + +fn read_line(stream: &mut UnixStream) -> Vec { + let mut out = Vec::new(); + let mut byte = [0u8; 1]; + while stream.read_exact(&mut byte).is_ok() && byte[0] != b'\n' { + out.push(byte[0]); + } + out +} + +#[test] +fn an_oversized_request_header_closes_that_connection_only() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut raw = raw_attach(&rig, &ticket.ticket); + raw.write_all(b"CFRQ").unwrap(); + raw.write_all(&(wire::BULK_HEADER_MAX + 1).to_le_bytes()).unwrap(); + let mut rest = Vec::new(); + // The server closes without a reply, so the read ends at EOF instead of timing out. + let n = raw.read_to_end(&mut rest).unwrap(); + assert_eq!(n, 0, "no reply expected for an oversized header"); + // The control connection and a fresh bulk connection still work. + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + assert!(bulk.request(0.0, 1, None).is_ok()); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_bad_magic_and_a_bad_header_do_not_panic_the_server() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut raw = raw_attach(&rig, &ticket.ticket); + raw.write_all(b"XXXX\0\0\0\0").unwrap(); + let mut rest = Vec::new(); + assert_eq!(raw.read_to_end(&mut rest).unwrap(), 0); + // A well-formed record with a bad header gets an error record, not a close. + let mut raw = raw_attach(&rig, &ticket.ticket); + wire::write_cfrq(&mut raw, &json!({"req": 5, "epoch": 0})).unwrap(); + let (header, payload) = wire::read_cfrm(&mut raw).unwrap(); + assert_eq!(header["req"], json!(5)); + assert!(header["error"].as_str().unwrap().contains("time_s"), "{header}"); + assert!(payload.is_empty()); + rig.server.shutdown().unwrap(); +} + +/// A one-shot fake server. Answers auth and attach, then sends `bytes` and closes. +fn fake_server(bytes: Vec) -> (InstanceRecord, std::thread::JoinHandle<()>, Tmp) { + static NEXT_FAKE: AtomicU64 = AtomicU64::new(0); + let serial = NEXT_FAKE.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("cl-fk-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("s.sock"); + let listener = UnixListener::bind(&path).unwrap(); + let record = InstanceRecord::new("fake", "0", std::process::id(), Endpoint::Unix { path: path.to_string_lossy().into_owned() }).unwrap(); + let handle = std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { return }; + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &wire::auth_success(&json!(0))).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &wire::reply_ok(&json!(1), json!({}))).unwrap(); + let mut request = [0u8; 8]; + stream.read_exact(&mut request).unwrap(); + let mut header = vec![0u8; u32::from_le_bytes([request[4], request[5], request[6], request[7]]) as usize]; + stream.read_exact(&mut header).unwrap(); + stream.write_all(&bytes).unwrap(); + }); + (record, handle, Tmp(dir)) +} + +fn cfrm(header: &Value, declared_payload: u64, payload: &[u8]) -> Vec { + let header = serde_json::to_vec(header).unwrap(); + let mut out = b"CFRM".to_vec(); + out.extend((header.len() as u32).to_le_bytes()); + out.extend(declared_payload.to_le_bytes()); + out.extend(header); + out.extend(payload); + out +} + +#[test] +fn truncated_replies_are_errors_not_panics() { + let good_header = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + let full = cfrm(&good_header, 48, &[7u8; 48]); + for cut in [2, 6, 10, 20, full.len() - 1] { + let (record, handle, _tmp) = fake_server(full[..cut].to_vec()); + let mut bulk = BulkClient::attach(&record, &"a".repeat(32)).unwrap(); + let err = bulk.request(0.0, 0, None).unwrap_err(); + assert!(matches!(err, LinkError::Io(_)), "cut {cut}: {err:?}"); + handle.join().unwrap(); + } + // The uncut record is accepted by the same fake, so the cuts above are the cause. + let (record, handle, _tmp) = fake_server(full); + let mut bulk = BulkClient::attach(&record, &"a".repeat(32)).unwrap(); + assert_eq!(bulk.request(0.0, 0, None).unwrap().pixels, vec![7u8; 48]); + handle.join().unwrap(); +} + +#[test] +fn oversized_or_inconsistent_replies_are_rejected_by_the_client() { + let good = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + let mut huge_header = b"CFRM".to_vec(); + huge_header.extend((wire::BULK_HEADER_MAX + 1).to_le_bytes()); + huge_header.extend(0u64.to_le_bytes()); + let huge_payload = cfrm(&good, wire::FRAME_PAYLOAD_MAX + 1, &[]); + let short_payload = cfrm(&good, 47, &[1u8; 47]); + let narrow = cfrm(&json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 1, "format": "rgba8_premul", "render_ms": 1.0}), 3, &[1u8; 3]); + let wrong_req = cfrm(&json!({"req": 99, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}), 48, &[1u8; 48]); + for (name, bytes, want_too_large) in [ + ("header", huge_header, true), + ("payload", huge_payload, true), + ("short", short_payload, false), + ("narrow", narrow, false), + ("req", wrong_req, false), + ] { + let (record, handle, _tmp) = fake_server(bytes); + let mut bulk = BulkClient::attach(&record, &"b".repeat(32)).unwrap(); + let err = bulk.request(0.0, 0, None).unwrap_err(); + assert_eq!(matches!(err, LinkError::TooLarge), want_too_large, "{name}: {err:?}"); + handle.join().unwrap(); + } +} diff --git a/crates/ipc/tests/connection_limits.rs b/crates/ipc/tests/connection_limits.rs new file mode 100644 index 000000000..5c0888340 --- /dev/null +++ b/crates/ipc/tests/connection_limits.rs @@ -0,0 +1,280 @@ +//! Connection limits and partial-line handling. Each test asserts the secure behavior: silent +//! peers cannot fill the server, and a line that arrives in pieces is not lost. +#![cfg(unix)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, Extensions, FrameOpen, Frames, Limits, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; + +struct Sink; +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({})); + Some(rx) + } +} +struct NoActivate; +impl Activate for NoActivate { + fn activate(&self, _t: Option, _p: Option, _o: Option) -> Activation { + Activation::No + } +} +struct NoFrames; +impl Frames for NoFrames { + fn open(&self, _p: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _t: &str, _h: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _t: &str) {} +} + +fn boot_with(label: &str, limits: Limits) -> (Server, PathBuf, String, PathBuf) { + let dir = std::env::temp_dir().join(format!("cl-ipc-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let runtime = RuntimeDir::at(&dir).unwrap(); + let server = Server::start_with_limits( + ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(runtime), + tcp_port: None, + sink: Arc::new(Sink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }, + Extensions::default(), + limits, + ) + .unwrap(); + let socket = match &server.record().endpoint { + craft_ipc::Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("unix endpoint expected, got {other:?}"), + }; + let token = server.record().token.clone(); + (server, socket, token, dir) +} + +fn boot(label: &str) -> (Server, PathBuf, String, PathBuf) { + boot_with(label, Limits::default()) +} + +fn auth_line(token: &str) -> String { + format!("{{\"id\":0,\"method\":\"auth\",\"params\":{{\"token\":\"{token}\"}}}}\n") +} + +const HELLO: &str = "{\"id\":1,\"method\":\"link.hello\",\"params\":{\"protocol\":{\"min\":1,\"max\":1}}}\n"; + +fn arm_read_timeout(stream: &UnixStream, secs: u64) { + let result = stream.set_read_timeout(Some(Duration::from_secs(secs))); + // Darwin refuses socket options after shutdown. The queued denial/EOF can still be read. + #[cfg(target_os = "macos")] + if result.as_ref().is_err_and(|err| err.raw_os_error() == Some(libc::EINVAL)) { + return; + } + result.unwrap(); +} + +fn read_text(stream: &mut UnixStream, secs: u64) -> String { + arm_read_timeout(stream, secs); + let mut text = String::new(); + let _ = stream.read_to_string(&mut text); + text +} + +fn next_line(reader: &mut BufReader) -> String { + let mut line = String::new(); + arm_read_timeout(reader.get_ref(), 5); + reader.read_line(&mut line).unwrap(); + line +} + +/// 16 silent connections must not lock the legitimate client out. At most 4 unauthenticated +/// connections are admitted, the rest are refused at once, and the admitted ones are closed at the +/// auth deadline. A client that authenticated before the flood keeps working. +#[test] +fn silent_connections_cannot_lock_out_clients() { + let (server, socket, token, dir) = boot("slots"); + let early = UnixStream::connect(&socket).unwrap(); + let mut early_writer = early.try_clone().unwrap(); + let mut early_reader = BufReader::new(early); + early_writer.write_all(auth_line(&token).as_bytes()).unwrap(); + assert!(next_line(&mut early_reader).contains("authenticated")); + + let mut hogs = Vec::new(); + for _ in 0..16 { + hogs.push(UnixStream::connect(&socket).unwrap()); + } + // The authenticated client still gets its answer while the flood is open. + early_writer.write_all(HELLO.as_bytes()).unwrap(); + let line = next_line(&mut early_reader); + assert!(line.contains("\"ok\":true"), "authenticated client lost service: {line:?}"); + + // Past the auth deadline every silent connection is closed and a new client is served. + std::thread::sleep(Duration::from_millis(3500)); + for hog in &mut hogs { + let text = read_text(hog, 1); + assert!(!text.contains("\"authenticated\""), "{text:?}"); + } + let mut legit = UnixStream::connect(&socket).unwrap(); + legit.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(legit); + let line = next_line(&mut reader); + assert!(line.contains("authenticated"), "legitimate client was not served: {line:?}"); + drop(hogs); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// the unauthenticated pool is small, and the refusal for an extra peer is immediate. +#[test] +fn unauthenticated_peers_beyond_the_cap_are_refused_at_once() { + let limits = Limits { max_unauthenticated: 2, ..Limits::default() }; + let (server, socket, token, dir) = boot_with("unauth-cap", limits); + let a = UnixStream::connect(&socket).unwrap(); + let b = UnixStream::connect(&socket).unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let mut third = UnixStream::connect(&socket).unwrap(); + let text = read_text(&mut third, 1); + assert!(text.contains("too many unauthenticated connections"), "{text:?}"); + // Authenticating frees an unauthenticated slot. + let mut a = a; + a.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(a); + assert!(next_line(&mut reader).contains("authenticated")); + let mut fourth = UnixStream::connect(&socket).unwrap(); + fourth.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(fourth); + assert!(next_line(&mut reader).contains("authenticated")); + drop(b); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a connection that sends no auth line is closed after the deadline (default 2 s). +#[test] +fn silent_connection_is_closed_at_the_auth_deadline() { + let (server, socket, _token, dir) = boot("deadline"); + let mut s = UnixStream::connect(&socket).unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut s, 6); + let waited = start.elapsed(); + assert!(text.contains("authentication required"), "{text:?}"); + assert!(waited < Duration::from_secs(4), "closed after {waited:?}"); + assert!(waited >= Duration::from_millis(1500), "closed too early: {waited:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// an auth line that completes after the deadline does not authenticate, even when it lands +/// between two read polls (250 ms apart) and the poll has not yet seen the deadline pass. +#[test] +fn an_auth_line_that_completes_after_the_deadline_is_refused() { + let limits = Limits { auth_deadline: Duration::from_millis(300), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("lateauth", limits); + let mut s = UnixStream::connect(&socket).unwrap(); + std::thread::sleep(Duration::from_millis(400)); + if let Err(err) = s.write_all(auth_line(&token).as_bytes()) { + assert!( + matches!(err.kind(), std::io::ErrorKind::BrokenPipe | std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::NotConnected), + "{err}" + ); + } + let text = read_text(&mut s, 6); + assert!(text.contains("authentication required"), "{text:?}"); + assert!(!text.contains("\"authenticated\""), "{text:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a peer that starts a line and stalls is cut at the partial-line deadline. +#[test] +fn a_stalled_partial_line_is_closed() { + let limits = Limits { partial_line: Duration::from_millis(800), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("stall", limits); + let mut s = UnixStream::connect(&socket).unwrap(); + s.write_all(auth_line(&token).as_bytes()).unwrap(); + s.write_all(b"{\"id\":1,\"method\":\"link.he").unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut s, 6); + assert!(text.contains("authenticated"), "{text:?}"); + assert!(start.elapsed() < Duration::from_secs(4), "the stalled line stayed open for {:?}", start.elapsed()); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// an authenticated connection with no subscription and no traffic is closed at the idle limit. +/// A subscribed connection stays open. +#[test] +fn idle_connections_are_closed_but_subscribers_stay() { + let limits = Limits { idle: Duration::from_millis(900), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("idle", limits); + let mut idle = UnixStream::connect(&socket).unwrap(); + idle.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut sub = UnixStream::connect(&socket).unwrap(); + sub.write_all(auth_line(&token).as_bytes()).unwrap(); + sub.write_all(b"{\"id\":2,\"method\":\"events.subscribe\",\"params\":{\"events\":[\"doc.changed\"]}}\n").unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut idle, 6); + assert!(text.contains("authenticated"), "{text:?}"); + assert!(start.elapsed() < Duration::from_secs(4), "idle connection stayed open for {:?}", start.elapsed()); + // The subscriber is still served after the idle limit passed. + std::thread::sleep(Duration::from_millis(300)); + sub.write_all(HELLO.as_bytes()).unwrap(); + let mut reader = BufReader::new(sub); + let mut seen = String::new(); + for _ in 0..3 { + seen.push_str(&next_line(&mut reader)); + if seen.contains("\"id\":1") { + break; + } + } + assert!(seen.contains("\"id\":1"), "subscriber was cut: {seen:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a request split at several points, with a pause longer than the 250 ms read poll, is served. +#[test] +fn a_line_split_across_the_250ms_read_poll_is_kept() { + let (server, socket, token, dir) = boot("split"); + for cut in [1usize, 23, HELLO.len() - 2, HELLO.len() - 1] { + let mut s = UnixStream::connect(&socket).unwrap(); + s.write_all(auth_line(&token).as_bytes()).unwrap(); + s.write_all(&HELLO.as_bytes()[..cut]).unwrap(); + std::thread::sleep(Duration::from_millis(600)); + s.write_all(&HELLO.as_bytes()[cut..]).unwrap(); + let mut reader = BufReader::new(s); + assert!(next_line(&mut reader).contains("authenticated"), "cut {cut}"); + let line = next_line(&mut reader); + assert!(line.contains("\"ok\":true"), "cut {cut}: split request was not served: {line:?}"); + } + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// the auth line itself may arrive in pieces. +#[test] +fn an_auth_line_split_across_the_read_poll_is_kept() { + let (server, socket, token, dir) = boot("authsplit"); + let mut s = UnixStream::connect(&socket).unwrap(); + let line = auth_line(&token); + s.write_all(&line.as_bytes()[..20]).unwrap(); + std::thread::sleep(Duration::from_millis(600)); + s.write_all(&line.as_bytes()[20..]).unwrap(); + let mut reader = BufReader::new(s); + assert!(next_line(&mut reader).contains("authenticated")); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/ipc/tests/discovery.rs b/crates/ipc/tests/discovery.rs new file mode 100644 index 000000000..7d9f4a05f --- /dev/null +++ b/crates/ipc/tests/discovery.rs @@ -0,0 +1,97 @@ +//! A record held by a child dies with the child. +//! +//! `hold_until_killed` returns at once unless `CRAFT_HOLD_DIR` is set. +//! The parent test spawns that child, checks the live record, sends SIGKILL, +//! and checks that `Discovery::scan` removed it. Drop does not run on SIGKILL. + +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +use craft_ipc::{Discovery, InstanceRecord, RuntimeDir}; + +#[test] +fn hold_until_killed() { + let Ok(dir) = std::env::var("CRAFT_HOLD_DIR") else { + return; + }; + let runtime = RuntimeDir::at(&dir).unwrap(); + let app = "effectcraft"; + let pid = std::process::id(); + let endpoint = runtime.unix_endpoint(app, pid).unwrap(); + let record = InstanceRecord::new(app, "0.8.3", pid, endpoint).unwrap(); + let held = Discovery::publish(&runtime, record).unwrap(); + std::fs::write(runtime.path().join("ready"), b"ready").unwrap(); + let _held = held; + // park can wake without an unpark. Loop so Drop does not run until SIGKILL. + loop { + std::thread::park(); + } +} + +#[test] +fn sigkill_drops_the_record() { + let path = hold_dir(); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let exe = std::env::current_exe().unwrap(); + let child = Command::new(exe).args(["--exact", "hold_until_killed", "--test-threads=1"]).env("CRAFT_HOLD_DIR", &path).spawn().unwrap(); + let mut guard = HoldCleanup { dir: path.clone(), child: Some(child) }; + let ready = runtime.path().join("ready"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if ready.is_file() { + break; + } + let child = guard.child.as_mut().unwrap(); + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before ready: {status}"); + } + if Instant::now() >= deadline { + panic!("child did not write ready"); + } + std::thread::sleep(Duration::from_millis(20)); + } + let child_pid = guard.child.as_ref().unwrap().id(); + let live = Discovery::scan(&runtime).unwrap(); + assert_eq!(live.len(), 1, "expected one live record"); + assert_eq!(live[0].pid, child_pid); + assert_eq!(live[0].app, "effectcraft"); + let mut child = guard.child.take().unwrap(); + child.kill().unwrap(); + let status = child.wait().unwrap(); + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + assert_eq!(status.signal(), Some(9), "SIGKILL so the child's Drop does not unlock"); + } + #[cfg(not(unix))] + { + let _ = status; + } + let live = Discovery::scan(&runtime).unwrap(); + assert!(live.is_empty(), "scan should drop the record after SIGKILL, got {live:?}"); + assert!(!runtime.record_file("effectcraft", child_pid).exists(), "record json should be gone"); +} + +struct HoldCleanup { + dir: PathBuf, + child: Option, +} + +impl Drop for HoldCleanup { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn hold_dir() -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + std::env::temp_dir().join(format!("cl-hold-{hex}")) +} diff --git a/crates/ipc/tests/fetch.rs b/crates/ipc/tests/fetch.rs new file mode 100644 index 000000000..8e1dcde73 --- /dev/null +++ b/crates/ipc/tests/fetch.rs @@ -0,0 +1,366 @@ +//! `FrameFetcher` against a real server with a fake `FrameSource`: cache, deadline, dedupe, +//! parallel connections, and a peer that goes away. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use craft_ipc::bulk::{FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; +use craft_ipc::fetch::{FetchOptions, Fetched, FrameFetcher, FrameKey, QUEUE_MAX}; +use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; +use craft_ipc::{InstanceRecord, LinkError, RuntimeDir}; + +const W: u32 = 4; +const H: u32 = 3; +const STRIDE: u32 = 16; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +/// The first pixel byte is the frame number asked for, so a wrong frame cannot match by accident. +struct Fake { + epoch: AtomicU64, + delay_ms: AtomicU64, + renders: AtomicUsize, + times: Mutex>, + fail_time: Mutex>, +} + +impl FrameSource for Fake { + fn open(&self, _spec: &FrameSpec) -> Result { + Ok(FrameLayout { w: W, h: H, stride: STRIDE, len: u64::from(STRIDE * H) }) + } + + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { + self.renders.fetch_add(1, Ordering::SeqCst); + self.times.lock().unwrap().push(request.time_s); + let delay = self.delay_ms.load(Ordering::SeqCst); + if delay > 0 { + std::thread::sleep(Duration::from_millis(delay)); + } + if *self.fail_time.lock().unwrap() == Some(request.time_s) { + return Err(LinkError::NotReady("comp is busy".into())); + } + let mut pixels = vec![0u8; (STRIDE * H) as usize]; + pixels[0] = request.time_s as u8; + Ok(RenderedFrame { + epoch: self.epoch.load(Ordering::SeqCst).max(request.epoch), + w: W, + h: H, + stride: STRIDE, + format: spec.format, + render_ms: 1.0, + pixels, + }) + } + + fn close(&self, _spec: &FrameSpec) {} +} + +struct Rig { + _tmp: Tmp, + server: Option, + record: InstanceRecord, + fake: Arc, +} + +fn boot() -> Rig { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-fch-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let fake = Arc::new(Fake { + epoch: AtomicU64::new(1), + delay_ms: AtomicU64::new(0), + renders: AtomicUsize::new(0), + times: Mutex::new(Vec::new()), + fail_time: Mutex::new(None), + }); + let tickets = Arc::new(FrameTickets::new(Arc::clone(&fake) as Arc)); + let server = Server::start(ServerConfig { + app: format!("ec-{serial}"), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(RuntimeDir::at(&path).unwrap()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: tickets as Arc, + }) + .unwrap(); + let record = server.record().clone(); + Rig { _tmp: Tmp(path), server: Some(server), record, fake } +} + +fn connect(rig: &Rig, connections: usize) -> FrameFetcher { + let options = FetchOptions { connections, io_timeout: Duration::from_secs(2), ..FetchOptions::default() }; + FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap() +} + +const fn key(frame: i64) -> FrameKey { + FrameKey { frame, scale_milli: 1000 } +} + +const WAIT: Duration = Duration::from_secs(2); + +#[test] +fn fetch_returns_the_frame_and_the_second_ask_is_a_cache_hit() { + let rig = boot(); + let fetcher = connect(&rig, 1); + let layout = fetcher.layout(); + assert_eq!((layout.w, layout.h, layout.stride), (W, H, STRIDE)); + let Fetched::Frame(first) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; + assert_eq!(first.pixels[0], 7); + assert_eq!(first.epoch, 1); + let Fetched::Frame(second) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; + assert!(Arc::ptr_eq(&first, &second), "a hit returns the cached frame"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); + let stats = fetcher.stats(); + assert_eq!((stats.hits, stats.renders), (1, 1)); +} + +#[test] +fn a_late_frame_misses_the_deadline_then_lands_in_the_cache() { + let rig = boot(); + rig.fake.delay_ms.store(300, Ordering::SeqCst); + let fetcher = connect(&rig, 1); + let t0 = Instant::now(); + assert!(matches!(fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(40)), Fetched::TimedOut)); + assert!(t0.elapsed() < Duration::from_millis(200), "the call must not wait for the render: {:?}", t0.elapsed()); + assert!(fetcher.cached(key(3), 1).is_none(), "not there yet"); + let deadline = Instant::now() + Duration::from_secs(3); + while fetcher.cached(key(3), 1).is_none() { + assert!(Instant::now() < deadline, "the late frame never reached the cache"); + std::thread::sleep(Duration::from_millis(10)); + } + let Fetched::Frame(frame) = fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(10)) else { panic!("no frame") }; + assert_eq!(frame.pixels[0], 3); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1, "the timed-out ask and the retry share one render"); + assert_eq!(fetcher.stats().timeouts, 1); +} + +#[test] +fn a_newer_epoch_asked_for_renders_again_and_an_older_cached_one_does_not_count() { + let rig = boot(); + let fetcher = connect(&rig, 1); + let Fetched::Frame(old) = fetcher.fetch(key(1), 1.0, 1, WAIT) else { panic!("no frame") }; + assert_eq!(old.epoch, 1); + rig.fake.epoch.store(5, Ordering::SeqCst); + // Epoch 1 is still good for a caller that wants epoch 1. + assert!(fetcher.cached(key(1), 1).is_some()); + assert!(fetcher.cached(key(1), 2).is_none(), "an epoch 1 frame cannot answer a request for epoch 2"); + let Fetched::Frame(fresh) = fetcher.fetch(key(1), 1.0, 2, WAIT) else { panic!("no frame") }; + assert_eq!(fresh.epoch, 5, "the source may answer with a newer epoch"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2); + // The newer frame answers an older request too. + assert!(fetcher.cached(key(1), 1).is_some_and(|f| f.epoch == 5)); +} + +#[test] +fn two_asks_for_one_key_render_once() { + let rig = boot(); + rig.fake.delay_ms.store(120, Ordering::SeqCst); + let fetcher = Arc::new(connect(&rig, 2)); + let a = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) + }; + let b = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) + }; + assert!(a.join().unwrap() && b.join().unwrap()); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); +} + +#[test] +fn four_connections_render_four_frames_at_once() { + let rig = boot(); + rig.fake.delay_ms.store(200, Ordering::SeqCst); + let fetcher = Arc::new(connect(&rig, 4)); + let t0 = Instant::now(); + let handles: Vec<_> = (0..4) + .map(|n| { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))) + }) + .collect(); + for handle in handles { + assert!(handle.join().unwrap()); + } + let took = t0.elapsed(); + assert!(took < Duration::from_millis(600), "four 200 ms renders on four connections took {took:?}"); +} + +#[test] +fn a_frame_error_fails_that_frame_and_the_fetcher_keeps_working() { + let rig = boot(); + *rig.fake.fail_time.lock().unwrap() = Some(4.0); + let fetcher = connect(&rig, 1); + match fetcher.fetch(key(4), 4.0, 1, WAIT) { + Fetched::Failed(message) => assert!(message.contains("comp is busy"), "{message}"), + other => panic!("expected Failed, got {other:?}"), + } + assert!(!fetcher.is_down()); + assert!(matches!(fetcher.fetch(key(5), 5.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn a_peer_that_goes_away_makes_the_fetcher_down_and_asks_return_at_once() { + let mut rig = boot(); + let fetcher = connect(&rig, 2); + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); + rig.server.take().unwrap().shutdown().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !fetcher.is_down() { + // A miss makes the workers touch the dead socket. + let _ = fetcher.fetch(key(100), 100.0, 1, Duration::from_millis(50)); + assert!(Instant::now() < deadline, "the fetcher never noticed the peer is gone"); + } + let t0 = Instant::now(); + assert!(matches!(fetcher.fetch(key(200), 200.0, 1, WAIT), Fetched::Down)); + assert!(t0.elapsed() < Duration::from_millis(100)); + // Frames already in the cache still answer. + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn the_cache_stays_inside_its_byte_budget() { + let rig = boot(); + let options = FetchOptions { connections: 1, cache_bytes: usize::try_from(STRIDE * H).unwrap() * 3, ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + for n in 0..6 { + assert!(matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))); + } + let kept = (0..6).filter(|n| fetcher.cached(key(*n), 1).is_some()).count(); + assert_eq!(kept, 3, "a budget of three frames keeps three"); + assert!(fetcher.cached(key(5), 1).is_some(), "the newest frame stays"); +} + +#[test] +fn queued_asks_nobody_waits_for_are_dropped_when_old() { + let rig = boot(); + rig.fake.delay_ms.store(150, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + // Five asks with no patience: frame 0 starts at once, frames 1 to 4 wait behind it. + for n in 0..5 { + assert!(matches!(fetcher.fetch(key(n), n as f64, 1, Duration::from_millis(1)), Fetched::TimedOut)); + } + std::thread::sleep(Duration::from_millis(900)); + let renders = rig.fake.renders.load(Ordering::SeqCst); + assert!(renders < 5, "frames that waited longer than max_age must be skipped, got {renders} renders"); +} + +#[test] +fn a_time_that_is_not_finite_fails_the_ask_and_keeps_the_workers() { + let rig = boot(); + let fetcher = connect(&rig, 1); + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + match fetcher.fetch(key(1), bad, 1, WAIT) { + Fetched::Failed(message) => assert!(message.contains("finite"), "{message}"), + other => panic!("expected Failed for {bad}, got {other:?}"), + } + } + assert!(!fetcher.is_down(), "a bad time must not end the worker"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 0, "nothing was queued"); + assert!(matches!(fetcher.fetch(key(2), 2.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn an_ask_with_a_waiting_caller_is_not_dropped_for_age() { + let rig = boot(); + rig.fake.delay_ms.store(300, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; + let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); + // Frame 0 keeps the only worker busy for 300 ms. Frame 1 waits in the queue past max_age, + // and its caller is still there when the worker reaches it. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + let waiter = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || fetcher.fetch(key(1), 1.0, 1, WAIT)) + }; + // A second caller for the same key joins the queued job and shares its render. + std::thread::sleep(Duration::from_millis(20)); + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + assert!(matches!(waiter.join().unwrap(), Fetched::Frame(_))); + let stats = fetcher.stats(); + assert_eq!(stats.skipped, 0, "a job a caller waits for is not skipped"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2, "each frame renders once"); +} + +#[test] +fn the_queue_holds_at_most_queue_max_asks() { + let rig = boot(); + rig.fake.delay_ms.store(500, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + // Frame 0 holds the only worker. The asks after it only queue. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + std::thread::sleep(Duration::from_millis(100)); + let extra = 40; + for n in 1..=i64::try_from(QUEUE_MAX + extra).unwrap() { + assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); + } + assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap(), "the oldest asks past the cap are dropped"); +} + +#[test] +fn a_caller_waiting_on_a_dropped_ask_gets_failed_and_does_not_queue_it_again() { + let rig = boot(); + rig.fake.delay_ms.store(1500, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; + let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); + // Frame 0 holds the only worker. Frame 1 queues behind it with a caller that waits. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + std::thread::sleep(Duration::from_millis(100)); + let (done, finished) = std::sync::mpsc::channel(); + let waiter = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || { + let started = Instant::now(); + let _ = done.send((fetcher.fetch(key(1), 1.0, 1, Duration::from_secs(20)), started.elapsed())); + }) + }; + std::thread::sleep(Duration::from_millis(100)); + // Fill the queue past its cap: frame 1 is the oldest, so it goes first. + let extra = 5; + for n in 2..=i64::try_from(QUEUE_MAX + extra).unwrap() { + assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); + } + let (outcome, waited) = finished.recv_timeout(Duration::from_secs(1)).expect("the dropped caller must return at once"); + assert!(matches!(&outcome, Fetched::Failed(message) if message.contains("dropped")), "{outcome:?}"); + assert!(waited < Duration::from_secs(5), "{waited:?}"); + waiter.join().unwrap(); + std::thread::sleep(Duration::from_millis(100)); + // Frame 1 plus the others make QUEUE_MAX + extra asks for QUEUE_MAX places. A caller that queued + // frame 1 again would evict more. + assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap()); + // A new ask for the dropped frame is a new ask, not a repeat of the failure. + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); +} diff --git a/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json b/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json new file mode 100644 index 000000000..9b0330361 --- /dev/null +++ b/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json @@ -0,0 +1,13 @@ +{ + "v": 1, + "link_id": "550e8400-e29b-41d4-a716-446655440000", + "target_app": "effectcraft", + "target_project": "links/shot.ecproj", + "target_comp": "comp-1", + "nested_sequence": "item-9", + "result_path": "Project Links/550e8400-e29b-41d4-a716-446655440000.mov", + "result_epoch": 7, + "format": { "w": 1920, "h": 1080, "rate_num": 24, "rate_den": 1, "duration_ticks": 240, "alpha": true, "tint": "warm" }, + "created": "2026-10-11T15:04:05Z", + "badge": "kept" +} diff --git a/crates/ipc/tests/fixtures/link-record-still.json b/crates/ipc/tests/fixtures/link-record-still.json new file mode 100644 index 000000000..20cf44e85 --- /dev/null +++ b/crates/ipc/tests/fixtures/link-record-still.json @@ -0,0 +1,15 @@ +{ + "v": 1, + "link_id": "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", + "target_app": "photocraft", + "target_project": "art/poster.photocraft", + "result_path": "Project Links/0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d.png", + "result_epoch": 3, + "asset": { + "kind": "still", + "moniker": "craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/flattened" + }, + "producer": { "app": "photocraft", "app_version": "0.4.2", "doc_id": "9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f" }, + "format": { "w": 4096, "h": 2160, "alpha": false }, + "created": "2026-10-23T09:30:00.250+02:00" +} diff --git a/crates/ipc/tests/hub.rs b/crates/ipc/tests/hub.rs new file mode 100644 index 000000000..0a9482b41 --- /dev/null +++ b/crates/ipc/tests/hub.rs @@ -0,0 +1,334 @@ +//! A hub is a peer. Two fake apps share one runtime directory: +//! `photocraft`, which owns a still and answers the asset methods, and `hub`, +//! which owns nothing, writes `caps: ["hub"]` and receives `app.activate` +//! with `open:{asset}`. A third party finds both through discovery. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use craft_ipc::asset::{Described, Refreshed, Resolved}; +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::moniker::Moniker; +use craft_ipc::record::AssetKind; +use craft_ipc::refresh::{Tracker, write_atomic}; +use craft_ipc::server::{Activate, Activation, Assets, ControlSink, Extensions, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{Discovery, InstanceRecord, LinkError, RuntimeDir}; + +// Both tests bind the same app and process named pipe on Windows, regardless of runtime directory. +static HUB_TEST: Mutex<()> = Mutex::new(()); + +const DOC: &str = "9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f"; +const LINK: &str = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +struct Tmp(PathBuf); + +impl Tmp { + fn new() -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-hub-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[derive(Default)] +struct RecordingSink { + seen: Mutex>, +} + +impl ControlSink for RecordingSink { + fn submit(&self, method: String, _params: Value) -> Option> { + self.seen.lock().unwrap().push(method); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({"forwarded": true})); + Some(rx) + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[derive(Default)] +struct RecordingActivate { + opens: Mutex>>, +} + +impl Activate for RecordingActivate { + fn activate(&self, _token: Option, _from_pid: Option, open: Option) -> Activation { + self.opens.lock().unwrap().push(open); + Activation::Requested + } +} + +/// A producer with one still. The real file watcher answers `link.refresh`. +struct Stills { + asset: Moniker, + file: PathBuf, + thumb: PathBuf, + tracker: Mutex, + links: HashMap, +} + +impl Assets for Stills { + fn resolve(&self, asset: &Moniker) -> Result { + self.known(asset)?; + let fp = craft_ipc::refresh::fingerprint_if_present(&self.file)?.ok_or_else(|| LinkError::Unknown("no result file yet".into()))?; + let epoch = self.tracker.lock().unwrap().epoch(); + Ok(Resolved::from_fingerprint(asset.clone(), &self.file.display().to_string(), epoch, &fp)) + } + + fn describe(&self, asset: &Moniker) -> Result { + self.known(asset)?; + let fp = craft_ipc::refresh::fingerprint_if_present(&self.file)?; + Ok(Described { + asset: asset.clone(), + name: "Poster".into(), + kind: AssetKind::Still, + epoch: self.tracker.lock().unwrap().epoch(), + w: Some(640), + h: Some(480), + hash: fp.as_ref().map(|f| f.hash.clone()), + thumb_path: Some(self.thumb.display().to_string()), + modified_ms: fp.map(|f| f.mtime_ms), + }) + } + + fn refresh(&self, link_id: &str) -> Result { + if !self.links.contains_key(link_id) { + return Err(LinkError::Unknown(format!("link {link_id} is not known"))); + } + let mut tracker = self.tracker.lock().unwrap(); + let change = tracker.poll(true)?; + Ok(Refreshed::new(link_id, change.epoch, change.state, &self.file.display().to_string(), tracker.seen())) + } +} + +impl Stills { + fn known(&self, asset: &Moniker) -> Result<(), LinkError> { + if *asset == self.asset { Ok(()) } else { Err(LinkError::Unknown(format!("{asset} is not an asset of this document"))) } + } +} + +fn config(app: &str, runtime: &RuntimeDir, sink: Arc, activate: Arc) -> ServerConfig { + ServerConfig { + app: app.into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink, + activate, + frames: Arc::new(NoFrames), + } +} + +fn dial(runtime: &RuntimeDir, app: &str) -> (Client, InstanceRecord) { + let found = Discovery::scan(runtime).unwrap(); + let record = found.into_iter().find(|r| r.app == app).unwrap_or_else(|| panic!("no live {app} record")); + let mut client = Client::connect(&record).unwrap(); + let identity = HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440000".into(), + protocol_min: 1, + protocol_max: 1, + }; + let hello = client.hello(&identity).unwrap(); + assert_eq!(hello["app"], json!(app)); + (client, record) +} + +fn caps_of(hello_or_record: &Value) -> Vec { + hello_or_record["caps"].as_array().unwrap().iter().map(|c| c.as_str().unwrap().to_string()).collect() +} + +/// The two fake apps of the hub test and the files behind them. +struct World { + runtime: RuntimeDir, + asset: Moniker, + file: PathBuf, + producer: Server, + producer_sink: Arc, + hub: Server, + hub_activate: Arc, +} + +fn start_world(tmp: &Tmp) -> World { + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let asset = Moniker::parse(&format!("craft://photocraft/{DOC}/image/flattened")).unwrap(); + let file = tmp.0.join("flat.png"); + write_atomic(&file, b"png-v1").unwrap(); + let stills = Arc::new(Stills { + asset: asset.clone(), + file: file.clone(), + thumb: tmp.0.join("thumb.png"), + tracker: Mutex::new(Tracker::new(&file, 3).unwrap()), + links: HashMap::from([(LINK.to_string(), ())]), + }); + + let producer_sink = Arc::new(RecordingSink::default()); + let producer = Server::start_with( + config("photocraft", &runtime, Arc::clone(&producer_sink), Arc::new(RecordingActivate::default())), + Extensions { assets: Some(stills), caps: Vec::new() }, + ) + .unwrap(); + let hub_activate = Arc::new(RecordingActivate::default()); + let hub = Server::start_with( + config("hub", &runtime, Arc::new(RecordingSink::default()), Arc::clone(&hub_activate)), + Extensions { assets: None, caps: vec!["hub".into()] }, + ) + .unwrap(); + World { runtime, asset, file, producer, producer_sink, hub, hub_activate } +} + +/// Discovery: the hub is an ordinary record with app "hub" and caps ["hub"]. `link.hello` +/// advertises the base caps plus the record caps. +fn check_discovery_and_hello(world: &World, hub_client: &mut Client, hub_record: &InstanceRecord, producer_client: &mut Client) { + let live = Discovery::scan(&world.runtime).unwrap(); + let mut names: Vec<&str> = live.iter().map(|r| r.app.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["hub", "photocraft"]); + assert_eq!(world.hub.record().caps(), vec!["hub".to_string()]); + assert_eq!(world.producer.record().caps(), vec!["assets".to_string()]); + + let hello = hub_client.call("link.hello", json!({"protocol": {"min": 1, "max": 1}})).unwrap(); + assert_eq!(hello["app"], json!("hub")); + assert!(caps_of(&hello).contains(&"hub".to_string()), "{hello}"); + assert!(caps_of(&hello).contains(&"activate".to_string()), "{hello}"); + assert_eq!(hub_record.docs.len(), 0, "a hub owns no document"); + + let hello = producer_client.call("link.hello", json!({"protocol": {"min": 1, "max": 1}})).unwrap(); + assert!(caps_of(&hello).contains(&"assets".to_string()), "{hello}"); +} + +/// The hub browses the producer with no document open. Returns the hash of the first version. +fn check_browse(world: &World, producer_client: &mut Client) -> String { + let asset = &world.asset; + let described = producer_client.call("asset.describe", json!({"asset": asset.to_string()})).unwrap(); + assert_eq!(described["name"], json!("Poster")); + assert_eq!(described["kind"], json!("still")); + assert_eq!(described["epoch"], json!(3)); + assert_eq!(described["w"], json!(640)); + assert_eq!(described["asset"], json!(asset.to_string())); + let resolved = producer_client.call("link.resolve", json!({"asset": asset.to_string()})).unwrap(); + assert_eq!(resolved["path"], json!(world.file.display().to_string())); + assert_eq!(resolved["size"], json!(6)); + let v1_hash = resolved["hash"].as_str().unwrap().to_string(); + assert_eq!(described["hash"], json!(v1_hash)); + assert!(v1_hash.starts_with("sha256:") && v1_hash.len() == 71, "{v1_hash}"); + assert!(world.producer_sink.seen.lock().unwrap().is_empty(), "asset methods must not reach the UI sink"); + v1_hash +} + +/// Errors map to the protocol's error codes. +fn check_error_codes(producer_client: &mut Client) { + let other = format!("craft://photocraft/{DOC}/image/other"); + assert!(matches!(producer_client.call("asset.describe", json!({"asset": other})), Err(LinkError::Unknown(_)))); + assert!(matches!(producer_client.call("link.resolve", json!({"asset": "craft://nope"})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.resolve", json!({})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.refresh", json!({"link_id": ""})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.refresh", json!({"link_id": "7b1e0000-0000-4000-8000-000000000000"})), Err(LinkError::Unknown(_)))); +} + +/// `link.refresh` sees a new version on disk and bumps the epoch once. +fn check_refresh(world: &World, producer_client: &mut Client, v1_hash: &str) { + let file = &world.file; + let current = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(current["state"], json!("current")); + assert_eq!(current["epoch"], json!(3)); + write_atomic(file, b"png-version-two").unwrap(); + let modified = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(modified["state"], json!("modified")); + assert_eq!(modified["epoch"], json!(4)); + assert_ne!(modified["hash"], json!(v1_hash)); + let again = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(again["epoch"], json!(4), "no second bump without a new edit"); + std::fs::remove_file(file).unwrap(); + let missing = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(missing["state"], json!("missing")); + assert!(missing.get("hash").is_none() && missing.get("result_path").is_none(), "{missing}"); + assert!(matches!(producer_client.call("link.resolve", json!({"asset": world.asset.to_string()})), Err(LinkError::Unknown(_)))); +} + +/// `app.activate` carries open:{asset} to the hub's Activate. +fn check_activate(world: &World, hub_client: &mut Client) { + let asset = &world.asset; + let reply = hub_client.call("app.activate", json!({"token": null, "from_pid": 4242, "open": {"asset": asset.to_string()}})).unwrap(); + assert_eq!(reply["activated"], json!("requested")); + let reply = hub_client.call("app.activate", json!({"open": {"link_id": LINK, "asset": asset.to_string()}})).unwrap(); + assert_eq!(reply["activated"], json!("requested")); + // Legacy shapes still pass through unchanged. + hub_client.call("app.activate", json!({"open": {"link_id": LINK}})).unwrap(); + hub_client.call("app.activate", json!({})).unwrap(); + // A bad moniker never reaches the app. + assert!(matches!(hub_client.call("app.activate", json!({"open": {"asset": "craft://bad"}})), Err(LinkError::BadParams(_)))); + assert!(matches!(hub_client.call("app.activate", json!({"open": {"asset": 7}})), Err(LinkError::BadParams(_)))); + let opens = world.hub_activate.opens.lock().unwrap().clone(); + assert_eq!( + opens, + vec![ + Some(json!({"asset": asset.to_string()})), + Some(json!({"link_id": LINK, "asset": asset.to_string()})), + Some(json!({"link_id": LINK})), + None, + ] + ); +} + +#[test] +fn hub_is_a_peer_and_producer_answers_asset_methods() { + let _guard = HUB_TEST.lock().unwrap(); + let tmp = Tmp::new(); + let world = start_world(&tmp); + let (mut hub_client, hub_record) = dial(&world.runtime, "hub"); + let (mut producer_client, _) = dial(&world.runtime, "photocraft"); + + check_discovery_and_hello(&world, &mut hub_client, &hub_record, &mut producer_client); + let v1_hash = check_browse(&world, &mut producer_client); + check_error_codes(&mut producer_client); + check_refresh(&world, &mut producer_client, &v1_hash); + check_activate(&world, &mut hub_client); + + world.producer.shutdown().unwrap(); + world.hub.shutdown().unwrap(); +} + +#[test] +fn server_without_a_provider_forwards_asset_methods_to_the_app() { + let _guard = HUB_TEST.lock().unwrap(); + let tmp = Tmp::new(); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let sink = Arc::new(RecordingSink::default()); + let server = Server::start(config("photocraft", &runtime, Arc::clone(&sink), Arc::new(RecordingActivate::default()))).unwrap(); + assert!(server.record().caps().is_empty()); + let (mut client, _) = dial(&runtime, "photocraft"); + let asset = format!("craft://photocraft/{DOC}/image/x"); + for method in ["link.resolve", "asset.describe", "link.refresh"] { + let reply = client.call(method, json!({"asset": asset, "link_id": LINK})).unwrap(); + assert_eq!(reply, json!({"forwarded": true}), "{method}"); + } + assert_eq!(sink.seen.lock().unwrap().as_slice(), ["link.resolve", "asset.describe", "link.refresh"]); + server.shutdown().unwrap(); +} diff --git a/crates/ipc/tests/native_roundtrip.rs b/crates/ipc/tests/native_roundtrip.rs new file mode 100644 index 000000000..7539d91cf --- /dev/null +++ b/crates/ipc/tests/native_roundtrip.rs @@ -0,0 +1,93 @@ +//! Exercise the real native transport: Unix sockets or Windows named pipes. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::PathBuf; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; +use serde_json::{Value, json}; + +struct Cleanup(PathBuf); +impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct Echo; +impl ControlSink for Echo { + fn submit(&self, method: String, params: Value) -> Option> { + let (tx, rx) = mpsc::channel(); + tx.send(json!({"ok": true, "result": {"method": method, "params": params}})).unwrap(); + Some(rx) + } +} + +struct NoActivate; +impl Activate for NoActivate { + fn activate(&self, _token: Option, _pid: Option, _options: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _hints: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[test] +fn native_transport_authenticates_and_delivers_commands() { + let mut nonce = [0u8; 4]; + getrandom::getrandom(&mut nonce).unwrap(); + let path = std::env::temp_dir().join(format!("cl-nr-{}-{}", std::process::id(), u32::from_le_bytes(nonce))); + let cleanup = Cleanup(path.clone()); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "test".into(), + instance: None, + runtime: Some(RuntimeDir::at(path).unwrap()), + tcp_port: None, + sink: Arc::new(Echo), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let mut client = Client::connect(server.record()).unwrap(); + client + .hello(&HelloClient { + app: "ec".into(), + app_version: "test".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440000".into(), + protocol_min: craft_ipc::wire::PROTOCOL_MIN, + protocol_max: craft_ipc::wire::PROTOCOL_MAX, + }) + .unwrap(); + let reply = client.call("engine.execute", json!({"command": "ping"})).unwrap(); + assert_eq!(reply, json!({"method": "engine.execute", "params": {"command": "ping"}})); + + client.subscribe(&["doc.changed"]).unwrap(); + let start = Instant::now(); + assert!(client.next_event(Some(Duration::from_millis(50))).unwrap().is_none()); + assert!(start.elapsed() < Duration::from_secs(2), "quiet native transport must respect its deadline"); + server.emit("doc.changed", json!({"after_timeout": true})).unwrap(); + let event = client.next_event(Some(Duration::from_secs(2))).unwrap().expect("event after timeout"); + assert_eq!(event.data, json!({"after_timeout": true})); + + let mut wrong = server.record().clone(); + let opposite = if wrong.token.starts_with('0') { "1" } else { "0" }; + wrong.token.replace_range(..1, opposite); + assert!(matches!(Client::connect(&wrong), Err(LinkError::Auth))); + drop(client); + drop(server); + drop(cleanup); +} diff --git a/crates/ipc/tests/open_payload.rs b/crates/ipc/tests/open_payload.rs new file mode 100644 index 000000000..1a6f3453b --- /dev/null +++ b/crates/ipc/tests/open_payload.rs @@ -0,0 +1,169 @@ +//! `link.open` payload validation over a real server and a real client. +//! A bad payload gets `payload_invalid` and never reaches the app sink. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{Discovery, LinkError, RuntimeDir}; + +const ASSET: &str = "craft://photocraft/550e8400-e29b-41d4-a716-446655440000/image/flat"; +const LINK: &str = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +struct Tmp(PathBuf); + +impl Tmp { + fn new() -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-open-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Records every forwarded request and answers with a finished reply payload. +#[derive(Default)] +struct RecordingSink { + seen: Mutex>, +} + +impl ControlSink for RecordingSink { + fn submit(&self, method: String, params: Value) -> Option> { + self.seen.lock().unwrap().push((method, params)); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({"comp": "c1"})); + Some(rx) + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +fn native_path(name: &str) -> String { + std::env::temp_dir().join(name).to_str().unwrap().to_owned() +} + +fn file_url(path: &std::path::Path) -> String { + let text = path.to_str().unwrap().replace('\\', "/"); + if text.starts_with('/') { format!("file://{text}") } else { format!("file:///{text}") } +} + +fn open_params(payload: Value) -> Value { + json!({"link_id": LINK, "payload": payload, "result_path": native_path("x.png")}) +} + +#[test] +fn bad_payloads_get_payload_invalid_and_good_ones_reach_the_sink() { + let tmp = Tmp::new(); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let sink = Arc::new(RecordingSink::default()); + let server = Server::start(ServerConfig { + app: "effectcraft".into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink: Arc::clone(&sink) as Arc, + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let record = Discovery::scan(&runtime).unwrap().into_iter().next().unwrap(); + let mut client = Client::connect(&record).unwrap(); + client + .hello(&HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440001".into(), + protocol_min: 1, + protocol_max: 1, + }) + .unwrap(); + + let bad = [ + ("unknown format", json!({"format": "svg", "data": ""})), + ("no format", json!({"data": "{}"})), + ("payload is a string", json!("otio")), + ("otio without data", json!({"format": "otio-json"})), + ("otio empty data", json!({"format": "otio-json", "data": ""})), + ("image-doc relative path", json!({"format": "image-doc", "path": "rel.png", "asset": ASSET})), + ("image-doc no path", json!({"format": "image-doc", "asset": ASSET})), + ("image-doc bad moniker", json!({"format": "image-doc", "path": native_path("a.png"), "asset": "craft://bad"})), + ("image-doc no asset", json!({"format": "image-doc", "path": native_path("a.png")})), + ]; + for (name, payload) in bad { + let err = client.call("link.open", open_params(payload)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{name}: {err:?}"); + } + let err = client.call("link.open", json!({"link_id": LINK})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "missing payload: {err:?}"); + // A payload that names a file that is not there gets media_missing with the paths. + let missing_path = tmp.0.join("missing-clip.mov"); + let missing = json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": file_url(&missing_path)}).to_string(); + let err = client.call("link.open", open_params(json!({"format": "otio-json", "data": missing}))).unwrap_err(); + assert_eq!(err, LinkError::MediaMissing { paths: vec![missing_path.to_str().unwrap().replace('\\', "/")] }); + // A path that climbs, is relative or is not a string never reaches the app. + let good = json!({"format": "otio-json", "data": "{}"}); + for result_path in [json!("../x.mov"), json!("/tmp/../etc/x.mov"), json!("rel/x.mov"), json!(7), json!("/tmp/a\u{0}b")] { + let params = json!({"link_id": LINK, "payload": good, "result_path": result_path}); + let err = client.call("link.open", params).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{result_path}: {err:?}"); + } + let climbing = json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": file_url(&tmp.0.join("..").join("outside.mov"))}).to_string(); + let err = client.call("link.open", open_params(json!({"format": "otio-json", "data": climbing}))).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // Params that are not an object are still bad_params. + let err = client.call("link.open", json!("nope")).unwrap_err(); + assert!(matches!(err, LinkError::BadParams(_)), "{err:?}"); + assert!(sink.seen.lock().unwrap().is_empty(), "a bad payload must not reach the app: {:?}", sink.seen.lock().unwrap()); + + let otio = open_params(json!({"format": "otio-json", "data": "{\"OTIO_SCHEMA\":\"Timeline.1\"}"})); + let still = open_params(json!({"format": "image-doc", "path": native_path("a.photocraft"), "asset": ASSET})); + for params in [otio, still] { + let reply = client.call("link.open", params.clone()).unwrap(); + assert_eq!(reply, json!({"comp": "c1"})); + let seen = sink.seen.lock().unwrap(); + let (method, got) = seen.last().unwrap(); + assert_eq!(method, "link.open"); + assert_eq!(got, ¶ms, "the app gets the request unchanged"); + } + assert_eq!(sink.seen.lock().unwrap().len(), 2); + + // The wire carries the contract code, not bad_params. + let err = client.call("link.open", open_params(json!({"format": "svg"}))).unwrap_err(); + assert_eq!(err.code(), "payload_invalid"); + // The connection stays usable after a refusal. + client.call("link.open", open_params(json!({"format": "otio-json", "data": "{}"}))).unwrap(); + assert_eq!(sink.seen.lock().unwrap().len(), 3); + + server.shutdown().unwrap(); +} diff --git a/crates/ipc/tests/protocol.rs b/crates/ipc/tests/protocol.rs new file mode 100644 index 000000000..d77a46017 --- /dev/null +++ b/crates/ipc/tests/protocol.rs @@ -0,0 +1,361 @@ +//! Session behaviour. The tests own the sink, the activate trait, and frames. +//! They do not use the client module. +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Read}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::wire::{self, MAX_LINE}; +use craft_ipc::{Endpoint, LinkError, RuntimeDir}; + +const PAYLOAD: &[u8] = b"craft-frame"; +const TICKET: &str = "00112233445566778899aabbccddeeff"; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct ReplySink { + result: Mutex, + seen: Mutex>, +} + +impl ControlSink for ReplySink { + fn submit(&self, method: String, _params: Value) -> Option> { + self.seen.lock().unwrap().push(method); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(self.result.lock().unwrap().clone()); + Some(rx) + } +} + +struct SwitchActivate { + mode: Mutex, + seen_token: Mutex>>, + seen_pid: Mutex>>, +} + +impl Activate for SwitchActivate { + fn activate(&self, token: Option, from_pid: Option, _open: Option) -> Activation { + *self.seen_token.lock().unwrap() = Some(token); + *self.seen_pid.lock().unwrap() = Some(from_pid); + *self.mode.lock().unwrap() + } +} + +struct TestFrames { + open_count: Mutex, + closed: Mutex>, +} + +impl Frames for TestFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + *self.open_count.lock().unwrap() += 1; + Ok(FrameOpen { ticket: TICKET.into(), w: 4, h: 2, stride: 16, len: PAYLOAD.len() as u64 }) + } + + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError> { + if ticket != TICKET { + return Err(LinkError::BadParams("unknown ticket".into())); + } + let req = header.get("req").cloned().unwrap_or(Value::Null); + Ok((json!({"req": req, "epoch": 3, "w": 4, "h": 2, "stride": 16, "format": "rgba8_premul", "render_ms": 1.5}), PAYLOAD.to_vec())) + } + + fn close_ticket(&self, ticket: &str) { + self.closed.lock().unwrap().push(ticket.to_string()); + } +} + +struct Harness { + tmp: Tmp, + server: Server, + socket: PathBuf, + token: String, + sink: Arc, + activate: Arc, + frames: Arc, +} + +fn boot() -> Harness { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-proto-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let sink = Arc::new(ReplySink { result: Mutex::new(json!({"commands": ["cut"]})), seen: Mutex::new(Vec::new()) }); + let activate = Arc::new(SwitchActivate { mode: Mutex::new(Activation::Requested), seen_token: Mutex::new(None), seen_pid: Mutex::new(None) }); + let frames = Arc::new(TestFrames { open_count: Mutex::new(0), closed: Mutex::new(Vec::new()) }); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(runtime), + tcp_port: None, + sink: Arc::clone(&sink) as Arc, + activate: Arc::clone(&activate) as Arc, + frames: Arc::clone(&frames) as Arc, + }) + .unwrap(); + let socket = match &server.record().endpoint { + Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + }; + let token = server.record().token.clone(); + Harness { tmp: Tmp(path), server, socket, token, sink, activate, frames } +} + +struct Client { + reader: BufReader, +} + +impl Client { + fn connect(socket: &Path, token: &str) -> Self { + let stream = UnixStream::connect(socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut reader = BufReader::new(stream); + let req = json!({"id": 0, "method": "auth", "params": {"token": token}}); + wire::write_line(reader.get_mut(), &req).unwrap(); + let reply = read_value(&mut reader).unwrap_or_else(|| panic!("no auth reply")); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"]["authenticated"], json!(true), "{reply}"); + Self { reader } + } + + fn call(&mut self, id: i64, method: &str, params: Value) -> Value { + let req = json!({"id": id, "method": method, "params": params}); + wire::write_line(self.reader.get_mut(), &req).unwrap(); + read_value(&mut self.reader).unwrap_or_else(|| panic!("no reply for {method}")) + } + + fn set_timeout(&mut self, timeout: Duration) { + self.reader.get_mut().set_read_timeout(Some(timeout)).unwrap(); + } +} + +fn read_value(reader: &mut BufReader) -> Option { + match wire::read_line(reader, MAX_LINE) { + Ok(Some(bytes)) => Some(serde_json::from_slice(&bytes).unwrap()), + Ok(None) => None, + Err(_) => None, + } +} + +fn read_raw_line(stream: &mut impl Read) -> Vec { + let mut out = Vec::new(); + let mut byte = [0u8; 1]; + loop { + stream.read_exact(&mut byte).unwrap(); + if byte[0] == b'\n' { + break; + } + out.push(byte[0]); + } + if out.last() == Some(&b'\r') { + out.pop(); + } + out +} + +#[test] +fn legacy_command_keeps_the_request_id() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let reply = client.call(41, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(41), "{reply}"); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"], json!({"commands": ["cut"]}), "{reply}"); + assert_eq!(harness.sink.seen.lock().unwrap().as_slice(), ["engine.commands"]); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// A sink reply longer than `MAX_LINE` cannot be read by a client. The server answers that request +/// with `payload_too_large` and the same id, and the connection keeps working. +#[test] +fn a_reply_over_the_line_limit_is_answered_with_an_error_and_the_connection_survives() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + *harness.sink.result.lock().unwrap() = json!({"blob": "x".repeat(MAX_LINE + 16)}); + let reply = client.call(5, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(5), "{reply}"); + assert_eq!(reply["ok"], json!(false), "{reply}"); + assert_eq!(reply["error"]["code"], json!("payload_too_large"), "{reply}"); + *harness.sink.result.lock().unwrap() = json!({"commands": ["cut"]}); + let next = client.call(6, "engine.commands", json!({})); + assert_eq!(next["ok"], json!(true), "the connection must still work: {next}"); + assert_eq!(next["id"], json!(6), "{next}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// The server polls each connection with a 250 ms read timeout. On Linux that timeout surfaces as +/// `WouldBlock` ("Resource temporarily unavailable"), which must mean "idle, keep waiting". A +/// connection that stays quiet for longer than one poll must still answer the next request. +#[test] +fn a_connection_idle_for_longer_than_the_read_poll_still_answers() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let first = client.call(1, "engine.commands", json!({})); + assert_eq!(first["ok"], json!(true), "{first}"); + std::thread::sleep(Duration::from_millis(900)); + let second = client.call(2, "engine.commands", json!({})); + assert_eq!(second["id"], json!(2), "the idle connection must still be open: {second}"); + assert_eq!(second["ok"], json!(true), "{second}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// A sink value that already has `ok` is the finished reply. The server inserts `id` and does not nest it. +#[test] +fn sink_reply_with_ok_is_not_wrapped() { + let harness = boot(); + *harness.sink.result.lock().unwrap() = json!({"ok": true, "result": null}); + let mut client = Client::connect(&harness.socket, &harness.token); + let reply = client.call(3, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(3), "{reply}"); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"], Value::Null, "{reply}"); + assert!(reply.get("error").is_none(), "{reply}"); + + *harness.sink.result.lock().unwrap() = json!({"ok": false, "error": "missing command"}); + let bad = client.call(4, "engine.commands", json!({})); + assert_eq!(bad["id"], json!(4), "{bad}"); + assert_eq!(bad["ok"], json!(false), "{bad}"); + assert_eq!(bad["error"], json!("missing command"), "{bad}"); + assert!(bad.get("result").is_none(), "an error reply must stay flat: {bad}"); + harness.server.shutdown().unwrap(); +} + +#[test] +fn events_wait_for_subscribe_then_one_event_is_delivered() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + client.set_timeout(Duration::from_millis(200)); + harness.server.emit("doc.changed", json!({"doc_id": "a"})).unwrap(); + assert!(read_value(&mut client.reader).is_none(), "event before subscribe"); + client.set_timeout(Duration::from_secs(2)); + let sub = client.call(2, "events.subscribe", json!({"events": ["doc.changed"]})); + assert_eq!(sub["id"], json!(2), "{sub}"); + assert_eq!(sub["result"], json!({"seq": 0}), "{sub}"); + harness.server.emit("doc.changed", json!({"doc_id": "a"})).unwrap(); + let event = read_value(&mut client.reader).unwrap_or_else(|| panic!("missing event")); + assert_eq!(event["event"], json!("doc.changed"), "{event}"); + assert_eq!(event["seq"], json!(1), "{event}"); + assert_eq!(event["data"], json!({"doc_id": "a"}), "{event}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +#[test] +fn oversized_event_delivers_an_overflow_marker() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let sub = client.call(2, "events.subscribe", json!({"events": ["doc.changed"]})); + assert_eq!(sub["result"]["seq"], json!(0), "{sub}"); + harness.server.emit("doc.changed", json!({"blob": "x".repeat(MAX_LINE)})).unwrap(); + let event = read_value(&mut client.reader).expect("missing overflow event"); + assert_eq!(event["event"], json!("events.overflow")); + assert!(event["data"]["since"].is_number()); + harness.server.shutdown().unwrap(); +} + +#[test] +fn bulk_round_trip_returns_the_frame_bytes() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let hello = client.call( + 1, + "link.hello", + json!({"client": {"app": "fc", "app_version": "0.9.1", "pid": 7, "instance": "00000000-0000-4000-8000-000000000001"}, "protocol": {"min": 1, "max": 1}}), + ); + assert_eq!(hello["ok"], json!(true), "{hello}"); + assert_eq!(hello["result"]["app"], json!("ec"), "{hello}"); + assert_eq!(hello["result"]["app_version"], json!("0.8.3"), "{hello}"); + assert_eq!(hello["result"]["protocol"], json!(1), "{hello}"); + assert_eq!(hello["result"]["caps"], json!(["events", "activate", "link.receive", "frames"]), "{hello}"); + let opened = client.call(8, "frames.open", json!({"link_id": "L", "format": "rgba8_premul", "scale": 1.0})); + assert_eq!(opened["ok"], json!(true), "{opened}"); + assert_eq!(opened["result"]["ticket"], json!(TICKET), "{opened}"); + assert_eq!(opened["result"]["w"], json!(4), "{opened}"); + assert_eq!(opened["result"]["len"], json!(PAYLOAD.len()), "{opened}"); + + let mut bulk = UnixStream::connect(&harness.socket).unwrap(); + bulk.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + bulk.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let auth = json!({"id": 0, "method": "auth", "params": {"token": harness.token}}); + wire::write_line(&mut bulk, &auth).unwrap(); + let auth_reply: Value = serde_json::from_slice(&read_raw_line(&mut bulk)).unwrap(); + assert_eq!(auth_reply["ok"], json!(true), "{auth_reply}"); + let attach = json!({"id": 9, "method": "bulk.attach", "params": {"ticket": TICKET}}); + wire::write_line(&mut bulk, &attach).unwrap(); + let attach_reply: Value = serde_json::from_slice(&read_raw_line(&mut bulk)).unwrap(); + assert_eq!(attach_reply["ok"], json!(true), "{attach_reply}"); + assert_eq!(attach_reply["id"], json!(9), "{attach_reply}"); + + let header = json!({"req": 7, "time_s": 1.5, "epoch": 3, "region": null}); + wire::write_cfrq(&mut bulk, &header).unwrap(); + let (got, payload) = wire::read_cfrm(&mut bulk).unwrap(); + assert_eq!(got["req"], json!(7), "{got}"); + assert_eq!(payload, PAYLOAD); + + drop(bulk); + drop(client); + harness.server.shutdown().unwrap(); + assert_eq!(harness.frames.closed.lock().unwrap().as_slice(), [TICKET]); + drop(harness.tmp); +} + +#[test] +fn activate_returns_requested_from_the_trait() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let params = json!({"token": "wayland-token", "from_pid": 4242, "open": {"link_id": "abc"}}); + let reply = client.call(3, "app.activate", params); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"]["activated"], json!("requested"), "{reply}"); + assert_eq!(harness.activate.seen_token.lock().unwrap().clone().unwrap().as_deref(), Some("wayland-token")); + assert_eq!(harness.activate.seen_pid.lock().unwrap().unwrap(), Some(4242)); + + *harness.activate.mode.lock().unwrap() = Activation::No; + let reply = client.call(4, "app.activate", json!({"token": null})); + assert_eq!(reply["result"]["activated"], json!(false), "{reply}"); + + *harness.activate.mode.lock().unwrap() = Activation::Yes; + let reply = client.call(5, "app.activate", json!({})); + assert_eq!(reply["result"]["activated"], json!(true), "{reply}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +#[test] +fn the_17th_connection_is_refused_and_does_not_stay_open() { + let harness = boot(); + let mut held = Vec::new(); + // Authenticated connections fill the cap. Unauthenticated ones are limited to a smaller pool. + for _ in 0..wire::MAX_CONNECTIONS { + held.push(Client::connect(&harness.socket, &harness.token)); + } + let mut extra = UnixStream::connect(&harness.socket).unwrap(); + extra.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut line = String::new(); + BufReader::new(&mut extra).read_line(&mut line).unwrap(); + let value: Value = serde_json::from_str(line.trim()).unwrap(); + assert_eq!(value["ok"], json!(false), "{line}"); + assert_eq!(value["error"], json!("too many control connections"), "{line}"); + drop(held); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} diff --git a/crates/ipc/tests/provider_stall.rs b/crates/ipc/tests/provider_stall.rs new file mode 100644 index 000000000..3407df05d --- /dev/null +++ b/crates/ipc/tests/provider_stall.rs @@ -0,0 +1,248 @@ +//! A provider that never returns must not hold up shutdown. `Assets` and `Activate` calls run on +//! helper threads with a deadline: the caller gets a timeout reply, the helper is detached, and a +//! normal call still works afterwards. +#![cfg(unix)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use craft_ipc::asset::{Described, Refreshed, Resolved}; +use craft_ipc::moniker::Moniker; +use craft_ipc::record::AssetKind; +use craft_ipc::refresh::FileState; +use craft_ipc::server::{Activate, Activation, Assets, ControlSink, Extensions, FrameOpen, Frames, Limits, Server, ServerConfig}; +use craft_ipc::{Endpoint, LinkError, RuntimeDir}; + +const DEADLINE: Duration = Duration::from_millis(400); +/// `link.refresh` with this link id blocks until the test releases it. `app.activate` with this +/// token blocks the same way. +const STUCK: &str = "stuck"; + +/// Blocks while `release` is false, so a test can end its helper threads when it is done. +struct Gate { + release: AtomicBool, + entered: AtomicUsize, +} + +impl Gate { + fn wait(&self) { + self.entered.fetch_add(1, Ordering::SeqCst); + while !self.release.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +struct StuckAssets(Arc); + +impl Assets for StuckAssets { + fn resolve(&self, _asset: &Moniker) -> Result { + Err(LinkError::Unknown("no such asset".into())) + } + + fn describe(&self, asset: &Moniker) -> Result { + Ok(Described { + asset: asset.clone(), + name: "x".into(), + kind: AssetKind::Still, + epoch: 1, + w: None, + h: None, + hash: None, + thumb_path: None, + modified_ms: None, + }) + } + + fn refresh(&self, link_id: &str) -> Result { + if link_id == STUCK { + self.0.wait(); + } + Ok(Refreshed::new(link_id, 1, FileState::Current, "/tmp/x", None)) + } +} + +struct StuckActivate(Arc); + +impl Activate for StuckActivate { + fn activate(&self, token: Option, _from_pid: Option, _open: Option) -> Activation { + if token.as_deref() == Some(STUCK) { + self.0.wait(); + } + Activation::Yes + } +} + +struct Sink; + +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +struct Rig { + server: Server, + gate: Arc, + reader: BufReader, + dir: PathBuf, +} + +impl Rig { + fn boot(label: &str, deadline: Duration) -> Self { + let dir = std::env::temp_dir().join(format!("cl-stall-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let gate = Arc::new(Gate { release: AtomicBool::new(false), entered: AtomicUsize::new(0) }); + let server = Server::start_with_limits( + ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(RuntimeDir::at(&dir).unwrap()), + tcp_port: None, + sink: Arc::new(Sink), + activate: Arc::new(StuckActivate(Arc::clone(&gate))), + frames: Arc::new(NoFrames), + }, + Extensions { assets: Some(Arc::new(StuckAssets(Arc::clone(&gate)))), caps: Vec::new() }, + Limits { provider_deadline: deadline, ..Limits::default() }, + ) + .unwrap(); + let Endpoint::Unix { path } = &server.record().endpoint else { panic!("unix endpoint expected") }; + let stream = UnixStream::connect(path).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(stream); + let token = server.record().token.clone(); + send(&mut reader, &json!({"id": 0, "method": "auth", "params": {"token": token}})); + assert_eq!(read(&mut reader)["ok"], json!(true)); + Self { server, gate, reader, dir } + } + + fn call(&mut self, id: u64, method: &str, params: Value) -> Value { + send(&mut self.reader, &json!({"id": id, "method": method, "params": params})); + read(&mut self.reader) + } + + /// Shut down on a thread and return how long it took. A hang fails the test after `limit`. + fn shutdown_within(self, limit: Duration) -> Duration { + let Self { server, gate, reader, dir } = self; + let (done, finished) = mpsc::channel(); + let started = Instant::now(); + std::thread::spawn(move || { + let result = server.shutdown(); + let _ = done.send(result.is_ok()); + }); + let waited = finished.recv_timeout(limit); + // Let the detached helpers end, so the test process leaves no spinning threads. + gate.release.store(true, Ordering::SeqCst); + drop(reader); + let _ = std::fs::remove_dir_all(dir); + assert!(waited.expect("shutdown did not finish: a stuck provider held it up"), "shutdown failed"); + started.elapsed() + } +} + +fn send(reader: &mut BufReader, value: &Value) { + let mut bytes = serde_json::to_vec(value).unwrap(); + bytes.push(b'\n'); + reader.get_mut().write_all(&bytes).unwrap(); +} + +fn read(reader: &mut BufReader) -> Value { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + serde_json::from_str(&line).unwrap_or_else(|err| panic!("reply {line:?}: {err}")) +} + +fn timed_out(reply: &Value) -> bool { + reply["ok"] == json!(false) && reply["error"]["message"] == json!("timeout") +} + +#[test] +fn a_stuck_asset_call_times_out_and_the_connection_keeps_working() { + let mut rig = Rig::boot("assets", DEADLINE); + let started = Instant::now(); + let reply = rig.call(1, "link.refresh", json!({"link_id": STUCK})); + let waited = started.elapsed(); + assert!(timed_out(&reply), "{reply}"); + assert_eq!(reply["id"], json!(1)); + assert!(waited >= DEADLINE && waited < DEADLINE * 5, "{waited:?}"); + // Other calls on the same connection still work, asset calls included. + let fine = rig.call(2, "link.refresh", json!({"link_id": "ok"})); + assert_eq!(fine["ok"], json!(true), "{fine}"); + assert_eq!(fine["result"]["link_id"], json!("ok")); + let hello = rig.call(3, "link.hello", json!({"protocol": {"min": 1, "max": 1}})); + assert_eq!(hello["ok"], json!(true), "{hello}"); + assert_eq!(rig.gate.entered.load(Ordering::SeqCst), 1); + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn shutdown_does_not_wait_for_a_stuck_asset_call() { + // The deadline is long, so shutdown meets the call still in flight. + let mut rig = Rig::boot("assets-stop", Duration::from_secs(60)); + send(&mut rig.reader, &json!({"id": 1, "method": "link.refresh", "params": {"link_id": STUCK}})); + while rig.gate.entered.load(Ordering::SeqCst) == 0 { + std::thread::sleep(Duration::from_millis(10)); + } + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn a_stuck_activate_call_times_out_and_shutdown_does_not_wait_for_it() { + let mut rig = Rig::boot("activate", DEADLINE); + let reply = rig.call(1, "app.activate", json!({"token": STUCK})); + assert!(timed_out(&reply), "{reply}"); + let fine = rig.call(2, "app.activate", json!({"token": "ok"})); + assert_eq!(fine["result"]["activated"], json!(true), "{fine}"); + // A second stuck call, then shut down while it is in flight. + send(&mut rig.reader, &json!({"id": 3, "method": "app.activate", "params": {"token": STUCK}})); + while rig.gate.entered.load(Ordering::SeqCst) < 2 { + std::thread::sleep(Duration::from_millis(10)); + } + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn stuck_helpers_are_capped_and_the_server_says_busy() { + let mut rig = Rig::boot("cap", Duration::from_millis(60)); + let mut busy_at = None; + for id in 1..=40u64 { + let reply = rig.call(id, "link.refresh", json!({"link_id": STUCK})); + if reply["error"]["code"] == json!("busy") { + busy_at = Some(id); + break; + } + assert!(timed_out(&reply), "call {id}: {reply}"); + } + let busy_at = busy_at.expect("the server kept starting helper threads past its cap"); + assert_eq!(busy_at, 33, "32 stuck helpers are allowed, the next call is refused"); + assert_eq!(rig.gate.entered.load(Ordering::SeqCst), 32, "a refused call starts no thread"); + // A call that needs no helper still works while the cap is full. + let hello = rig.call(100, "link.hello", json!({"protocol": {"min": 1, "max": 1}})); + assert_eq!(hello["ok"], json!(true), "{hello}"); + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} diff --git a/crates/ipc/tests/schemas.rs b/crates/ipc/tests/schemas.rs new file mode 100644 index 000000000..4dbcd251b --- /dev/null +++ b/crates/ipc/tests/schemas.rs @@ -0,0 +1,318 @@ +//! The JSON schemas in `schemas/` and the Rust types +//! must accept the same documents. Each table row is checked against both. +//! The fixtures are stored records. The first one is the FilmCraft to +//! EffectCraft shape written before the asset fields existed. + +use std::path::PathBuf; + +use jsonschema::Validator; +use serde_json::{Value, json}; + +use craft_ipc::InstanceRecord; +use craft_ipc::record::{AssetKind, LinkRecord}; + +fn schema_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schemas") +} + +fn validator(name: &str) -> Validator { + let path = schema_dir().join(name); + let schema: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + jsonschema::options().should_validate_formats(true).build(&schema).unwrap() +} + +fn fixture(name: &str) -> Value { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures").join(name); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +} + +/// Schema verdict and Rust verdict for one document. Both must equal `want`. +fn check_link(schema: &Validator, doc: &Value, want: bool, why: &str) { + let by_schema = schema.is_valid(doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, want, "schema: {why}: {doc}"); + assert_eq!(by_rust, want, "rust: {why}: {doc}"); +} + +#[test] +fn stored_filmcraft_effectcraft_record_stays_valid() { + let schema = validator("link-record.schema.json"); + let doc = fixture("link-record-filmcraft-effectcraft.json"); + check_link(&schema, &doc, true, "legacy fixture"); + let record: LinkRecord = serde_json::from_value(doc.clone()).unwrap(); + assert_eq!(record.target_app, "effectcraft"); + assert_eq!(record.target_comp.as_deref(), Some("comp-1")); + assert_eq!(record.nested_sequence.as_deref(), Some("item-9")); + assert_eq!(record.format.rate_num, Some(24)); + assert_eq!(record.format.duration_ticks, Some(240)); + assert!(record.asset.is_none() && record.producer.is_none()); + // Unknown fields survive a load and save, byte-for-byte as JSON values. + assert_eq!(serde_json::to_value(&record).unwrap(), doc); +} + +#[test] +fn still_record_has_no_video_fields_and_round_trips() { + let schema = validator("link-record.schema.json"); + let doc = fixture("link-record-still.json"); + check_link(&schema, &doc, true, "still fixture"); + let record: LinkRecord = serde_json::from_value(doc.clone()).unwrap(); + assert_eq!(record.target_app, "photocraft"); + assert!(record.nested_sequence.is_none() && record.target_comp.is_none()); + assert_eq!(record.format.rate_num, None); + assert_eq!(record.format.rate_den, None); + assert_eq!(record.format.duration_ticks, None); + assert_eq!(record.asset.as_ref().map(|a| a.kind), Some(AssetKind::Still)); + let out = serde_json::to_value(&record).unwrap(); + assert_eq!(out, doc); + let format = out["format"].as_object().unwrap(); + for key in ["rate_num", "rate_den", "duration_ticks"] { + assert!(!format.contains_key(key), "{key} must not be written as null"); + } +} + +#[test] +fn schema_and_rust_agree_on_edge_cases() { + let schema = validator("link-record.schema.json"); + let legacy = fixture("link-record-filmcraft-effectcraft.json"); + let still = fixture("link-record-still.json"); + + let edit = |base: &Value, f: &dyn Fn(&mut Value)| { + let mut doc = base.clone(); + f(&mut doc); + doc + }; + let cases: Vec<(Value, bool, &str)> = vec![ + (edit(&legacy, &|d| d["target_app"] = json!("photocraft")), true, "any app id"), + (edit(&legacy, &|d| d["target_app"] = json!("Film")), false, "uppercase app id"), + (edit(&legacy, &|d| d["target_app"] = json!("")), false, "empty app id"), + (edit(&legacy, &|d| d["v"] = json!(2)), false, "v 2"), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("nested_sequence"); + }), + true, + "nested_sequence optional", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("duration_ticks"); + }), + true, + "duration optional", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("rate_num"); + }), + false, + "rate_den without rate_num", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("rate_den"); + }), + false, + "rate_num without rate_den", + ), + (edit(&legacy, &|d| d["format"]["rate_num"] = json!(0)), false, "rate_num 0"), + (edit(&legacy, &|d| d["format"]["w"] = json!(0)), false, "w 0"), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("alpha"); + }), + false, + "alpha stays required", + ), + (edit(&legacy, &|d| d["result_epoch"] = json!(0)), true, "epoch 0"), + (edit(&legacy, &|d| d["created"] = json!("yesterday")), false, "bad date"), + (edit(&legacy, &|d| d["created"] = json!("2026-02-31T09:30:00Z")), false, "31 February"), + (edit(&legacy, &|d| d["created"] = json!("2026-02-29T09:30:00Z")), false, "29 February in a common year"), + (edit(&legacy, &|d| d["created"] = json!("2026-04-31T09:30:00+02:00")), false, "31 April"), + (edit(&legacy, &|d| d["created"] = json!("1900-02-29T00:00:00Z")), false, "29 February in a century year"), + (edit(&legacy, &|d| d["created"] = json!("2024-02-29T09:30:00Z")), true, "29 February in a leap year"), + (edit(&legacy, &|d| d["created"] = json!("2000-02-29T09:30:00.250Z")), true, "29 February in a 400 year"), + (edit(&legacy, &|d| d["created"] = json!("2026-12-31T23:59:59-05:30")), true, "last day of the year"), + (edit(&legacy, &|d| d["link_id"] = json!("nope")), false, "bad link id"), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("target_comp"); + }), + false, + "no comp and no asset", + ), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("target_project"); + }), + false, + "target_project required", + ), + ( + edit(&still, &|d| { + d.as_object_mut().unwrap().remove("asset"); + }), + false, + "still without asset or comp", + ), + (edit(&still, &|d| d["asset"]["kind"] = json!("video")), false, "unknown asset kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("vector")), true, "vector kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("sequence")), true, "sequence kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("comp")), true, "comp kind"), + (edit(&still, &|d| d["asset"]["moniker"] = json!("craft://photocraft/not-a-uuid/image/x")), false, "bad moniker doc"), + (edit(&still, &|d| d["asset"]["moniker"] = json!("photocraft/x")), false, "moniker without scheme"), + ( + edit(&still, &|d| d["asset"]["moniker"] = json!("craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/a/b")), + false, + "moniker with extra segment", + ), + (edit(&still, &|d| d["producer"]["app"] = json!("Bad App")), false, "bad producer app"), + ( + edit(&still, &|d| { + d.as_object_mut().unwrap().remove("producer"); + }), + true, + "producer optional", + ), + (edit(&still, &|d| d["asset"]["extra"] = json!(1)), true, "unknown asset field kept"), + ]; + for (doc, want, why) in &cases { + check_link(&schema, doc, *want, why); + } +} + +const MONIKER_BASE: &str = "craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/"; + +fn with_moniker(still: &Value, id: &str) -> Value { + let mut doc = still.clone(); + doc["asset"]["moniker"] = json!(format!("{MONIKER_BASE}{id}")); + doc +} + +/// The moniker id rules of `Moniker::parse` as the schema states them: unreserved characters and +/// two-digit escapes only, 1 to 512 decoded bytes, and decoded bytes that are valid UTF-8. +#[test] +fn schema_and_rust_agree_on_moniker_ids() { + let schema = validator("link-record.schema.json"); + let still = fixture("link-record-still.json"); + let escaped = |count: usize| "%41".repeat(count); + let cases: Vec<(String, bool, &str)> = vec![ + ("layer-3".into(), true, "plain"), + ("~._-AZaz09".into(), true, "every unreserved class"), + ("a%20b".into(), true, "escaped space"), + ("a%2fb".into(), true, "lowercase escape"), + ("%C3%A9".into(), true, "two byte character"), + ("%c3%a9".into(), true, "two byte character in lowercase hex"), + ("%E2%82%AC".into(), true, "three byte character"), + ("%F0%9F%98%80".into(), true, "four byte character"), + ("%F4%8F%BF%BF".into(), true, "highest scalar"), + ("%ED%9F%BF".into(), true, "just below the surrogates"), + ("%00".into(), true, "escaped NUL is a byte like any other"), + ("%GG".into(), false, "bad escape"), + ("%4".into(), false, "truncated escape"), + ("%".into(), false, "lone percent"), + ("a b".into(), false, "raw space"), + ("caf\u{e9}".into(), false, "raw non-ASCII"), + ("a:b".into(), false, "raw colon"), + ("a%2".into(), false, "truncated escape after text"), + ("".into(), false, "empty id"), + ("%FF".into(), false, "invalid UTF-8 byte"), + ("%C3".into(), false, "truncated two byte character"), + ("%80".into(), false, "lone continuation byte"), + ("%C0%80".into(), false, "overlong two byte form"), + ("%E0%80%80".into(), false, "overlong three byte form"), + ("%ED%A0%80".into(), false, "surrogate"), + ("%F0%80%80%80".into(), false, "overlong four byte form"), + ("%F4%90%80%80".into(), false, "above U+10FFFF"), + ("%F5%80%80%80".into(), false, "lead byte F5"), + ("a/b".into(), false, "raw slash"), + ("a?b".into(), false, "raw query"), + ("a#b".into(), false, "raw fragment"), + ("a".repeat(512), true, "512 raw bytes"), + ("a".repeat(513), false, "513 raw bytes"), + (escaped(512), true, "512 escaped bytes"), + (escaped(513), false, "513 escaped bytes"), + (format!("{}%C3%A9", "a".repeat(510)), true, "512 decoded bytes ending in a two byte character"), + (format!("{}%C3%A9", "a".repeat(511)), false, "513 decoded bytes ending in a two byte character"), + ("%C3%A9".repeat(256), true, "256 two byte characters make 512 bytes"), + ("%C3%A9".repeat(257), false, "257 two byte characters make 514 bytes"), + ]; + for (id, want, why) in &cases { + check_link(&schema, &with_moniker(&still, id), *want, why); + } + + // Every lead byte against a spread of second and third bytes: the two verdicts must match. + let probes = [0x00u8, 0x41, 0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0]; + let mut seen_valid = 0; + for lead in 0u8..=0xFF { + for second in probes { + for tail in [&[][..], &[0x80][..], &[0x80, 0x80][..]] { + for hex in [|b: u8| format!("%{b:02X}"), |b: u8| format!("%{b:02x}")] { + let id: String = [lead, second].iter().chain(tail).map(|byte| hex(*byte)).collect(); + let doc = with_moniker(&still, &id); + let by_schema = schema.is_valid(&doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, by_rust, "bytes {lead:02X} {second:02X} {tail:02X?} as {id}"); + seen_valid += usize::from(by_schema); + } + } + } + } + assert!(seen_valid > 100, "the probe must reach valid sequences too, got {seen_valid}"); +} + +fn instance_doc() -> Value { + let endpoint = if cfg!(windows) { + json!({"kind": "pipe", "name": "craft-hub-4242"}) + } else { + json!({"kind": "unix", "path": "/run/user/1000/craft/hub-4242.sock"}) + }; + json!({ + "v": 1, + "app": "hub", + "app_version": "0.1.0", + "pid": 4242, + "instance": "550e8400-e29b-41d4-a716-446655440000", + "endpoint": endpoint, + "token": "ab".repeat(32), + "started": 1_760_000_000u64, + "protocol": {"link": 1}, + "docs": [] + }) +} + +fn check_instance(schema: &Validator, doc: &Value, want: bool, why: &str) { + let by_schema = schema.is_valid(doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, want, "schema: {why}: {doc}"); + assert_eq!(by_rust, want, "rust: {why}: {doc}"); +} + +#[test] +fn instance_record_caps_agree_between_schema_and_rust() { + let schema = validator("instance-record.schema.json"); + let base = instance_doc(); + check_instance(&schema, &base, true, "no caps (existing records)"); + for (caps, want, why) in [ + (json!(["hub"]), true, "hub"), + (json!(["hub", "assets"]), true, "two caps"), + (json!([]), true, "empty caps"), + (json!(["Hub"]), false, "uppercase"), + (json!([""]), false, "empty cap"), + (json!(["hub", "hub"]), false, "duplicate"), + (json!("hub"), false, "not an array"), + (json!([1]), false, "not strings"), + ] { + let mut doc = base.clone(); + doc["caps"] = caps; + check_instance(&schema, &doc, want, why); + } + let mut hub: InstanceRecord = serde_json::from_value(base).unwrap(); + assert!(hub.caps().is_empty()); + hub.add_cap("hub").unwrap(); + hub.add_cap("hub").unwrap(); + assert_eq!(hub.caps(), vec!["hub".to_string()]); + assert!(hub.add_cap("Not Valid").is_err()); + let value = serde_json::to_value(&hub).unwrap(); + assert_eq!(value["caps"], json!(["hub"])); + assert!(schema.is_valid(&value)); +} diff --git a/crates/ipc/tests/security.rs b/crates/ipc/tests/security.rs new file mode 100644 index 000000000..0c42c599a --- /dev/null +++ b/crates/ipc/tests/security.rs @@ -0,0 +1,290 @@ +//! Auth refusal on the local socket. These tests speak bytes themselves. +//! They do not use the client module. +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct Sink; + +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({})); + Some(rx) + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + + fn close_ticket(&self, _ticket: &str) {} +} + +struct Running { + _tmp: Tmp, + server: Server, + socket: PathBuf, + token: String, +} + +fn boot() -> Running { + boot_with(None) +} + +fn boot_with(tcp_port: Option) -> Running { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-sec-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(runtime), + tcp_port, + sink: Arc::new(Sink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let socket = match &server.record().endpoint { + craft_ipc::Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + }; + let token = server.record().token.clone(); + Running { _tmp: Tmp(path), server, socket, token } +} + +fn read_closed_line(socket: &Path, payload: &[u8]) -> Value { + let mut stream = std::os::unix::net::UnixStream::connect(socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.write_all(payload).unwrap(); + let mut text = String::new(); + stream.read_to_string(&mut text).unwrap_or_else(|err| panic!("socket did not close after auth failure: {err}")); + let lines: Vec<_> = text.lines().filter(|line| !line.is_empty()).collect(); + assert_eq!(lines.len(), 1, "{text}"); + assert!(!lines[0].contains("bad JSON"), "{text}"); + serde_json::from_str(lines[0]).unwrap() +} + +fn assert_auth_failure(value: &Value) { + assert_eq!(value["id"], json!(0), "{value}"); + assert_eq!(value["ok"], json!(false), "{value}"); + assert_eq!(value["error"], json!("authentication required"), "{value}"); +} + +#[test] +fn no_token_is_auth_failure_not_bad_json() { + let running = boot(); + let value = read_closed_line(&running.socket, b"{\"id\":1,\"method\":\"engine.commands\",\"params\":{}}\n"); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn wrong_token_is_auth_failure() { + let running = boot(); + let mut bad = "ab".repeat(32); + if bad == running.token { + bad = "cd".repeat(32); + } + let line = json!({"id": 4, "method": "auth", "params": {"token": bad}}); + let mut bytes = serde_json::to_vec(&line).unwrap(); + bytes.push(b'\n'); + let value = read_closed_line(&running.socket, &bytes); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn malformed_first_line_is_auth_failure() { + let running = boot(); + let value = read_closed_line(&running.socket, b"{\n"); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_is_off_when_the_port_is_none() { + let running = boot(); + assert_eq!(running.server.tcp_port(), None); + running.server.shutdown().unwrap(); +} + +fn tcp_exchange(port: u16, payload: &[u8]) -> String { + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.write_all(payload).unwrap(); + let mut text = String::new(); + let _ = stream.read_to_string(&mut text); + text +} + +#[test] +fn tcp_without_a_token_is_refused_and_runs_nothing() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + assert_ne!(port, 0); + let text = tcp_exchange(port, b"{\"id\":7,\"method\":\"engine.commands\",\"params\":{}}\n"); + let first = text.lines().find(|line| !line.is_empty()).unwrap_or_else(|| panic!("tcp closed with no refusal line")); + assert_auth_failure(&serde_json::from_str(first).unwrap()); + assert!(!text.contains("\"result\""), "{text}"); + let bad_json = tcp_exchange(port, b"{\n"); + assert!(!bad_json.contains("bad JSON"), "{bad_json}"); + assert!(bad_json.contains("authentication required"), "{bad_json}"); + let mut wrong = "ab".repeat(32); + if wrong == running.token { + wrong = "cd".repeat(32); + } + let line = json!({"id": 0, "method": "auth", "params": {"token": wrong}}); + let mut bytes = serde_json::to_vec(&line).unwrap(); + bytes.push(b'\n'); + assert_auth_failure(&serde_json::from_str(tcp_exchange(port, &bytes).lines().next().unwrap()).unwrap()); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_with_the_record_token_works_and_the_record_publishes_the_port() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + match &running.server.record().endpoint { + craft_ipc::Endpoint::Unix { .. } => {} + other => panic!("the instance record must keep the unix socket, got {other:?}"), + } + assert_eq!(running.server.record().tcp_port(), Some(port)); + let mut record = running.server.record().clone(); + record.endpoint = craft_ipc::Endpoint::Tcp { port }; + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + let auth = json!({"id": 0, "method": "auth", "params": {"token": running.token}}); + let mut bytes = serde_json::to_vec(&auth).unwrap(); + bytes.extend_from_slice(b"\n{\"id\":7,\"method\":\"engine.commands\",\"params\":{}}\n"); + stream.write_all(&bytes).unwrap(); + let mut buf = Vec::new(); + while buf.iter().filter(|b| **b == b'\n').count() < 2 { + let mut byte = [0u8; 1]; + assert_ne!(stream.read(&mut byte).unwrap(), 0, "tcp closed before a reply"); + buf.push(byte[0]); + } + let text = String::from_utf8(buf).unwrap(); + let mut lines = text.lines(); + let ok: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + assert_eq!(ok["ok"], json!(true), "{ok}"); + let value: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + assert_eq!(value["id"], json!(7), "{value}"); + assert_eq!(value["ok"], json!(true), "{value}"); + assert_eq!(value["result"], json!({}), "{value}"); + // The library client does the same from the published record. + let mut client = craft_ipc::client::Client::connect_tcp(running.server.record()).unwrap(); + assert_eq!(client.call("engine.commands", json!({})).unwrap(), json!({})); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_is_not_published_when_it_is_off() { + let running = boot(); + assert_eq!(running.server.record().tcp_port(), None); + assert!(craft_ipc::client::Client::connect_tcp(running.server.record()).is_err()); + running.server.shutdown().unwrap(); +} + +#[test] +fn silent_tcp_connections_hit_the_unauthenticated_cap() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + let silent: Vec<_> = (0..craft_ipc::wire::MAX_UNAUTHENTICATED).map(|_| std::net::TcpStream::connect(("127.0.0.1", port)).unwrap()).collect(); + std::thread::sleep(Duration::from_millis(400)); + let text = tcp_exchange(port, b""); + assert!(text.contains("too many unauthenticated connections"), "{text}"); + drop(silent); + running.server.shutdown().unwrap(); +} + +#[test] +fn other_uid_is_refused_when_the_env_is_set() { + let Ok(user) = std::env::var("CRAFT_TEST_OTHER_USER") else { + return; + }; + let running = boot(); + // 0600 blocks a non-owner before the peer check. Loosen it so this user + // can connect. The assertion is still the auth failure line. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&running.socket, std::fs::Permissions::from_mode(0o666)).unwrap(); + } + let script = r#" +import socket, sys +s = socket.socket(socket.AF_UNIX) +s.settimeout(5) +s.connect(sys.argv[1]) +token = sys.argv[2] +msg = '{"id":1,"method":"auth","params":{"token":"%s"}}\n' % token +s.sendall(msg.encode()) +data = b"" +while b"\n" not in data: + chunk = s.recv(4096) + if not chunk: + break + data += chunk +sys.stdout.buffer.write(data) +"#; + let output = run_as(&user, script, &running.socket, &running.token); + if !output.status.success() { + panic!("run as {user} failed: status {:?} stderr {}", output.status.code(), String::from_utf8_lossy(&output.stderr)); + } + let text = String::from_utf8_lossy(&output.stdout); + assert!(!text.contains("bad JSON"), "{text}"); + let value: Value = serde_json::from_str(text.lines().next().unwrap()).unwrap(); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +fn run_as(user: &str, script: &str, socket: &Path, token: &str) -> std::process::Output { + let args = ["-u", user, "--", "python3", "-c", script]; + match std::process::Command::new("runuser").args(args).arg(socket).arg(token).output() { + Ok(output) => output, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + std::process::Command::new("sudo").args(args).arg(socket).arg(token).output().unwrap_or_else(|err| { + panic!("sudo failed to start: {err}"); + }) + } + Err(err) => panic!("runuser failed to start: {err}"), + } +} diff --git a/crates/ipc/tests/watch.rs b/crates/ipc/tests/watch.rs new file mode 100644 index 000000000..185990e92 --- /dev/null +++ b/crates/ipc/tests/watch.rs @@ -0,0 +1,361 @@ +//! Scheduled polling. A `ManualTicker` stands in for the clock, so no test +//! sleeps to wait for an interval. Real-time waits only bound a failure. + +use std::fs::OpenOptions; +use std::path::PathBuf; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::json; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::refresh::{Change, FileState, Tracker, write_atomic}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::watch::{DEEP_INTERVAL, IntervalTicker, Intervals, ManualTicker, Notice, SHALLOW_INTERVAL, Ticker, Watcher}; +use craft_ipc::{Discovery, LinkError, RuntimeDir}; + +const LIMIT: Duration = Duration::from_secs(10); +const SECOND: Duration = Duration::from_secs(1); + +struct Tmp(PathBuf); + +impl Tmp { + fn new(label: &str) -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-watch-{label}-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn forward(tx: Sender) -> impl Fn(Notice) + Send + Sync + 'static { + let tx = Mutex::new(tx); + move |notice| { + let _ = tx.lock().unwrap().send(notice); + } +} + +struct Rig { + _tmp: Tmp, + file: PathBuf, + clock: Arc, + notices: Receiver, + watcher: Watcher, +} + +impl Rig { + /// A watcher on `a.png` with content "one" and epoch 4, on the contract intervals. + fn new(label: &str) -> Self { + let tmp = Tmp::new(label); + let file = tmp.0.join("a.png"); + std::fs::write(&file, b"one").unwrap(); + let clock = Arc::new(ManualTicker::new()); + let (tx, notices) = channel(); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), forward(tx)).unwrap(); + watcher.watch("link-1", Tracker::new(&file, 4).unwrap()).unwrap(); + Self { _tmp: tmp, file, clock, notices, watcher } + } + + /// Move time and wait until the watcher thread has handled it. + fn advance(&self, by: Duration) { + self.clock.advance(by); + assert!(self.clock.settle(LIMIT), "the watcher did not settle"); + } + + fn quiet(&self) { + let got = self.notices.try_recv(); + assert!(got.is_err(), "unexpected notice {got:?}"); + } + + fn next(&self) -> Notice { + self.notices.try_recv().expect("a notice was due") + } +} + +fn changed(link: &str, state: FileState, epoch: u64, bumped: bool, deep: bool) -> Notice { + Notice::Changed { link_id: link.into(), change: Change { state, epoch, bumped }, deep } +} + +#[test] +fn contract_intervals_are_one_and_thirty_seconds() { + assert_eq!(SHALLOW_INTERVAL, Duration::from_secs(1)); + assert_eq!(DEEP_INTERVAL, Duration::from_secs(30)); + assert_eq!(Intervals::default(), Intervals { shallow: SHALLOW_INTERVAL, deep: DEEP_INTERVAL }); +} + +#[test] +fn edit_on_disk_bumps_the_epoch_on_the_first_shallow_tick_and_not_before() { + let rig = Rig::new("shallow"); + write_atomic(&rig.file, b"two!").unwrap(); + rig.advance(Duration::from_millis(500)); + rig.advance(Duration::from_millis(499)); + rig.quiet(); + assert_eq!(rig.watcher.poll_counts().shallow, 0, "no poll before the 1 s interval"); + rig.advance(Duration::from_millis(1)); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, false)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (1, 0)); + // No further edit, no further notice, and the next poll comes one interval later. + rig.advance(Duration::from_millis(999)); + assert_eq!(rig.watcher.poll_counts().shallow, 1); + rig.advance(Duration::from_millis(1)); + assert_eq!(rig.watcher.poll_counts().shallow, 2); + rig.quiet(); + // A second edit bumps again. + write_atomic(&rig.file, b"three").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 6, true, false)); +} + +#[test] +fn deep_poll_runs_at_thirty_seconds_and_finds_an_edit_that_keeps_size_and_mtime() { + let rig = Rig::new("deep"); + let mtime = std::fs::metadata(&rig.file).unwrap().modified().unwrap(); + std::fs::write(&rig.file, b"two").unwrap(); + OpenOptions::new().write(true).open(&rig.file).unwrap().set_modified(mtime).unwrap(); + for _ in 1..30 { + rig.advance(SECOND); + } + rig.quiet(); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (29, 0), "shallow polls are blind to this edit"); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, true)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (29, 1), "the deep poll replaces the shallow poll on its tick"); + // The schedule goes on: shallow at 31 s, deep again at 60 s. + rig.advance(SECOND); + assert_eq!(rig.watcher.poll_counts().shallow, 30); + rig.advance(Duration::from_secs(29)); + assert_eq!(rig.watcher.poll_counts().deep, 2); +} + +#[test] +fn a_large_time_jump_is_one_deep_poll_and_the_grid_stays_aligned() { + let rig = Rig::new("jump"); + rig.advance(Duration::from_secs(100)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (0, 1)); + rig.advance(Duration::from_millis(500)); + assert_eq!(rig.watcher.poll_counts().shallow, 0, "100.5 s is before the next grid point at 101 s"); + rig.advance(Duration::from_millis(500)); + assert_eq!(rig.watcher.poll_counts().shallow, 1, "101 s is the next grid point"); +} + +#[test] +fn producer_offline_missing_and_return_are_reported_without_a_bump() { + let rig = Rig::new("offline"); + // A touch alone is silent. + let file = OpenOptions::new().write(true).open(&rig.file).unwrap(); + file.set_modified(std::time::SystemTime::now() + Duration::from_secs(60)).unwrap(); + drop(file); + rig.advance(SECOND); + rig.quiet(); + std::fs::remove_file(&rig.file).unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Missing, 4, false, false)); + rig.advance(SECOND); + rig.quiet(); + std::fs::write(&rig.file, b"one").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Current, 4, false, false)); + rig.quiet(); +} + +#[test] +fn read_errors_are_reported_once_and_recover() { + let rig = Rig::new("error"); + std::fs::remove_file(&rig.file).unwrap(); + std::fs::create_dir(&rig.file).unwrap(); + rig.advance(SECOND); + let Notice::PollFailed { link_id, error } = rig.next() else { + panic!("expected a failure notice"); + }; + assert_eq!(link_id, "link-1"); + assert!(matches!(error, LinkError::Invalid(_)), "{error:?}"); + rig.advance(SECOND); + rig.quiet(); + std::fs::remove_dir(&rig.file).unwrap(); + std::fs::write(&rig.file, b"different").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, false)); +} + +#[test] +fn with_tracker_accept_silences_the_next_notice_and_unwatch_stops_polling() { + let rig = Rig::new("accept"); + write_atomic(&rig.file, b"two!").unwrap(); + rig.advance(SECOND); + rig.next(); + let state = rig.watcher.with_tracker("link-1", |tracker| { + tracker.accept(); + tracker.state() + }); + assert_eq!(state, Some(FileState::Current)); + rig.advance(SECOND); + rig.quiet(); + assert!(rig.watcher.with_tracker("nope", |_| ()).is_none()); + assert!(rig.watcher.watch("link-1", Tracker::new(&rig.file, 0).unwrap()).is_err(), "duplicate id"); + let tracker = rig.watcher.unwatch("link-1").unwrap(); + assert_eq!(tracker.epoch(), 5); + write_atomic(&rig.file, b"after unwatch").unwrap(); + rig.advance(SECOND); + rig.quiet(); + assert!(rig.watcher.unwatch("link-1").is_none()); +} + +#[test] +fn several_links_are_polled_together() { + let rig = Rig::new("many"); + let other = rig._tmp.0.join("b.png"); + std::fs::write(&other, b"b").unwrap(); + rig.watcher.watch("link-2", Tracker::new(&other, 0).unwrap()).unwrap(); + write_atomic(&other, b"bb").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-2", FileState::Modified, 1, true, false)); + rig.quiet(); +} + +#[test] +fn bad_intervals_are_refused() { + let clock = Arc::new(ManualTicker::new()); + let zero = Intervals { shallow: Duration::ZERO, deep: SECOND }; + assert!(Watcher::new(Arc::clone(&clock) as Arc, zero, |_| ()).is_err()); + let inverted = Intervals { shallow: Duration::from_secs(5), deep: SECOND }; + assert!(Watcher::new(clock as Arc, inverted, |_| ()).is_err()); + assert!(IntervalTicker::new(Duration::ZERO).is_err()); +} + +#[test] +fn drop_stops_the_thread_at_once_even_with_a_slow_ticker() { + let ticker = Arc::new(IntervalTicker::new(Duration::from_secs(3600)).unwrap()); + let watcher = + Watcher::new(ticker as Arc, Intervals { shallow: Duration::from_secs(3600), deep: Duration::from_secs(7200) }, |_| ()).unwrap(); + let (done, finished) = channel(); + std::thread::spawn(move || { + drop(watcher); + let _ = done.send(()); + }); + finished.recv_timeout(LIMIT).expect("drop must not wait for the next tick"); + + let clock = Arc::new(ManualTicker::new()); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), |_| ()).unwrap(); + assert!(clock.settle(LIMIT)); + drop(watcher); + assert!(clock.wait().is_none(), "a closed ticker stays closed"); + assert!(!clock.settle(Duration::from_millis(1))); +} + +#[test] +fn the_native_ticker_drives_a_watcher_without_a_manual_clock() { + let tmp = Tmp::new("native"); + let file = tmp.0.join("a.png"); + std::fs::write(&file, b"one").unwrap(); + let (tx, notices) = channel(); + let ticker = Arc::new(IntervalTicker::new(Duration::from_millis(10)).unwrap()); + let intervals = Intervals { shallow: Duration::from_millis(10), deep: Duration::from_millis(50) }; + let watcher = Watcher::new(ticker as Arc, intervals, forward(tx)).unwrap(); + watcher.watch("link-1", Tracker::new(&file, 0).unwrap()).unwrap(); + write_atomic(&file, b"two!").unwrap(); + let got = notices.recv_timeout(LIMIT).expect("a notice within the limit"); + // The first poll that sees the edit is shallow or deep, depending on when the 50 ms deep deadline falls. + let deep = matches!(got, Notice::Changed { deep: true, .. }); + assert_eq!(got, changed("link-1", FileState::Modified, 1, true, deep)); + drop(watcher); +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: serde_json::Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &serde_json::Value) -> Result<(serde_json::Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[test] +fn a_disk_edit_reaches_a_subscribed_client_as_link_changed() { + let tmp = Tmp::new("wire"); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let server = Arc::new( + Server::start(ServerConfig { + app: "filmcraft".into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(), + ); + let record = Discovery::scan(&runtime).unwrap().into_iter().next().unwrap(); + let mut client = Client::connect(&record).unwrap(); + client + .hello(&HelloClient { + app: "hub".into(), + app_version: "0.1.0".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440002".into(), + protocol_min: 1, + protocol_max: 1, + }) + .unwrap(); + client.subscribe(&["link.changed"]).unwrap(); + + let file = tmp.0.join("still.png"); + std::fs::write(&file, b"v1").unwrap(); + let clock = Arc::new(ManualTicker::new()); + let emitter = Arc::clone(&server); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), move |notice| { + if let Notice::Changed { link_id, change, .. } = notice + && change.bumped + { + emitter.emit("link.changed", json!({"link_id": link_id, "epoch": change.epoch})).unwrap(); + } + }) + .unwrap(); + watcher.watch("link-9", Tracker::new(&file, 7).unwrap()).unwrap(); + + clock.advance(SECOND); + assert!(clock.settle(LIMIT)); + assert!(client.next_event(Some(Duration::from_millis(50))).unwrap().is_none(), "no edit, no event"); + write_atomic(&file, b"v2 is longer").unwrap(); + clock.advance(SECOND); + assert!(clock.settle(LIMIT)); + let event = client.next_event(Some(LIMIT)).unwrap().expect("a link.changed event"); + assert_eq!(event.event, "link.changed"); + assert_eq!(event.data, json!({"link_id": "link-9", "epoch": 8})); + + drop(watcher); + Arc::try_unwrap(server).ok().expect("the watcher released the server").shutdown().unwrap(); +} From 3919a2858374bada1e6542f9cae3db50f701f6fb Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:55:27 +0100 Subject: [PATCH 055/209] svg: a scene in local coordinates, with fill rules and element ids (#778, #789) (#1859) * svg: a scene in local coordinates, with fill rules and element ids (#778, #789) Document::from_tree_local keeps every node in its own user space, with its transform relative to its parent, so an app can keep geometry editable apart from where it is placed (EffectCraft's shape layers). Paths now carry their fill rule, id and transform, and groups their id. from_tree is unchanged: segments, strokes and group transforms stay absolute, path transforms are the identity. * craft-svg: place local text and glyph groups in their own frames (#789) * svg: pattern paints carry their tile and content (#778, #789) (#1863) Paint::Pattern now holds the resolved tile: id, rect, transform and the content as scene nodes in tile space. Each pattern is converted once and shared by every use; content past the depth limit, or a pattern that refers back to itself, has none. In absolute scenes the tile transform maps to the document. * craft-svg: report scene-omitted group effects and unsupported elements (#789) (#1870) --- crates/svg/src/tree.rs | 37 ++ crates/svg/src/tree/document.rs | 457 ++++++++++++++++++------- crates/svg/src/tree/document_tests.rs | 349 +++++++++++++++++++ crates/svg/src/tree/primitive_tests.rs | 3 + 4 files changed, 715 insertions(+), 131 deletions(-) create mode 100644 crates/svg/src/tree/document_tests.rs diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 99b6f73a4..ab04a3002 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -174,12 +174,49 @@ impl Preflight<'_> { self.depth } + /// The SVG elements in the document that the parsed tree leaves out, by name and count, + /// sorted by name: animation, scripts, embedded HTML and media, and text when this build + /// converts no text (neither `tree-text` nor `tree-render`). For an import's "not imported" notes. + pub fn unsupported_elements(&self) -> Vec<(&'static str, usize)> { + const SVG_NS: &str = "http://www.w3.org/2000/svg"; + let mut counts: std::collections::BTreeMap<&'static str, usize> = std::collections::BTreeMap::new(); + for node in self.xml.descendants().filter(roxmltree::Node::is_element) { + if !matches!(node.tag_name().namespace(), None | Some(SVG_NS)) { + continue; + } + if let Some(name) = UNSUPPORTED.iter().copied().find(|name| *name == node.tag_name().name()) { + *counts.entry(name).or_default() += 1; + } + } + counts.into_iter().collect() + } + /// Converts this already-checked XML into the configured usvg tree. pub fn parse(self, options: &usvg::Options<'_>) -> Result { usvg::Tree::from_xmltree(&self.xml, options).map_err(|e| Error::Parse(e.to_string())) } } +/// Elements the parsed tree never holds: usvg reads static content only, and text needs fonts. +const UNSUPPORTED: &[&str] = &[ + "animate", + "animateColor", + "animateMotion", + "animateTransform", + "audio", + "canvas", + "discard", + "foreignObject", + "iframe", + "script", + "set", + // usvg converts text when built with its `text` feature: `tree-text`, or `tree-render` + // through resvg. + #[cfg(not(any(feature = "tree-text", feature = "tree-render")))] + "text", + "video", +]; + /// Checks already-decoded SVG text once, retaining the parsed XML for both depth policy and tree conversion. pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { let preflight_limits = crate::preflight::Limits { diff --git a/crates/svg/src/tree/document.rs b/crates/svg/src/tree/document.rs index 838f62e98..19f594173 100644 --- a/crates/svg/src/tree/document.rs +++ b/crates/svg/src/tree/document.rs @@ -1,8 +1,12 @@ //! A format-neutral vector scene made from a bounded SVG tree. //! //! The scene preserves paths, groups, paints, transforms, opacity, strokes and image sources -//! without depending on an application's document model. Coordinates and stroke widths use the -//! SVG tree's CSS-pixel coordinate system. +//! without depending on an application's document model. +//! +//! [`Document::from_tree`] gives every coordinate and stroke width in the document's CSS pixels +//! ([`Coordinates::Absolute`]). [`Document::from_tree_local`] keeps each node in its own user +//! space with its transform relative to its parent ([`Coordinates::Local`]), for an app that +//! keeps a node's geometry editable apart from where it is placed. #[cfg(feature = "tree-render")] use super::draw::Refusal; @@ -18,9 +22,42 @@ pub struct Document { pub images: usize, /// True if nodes deeper than the requested depth were omitted. pub too_deep: bool, + /// The space segments, transforms, bounds and stroke widths are given in. + pub coordinates: Coordinates, + /// Group effects the scene flags ([`GroupFeatures`]) but doesn't carry, counted. + pub omitted: Omitted, source: Option>, } +/// How many groups have an effect the scene doesn't carry: an app that draws only the scene +/// draws them without it, and can say so. Elements usvg itself leaves out (animation, scripts…) +/// are listed by [`super::Preflight::unsupported_elements`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Omitted { + /// Groups with `filter`. + pub filters: usize, + /// Groups with `clip-path`. + pub clip_paths: usize, + /// Groups with `mask`. + pub masks: usize, + /// Groups with a `mix-blend-mode` other than normal. + pub blends: usize, +} + +/// Where a scene's geometry is given. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Coordinates { + /// Document CSS pixels: path segments, stroke widths and dashes are already transformed, a + /// path's `transform` is the identity, and a group's or image's `transform` is its full + /// transform to the document. + #[default] + Absolute, + /// Each node in its own user space: path segments, stroke widths, dashes, gradients and + /// bounds are as the SVG gives them, and every `transform` maps the node into its parent + /// (the document for top-level nodes). + Local, +} + impl PartialEq for Document { fn eq(&self, other: &Self) -> bool { self.width == other.width @@ -28,6 +65,8 @@ impl PartialEq for Document { && self.nodes == other.nodes && self.images == other.images && self.too_deep == other.too_deep + && self.coordinates == other.coordinates + && self.omitted == other.omitted } } @@ -44,7 +83,16 @@ impl NodePath { /// A group or path in document order. #[derive(Clone, Debug, PartialEq)] pub enum Node { - Group { children: Vec, opacity: f32, transform: [f64; 6], bounds: [f64; 4], features: GroupFeatures, source_path: NodePath }, + Group { + children: Vec, + opacity: f32, + transform: [f64; 6], + bounds: [f64; 4], + features: GroupFeatures, + source_path: NodePath, + /// The element's `id`, empty when it has none. + id: String, + }, Path(Box), Image(Image), } @@ -80,13 +128,27 @@ pub enum ImageKind { Svg, } -/// A transformed path with its resolved fill and stroke. +/// A path with its resolved fill and stroke. #[derive(Clone, Debug, PartialEq)] pub struct Path { pub segments: Vec, pub fill: Option, + /// How the fill decides what is inside (`fill-rule`); `NonZero` when there is no fill. + pub fill_rule: FillRule, pub stroke: Option, pub source_path: NodePath, + /// The element's `id`, empty when it has none. + pub id: String, + /// Maps the segments into the parent ([`Coordinates`]); the identity in absolute scenes. + pub transform: [f64; 6], +} + +/// The `fill-rule` property. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FillRule { + #[default] + NonZero, + EvenOdd, } impl Path { @@ -97,7 +159,7 @@ impl Path { } } -/// A path segment in CSS pixels after its absolute transform. +/// A path segment, in the space the scene's [`Coordinates`] name. #[derive(Clone, Copy, Debug, PartialEq)] pub enum Segment { MoveTo([f64; 2]), @@ -107,12 +169,42 @@ pub enum Segment { Close, } -/// A resolved color or an unsupported SVG paint server. +/// A resolved color, gradient or pattern. #[derive(Clone, Debug, PartialEq)] pub enum Paint { - Color { rgb: [u8; 3], opacity: f32 }, - Gradient { kind: GradientKind, stops: Vec, transform: [f64; 6], spread: Spread, opacity: f32 }, - Pattern, + Color { + rgb: [u8; 3], + opacity: f32, + }, + Gradient { + kind: GradientKind, + stops: Vec, + transform: [f64; 6], + spread: Spread, + opacity: f32, + }, + /// A ``; `None` when its content is past the depth limit or refers back to the + /// pattern itself. + Pattern { + pattern: Option, + opacity: f32, + }, +} + +/// A resolved `` tile. usvg has already applied `patternUnits`, +/// `patternContentUnits` and `viewBox`, so the tile and its content share one space. +#[derive(Clone, Debug, PartialEq)] +pub struct Pattern { + /// The element's `id`. + pub id: String, + /// The tile `[x, y, width, height]` in tile space. + pub rect: [f64; 4], + /// Maps tile space into the painted path's space: `patternTransform` in a local scene; in + /// an absolute scene, `patternTransform` followed by the path's transform to the document. + pub transform: [f64; 6], + /// What one tile draws, in tile space ([`Coordinates::Local`] whatever the scene's + /// coordinates). Shared by every use of the same pattern. + pub content: std::sync::Arc>, } #[derive(Clone, Debug, PartialEq)] @@ -162,20 +254,32 @@ pub enum LineJoin { } impl Document { - /// Convert a bounded usvg tree to a scene, omitting nodes beyond `max_depth`. + /// Convert a bounded usvg tree to a scene in document pixels ([`Coordinates::Absolute`]), + /// omitting nodes beyond `max_depth`. pub fn from_tree(tree: &usvg::Tree, max_depth: usize) -> Self { - let size = tree.size(); - let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), ..Self::default() }; - document.nodes = convert_group(tree.root(), 0, max_depth, &mut document, &[]); + Self::convert(tree, max_depth, Coordinates::Absolute) + } + + /// Convert a bounded usvg tree to a scene that keeps every node in its own user space with a + /// transform relative to its parent ([`Coordinates::Local`]), omitting nodes beyond + /// `max_depth`. + pub fn from_tree_local(tree: &usvg::Tree, max_depth: usize) -> Self { + Self::convert(tree, max_depth, Coordinates::Local) + } + + /// Build a scene in document pixels that retains its source tree for fallback rendering. + pub fn from_owned_tree(tree: usvg::Tree, max_depth: usize) -> Self { + let mut document = Self::convert(&tree, max_depth, Coordinates::Absolute); + document.source = Some(Box::new(tree)); document } - /// Build a scene that retains its source tree for fallback rendering. - pub fn from_owned_tree(tree: usvg::Tree, max_depth: usize) -> Self { + fn convert(tree: &usvg::Tree, max_depth: usize, coordinates: Coordinates) -> Self { let size = tree.size(); - let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), ..Self::default() }; - document.nodes = convert_group(tree.root(), 0, max_depth, &mut document, &[]); - document.source = Some(Box::new(tree)); + let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), coordinates, ..Self::default() }; + let mut cx = Convert { doc: &mut document, max_depth, local: coordinates == Coordinates::Local, patterns: std::collections::HashMap::new() }; + let nodes = cx.group(tree.root(), tree.root().abs_transform(), 0, &[]); + document.nodes = nodes; document } @@ -231,7 +335,7 @@ fn spread(s: usvg::SpreadMethod) -> Spread { } } -fn color(paint: &usvg::Paint, opacity: f32) -> Paint { +fn color(paint: &usvg::Paint, opacity: f32, pattern: impl FnOnce(&std::sync::Arc) -> Option) -> Paint { match paint { usvg::Paint::Color(c) => Paint::Color { rgb: [c.red, c.green, c.blue], opacity }, usvg::Paint::LinearGradient(g) => Paint::Gradient { @@ -269,7 +373,7 @@ fn color(paint: &usvg::Paint, opacity: f32) -> Paint { spread: spread(g.spread_method()), opacity, }, - usvg::Paint::Pattern(_) => Paint::Pattern, + usvg::Paint::Pattern(source) => Paint::Pattern { pattern: pattern(source), opacity }, } } @@ -279,122 +383,213 @@ fn point(transform: usvg::Transform, p: usvg::tiny_skia_path::Point) -> [f64; 2] [f64::from(p.x), f64::from(p.y)] } -fn convert_group(group: &usvg::Group, depth: usize, max_depth: usize, doc: &mut Document, parent_path: &[usize]) -> Vec { - let mut result = Vec::new(); - for (index, node) in group.children().iter().enumerate() { - let mut source_path = parent_path.to_vec(); - source_path.push(index); - match node { - usvg::Node::Path(path) => { - if !path.is_visible() || path.data().is_empty() { - continue; +const IDENTITY: [f64; 6] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + +/// A node's transform relative to `parent` (both absolute): `parent⁻¹ · node`. The identity +/// when `parent` cannot be inverted (a degenerate group draws nothing anyway). +fn relative(parent: usvg::Transform, node: usvg::Transform) -> usvg::Transform { + parent.invert().map_or(usvg::Transform::identity(), |inverse| inverse.pre_concat(node)) +} + +struct Convert<'d> { + doc: &'d mut Document, + max_depth: usize, + local: bool, + /// Pattern content already converted, by the address of usvg's shared pattern; `None` while + /// a pattern is being converted, so a pattern that refers back to itself gets no content. + patterns: std::collections::HashMap>>>, +} + +impl Convert<'_> { + /// The transform a node with absolute transform `node` gets under a group whose absolute + /// transform is `parent`. + fn placement(&self, parent: usvg::Transform, node: usvg::Transform) -> [f64; 6] { + if self.local { transform(relative(parent, node)) } else { transform(node) } + } + + /// `parent` is the absolute transform `group`'s leaves are placed against: its own, except + /// for flattened text, whose outline paths carry the text's while the group has none. + fn group(&mut self, group: &usvg::Group, parent: usvg::Transform, depth: usize, parent_path: &[usize]) -> Vec { + let mut result = Vec::new(); + for (index, node) in group.children().iter().enumerate() { + let mut source_path = parent_path.to_vec(); + source_path.push(index); + match node { + usvg::Node::Path(path) => { + if let Some(path) = self.path(path, parent, source_path, depth) { + result.push(Node::Path(Box::new(path))); + } } - let transform = path.abs_transform(); - let segments = path - .data() - .segments() - .map(|segment| { - use usvg::tiny_skia_path::PathSegment as S; - match segment { - S::MoveTo(a) => Segment::MoveTo(point(transform, a)), - S::LineTo(a) => Segment::LineTo(point(transform, a)), - S::QuadTo(a, b) => Segment::QuadTo(point(transform, a), point(transform, b)), - S::CubicTo(a, b, c) => Segment::CubicTo(point(transform, a), point(transform, b), point(transform, c)), - S::Close => Segment::Close, + usvg::Node::Group(child) => { + if depth >= self.max_depth { + self.doc.too_deep = true; + continue; + } + let children = self.group(child, child.abs_transform(), depth + 1, &source_path); + if !children.is_empty() { + let omitted = &mut self.doc.omitted; + omitted.filters += usize::from(!child.filters().is_empty()); + omitted.clip_paths += usize::from(child.clip_path().is_some()); + omitted.masks += usize::from(child.mask().is_some()); + omitted.blends += usize::from(child.blend_mode() != usvg::BlendMode::Normal); + let bounds = if self.local { child.layer_bounding_box().to_rect() } else { child.abs_layer_bounding_box().to_rect() }; + result.push(Node::Group { + children, + opacity: child.opacity().get(), + // Locally, the group's own transform: usvg leaves the absolute one at + // the identity for colour and bitmap glyph groups in text. + transform: if self.local { transform(child.transform()) } else { transform(child.abs_transform()) }, + bounds: rect(bounds), + features: GroupFeatures { + filters: !child.filters().is_empty(), + clip_path: child.clip_path().is_some(), + mask: child.mask().is_some(), + non_normal_blend: child.blend_mode() != usvg::BlendMode::Normal, + isolation: child.should_isolate(), + }, + source_path: NodePath(source_path), + id: child.id().to_string(), + }); + } + } + usvg::Node::Text(text) => { + if depth >= self.max_depth { + self.doc.too_deep = true; + continue; + } + let flat = text.flattened(); + // The flattened group has no transform of its own, but its outline paths + // have the text's: place its leaves, and the wrapper, in the text's frame. + let frame = if self.local { text.abs_transform() } else { flat.abs_transform() }; + let children = self.group(flat, frame, depth + 1, &source_path); + if !children.is_empty() { + let bounds = if self.local { text.bounding_box() } else { text.abs_bounding_box() }; + result.push(Node::Group { + children, + opacity: 1.0, + transform: self.placement(parent, frame), + bounds: rect(bounds), + features: GroupFeatures::default(), + source_path: NodePath(source_path), + id: text.id().to_string(), + }); + } + } + usvg::Node::Image(image) => { + self.doc.images += 1; + let kind = match image.kind() { + usvg::ImageKind::JPEG(_) => ImageKind::Jpeg, + usvg::ImageKind::PNG(_) => ImageKind::Png, + usvg::ImageKind::GIF(_) => ImageKind::Gif, + usvg::ImageKind::WEBP(_) => ImageKind::WebP, + usvg::ImageKind::SVG(_) => ImageKind::Svg, + }; + let (encoded, svg) = match image.kind() { + usvg::ImageKind::JPEG(data) | usvg::ImageKind::PNG(data) | usvg::ImageKind::GIF(data) | usvg::ImageKind::WEBP(data) => { + (Some(data.clone()), None) } - }) - .collect(); - let fill = path.fill().map(|f| color(f.paint(), f.opacity().get())); - let stroke = path.stroke().map(|s| { - let scale = f64::from((transform.sx * transform.sy - transform.kx * transform.ky).abs().sqrt()); - Stroke { - paint: color(s.paint(), s.opacity().get()), - width: f64::from(s.width().get()) * scale, - cap: match s.linecap() { - usvg::LineCap::Butt => LineCap::Butt, - usvg::LineCap::Round => LineCap::Round, - usvg::LineCap::Square => LineCap::Square, - }, - join: match s.linejoin() { - usvg::LineJoin::Round => LineJoin::Round, - usvg::LineJoin::Bevel => LineJoin::Bevel, - usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => LineJoin::Miter, - }, - miter_limit: f64::from(s.miterlimit().get()), - dash_array: s.dasharray().map(|d| d.iter().map(|v| f64::from(*v) * scale).collect()), - dash_offset: f64::from(s.dashoffset()) * scale, - } - }); - result.push(Node::Path(Box::new(Path { segments, fill, stroke, source_path: NodePath(source_path) }))); - } - usvg::Node::Group(child) => { - if depth >= max_depth { - doc.too_deep = true; - continue; - } - let children = convert_group(child, depth + 1, max_depth, doc, &source_path); - if !children.is_empty() { - result.push(Node::Group { - children, - opacity: child.opacity().get(), - transform: transform(child.abs_transform()), - bounds: rect(child.abs_layer_bounding_box().to_rect()), - features: GroupFeatures { - filters: !child.filters().is_empty(), - clip_path: child.clip_path().is_some(), - mask: child.mask().is_some(), - non_normal_blend: child.blend_mode() != usvg::BlendMode::Normal, - isolation: child.should_isolate(), - }, + usvg::ImageKind::SVG(tree) => { + let max_depth = self.max_depth.saturating_sub(depth + 1); + let nested = if self.local { + Document::from_tree_local(tree, max_depth) + } else { + Document::from_owned_tree(tree.clone(), max_depth) + }; + (None, Some(Box::new(nested))) + } + }; + let bounds = if self.local { image.bounding_box() } else { image.abs_bounding_box() }; + result.push(Node::Image(Image { + kind, + visible: image.is_visible(), + transform: self.placement(parent, image.abs_transform()), + bounds: rect(bounds), + encoded, + svg, source_path: NodePath(source_path), - }); + })); } } - usvg::Node::Text(text) => { - if depth >= max_depth { - doc.too_deep = true; - continue; - } - let children = convert_group(text.flattened(), depth + 1, max_depth, doc, &source_path); - if !children.is_empty() { - result.push(Node::Group { - children, - opacity: 1.0, - transform: transform(text.abs_transform()), - bounds: rect(text.abs_bounding_box()), - features: GroupFeatures::default(), - source_path: NodePath(source_path), - }); - } - } - usvg::Node::Image(image) => { - doc.images += 1; - let kind = match image.kind() { - usvg::ImageKind::JPEG(_) => ImageKind::Jpeg, - usvg::ImageKind::PNG(_) => ImageKind::Png, - usvg::ImageKind::GIF(_) => ImageKind::Gif, - usvg::ImageKind::WEBP(_) => ImageKind::WebP, - usvg::ImageKind::SVG(_) => ImageKind::Svg, - }; - let (encoded, svg) = match image.kind() { - usvg::ImageKind::JPEG(data) | usvg::ImageKind::PNG(data) | usvg::ImageKind::GIF(data) | usvg::ImageKind::WEBP(data) => { - (Some(data.clone()), None) - } - usvg::ImageKind::SVG(tree) => { - (None, Some(Box::new(Document::from_owned_tree(tree.clone(), max_depth.saturating_sub(depth + 1))))) - } - }; - result.push(Node::Image(Image { - kind, - visible: image.is_visible(), - transform: transform(image.abs_transform()), - bounds: rect(image.abs_bounding_box()), - encoded, - svg, - source_path: NodePath(source_path), - })); - } } + result + } + + /// A visible, non-empty path. Its own absolute transform is its group's in usvg, so in a + /// local scene its segments stay as they are and its transform is normally the identity. + /// The tile of `pattern` painted on a path at `depth` whose own transform to the document is + /// `abs`. + fn pattern(&mut self, pattern: &std::sync::Arc, abs: usvg::Transform, depth: usize) -> Option { + let key = std::sync::Arc::as_ptr(pattern) as usize; + let content = match self.patterns.get(&key) { + Some(done) => done.clone(), + None if depth >= self.max_depth => { + self.doc.too_deep = true; + None + } + None => { + self.patterns.insert(key, None); + let local = std::mem::replace(&mut self.local, true); + let nodes = self.group(pattern.root(), pattern.root().abs_transform(), depth + 1, &[]); + self.local = local; + let content = Some(std::sync::Arc::new(nodes)); + self.patterns.insert(key, content.clone()); + content + } + }?; + let placement = if self.local { pattern.transform() } else { abs.pre_concat(pattern.transform()) }; + Some(Pattern { id: pattern.id().to_string(), rect: rect(pattern.rect().to_rect()), transform: transform(placement), content }) + } + + fn path(&mut self, path: &usvg::Path, parent: usvg::Transform, source_path: Vec, depth: usize) -> Option { + if !path.is_visible() || path.data().is_empty() { + return None; + } + let abs = path.abs_transform(); + let (to_points, scale, placement) = if self.local { + (usvg::Transform::identity(), 1.0, transform(relative(parent, abs))) + } else { + (abs, f64::from((abs.sx * abs.sy - abs.kx * abs.ky).abs().sqrt()), IDENTITY) + }; + let segments = path + .data() + .segments() + .map(|segment| { + use usvg::tiny_skia_path::PathSegment as S; + match segment { + S::MoveTo(a) => Segment::MoveTo(point(to_points, a)), + S::LineTo(a) => Segment::LineTo(point(to_points, a)), + S::QuadTo(a, b) => Segment::QuadTo(point(to_points, a), point(to_points, b)), + S::CubicTo(a, b, c) => Segment::CubicTo(point(to_points, a), point(to_points, b), point(to_points, c)), + S::Close => Segment::Close, + } + }) + .collect(); + let fill = path.fill().map(|f| color(f.paint(), f.opacity().get(), |p| self.pattern(p, abs, depth))); + let fill_rule = match path.fill().map(usvg::Fill::rule) { + Some(usvg::FillRule::EvenOdd) => FillRule::EvenOdd, + _ => FillRule::NonZero, + }; + let stroke = path.stroke().map(|s| Stroke { + paint: color(s.paint(), s.opacity().get(), |p| self.pattern(p, abs, depth)), + width: f64::from(s.width().get()) * scale, + cap: match s.linecap() { + usvg::LineCap::Butt => LineCap::Butt, + usvg::LineCap::Round => LineCap::Round, + usvg::LineCap::Square => LineCap::Square, + }, + join: match s.linejoin() { + usvg::LineJoin::Round => LineJoin::Round, + usvg::LineJoin::Bevel => LineJoin::Bevel, + usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => LineJoin::Miter, + }, + miter_limit: f64::from(s.miterlimit().get()), + dash_array: s.dasharray().map(|d| d.iter().map(|v| f64::from(*v) * scale).collect()), + dash_offset: f64::from(s.dashoffset()) * scale, + }); + Some(Path { segments, fill, fill_rule, stroke, source_path: NodePath(source_path), id: path.id().to_string(), transform: placement }) } - result } + +#[cfg(test)] +#[path = "document_tests.rs"] +mod tests; diff --git a/crates/svg/src/tree/document_tests.rs b/crates/svg/src/tree/document_tests.rs new file mode 100644 index 000000000..a130a6f6f --- /dev/null +++ b/crates/svg/src/tree/document_tests.rs @@ -0,0 +1,349 @@ +#![allow(clippy::unwrap_used, clippy::panic, clippy::indexing_slicing, clippy::float_cmp)] + +use super::*; +use crate::tree::{Limits, parse}; + +fn tree(body: &str) -> usvg::Tree { + let svg = format!(r#"{body}"#); + parse(svg.as_bytes(), &Limits::default(), &usvg::Options::default()).unwrap() +} + +fn apply(m: [f64; 6], p: [f64; 2]) -> [f64; 2] { + [m[0] * p[0] + m[2] * p[1] + m[4], m[1] * p[0] + m[3] * p[1] + m[5]] +} + +fn then(outer: [f64; 6], inner: [f64; 6]) -> [f64; 6] { + let [a, b, c, d, e, f] = outer; + let [p, q, r, s, t, u] = inner; + [a * p + c * q, b * p + d * q, a * r + c * s, b * r + d * s, a * t + c * u + e, b * t + d * u + f] +} + +fn close(a: [f64; 2], b: [f64; 2]) -> bool { + (a[0] - b[0]).abs() < 1e-3 && (a[1] - b[1]).abs() < 1e-3 +} + +/// Every path's points, mapped to document pixels through the scene's transforms. +fn document_points(nodes: &[Node], outer: [f64; 6], local: bool, out: &mut Vec<[f64; 2]>) { + for node in nodes { + match node { + Node::Group { children, transform, .. } => { + let next = if local { then(outer, *transform) } else { outer }; + document_points(children, next, local, out); + } + Node::Path(path) => { + let m = then(outer, path.transform); + for segment in &path.segments { + match segment { + Segment::MoveTo(p) | Segment::LineTo(p) => out.push(apply(m, *p)), + Segment::QuadTo(a, b) => out.extend([apply(m, *a), apply(m, *b)]), + Segment::CubicTo(a, b, c) => out.extend([apply(m, *a), apply(m, *b), apply(m, *c)]), + Segment::Close => {} + } + } + } + Node::Image(_) => {} + } + } +} + +fn paths(nodes: &[Node]) -> Vec<&Path> { + let mut out = Vec::new(); + for node in nodes { + match node { + Node::Group { children, .. } => out.extend(paths(children)), + Node::Path(path) => out.push(path), + Node::Image(_) => {} + } + } + out +} + +fn group_ids(nodes: &[Node]) -> Vec<(String, [f64; 6])> { + let mut out = Vec::new(); + for node in nodes { + if let Node::Group { children, id, transform, .. } = node { + out.push((id.clone(), *transform)); + out.extend(group_ids(children)); + } + } + out +} + +const NESTED: &str = r##" + + + + +"##; + +#[test] +fn a_local_scene_keeps_user_space_geometry_and_relative_transforms() { + let tree = tree(NESTED); + let local = Document::from_tree_local(&tree, 64); + assert_eq!(local.coordinates, Coordinates::Local); + let groups = group_ids(&local.nodes); + let outer = groups.iter().find(|(id, _)| id == "outer").unwrap(); + assert!(close([outer.1[0], outer.1[3]], [2.0, 2.0]) && close([outer.1[4], outer.1[5]], [10.0, 20.0]), "{outer:?}"); + let inner = groups.iter().find(|(id, _)| id == "inner").unwrap(); + assert!(close([inner.1[0], inner.1[1]], [0.0, 1.0]) && close([inner.1[4], inner.1[5]], [0.0, 0.0]), "relative, not absolute: {inner:?}"); + + let all = paths(&local.nodes); + let rect = all.iter().find(|p| p.id == "box").unwrap(); + let Segment::MoveTo(start) = rect.segments[0] else { panic!("{:?}", rect.segments) }; + assert!(close(start, [1.0, 2.0]), "user space: {start:?}"); + let stroke = rect.stroke.as_ref().unwrap(); + assert_eq!(stroke.width, 2.0, "not scaled by the groups"); + assert_eq!(stroke.dash_array.as_deref(), Some([1.0, 3.0].as_slice())); +} + +#[test] +fn an_absolute_scene_is_unchanged_by_the_local_option() { + let tree = tree(NESTED); + let absolute = Document::from_tree(&tree, 64); + assert_eq!(absolute.coordinates, Coordinates::Absolute); + let rect = paths(&absolute.nodes).into_iter().find(|p| p.id == "box").unwrap(); + assert_eq!(rect.transform, IDENTITY); + // (1, 2) rotated 90° is (-2, 1), then scaled by 2 and moved by (10, 20). + let Segment::MoveTo(start) = rect.segments[0] else { panic!("{:?}", rect.segments) }; + assert!(close(start, [6.0, 22.0]), "{start:?}"); + assert_eq!(rect.stroke.as_ref().unwrap().width, 4.0); +} + +#[test] +fn local_and_absolute_scenes_put_every_point_in_the_same_place() { + let tree = tree(NESTED); + let (mut local, mut absolute) = (Vec::new(), Vec::new()); + document_points(&Document::from_tree_local(&tree, 64).nodes, IDENTITY, true, &mut local); + document_points(&Document::from_tree(&tree, 64).nodes, IDENTITY, false, &mut absolute); + assert_eq!(local.len(), absolute.len()); + assert!(!local.is_empty()); + for (l, a) in local.iter().zip(&absolute) { + assert!(close(*l, *a), "{l:?} vs {a:?}"); + } +} + +#[test] +fn fill_rule_and_ids_are_kept() { + let tree = + tree(r##""##); + for document in [Document::from_tree(&tree, 64), Document::from_tree_local(&tree, 64)] { + let all = paths(&document.nodes); + assert_eq!(all.len(), 2); + assert_eq!((all[0].id.as_str(), all[0].fill_rule), ("hole", FillRule::EvenOdd)); + assert_eq!((all[1].id.as_str(), all[1].fill_rule), ("", FillRule::NonZero), "no fill: the default"); + assert!(all[1].fill.is_none()); + } +} + +#[test] +fn a_local_scene_respects_the_depth_limit() { + let deep = format!("{}{}", "".repeat(6), "".repeat(6)); + let tree = tree(&deep); + let shallow = Document::from_tree_local(&tree, 2); + assert!(shallow.too_deep); + assert!(paths(&shallow.nodes).is_empty()); + assert!(!Document::from_tree_local(&tree, 64).too_deep); +} + +#[test] +fn an_embedded_svg_in_a_local_scene_is_local_too() { + use std::io::Write as _; + let inner = r##""##; + let mut b64 = Vec::new(); + { + // A tiny base64 encoder keeps the test free of extra dependencies. + const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + for chunk in inner.as_bytes().chunks(3) { + let n = chunk.iter().enumerate().fold(0u32, |n, (i, b)| n | (u32::from(*b) << (16 - 8 * i))); + for i in 0..4 { + if i <= chunk.len() { + b64.push(A[((n >> (18 - 6 * i)) & 63) as usize]); + } else { + b64.push(b'='); + } + } + } + } + let mut body = Vec::new(); + write!(body, r#""#, String::from_utf8(b64).unwrap()).unwrap(); + let svg = format!(r#"{}"#, String::from_utf8(body).unwrap()); + let options = usvg::Options { image_href_resolver: crate::tree::image_resolver(Limits::default()), ..usvg::Options::default() }; + let tree = parse(svg.as_bytes(), &Limits::default(), &options).unwrap(); + let local = Document::from_tree_local(&tree, 64); + let image = local + .nodes + .iter() + .find_map(|n| match n { + Node::Image(image) => Some(image), + Node::Group { children, .. } => children.iter().find_map(|c| if let Node::Image(image) = c { Some(image) } else { None }), + Node::Path(_) => None, + }) + .unwrap(); + let nested = image.svg.as_ref().unwrap(); + assert_eq!(nested.coordinates, Coordinates::Local); + let rect = paths(&nested.nodes)[0]; + let Segment::LineTo(p) = rect.segments[1] else { panic!("{:?}", rect.segments) }; + assert!(close(p, [1.0, 0.0]), "inside its own scaled group: {p:?}"); +} + +fn fill_pattern(path: &Path) -> Option<&Pattern> { + match path.fill.as_ref()? { + Paint::Pattern { pattern, .. } => pattern.as_ref(), + _ => None, + } +} + +const PATTERN: &str = r##" + + +"##; + +#[test] +fn a_pattern_paint_carries_its_tile_and_content() { + let tree = tree(PATTERN); + let local = Document::from_tree_local(&tree, 64); + let all = paths(&local.nodes); + let a = all.iter().find(|p| p.id == "a").unwrap(); + let pattern = fill_pattern(a).unwrap(); + assert_eq!(pattern.id, "dots"); + assert_eq!(pattern.rect, [0.0, 0.0, 4.0, 4.0]); + assert!(close([pattern.transform[4], pattern.transform[5]], [1.0, 2.0]) && close([pattern.transform[0], pattern.transform[3]], [1.0, 1.0])); + let tile = paths(&pattern.content); + assert_eq!(tile.len(), 1); + assert_eq!(tile[0].fill, Some(Paint::Color { rgb: [255, 0, 0], opacity: 1.0 })); + + // Every use shares one converted tile, and keeps its own opacity. + let b = all.iter().find(|p| p.id == "b").unwrap(); + assert!(std::sync::Arc::ptr_eq(&pattern.content, &fill_pattern(b).unwrap().content)); + assert!(matches!(b.fill, Some(Paint::Pattern { opacity, .. }) if (opacity - 0.5).abs() < 1e-6)); +} + +#[test] +fn an_absolute_scene_maps_the_tile_to_the_document() { + let tree = tree(PATTERN); + let absolute = Document::from_tree(&tree, 64); + let a = paths(&absolute.nodes).into_iter().find(|p| p.id == "a").unwrap(); + let pattern = fill_pattern(a).unwrap(); + // translate(1 2), then the group's scale(2). + assert!(close([pattern.transform[0], pattern.transform[3]], [2.0, 2.0]), "{:?}", pattern.transform); + assert!(close([pattern.transform[4], pattern.transform[5]], [2.0, 4.0]), "{:?}", pattern.transform); + // The tile's content stays in tile space. + let Segment::LineTo(p) = paths(&pattern.content)[0].segments[1] else { panic!() }; + assert!(close(p, [2.0, 0.0]), "{p:?}"); +} + +#[test] +fn bounding_box_pattern_units_are_resolved() { + let tree = tree( + r##""##, + ); + let document = Document::from_tree_local(&tree, 64); + let pattern = fill_pattern(paths(&document.nodes)[0]).unwrap(); + assert_eq!(pattern.rect, [0.0, 0.0, 5.0, 10.0]); +} + +#[test] +fn nested_patterns_convert_within_the_depth_limit() { + let svg = r##" + + + "##; + let tree = tree(svg); + let deep = Document::from_tree_local(&tree, 64); + let outer = fill_pattern(paths(&deep.nodes)[0]).unwrap(); + let inner = fill_pattern(paths(&outer.content)[0]).unwrap(); + assert_eq!(inner.id, "inner"); + assert!(!deep.too_deep); + + let shallow = Document::from_tree_local(&tree, 1); + assert!(shallow.too_deep); + let outer = fill_pattern(paths(&shallow.nodes)[0]).unwrap(); + assert!(fill_pattern(paths(&outer.content)[0]).is_none(), "past the limit: no content"); +} + +#[test] +fn a_pattern_that_refers_to_itself_ends() { + let tree = tree( + r##""##, + ); + let document = Document::from_tree_local(&tree, 64); + // usvg may drop the reference; if it keeps it, the inner use has no content. + for path in paths(&document.nodes) { + if let Some(pattern) = fill_pattern(path) { + for inner in paths(&pattern.content) { + assert!(fill_pattern(inner).is_none()); + } + } + } +} + +#[test] +fn group_effects_the_scene_leaves_out_are_counted() { + let tree = tree( + r##" + + + + + + "##, + ); + for document in [Document::from_tree(&tree, 64), Document::from_tree_local(&tree, 64)] { + assert_eq!(document.omitted, Omitted { filters: 1, clip_paths: 2, masks: 1, blends: 1 }); + } + assert_eq!(Document::from_tree(&self::tree(r#""#), 64).omitted, Omitted::default()); +} + +#[test] +fn elements_usvg_leaves_out_are_listed() { + let svg = r##" + + + + + "##; + let checked = crate::tree::preflight(svg, &Limits::default()).unwrap(); + assert_eq!( + checked.unsupported_elements(), + vec![("animate", 2), ("animateTransform", 1), ("foreignObject", 1), ("script", 1), ("set", 1)], + "the XHTML div is not an SVG element" + ); + // The same checked XML still parses. + assert!(checked.parse(&usvg::Options::default()).is_ok()); + let plain = crate::tree::preflight(r#""#, &Limits::default()).unwrap(); + assert!(plain.unsupported_elements().is_empty()); +} + +#[cfg(feature = "tree-text")] +#[test] +fn local_text_is_placed_and_bounded_in_its_parents_frame() { + let mut options = usvg::Options::default(); + let fonts = options.fontdb_mut(); + fonts.load_font_data(include_bytes!("../../../ui/tests/fonts/NotoSans-test.ttf").to_vec()); + let family = fonts.faces().next().unwrap().families[0].0.clone(); + let svg = format!( + r#"Hi"# + ); + let tree = parse(svg.as_bytes(), &Limits::default(), &options).unwrap(); + let local = Document::from_tree_local(&tree, 64); + let absolute = Document::from_tree(&tree, 64); + + let groups = group_ids(&local.nodes); + let text = groups.iter().find(|(id, _)| id == "t").unwrap(); + assert!(close([text.1[0], text.1[3]], [1.0, 1.0]) && close([text.1[4], text.1[5]], [0.0, 0.0]), "text sits in its group's frame: {text:?}"); + + let bounds = |nodes: &[Node]| -> [f64; 4] { + let Node::Group { children, .. } = &nodes[0] else { panic!() }; + let Node::Group { bounds, .. } = &children[0] else { panic!() }; + *bounds + }; + let (l, a) = (bounds(&local.nodes), bounds(&absolute.nodes)); + assert!(close([l[0] + 50.0, l[1]], [a[0], a[1]]) && close([l[2], l[3]], [a[2], a[3]]), "local {l:?} vs absolute {a:?}"); + assert!(l[0] > 0.0 && l[0] < 10.0, "near x=5 in the group's frame: {l:?}"); + + let (mut lp, mut ap) = (Vec::new(), Vec::new()); + document_points(&local.nodes, IDENTITY, true, &mut lp); + document_points(&absolute.nodes, IDENTITY, false, &mut ap); + assert!(!lp.is_empty() && lp.len() == ap.len()); + assert!(lp.iter().zip(&ap).all(|(l, a)| close(*l, *a))); +} diff --git a/crates/svg/src/tree/primitive_tests.rs b/crates/svg/src/tree/primitive_tests.rs index 81f4c0979..ac417eb6c 100644 --- a/crates/svg/src/tree/primitive_tests.rs +++ b/crates/svg/src/tree/primitive_tests.rs @@ -109,8 +109,11 @@ fn hand_written_segments_are_recognised_too() { Segment::Close, ], fill: None, + fill_rule: crate::tree::document::FillRule::NonZero, stroke: None, source_path: crate::tree::document::NodePath::default(), + id: String::new(), + transform: [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], }; assert_eq!(path.primitive(), Some(Primitive::Ellipse { cx: 5.0, cy: 5.0, rx: 5.0, ry: 5.0 })); } From 472956a8d7626643b0148de5573083474a595934 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:33 -0500 Subject: [PATCH 056/209] feat: add craft-control (control channel protocol) (#1879) * feat: add craft-control (control channel protocol) Port from solidcraft: the control channel any app may share, no UI toolkit. - Per-launch token files (32 random bytes, user-only permissions) - Loopback server with token-based authentication - Client that talks to the loopback server - Request/response types (ControlRequest, ControlResponse) - Document watch for subscribed connections waiting on changes - Effect type for requests still running when they finish Desktop-only (wasm32 excluded): uses getrandom for token generation. No external dependencies beyond serde and serde_json. * fix: update test directory names from solidcraft-control to craft-control * Record control crate dependencies in workspace lockfile * re-trigger CI * chore(lock): keep the wasm-bindgen family at the version main's runners install --- Cargo.lock | 10 + crates/control/Cargo.toml | 23 + crates/control/src/auth.rs | 673 +++++++++++++ crates/control/src/cancel.rs | 118 +++ crates/control/src/client.rs | 616 ++++++++++++ crates/control/src/client_tests.rs | 190 ++++ crates/control/src/effect.rs | 27 + crates/control/src/lib.rs | 26 + crates/control/src/request.rs | 125 +++ crates/control/src/server.rs | 1433 ++++++++++++++++++++++++++++ 10 files changed, 3241 insertions(+) create mode 100644 crates/control/Cargo.toml create mode 100644 crates/control/src/auth.rs create mode 100644 crates/control/src/cancel.rs create mode 100644 crates/control/src/client.rs create mode 100644 crates/control/src/client_tests.rs create mode 100644 crates/control/src/effect.rs create mode 100644 crates/control/src/lib.rs create mode 100644 crates/control/src/request.rs create mode 100644 crates/control/src/server.rs diff --git a/Cargo.lock b/Cargo.lock index 5f28d1878..50b715040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1498,6 +1498,16 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "craft-control" +version = "0.1.0" +dependencies = [ + "getrandom 0.3.4", + "proptest", + "serde", + "serde_json", +] + [[package]] name = "craft-control-protocol" version = "0.1.0" diff --git a/crates/control/Cargo.toml b/crates/control/Cargo.toml new file mode 100644 index 000000000..0a18ccc99 --- /dev/null +++ b/crates/control/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "craft-control" +description = "The control channel any app may share: token files, the loopback server, and its requests; no UI toolkit" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + +# The loopback server is for desktop apps; the web app is driven through the page instead. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# The per-launch control token: 32 random bytes from the operating system (MIT OR Apache-2.0). +getrandom = "0.3" + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/control/src/auth.rs b/crates/control/src/auth.rs new file mode 100644 index 000000000..bb2cef86b --- /dev/null +++ b/crates/control/src/auth.rs @@ -0,0 +1,673 @@ +//! The control channel's per-launch token (docs/control-protocol.md). An app listening for +//! control makes a random token, writes it to a file only the user can read +//! ([`App::token_path`]) and refuses every connection whose first request doesn't carry it, so +//! other users on the same machine (loopback is shared, on Windows across remote-desktop sessions +//! too) can't drive the app. Clients find it with [`App::client_token`]. Next to it, `.pid` +//! names the process that wrote it, so a client refused with a leftover token can say the file is +//! stale. + +use std::ffi::OsString; +use std::io::{Error, ErrorKind, Read, Result, Write}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Shortest token accepted from the app's `CONTROL_TOKEN` variable (hex characters). +pub const MIN_TOKEN_LEN: usize = 32; + +/// Ports tried from the app's default port upward when it is taken (a second window, another +/// program). +pub const PORT_TRIES: u16 = 16; + +/// Which app a control channel belongs to: where its token files live, its environment and its +/// name in messages. +#[derive(Clone, Copy)] +pub struct App { + /// The app's name in messages ("restart SolidCraft"). + pub name: &'static str, + /// The per-user data folder; the token files go in its `control` folder. + pub data_dir: fn() -> Option, + /// The app's environment variable `name` (`CONTROL_DIR`, `CONTROL_TOKEN`), with the app's prefix. + pub env: fn(&str) -> Option, + /// A time in seconds since the Unix epoch, as the app writes UTC times in messages. + pub utc: fn(u64) -> String, + /// The port the control channel tries first when the launch gives none. Below the ephemeral + /// ranges of Linux (32768+) and Windows/macOS (49152+), so the system never hands it to an + /// outgoing connection. + pub default_port: u16, +} + +/// Where the control channel listens, from the launch's `--control`/environment port and the app's +/// allow-agents setting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Listen { + /// Exactly this port (`--control PORT`, the app's `CONTROL_PORT`); no fallback. + Exact(u16), + /// The app's [`App::default_port`], or the next free one of [`PORT_TRIES`]. + Default, +} + +/// What a launch listens on. An explicit port wins over the app's allow-agents setting +/// (`enabled`), so a scripted launch works with the setting off; with the setting off and no port, +/// nothing listens. +pub fn plan(explicit: Option, enabled: bool) -> Option { + match explicit { + Some(port) => Some(Listen::Exact(port)), + None if enabled => Some(Listen::Default), + None => None, + } +} + +/// Bind a loopback-only listener on `first`, or on the next free port when it is taken, up to +/// `tries` ports in all. +pub fn bind_with_fallback(first: u16, tries: u16) -> Result { + let mut last = Error::new(ErrorKind::AddrInUse, "no port to try"); + for port in (first..=u16::MAX).take(usize::from(tries.max(1))) { + match std::net::TcpListener::bind(("127.0.0.1", port)) { + Ok(l) => return Ok(l), + // In use; Windows reports a port another program holds exclusively as denied. + Err(e) if matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) => last = e, + Err(e) => return Err(e), + } + } + Err(last) +} + +impl App { + /// The folder of the token files: `/control` (the app's `CONTROL_DIR` replaces it). + fn control_dir(&self) -> Option { + match self.custom_control_dir() { + Some(d) => Some(PathBuf::from(d)), + None => Some((self.data_dir)()?.join("control")), + } + } + + /// The folder the app's `CONTROL_DIR` names in place of the default one. + fn custom_control_dir(&self) -> Option { + (self.env)("CONTROL_DIR").filter(|d| !d.is_empty()) + } + + /// The token file of the app listening on `port`: `/control/.token` + /// (the app's `CONTROL_DIR` replaces the folder). + pub fn token_path(&self, port: u16) -> Option { + Some(self.control_dir()?.join(format!("{port}.token"))) + } + + /// The discovery file: the port of the most recently started app, so a client with no port + /// finds it. It holds only the port; the token stays in its own file. + pub fn current_port_path(&self) -> Option { + Some(self.control_dir()?.join("current.port")) + } + + /// Record `port` as the running app's, readable only by the user like the token files. + pub fn write_current_port(&self, port: u16) -> Result { + let path = self.current_port_path().ok_or_else(|| Error::new(ErrorKind::NotFound, "no data folder"))?; + self.write_token_file(&path, &port.to_string())?; + Ok(path) + } + + /// The port the discovery file names, if there is one. + pub fn read_current_port(&self) -> Option { + parse_port(&std::fs::read_to_string(self.current_port_path()?).ok().filter(|t| t.len() <= 64)?) + } + + /// Remove the discovery file (and its process file) at exit, unless a later app has taken it + /// over: it names another port, or another launch now holds this port's token file. Under + /// Flatpak every instance has its own loopback, so two windows can listen on the same port + /// number and write the same file names; the one closing first must leave the other's (#8794). + /// Run it after [`remove_token_file`], which takes this launch's own token file away first. + pub fn remove_current_port(&self, port: u16, token: &str) { + if let (Some(current), Some(file)) = (self.current_port_path(), self.token_path(port)) { + remove_discovery(¤t, &file, port, token); + } + } + + /// A token set on purpose with the app's `CONTROL_TOKEN` (test harnesses, sandboxed packages + /// whose data folder the client can't see), if it is long enough. + pub fn env_token(&self) -> Option { + (self.env)("CONTROL_TOKEN").and_then(|v| v.into_string().ok()).map(|t| t.trim().to_string()).filter(|t| t.len() >= MIN_TOKEN_LEN) + } + + /// The token a client sends to the app on `port`: `explicit` (`--token`), else the app's + /// `CONTROL_TOKEN`, else the app's token file. + pub fn client_token(&self, port: u16, explicit: Option<&str>) -> Option { + if let Some(t) = explicit.map(str::trim).filter(|t| !t.is_empty()) { + return Some(ClientToken { token: t.to_string(), file: None, app: *self }); + } + if let Some(t) = self.env_token() { + return Some(ClientToken { token: t, file: None, app: *self }); + } + // Read directly, not through any agent's file scope: the app's own file. + read_token_file(&self.token_path(port)?, *self) + } + + /// Write `token` to `path`, readable and writable only by the user: the folder is made `0700` + /// and the file `0600` on Unix; on Windows the file's inherited permissions are replaced by one + /// entry for the user (`icacls`) before the token goes in; when that fails, the still empty file + /// is removed and no token is written. A file left by an earlier launch is replaced. The process + /// file beside it ([`pid_path`]) records this process, best effort. + pub fn write_token_file(&self, path: &Path, token: &str) -> Result<()> { + #[cfg(windows)] + let restrict = |p: &Path| restrict_to_user(p, self.custom_control_dir().is_some()); + #[cfg(not(windows))] + let restrict = |_: &Path| Ok(()); + write_token_file_with(path, token, restrict) + } +} + +fn parse_port(text: &str) -> Option { + text.trim().parse::().ok().filter(|p| *p != 0) +} + +/// [`App::remove_current_port`] on given files: `current` is the discovery file, `file` the token +/// file of `port`. +fn remove_discovery(current: &Path, file: &Path, port: u16, token: &str) { + let named = std::fs::read_to_string(current).ok().filter(|t| t.len() <= 64).and_then(|t| parse_port(&t)); + if named != Some(port) || held_by_another(file, token) { + return; + } + let _ = std::fs::remove_file(current); + let _ = std::fs::remove_file(pid_path(current)); +} + +/// A client token and, when it came from a token file, where that file is. +pub struct ClientToken { + pub token: String, + pub file: Option, + /// The app it is for, named in an explanation. + pub app: App, +} + +/// The token file a client read: its path, when it was written and the process that wrote it. +pub struct TokenSource { + pub path: PathBuf, + pub modified: Option, + /// From `.pid`: the writer's process id and, on Linux, its start time. + pub writer: Option<(u32, Option)>, +} + +impl ClientToken { + /// The app's `error`, and for a refused token read from a file, which file it was, when it + /// was written and whether the instance that wrote it has exited. Never the token itself. + pub fn explain_error(&self, error: &str) -> String { + let Some(src) = &self.file else { return error.to_string() }; + if !error.starts_with("unauthorized") { + return error.to_string(); + } + let written = src + .modified + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map(|t| (self.app.utc)(t.as_secs())) + .unwrap_or_else(|| "at an unknown time".into()); + let age = src.modified.and_then(|t| t.elapsed().ok()).map(|d| format!(" ({} s ago)", d.as_secs())).unwrap_or_default(); + let path = src.path.display(); + let name = self.app.name; + match src.writer.filter(|(pid, start)| writer_exited(*pid, *start)) { + Some((pid, _)) => format!( + "{error}; stale token file from an instance that exited (process {pid}): {path}, written {written}{age}; delete that file or restart {name}" + ), + None => format!("{error}; token from {path}, written {written}{age}; if {name} was force-quit, restart it or delete that file"), + } + } +} + +/// Files bigger than this are not token files. +const MAX_TOKEN_FILE: u64 = 4096; + +fn read_token_file(path: &Path, app: App) -> Option { + let f = std::fs::File::open(path).ok()?; + let modified = f.metadata().ok().and_then(|m| m.modified().ok()); + let mut text = String::new(); + f.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?; + if text.len() as u64 > MAX_TOKEN_FILE { + return None; + } + let token = text.trim(); + if token.is_empty() { + return None; + } + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let writer = read_pid_file(&pid_path(&path)); + Some(ClientToken { token: token.to_string(), file: Some(TokenSource { path, modified, writer }), app }) +} + +/// The process file beside the token file `path`: `7878.token` → `7878.pid`. +pub fn pid_path(path: &Path) -> PathBuf { + path.with_extension("pid") +} + +/// ` []`; anything else (or no file) is no information. +fn read_pid_file(path: &Path) -> Option<(u32, Option)> { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(256).read_to_string(&mut text).ok()?; + let mut fields = text.split_whitespace(); + let pid = fields.next()?.parse::().ok().filter(|&pid| pid > 0)?; + Some((pid, fields.next().and_then(|s| s.parse().ok()))) +} + +/// Whether the process that wrote a token file is known to be gone: not running, or its id now +/// belongs to a process that started at another time. Unknown counts as running. +fn writer_exited(pid: u32, start: Option) -> bool { + process_running(pid) == Some(false) || start.zip(process_start(pid)).is_some_and(|(saved, now)| saved != now) +} + +/// Whether process `pid` runs; `None` when that can't be found out. +fn process_running(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + if !Path::new("/proc/self").exists() { + return None; + } + match std::fs::metadata(format!("/proc/{pid}")) { + Ok(_) => Some(true), + Err(e) if e.kind() == ErrorKind::NotFound => Some(false), + Err(_) => None, + } + } + #[cfg(all(unix, not(target_os = "linux")))] + { + let output = std::process::Command::new("ps").args(["-p", &pid.to_string(), "-o", "pid="]).output().ok()?; + if output.status.success() { + return Some(true); + } + (output.status.code() == Some(1) && output.stdout.iter().all(u8::is_ascii_whitespace)).then_some(false) + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + let output = std::process::Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .creation_flags(0x0800_0000) + .output() + .ok()?; + if !output.status.success() || output.stdout.is_empty() { + return None; + } + // `"image","pid",…` per process; with no match, a message line without a pid. + let text = String::from_utf8_lossy(&output.stdout); + Some(text.lines().any(|line| line.split("\",\"").nth(1).and_then(|s| s.trim_matches('"').parse::().ok()) == Some(pid))) + } + #[cfg(not(any(unix, windows)))] + { + let _ = pid; + None + } +} + +/// Linux: the process's start time (clock ticks after boot), which tells a reused id apart. +fn process_start(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let mut stat = String::new(); + std::fs::File::open(format!("/proc/{pid}/stat")).ok()?.take(4096).read_to_string(&mut stat).ok()?; + stat.rsplit_once(')')?.1.split_whitespace().nth(19)?.parse().ok() + } + #[cfg(not(target_os = "linux"))] + { + let _ = pid; + None + } +} + +/// `a == b`, taking the same time wherever they differ. +pub fn tokens_match(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// [`App::write_token_file`], with `restrict` limiting the new file to the user (Windows). +fn write_token_file_with(path: &Path, token: &str, restrict: impl FnOnce(&Path) -> Result<()>) -> Result<()> { + let dir = path.parent().ok_or_else(|| Error::new(ErrorKind::InvalidInput, "no folder"))?; + std::fs::create_dir_all(dir)?; + let _ = std::fs::remove_file(path); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + opts.mode(0o600); + } + let opened = opts.open(path)?; + // On a refusal the file goes with the closure, so it is closed before it is removed. + let mut f = match restrict(path).map(|()| opened) { + Ok(f) => f, + Err(e) => { + let _ = std::fs::remove_file(path); + return Err(e); + } + }; + f.write_all(token.as_bytes())?; + f.write_all(b"\n")?; + f.sync_all()?; + let pid = std::process::id(); + let line = process_start(pid).map_or_else(|| format!("{pid}\n"), |start| format!("{pid} {start}\n")); + let _ = std::fs::write(pid_path(path), line); + Ok(()) +} + +/// Remove a token file this launch wrote, and its process file, while it still holds `token`. +/// Another launch may have written the same file since (a second window on the same port, +/// #8794); its file stays. The process id can't tell the two apart, as every Flatpak instance +/// has the same one, but each launch's token is its own. +pub fn remove_token_file(path: &Path, token: &str) { + if read_token(path).is_some_and(|held| tokens_match(&held, token.trim())) { + let _ = std::fs::remove_file(path); + let _ = std::fs::remove_file(pid_path(path)); + } +} + +/// The token a token file holds, trimmed; `None` when there is no readable token file. +fn read_token(path: &Path) -> Option { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?; + let token = text.trim(); + (text.len() as u64 <= MAX_TOKEN_FILE && !token.is_empty()).then(|| token.to_string()) +} + +/// Whether the token file at `path` holds a token other than `token`: another launch's. +fn held_by_another(path: &Path, token: &str) -> bool { + read_token(path).is_some_and(|held| !tokens_match(&held, token.trim())) +} + +/// How long `icacls` may take (#6145). It finishes at once; on some computers the new process +/// is left suspended (security software holding it), and waiting on it without a limit froze +/// the app. +#[cfg(windows)] +const ICACLS_LIMIT: std::time::Duration = std::time::Duration::from_secs(5); + +/// How a run of `icacls` ended. +#[cfg(any(windows, test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum IcaclsEnd { + Succeeded, + Failed, + /// It didn't finish within [`ICACLS_LIMIT`] and was ended. + TimedOut, +} + +/// Whether a token may go into a file after `icacls` ended this way. `custom_folder`: the file is +/// in the folder the app's `CONTROL_DIR` names. A timeout is accepted only in the default folder, +/// the user's own local app data, which other users can't read by default; nothing is known about +/// the permissions of any other folder. +#[cfg(any(windows, test))] +fn restriction_result(end: IcaclsEnd, custom_folder: bool, path: &Path) -> Result<()> { + match end { + IcaclsEnd::Succeeded => Ok(()), + IcaclsEnd::TimedOut if !custom_folder => Ok(()), + IcaclsEnd::TimedOut => Err(Error::new(ErrorKind::TimedOut, format!("icacls did not finish in time to restrict {}", path.display()))), + IcaclsEnd::Failed => Err(Error::other(format!("icacls failed to restrict {}", path.display()))), + } +} + +/// Windows: drop inherited entries and grant only the current user full access. An `icacls` +/// that doesn't finish in time is ended, and [`restriction_result`] decides what that means. +#[cfg(windows)] +fn restrict_to_user(path: &Path, custom_folder: bool) -> Result<()> { + use std::os::windows::process::CommandExt; + let user = std::env::var("USERNAME").map_err(|_| Error::other("USERNAME is not set"))?; + let domain = std::env::var("USERDOMAIN").ok().filter(|d| !d.is_empty()); + let who = domain.map_or(user.clone(), |d| format!("{d}\\{user}")); + let child = std::process::Command::new("icacls") + .arg(path) + .args(["/inheritance:r", "/grant:r", &format!("{who}:F")]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .creation_flags(0x0800_0000) + .spawn()?; + let end = match wait_within(child, ICACLS_LIMIT)? { + Some(status) if status.success() => IcaclsEnd::Succeeded, + Some(_) => IcaclsEnd::Failed, + None => IcaclsEnd::TimedOut, + }; + restriction_result(end, custom_folder, path) +} + +/// Wait for `child` at most `limit`: its exit status, or `None` (and the child ended) when it +/// hasn't finished by then. +#[cfg(any(windows, test))] +fn wait_within(mut child: std::process::Child, limit: std::time::Duration) -> Result> { + let start = std::time::Instant::now(); + loop { + if let Some(status) = child.try_wait()? { + return Ok(Some(status)); + } + if start.elapsed() >= limit { + // Not waited for afterwards: a suspended process may not end at once. + let _ = child.kill(); + return Ok(None); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +/// Removes the token file (path, token) when the app ends, while it still holds that token. +pub struct TokenFile(pub PathBuf, pub String); + +impl Drop for TokenFile { + fn drop(&mut self) { + remove_token_file(&self.0, &self.1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An app with no data folder and no environment, whose times read `t`. + const TEST: App = App { name: "TestCraft", data_dir: || None, env: |_| None, utc: |secs| format!("t{secs}"), default_port: 27878 }; + + /// #6145: a child that doesn't finish in time is given up on, not waited for forever. + #[cfg(unix)] + #[test] + fn a_slow_child_is_waited_for_only_so_long() { + use std::time::{Duration, Instant}; + let start = Instant::now(); + let slow = std::process::Command::new("sleep").arg("30").spawn().unwrap(); + assert!(wait_within(slow, Duration::from_millis(200)).unwrap().is_none()); + assert!(start.elapsed() < Duration::from_secs(5), "{:?}", start.elapsed()); + let quick = std::process::Command::new("true").spawn().unwrap(); + assert!(wait_within(quick, Duration::from_secs(5)).unwrap().is_some_and(|s| s.success())); + let failing = std::process::Command::new("false").spawn().unwrap(); + assert!(wait_within(failing, Duration::from_secs(5)).unwrap().is_some_and(|s| !s.success())); + } + + /// #6145's freeze fix stays in the default folder, the user's own; a folder picked with + /// the app's `CONTROL_DIR` gets no token unless the file was really restricted. + #[test] + fn only_the_default_folder_accepts_an_icacls_that_timed_out() { + let path = Path::new("control/7878.token"); + for custom in [false, true] { + assert!(restriction_result(IcaclsEnd::Succeeded, custom, path).is_ok()); + assert!(restriction_result(IcaclsEnd::Failed, custom, path).is_err()); + } + assert!(restriction_result(IcaclsEnd::TimedOut, false, path).is_ok()); + let e = restriction_result(IcaclsEnd::TimedOut, true, path).unwrap_err(); + assert_eq!(e.kind(), ErrorKind::TimedOut); + assert!(e.to_string().contains("7878.token"), "{e}"); + } + + /// A file that couldn't be restricted is removed, empty and with no process file beside it, and + /// the error goes to the caller. A restricted one gets its token. + #[test] + fn a_token_file_that_could_not_be_restricted_is_removed() { + let dir = std::env::temp_dir().join(format!("craft-control-restrict-{}", std::process::id())); + let path = dir.join("7879.token"); + let e = write_token_file_with(&path, "secret-token", |p| { + assert_eq!(std::fs::read(p).unwrap(), b"", "restricted before the token goes in"); + restriction_result(IcaclsEnd::TimedOut, true, p) + }) + .unwrap_err(); + assert_eq!(e.kind(), ErrorKind::TimedOut); + assert!(!path.exists() && !pid_path(&path).exists()); + write_token_file_with(&path, "secret-token", |p| restriction_result(IcaclsEnd::TimedOut, false, p)).unwrap(); + assert_eq!(read_token(&path).as_deref(), Some("secret-token")); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn tokens_compare_whole() { + assert!(tokens_match("abc123", "abc123")); + assert!(!tokens_match("abc123", "abc124")); + assert!(!tokens_match("abc123", "abc12")); + assert!(!tokens_match("", "a")); + } + + /// The preference starts the listener by default, off it doesn't, and a port given on the + /// command line or in the environment wins over it either way. + #[test] + fn the_preference_decides_unless_a_port_is_given() { + assert_eq!(plan(None, true), Some(Listen::Default)); + assert_eq!(plan(None, false), None); + assert_eq!(plan(Some(7878), false), Some(Listen::Exact(7878))); + assert_eq!(plan(Some(7878), true), Some(Listen::Exact(7878))); + } + + /// A taken port falls through to the next free one, on loopback only; a run of taken ports + /// is an error, not a hang. + #[test] + fn a_taken_port_falls_back_to_the_next_free_one() { + let first = bind_with_fallback(TEST.default_port, PORT_TRIES).unwrap(); + let a = first.local_addr().unwrap(); + assert!(a.ip().is_loopback() && (TEST.default_port..TEST.default_port + PORT_TRIES).contains(&a.port()), "{a}"); + let second = bind_with_fallback(a.port(), PORT_TRIES).unwrap(); + assert!(second.local_addr().unwrap().port() > a.port()); + assert!(bind_with_fallback(a.port(), 1).is_err()); + } + + #[test] + fn the_discovery_file_holds_a_port() { + assert_eq!(parse_port("27879\n"), Some(27879)); + assert_eq!(parse_port("0"), None); + assert_eq!(parse_port("70000"), None); + assert_eq!(parse_port("nope"), None); + let dir = std::env::temp_dir().join(format!("solidcraft-current-port-{}", std::process::id())); + let path = dir.join("current.port"); + TEST.write_token_file(&path, "27881").unwrap(); + assert_eq!(parse_port(&std::fs::read_to_string(&path).unwrap()), Some(27881)); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A one-line token file (and one with no process file) still reads. + #[test] + fn a_token_file_reads_with_and_without_its_process_file() { + let dir = std::env::temp_dir().join(format!("craft-control-read-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("7879.token"); + std::fs::write(&path, "legacy-token\n").unwrap(); + let client = read_token_file(&path, TEST).unwrap(); + assert_eq!(client.token, "legacy-token"); + let src = client.file.as_ref().unwrap(); + assert_eq!(src.path, std::fs::canonicalize(&path).unwrap()); + assert!(src.modified.is_some() && src.writer.is_none()); + std::fs::write(pid_path(&path), "not a pid\n").unwrap(); + assert!(read_token_file(&path, TEST).unwrap().file.unwrap().writer.is_none()); + std::fs::write(&path, "x".repeat(4097)).unwrap(); + assert!(read_token_file(&path, TEST).is_none()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A refused token's explanation names the app and writes the time as the app does. + #[test] + fn an_explanation_names_the_app() { + let file = Some(TokenSource { path: PathBuf::from("7878.token"), modified: Some(UNIX_EPOCH), writer: None }); + let client = ClientToken { token: "secret-token".into(), file, app: TEST }; + let error = client.explain_error("unauthorized"); + assert!(error.contains("written t0 (") && error.contains("if TestCraft was force-quit"), "{error}"); + assert!(!error.contains("secret-token"), "{error}"); + let explicit = TEST.client_token(0, Some("explicit-token")).unwrap(); + assert!(explicit.token == "explicit-token" && explicit.file.is_none()); + assert_eq!(explicit.explain_error("unauthorized"), "unauthorized"); + } + + /// A process file naming a process that has exited marks the token file stale. + /// Where the process inspector cannot run (a sandboxed test runner), the exit + /// is unknown, and the explanation must not claim it. + #[cfg(any(unix, windows))] + #[test] + fn a_token_file_from_an_exited_process_is_stale() { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "auth::tests::tokens_compare_whole"]) + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + assert!(child.wait().unwrap().success()); + let source = |writer| Some(TokenSource { path: PathBuf::from("7878.token"), modified: None, writer }); + let stale = ClientToken { token: "secret-token".into(), file: source(Some((pid, None))), app: TEST }; + let error = stale.explain_error("unauthorized"); + match process_running(pid) { + Some(false) => { + assert!(error.contains(&format!("stale token file from an instance that exited (process {pid}): 7878.token")), "{error}"); + } + None => assert!(!error.contains("stale") && error.contains("restart it or delete that file"), "{error}"), + Some(true) => panic!("the reaped child {pid} is reported as running"), + } + assert!(!error.contains("secret-token")); + let live = ClientToken { token: "t".into(), file: source(Some((std::process::id(), process_start(std::process::id())))), app: TEST }; + assert!(!live.explain_error("unauthorized").contains("stale")); + #[cfg(target_os = "linux")] + { + let reused = process_start(std::process::id()).map(|s| s + 1); + let reused = ClientToken { token: "t".into(), file: source(Some((std::process::id(), reused))), app: TEST }; + assert!(reused.explain_error("unauthorized").contains("stale token file")); + } + } + + /// The token file is the user's alone, and a client reads it back. + #[cfg(unix)] + #[test] + fn the_token_file_is_private_to_the_user() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("craft-control-auth-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let path = dir.join("control/7878.token"); + std::fs::create_dir_all(dir.join("control")).unwrap(); + std::fs::write(&path, "stale").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + TEST.write_token_file(&path, "0123456789abcdef0123456789abcdef").unwrap(); + let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&path), 0o600); + assert_eq!(mode(path.parent().unwrap()), 0o700); + assert_eq!(std::fs::read_to_string(&path).unwrap().trim(), "0123456789abcdef0123456789abcdef"); + let writer = read_token_file(&path, TEST).unwrap().file.unwrap().writer; + assert_eq!(writer.map(|(pid, _)| pid), Some(std::process::id())); + drop(TokenFile(path.clone(), "0123456789abcdef0123456789abcdef".into())); + assert!(!path.exists() && !pid_path(&path).exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// #8794: under Flatpak two windows can listen on the same port number (each has its own + /// loopback) and write the same files. The one closing first leaves the other's token, process + /// and discovery files; the last one takes them all away, `current.pid` too. + #[test] + fn a_closing_launch_leaves_another_launchs_files() { + let dir = std::env::temp_dir().join(format!("craft-control-two-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let (file, current) = (dir.join("27878.token"), dir.join("current.port")); + let (first, second) = ("a".repeat(64), "b".repeat(64)); + // The first window, then the second on the same port, each writing both files. + for token in [&first, &second] { + TEST.write_token_file(&file, token).unwrap(); + TEST.write_token_file(¤t, "27878").unwrap(); + } + // The first closes: the second's files stay. + remove_token_file(&file, &first); + remove_discovery(¤t, &file, 27878, &first); + assert_eq!(read_token(&file).as_deref(), Some(second.as_str())); + assert!(pid_path(&file).exists() && current.exists() && pid_path(¤t).exists()); + // Another port named in the discovery file is another app's too. + remove_discovery(¤t, &file, 27879, &second); + assert!(current.exists()); + // The second closes: nothing of either is left. + remove_token_file(&file, &format!(" {second}\n")); + remove_discovery(¤t, &file, 27878, &second); + assert!(!file.exists() && !pid_path(&file).exists() && !current.exists() && !pid_path(¤t).exists()); + // With no token file left (removed by hand, or never written), the discovery file of the + // port still goes; a missing file is no error. + TEST.write_token_file(¤t, "27878").unwrap(); + remove_token_file(&file, &second); + remove_discovery(¤t, &file, 27878, &second); + assert!(!current.exists()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/control/src/cancel.rs b/crates/control/src/cancel.rs new file mode 100644 index 000000000..e03052307 --- /dev/null +++ b/crates/control/src/cancel.rs @@ -0,0 +1,118 @@ +//! Where a control request is, so the connection that sent it can cancel it (`control.cancel`, +//! #1989): shared between the window, which moves it along, and the connection. Whoever changes +//! it first wins. + +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::{Mutex, PoisonError}; + +/// A request's place in the app, and how to stop the work computing it. +#[derive(Default)] +pub struct Ticket { + state: AtomicU8, + /// It is an export computing on a copy, whose worker writes its file. + writes_file: AtomicBool, + /// Tells its worker to stop: set by the window when it hands the request to one. + stopper: Mutex>>, +} + +impl std::fmt::Debug for Ticket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Ticket").field("state", &self.state).field("writes_file", &self.writes_file).finish_non_exhaustive() + } +} + +/// Waiting its turn (sent, or held behind another request). +const WAITING: u8 = 0; +/// Running on the window thread (or handed to something that can't stop it, such as the file +/// loader). +const RUNNING: u8 = 1; +/// Computing off the window thread. +const COMPUTING: u8 = 2; +const CANCELLED: u8 = 3; + +/// What [`Ticket::cancel`] did. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Cancel { + /// It was waiting its turn: it will not run. + Removed, + /// It was computing off the window thread: it is answered as stopped and nothing of it is + /// applied. An export's worker may still write its file. + Stopped { writes_file: bool }, + /// It runs on the window thread: it will finish. + Running, + /// It was cancelled or stopped before. + Already, +} + +impl Ticket { + /// Cancel the request, unless it already runs on the window thread. + pub fn cancel(&self) -> Cancel { + let mut state = self.state.load(Ordering::Acquire); + loop { + let done = match state { + WAITING => Cancel::Removed, + COMPUTING => Cancel::Stopped { writes_file: self.writes_file.load(Ordering::Acquire) }, + CANCELLED => return Cancel::Already, + _ => return Cancel::Running, + }; + match self.state.compare_exchange(state, CANCELLED, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => { + self.stop_worker(); + return done; + } + Err(now) => state = now, + } + } + } + + pub fn is_cancelled(&self) -> bool { + self.state.load(Ordering::Acquire) == CANCELLED + } + + /// The window takes the request to run it: false when it was cancelled. + pub fn take(&self) -> bool { + self.moves(WAITING, RUNNING) + } + + /// The work computing it stops on `stop` (a worker's own stop flag). Cancelled already, it is + /// stopped at once. + pub fn on_stop(&self, stop: impl Fn() + Send + Sync + 'static) { + *self.stopper.lock().unwrap_or_else(PoisonError::into_inner) = Some(Box::new(stop)); + if self.is_cancelled() { + self.stop_worker(); + } + } + + /// The window hands it to a worker (`writes_file`: an export on a copy). + pub fn computing(&self, writes_file: bool) { + self.writes_file.store(writes_file, Ordering::Release); + let _ = self.moves(RUNNING, COMPUTING) || self.moves(WAITING, COMPUTING); + } + + /// Its worker is done and it runs on the window thread now (or it never left): false when it + /// was cancelled meanwhile. Only the window moves it on from running, so running stays running. + pub fn back(&self) -> bool { + self.moves(COMPUTING, RUNNING) || self.state.load(Ordering::Acquire) == RUNNING + } + + /// Computed, but it waits its turn again (a file opening meanwhile): false when cancelled. + pub fn wait_again(&self) -> bool { + self.moves(COMPUTING, WAITING) + } + + /// The person at the app stopped it (Stop): its worker stops too. + pub fn stop(&self) { + self.state.store(CANCELLED, Ordering::Release); + self.stop_worker(); + } + + fn stop_worker(&self) { + if let Some(stop) = self.stopper.lock().unwrap_or_else(PoisonError::into_inner).as_ref() { + stop(); + } + } + + fn moves(&self, from: u8, to: u8) -> bool { + self.state.compare_exchange(from, to, Ordering::AcqRel, Ordering::Acquire).is_ok() + } +} diff --git a/crates/control/src/client.rs b/crates/control/src/client.rs new file mode 100644 index 000000000..cbad78850 --- /dev/null +++ b/crates/control/src/client.rs @@ -0,0 +1,616 @@ +//! The control channel's client (docs/control-protocol.md): a connection to an app's loopback +//! control port, with the app's token ([`App::client_token`]) on every request, one bounded +//! round trip per request, a fresh connection after one is lost, and the wait for a request the +//! app says is still running (#1618), through `control.pending`. Its errors tell an app that is +//! not running from one that went away during a call ([`CallError`]). What a request means, and +//! anything an app adds to its parameters, is the caller's. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::Effect; +use crate::auth::App; + +/// Longest reply line accepted from the app. +pub const MAX_REPLY: u64 = 64 << 20; + +/// How long a reply may take: kernel operations (booleans, fillets) on big parts can take a while. +pub const READ_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long the app may take to answer a request sent [`Request::once`]: a little longer than its +/// control server waits for a long request (3600 s), so its "still running" reply arrives first. +pub const ONCE_TIMEOUT: Duration = Duration::from_secs(3720); + +/// How long connecting to the app may take. +const CONNECT_TIMEOUT: Duration = Duration::from_millis(800); + +/// How often a long wait is reported ([`Connection::with_progress`]). +const PROGRESS_EVERY: Duration = Duration::from_secs(30); + +/// How an error starts when a request is still running in the app after the wait: the request +/// was not cancelled and its reply waits on the connection; [`Connection::pending`] gets it. +pub use crate::STILL_RUNNING; + +/// How long to wait for a request the app says is still running. +#[derive(Clone, Copy, Debug)] +pub struct Waits { + /// How long a call goes on waiting, asking `control.pending`, after the app said its request + /// is still running (the app says so after 60 s), before it says so itself. + pub pending: Duration, + /// How long a later call, or [`Connection::pending`], waits for a request still running from + /// an earlier call. + pub settle: Duration, + /// How often `control.pending` is asked while waiting. + pub poll: Duration, +} + +impl Default for Waits { + fn default() -> Self { + Waits { pending: Duration::from_secs(240), settle: Duration::from_secs(60), poll: Duration::from_millis(500) } + } +} + +/// Why a call has no result. Each holds the message for the caller (an agent reads it). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CallError { + /// Nothing listens at the address: the app is not running, or it quit before the call was + /// sent. The message says how to start it. + NotRunning(String), + /// The app went away or did not answer during the call, after running it or not. + Lost(String), + /// The request is still running in the app after the wait (it starts with [`STILL_RUNNING`]), + /// or an earlier one is and this one was not sent. + StillRunning(String), + /// The app answered with an error (a refused token says which token file it came from). + Refused(String), + /// The reply was not a control reply. + BadReply(String), +} + +impl CallError { + /// The message. + pub fn message(&self) -> &str { + match self { + CallError::NotRunning(m) | CallError::Lost(m) | CallError::StillRunning(m) | CallError::Refused(m) | CallError::BadReply(m) => m, + } + } +} + +impl std::fmt::Display for CallError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message()) + } +} + +impl std::error::Error for CallError {} + +impl From for String { + fn from(e: CallError) -> String { + e.message().to_string() + } +} + +/// A request: its method and parameters, how messages name it and what it does if it is still +/// running when its call returns. +#[derive(Clone, Debug)] +pub struct Request { + pub method: String, + pub params: Value, + /// The request in messages ("`engine.execute` (solid.fillet)"). + pub what: String, + /// What it still does when it finishes after its wait. + pub effect: Effect, + /// Never sent twice: on a fresh connection, waiting up to [`ONCE_TIMEOUT`], and once it is + /// being written any error is final, as the app may have run it. Otherwise a request that + /// finds its connection lost is sent once more on a fresh one (the app may have restarted). + pub once: bool, +} + +impl Request { + /// `method` with `params`, named "`method`" in messages, taken to change the document, sent + /// again once on a fresh connection when its connection was lost. + pub fn new(method: &str, params: Value) -> Self { + Request { method: method.to_string(), params, what: format!("`{method}`"), effect: Effect::Changes, once: false } + } +} + +/// A control reply as a call's result: its `result`, or its `error`. +pub fn outcome(v: &Value) -> Result { + if v.get("ok").and_then(Value::as_bool) == Some(true) { + Ok(v.get("result").cloned().unwrap_or(Value::Null)) + } else { + Err(v.get("error").and_then(Value::as_str).unwrap_or("error").to_string()) + } +} + +/// A request the app said is still running. Its reply will come on this connection, through +/// `control.pending`, so the connection is kept for it. +struct Outstanding { + what: String, + /// What it does when it finishes; `None`: not known (a request this client did not send). + effect: Option, + since: Instant, +} + +impl Outstanding { + fn secs(&self) -> u64 { + self.since.elapsed().as_secs() + } +} + +/// What became of a request still running when its call returned, once a later call waited it +/// out: kept for [`Connection::pending`]. +enum Settled { + Finished { what: String, reply: Value }, + Lost { what: String }, +} + +/// The end of a wait for an outstanding request. +enum Waited { + /// It finished: its own reply. + Done(Value), + /// The app holds nothing (any more) for this connection. + Nothing, + /// Still running when the wait ran out. + Running, +} + +/// A connection to the app listening at an address. It connects when a call needs it, so a +/// client started before the app reaches it once it runs. +pub struct Connection { + app: App, + /// What the app calls its document in messages ("design"). + document: &'static str, + addr: String, + port: u16, + conn: Option<(BufReader, TcpStream)>, + /// Counts the connections made, so a caller can tell a fresh one from the one it asked. + serial: u64, + next_id: u64, + /// The token given, when one was. Otherwise each request looks the token up again, since + /// every launch of the app makes a new one. + explicit_token: Option, + /// A request still running in the app; its reply waits on `conn`. + outstanding: Option, + /// An outstanding request's end, once a later call waited for it (for `pending`). + settled: Option, + waits: Waits, + /// Where a long wait is reported. + progress: fn(&str), +} + +impl Connection { + /// A connection to `app` at `addr` (`host:port`), with `token` when given, else the app's + /// token found the usual way ([`App::client_token`]); not connected yet. + pub fn new(app: App, addr: &str, token: Option<&str>) -> Self { + let port = addr.rsplit(':').next().and_then(|p| p.parse::().ok()).unwrap_or(0); + Self { + app, + document: "design", + addr: addr.to_string(), + port, + conn: None, + serial: 0, + next_id: 1, + explicit_token: token.map(str::to_string), + outstanding: None, + settled: None, + waits: Waits::default(), + progress: |_| {}, + } + } + + /// What the app calls its document in messages ("design", the default). + #[must_use] + pub fn with_document(mut self, document: &'static str) -> Self { + self.document = document; + self + } + + /// Report a long wait for a request still running to `progress`, every 30 s. + #[must_use] + pub fn with_progress(mut self, progress: fn(&str)) -> Self { + self.progress = progress; + self + } + + /// How long to wait for a request still running. + pub fn set_waits(&mut self, waits: Waits) { + self.waits = waits; + } + + /// The address it connects to. + pub fn addr(&self) -> &str { + &self.addr + } + + /// The port of the address (0 when it names none). + pub fn port(&self) -> u16 { + self.port + } + + /// The token given, if one was. + pub fn explicit_token(&self) -> Option<&str> { + self.explicit_token.as_deref() + } + + /// Whether a connection is open now. + pub fn is_connected(&self) -> bool { + self.conn.is_some() + } + + /// The number of connections made so far: the same number while the same connection stays + /// open. + pub fn serial(&self) -> u64 { + self.serial + } + + /// Whether a request of this connection is still running in the app. + pub fn has_outstanding(&self) -> bool { + self.outstanding.is_some() + } + + /// Close the connection. A request still running loses its reply with it. + pub fn disconnect(&mut self) { + self.conn = None; + if let Some(o) = self.outstanding.take() { + self.settled = Some(Settled::Lost { what: o.what }); + } + } + + /// Connect afresh. Failing to reach the app at all is `NotConnected`, so [`Connection::explain`] + /// can tell it from a connection lost during a call. + /// + /// # Errors + /// `NotConnected` when nothing listens at the address (or it does not resolve). + pub fn connect(&mut self) -> std::io::Result<()> { + self.disconnect(); + let not_connected = |e: std::io::Error| std::io::Error::new(std::io::ErrorKind::NotConnected, e); + let mut last = std::io::Error::new(std::io::ErrorKind::NotFound, format!("cannot resolve {}", self.addr)); + for sa in self.addr.to_socket_addrs().map_err(not_connected)? { + match TcpStream::connect_timeout(&sa, CONNECT_TIMEOUT) { + Ok(s) => { + s.set_nodelay(true).ok(); + s.set_read_timeout(Some(READ_TIMEOUT)).ok(); + let read = s.try_clone()?; + self.conn = Some((BufReader::new(read), s)); + self.serial += 1; + return Ok(()); + } + Err(e) => last = e, + } + } + Err(not_connected(last)) + } + + /// See whether the app is there now; the error says why not. The connection doesn't stay + /// open: the app closes one that shows no token within a few seconds, and the first call may + /// be long away. That call connects again. + /// + /// # Errors + /// As [`Connection::connect`]. + pub fn try_connect(&mut self) -> std::io::Result<()> { + self.connect()?; + self.disconnect(); + Ok(()) + } + + /// The error for a failed exchange: whether the app is not running (and how to start it) or + /// went away during the call. + pub fn explain(&self, e: &std::io::Error) -> CallError { + let (addr, port, name, document) = (&self.addr, self.port, self.app.name, self.document); + let start = format!("{} --control {port}", name.to_lowercase()); + if e.kind() == std::io::ErrorKind::NotConnected { + CallError::NotRunning(format!( + "no {name} app is listening at {addr} ({e}): it is not running, or it quit. Start it with `{start}` (or ask the user to); the \ + next call connects to it" + )) + } else { + CallError::Lost(format!( + "the {name} app at {addr} did not answer ({e}): it may have quit or crashed during this call, after running it or not. Inspect \ + the {document} before sending a change again; if the app quit, start it with `{start}`" + )) + } + } + + /// The request line for `method`: a fresh id and the app's token (looked up again for every + /// request: an app restarted mid-session has a new one). + pub fn request_line(&mut self, method: &str, params: Value) -> String { + let id = self.next_id; + self.next_id += 1; + let mut msg = json!({"id": id, "method": method, "params": params}); + if let Some(t) = self.app.client_token(self.port, self.explicit_token.as_deref()) { + msg["token"] = json!(t.token); + } + msg.to_string() + } + + /// One request line out and one reply line back, at most [`MAX_REPLY`] long, connecting + /// first when not connected. Any failure closes the connection (a timed-out read leaves a + /// late reply on the stream). + fn roundtrip(&mut self, line: &str) -> std::io::Result { + if self.conn.is_none() { + self.connect()?; + } + let Some((reader, writer)) = self.conn.as_mut() else { return Err(std::io::Error::other("not connected")) }; + // One write: a request that loses its newline still reaches the app as a request. + let sent = writer.write_all(format!("{line}\n").as_bytes()).and_then(|()| writer.flush()); + if let Err(e) = sent { + self.disconnect(); + return Err(e); + } + let mut reply = String::new(); + let n = match reader.by_ref().take(MAX_REPLY).read_line(&mut reply) { + Ok(n) => n, + Err(e) => { + self.disconnect(); + return Err(e); + } + }; + if n == 0 || !reply.ends_with('\n') { + self.disconnect(); + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "connection closed")); + } + Ok(reply) + } + + fn set_read_timeout(&self, t: Duration) { + if let Some((_, writer)) = &self.conn { + writer.set_read_timeout(Some(t)).ok(); + } + } + + /// A reply line as JSON. + fn parse(reply: &str) -> Result { + serde_json::from_str(reply).map_err(|e| CallError::BadReply(format!("bad reply from the app: {e}"))) + } + + /// A control reply as a call's result; a refused token says which token file it came from + /// (and whether it is stale). + /// + /// # Errors + /// [`CallError::Refused`] with the app's error. + pub fn outcome(&self, v: &Value) -> Result { + outcome(v).map_err(|error| { + CallError::Refused( + self.app.client_token(self.port, self.explicit_token.as_deref()).map_or_else(|| error.clone(), |t| t.explain_error(&error)), + ) + }) + } + + /// `method` with `params`, at once and once: not behind a request still running on this + /// connection, no second try. The app's whole reply (`ok`, `result` or `error`, and anything + /// it adds), for a request the app answers at once on any connection + /// (`control.capabilities`, `control.running`). + /// + /// # Errors + /// As [`Connection::explain`]; [`CallError::BadReply`]. + pub fn ask(&mut self, method: &str, params: Value) -> Result { + let line = self.request_line(method, params); + let reply = self.roundtrip(&line).map_err(|e| self.explain(&e))?; + Self::parse(&reply) + } + + /// `method` with `params`, waiting at most `timeout` for the reply and never behind a request + /// still running: for a request the app answers within a frame, asked again later when the + /// app is busy. One try. + /// + /// # Errors + /// [`CallError::StillRunning`] when a request of this connection is still running; as + /// [`Connection::ask`] and [`Connection::outcome`]. + pub fn quick(&mut self, method: &str, params: Value, timeout: Duration) -> Result { + if self.outstanding.is_some() { + return Err(CallError::StillRunning("a request is still running in the app".into())); + } + if self.conn.is_none() { + self.connect().map_err(|e| self.explain(&e))?; + } + let line = self.request_line(method, params); + self.set_read_timeout(timeout); + let reply = self.roundtrip(&line); + self.set_read_timeout(READ_TIMEOUT); + let reply = reply.map_err(|e| self.explain(&e))?; + let v = Self::parse(&reply)?; + self.outcome(&v) + } + + /// Ask the app, on this connection only, about the request it still runs for it. + fn ask_pending(&mut self) -> Result { + if self.conn.is_none() { + self.disconnect(); + return Err(CallError::Lost("the connection to the app was lost".into())); + } + let v = self.ask("control.pending", json!({}))?; + outcome(&v).map_err(CallError::Refused).inspect_err(|_| self.disconnect()) + } + + /// `control.cancel` for the request still running on this connection. Sent at once, without + /// waiting for that request first, and never on a fresh connection, which has nothing to + /// cancel. Once the app no longer holds it (removed, stopped or finished), later calls don't + /// wait for it. + /// + /// # Errors + /// As [`Connection::ask`]; [`CallError::Refused`]. + pub fn cancel(&mut self, params: Value) -> Result { + if self.conn.is_none() { + return Ok(json!({"cancelled": null, "note": "nothing to cancel: no request of this client is running in the app"})); + } + let v = self.ask("control.cancel", params)?; + let r = outcome(&v).map_err(CallError::Refused); + if r.is_ok() { + self.outstanding = None; + } + r + } + + /// Wait up to `limit` for the outstanding request, asking `control.pending` now and then and + /// reporting how long it takes. An error: the connection, and so the reply, was lost. + fn wait_outstanding(&mut self, limit: Duration) -> Result { + let start = Instant::now(); + let mut told = start; + loop { + let Some(o) = &self.outstanding else { return Ok(Waited::Nothing) }; + let what = o.what.clone(); + let r = self.ask_pending()?; + if r.get("pending").is_none_or(Value::is_null) { + self.outstanding = None; + return Ok(Waited::Nothing); + } + if r.get("done").and_then(Value::as_bool) == Some(true) { + self.outstanding = None; + return Ok(Waited::Done(r.get("reply").cloned().unwrap_or(Value::Null))); + } + let waited = start.elapsed(); + if waited >= limit { + return Ok(Waited::Running); + } + if told.elapsed() >= PROGRESS_EVERY { + let secs = self.outstanding.as_ref().map_or(0, Outstanding::secs); + (self.progress)(&format!("{what} is still running in the app ({secs} s); waiting for it")); + told = Instant::now(); + } + std::thread::sleep(self.waits.poll.min(limit.saturating_sub(waited))); + } + } + + /// The error for the outstanding request still running after its wait. + fn still_running(&self, earlier: bool) -> CallError { + let Some(o) = &self.outstanding else { return CallError::StillRunning(format!("{STILL_RUNNING} a request is still running in the app")) }; + let document = self.document; + let effect = + o.effect.map_or_else(|| format!("it was not cancelled and may still change the {document} when it finishes"), |e| e.when_done(document)); + let settle = self.waits.settle.as_secs(); + CallError::StillRunning(if earlier { + format!( + "{STILL_RUNNING} an earlier request, {}, is still running in the app ({} s; {effect}), so this call was not sent. Call the \ + `pending` tool to wait for its result, then send this call again", + o.what, + o.secs() + ) + } else { + format!( + "{STILL_RUNNING} {} is still running in the app after {} s ({effect}); don't send it again. Call the `pending` tool to \ + wait for its result (up to {settle} s per call); other calls wait for it first", + o.what, + o.secs() + ) + }) + } + + /// Before a new request: wait for one still running from an earlier call, keeping its end for + /// `pending`. Still running after the wait: the new request is not sent. + fn settle_earlier(&mut self) -> Result<(), CallError> { + let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return Ok(()) }; + match self.wait_outstanding(self.waits.settle) { + Ok(Waited::Done(reply)) => self.settled = Some(Settled::Finished { what, reply }), + Ok(Waited::Nothing) => {} + Ok(Waited::Running) => return Err(self.still_running(true)), + // The connection was lost, and the earlier reply with it (`pending` says so): the new + // request goes on a fresh connection. + Err(_) => {} + } + Ok(()) + } + + /// `control.pending` for the client: the end of the request still running from an earlier + /// call, waiting for it up to the settle wait: `{pending: what | null, done?, reply?, lost?, + /// running_s?}`. Never sent on a fresh connection, which has nothing pending. + pub fn pending(&mut self) -> Value { + if let Some(s) = self.settled.take() { + return match s { + Settled::Finished { what, reply } => json!({"pending": what, "done": true, "reply": reply}), + Settled::Lost { what } => json!({"pending": what, "lost": true}), + }; + } + let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return json!({"pending": null}) }; + match self.wait_outstanding(self.waits.settle) { + Ok(Waited::Done(reply)) => json!({"pending": what, "done": true, "reply": reply}), + Ok(Waited::Nothing) => json!({"pending": null}), + Ok(Waited::Running) => json!({"pending": what, "done": false, "running_s": self.outstanding.as_ref().map_or(0, Outstanding::secs)}), + Err(_) => { + self.settled = None; + json!({"pending": what, "lost": true}) + } + } + } + + /// Send `request` and wait for its reply: the app's whole reply (`ok`, `result` or `error`, + /// and anything it adds). First a request still running from an earlier call is waited for + /// (except for `control.running`, which the app answers at once); a request the app says is + /// still running is waited for through `control.pending`; a request refused unrun as busy + /// behind one this client did not know of is sent once more after that one, when + /// `retry_busy`. + /// + /// # Errors + /// As [`Connection::explain`]: a request whose connection is lost is sent once more on a fresh + /// one (unless [`Request::once`]), and when nothing listens any more the app went away, maybe + /// while running it ([`CallError::Lost`]); [`CallError::StillRunning`]; [`CallError::BadReply`]. + pub fn exchange(&mut self, request: Request, retry_busy: bool) -> Result { + let Request { method, params, what, effect, once } = request; + if method != "control.running" { + self.settle_earlier()?; + } + let again = retry_busy.then(|| params.clone()); + let sent = Instant::now(); + let line = self.request_line(&method, params); + let reply = if once { + // A fresh connection (an idle one may have been closed by an app restart; connecting + // again sends nothing). + self.connect().or_else(|_| self.connect()).and_then(|()| { + self.set_read_timeout(ONCE_TIMEOUT); + let r = self.roundtrip(&line); + self.set_read_timeout(READ_TIMEOUT); + r + }) + } else { + self.roundtrip(&line).or_else(|lost| { + self.roundtrip(&line).map_err(|e| if e.kind() == std::io::ErrorKind::NotConnected && lost.kind() != e.kind() { lost } else { e }) + }) + } + .map_err(|e| self.explain(&e))?; + let v = Self::parse(&reply)?; + // Timed out in the app, still running: wait for its result on this connection. + if v.get("running").and_then(Value::as_bool) == Some(true) { + self.outstanding = Some(Outstanding { what: what.clone(), effect: Some(effect), since: sent }); + (self.progress)(&format!( + "{what} is still running in the app after {} s; waiting up to {} s more for it", + sent.elapsed().as_secs(), + self.waits.pending.as_secs() + )); + let document = self.document; + return match self.wait_outstanding(self.waits.pending)? { + Waited::Done(reply) => Ok(reply), + Waited::Nothing => Err(CallError::Lost(format!( + "{what} timed out in the app, which kept no result for it: inspect the {document} before sending it again" + ))), + Waited::Running => Err(self.still_running(false)), + }; + } + // Refused unrun behind a request this client did not know of: wait for that one, then + // send this one once more. + if v.get("busy").and_then(Value::as_bool) == Some(true) + && let Some(params) = again + { + self.outstanding = Some(Outstanding { what: "an earlier request".into(), effect: None, since: Instant::now() }); + self.settle_earlier()?; + return self.exchange(Request { method, params, what, effect, once }, false); + } + Ok(v) + } + + /// [`Connection::exchange`] (retrying a busy refusal once) and the reply's result. + /// + /// # Errors + /// As [`Connection::exchange`] and [`Connection::outcome`]. + pub fn call(&mut self, request: Request) -> Result { + let v = self.exchange(request, true)?; + self.outcome(&v) + } +} + +#[cfg(test)] +#[path = "client_tests.rs"] +mod tests; diff --git a/crates/control/src/client_tests.rs b/crates/control/src/client_tests.rs new file mode 100644 index 000000000..f5af19a00 --- /dev/null +++ b/crates/control/src/client_tests.rs @@ -0,0 +1,190 @@ +//! The client against the shared server ([`crate::server::Hub`]) on an ephemeral loopback port. + +use super::*; +use crate::ControlRequest; +use crate::server::{Hub, Protocol, Stop, Summary, Waits as ServerWaits}; +use std::ffi::OsString; +use std::net::TcpListener; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::mpsc::channel; + +const TOKEN: &str = "0123456789abcdef0123456789abcdef"; + +fn no_dir() -> Option { + None +} + +fn no_env(_: &str) -> Option { + None +} + +fn utc(secs: u64) -> String { + secs.to_string() +} + +/// An app with no token files and no environment: a client sends only the token it is given. +const APP: App = App { name: "Probe", data_dir: no_dir, env: no_env, utc, default_port: 0 }; + +/// An app whose requests all may change its document. +struct Probe; + +impl Protocol for Probe { + fn summary(&self, method: &str, _: &Value) -> Summary { + Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false } + } + fn document(&self) -> &'static str { + "design" + } + fn change_event(&self) -> &'static str { + "design.changed" + } + fn revision_method(&self) -> &'static str { + "document.revision" + } + fn docs(&self) -> &'static str { + "docs/control-protocol.md" + } + fn capabilities(&self) -> Value { + json!({}) + } +} + +/// The app on `port` (0: an ephemeral one) behind [`TOKEN`]: its window answers each request after +/// `delay` with the request's method and parameters, and the server says a request is still +/// running after `wait`. +fn serve(port: u16, delay: Duration, wait: Duration) -> Stop { + // A port just closed may take a moment to be free again on a loaded machine. + let started = Instant::now(); + let mut bound = TcpListener::bind(("127.0.0.1", port)); + while bound.is_err() && started.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(10)); + bound = TcpListener::bind(("127.0.0.1", port)); + } + let listener = bound.unwrap(); + let (tx, rx) = channel::(); + std::thread::spawn(move || { + for req in rx { + std::thread::sleep(delay); + let _ = req.reply.send(json!({"ok": true, "result": {"method": req.method, "params": req.params}})); + } + }); + let waits = ServerWaits { short: wait, long: wait, token: Duration::from_secs(5) }; + let hub = Arc::new(Hub::new(Probe, || {}, 1 << 20).with_waits(waits)); + hub.listen_stoppable(listener, TOKEN.into(), tx, false).unwrap() +} + +fn addr(port: u16) -> String { + format!("127.0.0.1:{port}") +} + +/// A port nothing listens on. +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() +} + +#[test] +fn a_call_is_one_round_trip_with_the_token() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + assert!(!c.is_connected()); + let r = c.call(Request::new("document.inspect", json!({"a": 1}))).unwrap(); + assert_eq!(r, json!({"method": "document.inspect", "params": {"a": 1}})); + assert!(c.is_connected()); + let serial = c.serial(); + // the next call goes on the same connection + c.call(Request::new("document.inspect", json!({}))).unwrap(); + assert_eq!(c.serial(), serial); + // `try_connect` sees the app there and leaves no connection open + let mut fresh = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + fresh.try_connect().unwrap(); + assert!(!fresh.is_connected()); +} + +#[test] +fn a_wrong_token_or_none_is_refused() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + for token in [Some("fedcba9876543210fedcba9876543210"), None] { + let mut c = Connection::new(APP, &addr(app.port()), token); + match c.call(Request::new("document.inspect", json!({}))) { + Err(CallError::Refused(e)) => assert!(e.starts_with("unauthorized"), "{token:?}: {e}"), + other => panic!("{token:?}: {other:?}"), + } + } +} + +#[test] +fn an_app_not_running_says_how_to_start_it() { + let port = free_port(); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + let CallError::NotRunning(m) = &e else { panic!("{e:?}") }; + assert!(m.starts_with(&format!("no Probe app is listening at 127.0.0.1:{port}")), "{m}"); + assert!(m.contains(&format!("`probe --control {port}`")), "{m}"); + assert_eq!(c.try_connect().unwrap_err().kind(), std::io::ErrorKind::NotConnected); +} + +/// An app that goes away during a call is lost, not "not running"; the next call says it is not +/// running, and once it runs again the next call reaches it. +#[test] +fn an_app_gone_during_a_call_is_lost_and_reached_again_once_back() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + let port = app.port(); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + c.call(Request::new("document.inspect", json!({}))).unwrap(); + app.stop(); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + let CallError::Lost(m) = &e else { panic!("{e:?}") }; + assert!(m.contains("may have quit") && m.contains("Inspect the design") && m.contains(&format!("`probe --control {port}`")), "{m}"); + assert!(matches!(c.call(Request::new("document.inspect", json!({}))), Err(CallError::NotRunning(_)))); + let _back = serve(port, Duration::ZERO, Duration::from_secs(10)); + assert_eq!(c.call(Request::new("ui.view", json!({}))).unwrap()["method"], json!("ui.view")); +} + +/// A server that reads a request and closes the connection without a reply: the call is lost +/// (after one more try on a fresh connection), and the reply is never taken for "not running". +#[test] +fn a_connection_closed_without_a_reply_is_lost() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let mut line = String::new(); + let _ = BufReader::new(stream).read_line(&mut line); + } + }); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + assert!(matches!(e, CallError::Lost(_)), "{e:?}"); + assert!(!c.is_connected()); +} + +/// A request the app says is still running is waited for through `control.pending`, on the same +/// connection, and its own reply is the call's result. +#[test] +fn a_request_still_running_is_waited_for_through_pending() { + let app = serve(0, Duration::from_millis(400), Duration::from_millis(50)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + c.set_waits(Waits { pending: Duration::from_secs(10), settle: Duration::from_secs(10), poll: Duration::from_millis(5) }); + let r = c.call(Request::new("solid.slow", json!({"n": 2}))).unwrap(); + assert_eq!(r, json!({"method": "solid.slow", "params": {"n": 2}})); + assert!(!c.has_outstanding()); + assert_eq!(c.pending(), json!({"pending": null})); +} + +/// Past the wait, the call says the request is still running, and `pending` later has its end. +#[test] +fn a_request_still_running_past_the_wait_is_reported_and_its_end_kept() { + let app = serve(0, Duration::from_millis(400), Duration::from_millis(20)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + c.set_waits(Waits { pending: Duration::from_millis(30), settle: Duration::from_secs(10), poll: Duration::from_millis(5) }); + let mut slow = Request::new("solid.slow", json!({})); + slow.what = "`solid.slow` (now)".into(); + let e = c.call(slow).unwrap_err(); + let CallError::StillRunning(m) = &e else { panic!("{e:?}") }; + assert!(m.starts_with(STILL_RUNNING) && m.contains("`solid.slow` (now)") && m.contains("will still apply"), "{m}"); + assert!(c.has_outstanding()); + let end = c.pending(); + assert_eq!(end["done"], json!(true), "{end}"); + assert_eq!(end["reply"]["result"]["method"], json!("solid.slow"), "{end}"); +} diff --git a/crates/control/src/effect.rs b/crates/control/src/effect.rs new file mode 100644 index 000000000..aed5cbe59 --- /dev/null +++ b/crates/control/src/effect.rs @@ -0,0 +1,27 @@ +//! What a client is told about a control request that outlasts its wait (docs/control-protocol.md, +//! #1618): the app's control server and a bridged client both say what the request still does when +//! it finishes. Which requests do what is the app's to say. + +/// What a request still running does when it finishes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Effect { + /// It only reads (the document, the window or the command list): nothing changes. + ReadsOnly, + /// It writes a file (an export, a save, a render to a path); the document stays as it is. + WritesFile, + /// It changes the document (or may: anything not known to read or only write a file). + Changes, +} + +impl Effect { + /// What happens when the request finishes, for a message: "it only reads: nothing changes", + /// "it was not cancelled and will still write the file when it finishes; the design does not + /// change", … `document` is what the app calls its document ("design"). + pub fn when_done(self, document: &str) -> String { + match self { + Effect::ReadsOnly => "it only reads: nothing changes".into(), + Effect::WritesFile => format!("it was not cancelled and will still write the file when it finishes; the {document} does not change"), + Effect::Changes => "it was not cancelled and will still apply when it finishes".into(), + } + } +} diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs new file mode 100644 index 000000000..26727a97c --- /dev/null +++ b/crates/control/src/lib.rs @@ -0,0 +1,26 @@ +//! The control channel any app may share (docs/control-protocol.md): the per-launch token and its +//! files ([`auth`]), the loopback server that admits a connection by the token and hands each +//! request to the window ([`server`]), the client that talks to it ([`client`]), the request the +//! window answers ([`ControlRequest`]), the document watch subscribed connections wait on +//! ([`DocumentWatch`]), and what a request still running does when it finishes ([`Effect`]). It +//! needs no UI toolkit; the window's own requests over egui (`ui.screenshot`, `ui.resize`) are +//! `craft-control-egui`. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +pub mod auth; +mod cancel; +#[cfg(not(target_arch = "wasm32"))] +pub mod client; +mod effect; +mod request; +#[cfg(not(target_arch = "wasm32"))] +pub mod server; + +pub use cancel::{Cancel, Ticket}; +pub use effect::Effect; +pub use request::{ControlRequest, ControlResponse, DOCUMENT_WATCH, DocumentWatch, Subscriber, cancelled_chat_response, chat_turn_cancelled}; + +/// How an error from the client (`client`, native only) starts when a request is still running +/// in the app after the wait. Outside the client so callers on every target can test for it. +pub const STILL_RUNNING: &str = "still running:"; diff --git a/crates/control/src/request.rs b/crates/control/src/request.rs new file mode 100644 index 000000000..b97c90b76 --- /dev/null +++ b/crates/control/src/request.rs @@ -0,0 +1,125 @@ +//! A control request as the window gets it, and the design watch subscribed connections wait on. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::Sender; +use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use serde_json::{Value, json}; + +pub type ControlResponse = Value; + +pub struct ControlRequest { + pub method: String, + pub params: Value, + pub reply: Sender, + /// Set once the window starts on it; until then it waits its turn (`control.running`, #3654). + pub started: std::sync::Arc, + /// Chat requests already admitted by the gate may outlive a tab switch. This turn's + /// validity follows them through queues, computation and cursor animation. + pub chat_turn: Option>, + /// Where the request is, shared with the connection that sent it so it can cancel it + /// (`control.cancel`, #1989). + pub ticket: std::sync::Arc, + /// Told apart from every other request of this run: names its row while it waits. + pub serial: u64, +} + +impl ControlRequest { + pub fn new(method: impl Into, params: Value) -> (Self, std::sync::mpsc::Receiver) { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let (tx, rx) = std::sync::mpsc::channel(); + let serial = NEXT.fetch_add(1, Ordering::Relaxed); + (Self { method: method.into(), params, reply: tx, started: Default::default(), chat_turn: None, ticket: Default::default(), serial }, rx) + } + + /// Answer a request whose agent chat turn has ended, and say so: such a request is not run. + pub fn refuse_cancelled_chat(&self) -> bool { + if chat_turn_cancelled(self.chat_turn.as_deref()) { + let _ = self.reply.send(cancelled_chat_response()); + true + } else { + false + } + } +} + +/// Whether the agent chat turn a request belongs to has ended (`None`: it belongs to none). +pub fn chat_turn_cancelled(turn: Option<&std::sync::atomic::AtomicBool>) -> bool { + turn.is_some_and(|active| !active.load(std::sync::atomic::Ordering::Relaxed)) +} + +/// The reply to a request whose agent chat turn has ended. +pub fn cancelled_chat_response() -> Value { + json!({"ok": false, "error": "not run: the agent chat turn ended or its document tab was left"}) +} + +/// The active design's id and revision for control connections that sent `control.subscribe` +/// (#6985): the window publishes them each frame while one is subscribed, and each such +/// connection waits for them to move, then sends the app's change event (`Protocol::change_event`). +#[derive(Default)] +pub struct DocumentWatch { + subscribers: AtomicUsize, + now: Mutex>, + moved: Condvar, +} + +/// The app's [`DocumentWatch`]. +pub static DOCUMENT_WATCH: DocumentWatch = DocumentWatch::new(); + +impl DocumentWatch { + pub const fn new() -> Self { + DocumentWatch { subscribers: AtomicUsize::new(0), now: Mutex::new(None), moved: Condvar::new() } + } + + fn now(&self) -> MutexGuard<'_, Option<(String, u64)>> { + self.now.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The design as last published (`None` before anything was). + pub fn current(&self) -> Option<(String, u64)> { + self.now().clone() + } + + /// The design as it is now; waiters wake only when it moved. Only the latest is kept, so + /// changes made faster than a connection sends them go out as one. + pub fn publish(&self, document_id: &str, revision: u64) { + let mut now = self.now(); + if now.as_ref().is_some_and(|(id, r)| id == document_id && *r == revision) { + return; + } + *now = Some((document_id.to_string(), revision)); + self.moved.notify_all(); + } + + /// Whether a connection is subscribed: the window publishes only then. + pub fn subscribed(&self) -> bool { + self.subscribers.load(Ordering::Relaxed) > 0 + } + + /// Count a subscribed connection until the returned guard is dropped. + pub fn join(&'static self) -> Subscriber { + self.subscribers.fetch_add(1, Ordering::Relaxed); + Subscriber(self) + } + + /// Wait up to `wait` for the design to differ from `seen`, or for `stop`; the design then + /// (`None` before anything was published). + pub fn wait(&self, seen: &(String, u64), stop: &AtomicBool, wait: Duration) -> Option<(String, u64)> { + let unmoved = |now: &mut Option<(String, u64)>| now.as_ref() == Some(seen) && !stop.load(Ordering::Relaxed); + let (now, _) = self.moved.wait_timeout_while(self.now(), wait, unmoved).unwrap_or_else(PoisonError::into_inner); + now.clone() + } +} + +/// A subscribed connection, counted by its [`DocumentWatch`] while this lives. +pub struct Subscriber(&'static DocumentWatch); + +impl Drop for Subscriber { + fn drop(&mut self) { + self.0.subscribers.fetch_sub(1, Ordering::Relaxed); + // Under the lock, so a connection about to wait sees its `stop` set before it does. + let _now = self.0.now(); + self.0.moved.notify_all(); + } +} diff --git a/crates/control/src/server.rs b/crates/control/src/server.rs new file mode 100644 index 000000000..85664a1aa --- /dev/null +++ b/crates/control/src/server.rs @@ -0,0 +1,1433 @@ +//! The loopback JSON-lines control server: one request per line, one reply per line, behind the +//! launch's token (docs/control-protocol.md). Agents and tests drive an app through it (adapted +//! from CADCraft, MIT OR Apache-2.0). What the app's requests mean is the app's [`Protocol`]; the +//! window answers them from the [`ControlRequest`]s it is sent. + +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::Effect; +use crate::auth::tokens_match; +use crate::cancel::{Cancel, Ticket}; +use crate::request::{ControlRequest, DOCUMENT_WATCH, DocumentWatch, Subscriber}; + +/// What the server needs to know of a request before it sends it to the window: how to name it +/// in messages, what it does if it outlasts its wait, and what `control.running` shows of it. +#[derive(Clone, Debug)] +pub struct Summary { + /// The request in messages ("`engine.execute` (solid.fillet)"). + pub what: String, + /// What it still does when it finishes after its wait. + pub effect: Effect, + /// The command it runs, for `control.running` (never its parameters). + pub command: Option<&'static str>, + /// The number of steps it runs, for `control.running`. + pub steps: Option, + /// It may take as long as a batch. + pub long: bool, +} + +/// What an app's requests mean to its control server. +pub trait Protocol: Send + Sync { + fn summary(&self, method: &str, params: &Value) -> Summary; + /// What the app calls its document in messages ("design"). + fn document(&self) -> &'static str; + /// The event a subscribed connection is sent when the document moves ("design.changed"). + fn change_event(&self) -> &'static str; + /// The method that answers the document's id and revision now ("document.revision"). + fn revision_method(&self) -> &'static str; + /// Where a client reads how to talk to the channel, named when its token is refused + /// ("docs/control-protocol.md"). + fn docs(&self) -> &'static str; + /// The `control.capabilities` result: what this app's channel does beyond the methods every + /// version has, for a client to find out before it relies on it (an older app answers + /// `control.capabilities` with "unknown method"). + fn capabilities(&self) -> Value; +} + +/// An app's control server: its [`Protocol`], how it wakes the window for a request, and the +/// requests every connection of its listeners has the window running (`control.running`). +pub struct Hub { + protocol: Box, + wake: Box, + /// Longest request line accepted. + max_line: usize, + waits: Waits, + watch: &'static DocumentWatch, + running: Arc, + open: AtomicUsize, +} + +impl Hub { + /// A server for `protocol`'s requests; `wake` makes the window look at its requests now. + pub fn new(protocol: impl Protocol + 'static, wake: impl Fn() + Send + Sync + 'static, max_line: usize) -> Self { + Hub { + protocol: Box::new(protocol), + wake: Box::new(wake), + max_line, + waits: Waits::default(), + watch: &DOCUMENT_WATCH, + running: Arc::new(Running::new()), + open: AtomicUsize::new(0), + } + } + + /// Other waits (tests). + pub fn with_waits(mut self, waits: Waits) -> Self { + self.waits = waits; + self + } + + /// Another document watch than the app's [`DOCUMENT_WATCH`] (tests). + pub fn with_watch(mut self, watch: &'static DocumentWatch) -> Self { + self.watch = watch; + self + } + + /// Serve `listener` with `token` on threads of their own, at most [`MAX_CONNECTIONS`] + /// connections at once. `patient`: every request may wait as long as a batch (a person decides + /// on it). A listener that is not on a loopback address is refused: the channel never goes on + /// the network. + pub fn listen(self: &Arc, listener: TcpListener, token: String, requests: Sender, patient: bool) -> std::io::Result<()> { + self.listen_stoppable(listener, token, requests, patient).map(drop) + } + + /// [`Hub::listen`], and a [`Stop`] that closes the listener and the connections it accepted + /// (a channel turned off in the app's settings). Dropping the [`Stop`] leaves it serving. + pub fn listen_stoppable( + self: &Arc, + listener: TcpListener, + token: String, + requests: Sender, + patient: bool, + ) -> std::io::Result { + let addr = listener.local_addr()?; + if !addr.ip().is_loopback() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("the control channel listens on loopback only, not {addr}"))); + } + let hub = Arc::clone(self); + let token = Arc::new(token); + let shared = Arc::new(Accepted::default()); + let (closed_tx, closed) = channel::<()>(); + let stop = Stop { addr, shared: Arc::clone(&shared), closed }; + std::thread::spawn(move || { + // Dropped when this thread ends, with the listener: `Stop::stop` waits for that. + let _closed = closed_tx; + for stream in listener.incoming().flatten() { + if shared.stopped.load(Ordering::SeqCst) { + break; + } + if hub.open.fetch_add(1, Ordering::Relaxed) >= MAX_CONNECTIONS { + hub.open.fetch_sub(1, Ordering::Relaxed); + let mut refused = stream; + let _ = send(&mut refused, &json!({"ok": false, "error": "too many connections"})); + continue; + } + let id = shared.next.fetch_add(1, Ordering::Relaxed); + if let Ok(copy) = stream.try_clone() { + shared.lock().push((id, copy)); + } + // Stopped while this one was being accepted: it is closed with the rest. + if shared.stopped.load(Ordering::SeqCst) { + shared.close_all(); + } + let (hub, tx, token, shared) = (Arc::clone(&hub), requests.clone(), Arc::clone(&token), Arc::clone(&shared)); + std::thread::spawn(move || { + hub.serve(stream, &token, tx, patient); + hub.open.fetch_sub(1, Ordering::Relaxed); + shared.lock().retain(|(i, _)| *i != id); + }); + } + // A captured value drops after the body's locals, so without this `_closed` would tell + // `Stop::stop` the port is free while the listener still holds it (#14752). + drop(listener); + }); + Ok(stop) + } +} + +/// The connections a stoppable listener accepted, so stopping it can close them too. +#[derive(Default)] +struct Accepted { + stopped: AtomicBool, + next: AtomicU64, + streams: Mutex>, +} + +impl Accepted { + fn lock(&self) -> MutexGuard<'_, Vec<(u64, TcpStream)>> { + self.streams.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn close_all(&self) { + for (_, stream) in self.lock().drain(..) { + let _ = stream.shutdown(std::net::Shutdown::Both); + } + } +} + +/// Stops a listener started with [`Hub::listen_stoppable`]. +pub struct Stop { + addr: std::net::SocketAddr, + shared: Arc, + closed: Receiver<()>, +} + +impl Stop { + /// The port the listener is on. + pub fn port(&self) -> u16 { + self.addr.port() + } + + /// Stop accepting, close the port and end every connection it accepted. Returns once the + /// port is closed (at most a couple of seconds), so the same port can be bound again at once. + pub fn stop(self) { + self.shared.stopped.store(true, Ordering::SeqCst); + // The listener waits in `accept`: a connection of our own wakes it to see the flag. + let _ = TcpStream::connect_timeout(&self.addr, Duration::from_secs(1)); + let _ = self.closed.recv_timeout(Duration::from_secs(2)); + self.shared.close_all(); + } +} + +/// Connections served at once, so a flood must not be able to spawn threads without end. +pub const MAX_CONNECTIONS: usize = 16; +/// Lines that are not JSON before the connection is closed, so garbage cannot loop forever. +const MAX_BAD_LINES: usize = 8; +/// How long a subscribed connection waits after a `design.changed` line before the next: what +/// changes meanwhile (a parameter dragged) goes out as one line, the latest (#6985). +const EVENT_GAP: Duration = Duration::from_millis(100); +/// How long a new connection has to show the token. A client connects when it has a request to +/// send, so this is plenty; one that doesn't send it loses the connection, and its seat among +/// [`MAX_CONNECTIONS`] goes to someone else. +const TOKEN_WAIT: Duration = Duration::from_secs(5); + +/// A new random token: 32 bytes from the operating system, as hex. +pub fn new_token() -> Option { + let mut bytes = [0u8; 32]; + getrandom::fill(&mut bytes).ok()?; + Some(bytes.iter().map(|b| format!("{b:02x}")).collect()) +} + +/// One line without its newline, or `None` at the end of the stream. A client that sends +/// gigabytes with no newline would otherwise make this thread allocate until the process is +/// killed, so the line is refused as soon as it passes `max`. +fn read_line(r: &mut impl BufRead, max: usize) -> std::io::Result>> { + let mut line = Vec::new(); + loop { + let (done, used) = { + let available = r.fill_buf()?; + if available.is_empty() { + return Ok((!line.is_empty()).then_some(line)); + } + match available.iter().position(|b| *b == b'\n') { + Some(i) => { + line.extend_from_slice(available.get(..i).unwrap_or(&[])); + (true, i + 1) + } + None => { + line.extend_from_slice(available); + (false, available.len()) + } + } + }; + r.consume(used); + if done { + return Ok(Some(line)); + } + if line.len() > max { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "request too large")); + } + } +} + +/// A reply line, written to the socket at once. `writeln!` with a JSON value hands the socket each +/// of its pieces apart, a system call for every string, number and comma: half a second for a +/// design's component tree when it has thousands of components. +fn send(out: &mut impl Write, reply: &Value) -> std::io::Result<()> { + let mut line = serde_json::to_vec(reply).map_err(std::io::Error::other)?; + line.push(b'\n'); + out.write_all(&line) +} + +/// The `"token"` of a request line that is a JSON object, read without building the rest of it. +/// Until a connection shows the token, a line costs only its own bytes: parsed whole, a 16 MiB +/// line of `[0,0,…]` is hundreds of MB, and 16 such connections could exhaust memory before any +/// token was checked. +fn token_of(line: &str) -> Option { + struct Token; + impl<'de> serde::de::Visitor<'de> for Token { + type Value = Option; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a request object") + } + fn visit_map>(self, mut map: A) -> Result { + let mut token = None; + while let Some(key) = map.next_key::()? { + if key == "token" { + token = Some(map.next_value::()?); + } else { + map.next_value::()?; + } + } + Ok(token) + } + } + let mut de = serde_json::Deserializer::from_str(line); + let token = serde::Deserializer::deserialize_map(&mut de, Token).ok()?; + de.end().ok()?; + token +} + +/// The reply to a connection that didn't show the token, naming the app's `docs`; the +/// connection is then closed. +fn unauthorized(docs: &str) -> Value { + json!({"ok": false, "error": format!("unauthorized: send the control token as \"token\" (see {docs})")}) +} + +/// Refuse the connection: the reply, then the end of what we send, then what the client had +/// already sent read and dropped (a little, briefly). Closing a socket with unread input resets +/// the connection, and a reset can reach the client before it has read the refusal. +fn refuse(out: &mut TcpStream, reader: &mut BufReader, docs: &str) { + use std::io::Read; + let _ = send(out, &unauthorized(docs)); + let _ = out.shutdown(std::net::Shutdown::Write); + reader.get_mut().deadline = Some(Instant::now() + Duration::from_millis(500)); + let _ = std::io::copy(&mut reader.by_ref().take(64 * 1024), &mut std::io::sink()); +} + +/// What a client sends. Until it shows the token, all of its reads share one deadline, so neither +/// blank lines nor a line sent a byte at a time buy it more time. After that it may stay idle as +/// long as it likes. +struct ClientInput { + socket: TcpStream, + deadline: Option, +} + +impl ClientInput { + /// No deadline any more: the client showed the token. + fn admit(&mut self) -> std::io::Result<()> { + self.deadline = None; + self.socket.set_read_timeout(None) + } +} + +impl std::io::Read for ClientInput { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(deadline) = self.deadline { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(std::io::ErrorKind::TimedOut.into()); + } + self.socket.set_read_timeout(Some(left))?; + } + self.socket.read(buf) + } +} + +/// How long a request waits for the app's reply: most requests, and a batch (or every request, +/// on a patient connection); and how long a new connection has to show the token. +#[derive(Clone, Copy, Debug)] +pub struct Waits { + pub short: Duration, + pub long: Duration, + pub token: Duration, +} + +impl Default for Waits { + fn default() -> Self { + Waits { short: Duration::from_secs(60), long: Duration::from_secs(3600), token: TOKEN_WAIT } + } +} + +/// A request that timed out but is still running in the app (#1618): its id and method, and where +/// its reply will land. +struct Pending { + id: Value, + method: String, + /// The request in messages ("`engine.execute` (solid.fillet)"). + what: String, + reply: Receiver, + /// Where it is in the app, to cancel it (`control.cancel`). + job: Arc, +} + +/// `control.cancel {id?}`: cancel the request this connection still waits for (the one +/// `control.pending` reports; `id`, when given, must name it). A connection can only cancel its +/// own requests. Waiting its turn, it is removed; computing off the window thread, it is stopped +/// as the app's Stop stops it. Either way nothing of it is applied and the connection is free. +/// One that runs on the window thread can't be cancelled; one that finished is answered. +fn cancel(pending: &mut Option, late: Option, id: Option<&Value>, document: &str) -> Value { + let id = id.filter(|v| !v.is_null()); + let mine = pending.as_ref().filter(|p| id.is_none_or(|id| *id == p.id)); + let Some(p) = mine else { + return match id { + None => json!({"ok": true, "result": {"cancelled": null, "note": "nothing to cancel: no request of this connection is running"}}), + Some(id) => json!({"ok": false, "error": format!( + "nothing to cancel: request {} is not running on this connection (a connection can only cancel its own requests)", + id.to_string().chars().take(64).collect::() + )}), + }; + }; + let (what, id) = (format!("request {} ({})", p.id, p.what), p.id.clone()); + let finished = |p: Pending, reply: Value| { + json!({"ok": true, "result": {"cancelled": null, "pending": p.id, "method": p.method, "done": true, "reply": reply, + "note": format!("nothing to cancel: {what} had already finished; its reply is here")}}) + }; + if let Some(reply) = late { + return pending.take().map_or(Value::Null, |p| finished(p, reply)); + } + let outcome = p.job.cancel(); + let note = match outcome { + Cancel::Removed => format!("cancelled: {what} was still waiting its turn and will not run; nothing was applied"), + Cancel::Stopped { writes_file: false } => format!("stopped: {what} was computing off the window thread; nothing was applied"), + Cancel::Stopped { writes_file: true } => { + format!("stopped: {what} was computing on a copy of the {document}, which is unchanged; its worker may still write the file") + } + Cancel::Already => format!("{what} was already cancelled or stopped; nothing of it was applied"), + Cancel::Running => { + // It may have just finished. + if let Some(reply) = pending.as_ref().and_then(|p| p.reply.try_recv().ok()) { + return pending.take().map_or(Value::Null, |p| finished(p, reply)); + } + return json!({"ok": false, "cancellable": false, "error": format!( + "cannot be cancelled: {what} already runs in the app (on its window thread, or as a file it opens) and will \ + still finish; ask `control.pending` for its result. Only a request still waiting its turn, or a change or \ + export computing off the window thread, can be cancelled" + )}); + } + }; + let method = pending.take().map(|p| p.method); + let how = match outcome { + Cancel::Removed => "removed", + Cancel::Stopped { .. } => "stopped", + _ => "already", + }; + json!({"ok": true, "result": {"cancelled": id, "method": method, "outcome": how, "note": note}}) +} + +/// Every request the app has been sent and not answered yet, from every connection, for +/// `control.running` (#3654). Each connection's own thread answers that, so it never waits behind +/// a running request. +struct Running { + list: Mutex>, + connections: AtomicU64, + keys: AtomicU64, +} + +/// What `control.running` says about a request: never its parameters beyond the command id. +struct InFlight { + key: u64, + connection: u64, + id: Value, + method: String, + command: Option<&'static str>, + steps: Option, + since: Instant, + started: Arc, +} + +impl Running { + fn new() -> Self { + Running { list: Mutex::new(Vec::new()), connections: AtomicU64::new(0), keys: AtomicU64::new(0) } + } + + fn lock(&self) -> MutexGuard<'_, Vec> { + self.list.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// List `req`, sent on `connection`, until the returned ticket is dropped; `command` and + /// `steps` as the app's [`Protocol`] summarised it. + fn add(self: &Arc, connection: u64, id: &Value, req: &ControlRequest, command: Option<&'static str>, steps: Option) -> Listing { + let key = self.keys.fetch_add(1, Ordering::Relaxed); + // Ids and methods come from the client: only short ones are repeated to the others. + let id = match id { + Value::Number(_) => id.clone(), + Value::String(s) if s.len() <= 64 => id.clone(), + _ => Value::Null, + }; + let method = req.method.chars().take(64).collect(); + let started = req.started.clone(); + self.lock().push(InFlight { key, connection, id, method, command, steps, since: Instant::now(), started }); + Listing { running: Arc::clone(self), key } + } + + /// The `control.running` result, as `asker`'s connection sees it. + fn result(&self, asker: u64) -> Value { + let rows: Vec = self + .lock() + .iter() + .map(|e| { + let mut r = json!({ + "connection": e.connection, + "id": e.id, + "method": e.method, + "running_s": (e.since.elapsed().as_secs_f64() * 10.0).round() / 10.0, + "queued": !e.started.load(Ordering::Relaxed), + }); + if let Some(c) = e.command { + r["command"] = json!(c); + } + if let Some(n) = e.steps { + r["steps"] = json!(n); + } + if e.connection == asker { + r["self"] = json!(true); + } + r + }) + .collect(); + json!({"running": rows}) + } +} + +/// A listed request; dropped when its reply arrives. +struct Listing { + running: Arc, + key: u64, +} + +impl Drop for Listing { + fn drop(&mut self) { + let key = self.key; + self.running.lock().retain(|e| e.key != key); + } +} + +/// The reply to a timed-out request, received on a thread of its own so the request stays listed +/// until the app answers, also once its connection has closed (#3654). +fn watch(reply: Receiver, ticket: Listing) -> Receiver { + let (late_tx, late_rx) = channel(); + // A small thread per timed-out request (each connection has at most one). If none can be + // made, the reply is lost and `control.pending` says the app ended the request without one. + // The reply is passed on before the request leaves the list: once `control.running` no longer + // shows it, its connection's `control.pending` has it. + let _ = std::thread::Builder::new().stack_size(64 * 1024).spawn(move || { + if let Ok(v) = reply.recv() { + let _ = late_tx.send(v); + } + drop(late_tx); + drop(ticket); + }); + late_rx +} + +/// The reply text for a request that outlasts its wait. A request that only reads changes +/// nothing when it finishes; an export or a save still writes its file, and any other request +/// still applies, as it was not cancelled. +pub fn timeout_message(what: &str, effect: Effect, document: &str) -> String { + match effect { + Effect::ReadsOnly => { + format!("timeout: {what} is still running in the app ({}); ask `control.pending` for its answer", effect.when_done(document)) + } + Effect::WritesFile => format!( + "timeout: {what} is still running in the app and will still write the file when it finishes (it was not cancelled: \ + don't send it again; the {document} does not change); ask `control.pending` for its result, or `control.cancel` to \ + cancel it" + ), + Effect::Changes => format!( + "timeout: {what} is still running in the app and will still apply when it finishes (it was not cancelled: don't send it \ + again); ask `control.pending` for its result, or `control.cancel` to cancel it" + ), + } +} + +/// The reply text for a request refused because an earlier one on the connection still runs. +fn busy_message(p: &Pending, method: &str) -> String { + let method: String = method.chars().take(128).collect(); + format!( + "busy: request {} ({}) is still running, so `{method}` was not run; ask `control.pending` for that request's result, then \ + send this one again", + p.id, p.what + ) +} + +/// A subscribed connection's socket, for its replies and its events: each line is written whole +/// and alone, so an event never lands inside a reply (#6985). +struct Lines { + socket: TcpStream, + turn: Mutex<()>, +} + +impl Lines { + fn send(&self, line: &Value) -> std::io::Result<()> { + let _turn = self.turn.lock().unwrap_or_else(PoisonError::into_inner); + send(&mut &self.socket, line) + } +} + +/// A subscribed connection's seat on the watch: it leaves exactly once, whichever thread sees +/// the connection's end first — the connection's own thread at the end of the stream, or the +/// events thread at the first write a closed client refuses. The seat used to wait for the +/// events thread alone, and that thread only leaves its wait when the design moves, so the +/// count could outlive the connection by a second or more (#8315). +struct Seat(Mutex>); + +impl Seat { + fn new(counted: Subscriber) -> Self { + Seat(Mutex::new(Some(counted))) + } + + /// Leave the watch now: the seat goes once, whoever gets here first. + fn release(&self) { + let counted = self.0.lock().unwrap_or_else(PoisonError::into_inner).take(); + drop(counted); + } +} + +impl Drop for Seat { + fn drop(&mut self) { + self.release(); + } +} + +/// A connection's `control.subscribe` (#6985): its `design.changed` lines, sent from a thread of +/// their own until the connection ends. +struct Subscription { + lines: Arc, + stop: Arc, + seat: Arc, +} + +impl Subscription { + /// Write `reply`, the window's answer to `control.subscribe` naming the design `seen`, on + /// `out`, then send `design`'s changes after it: the reply goes out before any event. `None`, + /// with nothing written, when no thread or socket was left for the events. + fn start( + watch: &'static DocumentWatch, + event: &'static str, + counted: Subscriber, + out: &TcpStream, + seen: (String, u64), + reply: &Value, + ) -> Option<(Self, std::io::Result<()>)> { + let lines = Arc::new(Lines { socket: out.try_clone().ok()?, turn: Mutex::new(()) }); + let stop = Arc::new(AtomicBool::new(false)); + let seat = Arc::new(Seat::new(counted)); + let first = lines.turn.lock().unwrap_or_else(PoisonError::into_inner); + let (l, s, watch_seat) = (lines.clone(), stop.clone(), Arc::clone(&seat)); + // The connection's seat on the watch rides the events thread: it leaves when the thread + // does, so the next failed write to a closed client retires it even while the connection's + // own thread is slow to see the end of the stream (#7226). The connection's own thread + // leaves it too, so a closed connection stops counting at once (#8315). + std::thread::Builder::new().stack_size(64 * 1024).spawn(move || send_events(watch, event, &l, &s, seen, watch_seat)).ok()?; + let sent = send(&mut &lines.socket, reply); + drop(first); + Some((Subscription { lines, stop, seat }, sent)) + } +} + +/// The design a `control.subscribe` reply names. +fn document_of(reply: &Value) -> Option<(String, u64)> { + let result = reply.get("result")?; + Some((result.get("document_id")?.as_str()?.to_string(), result.get("revision")?.as_u64()?)) +} + +impl Drop for Subscription { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + // The seat on the watch goes now: the connection is over, and nothing counts it any more + // (#8315). + self.seat.release(); + // The socket closes now, not when the thread next looks; a line blocked on a client that + // stopped reading ends too. + let _ = self.lines.socket.shutdown(std::net::Shutdown::Both); + } +} + +/// The `design.changed` lines of a subscribed connection, each time the design differs from the +/// last one it was told of (`seen`), until `stop`. Holds the connection's seat on the watch +/// (`seat`), which it leaves when the thread ends: on `stop`, or on the first write a closed +/// client refuses (#7226). +fn send_events(watch: &DocumentWatch, event: &str, lines: &Lines, stop: &AtomicBool, mut seen: (String, u64), seat: Arc) { + while !stop.load(Ordering::Relaxed) { + let Some(now) = watch.wait(&seen, stop, Duration::from_secs(1)).filter(|now| *now != seen) else { continue }; + if lines.send(&json!({"event": event, "params": {"document_id": now.0, "revision": now.1}})).is_err() { + break; + } + seen = now; + std::thread::sleep(EVENT_GAP); + } + // The client is gone: it is not counted on the watch any more, whatever the connection's own + // thread still has to notice (#8315). + seat.release(); +} + +/// Write `reply` on the connection: through its subscription once it has one. +fn answer(out: &mut TcpStream, events: Option<&Subscription>, reply: &Value) -> std::io::Result<()> { + match events { + Some(e) => e.lines.send(reply), + None => send(out, reply), + } +} + +impl Hub { + /// Serve one connection until it closes: the token first, then one request per line, each + /// sent to the window on `requests` and its reply written back. + pub fn serve(&self, stream: TcpStream, token: &str, requests: Sender, patient: bool) { + let (waits, running, documents, tx) = (self.waits, &self.running, self.watch, requests); + let connection = running.connections.fetch_add(1, Ordering::Relaxed) + 1; + let Ok(read) = stream.try_clone() else { return }; + let mut out = stream; + let mut reader = BufReader::new(ClientInput { socket: read, deadline: Some(Instant::now() + waits.token) }); + let mut bad = 0usize; + let mut authenticated = false; + let mut pending: Option = None; + let mut events: Option = None; + loop { + // A first `control.subscribe` on its way to the window (#6985). + let mut joining = None; + let line = match read_line(&mut reader, self.max_line) { + Ok(Some(l)) => String::from_utf8_lossy(&l).into_owned(), + Ok(None) => break, + // Closed without a word: a client that connected early and only now sends its first + // request would read a line here as the reply to it. It finds the connection closed + // and opens another. + Err(e) if !authenticated && matches!(e.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) => break, + Err(e) => { + let _ = answer(&mut out, events.as_ref(), &json!({"ok": false, "error": e.to_string()})); + break; + } + }; + if line.trim().is_empty() { + continue; + } + // The first request must carry the token; later ones on the same connection may. + if !authenticated { + match token_of(&line) { + Some(t) if tokens_match(&t, token) => { + if reader.get_mut().admit().is_err() { + break; + } + authenticated = true; + } + _ => { + refuse(&mut out, &mut reader, self.protocol.docs()); + break; + } + } + } + let reply = match serde_json::from_str::(&line) { + Ok(msg) => { + let id = msg.get("id").cloned().unwrap_or(Value::Null); + let method = msg.get("method").and_then(Value::as_str).unwrap_or("").to_string(); + let params = msg.get("params").cloned().unwrap_or(json!({})); + // A request that timed out may have finished since (#1618). `control.running` and + // `control.capabilities` only look, so they leave that reply where it is. + let late = pending.as_ref().filter(|_| !matches!(method.as_str(), "control.running" | "control.capabilities")).and_then(|p| { + match p.reply.try_recv() { + Ok(reply) => Some(reply), + Err(TryRecvError::Disconnected) => Some(json!({"ok": false, "error": "the app ended the request without a reply"})), + Err(TryRecvError::Empty) => None, + } + }); + let mut r = if method == "control.running" { + json!({"ok": true, "result": running.result(connection)}) + } else if method == "control.capabilities" { + json!({"ok": true, "result": self.protocol.capabilities()}) + } else if method == "control.cancel" { + let r = cancel(&mut pending, late, params.get("id"), self.protocol.document()); + (self.wake)(); + r + } else if method == "control.pending" { + match (pending.take(), late) { + (None, _) => json!({"ok": true, "result": {"pending": null}}), + (Some(p), Some(reply)) => { + json!({"ok": true, "result": {"pending": p.id, "method": p.method, "done": true, "reply": reply}}) + } + (Some(p), None) => { + let r = json!({"ok": true, "result": {"pending": p.id, "method": p.method, "done": false}}); + pending = Some(p); + r + } + } + } else if let Some(p) = pending.as_ref().filter(|_| late.is_none()) { + // Nothing more is queued behind it, to run unseen when it ends. + json!({"ok": false, "busy": true, "error": busy_message(p, &method)}) + } else if method == "control.subscribe" && events.is_some() { + if late.is_some() { + // As below: finished, and the client moved on without asking. + pending = None; + } + // Its events already say how the design moves; a second answer could only + // arrive behind a newer one. + { + let (event, revision, document) = + (self.protocol.change_event(), self.protocol.revision_method(), self.protocol.document()); + json!({"ok": false, "error": format!("already subscribed: this connection is sent {event} lines; ask {revision} for the {document} now")}) + } + } else { + if late.is_some() { + // Finished, and the client moved on without asking: it is dropped. + pending = None; + } + // A batch replaces many short requests: the whole of it may take long (#743). + let Summary { what, effect, command, steps, long } = self.protocol.summary(&method, ¶ms); + let wait = if patient || long { waits.long } else { waits.short }; + // Counted before the window answers, so it publishes every change from its + // answer on. + joining = (method == "control.subscribe").then(|| documents.join()); + let (req, rrx) = ControlRequest::new(method.clone(), params); + let job = Arc::clone(&req.ticket); + let ticket = running.add(connection, &id, &req, command, steps); + if tx.send(req).is_err() { + break; + } + (self.wake)(); + match rrx.recv_timeout(wait) { + Ok(r) => r, + Err(_) => { + let r = json!({"ok": false, "running": true, "error": timeout_message(&what, effect, self.protocol.document())}); + pending = Some(Pending { id: id.clone(), method, what, reply: watch(rrx, ticket), job }); + r + } + } + }; + if let Some(o) = r.as_object_mut() { + o.insert("id".into(), id); + } + r + } + Err(e) => { + bad += 1; + json!({"ok": false, "error": format!("bad JSON: {e}")}) + } + }; + // Subscribed once the window named the design (not when it refused or timed out). + let sent = match joining.zip(document_of(&reply)) { + Some((counted, seen)) => match Subscription::start(documents, self.protocol.change_event(), counted, &out, seen, &reply) { + Some((subscription, sent)) => { + events = Some(subscription); + sent + } + None => send( + &mut out, + &json!({"id": reply.get("id"), "ok": false, "error": "not subscribed: no thread or socket left for the events"}), + ), + }, + None => answer(&mut out, events.as_ref(), &reply), + }; + if sent.is_err() { + break; + } + if bad >= MAX_BAD_LINES { + break; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The longest request line, as an app might accept. + const MAX_LINE: usize = 16 << 20; + + /// An app whose requests all may change its document and take the short wait. + struct Plain; + + impl Protocol for Plain { + fn summary(&self, method: &str, _: &Value) -> Summary { + Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false } + } + fn document(&self) -> &'static str { + "design" + } + fn change_event(&self) -> &'static str { + "design.changed" + } + fn revision_method(&self) -> &'static str { + "document.revision" + } + fn docs(&self) -> &'static str { + "docs/control-protocol.md" + } + fn capabilities(&self) -> Value { + json!({}) + } + } + + /// A server for [`Plain`] with these waits and this document watch. + fn hub(waits: Waits, watch: &'static DocumentWatch) -> Arc { + Arc::new(Hub::new(Plain, || {}, MAX_LINE).with_waits(waits).with_watch(watch)) + } + + /// The channel never goes on the network: a listener on any address but loopback is refused, + /// and nothing is served on it. + #[test] + fn a_listener_off_loopback_is_refused() { + let (tx, _rx) = channel::(); + let open = TcpListener::bind("0.0.0.0:0").unwrap(); + let e = hub(Waits::default(), &DOCUMENT_WATCH).listen(open, "0123456789abcdef0123456789abcdef".into(), tx.clone(), false).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput, "{e}"); + assert!(e.to_string().contains("loopback only"), "{e}"); + let local = TcpListener::bind("127.0.0.1:0").unwrap(); + assert!(hub(Waits::default(), &DOCUMENT_WATCH).listen(local, "0123456789abcdef0123456789abcdef".into(), tx, false).is_ok()); + } + + /// A connection that doesn't show the token in time is closed without a word, however it fills + /// the time. One that does keeps working when it idles for longer than that. + #[test] + fn the_token_has_a_time_limit_and_an_authenticated_client_may_idle() { + use std::io::Read; + let token = "0123456789abcdef0123456789abcdef"; + let limit = Duration::from_millis(500); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + std::thread::spawn(move || { + for req in rx { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + }); + let waits = Waits { short: Duration::from_secs(10), long: Duration::from_secs(10), token: limit }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let open = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + s + }; + // Sends `drip` every 50 ms until the server closes; what it read, and when it closed. + let closed_after = |drip: &[u8]| { + let mut s = open(); + let start = Instant::now(); + s.set_read_timeout(Some(Duration::from_millis(50))).unwrap(); + let mut got = Vec::new(); + loop { + let mut buf = [0u8; 256]; + match s.read(&mut buf) { + Ok(0) => break, + Ok(n) => got.extend_from_slice(buf.get(..n).unwrap()), + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => { + assert!(start.elapsed() < Duration::from_secs(20), "still open"); + let _ = s.write_all(drip); + } + Err(_) => break, + } + } + (got, start.elapsed()) + }; + for drip in [&b""[..], b"\n", b"\r\n \n", b" ", b"{"] { + let (got, took) = closed_after(drip); + assert!(got.is_empty(), "{drip:?} was answered: {}", String::from_utf8_lossy(&got)); + assert!(took >= limit.saturating_sub(Duration::from_millis(50)), "{drip:?} closed after {took:?}"); + } + + let mut s = open(); + let mut lines = BufReader::new(s.try_clone().unwrap()); + let mut ask = |line: Value| -> Value { + writeln!(s, "{line}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + assert_eq!(ask(json!({"id": 1, "method": "ui.view", "token": token}))["ok"], true); + std::thread::sleep(limit * 3); + assert_eq!(ask(json!({"id": 2, "method": "ui.view"}))["ok"], true, "idle for longer than the limit"); + + // A wrong token is still answered, at once. + let mut wrong = open(); + writeln!(wrong, "{}", json!({"id": 1, "method": "ui.view", "token": "nope"})).unwrap(); + let mut l = String::new(); + BufReader::new(wrong).read_line(&mut l).unwrap(); + assert!(l.contains("unauthorized"), "{l}"); + } + + /// #1989: `control.cancel` cancels the connection's own outstanding request. One still + /// waiting its turn is removed and never runs, and the connection is free at once; one that + /// already runs in the app can't be, and says so; another connection's request is refused; + /// with nothing outstanding there is nothing to cancel. + #[test] + fn control_cancel_cancels_only_this_connections_request() { + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + let (gate_tx, gate_rx) = channel::<()>(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + // A stand-in for the window thread: `queued` waits behind the others (never taken until + // `slow` is done), `slow` runs until the gate opens. + std::thread::spawn(move || { + let mut held = Vec::new(); + for req in rx { + if req.method == "queued" { + held.push(req); + continue; + } + if !req.ticket.take() { + continue; + } + seen.lock().unwrap().push(req.method.clone()); + if req.method == "slow" { + let _ = gate_rx.recv(); + } + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + if req.method == "slow" { + for req in held.drain(..) { + if req.ticket.take() { + seen.lock().unwrap().push(req.method.clone()); + } + } + } + } + }); + let waits = Waits { short: Duration::from_millis(100), long: Duration::from_millis(100), token: TOKEN_WAIT }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let connect = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let out = s.try_clone().unwrap(); + (out, BufReader::new(s)) + }; + let ask = |(out, lines): &mut (TcpStream, BufReader), v: Value| -> Value { + writeln!(out, "{v}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let mut a = connect(); + let mut b = connect(); + let r = ask(&mut b, json!({"id": 1, "method": "control.cancel", "token": token})); + assert!(r["ok"] == true && r["result"]["cancelled"].is_null(), "nothing to cancel: {r}"); + + // Waiting its turn: removed, never run, and the connection is free. + let r = ask(&mut a, json!({"id": 1, "method": "queued", "token": token})); + assert_eq!(r["running"], true, "{r}"); + let r = ask(&mut a, json!({"id": 2, "method": "control.cancel", "params": {"id": 1}})); + assert_eq!((&r["ok"], &r["result"]["cancelled"], &r["result"]["outcome"]), (&json!(true), &json!(1), &json!("removed")), "{r}"); + assert!(r["result"]["note"].as_str().is_some_and(|n| n.contains("will not run")), "{r}"); + let r = ask(&mut a, json!({"id": 3, "method": "control.pending"})); + assert_eq!(r["result"]["pending"], Value::Null, "{r}"); + + // Already running in the app: it can't be, and it still answers. + let r = ask(&mut a, json!({"id": 4, "method": "slow"})); + assert_eq!(r["running"], true, "{r}"); + let r = ask(&mut b, json!({"id": 2, "method": "control.cancel", "params": {"id": 4}})); + assert!(r["ok"] == false && r["error"].as_str().is_some_and(|e| e.contains("its own requests")), "another connection's: {r}"); + let r = ask(&mut a, json!({"id": 5, "method": "control.cancel", "params": {"id": 99}})); + assert!(r["ok"] == false && r["error"].as_str().is_some_and(|e| e.starts_with("nothing to cancel")), "{r}"); + let r = ask(&mut a, json!({"id": 6, "method": "control.cancel"})); + assert!(r["ok"] == false && r["cancellable"] == false, "{r}"); + assert!(r["error"].as_str().is_some_and(|e| e.starts_with("cannot be cancelled") && e.contains("control.pending")), "{r}"); + gate_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let r = loop { + let r = ask(&mut a, json!({"id": 7, "method": "control.pending"})); + if r["result"]["done"] == true || Instant::now() > deadline { + break r; + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(r["result"]["reply"]["result"], "slow", "{r}"); + let r = ask(&mut a, json!({"id": 8, "method": "control.cancel"})); + assert!(r["ok"] == true && r["result"]["cancelled"].is_null(), "{r}"); + assert_eq!(*ran.lock().unwrap(), ["slow"], "the cancelled request never ran"); + } + + /// #1618: a request that outlasts its wait says it is still running; the connection keeps its + /// reply for `control.pending`, and refuses other requests as `busy` until it lands, so none + /// is queued to run unseen behind it. + #[test] + fn a_timed_out_request_can_be_asked_for_later() { + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The slow request is held until the test has seen it time out, so the test does not + // race the clock on a loaded machine. + let (gate_tx, gate_rx) = channel::<()>(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + std::thread::spawn(move || { + for req in rx { + seen.lock().unwrap().push(req.method.clone()); + if req.method == "slow" { + let _ = gate_rx.recv(); + } + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + }); + let waits = Waits { short: Duration::from_millis(100), long: Duration::from_millis(100), token: TOKEN_WAIT }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + if let Some(stream) = listener.incoming().flatten().next() { + hub.serve(stream, token, tx, false); + } + }); + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut out = s.try_clone().unwrap(); + let mut lines = BufReader::new(s); + let mut ask = |v: Value| -> Value { + writeln!(out, "{v}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let r = ask(json!({"id": 1, "method": "slow", "token": token})); + assert!(r["ok"] == false && r["running"] == true && r["error"].as_str().is_some_and(|e| e.starts_with("timeout")), "{r}"); + let r = ask(json!({"id": 2, "method": "ui.view"})); + assert!(r["busy"] == true && r["id"] == 2, "{r}"); + let r = ask(json!({"id": 3, "method": "control.pending"})); + assert_eq!((&r["result"]["pending"], &r["result"]["done"]), (&json!(1), &json!(false)), "{r}"); + gate_tx.send(()).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let r = loop { + let r = ask(json!({"id": 4, "method": "control.pending"})); + if r["result"]["done"] == true || std::time::Instant::now() > deadline { + break r; + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!( + (&r["result"]["pending"], &r["result"]["done"], &r["result"]["reply"]["result"]), + (&json!(1), &json!(true), &json!("slow")), + "{r}" + ); + let r = ask(json!({"id": 5, "method": "control.pending"})); + assert_eq!(r["result"]["pending"], Value::Null, "{r}"); + let r = ask(json!({"id": 6, "method": "ui.view"})); + assert_eq!((&r["ok"], &r["result"]), (&json!(true), &json!("ui.view")), "{r}"); + assert_eq!(*ran.lock().unwrap(), ["slow", "ui.view"], "the busy request never reached the app"); + } + + /// #6985: a connection that sends `control.subscribe` gets the design in the reply, then a + /// `design.changed` line each time the window publishes another: never before the reply, + /// none for the same design, the latest after a burst, each line whole between its replies. + /// A second subscribe is refused without reaching the window. A connection that did not + /// subscribe gets no events, and one that closed is no longer counted. + #[test] + fn subscribed_connections_are_told_of_design_changes() { + static WATCH: DocumentWatch = DocumentWatch::new(); + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The window: it answers `control.subscribe` as the app does, publishing the design first; + // then the design changes at once, before the reply may have been written. + std::thread::spawn(move || { + for req in rx { + if req.method == "control.subscribe" { + WATCH.publish("d-1", 3); + let _ = req.reply.send(json!({"ok": true, "result": {"document_id": "d-1", "revision": 3}})); + WATCH.publish("d-1", 4); + } else { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + } + }); + let hub = hub(Waits::default(), &WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let open = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + (s.try_clone().unwrap(), BufReader::new(s)) + }; + let next = |lines: &mut BufReader| -> Value { + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let changed = |id: &str, revision: u64| json!({"event": "design.changed", "params": {"document_id": id, "revision": revision}}); + + let (mut quiet_out, mut quiet) = open(); + writeln!(quiet_out, "{}", json!({"id": 1, "method": "ui.view", "token": token})).unwrap(); + assert_eq!(next(&mut quiet)["id"], 1); + + let (mut out, mut lines) = open(); + writeln!(out, "{}", json!({"id": 1, "method": "control.subscribe", "token": token})).unwrap(); + assert_eq!(next(&mut lines), json!({"id": 1, "ok": true, "result": {"document_id": "d-1", "revision": 3}})); + assert_eq!(next(&mut lines), changed("d-1", 4), "the change after the window's answer comes after the reply"); + assert!(WATCH.subscribed()); + // The design it was told of is no news; another revision is. + WATCH.publish("d-1", 4); + WATCH.publish("d-1", 5); + assert_eq!(next(&mut lines), changed("d-1", 5)); + // A burst ends with its latest. + for revision in 6..=10 { + WATCH.publish("d-1", revision); + } + let mut last = 5; + while last < 10 { + let e = next(&mut lines); + let revision = e["params"]["revision"].as_u64().unwrap(); + assert!(e["event"] == "design.changed" && revision > last, "{e} after {last}"); + last = revision; + } + // Another tab: another id, whatever its revision. + WATCH.publish("d-2", 2); + assert_eq!(next(&mut lines), changed("d-2", 2)); + // Subscribed already: refused here (the window would have published d-1 again). + writeln!(out, "{}", json!({"id": 2, "method": "control.subscribe"})).unwrap(); + let r = next(&mut lines); + assert!(r["id"] == 2 && r["ok"] == false && r["error"].as_str().is_some_and(|e| e.starts_with("already subscribed")), "{r}"); + // Replies still come, whole, and nothing else was sent meanwhile. + writeln!(out, "{}", json!({"id": 3, "method": "ui.view"})).unwrap(); + assert_eq!(next(&mut lines), json!({"id": 3, "ok": true, "result": "ui.view"})); + writeln!(quiet_out, "{}", json!({"id": 2, "method": "ui.view"})).unwrap(); + assert_eq!(next(&mut quiet), json!({"id": 2, "ok": true, "result": "ui.view"}), "no events without subscribing"); + // Closed, it is no longer counted. Each publish asks the events thread for one more line, + // and its write to the closed connection is what retires the seat; that keeps the wait off + // the connection's own thread, which anything else this process is doing may hold up past + // the deadline (#7226). + drop((out, lines)); + let mut revision = 11; + let deadline = Instant::now() + Duration::from_secs(10); + while WATCH.subscribed() && Instant::now() < deadline { + WATCH.publish("d-1", revision); + revision += 1; + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!WATCH.subscribed()); + } + + /// #8315: a closed connection leaves the watch at once. The seat of a subscribed connection + /// goes when the connection's own thread sees the end of the stream, not only when the + /// events thread notices: that thread waits for the design to move, so a connection that + /// closed while the design stood still kept the watch counting it for a second or more. + #[test] + fn a_closed_connection_leaves_the_watch_at_once() { + static WATCH: DocumentWatch = DocumentWatch::new(); + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The window names the design once and then leaves it alone: nothing moves it, so the + // events thread has nothing to write and only the end of the connection can retire it. + std::thread::spawn(move || { + for req in rx { + if req.method == "control.subscribe" { + WATCH.publish("d-1", 3); + let _ = req.reply.send(json!({"ok": true, "result": {"document_id": "d-1", "revision": 3}})); + } else { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + } + }); + let hub = hub(Waits::default(), &WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut out = s.try_clone().unwrap(); + let mut lines = BufReader::new(s); + writeln!(out, "{}", json!({"id": 1, "method": "control.subscribe", "token": token})).unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + let reply: Value = serde_json::from_str(&l).unwrap(); + assert!(reply["ok"] == true, "{reply}"); + assert!(WATCH.subscribed(), "the connection is subscribed"); + // Let the events thread settle into its wait: the design stands still, so it has nothing + // to write, and only the end of the connection can retire it. + std::thread::sleep(Duration::from_millis(200)); + // The client goes away without asking anything else. + drop((out, lines)); + let deadline = Instant::now() + Duration::from_millis(400); + while WATCH.subscribed() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!WATCH.subscribed(), "a closed connection is not counted on the watch"); + } + + /// The token is read from a request object without building the rest of it: other fields, + /// however large or nested, are only skipped. Anything but an object with a string token is + /// no token. + #[test] + fn token_is_read_without_parsing_the_request() { + assert_eq!(token_of(r#"{"id":1,"token":"abc","params":{"a":[1,[2,{"b":null}]]}}"#).as_deref(), Some("abc")); + let flood = format!(r#"{{"params":[{}0],"token":"t"}}"#, "0,".repeat(1 << 20)); + assert_eq!(token_of(&flood).as_deref(), Some("t")); + for no in [r#"["abc"]"#, r#"{"token":5}"#, r#"{"token":"a"} x"#, r#"{"id":1}"#, "GET / HTTP/1.1", r#""token""#] { + assert_eq!(token_of(no), None, "{no}"); + } + } + + /// #205: a connection runs nothing until a request carries the launch's token; a wrong or + /// missing token, or garbage, is refused and the connection closed. Once shown, the token + /// need not be repeated on that connection. + #[test] + fn requests_run_only_after_the_token() { + use std::io::Read; + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + std::thread::spawn(move || { + for req in rx { + seen.lock().unwrap().push(req.method.clone()); + let _ = req.reply.send(json!({"ok": true, "result": null})); + } + }); + let hub = hub(Waits::default(), &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let session = |lines: &[String]| { + let mut s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + for l in lines { + let _ = writeln!(s, "{l}"); + } + let _ = s.shutdown(std::net::Shutdown::Write); + let mut text = String::new(); + let _ = s.read_to_string(&mut text); + text.lines().map(|l| serde_json::from_str::(l).unwrap()).collect::>() + }; + for first in [ + json!({"id": 1, "method": "app.quit", "params": {"force": true}}).to_string(), + json!({"id": 1, "method": "app.quit", "token": "0123456789abcdef0123456789abcdee"}).to_string(), + json!({"id": 1, "method": "app.quit", "token": "0123"}).to_string(), + json!({"id": 1, "method": "app.quit", "token": 7}).to_string(), + "not json".to_string(), + ] { + let replies = session(&[first.clone(), json!({"id": 2, "method": "app.quit", "token": token}).to_string()]); + assert_eq!(replies.len(), 1, "{first}: closed after refusing: {replies:?}"); + assert!(replies[0]["error"].as_str().is_some_and(|e| e.starts_with("unauthorized")), "{first}: {replies:?}"); + } + assert!(ran.lock().unwrap().is_empty(), "nothing ran without the token"); + let replies = session(&[ + json!({"id": 1, "method": "ui.inspect", "token": token}).to_string(), + json!({"id": 2, "method": "engine.commands"}).to_string(), + ]); + assert_eq!(replies.len(), 2, "{replies:?}"); + assert!(replies.iter().all(|r| r["ok"] == true), "{replies:?}"); + assert_eq!(*ran.lock().unwrap(), ["ui.inspect", "engine.commands"]); + } + + /// A request longer than the cap is refused instead of allocated. + #[test] + fn lines_past_the_cap_are_refused() { + let flood = vec![b'A'; MAX_LINE + 2]; + let mut r = BufReader::new(flood.as_slice()); + let e = read_line(&mut r, MAX_LINE).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert!(e.to_string().contains("request too large"), "{e}"); + } + + /// Ordinary framing: lines split on newlines, blank lines kept, no trailing newline needed. + #[test] + fn lines_split_on_newlines() { + let text = b"{\"a\":1}\n\n{\"b\":2}\nlast".to_vec(); + let mut r = BufReader::new(text.as_slice()); + let mut got = Vec::new(); + while let Some(line) = read_line(&mut r, MAX_LINE).unwrap() { + got.push(String::from_utf8(line).unwrap()); + } + assert_eq!(got, vec!["{\"a\":1}", "", "{\"b\":2}", "last"]); + assert!(read_line(&mut r, MAX_LINE).unwrap().is_none()); + } + + /// A line that spans several reads of the underlying stream still arrives whole. + #[test] + fn lines_longer_than_one_buffer_arrive_whole() { + let long = "x".repeat(200_000); + let text = format!("{long}\n{long}\n").into_bytes(); + let mut r = BufReader::with_capacity(64, text.as_slice()); + assert_eq!(read_line(&mut r, MAX_LINE).unwrap().unwrap().len(), long.len()); + assert_eq!(read_line(&mut r, MAX_LINE).unwrap().unwrap().len(), long.len()); + assert!(read_line(&mut r, MAX_LINE).unwrap().is_none()); + // The same line against a smaller cap is refused. + let mut r = BufReader::with_capacity(64, text.as_slice()); + assert!(read_line(&mut r, 1000).is_err()); + } + + /// A large reply reaches the socket in one write, not one per JSON piece (a component tree of + /// thousands of components took half a second to send), and reads back as it was. + #[test] + fn a_large_reply_is_written_at_once() { + struct Counted { + writes: usize, + bytes: Vec, + } + impl Write for Counted { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.writes += 1; + self.bytes.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let components: Vec = (0..2000).map(|i| json!({"id": i, "name": format!("Part {i}"), "bodies": ["a", "b"], "parent": 0})).collect(); + let reply = json!({"ok": true, "id": 7, "result": {"components": components}}); + let mut out = Counted { writes: 0, bytes: Vec::new() }; + send(&mut out, &reply).unwrap(); + assert_eq!(out.writes, 1); + assert_eq!(out.bytes.last(), Some(&b'\n')); + assert_eq!(serde_json::from_slice::(&out.bytes).unwrap(), reply); + } + + /// A stopped listener frees its port at once and ends the connections it had, so a channel + /// turned off in the settings leaves nothing listening and can be turned on again on the + /// same port. + #[test] + fn a_stopped_listener_closes_its_port_and_its_connections() { + use std::io::Read; + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let (tx, _rx) = channel(); + let stop = hub(Waits::default(), &DOCUMENT_WATCH).listen_stoppable(listener, "0123456789abcdef0123456789abcdef".into(), tx, false).unwrap(); + assert_eq!(stop.port(), port); + let mut client = TcpStream::connect(("127.0.0.1", port)).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(4))).unwrap(); + // Wait until the server holds the connection, so stopping has one to close. + let held = Instant::now(); + while stop.shared.lock().is_empty() && held.elapsed() < Duration::from_secs(4) { + std::thread::sleep(Duration::from_millis(5)); + } + let started = Instant::now(); + stop.stop(); + let mut byte = [0u8; 1]; + match client.read(&mut byte) { + Ok(n) => assert_eq!(n, 0, "the connection ends"), + Err(e) => { + assert!(!matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut), "the connection ends, not waits: {e}") + } + } + assert!(started.elapsed() < Duration::from_secs(3), "before the token wait runs out"); + // The listener thread closes its socket when it notices the stop; on a loaded machine that can + // take a moment after `stop()` returns. Wait for the port to be free, not for one instant. + let freed = Instant::now(); + let mut bound = TcpListener::bind(("127.0.0.1", port)); + while bound.is_err() && freed.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(10)); + bound = TcpListener::bind(("127.0.0.1", port)); + } + assert!(bound.is_ok(), "the port is free again: {:?}", bound.err()); + } +} From bea91b1861eea3889bec9d26338076f2ff31c4ac Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:55:38 +0100 Subject: [PATCH 057/209] raw: charge each DNG chunk's own output allocation against the aggregate budget (craft-libs #778) (#1884) A lossy JPEG smaller than its tile is now budgeted by its frame, but decoding still fills the whole tile, so the chunk's w x h x spp output is charged as well (and refused over MAX_SAMPLES). Regression test: a 256x256 JPEG padded with comments in a 16384x16384 tile is TooLarge instead of allocating ~1.5 GiB. --- crates/raw/src/dng_lines.rs | 8 +++++++- crates/raw/tests/it/dng_lines.rs | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/raw/src/dng_lines.rs b/crates/raw/src/dng_lines.rs index 83877492e..8da1f45d8 100644 --- a/crates/raw/src/dng_lines.rs +++ b/crates/raw/src/dng_lines.rs @@ -342,7 +342,13 @@ pub fn read(t: &Tiff, ifd: &Ifd) -> Result { for c in &chunks { let n = c.w.checked_mul(c.h).and_then(|v| v.checked_mul(spp)).ok_or(Error::TooLarge)?; let budget = decode_budget(c, &l).filter(|&b| b <= MAX_SAMPLES).ok_or(Error::TooLarge)?; - decoded_samples = decoded_samples.checked_add(budget).ok_or(Error::TooLarge)?; + // the chunk's own `w` × `h` × `spp` output is allocated whatever its coded frame declares + // (a lossy JPEG smaller than its tile still fills a whole tile), so it is charged too + let alloc = budget.max(n); + if alloc > MAX_SAMPLES { + return Err(Error::TooLarge); + } + decoded_samples = decoded_samples.checked_add(alloc).ok_or(Error::TooLarge)?; if decoded_samples > MAX_DECODED_TILE_SAMPLES { return Err(Error::TooLarge); } diff --git a/crates/raw/tests/it/dng_lines.rs b/crates/raw/tests/it/dng_lines.rs index 1cf266d67..dd10c763c 100644 --- a/crates/raw/tests/it/dng_lines.rs +++ b/crates/raw/tests/it/dng_lines.rs @@ -277,3 +277,25 @@ fn short_lossy_jpeg_strips_are_charged_their_own_rows() { let band = w * rows * 3; assert_eq!(&img.data[..band], &img.data[img.data.len() - band..]); } + +#[test] +fn a_small_lossy_jpeg_in_a_huge_tile_is_charged_the_whole_tile() { + // a 256 x 256 RGB image in one 16384 x 16384 tile whose JPEG is only 256 x 256 (padded with + // comments to pass the bytes-per-sample guard): the JPEG's frame is small, but decoding fills + // the whole tile (about 1.5 GiB of samples), so it is refused before anything is allocated + let (w, tile) = (256usize, 16384usize); + let px: Vec = (0..w * w).flat_map(|i| [(i % 251) as u8, 90, 160]).collect(); + let mut jpeg = Vec::new(); + jpeg_encoder::Encoder::new(&mut jpeg, 90).encode(&px, w as u16, w as u16, jpeg_encoder::ColorType::Rgb).unwrap(); + let mut padded = jpeg[..2].to_vec(); + let need = tile * tile * 3 / 768 + 1; + while padded.len() < need { + padded.extend_from_slice(&[0xFF, 0xFE, 0xFF, 0xFF]); + padded.extend(std::iter::repeat_n(0u8, 0xFFFF - 2)); + } + padded.extend_from_slice(&jpeg[2..]); + let mut raw = base_ifd(w, w, 8, 3, false); + raw.set(t::COMPRESSION, Value::Short(vec![34892])); + raw.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles: vec![padded] }); + assert_eq!(lines(&dng(raw, ByteOrder::Little)).unwrap_err(), Error::TooLarge); +} From b774e4d1680a69ab0dc17bc437db93fe530cce36 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:44 -0500 Subject: [PATCH 058/209] psd: add bounded PackBits recovery decoding for app cutovers (#1885) * psd: expose bounded PackBits recovery row decoding * Clarify recovery row budget and partial-error contract --- crates/psd/src/compression.rs | 32 +++++++++++++++++++++++++---- crates/psd/src/compression/tests.rs | 22 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/psd/src/compression.rs b/crates/psd/src/compression.rs index b41936a04..76891a421 100644 --- a/crates/psd/src/compression.rs +++ b/crates/psd/src/compression.rs @@ -206,10 +206,34 @@ pub mod packbits { /// `-128` no-op bytes (which are always permitted). Errors, rather than /// overflowing, when `out.len() + expected` does not fit in `usize`. pub(crate) fn decode_into(src: &[u8], expected: usize, out: &mut Vec) -> Result<()> { + decode_row(src, expected, out, false) + } + + /// Appends a recovery row: complete oversized packets are clipped and a short + /// row is zero-padded. Truncated packet payloads still fail; trailing packets + /// after the row is full are ignored. Strict decoding remains the default. + /// + /// Appended row bytes are bounded by `max_row_bytes` and the PSD decoded-size + /// limit; callers enforce their aggregate image budget. Allocation is fallible. + /// Existing output is retained; an error may leave a partially appended row. + pub fn decode_padded_into(src: &[u8], expected: usize, max_row_bytes: usize, out: &mut Vec) -> Result<()> { + if expected > max_row_bytes || expected as u64 > super::MAX_DECODED_BYTES { + return Err(PsdError::LimitExceeded("PackBits recovery row exceeds limit")); + } + out.len().checked_add(expected).ok_or(PsdError::LimitExceeded("PackBits output length overflows usize"))?; + out.try_reserve_exact(expected).map_err(|_| PsdError::OutOfMemory("PackBits recovery row allocation failed"))?; + decode_row(src, expected, out, true) + } + + fn decode_row(src: &[u8], expected: usize, out: &mut Vec, padded: bool) -> Result<()> { let target = out.len().checked_add(expected).ok_or(PsdError::LimitExceeded("PackBits output length overflows usize"))?; let mut i = 0; while out.len() < target { let Some(&h) = src.get(i) else { + if padded { + out.resize(target, 0); + break; + } return Err(PsdError::Decompress("PackBits row ended early".into())); }; i += 1; @@ -219,10 +243,10 @@ pub mod packbits { let Some(lit) = src.get(i..i + len) else { return Err(PsdError::Decompress("PackBits literal truncated".into())); }; - if len > target - out.len() { + if !padded && len > target - out.len() { return Err(PsdError::Decompress("PackBits literal overflows row".into())); } - out.extend_from_slice(lit); + out.extend_from_slice(&lit[..len.min(target - out.len())]); i += len; } else if h != -128 { let len = 1 - h as isize; @@ -231,10 +255,10 @@ pub mod packbits { return Err(PsdError::Decompress("PackBits run truncated".into())); }; i += 1; - if len > target - out.len() { + if !padded && len > target - out.len() { return Err(PsdError::Decompress("PackBits run overflows row".into())); } - out.extend(std::iter::repeat_n(b, len)); + out.extend(std::iter::repeat_n(b, len.min(target - out.len()))); } } Ok(()) diff --git a/crates/psd/src/compression/tests.rs b/crates/psd/src/compression/tests.rs index 6e0d86a7e..7cf876321 100644 --- a/crates/psd/src/compression/tests.rs +++ b/crates/psd/src/compression/tests.rs @@ -771,3 +771,25 @@ fn rle_chunks_use_the_callers_pool_and_report_panics() { assert!(map_chunks(2, vec![0, 1], |_| panic!("worker")).is_err()); }); } + +#[test] +fn padded_packbits_preserves_recovery_without_weakening_strict_decode() { + for (input, expected) in + [(&[0, 7][..], vec![7, 0, 0]), (&[3, 1, 2, 3, 4][..], vec![1, 2, 3]), (&[253, 9][..], vec![9, 9, 9]), (&[128][..], vec![0, 0, 0])] + { + let mut out = vec![42]; + super::packbits::decode_padded_into(input, 3, 3, &mut out).unwrap(); + assert_eq!(&out[1..], expected); + assert_eq!(out[0], 42); + assert!(super::packbits::decode(input, 3).is_err()); + } + for input in [&[3, 1][..], &[255][..]] { + assert!(super::packbits::decode_padded_into(input, 1, 1, &mut vec![]).is_err()); + } + let mut out = vec![42]; + assert!(super::packbits::decode_padded_into(&[], 4, 3, &mut out).is_err()); + assert_eq!(out, [42]); + super::packbits::decode_padded_into(&[255], 0, 0, &mut out).unwrap(); + super::packbits::decode_padded_into(&[0, 7, 255], 1, 1, &mut out).unwrap(); + assert_eq!(out, [42, 7]); +} From e44720af47b02670b3ec42da3b65473de77a78c1 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:02:40 +0100 Subject: [PATCH 059/209] media: refuse a truncated file on a FileReader cache hit file() only checked length when reopening after eviction, so a prefix read could still succeed if the handle was pooled. Check metadata on the cached handle too, and drop the entry if the file is shorter or no longer a regular file. MAX_OPEN_FILES is documented as a bound on retained pool entries, not in-flight Arc handles. --- crates/media/src/file.rs | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/media/src/file.rs b/crates/media/src/file.rs index d87b323c0..f343737b3 100644 --- a/crates/media/src/file.rs +++ b/crates/media/src/file.rs @@ -1,8 +1,9 @@ //! Pooled file [`ByteSource`]s: many clips can be open at once without holding a descriptor each. //! -//! Past [`MAX_OPEN_FILES`] the least recently read file is closed and reopened by path on demand, -//! so a project with hundreds of sources does not hit the process file-descriptor limit (often 256 -//! on macOS, 1024 on Linux). +//! Past [`MAX_OPEN_FILES`] retained pool entries, the least recently read file is closed and +//! reopened by path on demand, so a project with hundreds of sources does not hit the process +//! file-descriptor limit (often 256 on macOS, 1024 on Linux). The cap does not bound in-flight +//! `Arc` handles: a file another thread is mid-read stays open until that read ends. #![cfg(not(target_arch = "wasm32"))] @@ -63,8 +64,10 @@ fn not_a_file(path: &Path) -> io::Error { io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())) } -/// Files [`FileReader`]s keep open at most, together. Past this many, the least recently read file -/// is closed and its reader reopens it on demand. +/// Retained pool entries [`FileReader`]s keep open at most, together. Past this many, the least +/// recently read file is closed and its reader reopens it on demand. The cap does not bound +/// in-flight `Arc` handles: a file another thread is mid-read stays open until that read ends, +/// even if the pool has already evicted its entry. pub const MAX_OPEN_FILES: usize = 128; /// The open files of a set of [`FileReader`]s, least recently used first. @@ -172,7 +175,12 @@ impl FileReader { /// being written) is read up to the opened length. fn file(&self) -> io::Result> { if let Some(file) = self.pool.get(self.id) { - return Ok(file); + let m = file.metadata()?; + if m.is_file() && m.len() >= self.len { + return Ok(file); + } + self.pool.remove(self.id); + return Err(io::Error::other(format!("{} changed since it was opened", self.path.display()))); } let file = open_nonblocking(&self.path)?; let m = file.metadata()?; @@ -331,4 +339,23 @@ mod tests { ); std::fs::remove_dir_all(&dir).ok(); } + + /// A file truncated while its handle is still in the pool is refused, same as after eviction. + #[test] + fn a_truncated_file_is_refused_while_still_pooled() { + let dir = craft_media_testkit::private_dir("file-trunc-hit"); + let path = dir.join("clip.bin"); + std::fs::write(&path, b"0123456789").unwrap(); + let pool = Arc::new(FilePool::new(2)); + let reader = FileReader::open_in(&path, &pool).unwrap(); + assert_eq!(pool.open_count(), 1); + let mut b = [0u8; 5]; + reader.read_at(0, &mut b).unwrap(); + assert_eq!(&b, b"01234"); + std::fs::write(&path, b"short").unwrap(); + assert!(reader.read_at(0, &mut b).is_err(), "truncation must be refused on a cache hit too"); + drop(reader); + assert_eq!(pool.open_count(), 0); + std::fs::remove_dir_all(&dir).ok(); + } } From ed850ede0f26ef0b8ff9b4d1fc08466a1cf21375 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 10:06:40 -0500 Subject: [PATCH 060/209] stream-codecs: append TIFF PackBits rows with shared packet encoder --- crates/stream-codecs/src/runlength.rs | 39 ++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/stream-codecs/src/runlength.rs b/crates/stream-codecs/src/runlength.rs index a6df9e19d..dceb78f66 100644 --- a/crates/stream-codecs/src/runlength.rs +++ b/crates/stream-codecs/src/runlength.rs @@ -43,6 +43,21 @@ pub fn decode(data: &[u8], max: usize) -> Step { pub fn encode(data: &[u8]) -> Vec { let mut out = Vec::with_capacity(data.len() + data.len() / 128 + 2); + encode_packets(data, &mut out, 2); + out.push(128); + out +} + +/// Append a TIFF PackBits row without an end-of-data marker. +/// +/// Runs of three or more bytes use repeat packets; pairs remain in literal packets. +/// Existing output is retained, permitting rows to be appended directly to a strip. +/// PostScript callers can append their own EOD byte after the final row. +pub fn encode_packbits_into(data: &[u8], out: &mut Vec) { + encode_packets(data, out, 3); +} + +fn encode_packets(data: &[u8], out: &mut Vec, repeat_min: usize) { let n = data.len(); let mut i = 0; while i < n { @@ -50,7 +65,7 @@ pub fn encode(data: &[u8]) -> Vec { while i + run < n && run < 128 && data[i + run] == data[i] { run += 1; } - if run >= 2 { + if run >= repeat_min { out.push((257 - run) as u8); out.push(data[i]); i += run; @@ -59,15 +74,14 @@ pub fn encode(data: &[u8]) -> Vec { // Literal: extend until a repeat begins or 128 bytes. let start = i; let mut j = i + 1; - while j < n && j - start < 128 && !(j + 1 < n && data[j] == data[j + 1]) { + while j < n && j - start < 128 && !data.get(j..).and_then(|rest| rest.get(..repeat_min)).is_some_and(|run| run.iter().all(|b| *b == data[j])) + { j += 1; } out.push((j - start - 1) as u8); out.extend_from_slice(&data[start..j]); i = j; } - out.push(128); - out } #[cfg(test)] @@ -114,4 +128,21 @@ mod tests { assert_eq!((e[0], e[129]), (127, 71)); assert_eq!(dec(&e).unwrap(), lit); } + #[test] + fn packbits_appends_rows_without_eod_and_keeps_pairs_literal() { + let mut out = vec![42]; + encode_packbits_into(b"aabbccc", &mut out); + assert_eq!(out, [42, 3, b'a', b'a', b'b', b'b', 254, b'c']); + let before = out.clone(); + encode_packbits_into(b"", &mut out); + assert_eq!(out, before); + assert_eq!(encode(b"aa"), [255, b'a', 128]); + for n in [1, 2, 3, 127, 128, 129, 255, 256, 300] { + for row in [vec![7; n], (0..n).map(|i| (i % 251) as u8).collect()] { + let mut encoded = Vec::new(); + encode_packbits_into(&row, &mut encoded); + assert_eq!(decode(&encoded, n).unwrap(), row); + } + } + } } From 9fc471fc5c18f467394a043adf17f021c2b27cae Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:09:54 +0100 Subject: [PATCH 061/209] psd: add Descriptor::parse_prefix for mid-stream reads (#1887) from_bytes still requires the slice to be fully consumed. parse_prefix returns the descriptor and the number of bytes used, matching VersionedDescriptor, so apps can read a descriptor that is followed by more data (layer effects, fill and adjustment blocks). Part of #778 and effectcraft#786. --- crates/psd/src/descriptor.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs index f3afae64a..96ad47a7b 100644 --- a/crates/psd/src/descriptor.rs +++ b/crates/psd/src/descriptor.rs @@ -300,6 +300,15 @@ impl Descriptor { Ok(d) } + /// Parses a descriptor at the start of `data`, returning it and the number of bytes consumed + /// (trailing data is allowed). Mid-stream reads of layer effects, fill and adjustment + /// descriptors use this; [`from_bytes`] still requires the slice to be fully consumed. + pub fn parse_prefix(data: &[u8]) -> Result<(Self, usize)> { + let mut r = Reader::new(data); + let d = Self::read(&mut r)?; + Ok((d, r.pos())) + } + /// Serializes the descriptor. pub fn to_bytes(&self) -> Vec { let mut v = Vec::new(); @@ -705,6 +714,18 @@ mod tests { assert_eq!(VersionedDescriptor::from_bytes(&b).unwrap(), v); } + #[test] + fn parse_prefix() { + let d = sample(); + let mut b = d.to_bytes(); + let n = b.len(); + b.extend_from_slice(&[1, 2, 3]); + let (p, used) = Descriptor::parse_prefix(&b).unwrap(); + assert_eq!(p, d); + assert_eq!(used, n); + assert!(Descriptor::from_bytes(&b).is_err(), "from_bytes still refuses trailing bytes"); + } + #[test] fn versioned_parse_prefix() { let v = VersionedDescriptor::new(sample()); From c5f217e76518f15ae27ac5ec615ed5d0735d6a90 Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 16:10:03 +0100 Subject: [PATCH 062/209] craft-docx: a shape's VML fallback takes its anchor's relativeHeight Main's VML fallback (#1864) read the writer's z counter after the anchor was written. That counter now only counts objects without a stored order, so a stored order's fallback got the wrong z-index. The anchor's relativeHeight is now kept in anchor_z for the fallback, and the wps_fallback tests expect the written relativeHeight (one less than Float::z). --- crates/docx/src/write.rs | 3 +++ crates/docx/src/write/parts.rs | 1 + crates/docx/src/write/story/drawing.rs | 1 + crates/docx/src/write/story/object.rs | 5 +++-- crates/docx/tests/it/wps_fallback.rs | 6 ++++-- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/docx/src/write.rs b/crates/docx/src/write.rs index 634f5f833..0391ba615 100644 --- a/crates/docx/src/write.rs +++ b/crates/docx/src/write.rs @@ -120,6 +120,9 @@ pub(crate) struct Writer<'d> { /// Stored stacking orders written as other `relativeHeight`s (see [`stacking`]); empty: each /// as its own. zmap: BTreeMap, + /// The `relativeHeight` the last `wp:anchor` was written with, which a shape's VML fallback + /// gives as its `z-index`. + anchor_z: u32, /// Note part id → file note id, in first-reference order. footnotes: Vec, endnotes: Vec, diff --git a/crates/docx/src/write/parts.rs b/crates/docx/src/write/parts.rs index 00826f4b5..fa6d6b693 100644 --- a/crates/docx/src/write/parts.rs +++ b/crates/docx/src/write/parts.rs @@ -33,6 +33,7 @@ pub(crate) fn write_parts(doc: &Document, flavor: Flavor) -> Result { self.z = self.z.saturating_add(1); self.z }; + self.anchor_z = z; let behind = if float.wrap == Wrap::BehindText { "1" } else { "0" }; w.open( "wp:anchor", diff --git a/crates/docx/src/write/story/object.rs b/crates/docx/src/write/story/object.rs index 4b0ae856a..3a42a8b04 100644 --- a/crates/docx/src/write/story/object.rs +++ b/crates/docx/src/write/story/object.rs @@ -398,8 +398,9 @@ impl Writer<'_> { } let float = &float; self.drawing_open(w, DrawingInfo { float, size: (*sw, *sh), docpr: &docpr, name: &name, alt: o.alt_text(), link: link.as_deref() }); - // This shape's stacking order: floating objects in its text box move `self.z` on. - let z = self.z; + // This shape's stacking order, as its anchor gave it: floating objects in its text box + // write anchors of their own. + let z = self.anchor_z; w.empty("wp:cNvGraphicFramePr", &[]); w.open("a:graphic", &[]); w.open("a:graphicData", &[("uri", "http://schemas.microsoft.com/office/word/2010/wordprocessingShape")]); diff --git a/crates/docx/tests/it/wps_fallback.rs b/crates/docx/tests/it/wps_fallback.rs index 9b209082b..6d3deddb1 100644 --- a/crates/docx/tests/it/wps_fallback.rs +++ b/crates/docx/tests/it/wps_fallback.rs @@ -107,7 +107,8 @@ fn the_fallback_keeps_order_visibility_and_direction() { // is copied into the outer one's text, so it comes after it). let tb = xml.find(r#""#).expect("the outer box's text, turned"); let rect = &xml[xml[..tb].rfind(" Date: Mon, 16 Nov 2026 10:21:41 -0500 Subject: [PATCH 063/209] test: report PackBits codec errors without requiring Failure Debug --- crates/stream-codecs/src/runlength.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/stream-codecs/src/runlength.rs b/crates/stream-codecs/src/runlength.rs index dceb78f66..a13a1d4fb 100644 --- a/crates/stream-codecs/src/runlength.rs +++ b/crates/stream-codecs/src/runlength.rs @@ -141,7 +141,7 @@ mod tests { for row in [vec![7; n], (0..n).map(|i| (i % 251) as u8).collect()] { let mut encoded = Vec::new(); encode_packbits_into(&row, &mut encoded); - assert_eq!(decode(&encoded, n).unwrap(), row); + assert_eq!(decode(&encoded, n).map_err(|failure| failure.error).unwrap(), row); } } } From 45ad9df6370213c1e1b0ad242d1c74c413963e15 Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 16:21:55 +0100 Subject: [PATCH 064/209] craft-xlsx: decode XLSB defined names after the workbook's tables are indexed A defined name's formula may use a structured reference (PtgList). Names were decoded inside xlsb::workbook, before Package::workbook indexed the table parts, so such a name was left out and the cells using it kept only their saved results. xlsb::workbook now returns the names undecoded (Names), and Package::workbook decodes them with xlsb::read_names after index_tables. Regression: a_name_using_a_structured_reference_is_decoded_after_tables_are_indexed fails without the reorder. --- crates/xlsx/src/package.rs | 4 +++- crates/xlsx/src/xlsb.rs | 28 +++++++++++++++++++++------- crates/xlsx/tests/xlsb.rs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index ce7d7756f..a75874e71 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -149,10 +149,12 @@ impl<'a> Package<'a> { if self.binary.is_some() { let bytes = self.required(&self.workbook_part.clone())?; let rels = self.workbook_rels()?; - let (wb, mut links) = xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings)?; + let (mut wb, mut links, names) = xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings)?; if let Some(links) = links.as_mut() { self.index_tables(&wb, links)?; } + // Names after tables: a name's formula may use a structured reference. + xlsb::read_names(&mut wb, names, links.as_mut(), &self.limits, &self.budget, &mut self.warnings); self.binary_links = Some(links); return Ok(wb); } diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 7ec6be839..b45b48808 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -289,14 +289,16 @@ impl<'a, 'l> Records<'a, 'l> { /// The workbook part: BrtBeginBook, BrtWbProp, the BrtBundleSh list. /// The workbook part, and the tables its formulas refer to ([`Links`]): `None` when those can't -/// be read, so that the workbook still opens, with every formula kept as its saved result. +/// be read, so that the workbook still opens, with every formula kept as its saved result. Its +/// defined names come back undecoded ([`Names`]): [`read_names`] decodes them once the workbook's +/// tables are indexed, so a name's formula can use a structured reference. pub(crate) fn workbook( bytes: &[u8], rels: &Rels, limits: &Limits, budget: &Budget, warnings: &mut Warnings, -) -> Result<(Workbook, Option), Error> { +) -> Result<(Workbook, Option, Names), Error> { let mut records = Records::new(bytes, limits, budget); records.first(131, warnings)?; let mut wb = Workbook::default(); @@ -320,11 +322,7 @@ pub(crate) fn workbook( if wb.sheets.is_empty() { return Err(invalid("no supported worksheets")); } - match links.as_mut() { - Some(l) => wb.defined_names = defined_names(pending, &bundle_sheets, l, limits, budget, warnings), - None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), - } - return Ok((wb, links)); + return Ok((wb, links, Names { pending, sheets: bundle_sheets })); } 39 | 355 | 357 | 358 | 362 | 667 => { if let Some(l) = links.as_mut() @@ -423,6 +421,22 @@ struct PendingName { help: Option, } +/// The workbook part's defined names, not yet decoded, and each BrtBundleSh's place in +/// [`Workbook::sheets`] (`None` for a skipped one). +pub(crate) struct Names { + pending: Vec, + sheets: Vec>, +} + +/// Decodes the workbook part's defined names `names` into `wb`, after the tables in `links` are +/// indexed. Without `links` none can be read, and formulas keep their saved results. +pub(crate) fn read_names(wb: &mut Workbook, names: Names, links: Option<&mut Links>, limits: &Limits, budget: &Budget, warnings: &mut Warnings) { + match links { + Some(l) => wb.defined_names = defined_names(names.pending, &names.sheets, l, limits, budget, warnings), + None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), + } +} + /// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the /// hidden ones for newer functions and LET parameters, which Excel doesn't write there. `sheets` /// maps a name's `itab` (its sheet's place in `BrtBundleSh` order) to [`Workbook::sheets`]. diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 16220df9e..cd5494c03 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1803,6 +1803,42 @@ fn tables_of(bytes: &[u8]) -> (Vec
, Vec) { (tables, p.warnings().to_vec()) } +#[test] +fn a_name_using_a_structured_reference_is_decoded_after_tables_are_indexed() { + // `Vals` = `Sales[Val]`. Names are decoded after the workbook's tables are indexed, so + // the formula and a cell that uses it are kept, instead of only their saved results. + let name = brt_name(0, 0, WORKBOOK, "Vals", &ptg_list(0, 1, 1, 1, 1), None); + let list = [record(660, &1u32.to_le_bytes()), record(661, &wide("t1")), record(662, &[])]; + let mut data = row(0); + data.extend(formula_cell(1, &[0x23, 5, 0, 0, 0], &[])); // =Vals, after the four names of `book_with_links` + let types = { + let t = TYPES.trim_end_matches(""); + format!(r#"{t}"#) + }; + let sheet_rels = r#""#; + let (one, two, strings, table) = (worksheet(&data, &list.concat()), worksheet(&[], &[]), sst(), sales(&sales_filters())); + let book = { + let plain = book_with_links(&xti()); + let end = record(132, &[]); + [&plain[..plain.len() - end.len()], name.as_slice(), end.as_slice()].concat() + }; + let bytes = zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT.as_bytes()), + ("records/book.bin", book.as_slice()), + ("records/_rels/book.bin.rels", RELS.as_bytes()), + ("data/one.bin", one.as_slice()), + ("data/_rels/one.bin.rels", sheet_rels.as_bytes()), + ("data/two.bin", two.as_slice()), + ("text/strings.bin", strings.as_slice()), + ("tables/t1.bin", table.as_slice()), + ]); + let names = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap().defined_names; + assert_eq!(names.iter().find(|n| n.name == "Vals").map(|n| n.formula.as_str()), Some("Sales[Val]")); + let wb = read(&bytes).unwrap(); + assert_eq!(wb.sheets[0].formula(CellRef::new(0, 1)), Some("Vals"), "{:?}", wb.warnings); +} + #[test] fn tables_are_read_as_their_xlsx_spelling() { // Spacing flags: 0x80 inside the outer brackets, 0x100 after commas. From dd3015c678052b9a1f1732e70f83576c0a72e75f Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:23:22 +0100 Subject: [PATCH 065/209] wordmodel: keep the kashida length of a justified paragraph (#1875) * wordmodel: keep the kashida length of a justified paragraph ST_Jc lowKashida, mediumKashida and highKashida were all read as Align::Justify, so the length was lost. ParaProps and ResolvedPara now keep it; a direct both/justify cancels an inherited length. The DOCX reader and writer round-trip the three values. * docx: keep a style's kashida length when writing justify w:jc encodes alignment and kashida together. A paragraph or style with Align::Justify and kashida unset still inherits a length from the style chain; writing "both" would read back as Some(Kashida::None) and cancel it. Use the resolved length in that case. --- crates/docx/src/read/props.rs | 11 +- crates/docx/src/write.rs | 5 + crates/docx/src/write/props.rs | 11 +- crates/docx/src/write/story/para.rs | 6 + crates/docx/src/write/styles.rs | 8 +- crates/docx/tests/it/kashida.rs | 136 +++++++++++++++++++++++ crates/docx/tests/it/main.rs | 1 + crates/docx/tests/it/roundtrip.rs | 5 +- crates/wordmodel/src/props.rs | 2 +- crates/wordmodel/src/props/para_props.rs | 45 +++++++- crates/wordmodel/src/resolve.rs | 35 +++++- 11 files changed, 254 insertions(+), 11 deletions(-) create mode 100644 crates/docx/tests/it/kashida.rs diff --git a/crates/docx/src/read/props.rs b/crates/docx/src/read/props.rs index dc61b3a28..4d86ae164 100644 --- a/crates/docx/src/read/props.rs +++ b/crates/docx/src/read/props.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use craft_wordmodel::para::cap_alt; use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CellProps, CharProps, CombineBrackets, EastAsianLayout, HeightRule, Highlight, LineSpacing, NumRef, + Align, Border, BorderStyle, Borders, CellProps, CharProps, CombineBrackets, EastAsianLayout, HeightRule, Highlight, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, RowProps, TabAlign, TabLeader, TabStop, TableFloat, TableLook, TableProps, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, }; @@ -465,7 +465,14 @@ impl PropCtx { _ => {} } } - "w:jc" => p.align = k.attr("w:val").map(align), + "w:jc" => { + if let Some(v) = k.attr("w:val") { + p.align = Some(align(v)); + if let Some(kashida) = Kashida::from_jc(v) { + p.kashida = Some(kashida); + } + } + } "w:outlineLvl" => p.outline_level = k.attr("w:val").and_then(u32_of).map(|v| v.min(9) as u8), "w:rPr" => mark = self.rpr(k), _ => {} diff --git a/crates/docx/src/write.rs b/crates/docx/src/write.rs index 1d9eb0818..4ba486a6c 100644 --- a/crates/docx/src/write.rs +++ b/crates/docx/src/write.rs @@ -151,6 +151,11 @@ pub fn write_as(doc: &Document, flavor: Flavor) -> Result, DocxError> { } impl Writer<'_> { + /// The document's styles, for writers that need the resolved style chain. + pub(crate) fn styles(&self) -> &craft_wordmodel::styles::StyleSheet { + &self.doc.styles + } + fn assign_media(&mut self) { let mut used: std::collections::HashSet = std::collections::HashSet::new(); for (key, bytes) in &self.doc.media { diff --git a/crates/docx/src/write/props.rs b/crates/docx/src/write/props.rs index 0b92ce191..fb76da8d8 100644 --- a/crates/docx/src/write/props.rs +++ b/crates/docx/src/write/props.rs @@ -1,8 +1,8 @@ //! Property writers, in ECMA-376 schema element order. use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CellProps, CharProps, EastAsianLayout, HeightRule, Highlight, LineSpacing, ParaProps, RowProps, TabAlign, - TabLeader, TableFloat, TableLook, TableProps, TextColor, VAlign, VMerge, VertAlign, + Align, Border, BorderStyle, Borders, CellProps, CharProps, EastAsianLayout, HeightRule, Highlight, Kashida, LineSpacing, ParaProps, RowProps, + TabAlign, TabLeader, TableFloat, TableLook, TableProps, TextColor, VAlign, VMerge, VertAlign, }; use crate::units::{n, twips}; @@ -201,6 +201,11 @@ pub fn align_val(a: Align) -> &'static str { } } +/// `w:jc` for a paragraph, including the kashida length of a justified one. +pub fn jc_val(a: Align, kashida: Option) -> &'static str { + if a == Align::Justify { kashida.unwrap_or(Kashida::None).jc() } else { align_val(a) } +} + /// A tab stop's or positional tab's `w:leader` value. pub fn tab_leader(l: TabLeader) -> &'static str { match l { @@ -287,7 +292,7 @@ pub fn ppr_inner(w: &mut W, p: &ParaProps, framed: bool, num_change: Option<&Cha toggle(w, "w:contextualSpacing", p.contextual_spacing); toggle(w, "w:mirrorIndents", p.mirror_indents); if let Some(a) = p.align { - w.val("w:jc", align_val(a)); + w.val("w:jc", jc_val(a, p.kashida)); } if let Some(direction) = p.text_direction { w.val("w:textDirection", direction.ooxml()); diff --git a/crates/docx/src/write/story/para.rs b/crates/docx/src/write/story/para.rs index e0242d8eb..aa973184e 100644 --- a/crates/docx/src/write/story/para.rs +++ b/crates/docx/src/write/story/para.rs @@ -76,6 +76,12 @@ impl Writer<'_> { a.push(("w:original", c.original.clone())); a }); + // `w:jc` encodes alignment and kashida length together. A justified paragraph + // whose kashida is unset (`None`) still inherits a length from its style; writing + // `both` would read back as `Some(Kashida::None)` and cancel that length. + if pp.align == Some(craft_wordmodel::props::Align::Justify) && pp.kashida.is_none() { + pp.kashida = Some(self.styles().resolve_para(&pp).kashida); + } ppr_inner(w, &pp, framed, num_change.as_ref()); if has_mark { self.write_mark_rpr(w, &p.mark, &mark_revs); diff --git a/crates/docx/src/write/styles.rs b/crates/docx/src/write/styles.rs index 3bcf9ab97..0fddf33b4 100644 --- a/crates/docx/src/write/styles.rs +++ b/crates/docx/src/write/styles.rs @@ -184,7 +184,13 @@ fn style_property_xml(wr: &mut Writer<'_>, w: &mut W, st: &Style) { let para_change = st.para_change.as_deref().filter(|ch| ch.old.formatting() != st.para.formatting()); if (!st.para.is_empty() || para_change.is_some()) && st.kind != StyleKind::Character { w.open("w:pPr", &[]); - props::ppr_inner(w, &st.para, false, None); + let mut ppr = st.para.clone(); + if ppr.align == Some(craft_wordmodel::props::Align::Justify) && ppr.kashida.is_none() { + let mut probe = ppr.clone(); + probe.style = Some(st.id.clone()); + ppr.kashida = Some(wr.styles().resolve_para(&probe).kashida); + } + props::ppr_inner(w, &ppr, false, None); if let Some(ch) = para_change { let a = wr.change_attrs(ch.rev); open_change(w, "w:pPrChange", &a); diff --git a/crates/docx/tests/it/kashida.rs b/crates/docx/tests/it/kashida.rs new file mode 100644 index 000000000..02eaf2a49 --- /dev/null +++ b/crates/docx/tests/it/kashida.rs @@ -0,0 +1,136 @@ +//! Justified paragraphs keep their kashida length (`ST_Jc` `lowKashida`, `mediumKashida`, +//! `highKashida`, and ordinary `both` / `justify`) through read and write. + +use std::io::{Read, Write}; + +use std::sync::Arc; + +use craft_wordmodel::props::{Align, Kashida, ParaProps}; +use craft_wordmodel::styles::{Style, StyleKind, StyleSheet}; +use craft_wordmodel::{Document, Paragraph, para_block}; + +fn zip(entries: &[(&str, &[u8])]) -> Vec { + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, bytes) in entries { + zw.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +fn docx(body: &str) -> Vec { + let root = r#""#; + let doc = format!( + r#"{body}"# + ); + zip(&[("_rels/.rels", root.as_bytes()), ("word/document.xml", doc.as_bytes())]) +} + +fn paras(d: &Document) -> Vec<&Paragraph> { + d.body.iter().filter_map(|b| b.as_para()).collect() +} + +fn document_xml(bytes: &[u8]) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut s = String::new(); + z.by_name("word/document.xml").unwrap().read_to_string(&mut s).unwrap(); + s +} + +fn jc_para(body: &str, val: &str) -> String { + format!(r#"{body}"#) +} + +#[test] +fn kashida_jc_values_are_read() { + let body = [ + jc_para("both", "both"), + jc_para("justify", "justify"), + jc_para("low", "lowKashida"), + jc_para("medium", "mediumKashida"), + jc_para("high", "highKashida"), + jc_para("left", "left"), + ] + .join(""); + let d = craft_docx::read(&docx(&body)).unwrap(); + let ps = paras(&d); + let got: Vec<_> = ps.iter().map(|p| (p.props.align, p.props.kashida)).collect(); + assert_eq!( + got, + vec![ + (Some(Align::Justify), Some(Kashida::None)), + (Some(Align::Justify), Some(Kashida::None)), + (Some(Align::Justify), Some(Kashida::Short)), + (Some(Align::Justify), Some(Kashida::Medium)), + (Some(Align::Justify), Some(Kashida::Long)), + (Some(Align::Left), None), + ] + ); +} + +#[test] +fn kashida_length_round_trips() { + let cases = [ + (Some(Kashida::None), "w:val=\"both\""), + (Some(Kashida::Short), "w:val=\"lowKashida\""), + (Some(Kashida::Medium), "w:val=\"mediumKashida\""), + (Some(Kashida::Long), "w:val=\"highKashida\""), + ]; + for (kashida, token) in cases { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { align: Some(Align::Justify), kashida, ..Default::default() }; + let mut d = Document::new(); + d.body = vec![para_block(p.clone())]; + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains(token), "expected {token} in {xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.align, Some(Align::Justify)); + assert_eq!(q.props.kashida, kashida, "round-trip of {token}"); + } +} + +fn body_style(kashida: Kashida) -> StyleSheet { + let mut sheet = StyleSheet::empty(); + sheet.styles.push(Style { + id: "Body".into(), + name: "Body".into(), + kind: StyleKind::Paragraph, + para: ParaProps { align: Some(Align::Justify), kashida: Some(kashida), ..ParaProps::default() }, + ..Style::default() + }); + sheet +} + +#[test] +fn style_kashida_length_survives_a_write_when_the_paragraph_has_no_direct_jc() { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { style: Some("Body".into()), ..ParaProps::default() }; + let mut d = Document::new(); + d.styles = Arc::new(body_style(Kashida::Short)); + d.body = vec![para_block(p)]; + assert_eq!(d.styles.resolve_para(&d.body[0].as_para().unwrap().props).kashida, Kashida::Short); + let bytes = craft_docx::write(&d).unwrap(); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.kashida, None, "no direct w:jc on the paragraph"); + assert_eq!(r.styles.resolve_para(&q.props).kashida, Kashida::Short); +} + +#[test] +fn writing_justify_without_a_local_kashida_keeps_the_style_length() { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { style: Some("Body".into()), align: Some(Align::Justify), kashida: None, ..ParaProps::default() }; + let mut d = Document::new(); + d.styles = Arc::new(body_style(Kashida::Medium)); + d.body = vec![para_block(p)]; + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains("w:val=\"mediumKashida\""), "expected mediumKashida in {xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.align, Some(Align::Justify)); + assert_eq!(q.props.kashida, Some(Kashida::Medium)); + assert_eq!(r.styles.resolve_para(&q.props).kashida, Kashida::Medium); +} diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index 8696981c7..8d6be8a97 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -20,6 +20,7 @@ mod emphasis_marks; mod fixtures; mod glossary; mod heading_levels; +mod kashida; mod legacy_forms; mod ligatures; mod line_numbering; diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 65a01e6b4..2037f9849 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use craft_wordmodel::numbering::ListKind; use craft_wordmodel::para::{Anchor, Float, FloatAlign, NoteKind, ShapeKind, Wrap}; use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CharProps, HeightRule, Highlight, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, RowProps, - TabAlign, TabLeader, TabStop, TableLook, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, + Align, Border, BorderStyle, Borders, CharProps, HeightRule, Highlight, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, + RowProps, TabAlign, TabLeader, TabStop, TableLook, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, }; use craft_wordmodel::section::{Columns, DocGrid, DocGridType, LineNumberRestart, LineNumbering, NumFormat, PageVAlign, SectionProps, SectionStart}; use craft_wordmodel::styles::{Style, StyleKind}; @@ -163,6 +163,7 @@ fn every_para_prop_round_trips() { let pp = ParaProps { style: Some("Heading1".into()), align: Some(Align::Justify), + kashida: Some(Kashida::Medium), indent_left: Some(36.0), indent_right: Some(18.0), indent_first: Some(-18.0), diff --git a/crates/wordmodel/src/props.rs b/crates/wordmodel/src/props.rs index 65a59592c..318779acb 100644 --- a/crates/wordmodel/src/props.rs +++ b/crates/wordmodel/src/props.rs @@ -27,7 +27,7 @@ pub use char_props::{ }; pub use color::{Rgb, TextColor}; pub use para_props::{ - Align, Border, BorderStyle, Borders, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, + Align, Border, BorderStyle, Borders, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, TextAlignment, TextFrame, }; pub use table_props::{ diff --git a/crates/wordmodel/src/props/para_props.rs b/crates/wordmodel/src/props/para_props.rs index a48540248..b3dec00ad 100644 --- a/crates/wordmodel/src/props/para_props.rs +++ b/crates/wordmodel/src/props/para_props.rs @@ -26,6 +26,45 @@ impl Align { } } +/// The kashida (tatweel) length a justified paragraph uses (`ST_Jc` in ECMA-376 §17.18.44). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Kashida { + /// No kashidas: justified lines widen their word spaces (`both`, `justify`). + #[default] + None, + /// `lowKashida`. + Short, + /// `mediumKashida`. + Medium, + /// `highKashida`. + Long, +} + +impl Kashida { + /// The length named by a `w:jc` value, if that value is a kashida or ordinary justify form. + /// Other alignments return `None` so they leave an inherited length. + pub fn from_jc(value: &str) -> Option { + match value { + "both" | "justify" => Some(Kashida::None), + "lowKashida" => Some(Kashida::Short), + "mediumKashida" => Some(Kashida::Medium), + "highKashida" => Some(Kashida::Long), + _ => None, + } + } + + /// The `w:jc` value for a justified paragraph of this length. + pub fn jc(self) -> &'static str { + match self { + Kashida::None => "both", + Kashida::Short => "lowKashida", + Kashida::Medium => "mediumKashida", + Kashida::Long => "highKashida", + } + } +} + /// Paragraph-level text flow (`w:pPr/w:textDirection`, ECMA-376 §17.3.1.41). /// /// Kept separate from [`super::TextDirection`], whose three values model the table-cell direction @@ -383,6 +422,10 @@ pub struct ParaProps { pub style: Option, #[serde(skip_serializing_if = "Option::is_none")] pub align: Option, + /// Kashida length for a justified paragraph. `None` inherits; `Some(Kashida::None)` is + /// ordinary justification (`both` / `justify`) and cancels an inherited length. + #[serde(skip_serializing_if = "Option::is_none")] + pub kashida: Option, /// Left indent, points. #[serde(skip_serializing_if = "Option::is_none")] pub indent_left: Option, @@ -516,7 +559,7 @@ impl ParaProps { } fn overlay_layout(&mut self, patch: &ParaProps) { - overlay_fields!(self, patch; style, align, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, + overlay_fields!(self, patch; style, align, kashida, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, space_before, space_after, space_before_auto, space_after_auto, line_spacing, contextual_spacing, mirror_indents, adjust_right_indent, tabs, shading, borders); } diff --git a/crates/wordmodel/src/resolve.rs b/crates/wordmodel/src/resolve.rs index 7ef12ccdc..85a7a1550 100644 --- a/crates/wordmodel/src/resolve.rs +++ b/crates/wordmodel/src/resolve.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::props::{ - Align, Border, Borders, CharProps, Highlight, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, + Align, Border, Borders, CharProps, Highlight, Kashida, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, }; use crate::styles::{StyleKind, StyleSheet}; @@ -173,6 +173,9 @@ fn default_adjust_right_indent() -> bool { pub struct ResolvedPara { pub style: String, pub align: Align, + /// Kashida length when `align` is Justify; ignored for other alignments. + #[serde(default)] + pub kashida: Kashida, pub indent_left: f32, pub indent_right: f32, pub indent_first: f32, @@ -268,6 +271,7 @@ impl ResolvedPara { ResolvedPara { style: style.to_string(), align: p.align.unwrap_or_default(), + kashida: p.kashida.unwrap_or_default(), indent_left: clampi(p.indent_left), indent_right: clampi(p.indent_right), indent_first: clampi(p.indent_first), @@ -634,4 +638,33 @@ mod tests { assert_eq!(back.padding, [0.0; 4]); assert_eq!(back, ResolvedPara { padding: [0.0; 4], ..p }); } + + /// A style's kashida length is inherited, and a direct `both` / `justify` cancels it. + #[test] + fn kashida_length_follows_the_style_chain_and_direct_formatting() { + use crate::styles::{Style, StyleKind}; + let mut styles = StyleSheet::builtin(); + styles.upsert(Style { + id: "Body".into(), + name: "Body".into(), + kind: StyleKind::Paragraph, + based_on: Some("Normal".into()), + para: ParaProps { align: Some(Align::Justify), kashida: Some(Kashida::Short), ..Default::default() }, + ..Default::default() + }); + let inherited = styles.resolve_para(&ParaProps { style: Some("Body".into()), ..Default::default() }); + assert_eq!(inherited.align, Align::Justify); + assert_eq!(inherited.kashida, Kashida::Short); + + let cleared = styles.resolve_para(&ParaProps { + style: Some("Body".into()), + align: Some(Align::Justify), + kashida: Some(Kashida::None), + ..Default::default() + }); + assert_eq!(cleared.kashida, Kashida::None); + + let longer = styles.resolve_para(&ParaProps { style: Some("Body".into()), kashida: Some(Kashida::Long), ..Default::default() }); + assert_eq!(longer.kashida, Kashida::Long); + } } From a58989cae9173c7402a125bdd42e48fb5c878e09 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:26:36 +0100 Subject: [PATCH 066/209] image-codecs: skip a SubIFDs array whose offset is cut off (#1889) camera_raw_mode read a missing SubIFDs offset as 0, so it walked the TIFF header as the pointer array and could classify a malformed TIFF as RAW. Skip the entry instead, matching LightCraft's classifier. --- crates/image-codecs/src/codecs/tiff.rs | 6 ++++-- crates/image-codecs/tests/legacy_tiff_raw.rs | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/src/codecs/tiff.rs b/crates/image-codecs/src/codecs/tiff.rs index bf59157eb..0f53bb3da 100644 --- a/crates/image-codecs/src/codecs/tiff.rs +++ b/crates/image-codecs/src/codecs/tiff.rs @@ -207,8 +207,10 @@ fn camera_raw_mode(b: &[u8], legacy: bool) -> bool { } TAG_SUB_IFDS if count == 1 => queue.extend(value(e)), TAG_SUB_IFDS => { - // More than one offset: they are stored at the entry's offset. - let at = u32_at(e.saturating_add(8)).unwrap_or(0) as usize; + // More than one offset: they are stored at the entry's offset. A cut-off + // offset field means no array, not one at the start of the file. + let Some(at) = u32_at(e.saturating_add(8)) else { continue }; + let at = at as usize; for k in 0..count.min(8) as usize { queue.extend(u32_at(at.saturating_add(4 * k))); } diff --git a/crates/image-codecs/tests/legacy_tiff_raw.rs b/crates/image-codecs/tests/legacy_tiff_raw.rs index 770801734..27cf6b881 100644 --- a/crates/image-codecs/tests/legacy_tiff_raw.rs +++ b/crates/image-codecs/tests/legacy_tiff_raw.rs @@ -63,3 +63,23 @@ fn oversized_or_cut_directories_are_skipped() { cut.truncate(20); assert!(!legacy_tiff_raw_hint(&cut)); } + +#[test] +fn a_subifds_array_without_its_offset_is_skipped() { + // IFD0 at 64 holds one SubIFDs entry (LONG, count 3) whose offset field is cut off. A DNG + // directory sits at 32, and the bytes at 8 point to it. Reading the missing offset as 0 + // would treat the TIFF header as the pointer array and follow 8 -> 32. + let mut b = vec![0u8; 64]; + b[..8].copy_from_slice(b"II\x2a\0\x40\0\0\0"); + b[8..12].copy_from_slice(&32u32.to_le_bytes()); + b[32..34].copy_from_slice(&1u16.to_le_bytes()); + b[34..46].copy_from_slice(&[0x12, 0xc6, 1, 0, 4, 0, 0, 0, 1, 4, 0, 0]); + b.extend_from_slice(&1u16.to_le_bytes()); + b.extend_from_slice(&[0x4a, 0x01, 4, 0, 3, 0, 0, 0]); + assert_eq!(b.len(), 74); + assert!(!legacy_tiff_raw_hint(&b)); + // With the offset present (the array at 8), the same SubIFD is followed. + let mut whole = b.clone(); + whole.extend_from_slice(&8u32.to_le_bytes()); + assert!(legacy_tiff_raw_hint(&whole)); +} From fb2b27b1f02b603d696480d03345827e8b44124a Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 16:50:42 +0100 Subject: [PATCH 067/209] craft-xlsx: preserve spacing in simple structured references --- crates/xlsx/src/xlsb/formula.rs | 6 +++++- crates/xlsx/tests/xlsb.rs | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 5d4cfa619..40d461db1 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -681,17 +681,22 @@ fn list(t: &mut Bytes<'_>, links: &Links) -> Result { 2 if first <= last => Some((column(first)?, Some(column(last)?))), _ => return Err(Unsupported("invalid structured reference columns")), }; + let pad = if bits & 0x80 != 0 { " " } else { "" }; let mut s = table.name.clone(); match (rows, &columns) { ([], None) => s.push_str("[]"), ([], Some((c, None))) => { s.push('['); + s.push_str(pad); s.push_str(c); + s.push_str(pad); s.push(']'); } ([row], None) => { s.push('['); + s.push_str(pad); s.push_str(row); + s.push_str(pad); s.push(']'); } _ => { @@ -701,7 +706,6 @@ fn list(t: &mut Bytes<'_>, links: &Links) -> Result { Some((a, None)) => items.push(format!("[{a}]")), None => {} } - let pad = if bits & 0x80 != 0 { " " } else { "" }; let comma = if bits & 0x100 != 0 { ", " } else { "," }; s.push('['); s.push_str(pad); diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index cd5494c03..a7253df22 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1839,6 +1839,27 @@ fn a_name_using_a_structured_reference_is_decoded_after_tables_are_indexed() { assert_eq!(wb.sheets[0].formula(CellRef::new(0, 1)), Some("Vals"), "{:?}", wb.warnings); } +#[test] +fn simple_table_references_preserve_square_bracket_spacing() { + let cases = [ + (1, "Sales[Val]"), + (1 | 0x80, "Sales[ Val ]"), + (rows(1), "Sales[#All]"), + (rows(1) | 0x80, "Sales[ #All ]"), + (1 | 0x100, "Sales[Val]"), + (rows(1) | 0x100, "Sales[#All]"), + ]; + let mut data = row(0); + for (i, (bits, _)) in cases.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, &ptg_list(0, *bits, 1, 1, 1), &[])); + } + let bytes = book_with_tables(&data, &[sales(&sales_filters())]); + let wb = read(&bytes).unwrap(); + for (i, (_, want)) in cases.iter().enumerate() { + assert_eq!(wb.sheets[0].formula(CellRef::new(0, i as u32 + 1)), Some(*want), "{:?}", wb.warnings); + } +} + #[test] fn tables_are_read_as_their_xlsx_spelling() { // Spacing flags: 0x80 inside the outer brackets, 0x100 after commas. From b84343883d546731e658df8a9a8b77b71ffa1489 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:04:27 +0100 Subject: [PATCH 068/209] Validate compositor frame inputs and return allocation errors --- crates/color/src/lib.rs | 45 ++++++-- crates/frame/src/lib.rs | 226 +++++++++++++++++++++++++++++++--------- 2 files changed, 211 insertions(+), 60 deletions(-) diff --git a/crates/color/src/lib.rs b/crates/color/src/lib.rs index 5bf6426b8..01fbf3b18 100644 --- a/crates/color/src/lib.rs +++ b/crates/color/src/lib.rs @@ -154,24 +154,41 @@ pub fn rgb_to_ycbcr(r: f32, g: f32, b: f32, m: Matrix) -> [f32; 3] { [y, (b - y) / (2.0 * (1.0 - kb)), (r - y) / (2.0 * (1.0 - kr))] } +/// Unsupported integer sample depth. Normalization supports 8 through 16 bits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InvalidBitDepth(pub u32); + +impl std::fmt::Display for InvalidBitDepth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "unsupported sample bit depth {}; expected 8..=16", self.0) + } +} +impl std::error::Error for InvalidBitDepth {} + /// Normalise an integer luma code value to 0..1 given bit depth and range. #[inline] -pub fn normalize_y(v: u32, bits: u32, range: Range) -> f32 { +pub fn normalize_y(v: u32, bits: u32, range: Range) -> Result { + if !(8..=16).contains(&bits) { + return Err(InvalidBitDepth(bits)); + } let scale = (1u32 << (bits - 8)) as f32; - match range { + Ok(match range { Range::Limited => (v as f32 - 16.0 * scale) / (219.0 * scale), Range::Full => v as f32 / ((1u32 << bits) - 1) as f32, - } + }) } /// Normalise an integer chroma code value to −0.5..0.5 given bit depth and range. #[inline] -pub fn normalize_c(v: u32, bits: u32, range: Range) -> f32 { +pub fn normalize_c(v: u32, bits: u32, range: Range) -> Result { + if !(8..=16).contains(&bits) { + return Err(InvalidBitDepth(bits)); + } let scale = (1u32 << (bits - 8)) as f32; - match range { + Ok(match range { Range::Limited => (v as f32 - 128.0 * scale) / (224.0 * scale), Range::Full => (v as f32 - (1u32 << (bits - 1)) as f32) / ((1u32 << bits) - 1) as f32, - } + }) } /// sRGB electro-optical transfer (encoded → linear). @@ -355,12 +372,20 @@ mod tests { } } + #[test] + fn invalid_bit_depths_are_errors_not_shifts() { + for bits in [0, 1, 7, 17, 31, 32, u32::MAX] { + assert_eq!(normalize_y(0, bits, Range::Full), Err(InvalidBitDepth(bits))); + assert_eq!(normalize_c(0, bits, Range::Limited), Err(InvalidBitDepth(bits))); + } + } + #[test] fn limited_range_levels() { - assert_eq!(normalize_y(16, 8, Range::Limited), 0.0); - assert_eq!(normalize_y(235, 8, Range::Limited), 1.0); - assert_eq!(normalize_y(940, 10, Range::Limited), 1.0); - assert_eq!(normalize_c(128, 8, Range::Limited), 0.0); + assert_eq!(normalize_y(16, 8, Range::Limited).unwrap(), 0.0); + assert_eq!(normalize_y(235, 8, Range::Limited).unwrap(), 1.0); + assert_eq!(normalize_y(940, 10, Range::Limited).unwrap(), 1.0); + assert_eq!(normalize_c(128, 8, Range::Limited).unwrap(), 0.0); } #[test] diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs index 09f95e0b8..ad294b10f 100644 --- a/crates/frame/src/lib.rs +++ b/crates/frame/src/lib.rs @@ -23,6 +23,53 @@ use craft_time::Tick; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +/// Default per-buffer allocation ceiling (1 GiB), permitting an 8K RGBA-f32 frame. +/// Callers with a smaller budget should use the `*_with_limit` conversions. +pub const DEFAULT_MAX_FRAME_BYTES: usize = 1 << 30; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameError { + InvalidLayout, + InvalidBitDepth, + TooLarge, + Allocation, +} +impl std::fmt::Display for FrameError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::InvalidLayout => "frame storage does not match its dimensions", + Self::InvalidBitDepth => "unsupported frame sample bit depth", + Self::TooLarge => "frame exceeds the allocation limit", + Self::Allocation => "unable to allocate frame buffer", + }) + } +} +impl std::error::Error for FrameError {} +impl From for FrameError { + fn from(_: craft_color::InvalidBitDepth) -> Self { + Self::InvalidBitDepth + } +} +pub type Result = std::result::Result; + +fn elements(width: u32, height: u32, channels: usize) -> Result { + usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .and_then(|n| n.checked_mul(channels)) + .ok_or(FrameError::TooLarge) +} + +fn buffer(len: usize, max_bytes: usize) -> Result> { + if len.checked_mul(std::mem::size_of::()).is_none_or(|bytes| bytes > max_bytes || bytes > isize::MAX as usize) { + return Err(FrameError::TooLarge); + } + let mut out = Vec::new(); + out.try_reserve_exact(len).map_err(|_| FrameError::Allocation)?; + out.resize(len, T::default()); + Ok(out) +} + /// Chroma subsampling of planar Y'CbCr. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Chroma { @@ -103,20 +150,51 @@ impl Region { impl VideoFrame { /// An sRGB-full, 1:1, zero-pts RGBA8 frame. `data` is `width × height × 4` bytes. - pub fn rgba8(width: u32, height: u32, data: Vec) -> Self { - debug_assert_eq!(data.len(), (width * height * 4) as usize); - Self { width, height, data: PixelData::Rgba8(Arc::new(data)), color: ColorInfo::SRGB_FULL, par: (1, 1), pts: Tick::ZERO } + pub fn rgba8(width: u32, height: u32, data: Vec) -> Result { + let frame = Self { width, height, data: PixelData::Rgba8(Arc::new(data)), color: ColorInfo::SRGB_FULL, par: (1, 1), pts: Tick::ZERO }; + frame.validate()?; + Ok(frame) } /// A linear premultiplied RGBA f32 frame, sRGB-full tags, 1:1, zero pts. - pub fn rgba_f32(width: u32, height: u32, data: Vec) -> Self { - debug_assert_eq!(data.len(), (width * height * 4) as usize); - Self { width, height, data: PixelData::RgbaF32(Arc::new(data)), color: ColorInfo::SRGB_FULL, par: (1, 1), pts: Tick::ZERO } + pub fn rgba_f32(width: u32, height: u32, data: Vec) -> Result { + let frame = Self { width, height, data: PixelData::RgbaF32(Arc::new(data)), color: ColorInfo::SRGB_FULL, par: (1, 1), pts: Tick::ZERO }; + frame.validate()?; + Ok(frame) } /// A transparent linear RGBA f32 frame of the given size. - pub fn transparent_f32(width: u32, height: u32) -> Self { - Self::rgba_f32(width, height, vec![0.0; (width * height * 4) as usize]) + pub fn transparent_f32(width: u32, height: u32) -> Result { + Self::rgba_f32(width, height, buffer(elements(width, height, 4)?, DEFAULT_MAX_FRAME_BYTES)?) + } + + /// Validate all public fields before indexing pixel storage. Rechecked by conversions because + /// callers can mutate these fields after construction. Padding beyond each required plane is allowed. + pub fn validate(&self) -> Result<()> { + let pixels = elements(self.width, self.height, 1)?; + let rgba = pixels.checked_mul(4).ok_or(FrameError::TooLarge)?; + let planar = |lengths: [usize; 3], chroma: Chroma, alpha: Option| { + let (sx, sy) = chroma.shifts(); + let cw = (self.width as usize).div_ceil(1 << sx); + let ch = (self.height as usize).div_ceil(1 << sy); + let cp = cw.checked_mul(ch).ok_or(FrameError::TooLarge)?; + if lengths[0] < pixels || lengths[1] < cp || lengths[2] < cp || alpha.is_some_and(|n| n < pixels) { + return Err(FrameError::InvalidLayout); + } + Ok(()) + }; + match &self.data { + PixelData::Rgba8(d) if d.len() == rgba => Ok(()), + PixelData::RgbaF32(d) if d.len() == rgba => Ok(()), + PixelData::Yuv8 { planes, chroma, alpha } => planar(planes.each_ref().map(|p| p.len()), *chroma, alpha.as_ref().map(|p| p.len())), + PixelData::Yuv16 { planes, chroma, bits, alpha } => { + if !(9..=16).contains(bits) { + return Err(FrameError::InvalidBitDepth); + } + planar(planes.each_ref().map(|p| p.len()), *chroma, alpha.as_ref().map(|p| p.len())) + } + _ => Err(FrameError::InvalidLayout), + } } /// Sets the presentation timestamp and returns the frame. @@ -139,18 +217,27 @@ impl VideoFrame { pub fn byte_size(&self) -> usize { match &self.data { PixelData::Rgba8(d) => d.len(), - PixelData::RgbaF32(d) => d.len() * 4, - PixelData::Yuv8 { planes, alpha, .. } => planes.iter().map(|p| p.len()).sum::() + alpha.as_ref().map_or(0, |a| a.len()), - PixelData::Yuv16 { planes, alpha, .. } => (planes.iter().map(|p| p.len()).sum::() + alpha.as_ref().map_or(0, |a| a.len())) * 2, + PixelData::RgbaF32(d) => d.len().saturating_mul(4), + PixelData::Yuv8 { planes, alpha, .. } => { + planes.iter().fold(0usize, |n, p| n.saturating_add(p.len())).saturating_add(alpha.as_ref().map_or(0, |a| a.len())) + } + PixelData::Yuv16 { planes, alpha, .. } => { + planes.iter().fold(0usize, |n, p| n.saturating_add(p.len())).saturating_add(alpha.as_ref().map_or(0, |a| a.len())).saturating_mul(2) + } } } /// Premultiplied linear-light RGBA f32, one pixel per source pixel, row-major, 4 samples each. - pub fn to_linear_f32(&self) -> Vec { + pub fn to_linear_f32(&self) -> Result> { + self.to_linear_f32_with_limit(DEFAULT_MAX_FRAME_BYTES) + } + + pub fn to_linear_f32_with_limit(&self, max_bytes: usize) -> Result> { + self.validate()?; let (w, h) = (self.width as usize, self.height as usize); - let mut out = vec![0.0; w.saturating_mul(h).saturating_mul(4)]; + let mut out = buffer::(elements(self.width, self.height, 4)?, max_bytes)?; if w == 0 || h == 0 { - return out; + return Ok(out); } match &self.data { PixelData::RgbaF32(d) => { @@ -173,8 +260,8 @@ impl VideoFrame { let info = self.color; let lin: Vec = (0..4096).map(|i| to_linear(i as f32 / 4095.0, info.transfer)).collect(); let q = |v: f32| lin[(v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize]; - let ytab: Vec = (0..256).map(|v| normalize_y(v, 8, info.range)).collect(); - let ctab: Vec = (0..256).map(|v| normalize_c(v, 8, info.range)).collect(); + let ytab: Vec = (0..256).map(|v| normalize_y(v, 8, info.range)).collect::>()?; + let ctab: Vec = (0..256).map(|v| normalize_c(v, 8, info.range)).collect::>()?; let rgb_of = |y: usize, x: usize| { let yy = ytab[planes[0][y * w + x] as usize]; let u = ctab[planes[1][(y >> sy) * cw + (x >> sx)] as usize]; @@ -200,9 +287,9 @@ impl VideoFrame { let q = |v: f32| lin[(v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize]; for y in 0..h { for x in 0..w { - let yy = normalize_y(planes[0][y * w + x] as u32, bits, info.range); - let u = normalize_c(planes[1][(y >> sy) * cw + (x >> sx)] as u32, bits, info.range); - let v = normalize_c(planes[2][(y >> sy) * cw + (x >> sx)] as u32, bits, info.range); + let yy = normalize_y(planes[0][y * w + x] as u32, bits, info.range)?; + let u = normalize_c(planes[1][(y >> sy) * cw + (x >> sx)] as u32, bits, info.range)?; + let v = normalize_c(planes[2][(y >> sy) * cw + (x >> sx)] as u32, bits, info.range)?; let rgb = ycbcr_to_rgb(yy, u, v, info.matrix); let a = alpha.as_ref().map_or(1.0, |al| al[y * w + x] as f32 / amax); let o = (y * w + x) * 4; @@ -214,19 +301,28 @@ impl VideoFrame { } } } - out + Ok(out) } /// Convert to straight-alpha sRGB RGBA8 for display (fast path for 8-bit Rec.709/sRGB Y'CbCr /// without alpha, and a copy for RGBA8). - pub fn to_rgba8(&self) -> Vec { + pub fn to_rgba8(&self) -> Result> { + self.to_rgba8_with_limit(DEFAULT_MAX_FRAME_BYTES) + } + + pub fn to_rgba8_with_limit(&self, max_bytes: usize) -> Result> { + self.validate()?; let (w, h) = (self.width as usize, self.height as usize); // Empty pictures have no rows; Rayon rejects a zero-sized row chunk. if w == 0 || h == 0 { - return Vec::new(); + return Ok(Vec::new()); } match &self.data { - PixelData::Rgba8(d) => d.as_ref().clone(), + PixelData::Rgba8(d) => { + let mut out = buffer::(d.len(), max_bytes)?; + out.copy_from_slice(d); + Ok(out) + } PixelData::Yuv8 { planes, chroma, alpha: None } if matches!(self.color.transfer, Transfer::Bt709 | Transfer::Srgb) => { let (sx, sy) = chroma.shifts(); let cw = w.div_ceil(1 << sx); @@ -238,7 +334,7 @@ impl VideoFrame { }; let (crr, cbb) = (2.0 * (1.0 - kr) * cs, 2.0 * (1.0 - kb) * cs); let (cgr, cgb) = (crr * kr / kg, cbb * kb / kg); - let mut out = vec![0u8; w * h * 4]; + let mut out = buffer::(elements(self.width, self.height, 4)?, max_bytes)?; out.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| { let cy = y >> sy; let yrow = &planes[0][y * w..y * w + w]; @@ -255,30 +351,33 @@ impl VideoFrame { o[3] = 255; } }); - out + Ok(out) } _ => { - let lin = self.to_linear_f32(); - let mut out = vec![0u8; w * h * 4]; + let lin = self.to_linear_f32_with_limit(max_bytes)?; + let mut out = buffer::(elements(self.width, self.height, 4)?, max_bytes)?; out.par_chunks_mut(w * 4).zip(lin.par_chunks(w * 4)).for_each(|(o, s)| linear_premul_to_srgb8(s, o)); - out + Ok(out) } } } /// Luma plane (8-bit, for scopes and thumbnails). Y'CbCr 8-bit returns the Y plane; others go /// through [`Self::to_rgba8`] and Rec.709 luma of the encoded RGB. - pub fn luma8(&self) -> Vec { + pub fn luma8(&self) -> Result> { + self.validate()?; + let pixels = elements(self.width, self.height, 1)?; + let mut out = buffer::(pixels, DEFAULT_MAX_FRAME_BYTES)?; match &self.data { - PixelData::Yuv8 { planes, .. } => planes[0].as_ref().clone(), - _ => self - .to_rgba8() - .as_chunks::<4>() - .0 - .iter() - .map(|p| (0.2126 * p[0] as f32 + 0.7152 * p[1] as f32 + 0.0722 * p[2] as f32) as u8) - .collect(), + PixelData::Yuv8 { planes, .. } => out.copy_from_slice(&planes[0][..pixels]), + _ => { + let rgba = self.to_rgba8()?; + for (o, p) in out.iter_mut().zip(rgba.as_chunks::<4>().0) { + *o = (0.2126 * p[0] as f32 + 0.7152 * p[1] as f32 + 0.0722 * p[2] as f32) as u8; + } + } } + Ok(out) } } @@ -307,22 +406,49 @@ pub fn default_matrix(width: u32, height: u32) -> Matrix { mod tests { use super::*; + #[test] + fn malformed_storage_and_bit_depth_are_errors() { + let make = |data| VideoFrame { width: 2, height: 2, data, color: ColorInfo::REC709, par: (1, 1), pts: Tick::ZERO }; + let short = make(PixelData::Yuv8 { planes: std::array::from_fn(|_| Arc::new(vec![0])), chroma: Chroma::C420, alpha: None }); + assert_eq!(short.to_rgba8(), Err(FrameError::InvalidLayout)); + assert_eq!(short.to_linear_f32(), Err(FrameError::InvalidLayout)); + assert_eq!(short.luma8(), Err(FrameError::InvalidLayout)); + for bits in [0, 8, 17, 32, u32::MAX] { + let frame = make(PixelData::Yuv16 { planes: std::array::from_fn(|_| Arc::new(vec![0; 4])), chroma: Chroma::C444, bits, alpha: None }); + assert_eq!(frame.to_linear_f32(), Err(FrameError::InvalidBitDepth)); + } + assert!(VideoFrame::rgba8(2, 2, vec![0; 15]).is_err()); + assert!(VideoFrame::rgba_f32(2, 2, vec![0.0; 15]).is_err()); + let mut valid = VideoFrame::rgba8(1, 1, vec![0; 4]).unwrap(); + valid.width = 2; // public mutations must be checked again, not just by constructors. + assert_eq!(valid.to_linear_f32(), Err(FrameError::InvalidLayout)); + } + + #[test] + fn frame_allocations_are_checked_before_reserving() { + assert!(matches!(VideoFrame::transparent_f32(u32::MAX, u32::MAX), Err(FrameError::TooLarge))); + let frame = VideoFrame::rgba8(1, 1, vec![0; 4]).unwrap(); + assert_eq!(frame.to_rgba8_with_limit(3), Err(FrameError::TooLarge)); + assert_eq!(frame.to_linear_f32_with_limit(15), Err(FrameError::TooLarge)); + assert_eq!(frame.to_linear_f32_with_limit(16).unwrap(), vec![0.0; 4]); + } + #[test] fn empty_frames_convert_without_zero_sized_chunks() { for (w, h) in [(0, 0), (0, 1), (1, 0)] { - let frame = VideoFrame::transparent_f32(w, h); - assert!(frame.to_rgba8().is_empty()); - assert!(frame.to_linear_f32().is_empty()); - assert!(frame.luma8().is_empty()); + let frame = VideoFrame::transparent_f32(w, h).unwrap(); + assert!(frame.to_rgba8().unwrap().is_empty()); + assert!(frame.to_linear_f32().unwrap().is_empty()); + assert!(frame.luma8().unwrap().is_empty()); } } #[test] fn rgba8_roundtrips_through_to_rgba8() { let px = vec![10u8, 20, 30, 255, 0, 0, 0, 0]; - let f = VideoFrame::rgba8(2, 1, px.clone()).with_pts(Tick(42)); + let f = VideoFrame::rgba8(2, 1, px.clone()).unwrap().with_pts(Tick(42)); assert_eq!(f.pts, Tick(42)); - assert_eq!(f.to_rgba8(), px); + assert_eq!(f.to_rgba8().unwrap(), px); assert_eq!(f.format_label(), "RGBA 8-bit"); assert_eq!(f.byte_size(), 8); assert_eq!(f.color, ColorInfo::SRGB_FULL); @@ -331,14 +457,14 @@ mod tests { #[test] fn rgba8_to_linear_premultiplies() { // 50% red, 50% alpha, sRGB encoded. Linear of 128/255, times 0.5 alpha. - let f = VideoFrame::rgba8(1, 1, vec![128, 0, 0, 128]); - let lin = f.to_linear_f32(); + let f = VideoFrame::rgba8(1, 1, vec![128, 0, 0, 128]).unwrap(); + let lin = f.to_linear_f32().unwrap(); let a = 128.0 / 255.0; let r = srgb_u8_to_linear_table()[128] * a; assert!((lin[0] - r).abs() < 1e-6); assert_eq!(lin[1], 0.0); assert!((lin[3] - a).abs() < 1e-6); - let back = f.to_rgba8(); + let back = f.to_rgba8().unwrap(); assert_eq!(back[0], 128); assert_eq!(back[3], 128); } @@ -356,14 +482,14 @@ mod tests { par: (1, 1), pts: Tick::ZERO, }; - let rgba = f.to_rgba8(); + let rgba = f.to_rgba8().unwrap(); for p in rgba.as_chunks::<4>().0 { assert!((p[0] as i16 - p[1] as i16).abs() <= 1, "{p:?}"); assert!((p[1] as i16 - p[2] as i16).abs() <= 1, "{p:?}"); assert!(p[0] > 90 && p[0] < 160, "{p:?}"); assert_eq!(p[3], 255); } - assert_eq!(f.luma8(), vec![126, 126, 126, 126]); + assert_eq!(f.luma8().unwrap(), vec![126, 126, 126, 126]); assert_eq!(f.format_label(), "YUV 4:4:4 8-bit"); } @@ -383,8 +509,8 @@ mod tests { #[test] fn transparent_float_encodes_as_zero_rgba8() { - let f = VideoFrame::transparent_f32(2, 1); - assert_eq!(f.to_rgba8(), vec![0; 8]); + let f = VideoFrame::transparent_f32(2, 1).unwrap(); + assert_eq!(f.to_rgba8().unwrap(), vec![0; 8]); assert_eq!(f.byte_size(), 32); // 2 pixels × 4 f32 channels × 4 bytes } } From d839db7d4a1a24cb21bcd44162aba1a5055ad2f0 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:06:42 +0100 Subject: [PATCH 069/209] Test odd planar layouts and independent storage validation --- crates/frame/src/lib.rs | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs index ad294b10f..7f3a993c7 100644 --- a/crates/frame/src/lib.rs +++ b/crates/frame/src/lib.rs @@ -424,6 +424,52 @@ mod tests { assert_eq!(valid.to_linear_f32(), Err(FrameError::InvalidLayout)); } + #[test] + fn odd_subsampled_frames_validate_chroma_and_alpha_independently() { + let make = |bits| VideoFrame { + width: 3, + height: 3, + data: PixelData::Yuv16 { + planes: [Arc::new(vec![128; 9]), Arc::new(vec![128; 4]), Arc::new(vec![128; 4])], + chroma: Chroma::C420, + bits, + alpha: Some(Arc::new(vec![255; 9])), + }, + color: ColorInfo::REC709, + par: (1, 1), + pts: Tick::ZERO, + }; + for bits in [9, 16] { + let mut valid = make(bits); + assert_eq!(valid.to_linear_f32().unwrap().len(), 36); + assert_eq!(valid.to_rgba8().unwrap().len(), 36); + if let PixelData::Yuv16 { planes, .. } = &mut valid.data { + planes[1] = Arc::new(vec![128; 3]); + } + assert_eq!(valid.to_linear_f32(), Err(FrameError::InvalidLayout)); + let mut valid = make(bits); + if let PixelData::Yuv16 { alpha, .. } = &mut valid.data { + *alpha = Some(Arc::new(vec![255; 8])); + } + assert_eq!(valid.to_rgba8(), Err(FrameError::InvalidLayout)); + } + let mut fast = VideoFrame { + width: 3, + height: 3, + data: PixelData::Yuv8 { + planes: [Arc::new(vec![128; 9]), Arc::new(vec![128; 4]), Arc::new(vec![128; 4])], + chroma: Chroma::C420, + alpha: None, + }, + color: ColorInfo::REC709, + par: (1, 1), + pts: Tick::ZERO, + }; + assert_eq!(fast.to_rgba8().unwrap().len(), 36); + fast.height = 4; + assert_eq!(fast.to_rgba8(), Err(FrameError::InvalidLayout)); + } + #[test] fn frame_allocations_are_checked_before_reserving() { assert!(matches!(VideoFrame::transparent_f32(u32::MAX, u32::MAX), Err(FrameError::TooLarge))); From 10a585c3e2f7486b8bbd179edb6dc167e0eaea99 Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 17:14:31 +0100 Subject: [PATCH 070/209] fix(psd): preserve consumer resource limits (#786) --- crates/psd/src/descriptor.rs | 60 ++++++++++++++++++----- crates/psd/src/float.rs | 28 ++++++++--- crates/psd/src/smart.rs | 7 ++- crates/psd/tests/resource_limits.rs | 76 +++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 18 deletions(-) create mode 100644 crates/psd/tests/resource_limits.rs diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs index 96ad47a7b..1a6f2000e 100644 --- a/crates/psd/src/descriptor.rs +++ b/crates/psd/src/descriptor.rs @@ -12,6 +12,26 @@ mod lookup; /// Maximum nesting depth accepted when parsing (fuzz safety). pub const MAX_DEPTH: usize = 64; +/// Per-container limits for parsing descriptor trees. +/// +/// Limits apply to every nested descriptor and `VlLs` list, before allocation. +/// They are not a total memory budget; byte strings and other value types retain +/// their existing input-length checks. Defaults preserve the unrestricted counts +/// accepted by the original API. Nesting remains bounded by [`MAX_DEPTH`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParseLimits { + /// Maximum key/value pairs in each descriptor. + pub max_descriptor_items: u32, + /// Maximum values in each `VlLs` list. + pub max_list_items: u32, +} + +impl Default for ParseLimits { + fn default() -> Self { + Self { max_descriptor_items: u32::MAX, max_list_items: u32::MAX } + } +} + /// A Photoshop Unicode string: UTF-16 code units, preserved exactly /// (including any trailing NUL and unpaired surrogates). #[derive(Debug, Clone, PartialEq, Eq, Default, Hash)] @@ -309,6 +329,14 @@ impl Descriptor { Ok((d, r.pos())) } + /// Parses a descriptor prefix with explicit per-container limits. + /// Returns the descriptor and bytes consumed, allowing trailing data. + pub fn parse_prefix_with_limits(data: &[u8], limits: ParseLimits) -> Result<(Self, usize)> { + let mut r = Reader::new(data); + let d = Self::read_with_limits(&mut r, limits)?; + Ok((d, r.pos())) + } + /// Serializes the descriptor. pub fn to_bytes(&self) -> Vec { let mut v = Vec::new(); @@ -317,7 +345,11 @@ impl Descriptor { } pub(crate) fn read(r: &mut Reader<'_>) -> Result { - read_descriptor(r, 0) + Self::read_with_limits(r, ParseLimits::default()) + } + + pub(crate) fn read_with_limits(r: &mut Reader<'_>, limits: ParseLimits) -> Result { + read_descriptor(r, 0, limits) } pub(crate) fn write(&self, out: &mut Vec) { @@ -391,33 +423,36 @@ pub(crate) fn vec_for_count(n: u32) -> Vec { Vec::with_capacity((n as usize).min(MAX_PREALLOC)) } -fn read_descriptor(r: &mut Reader<'_>, depth: usize) -> Result { +fn read_descriptor(r: &mut Reader<'_>, depth: usize, limits: ParseLimits) -> Result { if depth > MAX_DEPTH { return Err(PsdError::LimitExceeded("descriptor nesting too deep")); } let name = UnicodeString::read(r)?; let class_id = Id::read(r)?; let n = r.u32()?; + if n > limits.max_descriptor_items { + return Err(PsdError::LimitExceeded("too many descriptor items")); + } // Each item needs at least 5 (1-byte string key) + 4 (type) + 1 (bool). r.check_count(u64::from(n), 10)?; let mut items = vec_for_count(n); for _ in 0..n { let key = Id::read(r)?; - let v = read_value(r, depth + 1)?; + let v = read_value(r, depth + 1, limits)?; items.push((key, v)); } Ok(Descriptor { name, class_id, items }) } -fn read_value(r: &mut Reader<'_>, depth: usize) -> Result { +fn read_value(r: &mut Reader<'_>, depth: usize, limits: ParseLimits) -> Result { if depth > MAX_DEPTH { return Err(PsdError::LimitExceeded("descriptor nesting too deep")); } let ty = r.array::<4>()?; - read_typed_value(r, &ty, depth) + read_typed_value(r, &ty, depth, limits) } -fn read_typed_value(r: &mut Reader<'_>, ty: &[u8; 4], depth: usize) -> Result { +fn read_typed_value(r: &mut Reader<'_>, ty: &[u8; 4], depth: usize, limits: ParseLimits) -> Result { Ok(match ty { b"obj " => { let n = r.u32()?; @@ -428,14 +463,17 @@ fn read_typed_value(r: &mut Reader<'_>, ty: &[u8; 4], depth: usize) -> Result Value::Descriptor(read_descriptor(r, depth + 1)?), - b"GlbO" => Value::GlobalObject(read_descriptor(r, depth + 1)?), + b"Objc" => Value::Descriptor(read_descriptor(r, depth + 1, limits)?), + b"GlbO" => Value::GlobalObject(read_descriptor(r, depth + 1, limits)?), b"VlLs" => { let n = r.u32()?; + if n > limits.max_list_items { + return Err(PsdError::LimitExceeded("too many descriptor list items")); + } r.check_count(u64::from(n), 4)?; let mut items = vec_for_count(n); for _ in 0..n { - items.push(read_value(r, depth + 1)?); + items.push(read_value(r, depth + 1, limits)?); } Value::List(items) } @@ -472,7 +510,7 @@ fn read_typed_value(r: &mut Reader<'_>, ty: &[u8; 4], depth: usize) -> Result { let prefix = r.u32()?; - Value::ObjectArray(ObjectArray { prefix, body: read_descriptor(r, depth + 1)? }) + Value::ObjectArray(ObjectArray { prefix, body: read_descriptor(r, depth + 1, limits)? }) } other => { return Err(PsdError::Unsupported(format!("descriptor OSType {:?}", String::from_utf8_lossy(other)))); @@ -625,7 +663,7 @@ mod tests { data.extend_from_slice(b"bool\x01"); } let mut r = Reader::new(&data); - match read_typed_value(&mut r, b"VlLs", 0).unwrap() { + match read_typed_value(&mut r, b"VlLs", 0, ParseLimits::default()).unwrap() { Value::List(v) => assert_eq!(v.len(), n as usize), other => panic!("expected list, got {other:?}"), } diff --git a/crates/psd/src/float.rs b/crates/psd/src/float.rs index 6e26cd75b..658538bc2 100644 --- a/crates/psd/src/float.rs +++ b/crates/psd/src/float.rs @@ -436,8 +436,22 @@ impl SmartObject { /// exceed [`MAX_BAKE_PIXELS`], or the mesh needs more rasterising work than eight times the /// output allows (a hostile mesh whose triangles each cover the whole output). pub fn render_placed(&self, content: &FloatImage, k: f64, size: Option<(u32, u32)>) -> Option<(FloatImage, [f64; 2])> { - let (cw, ch) = (f64::from(content.width), f64::from(content.height)); - if content.width == 0 || content.height == 0 { + self.render_placed_pixels(content.width, content.height, &content.data, k, size) + } + + /// Bakes borrowed straight RGBA pixels without copying the source image. + /// Uses the same placement, output and work limits as [`Self::render_placed`]. + /// Missing pixels in a short slice read as transparent; extra pixels are ignored. + pub fn render_placed_pixels( + &self, + width: u32, + height: u32, + data: &[[f32; 4]], + k: f64, + size: Option<(u32, u32)>, + ) -> Option<(FloatImage, [f64; 2])> { + let (cw, ch) = (f64::from(width), f64::from(height)); + if width == 0 || height == 0 { return None; } // The quad is solved once for the bounds, the size and the whole grid. @@ -464,7 +478,7 @@ impl SmartObject { let mut out: Vec<[f32; 4]> = Vec::new(); out.try_reserve_exact(count).ok()?; out.resize(count, [0.0; 4]); - let sampler = Sampler { content, cw, ch }; + let sampler = Sampler { width, height, data, cw, ch }; let idx = |i: usize, j: usize| j * (n + 1) + i; // A sane mesh's triangles each span about one grid cell (ow/n by oh/n), rounded out to // whole pixels. A thin output (16384×1) stacks every grid row onto the same pixel row, so @@ -498,7 +512,9 @@ impl SmartObject { /// Bilinear, premultiplied sampling of the content at content-pixel coordinates. struct Sampler<'a> { - content: &'a FloatImage, + width: u32, + height: u32, + data: &'a [[f32; 4]], cw: f64, ch: f64, } @@ -506,14 +522,14 @@ struct Sampler<'a> { impl Sampler<'_> { fn at(&self, x: usize, y: usize) -> [f32; 4] { // `data` is a public field: a short one reads as transparent. - let p = self.content.data.get(y * self.content.width as usize + x).copied().unwrap_or([0.0; 4]); + let p = y.checked_mul(self.width as usize).and_then(|row| row.checked_add(x)).and_then(|i| self.data.get(i)).copied().unwrap_or([0.0; 4]); [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]] } fn sample(&self, x: f64, y: f64) -> [f32; 4] { let (fx, fy) = ((x - 0.5).clamp(0.0, self.cw - 1.0), (y - 0.5).clamp(0.0, self.ch - 1.0)); let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); - let (x1, y1) = ((x0 + 1).min(self.content.width as usize - 1), (y0 + 1).min(self.content.height as usize - 1)); + let (x1, y1) = ((x0 + 1).min(self.width as usize - 1), (y0 + 1).min(self.height as usize - 1)); let (a, b) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32); let (p00, p10, p01, p11) = (self.at(x0, y0), self.at(x1, y0), self.at(x0, y1), self.at(x1, y1)); std::array::from_fn(|c| (p00[c] * (1.0 - a) + p10[c] * a) * (1.0 - b) + (p01[c] * (1.0 - a) + p11[c] * a) * b) diff --git a/crates/psd/src/smart.rs b/crates/psd/src/smart.rs index a2195bf5d..d4d98c695 100644 --- a/crates/psd/src/smart.rs +++ b/crates/psd/src/smart.rs @@ -259,11 +259,16 @@ pub struct SmartObject { impl SmartObject { /// Reads the data of a `SoLd` / `SoLE` block. pub fn from_placed_layer_data(data: &[u8]) -> Result { + Self::from_placed_layer_data_with_limits(data, crate::descriptor::ParseLimits::default()) + } + + /// Reads a `SoLd` / `SoLE` block with explicit descriptor container limits. + pub fn from_placed_layer_data_with_limits(data: &[u8], limits: crate::descriptor::ParseLimits) -> Result { let mut r = Reader::new(data); let _kind = r.array::<4>()?; let _version = r.u32()?; let _descriptor_version = r.u32()?; - let d = Descriptor::read(&mut r)?; + let d = Descriptor::read_with_limits(&mut r, limits)?; let uuid = d.get_text("Idnt").unwrap_or_default(); let t = eight(&d, "Trnf").unwrap_or([0.0; 8]); // The non-affine transform when present (perspective), else the transform. diff --git a/crates/psd/tests/resource_limits.rs b/crates/psd/tests/resource_limits.rs new file mode 100644 index 000000000..624aaf287 --- /dev/null +++ b/crates/psd/tests/resource_limits.rs @@ -0,0 +1,76 @@ +use craft_psd::descriptor::{Descriptor, ObjectArray, ParseLimits, Value}; +use craft_psd::float::FloatImage; +use craft_psd::smart::SmartObject; + +#[test] +fn descriptor_limits_apply_at_boundary_and_through_every_nested_container() { + let limits = ParseLimits { max_descriptor_items: 1, max_list_items: 1 }; + let one = Descriptor::new("test").with("one", Value::Boolean(true)); + let two = one.clone().with("two", Value::Boolean(false)); + assert!(Descriptor::parse_prefix_with_limits(&one.to_bytes(), limits).is_ok()); + assert!(Descriptor::parse_prefix_with_limits(&two.to_bytes(), limits).is_err()); + assert!(Descriptor::parse_prefix(&two.to_bytes()).is_ok()); + for nested in [ + Value::Descriptor(two.clone()), + Value::GlobalObject(two.clone()), + Value::ObjectArray(ObjectArray { prefix: 0, body: two.clone() }), + Value::List(vec![Value::Descriptor(two.clone())]), + ] { + let bytes = Descriptor::new("test").with("nest", nested).to_bytes(); + assert!(Descriptor::parse_prefix_with_limits(&bytes, limits).is_err()); + assert!(Descriptor::parse_prefix(&bytes).is_ok()); + } + for n in [0, 1, 2] { + let d = Descriptor::new("test").with("list", Value::List(vec![Value::Boolean(true); n])); + assert_eq!(Descriptor::parse_prefix_with_limits(&d.to_bytes(), limits).is_ok(), n <= 1); + } + let nested_list = Descriptor::new("test").with("list", Value::List(vec![Value::List(vec![Value::Boolean(true); 2])])); + assert!(Descriptor::parse_prefix_with_limits(&nested_list.to_bytes(), limits).is_err()); + let mut bytes = one.to_bytes(); + let used = bytes.len(); + bytes.extend_from_slice(b"tail"); + assert_eq!(Descriptor::parse_prefix_with_limits(&bytes, limits).unwrap().1, used); + let zero = ParseLimits { max_descriptor_items: 0, max_list_items: 0 }; + assert!(Descriptor::parse_prefix_with_limits(&Descriptor::new("test").to_bytes(), zero).is_ok()); + assert!(Descriptor::parse_prefix_with_limits(&one.to_bytes(), zero).is_err()); +} + +#[test] +fn count_limits_reject_before_reading_declared_elements() { + let limits = ParseLimits { max_descriptor_items: 1, max_list_items: 1 }; + let mut bytes = Descriptor::new("test").to_bytes(); + let end = bytes.len(); + bytes[end - 4..].copy_from_slice(&2u32.to_be_bytes()); + assert!(matches!(Descriptor::parse_prefix_with_limits(&bytes, limits), Err(craft_psd::PsdError::LimitExceeded(_)))); + let mut bytes = Descriptor::new("test").with("list", Value::List(vec![])).to_bytes(); + let end = bytes.len(); + bytes[end - 4..].copy_from_slice(&2u32.to_be_bytes()); + assert!(matches!(Descriptor::parse_prefix_with_limits(&bytes, limits), Err(craft_psd::PsdError::LimitExceeded(_)))); +} + +#[test] +fn placed_layer_data_honors_descriptor_limits() { + let d = Descriptor::new("test").with("one", Value::Boolean(true)).with("two", Value::Boolean(false)); + let mut bytes = b"soLD".to_vec(); + bytes.extend_from_slice(&4u32.to_be_bytes()); + bytes.extend_from_slice(&16u32.to_be_bytes()); + bytes.extend_from_slice(&d.to_bytes()); + assert!(SmartObject::from_placed_layer_data(&bytes).is_ok()); + assert!(SmartObject::from_placed_layer_data_with_limits(&bytes, ParseLimits { max_descriptor_items: 1, max_list_items: 1 }).is_err()); +} + +#[test] +fn borrowed_pixels_match_owned_render_and_preserve_short_slice_behavior() { + let smart = SmartObject { uuid: String::new(), quad: [[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 2.0]], size: None, warp: None }; + for data in [vec![], vec![[1.0, 0.5, 0.25, 0.5]], vec![[1.0, 0.5, 0.25, 0.5]; 4]] { + let image = FloatImage { width: 2, height: 2, data }; + let owned = smart.render_placed(&image, 1.0, Some((2, 2))); + let borrowed = smart.render_placed_pixels(2, 2, &image.data, 1.0, Some((2, 2))); + assert!(owned.is_some()); + assert_eq!(owned, borrowed); + } + assert!(smart.render_placed_pixels(0, 2, &[], 1.0, None).is_none()); + assert!(smart.render_placed_pixels(u32::MAX, u32::MAX, &[], 1.0, Some((u32::MAX, u32::MAX))).is_none()); + // A short source remains safely transparent even with enormous declared dimensions. + assert!(smart.render_placed_pixels(u32::MAX, u32::MAX, &[], 1.0, Some((1, 1))).is_some()); +} From ecb92f4554042b2a6a5e7fb07b9b56be8fc67a31 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 16 Nov 2026 11:24:32 -0500 Subject: [PATCH 071/209] timeline: extract attributed allocation-free XML and JSON preflight scans --- crates/timeline/src/lib.rs | 1 + crates/timeline/src/preflight.rs | 82 ++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 crates/timeline/src/preflight.rs diff --git a/crates/timeline/src/lib.rs b/crates/timeline/src/lib.rs index b638fd3ec..8c409120e 100644 --- a/crates/timeline/src/lib.rs +++ b/crates/timeline/src/lib.rs @@ -34,6 +34,7 @@ pub mod model; #[cfg(feature = "otio")] pub mod otio; pub mod path; +pub mod preflight; pub mod rate; pub mod report; pub mod time; diff --git a/crates/timeline/src/preflight.rs b/crates/timeline/src/preflight.rs new file mode 100644 index 000000000..49995667c --- /dev/null +++ b/crates/timeline/src/preflight.rs @@ -0,0 +1,82 @@ +//! Allocation-free lexical nesting preflight for timeline XML and JSON. +//! +//! Extracted from EffectCraft's interchange reader, originally by Brandon Thomas. +//! These functions preserve its lightweight byte-scanner behavior. They do not validate +//! syntax, expand entities, decode encodings, or replace a bounded format parser. In +//! particular XML self-closing elements do not increase the count, and declarations are +//! skipped lexically rather than parsed as DTDs. Apply document/encoding/entity policy +//! separately; callers choose the maximum acceptable nesting for their parser. + +/// The deepest element nesting of an XML document (a quick scan that skips comments, CDATA, +/// processing instructions, declarations and quoted attribute values). +pub fn xml_depth(b: &[u8]) -> usize { + let (mut i, mut depth, mut max) = (0usize, 0usize, 0usize); + let skip_to = + |from: usize, end: &[u8]| b.get(from..).and_then(|r| r.windows(end.len()).position(|w| w == end)).map_or(b.len(), |p| from + p + end.len()); + while let Some(&c) = b.get(i) { + if c != b'<' { + i += 1; + continue; + } + let rest = b.get(i..).unwrap_or_default(); + if rest.starts_with(b""); + } else if rest.starts_with(b""); + } else if rest.starts_with(b""); + } else if rest.starts_with(b""); + } else if rest.starts_with(b""); + } else { + // A start tag: find its end outside quotes; `/>` closes it at once. + let mut j = i + 1; + let mut quote = None; + while let Some(&c) = b.get(j) { + match quote { + Some(q) if c == q => quote = None, + Some(_) => {} + None if c == b'"' || c == b'\'' => quote = Some(c), + None if c == b'>' => break, + None => {} + } + j += 1; + } + if b.get(j.wrapping_sub(1)) != Some(&b'/') { + depth += 1; + max = max.max(depth); + } + i = j + 1; + } + } + max +} + +/// The deepest array / object nesting of a JSON document (strings skipped). +pub fn json_depth(b: &[u8]) -> usize { + let (mut depth, mut max, mut in_str, mut esc) = (0usize, 0usize, false, false); + for &c in b { + if in_str { + if esc { + esc = false; + } else if c == b'\\' { + esc = true; + } else if c == b'"' { + in_str = false; + } + continue; + } + match c { + b'"' => in_str = true, + b'[' | b'{' => { + depth += 1; + max = max.max(depth); + } + b']' | b'}' => depth = depth.saturating_sub(1), + _ => {} + } + } + max +} From a2560df05b4c58631f53a57e65fb05f0e0e6cf76 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 11:25:03 -0500 Subject: [PATCH 072/209] timeline: cover lexical preflight compatibility and deep inputs --- crates/timeline/tests/preflight.rs | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 crates/timeline/tests/preflight.rs diff --git a/crates/timeline/tests/preflight.rs b/crates/timeline/tests/preflight.rs new file mode 100644 index 000000000..bbe485d67 --- /dev/null +++ b/crates/timeline/tests/preflight.rs @@ -0,0 +1,31 @@ +use craft_timeline::preflight::{json_depth, xml_depth}; + +#[test] +fn xml_skips_markup_inside_quotes_comments_and_cdata() { + let input = br#"]]>"#; + assert_eq!(xml_depth(input), 2); + assert_eq!(xml_depth(b""), 0); +} + +#[test] +fn lexical_scans_do_not_claim_to_validate_malformed_documents() { + assert_eq!(xml_depth(b""), 1); + assert_eq!(xml_depth(b"

"#; + let expected = br#"

"#; + assert_eq!(remap_orders(xml, |n| n.saturating_sub(1)).unwrap().as_ref(), expected); + assert!(matches!(remap_orders(xml, |n| n).unwrap(), std::borrow::Cow::Borrowed(_))); + let late = format!("", (0..128).map(|i| format!("a{i}='x'")).collect::>().join(" ")); + assert_eq!(remap_orders(late.as_bytes(), |n| n - 1).unwrap().as_ref(), late.as_bytes()); + for broken in ["=bad", "x=bad", "x==bad", "x", "x='a' x", "x=\"bad\"", "'odd'", "/odd", "x='a>b'"] { + let input = format!(""); + let mut seen = false; + events(input.as_bytes(), |e| { + if let Ev::Start(_, a) = e { + seen |= get(&a, "draw:z-index") == Some("99"); + } + }); + if seen { + let changed = remap_orders(input.as_bytes(), |n| n - 1).unwrap_or_else(|e| panic!("{input}: {e}")); + let mut found = false; + events(&changed, |e| { + if let Ev::Start(_, a) = e { + found |= get(&a, "draw:z-index") == Some("98"); + } + }); + assert!(found, "{input}"); + } + } + // Lenient malformed attributes are skipped just as in the reader. + let malformed = b""; + assert_eq!(remap_orders(malformed, |n| n - 1).unwrap().as_ref(), b""); + } +} From dca3593452635b2e8c75da2795fda830c3be451f Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan <126534658+scks1@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:37:17 -0500 Subject: [PATCH 085/209] craft-latex: skip the blanks after a control space TeX skips the blanks after a control space, `\ ` or `\` and a tab, as it does after `\` at a line end. The lexer left them to the whitespace branch, which added a second space, so text taken as written, such as a reference or an `\item` label, read `one two`. The control-space branch now skips them the way the line-end branch does, and a blank line after one still ends the paragraph. --- crates/latex/src/lex.rs | 10 ++++++++-- crates/latex/src/tests.rs | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/latex/src/lex.rs b/crates/latex/src/lex.rs index 82eabe15f..5040f9d9f 100644 --- a/crates/latex/src/lex.rs +++ b/crates/latex/src/lex.rs @@ -151,8 +151,14 @@ impl Lexer<'_> { } let Some(sym) = self.bump() else { break }; match sym { - // `\^^I`, likewise. - '\t' => self.out.push(Tok::Cmd(" ".to_string())), + // `\ ` and `\^^I` (a tab) are control spaces too. TeX skips the blanks + // after them, but a blank line after one still ends the paragraph. + ' ' | '\t' => { + self.out.push(Tok::Cmd(" ".to_string())); + if self.blanks() >= 2 { + self.out.push(Tok::Par); + } + } '(' => { let m = self.until("\\)"); self.out.push(Tok::Math(m, false)); diff --git a/crates/latex/src/tests.rs b/crates/latex/src/tests.rs index 5f6311094..85fee24e4 100644 --- a/crates/latex/src/tests.rs +++ b/crates/latex/src/tests.rs @@ -304,6 +304,25 @@ fn a_control_space_at_a_line_end_adds_no_second_space() { assert_eq!(texts("\\begin{document}\\ref{one\\\r\n two}\\end{document}"), vec!["one two".to_string()]); } +#[test] +fn blanks_after_a_control_space_add_no_second_space() { + // TeX skips the blanks after `\ ` and after `\` and a tab, but they were read as a space of + // their own, so text taken as written had two spaces. + for cs in ["\\ ", "\\\t"] { + assert_eq!(tokenize(&format!("a{cs} \tb")), vec![Tok::Text("a".into()), Tok::Cmd(" ".into()), Tok::Text("b".into())], "{cs:?}"); + assert_eq!(texts(&format!("\\begin{{document}}\\ref{{one{cs}\n two}}\\end{{document}}")), vec!["one two".to_string()], "{cs:?}"); + assert_eq!(texts(&format!("\\begin{{document}}\\ref{{one{cs} two}}\\end{{document}}")), vec!["one two".to_string()], "{cs:?}"); + let t = texts(&format!("\\begin{{document}}\\begin{{description}}\\item[one{cs} two] x\\end{{description}}\\end{{document}}")); + assert_eq!(t.iter().map(|s| s.trim()).collect::>(), vec!["one two x"], "{cs:?}"); + // A blank line after it still ends the paragraph. + assert_eq!( + texts(&format!("\\begin{{document}}first{cs}\n\nsecond\\end{{document}}")), + vec!["first".to_string(), "second".to_string()], + "{cs:?}" + ); + } +} + #[test] fn a_control_space_in_an_item_label_is_a_space() { // The optional argument kept a control symbol as written, so the label read `one\ two`. From 01fcf22cb019c5f1be92a6d970af038ef84d9afb Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Mon, 16 Nov 2026 11:23:53 -0800 Subject: [PATCH 086/209] craft-svg: comment markers in a declaration's literal hide no later declaration declared() skipped a comment or CDATA section wherever `"), + _ if rest.starts_with(b" past(b, i + 2, b"?>"), + _ => i + 1, + }; + } + b.len() +} + +/// Past the comment, character data or instruction that starts at `at` inside a declaration's +/// literal ending at `end`, when it closes before `end`. `missed` holds, for each kind, the literal +/// end at which its close was last not found, so a literal full of openers is searched once. +fn closed_in_literal(b: &[u8], at: usize, end: usize, missed: &mut [usize; 3]) -> Option { + const KINDS: [(&[u8], &[u8]); 3] = [(b""), (b""), (b"")]; + let rest = b.get(at..end)?; + let (kind, (open, close)) = KINDS.iter().enumerate().find(|(_, (open, _))| rest.starts_with(open))?; + let miss = missed.get_mut(kind)?; + if *miss == end { + return None; + } + let found = rest.get(open.len()..).and_then(|r| r.windows(close.len()).position(|w| w == *close)).map(|n| at + open.len() + n + close.len()); + if found.is_none() { + *miss = end; + } + found +} + /// The internal general entities declared in `svg`: name → every value declared for it. Text that /// only looks like a declaration in another entity's value is kept too: the largest value counts, /// so it can only make the estimate larger. Comments and CDATA are skipped, so markers there are -/// not treated as declarations. +/// not treated as declarations. A `") { - Some(n) => at + 1 + 3 + n + 3, - None => svg.len(), - }; - continue; - } - if let Some(body) = after.strip_prefix("![CDATA[") { - i = match body.find("]]>") { - Some(n) => at + 1 + 8 + n + 3, - None => svg.len(), - }; - continue; + if at >= literal_end { + if let Some(body) = after.strip_prefix("!--") { + i = match body.find("-->") { + Some(n) => at + 1 + 3 + n + 3, + None => svg.len(), + }; + continue; + } + if let Some(body) = after.strip_prefix("![CDATA[") { + i = match body.find("]]>") { + Some(n) => at + 1 + 8 + n + 3, + None => svg.len(), + }; + continue; + } + if after.starts_with('?') { + literal_end = past(b, at + 2, b"?>"); + } else if after.starts_with('!') { + literal_end = declaration_end(b, at); + } } if let Some(rest) = after.strip_prefix("!ENTITY") { let rest = rest.trim_start(); @@ -199,11 +266,10 @@ pub fn too_wide(svg: &str, most: usize, most_work: usize) -> bool { } else if rest.starts_with(b""); } else if rest.starts_with(b"` closing its subset (the first `>` when it has none). - let end = match (rest.iter().position(|b| *b == b'['), rest.iter().position(|b| *b == b'>')) { - (Some(o), Some(g)) if o < g => find(i, b"]>"), - _ => find(i, b">"), - }; + // To the `>` after the `]` closing its subset (its first `>` when it has none), + // quoted literals, comments and instructions skipped. + let head = declaration_end(data, i); + let end = if data.get(head.wrapping_sub(1)) == Some(&b'[') { find(subset_end(data, head), b">") } else { head }; let quotes = data.get(i..end).unwrap_or(&[]).iter().filter(|b| matches!(**b, b'"' | b'\'')).count(); if over(quotes / 2) { return true; @@ -318,24 +384,40 @@ pub fn prescan(svg: &str, most: usize) -> Prescan<'_> { // (now, deepest) of the elements, and of the markup inside the document type. let (mut elements, mut entities, mut open_depth) = ((0usize, 0usize), (0usize, 0usize), 0usize); let mut doctype_end = 0; + // The end of the declaration the scan is in: a `").unwrap_or(b.len()) - } else if rest.starts_with(b"").unwrap_or(b.len()) - } else if rest.starts_with(b"").unwrap_or(b.len()) - } else if rest.starts_with(b"` that ends it: after its internal subset (`[` … `]`) when it has one. - let plain = tag_end(at, false).0; - let subset = b.get(at..plain).is_some_and(|head| head.contains(&b'[')); - doctype_end = if subset { find(at, b"]>").unwrap_or(b.len()) } else { plain }; + } else if let Some(end) = (!open).then(|| closed_in_literal(b, at, literal_end, &mut missed)).flatten() { + // A comment, character data or instruction that closes inside the literal holds no + // element of the entity's markup. + end + } else if open && rest.starts_with(b"`. + let head = declaration_end(b, at); + literal_end = head; + doctype_end = if b.get(head.wrapping_sub(1)) == Some(&b'[') { subset_end(b, head) } else { head }; at + 2 } else if rest.starts_with(b""); + for hidden in [ + format!("{large}]>"), + format!("{large}]>"), + format!(""), + format!("{large}]>"), + format!(""), + format!("{large}]>"), + ] { + assert!(expansion_too_large(&format!("{hidden}"))); + } + assert!(entity_holds_markup("'>]>")); + // A real comment still hides a declaration-like text. + assert!(!entity_holds_markup("'> -->]>")); + } + + #[test] + fn markers_in_a_literal_hide_no_markup_from_the_depth_or_attribute_counts() { + let tenth = DEPTH / 10 + 1; + let nested = format!("{}{}", "".repeat(tenth), "".repeat(tenth)); + for bait in ["", "", "\">"] { + assert!(too_deep(&format!("]>&a;")), "{bait}"); + } + // Comments and instructions in a value open and close no element. + let quiet = format!("", "".repeat(DEPTH)); + assert!(!too_deep(&format!("{}", svg(&groups(8))))); + let closers = format!("", "?></g>".repeat(tenth)); + assert!(too_deep(&format!("&a;"))); + // Openers that never close in the value are read once each. + let openers = format!("", ""), _ if rest.starts_with(b" past(b, i + 2, b"?>"), _ => i + 1, @@ -552,6 +563,7 @@ mod tests { format!("{large}]>"), format!(""), format!("{large}]>"), + format!("{large}]>"), ] { assert!(expansion_too_large(&format!("{hidden}"))); } @@ -564,7 +576,7 @@ mod tests { fn markers_in_a_literal_hide_no_markup_from_the_depth_or_attribute_counts() { let tenth = DEPTH / 10 + 1; let nested = format!("{}{}", "".repeat(tenth), "".repeat(tenth)); - for bait in ["", "", "\">"] { + for bait in ["", "", "\">", ""] { assert!(too_deep(&format!("]>&a;")), "{bait}"); } // Comments and instructions in a value open and close no element. @@ -577,6 +589,8 @@ mod tests { assert!(!too_deep(&format!("{}", svg(&groups(8))))); let many = (0..5000).map(|i| format!(" a{i}=\"1\"")).collect::(); assert!(wide(&format!("\">'>]>&e;"))); + // An element declaration ends at its first `>`, so a `]` inside it does not end the subset. + assert!(wide(&format!("'>]>&e;"))); } #[test] From 227ea620b1388a8e7ac62fcb3b2051af2fc152e2 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Mon, 16 Nov 2026 12:05:10 -0800 Subject: [PATCH 090/209] craft-metafile: count refused bitmaps and lost pen textures, empty line clips - A bitmap that the item or byte limit refuses is counted in dropped_images once, however many pieces a clip cuts it into. - An EMF+ pen whose texture brush was not read is counted in dropped_images, as a lost texture brush object is. - An EMF+ clip path or region path whose figures all lie on a line encloses no area, so it gives an empty area instead of its bounding rectangle. --- crates/metafile/README.md | 2 +- crates/metafile/src/canvas.rs | 5 +++++ crates/metafile/src/emfplus.rs | 16 ++++++++++++---- crates/metafile/src/lib.rs | 8 ++++---- crates/metafile/src/tests.rs | 28 ++++++++++++++++++++++++++++ crates/metafile/src/tests/model.rs | 30 ++++++++++++++++++++++++++++++ 6 files changed, 80 insertions(+), 9 deletions(-) diff --git a/crates/metafile/README.md b/crates/metafile/README.md index 4e246de51..226c956ad 100644 --- a/crates/metafile/README.md +++ b/crates/metafile/README.md @@ -39,7 +39,7 @@ differently: | Field | Reports | |---|---| | `truncated` | The parse stopped at a limit that ends it (items, bytes kept, `SAVEDC` depth, clip paths), in the file or in an EMF+ metafile image drawn in it. | -| `dropped_images` | Images that were not read: bitmaps past a size, byte or decode limit or not decodable, EMF+ metafile images that do not parse or are nested too deep, and pattern or texture brushes whose bitmap was not read (counted when the brush is created). An image with no pixels, or drawn from a source rectangle that selects none, is not counted. | +| `dropped_images` | Images that were not read: bitmaps past a size, item, byte or decode limit or not decodable, EMF+ metafile images that do not parse or are nested too deep, and pattern or texture brushes whose bitmap was not read, a pen's brush included (counted when the brush or pen is created). An image with no pixels, or drawn from a source rectangle that selects none, is not counted. | | `ignored_clips` | Clip changes not applied as the file asks: `SELECTCLIPPATH` with `RGN_OR`, `RGN_XOR` or `RGN_DIFF`, EMF+ clips combined with XOR, exclude or complement, EMF+ unions drawn as the rectangle around both areas, and EMF+ clips whose area stands in for the one the file gives (a path that is not one convex figure, drawn as its rectangle, or a region built with XOR, exclude or complement). A change that cannot alter the clip is not counted: an area whose figures all lie on a line (except in an EMF+ complement of a clip that encloses some area, which would empty it), `RGN_OR` on a clip that is still the whole surface, and a union whose rectangle is no larger than the union. | | `unplayed_raster_ops` | Raster operations not played: bitmaps drawn as plain copies because their operation is not one of `SRCCOPY`, `SRCAND`, `SRCPAINT` and `SRCINVERT`, and operations without a source (such as `DSTINVERT` or `PATINVERT`) that are left out. | diff --git a/crates/metafile/src/canvas.rs b/crates/metafile/src/canvas.rs index 67cc43f0f..916e8ac9a 100644 --- a/crates/metafile/src/canvas.rs +++ b/crates/metafile/src/canvas.rs @@ -1335,7 +1335,12 @@ impl Canvas { let (x0, x1) = (xs.iter().copied().fold(f64::INFINITY, f64::min), xs.iter().copied().fold(f64::NEG_INFINITY, f64::max)); let (y0, y1) = (ys.iter().copied().fold(f64::INFINITY, f64::min), ys.iter().copied().fold(f64::NEG_INFINITY, f64::max)); let dev = if xf[1] == 0.0 && xf[2] == 0.0 { dev } else { [x0, y0, x1 - x0, y1 - y0] }; + let was_full = self.full; self.push(Raw::Bitmap { rect: dev, xf, img, region: RawClip::default() }); + // A bitmap the item or byte limit refuses is counted once, however many pieces a clip cuts it into. + if self.full && !was_full { + self.drop_image(); + } } /// Draws a bitmap that a record asked to combine with raster operation `rop` (already applied by diff --git a/crates/metafile/src/emfplus.rs b/crates/metafile/src/emfplus.rs index 32786bc5f..505149371 100644 --- a/crates/metafile/src/emfplus.rs +++ b/crates/metafile/src/emfplus.rs @@ -1181,7 +1181,11 @@ fn pen(r: Bytes) -> Option { off += 4 + (r.u32(off)? as usize).min(1 << 20); } } - let color = brush(r, off)?.color(); + // A pen whose texture brush was not read is counted like a lost brush object. + let Some(fill) = brush(r, off) else { + return lost_texture(r.tail(off)?).then_some(Obj::LostBrush); + }; + let color = fill.color(); Some(Obj::Pen { color, width: if width.is_finite() { width.clamp(0.0, 1.0e5) } else { 0.0 } }) } @@ -1290,16 +1294,20 @@ fn region_node(r: Bytes, off: usize, depth: usize, work: &mut usize) -> Option<( } } -/// The area a path encloses: its outline when that is one convex figure, else the rectangle -/// around it. The flag is set when the rectangle stands in for a path that encloses some area. +/// The area a path encloses: its outline when that is one convex figure, none when its figures all +/// lie on a line, else the rectangle around it. The flag is set when the rectangle stands in for +/// the path. fn area_of_ops(ops: &[Op]) -> (Vec

, bool) { + if !crate::geom::encloses_area(ops) { + return (Vec::new(), false); + } let subs = flatten(ops); if let [(pts, _)] = subs.as_slice() && let Some(c) = convex(pts) { return (c, false); } - (ops_bbox(ops).map(rect_poly).unwrap_or_default(), crate::geom::encloses_area(ops)) + (ops_bbox(ops).map(rect_poly).unwrap_or_default(), true) } /// Two areas' intersection, and whether it ran out of work and kept the first area instead. diff --git a/crates/metafile/src/lib.rs b/crates/metafile/src/lib.rs index 6a2338a78..26a406d31 100644 --- a/crates/metafile/src/lib.rs +++ b/crates/metafile/src/lib.rs @@ -78,10 +78,10 @@ pub struct Picture { /// the picture holds what came before. The limits that cut one path, string, region or dash /// pattern short do not set it. pub truncated: bool, - /// Images the file asks to draw that the reader did not read: bitmaps past a size, byte or decode - /// limit or in a form the reader cannot decode, EMF+ metafile images that do not parse or are - /// nested deeper than 3 levels, and pattern or texture brushes whose bitmap was not read (counted - /// when the brush is created). An image whose source rectangle selects none of its pixels, or that + /// Images the file asks to draw that the reader did not read: bitmaps past a size, item, byte or + /// decode limit or in a form the reader cannot decode, EMF+ metafile images that do not parse or + /// are nested deeper than 3 levels, and pattern or texture brushes whose bitmap was not read, a + /// pen's brush included (counted when the brush or pen is created). An image whose source rectangle selects none of its pixels, or that /// has no pixels, draws nothing and is not counted. Bitmaps left out while clipping to an EMF+ /// clip (see the README's limits) are not counted here. pub dropped_images: u32, diff --git a/crates/metafile/src/tests.rs b/crates/metafile/src/tests.rs index fa1cbb33a..9f929108b 100644 --- a/crates/metafile/src/tests.rs +++ b/crates/metafile/src/tests.rs @@ -486,6 +486,34 @@ fn canvas_stops_at_the_byte_budget() { assert_eq!(cv.items.len(), 2); } +#[test] +fn a_bitmap_the_byte_budget_refuses_is_counted_once() { + let img = || Rgba { w: 4, h: 4, px: vec![0u8; 64] }; + let diamond = || Some(vec![(2.0, 0.0), (4.0, 2.0), (2.0, 4.0), (0.0, 2.0)]); + let mut cv = Canvas::new(); + cv.set_clip(diamond()); + cv.bitmap([0.0, 0.0, 4.0, 4.0], img()); + assert!(!cv.full() && !cv.items.is_empty() && cv.dropped_images == 0, "a bitmap that is kept"); + // The clip cuts the bitmap into pieces, and none of them fits: one image is counted. + let mut cv = Canvas::new(); + cv.hold(MAX_BYTES - 1); + cv.set_clip(diamond()); + cv.bitmap([0.0, 0.0, 4.0, 4.0], img()); + assert!(cv.full() && cv.items.is_empty()); + assert_eq!(cv.dropped_images, 1); + // A clip that leaves none of the bitmap refuses nothing. + let mut cv = Canvas::new(); + cv.hold(MAX_BYTES - 1); + cv.set_clip(Some(vec![(100.0, 100.0), (110.0, 100.0), (110.0, 110.0)])); + cv.bitmap([0.0, 0.0, 4.0, 4.0], img()); + assert!(!cv.full() && cv.dropped_images == 0); + // A bitmap drawn with a raster operation is counted the same way. + let mut cv = Canvas::new(); + cv.hold(MAX_BYTES - 1); + cv.bitmap_rop([0.0, 0.0, 4.0, 4.0], img(), 0x00CC_0020); + assert!(cv.full() && cv.dropped_images == 1 && cv.unplayed_raster_ops == 0); +} + #[test] fn oversized_dib_header_size_is_rejected_without_overflow() { let mut info = info_header(2, 2, 24, 16); diff --git a/crates/metafile/src/tests/model.rs b/crates/metafile/src/tests/model.rs index 73186cb4c..5053606c4 100644 --- a/crates/metafile/src/tests/model.rs +++ b/crates/metafile/src/tests/model.rs @@ -485,6 +485,17 @@ fn pattern_and_texture_brushes_that_are_not_read_are_counted() { let none = cat(vec![h32(0xDBC0_1002), h32(1), h32(0), h32(4), h32(16), h32(0x0026_200A), h32(0)]); assert_eq!(parse(&texture(none)).unwrap().dropped_images, 0); assert_eq!(parse(&texture(bitmap_2x2())).unwrap().dropped_images, 0); + // A pen whose brush is such a texture brush. + let pen = |image: Vec| { + let brush = cat(vec![h32(0xDBC0_1002), h32(2), h32(0), h32(0), image]); + let pen = cat(vec![h32(0xDBC0_1002), h32(0), h32(0), h32(0), f32s(&[1.0]), brush]); + emf(&[plus_comment(&[plus_header(false), prec(0x4008, 0x0200 | 5, &pen)])], [0; 4], [0; 4]) + }; + let short = cat(vec![h32(0xDBC0_1002), h32(1), h32(4), h32(4), h32(16), h32(0x0026_200A), h32(0), vec![1, 2, 3]]); + assert_eq!(parse(&pen(short)).unwrap().dropped_images, 1); + let none = cat(vec![h32(0xDBC0_1002), h32(1), h32(0), h32(4), h32(16), h32(0x0026_200A), h32(0)]); + assert_eq!(parse(&pen(none)).unwrap().dropped_images, 0); + assert_eq!(parse(&pen(bitmap_2x2())).unwrap().dropped_images, 0); } #[test] @@ -643,6 +654,25 @@ fn emfplus_clips_that_are_not_applied_are_counted() { assert_eq!(plus(vec![clip(1, [0.0, 0.0, 50.0, 50.0]), clip(0, [1.0, 1.0, 5.0, 5.0])]), 0, "intersect and replace are applied"); } +#[test] +fn an_emfplus_clip_on_a_diagonal_line_encloses_nothing() { + let fill = prec(0x400A, 0x8000, &cat(vec![h32(0xFF00_0000), h32(1), f32s(&[0.0, 0.0, 100.0, 100.0])])); + let drawn = |clip: Vec>| { + let mut all = vec![plus_header(false)]; + all.extend(clip); + all.push(fill.clone()); + let p = parse(&emf(&[plus_comment(&all)], [0; 4], [0; 4])).unwrap(); + (p.items.len(), p.ignored_clips) + }; + let line = cat(vec![h32(0xDBC0_1002), h32(2), h32(0), f32s(&[0.0, 0.0, 10.0, 10.0]), vec![0, 1]]); + // SetClipPath with the line replaces the clip with nothing. + assert_eq!(drawn(vec![prec(0x4008, 0x0300 | 9, &line), prec(0x4033, 9, &[])]), (0, 0)); + // A region whose one node is the line. + let node = cat(vec![h32(0x1000_0001), h32(line.len() as u32), line]); + let region = cat(vec![h32(0xDBC0_1002), h32(1), node]); + assert_eq!(drawn(vec![prec(0x4008, 0x0400 | 6, ®ion), prec(0x4034, 6, &[])]), (0, 0)); +} + #[test] fn a_parse_stopped_at_a_limit_is_truncated() { assert!(!parse_emf(&[rect(0, 0, 10, 10)]).truncated); From 443a2b05ffa2588b7737f70b913126c7ad1899ea Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 15:51:23 -0500 Subject: [PATCH 091/209] docx: preserve partial table margins in current and tracked properties (#1924) --- crates/docx/src/read/props.rs | 11 ++++++ crates/docx/src/read/story/blocks.rs | 27 +++++++++++++- .../docx/tests/it/cell_margin_inheritance.rs | 15 ++++++++ crates/docx/tests/it/main.rs | 1 + crates/docx/tests/it/partial_table_margins.rs | 36 +++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 crates/docx/tests/it/partial_table_margins.rs diff --git a/crates/docx/src/read/props.rs b/crates/docx/src/read/props.rs index 4d86ae164..618ef8c10 100644 --- a/crates/docx/src/read/props.rs +++ b/crates/docx/src/read/props.rs @@ -562,6 +562,17 @@ pub fn margins(e: &El) -> [f32; 4] { margin_sides(e).map(|side| side.unwrap_or(0.0)) } +/// Word's own cell margins (the Normal Table style): 0.08" left and right, none above or below. +pub const WORD_CELL_MARGINS: [f32; 4] = [0.0, 5.4, 0.0, 5.4]; + +/// A table's `w:tblCellMar` over the margins it would have without one (its +/// style's, else Word's). A side the element leaves out keeps the underlying margin, as in Word: +/// a form table that sets only `w:left` keeps its style's right margin, not none. +pub fn margins_over(e: &El, base: [f32; 4]) -> [f32; 4] { + let sides = margin_sides(e); + std::array::from_fn(|i| sides.get(i).copied().flatten().or_else(|| base.get(i).copied()).unwrap_or(0.0)) +} + fn margin_sides(e: &El) -> [Option; 4] { let get = |names: &[&str]| names.iter().find_map(|n| e.child(n)).and_then(|c| tw(c, "w:w")).map(|v| v.clamp(0.0, 1584.0)); [get(&["w:top"]), get(&["w:left", "w:start"]), get(&["w:bottom"]), get(&["w:right", "w:end"])] diff --git a/crates/docx/src/read/story/blocks.rs b/crates/docx/src/read/story/blocks.rs index 878429454..91c54714f 100644 --- a/crates/docx/src/read/story/blocks.rs +++ b/crates/docx/src/read/story/blocks.rs @@ -333,12 +333,37 @@ impl Reader<'_> { } } + /// Omitted table-margin sides inherit from that property set's style. + fn table_margin_base(&self, style: Option<&str>) -> [f32; 4] { + style.and_then(|id| self.doc.styles.table_style(id)).and_then(|st| st.parts.cell_margins).unwrap_or(if self.no_default_table_style { + [0.0; 4] + } else { + crate::read::props::WORD_CELL_MARGINS + }) + } + fn read_table(&mut self, sc: &mut StoryCtx, t: &El, rels: &Rels, depth: usize) -> Option

{ let mut table = Table::default(); if let Some(p) = t.child("w:tblPr") { table.props = self.pc.tblpr(p); + // Sides `w:tblCellMar` leaves out keep the style's margins; without a style, Word's + // 0.08" left and right, or none in a file with no default table style. + if let Some(m) = p.child("w:tblCellMar") { + let base = self.table_margin_base(table.props.style.as_deref()); + table.props.cell_margins = Some(crate::read::props::margins_over(m, base)); + } if let Some(ch) = p.child("w:tblPrChange") { - let mut old = ch.child("w:tblPr").map(|o| self.pc.tblpr(o)).unwrap_or_default(); + let mut old = ch + .child("w:tblPr") + .map(|o| { + let mut props = self.pc.tblpr(o); + if let Some(m) = o.child("w:tblCellMar") { + let base = self.table_margin_base(props.style.as_deref()); + props.cell_margins = Some(crate::read::props::margins_over(m, base)); + } + props + }) + .unwrap_or_default(); // Rejecting the change brings back these properties: with the same margins. self.no_default_margins(&mut old); table.props.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, ch), old); diff --git a/crates/docx/tests/it/cell_margin_inheritance.rs b/crates/docx/tests/it/cell_margin_inheritance.rs index 83aafc7d0..dc5cba644 100644 --- a/crates/docx/tests/it/cell_margin_inheritance.rs +++ b/crates/docx/tests/it/cell_margin_inheritance.rs @@ -98,3 +98,18 @@ fn no_default_table_style_means_no_cell_margins() { let reopened = craft_docx::read(&bytes).unwrap(); assert_eq!(reopened.body[0].as_table().unwrap().props.cell_margins, Some([0.0; 4])); } + +#[test] +fn tracked_partial_table_margins_keep_the_old_styles_other_sides() { + let body = r#""#; + let styles = r#""#; + let doc = craft_docx::read(&package(body, Some(styles))).unwrap(); + let table = doc.body[0].as_table().unwrap(); + assert_eq!(table.props.cell_margins, Some([3.0, 5.0, 0.0, 9.0])); + let old = &table.props.fmt_change.as_ref().unwrap().old; + assert_eq!( + old.cell_margins, + Some([1.0, 0.0, 2.0, 8.0]), + "explicit zero overrides left, omitted sides come from the old style, not the current one" + ); +} diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index 8d6be8a97..6d521effb 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -29,6 +29,7 @@ mod malformed; mod missing_pictures; mod nested_block_controls; mod note_comments; +mod partial_table_margins; mod passthrough; mod picture_links; mod positional_tabs; diff --git a/crates/docx/tests/it/partial_table_margins.rs b/crates/docx/tests/it/partial_table_margins.rs new file mode 100644 index 000000000..7cf3c5f49 --- /dev/null +++ b/crates/docx/tests/it/partial_table_margins.rs @@ -0,0 +1,36 @@ +//! a table's partial cell margins keep the other sides. + +use std::io::Write; + +#[test] +fn a_left_only_table_margin_keeps_the_other_sides() { + let ns = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main""#; + let root = r#""#; + let rels = r#""#; + let styles = format!( + r#""# + ); + let table = |style: &str| { + format!( + r#"{style}Cell"# + ) + }; + let document = format!(r#"{}{}"#, table(r#""#), table("")); + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, data) in [ + ("_rels/.rels", root), + ("word/_rels/document.xml.rels", rels), + ("word/document.xml", document.as_str()), + ("word/styles.xml", styles.as_str()), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(data.as_bytes()).unwrap(); + } + let doc = craft_docx::read(&zip.finish().unwrap().into_inner()).unwrap(); + let margins: Vec<_> = doc.body.iter().filter_map(|b| b.as_table()).map(|t| t.props.cell_margins).collect(); + assert_eq!( + margins, + vec![Some([1.0, 5.35, 2.0, 8.0]), Some([0.0, 5.35, 0.0, 0.0])], + "the left side is the table's, the rest the style's (none without one: the file has no default table style)" + ); +} From 20220807a4c33a35a72c7b4dfad90edd29205e34 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 15:51:27 -0500 Subject: [PATCH 092/209] psd: preserve PhotoCraft lenient TySh prefix parsing (#1923) --- crates/psd/src/text/envelope.rs | 75 +++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/crates/psd/src/text/envelope.rs b/crates/psd/src/text/envelope.rs index 73345fa83..16fac8737 100644 --- a/crates/psd/src/text/envelope.rs +++ b/crates/psd/src/text/envelope.rs @@ -29,6 +29,33 @@ pub fn transform_is_safe(t: &[f64; 6]) -> bool { t.iter().all(|v| v.is_finite() && v.abs() <= f64::from(i32::MAX)) } +/// Recovers the text and raw transform of a `TySh` prefix, even if its descriptor is damaged. +/// +/// This is PhotoCraft's preservation fallback, not a rendering-ready type layer. It keeps +/// line endings, the trailing paragraph terminator and non-finite transform values verbatim. +/// Callers must validate geometry before rendering. A complete 52-byte prefix is required; +/// a missing, unsupported or malformed text descriptor produces empty text. Warp and bounds +/// are not interpreted. Above [`MAX_BYTES`], the descriptor is not parsed and text is empty, +/// retaining the fixed transform so consumers can preserve the original block without decoding +/// an oversized descriptor. Inputs within that cap retain the shared parser's allocation limits. +pub fn parse_tysh_text_prefix(data: &[u8]) -> Option<(String, [f64; 6])> { + let f = |at: usize| data.get(at..at + 8).map(|b| f64::from_be_bytes(b.try_into().unwrap_or([0; 8]))); + let mut m = [0.0; 6]; + for (i, v) in m.iter_mut().enumerate() { + *v = f(2 + i * 8)?; + } + // text version (2) at 50, descriptor version (4) at 52, descriptor at 56. + let rest = data.get(52..)?; + if data.len() > MAX_BYTES { + return Some((String::new(), m)); + } + let text = VersionedDescriptor::parse_prefix(rest).ok().and_then(|(d, _)| match d.descriptor.get("Txt ") { + Some(D::Text(t)) => Some(t.to_string_lossy()), + _ => None, + }); + Some((text.unwrap_or_default(), m)) +} + /// Parses a `TySh` block. pub fn parse_tysh(data: &[u8]) -> Option { if data.len() > MAX_BYTES { @@ -246,6 +273,54 @@ mod tests { bounds: [-1.0, 2.0, 30.0, 40.0], } } + #[test] + fn recovery_prefix_preserves_text_and_ignores_damaged_tail() { + let mut t = sample(); + t.text = t.text.with("Txt ", D::Text(UnicodeString::new_nul("Hello\r\nWorld\r"))); + let mut bytes = write_tysh(&t).unwrap(); + let (_, n) = VersionedDescriptor::parse_prefix(&bytes[52..]).unwrap(); + bytes.truncate(52 + n); + bytes.extend_from_slice(b"broken warp"); + assert_eq!(parse_tysh_text_prefix(&bytes), Some(("Hello\r\nWorld\r".into(), t.transform))); + } + + #[test] + fn recovery_prefix_needs_only_complete_fixed_header() { + let bytes = write_tysh(&sample()).unwrap(); + for cut in 0..52 { + assert!(parse_tysh_text_prefix(&bytes[..cut]).is_none(), "{cut}"); + } + for cut in 52..56 { + assert_eq!(parse_tysh_text_prefix(&bytes[..cut]), Some((String::new(), sample().transform))); + } + let mut unsupported = bytes.clone(); + unsupported[52..56].copy_from_slice(&99u32.to_be_bytes()); + assert_eq!(parse_tysh_text_prefix(&unsupported), Some((String::new(), sample().transform))); + assert_eq!(parse_tysh_text_prefix(&bytes), Some((String::new(), sample().transform))); + } + + #[test] + fn recovery_prefix_bounds_descriptor_work_without_losing_preservation_geometry() { + let mut t = sample(); + t.text = t.text.with("Txt ", D::Text(UnicodeString::new_nul("kept"))); + let mut bytes = write_tysh(&t).unwrap(); + bytes.resize(MAX_BYTES, 0); + assert_eq!(parse_tysh_text_prefix(&bytes), Some(("kept".into(), t.transform))); + bytes.push(0); + assert_eq!(parse_tysh_text_prefix(&bytes), Some((String::new(), t.transform))); + assert!(parse_tysh(&bytes).is_none()); + } + + #[test] + fn recovery_prefix_retains_geometry_but_is_not_render_validation() { + let mut bytes = write_tysh(&sample()).unwrap(); + bytes[2..10].copy_from_slice(&f64::NAN.to_be_bytes()); + let (_, transform) = parse_tysh_text_prefix(&bytes).unwrap(); + assert!(transform[0].is_nan()); + assert!(!transform_is_safe(&transform)); + assert!(parse_tysh(&bytes).is_none()); + } + #[test] fn roundtrip_preserves_descriptors_transform_and_bounds() { let t = sample(); From 617a3cea3ba6a7fbad366b7e834109d9648376cc Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:51:32 +0100 Subject: [PATCH 093/209] Add craft-media-source: MediaSource, open_bytes, and open_reader (#1891) --- Cargo.lock | 14 + Cargo.toml | 1 + crates/media-source/Cargo.toml | 23 ++ crates/media-source/src/lib.rs | 563 +++++++++++++++++++++++++++++++++ 4 files changed, 601 insertions(+) create mode 100644 crates/media-source/Cargo.toml create mode 100644 crates/media-source/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 9275d4646..8679e4eac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2281,6 +2281,20 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-media-source" +version = "0.1.0" +dependencies = [ + "craft-audio", + "craft-color", + "craft-frame", + "craft-media", + "craft-time", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-media-testkit" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bdcb8201d..c8ae36287 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ craft-macho = { path = "crates/macho" } craft-math = { path = "crates/math" } # No default features: craft-audio turns on the craft-media modules each of its features needs. craft-media = { path = "crates/media", default-features = false } +craft-media-source = { path = "crates/media-source" } craft-materialx = { path = "crates/materialx" } craft-media-testkit = { path = "crates/media-testkit" } craft-midi = { path = "crates/midi" } diff --git a/crates/media-source/Cargo.toml b/crates/media-source/Cargo.toml new file mode 100644 index 000000000..3edda0b2e --- /dev/null +++ b/crates/media-source/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "craft-media-source" +description = "Compositor MediaSource trait and a pluggable open_bytes / open_reader registry" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-audio = { workspace = true } +craft-color = { workspace = true } +craft-frame = { workspace = true } +craft-media = { workspace = true } +craft-time = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/crates/media-source/src/lib.rs b/crates/media-source/src/lib.rs new file mode 100644 index 000000000..c27aea688 --- /dev/null +++ b/crates/media-source/src/lib.rs @@ -0,0 +1,563 @@ +//! Compositor media sources and a pluggable opener registry. +//! +//! This crate is the EffectCraft / FilmCraft opener hub: a [`MediaSource`] yields compositor +//! [`VideoFrame`]s and [`AudioBuffer`]s, and [`open_bytes`] / [`open_reader`] try caller-supplied +//! openers in order. It lives in its own crate rather than `craft-media` so it can depend on +//! `craft-audio` without a cycle (`craft-audio` already depends on `craft-media`). +//! +//! Built-in still-image, WAV, and generator openers stay in the apps. [`open_bytes`] only runs the +//! `extra` openers; [`open_reader`] tries `reader_openers` and otherwise reads the file whole +//! (up to `max_whole`) and falls back to [`open_bytes`]. Container sources register themselves +//! from later PRs. +//! +//! [`SharedSource`] is `Arc`. It is not `craft_audio::SharedSource`, which is a +//! `ByteSource` marker used by the audio decoder. + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +use std::io; +use std::path::Path; +use std::sync::Arc; + +use craft_audio::AudioBuffer; +use craft_color::{ColorInfo, HdrMetadata}; +use craft_frame::VideoFrame; +use craft_media::ByteSource; +use craft_time::{FrameRate, Tick}; +use serde::{Deserialize, Serialize}; + +/// Errors from opening or reading a media source. +#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] +pub enum MediaError { + #[error("unsupported media: {0}")] + Unsupported(String), + #[error("I/O error: {0}")] + Io(String), + #[error("decode error: {0}")] + Decode(String), + #[error("no {0} stream")] + NoStream(&'static str), + #[error("media offline: {0}")] + Offline(String), + /// The request was cancelled. + #[error("cancelled")] + Cancelled, +} + +impl From for MediaError { + fn from(e: io::Error) -> Self { + MediaError::Io(e.to_string()) + } +} + +pub type Result = std::result::Result; + +/// Kind of media a source reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum MediaKind { + /// Video, possibly with audio. + Movie, + AudioOnly, + Still, + ImageSequence, + Synthetic, +} + +/// Video stream metadata a source reports. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct VideoStreamInfo { + pub width: u32, + pub height: u32, + pub frame_rate: FrameRate, + pub par: (u32, u32), + pub codec: String, + pub pixel_format: String, + pub color: ColorInfo, + pub has_alpha: bool, + /// Bitrate in bits/s when known. + pub bitrate: Option, + /// HDR static metadata when the file carries it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hdr: Option, +} + +/// Most audio streams a source reports (the count comes from the file: never trusted). +pub const MAX_AUDIO_STREAMS: usize = 64; + +/// Audio stream metadata a source reports. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AudioStreamInfo { + pub sample_rate: u32, + pub channels: u32, + pub codec: String, + pub bits_per_sample: Option, +} + +/// Identity and streams of an open source. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct MediaInfo { + pub name: String, + pub kind: MediaKind, + /// Media duration. Stills typically report a default duration chosen by the app. + pub duration: Tick, + pub video: Option, + /// Every audio stream the container carries, in file order. Stream 0 is the one + /// [`MediaSource::audio`] reads; the others are read with [`MediaSource::audio_stream`]. + /// Project files written before multi-stream support hold a single `audio` object (or + /// `null`): both still load. + #[serde(default, alias = "audio", deserialize_with = "deserialize_audio_streams")] + pub audio_streams: Vec, + pub container: String, + /// Timecode of the first frame (in frames at `video.frame_rate`), if the file carries one. + pub start_timecode: Option, + pub file_size: Option, +} + +/// Reads `audio_streams` from a list, or from the single optional stream older project files wrote. +fn deserialize_audio_streams<'de, D: serde::Deserializer<'de>>(d: D) -> std::result::Result, D::Error> { + struct StreamsVisitor; + struct RejectExtra; + impl<'de> serde::de::DeserializeSeed<'de> for RejectExtra { + type Value = (); + fn deserialize>(self, _d: D) -> std::result::Result<(), D::Error> { + Err(serde::de::Error::custom("audio stream count exceeds MAX_AUDIO_STREAMS")) + } + } + impl<'de> serde::de::Visitor<'de> for StreamsVisitor { + type Value = Vec; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("at most 64 audio streams, a legacy audio object, or null") + } + fn visit_unit(self) -> std::result::Result { + Ok(Vec::new()) + } + fn visit_none(self) -> std::result::Result { + Ok(Vec::new()) + } + fn visit_map>(self, map: A) -> std::result::Result { + AudioStreamInfo::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(|stream| vec![stream]) + } + fn visit_seq>(self, mut seq: A) -> std::result::Result { + let mut streams = Vec::new(); + // Do not reserve from an untrusted size hint or buffer an untagged sequence. + while streams.len() < MAX_AUDIO_STREAMS { + let Some(stream) = seq.next_element()? else { return Ok(streams) }; + streams.push(stream); + } + // Detect an excess entry without deserializing or allocating its contents. + seq.next_element_seed(RejectExtra)?; + Ok(streams) + } + } + d.deserialize_any(StreamsVisitor) +} + +impl MediaInfo { + pub fn frame_rate(&self) -> FrameRate { + self.video.as_ref().map(|v| v.frame_rate).unwrap_or_default() + } + pub fn has_video(&self) -> bool { + self.video.is_some() + } + pub fn has_audio(&self) -> bool { + !self.audio_streams.is_empty() + } + /// The primary audio stream (stream 0), the one [`MediaSource::audio`] reads. + pub fn audio(&self) -> Option<&AudioStreamInfo> { + self.audio_streams.first() + } +} + +/// A request for a video frame. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FrameRequest { + /// Media time. + pub time: Tick, + /// Desired scale relative to full resolution (1.0 = full). Sources may return a larger frame. + pub scale: f32, +} + +impl FrameRequest { + pub fn full(time: Tick) -> Self { + Self { time, scale: 1.0 } + } +} + +/// A decoded or generated media source that can yield compositor frames and audio. +pub trait MediaSource: Send + Sync { + fn info(&self) -> &MediaInfo; + /// The frame displayed at media time `req.time`. + fn video_frame(&self, req: FrameRequest) -> Result>; + /// `frames` audio frames starting at sample index `start` (at `sample_rate`, media time). + fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result; + /// Like [`audio`](Self::audio) for audio stream `stream` (an index into + /// [`MediaInfo::audio_streams`]). Sources with a single audio stream need not override this: + /// stream 0 is `audio`, any other index is [`MediaError::NoStream`]. + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + if stream == 0 { self.audio(start, frames, sample_rate) } else { Err(MediaError::NoStream("audio")) } + } +} + +/// Shared compositor source. Distinct from [`craft_audio::SharedSource`], a `ByteSource` marker. +pub type SharedSource = Arc; + +/// Random-access bytes used by [`open_reader`]. +pub type SharedBytes = Arc; + +/// File extensions recognised at import (the media browser filters on these). +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "tga", "dpx", "exr", "hdr", "dng"]; +pub const AUDIO_EXTENSIONS: &[&str] = &["wav", "wave", "bwf", "aif", "aiff", "mp3", "mp2", "m4a", "aac", "flac", "ogg", "opus"]; +pub const VIDEO_EXTENSIONS: &[&str] = &[ + "mp4", "m4v", "mov", "mkv", "webm", "avi", "mxf", "mts", "m2ts", "m2t", "ts", "mpg", "mpeg", "vob", "mod", "tod", "m2v", "m1v", "mpv", "3gp", + "y4m", "apv", +]; + +/// True when `path` has a still, audio, or video extension we recognise, ignoring case. +pub fn is_importable(path: &Path) -> bool { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + STILL_EXTENSIONS.contains(&ext.as_str()) || AUDIO_EXTENSIONS.contains(&ext.as_str()) || VIDEO_EXTENSIONS.contains(&ext.as_str()) +} + +/// A pluggable in-memory opener. Returns `None` when the format is not this opener's. +pub type Opener = fn(name: &str, bytes: Arc<[u8]>) -> Option>; + +/// Opens a container from a reader: `head` is the first bytes of the file (for sniffing). +/// Returns `None` when the format is not this opener's. +pub type ReaderOpener = fn(name: &str, head: &[u8], reader: &SharedBytes) -> Option>; + +/// Bytes sniffed by [`open_reader`]. +pub const HEAD_LEN: usize = 64 * 1024; + +/// Default ceiling for a whole-file fallback or an offline output buffer (256 MiB). +/// Streaming openers are tried first. Use open_reader_within for an explicit different file budget. +pub const DEFAULT_MAX_MEDIA_BYTES: usize = 256 << 20; + +fn allocate(len: usize, max_bytes: usize) -> io::Result> { + if len.checked_mul(std::mem::size_of::()).is_none_or(|bytes| bytes > max_bytes || bytes > isize::MAX as usize) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "media buffer exceeds allocation limit")); + } + let mut out = Vec::new(); + out.try_reserve_exact(len).map_err(|e| io::Error::other(format!("cannot allocate media buffer: {e}")))?; + out.resize(len, T::default()); + Ok(out) +} + +fn io_named(name: &str, e: io::Error) -> MediaError { + MediaError::Io(format!("{name}: {e}")) +} + +/// Read `len` bytes at `offset` (fewer at the end of the source). +pub fn read_range(r: &dyn ByteSource, offset: u64, len: usize) -> io::Result> { + read_range_within(r, offset, len, DEFAULT_MAX_MEDIA_BYTES) +} + +fn read_range_within(r: &dyn ByteSource, offset: u64, len: usize, max_bytes: usize) -> io::Result> { + let n = usize::try_from(r.len().saturating_sub(offset).min(len as u64)) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "media length is not representable"))?; + let mut v = allocate(n, max_bytes)?; + r.read_at(offset, &mut v)?; + Ok(v) +} + +/// Open media from bytes by trying `extra` openers in order. There is no built-in fallback: +/// stills, WAV, and generators stay in the apps and register here. +pub fn open_bytes(name: &str, bytes: Arc<[u8]>, extra: &[Opener]) -> Result { + for o in extra { + if let Some(r) = o(name, bytes.clone()) { + return r; + } + } + Err(MediaError::Unsupported(format!("{name}: unrecognised format"))) +} + +/// Open media from a reader: `reader_openers` first; otherwise read the whole file (if it fits in +/// `max_whole`) and use [`open_bytes`] with `extra`. +pub fn open_reader(name: &str, reader: SharedBytes, reader_openers: &[ReaderOpener], extra: &[Opener]) -> Result { + open_reader_within(name, reader, reader_openers, extra, DEFAULT_MAX_MEDIA_BYTES as u64) +} + +/// Like [`open_reader`], but a file no reader opener takes is read whole only up to `max_whole` +/// bytes; a larger one is refused instead. For looking at files rather than importing them: +/// reading a multi-gigabyte file into memory to show its duration, or to find out it isn't +/// supported, thrashes the disk. +pub fn open_reader_within( + name: &str, + reader: SharedBytes, + reader_openers: &[ReaderOpener], + extra: &[Opener], + max_whole: u64, +) -> Result { + let head = read_range(&*reader, 0, HEAD_LEN).map_err(|e| io_named(name, e))?; + for o in reader_openers { + if let Some(r) = o(name, &head, &reader) { + return r; + } + } + if extra.is_empty() { + return Err(MediaError::Unsupported(format!("{name}: unrecognised format"))); + } + if reader.len() > max_whole { + return Err(MediaError::Unsupported(format!("{name}: no streaming reader for this format, and it is too large to read whole here"))); + } + let len = usize::try_from(reader.len()).map_err(|_| MediaError::Unsupported(format!("{name}: media length is not representable")))?; + let budget = usize::try_from(max_whole).unwrap_or(usize::MAX); + let all = read_range_within(&*reader, 0, len, budget).map_err(|e| io_named(name, e))?; + open_bytes(name, all.into(), extra) +} + +/// Open a regular file through the shared pooled FileReader. Non-files (including FIFOs) +/// are refused without a blocking open, using the same contract as other media readers. +#[cfg(not(target_arch = "wasm32"))] +pub fn open_path(path: &Path, reader_openers: &[ReaderOpener], extra: &[Opener]) -> Result { + let file = craft_media::FileReader::open(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + open_reader(&name, Arc::new(file), reader_openers, extra) +} + +fn audio_channels(info: &MediaInfo) -> u16 { + info.audio().map_or(2, |a| u16::try_from(a.channels).unwrap_or(u16::MAX).max(1)) +} + +/// A source that renders a solid slate (used for offline media). +pub struct OfflineSource { + pub info: MediaInfo, +} + +impl MediaSource for OfflineSource { + fn info(&self) -> &MediaInfo { + &self.info + } + fn video_frame(&self, req: FrameRequest) -> Result> { + let v = self.info.video.as_ref().ok_or(MediaError::NoStream("video"))?; + if !req.scale.is_finite() || req.scale <= 0.0 || v.width == 0 || v.height == 0 { + return Err(MediaError::Decode("invalid offline frame dimensions or scale".into())); + } + let s = req.scale.clamp(0.02, 1.0); + let (w, h) = (((v.width as f32 * s) as u32).max(2), ((v.height as f32 * s) as u32).max(2)); + let samples = usize::try_from(w) + .ok() + .and_then(|w| usize::try_from(h).ok().and_then(|h| w.checked_mul(h))) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or_else(|| MediaError::Decode("offline frame dimensions overflow".into()))?; + let mut px = allocate::(samples, DEFAULT_MAX_MEDIA_BYTES)?; + for p in px.as_chunks_mut::<4>().0 { + p.copy_from_slice(&[140, 16, 16, 255]); + } + Ok(Arc::new(VideoFrame::rgba8(w, h, px).map_err(|e| MediaError::Decode(e.to_string()))?.with_pts(req.time))) + } + fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> Result { + let channels = usize::from(audio_channels(&self.info)); + let bytes = frames + .checked_mul(channels) + .and_then(|n| n.checked_mul(std::mem::size_of::())) + .ok_or_else(|| MediaError::Decode("offline audio dimensions overflow".into()))?; + if bytes > DEFAULT_MAX_MEDIA_BYTES { + return Err(MediaError::Decode("offline audio exceeds allocation limit".into())); + } + let mut planes = Vec::new(); + planes.try_reserve_exact(channels).map_err(|e| MediaError::Io(e.to_string()))?; + for _ in 0..channels { + planes.push(allocate::(frames, DEFAULT_MAX_MEDIA_BYTES)?); + } + Ok(AudioBuffer { sample_rate, channels: planes }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn silent(name: &str, kind: MediaKind) -> OfflineSource { + OfflineSource { + info: MediaInfo { + name: name.into(), + kind, + duration: Tick::from_seconds_f64(1.0), + video: Some(VideoStreamInfo { + width: 16, + height: 8, + frame_rate: FrameRate::FPS_24, + par: (1, 1), + codec: "slate".into(), + pixel_format: "rgba8".into(), + color: ColorInfo::default(), + has_alpha: false, + bitrate: None, + hdr: None, + }), + audio_streams: vec![AudioStreamInfo { sample_rate: 48000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(32) }], + container: "none".into(), + start_timecode: None, + file_size: None, + }, + } + } + + fn accept_ok(name: &str, _bytes: Arc<[u8]>) -> Option> { + if name.ends_with(".ok") { Some(Ok(Arc::new(silent(name, MediaKind::Synthetic)))) } else { None } + } + + fn reject_all(_name: &str, _bytes: Arc<[u8]>) -> Option> { + None + } + + fn boom(name: &str, _bytes: Arc<[u8]>) -> Option> { + Some(Err(MediaError::Decode(format!("{name}: boom")))) + } + + fn reader_ok(name: &str, head: &[u8], _reader: &SharedBytes) -> Option> { + if head.starts_with(b"OK") { Some(Ok(Arc::new(silent(name, MediaKind::Movie)))) } else { None } + } + + struct HugeSource; + impl ByteSource for HugeSource { + fn len(&self) -> u64 { + u64::MAX + } + fn read_at(&self, _offset: u64, buf: &mut [u8]) -> io::Result<()> { + assert!(buf.len() <= HEAD_LEN, "must reject whole-file allocation before reading"); + buf.fill(0); + Ok(()) + } + } + + #[test] + fn whole_file_fallback_and_explicit_ranges_are_bounded() { + let reader: SharedBytes = Arc::new(HugeSource); + assert!(open_reader("large.ok", reader.clone(), &[], &[accept_ok]).is_err()); + assert!(open_reader_within("large.ok", reader.clone(), &[], &[accept_ok], u64::MAX).is_err()); + assert!(read_range(&HugeSource, 0, usize::MAX).is_err()); + assert!(read_range_within(&HugeSource, 0, 10, 9).is_err()); + assert_eq!(read_range_within(&HugeSource, 0, 10, 10).unwrap(), vec![0; 10]); + } + + struct SniffOnlySource(std::sync::atomic::AtomicUsize); + impl ByteSource for SniffOnlySource { + fn len(&self) -> u64 { + u64::MAX + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + assert_eq!(offset, 0); + assert_eq!(buf.len(), HEAD_LEN); + assert_eq!(self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst), 0, "must only sniff once"); + buf.fill(0); + buf[..2].copy_from_slice(b"OK"); + Ok(()) + } + } + + #[test] + fn no_fallback_stops_after_sniff_and_streaming_opener_precedes_size_cap() { + let reader = Arc::new(SniffOnlySource(std::sync::atomic::AtomicUsize::new(0))); + let error = open_reader("unknown", reader.clone(), &[], &[]).err().unwrap(); + assert!(matches!(error, MediaError::Unsupported(_))); + assert_eq!(reader.0.load(std::sync::atomic::Ordering::SeqCst), 1); + let reader = Arc::new(SniffOnlySource(std::sync::atomic::AtomicUsize::new(0))); + let source = open_reader_within("large.mov", reader.clone(), &[reader_ok], &[accept_ok], 0).unwrap(); + assert_eq!(source.info().kind, MediaKind::Movie); + assert_eq!(reader.0.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[test] + fn hostile_offline_metadata_and_requests_are_errors() { + let mut source = silent("bad", MediaKind::Synthetic); + source.info.video.as_mut().unwrap().width = u32::MAX; + source.info.video.as_mut().unwrap().height = u32::MAX; + assert!(source.video_frame(FrameRequest::full(Tick::ZERO)).is_err()); + assert!(source.audio(0, usize::MAX, 48000).is_err()); + let source = silent("bad-scale", MediaKind::Synthetic); + for scale in [f32::NAN, f32::INFINITY, 0.0, -1.0] { + assert!(source.video_frame(FrameRequest { time: Tick::ZERO, scale }).is_err()); + } + } + + #[cfg(unix)] + #[test] + fn open_path_refuses_a_fifo_without_waiting_for_a_writer() { + let root = std::env::temp_dir().join(format!("craft-media-source-fifo-{}", std::process::id())); + std::fs::create_dir(&root).unwrap(); + let path = root.join("fifo"); + if std::process::Command::new("mkfifo").arg(&path).status().is_ok_and(|s| s.success()) { + assert!(open_path(&path, &[], &[]).is_err()); + std::fs::remove_file(&path).unwrap(); + } + std::fs::remove_dir(&root).unwrap(); + } + + #[test] + fn importable_extensions_match_regardless_of_case() { + for name in ["clip.mov", "clip.MOV", "clip.Mov", "shot.MP4", "IMG_0001.JPG", "sound.WAV", "take.M2TS"] { + assert!(is_importable(Path::new(name)), "{name}"); + } + assert!(!is_importable(Path::new("notes.txt"))); + } + + #[test] + fn open_bytes_tries_extra_in_order_and_otherwise_is_unsupported() { + let bytes: Arc<[u8]> = Arc::from(&b"xxxx"[..]); + let err = open_bytes("a.bin", bytes.clone(), &[]).err().expect("expected error"); + assert!(matches!(err, MediaError::Unsupported(_))); + let err = open_bytes("a.bin", bytes.clone(), &[reject_all, accept_ok]).err().expect("expected error"); + assert!(matches!(err, MediaError::Unsupported(_))); + let src = open_bytes("clip.ok", bytes.clone(), &[reject_all, accept_ok]).unwrap(); + assert_eq!(src.info().name, "clip.ok"); + let err = open_bytes("clip.ok", bytes, &[boom, accept_ok]).err().expect("expected error"); + assert!(matches!(err, MediaError::Decode(_))); + } + + #[test] + fn open_reader_uses_reader_openers_then_falls_back_within_max_whole() { + let small: SharedBytes = Arc::new(b"HELLO".to_vec()); + let err = open_reader("a.bin", small.clone(), &[], &[]).err().expect("expected error"); + assert!(matches!(err, MediaError::Unsupported(_))); + + let ok: SharedBytes = Arc::new(b"OK-file".to_vec()); + let src = open_reader("clip.mov", ok, &[reader_ok], &[]).unwrap(); + assert_eq!(src.info().kind, MediaKind::Movie); + + let tagged: SharedBytes = Arc::new(b"xxxx".to_vec()); + let src = open_reader("clip.ok", tagged.clone(), &[], &[accept_ok]).unwrap(); + assert_eq!(src.info().name, "clip.ok"); + + let big = vec![0u8; HEAD_LEN + 8]; + let r: SharedBytes = Arc::new(big); + let e = open_reader_within("a.avi", r, &[], &[accept_ok], HEAD_LEN as u64).err().expect("expected error"); + assert!(matches!(e, MediaError::Unsupported(s) if s.contains("too large"))); + } + + #[test] + fn audio_streams_serde_accepts_legacy_single_object() { + let stream = r#"{"sample_rate":48000,"channels":2,"codec":"pcm","bits_per_sample":null}"#; + let prefix = r#"{"name":"a","kind":"Movie","duration":0,"video":null,"audio_streams":["#; + let suffix = r#"],"container":"wav","start_timecode":null,"file_size":null}"#; + let full = std::iter::repeat_n(stream, MAX_AUDIO_STREAMS).collect::>().join(","); + let info: MediaInfo = serde_json::from_str(&format!("{prefix}{full}{suffix}")).unwrap(); + assert_eq!(info.audio_streams.len(), MAX_AUDIO_STREAMS); + for extra in [stream, r#"{"malformed":"not an audio stream"}"#] { + let error = serde_json::from_str::(&format!("{prefix}{full},{extra}{suffix}")).unwrap_err(); + assert!(error.to_string().contains("MAX_AUDIO_STREAMS"), "{error}"); + } + let many = r#"{"name":"a","kind":"Movie","duration":0,"video":null,"audio_streams":[{"sample_rate":48000,"channels":2,"codec":"pcm","bits_per_sample":null}],"container":"wav","start_timecode":null,"file_size":null}"#; + let info: MediaInfo = serde_json::from_str(many).unwrap(); + assert_eq!(info.audio_streams.len(), 1); + let one = r#"{"name":"a","kind":"AudioOnly","duration":0,"video":null,"audio":{"sample_rate":44100,"channels":1,"codec":"pcm","bits_per_sample":16},"container":"wav","start_timecode":null,"file_size":null}"#; + let info: MediaInfo = serde_json::from_str(one).unwrap(); + assert_eq!(info.audio().map(|a| a.sample_rate), Some(44100)); + let none = r#"{"name":"a","kind":"Still","duration":0,"video":null,"audio":null,"container":"png","start_timecode":null,"file_size":null}"#; + let info: MediaInfo = serde_json::from_str(none).unwrap(); + assert!(info.audio_streams.is_empty()); + } + + #[test] + fn offline_source_returns_slate_and_silence() { + let src = silent("off", MediaKind::Synthetic); + let frame = src.video_frame(FrameRequest::full(Tick::ZERO)).unwrap(); + assert_eq!((frame.width, frame.height), (16, 8)); + assert_eq!(frame.pts, Tick::ZERO); + let audio = src.audio(0, 8, 48000).unwrap(); + assert_eq!(audio.sample_rate, 48000); + assert_eq!(audio.num_channels(), 2); + assert_eq!(audio.frames(), 8); + assert!(src.audio_stream(1, 0, 8, 48000).is_err()); + } +} From f200f1c91df2c8b5ba8ef3c3c853382ced75df1f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 21:53:12 +0100 Subject: [PATCH 094/209] dds: 16-bit input and output (R16, R16G16, R16G16B16A16; BC4/BC5 fitted to 16-bit values) encode16 writes 16-bit pixels uncompressed as R16_UNORM, R16G16_UNORM or R16G16B16A16_UNORM with the DX10 header, and fits BC4/BC5 endpoints and indices against the 16-bit values; BC1-BC3 round to 8 bits first. Texture::decode16 reads these formats (and legacy FourCC 36, wide channel masks, BC4/BC5 interpolated at 16 bits) without losing precision. --- crates/dds/README.md | 18 +- crates/dds/src/bc.rs | 120 ++++++++++- crates/dds/src/lib.rs | 453 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 545 insertions(+), 46 deletions(-) diff --git a/crates/dds/README.md b/crates/dds/README.md index a3f81e50b..7a25eb36e 100644 --- a/crates/dds/README.md +++ b/crates/dds/README.md @@ -11,22 +11,32 @@ craft-dds = { git = "https://github.com/storytold/craft-libs", rev = "" | | Read | Write | |---|---|---| -| Uncompressed | 8, 16, 24 or 32 bits per pixel described by channel masks, RGB or luminance, with or without alpha; padded rows (`DDSD_PITCH`) | BGRA, BGRX, luminance, luminance with alpha | +| Uncompressed | 8, 16, 24 or 32 bits per pixel described by channel masks, RGB or luminance, with or without alpha; padded rows (`DDSD_PITCH`); 16 bits per channel: `R16_UNORM`, `R16G16_UNORM`, `R16G16B16A16_UNORM` (DXGI 56, 35, 11 and their typeless forms; legacy FourCC 36) | BGRA, BGRX, luminance, luminance with alpha; from 16-bit pixels `R16_UNORM`, `R16G16_UNORM`, `R16G16B16A16_UNORM` | | Block-compressed | BC1, BC2, BC3 (DXT1, DXT3, DXT5); unsigned BC4 (ATI1, BC4U, DXGI 79 and 80) and BC5 (ATI2, BC5U, DXGI 82 and 83) | BC1, BC2, BC3, BC4, BC5 | -| Headers | legacy and DX10 | legacy; DX10 for BC4 and BC5 | +| Headers | legacy and DX10 | legacy; DX10 for BC4, BC5 and the 16-bit formats | | Mip levels | the top level only | optional full chain, box-filtered | Cubemaps, texture arrays, volume textures, premultiplied-alpha textures (DXT2/DXT4, or DX10 alpha mode 2), signed (SNORM) BC4/BC5 and -the other DXGI formats (BC6H, BC7, float) are refused with `DdsError::Unsupported` rather than read +the other DXGI formats (BC6H, BC7, float, signed or integer 16-bit) are refused with `DdsError::Unsupported` rather than read in part. -Pixels are 8 bits per channel, interleaved and top row first, in one of four `Layout`s (`Gray`, +Pixels are 8 bits per channel (`encode`, `Texture::decode`) or 16 bits per channel (`encode16`, +`Texture::decode16`), interleaved and top row first, in one of four `Layout`s (`Gray`, `GrayA`, `Rgb`, `Rgba`). BC1 to BC3 are written from RGBA, BC4 from gray and BC5 from RGB (`DdsCompression::layout` names it). BC4 reads back as gray. BC5 stores red and green only, so it reads back as RGB with blue 0: for a tangent-space normal map, X and Y, with Z left to the renderer. ATI2 files are read as Microsoft's tools read them (red in the first block), not with the two blocks swapped as FFmpeg does. +From 16-bit pixels, uncompressed textures keep 16 bits per channel: gray as `R16_UNORM`, gray +and alpha (or any two channels, such as a normal map's X and Y) as `R16G16_UNORM`, RGB (alpha +65535) and RGBA as `R16G16B16A16_UNORM`, all with the DX10 header; they read back as gray, gray +and alpha, and RGBA. BC4 and BC5 keep 8-bit endpoints, as the formats store them, but pick them +and the indices against the 16-bit values, so a smooth height or normal map has less error than +when its 8-bit rounding is compressed. BC1 to BC3 round the pixels to 8 bits first. +`Texture::decode` rounds 16-bit formats to 8 bits; `Texture::decode16` returns them exactly, +interpolates BC4 and BC5 at 16 bits, and scales every 8-bit format by 257. + ```rust use craft_dds::{DdsCompression, DdsOptions, Layout, Texture}; diff --git a/crates/dds/src/bc.rs b/crates/dds/src/bc.rs index 863aa3077..c19a393f3 100644 --- a/crates/dds/src/bc.rs +++ b/crates/dds/src/bc.rs @@ -4,7 +4,9 @@ //! knows about DDS headers, so other containers of the same blocks can use it. //! //! BC4 is one block of 16 single-channel values, the same block BC3 stores its alpha in; BC5 is -//! two of them, red then green. [`value_block`] and [`decode_values`] serve all three. +//! two of them, red then green. [`value_block`] and [`decode_values`] serve all three; +//! [`value_block16`] and [`decode_values16`] are the same blocks for 16-bit values (BC4 and BC5 +//! from 16-bit sources). //! //! The encoder fits a block's colours with two RGB565 endpoints along their principal axis, //! refines them by least squares, and scores every candidate against the exact palette the @@ -69,6 +71,26 @@ pub(crate) fn decode_colour(block: &[u8], one_bit_alpha: bool) -> Block { std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) } +/// [`alpha_palette`] at 16 bits: the same eight values before they are rounded to 8 bits, as a +/// decoder that interpolates at a higher precision shows them (endpoints scaled by 257, so 255 +/// is 65535). +pub(crate) fn alpha_palette16(a0: u8, a1: u8) -> [u16; 8] { + let (a0, a1) = (u32::from(a0) * 257, u32::from(a1) * 257); + let mut palette = [a0, a1, 0, 0, 0, 0, 0, 65535]; + if a0 > a1 { + for (k, v) in palette.iter_mut().enumerate().skip(2) { + let k = k as u32; + *v = ((8 - k) * a0 + (k - 1) * a1 + 3) / 7; + } + } else { + for (k, v) in palette.iter_mut().enumerate().take(6).skip(2) { + let k = k as u32; + *v = ((6 - k) * a0 + (k - 1) * a1 + 2) / 5; + } + } + palette.map(|v| v as u16) +} + /// BC2's alpha: four explicit bits per texel. pub(crate) fn decode_bc2_alpha(block: &[u8]) -> [u8; 16] { let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0))); @@ -84,6 +106,14 @@ pub(crate) fn decode_values(block: &[u8]) -> [u8; 16] { std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) } +/// [`decode_values`] at 16 bits, from [`alpha_palette16`]. +pub(crate) fn decode_values16(block: &[u8]) -> [u16; 16] { + let byte = |i: usize| block.get(i).copied().unwrap_or(0); + let palette = alpha_palette16(byte(0), byte(1)); + let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]); + std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) +} + /// A BC1 block. Texels under half opacity become BC1's transparent black, which needs the /// three-colour mode; an opaque block takes whichever mode fits its colours better. pub(crate) fn encode_bc1(block: &Block, inside: Mask) -> [u8; 8] { @@ -369,6 +399,64 @@ fn value_fit(values: &[u8; 16], inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u32) ([a0, a1, b[0], b[1], b[2], b[3], b[4], b[5]], error) } +/// [`value_block`] for 16-bit values: the endpoints are still 8 bits, but each end of each mode +/// tries the 8-bit levels just below and above the 16-bit extreme, and every candidate is scored +/// against the 16-bit palette ([`alpha_palette16`]). A block that spans less than one 8-bit level +/// is flat from 8-bit values but gets the interpolated steps between two adjacent levels here. +pub(crate) fn value_block16(values: &[u16; 16], inside: Mask) -> [u8; 8] { + let mut all: Option<(u16, u16)> = None; + let mut mid = all; + let widen = |r: Option<(u16, u16)>, v: u16| Some(r.map_or((v, v), |(lo, hi)| (lo.min(v), hi.max(v)))); + for (t, &v) in values.iter().enumerate() { + if inside >> t & 1 != 0 { + all = widen(all, v); + if !matches!(v, 0 | 65535) { + mid = widen(mid, v); + } + } + } + let (lo, hi) = all.unwrap_or((0, 0)); + let (mid_lo, mid_hi) = mid.unwrap_or((0, 0)); + let (down, up) = (|v: u16| (v / 257) as u8, |v: u16| v.div_ceil(257) as u8); + // The eight-value mode's pairs first (a0 the high end), then the six-value mode's: on a tie + // the eight-value mode wins, as in `value_block`. Each pair's palette follows its order, so a + // pair that lands in the other mode is still scored as the decoder reads it. + let candidates = [ + (up(hi), down(lo)), + (up(hi), up(lo)), + (down(hi), down(lo)), + (down(hi), up(lo)), + (down(mid_lo), down(mid_hi)), + (down(mid_lo), up(mid_hi)), + (up(mid_lo), down(mid_hi)), + (up(mid_lo), up(mid_hi)), + ]; + let mut best: Option<([u8; 8], u64)> = None; + for (a0, a1) in candidates { + let fit = value_fit16(values, inside, a0, a1); + if best.is_none_or(|(_, e)| fit.1 < e) { + best = Some(fit); + } + } + best.map_or([0; 8], |(bytes, _)| bytes) +} + +fn value_fit16(values: &[u16; 16], inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u64) { + let palette = alpha_palette16(a0, a1); + let mut bits = 0u64; + let mut error = 0u64; + for (t, &v) in values.iter().enumerate() { + if inside >> t & 1 == 0 { + continue; + } + let (i, d) = nearest(&palette, |&p| u32::from(p.abs_diff(v)).pow(2)); + error += u64::from(d); + bits |= (i as u64) << (3 * t); + } + let b = bits.to_le_bytes(); + ([a0, a1, b[0], b[1], b[2], b[3], b[4], b[5]], error) +} + #[cfg(test)] mod tests { use super::*; @@ -484,6 +572,36 @@ mod tests { assert_eq!(nearest::(&[], |_| 7), (0, 0)); } + #[test] + fn value_blocks_decode_the_frozen_vectors_at_16_bits() { + // Eight-value mode (200 > 50): index 2 is (6 * 200 + 50) / 7 of 255 at 16 bits, which + // the 8-bit palette rounds to 179. + let bits: u64 = (0..16).fold(0, |b, t| b | 2 << (3 * t)); + let b = bits.to_le_bytes(); + let block = [200, 50, b[0], b[1], b[2], b[3], b[4], b[5]]; + assert_eq!(decode_values16(&block), [((6 * 200 + 50) * 257 + 3) / 7; 16]); + assert_eq!(decode_values(&block), [179; 16]); + // Six-value mode's indices 6 and 7 are 0 and 65535. + let bits: u64 = (0..16).fold(0, |b, t| b | (6 + (t % 2)) << (3 * t)); + let b = bits.to_le_bytes(); + let got = decode_values16(&[10, 250, b[0], b[1], b[2], b[3], b[4], b[5]]); + assert_eq!(got, std::array::from_fn(|t| if t % 2 == 0 { 0 } else { 65535 })); + } + + #[test] + fn value_blocks_from_16_bits_step_between_two_8_bit_levels() { + // A ramp across 0.8 of one 8-bit level: from 8 bits it is one or two flat values; from 16 + // bits the endpoints straddle it and the steps between them follow the ramp. + let values: [u16; 16] = std::array::from_fn(|t| 25_700 + (t as u16) * 13); + let error = |got: [u16; 16]| values.iter().zip(got).map(|(a, b)| u64::from(a.abs_diff(b)).pow(2)).sum::(); + let ours = error(decode_values16(&value_block16(&values, u16::MAX))); + let from_8_bits = error(decode_values16(&value_block(&values.map(|v| ((u32::from(v) + 128) / 257) as u8), u16::MAX))); + assert!(ours * 4 < from_8_bits, "{ours} vs {from_8_bits} from 8 bits"); + // Values outside the image don't count. + let solid = value_block16(&std::array::from_fn(|t| if t == 0 { 1000 } else { 60_000 }), 1); + assert!(decode_values16(&solid)[0].abs_diff(1000) < 129); + } + #[test] fn texels_outside_the_image_are_ignored() { // Only the top-left texel is inside: its colour alone decides the endpoints. diff --git a/crates/dds/src/lib.rs b/crates/dds/src/lib.rs index f7b44fc7a..336994285 100644 --- a/crates/dds/src/lib.rs +++ b/crates/dds/src/lib.rs @@ -2,7 +2,8 @@ //! "DDS_HEADER", "DDS_PIXELFORMAT", "DDS_HEADER_DXT10"). //! //! Reading: the top mip level of a 2D texture, either uncompressed (8 to 32 bits per pixel -//! described by channel masks, RGB or luminance) or block-compressed BC1/BC2/BC3 +//! described by channel masks, RGB or luminance, or 16 bits per channel: `R16_UNORM`, +//! `R16G16_UNORM`, `R16G16B16A16_UNORM`) or block-compressed BC1/BC2/BC3 //! (DXT1/DXT3/DXT5) or unsigned BC4/BC5 (ATI1/ATI2), with or without the DX10 header. BC4 opens //! as gray and BC5 as RGB with blue 0: BC5 stores only red and green, and a normal map's Z is //! left for the caller to reconstruct. Cubemaps, texture arrays and volume textures are refused @@ -14,7 +15,12 @@ //! legacy FourCCs are vendor names that programs read differently (ATI2's two blocks are swapped //! by some). BC6H, BC7 and sRGB formats aren't written. //! -//! Pixels are 8 bits per channel, interleaved, top row first, in one of the [`Layout`]s. The +//! Pixels are 8 bits per channel ([`encode`], [`Texture::decode`]) or 16 bits per channel +//! ([`encode16`], [`Texture::decode16`]), interleaved, top row first, in one of the [`Layout`]s. +//! Uncompressed 16-bit pixels are written with the DX10 header as `R16_UNORM` (gray), +//! `R16G16_UNORM` (gray and alpha, or any two channels such as a normal map's X and Y) or +//! `R16G16B16A16_UNORM` (RGB, with alpha 65535, and RGBA). BC4 and BC5 fit their 8-bit endpoints +//! and their indices to the 16-bit values; BC1 to BC3 round the pixels to 8 bits first. The //! crate has no dependencies; the `parallel` feature encodes and decodes rows of blocks (and mip //! levels) on several threads with rayon. //! @@ -38,7 +44,7 @@ use std::fmt; use bc::{Block, Mask}; -/// The channels of a pixel, 8 bits each, in this order. +/// The channels of a pixel, 8 or 16 bits each, in this order. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Layout { Gray, @@ -187,6 +193,8 @@ const DDSCAPS2_CUBEMAP_FACES: u32 = 0xfc00; const DDSCAPS2_VOLUME: u32 = 0x20_0000; const DIMENSION_TEXTURE2D: u32 = 3; const MISC_TEXTURECUBE: u32 = 0x4; +/// The legacy FourCC (a number, not text) of 16-bit RGBA. +const D3DFMT_A16B16G16R16: u32 = 36; fn bad(reason: impl fmt::Display) -> DdsError { DdsError::Malformed(reason.to_string()) @@ -242,6 +250,12 @@ impl Field { let max = u64::from(self.max); ((v * 255 + max / 2) / max) as u8 } + + fn get16(self, pixel: u32) -> u16 { + let v = u64::from((pixel >> self.shift) & self.max); + let max = u64::from(self.max); + ((v * 65535 + max / 2) / max) as u16 + } } #[derive(Clone, Copy)] @@ -255,6 +269,8 @@ enum Encoding { Bc1, Bc2, Bc3, + /// 16 bits per channel, little-endian: gray, gray and alpha (or red and green), or RGBA. + Unorm16 { channels: usize }, /// One value block: gray. Bc4, /// Two value blocks: red, then green. @@ -282,6 +298,8 @@ impl Encoding { Encoding::Masks { gray: true, fields, .. } if fields[3].is_some() => Layout::GrayA, Encoding::Masks { gray: true, .. } => Layout::Gray, Encoding::Masks { fields, .. } if fields[3].is_none() => Layout::Rgb, + Encoding::Unorm16 { channels: 1 } => Layout::Gray, + Encoding::Unorm16 { channels: 2 } => Layout::GrayA, Encoding::Bc4 => Layout::Gray, Encoding::Bc5 => Layout::Rgb, _ => Layout::Rgba, @@ -291,7 +309,7 @@ impl Encoding { /// Bytes per 4×4 block; 0 for uncompressed pixels. fn block_size(&self) -> usize { match self { - Encoding::Masks { .. } => 0, + Encoding::Masks { .. } | Encoding::Unorm16 { .. } => 0, Encoding::Bc1 | Encoding::Bc4 => 8, Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc5 => 16, } @@ -301,9 +319,14 @@ impl Encoding { fn level_layout(&self, width: u32, height: u32, pitch: Option) -> Result<(usize, usize), DdsError> { let overflow = || bad("pixel data size overflow"); let (w, h) = (usize::try_from(width).map_err(|_| overflow())?, usize::try_from(height).map_err(|_| overflow())?); - let (row_bytes, rows) = match self { - Encoding::Masks { bytes, .. } => { - let packed = w.checked_mul(*bytes).ok_or_else(overflow)?; + let pixel_bytes = match self { + Encoding::Masks { bytes, .. } => Some(*bytes), + Encoding::Unorm16 { channels } => Some(2 * channels), + _ => None, + }; + let (row_bytes, rows) = match pixel_bytes { + Some(bytes) => { + let packed = w.checked_mul(bytes).ok_or_else(overflow)?; let row_bytes = match pitch { Some(pitch) => usize::try_from(pitch).map_err(|_| overflow())?, None => packed, @@ -313,9 +336,7 @@ impl Encoding { } (row_bytes, h) } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc4 | Encoding::Bc5 => { - (w.div_ceil(4).checked_mul(self.block_size()).ok_or_else(overflow)?, h.div_ceil(4)) - } + None => (w.div_ceil(4).checked_mul(self.block_size()).ok_or_else(overflow)?, h.div_ceil(4)), }; let size = row_bytes.checked_mul(rows).ok_or_else(overflow)?; Ok((row_bytes, size)) @@ -327,6 +348,10 @@ fn legacy_encoding(b: &[u8]) -> Result { let (flags, fourcc, bits) = (le32(b, 80)?, le32(b, 84)?, le32(b, 88)?); let alpha = if flags & DDPF_ALPHAPIXELS != 0 { le32(b, 104)? } else { 0 }; if flags & DDPF_FOURCC != 0 { + // D3DFMT_A16B16G16R16, the legacy name of R16G16B16A16_UNORM (DirectXTex writes it so). + if fourcc == D3DFMT_A16B16G16R16 { + return Ok(Encoding::Unorm16 { channels: 4 }); + } return match &fourcc.to_le_bytes() { b"DXT1" => Ok(Encoding::Bc1), b"DXT3" => Ok(Encoding::Bc2), @@ -364,7 +389,7 @@ fn dx10_encoding(b: &[u8]) -> Result { } // miscFlags2 bits 0-2 are the alpha mode; 2 is premultiplied. Only the formats with a real // alpha channel are affected, and the legacy path refuses their DXT2/DXT4 equivalents too. - if le32(b, HEADER + 16)? & 0x7 == 2 && matches!(dxgi, 27..=29 | 73..=78 | 87 | 90 | 91) { + if le32(b, HEADER + 16)? & 0x7 == 2 && matches!(dxgi, 9 | 11 | 27..=29 | 73..=78 | 87 | 90 | 91) { return Err(unsupported(format!("premultiplied-alpha DX10 textures are not supported (DXGI format {dxgi})"))); } const RGBA: [u32; 4] = [0xff, 0xff00, 0xff_0000, 0xff00_0000]; @@ -372,6 +397,10 @@ fn dx10_encoding(b: &[u8]) -> Result { const BGRX: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0]; match dxgi { 27..=29 => Encoding::masks(32, RGBA, false), + // Typeless, then UNORM, as for BC4 and BC5 below. + 9 | 11 => Ok(Encoding::Unorm16 { channels: 4 }), + 33 | 35 => Ok(Encoding::Unorm16 { channels: 2 }), + 53 | 56 => Ok(Encoding::Unorm16 { channels: 1 }), 87 | 90 | 91 => Encoding::masks(32, BGRA, false), 88 | 92 | 93 => Encoding::masks(32, BGRX, false), 70..=72 => Ok(Encoding::Bc1), @@ -382,7 +411,7 @@ fn dx10_encoding(b: &[u8]) -> Result { 79 | 80 => Ok(Encoding::Bc4), 82 | 83 => Ok(Encoding::Bc5), 81 | 84 => Err(unsupported(format!("signed (SNORM) BC4 and BC5 textures are not supported (DXGI format {dxgi})"))), - _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1 to BC5 and 8-bit RGBA or BGRA are)"))), + _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1 to BC5, 8-bit RGBA or BGRA and 16-bit R, RG or RGBA are)"))), } } @@ -448,6 +477,65 @@ impl<'a> Texture<'a> { /// Decode with an explicit per-buffer output byte budget; allocation remains fallible. pub fn decode_with_limit(&self, max_bytes: u64) -> Result, DdsError> { + let encoding = self.encoding; + let (data, row_pitch, total) = self.top_level(1, max_bytes)?; + let (w, channels) = (self.width as usize, encoding.layout().channels()); + let row_size = w * channels; + let mut pixels = alloc_zeroed(total)?; + match encoding { + Encoding::Masks { bytes, gray, fields } => decode_masks(Masks { bytes, gray, fields }, data, row_pitch, &mut pixels, row_size), + // Each 16-bit sample rounded to the nearest 8-bit level. + Encoding::Unorm16 { .. } => rows(&mut pixels, row_size, |y, dst| { + let src = data.get(y * row_pitch..).unwrap_or_default(); + for (d, s) in dst.iter_mut().zip(src.as_chunks::<2>().0) { + *d = to8(u16::from_le_bytes(*s)); + } + }), + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc4 | Encoding::Bc5 => { + decode_blocks(encoding, data, (w, channels), &mut pixels) + } + } + Ok(pixels) + } + + /// The top mip level's pixels in [`Texture::layout`] at 16 bits per channel, bounded by + /// [`DEFAULT_MAX_DECODE_BYTES`]. 16-bit formats come back exactly, as do channel masks wider + /// than 8 bits; BC4 and BC5 values are interpolated at 16 bits; every other format is its + /// 8-bit value times 257 (so 255 is 65535). + pub fn decode16(&self) -> Result, DdsError> { + self.decode16_with_limit(DEFAULT_MAX_DECODE_BYTES) + } + + /// [`Texture::decode16`] with an explicit per-buffer output byte budget (two bytes a sample). + pub fn decode16_with_limit(&self, max_bytes: u64) -> Result, DdsError> { + let encoding = self.encoding; + let (data, row_pitch, total) = self.top_level(2, max_bytes)?; + let (w, channels) = (self.width as usize, encoding.layout().channels()); + let row_size = w * channels; + let mut pixels = alloc_zeroed(total / 2)?; + match encoding { + Encoding::Masks { bytes, gray, fields } => decode_masks16(Masks { bytes, gray, fields }, data, row_pitch, &mut pixels, row_size), + Encoding::Unorm16 { .. } => rows(&mut pixels, row_size, |y, dst| { + let src = data.get(y * row_pitch..).unwrap_or_default(); + for (d, s) in dst.iter_mut().zip(src.as_chunks::<2>().0) { + *d = u16::from_le_bytes(*s); + } + }), + Encoding::Bc4 | Encoding::Bc5 => decode_value_blocks16(encoding, data, (w, channels), &mut pixels), + Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { + // The limit was checked above for the 16-bit buffer, which is twice this one. + let narrow = self.decode_with_limit(u64::MAX)?; + for (d, v) in pixels.iter_mut().zip(narrow) { + *d = u16::from(v) * 257; + } + } + } + Ok(pixels) + } + + /// The top level's stored bytes and row stride, and the decoded size in bytes at + /// `sample_bytes` a sample, checked against `max_bytes` before the stored data is. + fn top_level(&self, sample_bytes: usize, max_bytes: u64) -> Result<(&'a [u8], usize, usize), DdsError> { let (b, flags, encoding) = (self.bytes, self.flags, self.encoding); let (width, height) = (self.width, self.height); // The same header field is a total level size for BC, but a row stride when DDSD_PITCH is set. @@ -455,23 +543,21 @@ impl<'a> Texture<'a> { let (row_pitch, size) = encoding.level_layout(width, height, pitch)?; let (w, h) = (width as usize, height as usize); let channels = encoding.layout().channels(); - let row_size = w.checked_mul(channels).ok_or_else(|| bad("pixel size overflow"))?; + let row_size = w.checked_mul(channels).and_then(|n| n.checked_mul(sample_bytes)).ok_or_else(|| bad("pixel size overflow"))?; let total = row_size.checked_mul(h).ok_or_else(|| bad("pixel size overflow"))?; if total as u64 > max_bytes { return Err(DdsError::LimitExceeded { required: total, limit: max_bytes }); } let data = b.get(self.start..).and_then(|d| d.get(..size)).ok_or_else(|| bad(format!("truncated pixel data: {size} bytes needed")))?; - let mut pixels = alloc_zeroed(total)?; - match encoding { - Encoding::Masks { bytes, gray, fields } => decode_masks(Masks { bytes, gray, fields }, data, row_pitch, &mut pixels, row_size), - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 | Encoding::Bc4 | Encoding::Bc5 => { - decode_blocks(encoding, data, (w, channels), &mut pixels) - } - } - Ok(pixels) + Ok((data, row_pitch, total)) } } +/// A 16-bit sample rounded to the nearest 8-bit level. +fn to8(v: u16) -> u8 { + ((u32::from(v) + 128) / 257) as u8 +} + /// An uncompressed pixel format: bytes per pixel, luminance or RGB, and the red (or /// luminance), green, blue and alpha fields. struct Masks { @@ -531,6 +617,51 @@ fn decode_masks(m: Masks, data: &[u8], row_pitch: usize, pixels: &mut [u8], row_ }); } +/// [`decode_masks`] at 16 bits: every field scaled to 0..=65535, so fields wider than 8 bits +/// (luminance or red and green of 16 bits) keep their precision. +fn decode_masks16(m: Masks, data: &[u8], row_pitch: usize, pixels: &mut [u16], row_size: usize) { + let Masks { bytes, gray, fields } = m; + let channels = if gray { 1 + usize::from(fields[3].is_some()) } else { 3 + usize::from(fields[3].is_some()) }; + rows(pixels, row_size, |y, dst_row| { + let src_row = data.get(y * row_pitch..).unwrap_or_default(); + for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { + let pixel = src.iter().rev().fold(0u32, |v, &byte| v << 8 | u32::from(byte)); + let channel = |i: usize| fields[i].map_or(0, |f| f.get16(pixel)); + let values = if gray { [channel(0), channel(3), 0, 0] } else { [channel(0), channel(1), channel(2), channel(3)] }; + dst.copy_from_slice(&values[..channels]); + } + }); +} + +/// BC4 or BC5 blocks into 16-bit `pixels` of `width` × `channels` samples a row: gray, or red and +/// green with blue 0, interpolated at 16 bits. +fn decode_value_blocks16(encoding: Encoding, data: &[u8], (width, channels): (usize, usize), pixels: &mut [u16]) { + let block_size = encoding.block_size().max(1); + let blocks_x = width.div_ceil(4); + let row_size = width * channels; + rows(pixels, row_size.saturating_mul(4), |by, band| { + let src = data.get(by * blocks_x * block_size..).unwrap_or_default(); + for (bx, block) in src.chunks_exact(block_size).take(blocks_x).enumerate() { + let red = bc::decode_values16(block); + let green = if matches!(encoding, Encoding::Bc5) { bc::decode_values16(block.get(8..).unwrap_or_default()) } else { [0; 16] }; + let x0 = bx * 4; + for (ty, line) in band.chunks_exact_mut(row_size).enumerate() { + let dst = line.get_mut(x0 * channels..).unwrap_or_default(); + for (tx, d) in dst.chunks_exact_mut(channels).take(4).enumerate() { + let t = ty * 4 + tx; + for (c, v) in d.iter_mut().enumerate() { + *v = match c { + 0 => red[t], + 1 => green[t], + _ => 0, + }; + } + } + } + } + }); +} + /// Block-compressed data into `pixels` of `width` × `channels` bytes a row, one row of blocks /// per task. Blocks wholly inside the image copy whole rows of texels; only those on the right /// edge check each texel. @@ -593,19 +724,47 @@ const DDSCAPS_TEXTURE: u32 = 0x1000; const DDSCAPS_MIPMAP: u32 = 0x40_0000; const DXGI_BC4_UNORM: u32 = 80; const DXGI_BC5_UNORM: u32 = 83; +const DXGI_R16G16B16A16_UNORM: u32 = 11; +const DXGI_R16G16_UNORM: u32 = 35; +const DXGI_R16_UNORM: u32 = 56; fn encode_err(reason: impl fmt::Display) -> DdsError { DdsError::Encode(reason.to_string()) } -/// A zeroed buffer of `len` bytes, or [`DdsError::Alloc`] when it can't be reserved. -fn alloc_zeroed(len: usize) -> Result, DdsError> { +/// A zeroed buffer of `len` samples, or [`DdsError::Alloc`] (in bytes) when it can't be reserved. +fn alloc_zeroed(len: usize) -> Result, DdsError> { let mut v = Vec::new(); - v.try_reserve_exact(len).map_err(|_| DdsError::Alloc(len))?; - v.resize(len, 0); + v.try_reserve_exact(len).map_err(|_| DdsError::Alloc(len.saturating_mul(size_of::())))?; + v.resize(len, T::default()); Ok(v) } +/// A sample of 8 or 16 bits: what the mip chain averages and the buffers hold. +trait Sample: Copy + Default + Send + Sync { + fn wide(self) -> u32; + /// An average already within the sample's range. + fn from_average(v: u32) -> Self; +} + +impl Sample for u8 { + fn wide(self) -> u32 { + u32::from(self) + } + fn from_average(v: u32) -> u8 { + v as u8 + } +} + +impl Sample for u16 { + fn wide(self) -> u32 { + u32::from(self) + } + fn from_average(v: u32) -> u16 { + v as u16 + } +} + /// How a written texture stores its pixels. #[derive(Clone, Copy, PartialEq, Eq)] enum Store { @@ -622,6 +781,14 @@ enum Store { Bc3, Bc4, Bc5, + /// 16-bit gray (`R16_UNORM`). + R16, + /// 16-bit gray and alpha, or any two channels (`R16G16_UNORM`). + Rg16, + /// 16-bit RGB with alpha 65535 (`R16G16B16A16_UNORM`). + Rgb16, + /// 16-bit RGBA (`R16G16B16A16_UNORM`). + Rgba16, } impl Store { @@ -642,12 +809,24 @@ impl Store { }) } + /// [`Store::new`] for 16-bit pixels: uncompressed ones keep 16 bits per channel. + fn new16(compression: DdsCompression, layout: Layout) -> Result { + Ok(match (compression, layout) { + (DdsCompression::None, Layout::Gray) => Store::R16, + (DdsCompression::None, Layout::GrayA) => Store::Rg16, + (DdsCompression::None, Layout::Rgb) => Store::Rgb16, + (DdsCompression::None, Layout::Rgba) => Store::Rgba16, + _ => Store::new(compression, layout)?, + }) + } + /// Bytes per pixel, or per 4×4 block for a block compression. fn unit(self) -> usize { match self { Store::Luminance => 1, - Store::LuminanceAlpha => 2, - Store::Bgrx | Store::Bgra => 4, + Store::LuminanceAlpha | Store::R16 => 2, + Store::Bgrx | Store::Bgra | Store::Rg16 => 4, + Store::Rgb16 | Store::Rgba16 => 8, Store::Bc1 | Store::Bc4 => 8, Store::Bc2 | Store::Bc3 | Store::Bc5 => 16, } @@ -662,6 +841,9 @@ impl Store { match self { Store::Bc4 => Some(DXGI_BC4_UNORM), Store::Bc5 => Some(DXGI_BC5_UNORM), + Store::R16 => Some(DXGI_R16_UNORM), + Store::Rg16 => Some(DXGI_R16G16_UNORM), + Store::Rgb16 | Store::Rgba16 => Some(DXGI_R16G16B16A16_UNORM), _ => None, } } @@ -686,7 +868,7 @@ impl Store { Store::Bc1 => (DDPF_FOURCC, *b"DXT1", 0, [0; 4]), Store::Bc2 => (DDPF_FOURCC, *b"DXT3", 0, [0; 4]), Store::Bc3 => (DDPF_FOURCC, *b"DXT5", 0, [0; 4]), - Store::Bc4 | Store::Bc5 => (DDPF_FOURCC, *b"DX10", 0, [0; 4]), + Store::Bc4 | Store::Bc5 | Store::R16 | Store::Rg16 | Store::Rgb16 | Store::Rgba16 => (DDPF_FOURCC, *b"DX10", 0, [0; 4]), } } } @@ -708,11 +890,40 @@ pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: D return Err(encode_err("texture dimensions must be nonzero")); } let store = Store::new(options.compression, layout)?; + encode_levels((width, height), layout, pixels, store, options.mipmaps, write_level) +} + +/// [`encode`] for 16-bit pixels. Uncompressed, they keep 16 bits per channel: gray as +/// `R16_UNORM`, gray and alpha (or any two channels, such as a normal map's X and Y) as +/// `R16G16_UNORM`, RGB (with alpha 65535) and RGBA as `R16G16B16A16_UNORM`, all with the DX10 +/// header. BC4 and BC5 pick their endpoints and indices against the 16-bit values; BC1 to BC3 +/// round the pixels to 8 bits and write what [`encode`] writes from them. +pub fn encode16(width: u32, height: u32, layout: Layout, pixels: &[u16], options: DdsOptions) -> Result, DdsError> { + if width == 0 || height == 0 { + return Err(encode_err("texture dimensions must be nonzero")); + } + let store = Store::new16(options.compression, layout)?; + if matches!(store, Store::Bc1 | Store::Bc2 | Store::Bc3) { + let mut narrow = alloc_zeroed(pixels.len())?; + for (d, &v) in narrow.iter_mut().zip(pixels) { + *d = to8(v); + } + return encode(width, height, layout, &narrow, options); + } + encode_levels((width, height), layout, pixels, store, options.mipmaps, write_level16) +} + +/// A level writer: one level of `width` × `height` pixels (in the image's layout) into `dst`. +type WriteLevel = fn(Store, &[T], (usize, usize), &mut [u8]) -> Result<(), DdsError>; + +/// The header and every level of `pixels`, each written by `write`. +fn encode_levels((width, height): (u32, u32), layout: Layout, pixels: &[T], store: Store, mipmaps: bool, write: WriteLevel) -> Result, DdsError> { let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(layout.channels())); if expected != Some(pixels.len()) { - return Err(encode_err(format!("{} bytes of pixels for a {width}x{height} {layout:?} texture", pixels.len()))); + let unit = if size_of::() == 1 { "bytes" } else { "16-bit samples" }; + return Err(encode_err(format!("{} {unit} of pixels for a {width}x{height} {layout:?} texture", pixels.len()))); } - let levels = if options.mipmaps { mip_levels(width, height) } else { 1 }; + let levels = if mipmaps { mip_levels(width, height) } else { 1 }; let top = (width as usize, height as usize); // Every level's size, checked before anything is allocated. let mut sizes = Vec::new(); @@ -724,7 +935,7 @@ pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: D let head = if store.dxgi().is_some() { HEADER + DX10_HEADER } else { HEADER }; let total = sizes.iter().try_fold(head, |total, &s| total.checked_add(s)).ok_or_else(|| encode_err("texture size overflow"))?; let header = header(width, height, store, levels, sizes.first().copied().unwrap_or(0))?; - let mut out = alloc_zeroed(total)?; + let mut out = alloc_zeroed::(total)?; let (Some(dst), Some(src)) = (out.get_mut(..head), header.get(..head)) else { return Err(encode_err("output buffer too small")); }; @@ -735,7 +946,7 @@ pub fn encode(width: u32, height: u32, layout: Layout, pixels: &[u8], options: D let (mut size, mut at) = (top, head); for (i, &bytes) in sizes.iter().enumerate() { let dst = out.get_mut(at..at + bytes).ok_or_else(|| encode_err("output buffer too small"))?; - write_level(store, &level, size, dst)?; + write(store, &level, size, dst)?; at += bytes; if i + 1 < sizes.len() { let next = half(size); @@ -777,7 +988,8 @@ fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Res } put(108, DDSCAPS_TEXTURE | if mipmapped { DDSCAPS_COMPLEX | DDSCAPS_MIPMAP } else { 0 }); if let Some(dxgi) = store.dxgi() { - // One 2D texture, no misc flags, alpha mode unknown (the formats have no alpha). + // One 2D texture, no misc flags, alpha mode unknown (straight for the one format with + // alpha, RGBA16, as readers take it). put(HEADER, dxgi); put(HEADER + 4, DIMENSION_TEXTURE2D); put(HEADER + 12, 1); @@ -788,7 +1000,7 @@ fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Res /// Runs `f` on each `len`-byte row of `out` with its index, in parallel with the `parallel` /// feature. -fn rows(out: &mut [u8], len: usize, f: impl Fn(usize, &mut [u8]) + Send + Sync) { +fn rows(out: &mut [T], len: usize, f: impl Fn(usize, &mut [T]) + Send + Sync) { #[cfg(feature = "parallel")] { use rayon::prelude::*; @@ -804,6 +1016,8 @@ fn rows(out: &mut [u8], len: usize, f: impl Fn(usize, &mut [u8]) + Send + Sync) fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst: &mut [u8]) -> Result<(), DdsError> { let row_bytes = store.row_bytes(width).ok_or_else(|| encode_err("texture size overflow"))?; match store { + // 16-bit stores are written by `write_level16`. + Store::R16 | Store::Rg16 | Store::Rgb16 | Store::Rgba16 => return Err(encode_err("16-bit storage needs 16-bit pixels")), Store::Luminance | Store::LuminanceAlpha => { // The image's gray (and alpha) samples, in the same order. let src = pixels.get(..dst.len()).ok_or_else(|| encode_err("the level is smaller than its size"))?; @@ -851,10 +1065,44 @@ fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst Ok(()) } +/// [`write_level`] for 16-bit pixels: little-endian samples, or BC4/BC5 blocks fitted to them. +fn write_level16(store: Store, pixels: &[u16], (width, height): (usize, usize), dst: &mut [u8]) -> Result<(), DdsError> { + let row_bytes = store.row_bytes(width).ok_or_else(|| encode_err("texture size overflow"))?; + match store { + Store::R16 | Store::Rg16 | Store::Rgba16 => { + // The image's samples, in the same order. + for (d, s) in dst.as_chunks_mut::<2>().0.iter_mut().zip(pixels) { + *d = s.to_le_bytes(); + } + } + Store::Rgb16 => rows(dst, row_bytes, |y, row| { + let src = pixels.get(y * width * 3..).unwrap_or_default(); + for (d, s) in row.as_chunks_mut::<8>().0.iter_mut().zip(src.as_chunks::<3>().0) { + let [r, g, b, a] = [s[0], s[1], s[2], u16::MAX].map(u16::to_le_bytes); + *d = [r[0], r[1], g[0], g[1], b[0], b[1], a[0], a[1]]; + } + }), + Store::Bc4 | Store::Bc5 => { + let channels = if store == Store::Bc4 { 1 } else { 3 }; + rows(dst, row_bytes, |by, row| { + for (bx, d) in row.chunks_exact_mut(store.unit()).enumerate() { + for (c, d) in d.as_chunks_mut::<8>().0.iter_mut().enumerate() { + let (values, inside) = gather_channel(pixels, (width, height), channels, c, (bx, by)); + *d = bc::value_block16(&values, inside); + } + } + }); + } + // `encode16` rounds BC1 to BC3 to 8 bits; the 8-bit stores take 8-bit pixels. + _ => return Err(encode_err("8-bit storage needs 8-bit pixels")), + } + Ok(()) +} + /// Channel `c` of block (`bx`, `by`) of pixels with `channels` channels, and which of its texels /// lie inside the image. -fn gather_channel(pixels: &[u8], (width, height): (usize, usize), channels: usize, c: usize, (bx, by): (usize, usize)) -> ([u8; 16], Mask) { - let mut values = [0u8; 16]; +fn gather_channel(pixels: &[T], (width, height): (usize, usize), channels: usize, c: usize, (bx, by): (usize, usize)) -> ([T; 16], Mask) { + let mut values = [T::default(); 16]; let mut inside: Mask = 0; for (t, v) in values.iter_mut().enumerate() { let (x, y) = (bx * 4 + t % 4, by * 4 + t / 4); @@ -902,7 +1150,7 @@ fn gather(texels: &[[u8; 4]], (width, height): (usize, usize), bx: usize, by: us /// The next mip level: each pixel the average of the 2×2 pixels it covers, or of 2×3, 3×2 or /// 3×3 along the last column and row of an odd-sized level, so no pixel is dropped. -fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), channels: usize) -> Result, DdsError> { +fn downsample(src: &[T], (w, h): (usize, usize), (nw, nh): (usize, usize), channels: usize) -> Result, DdsError> { let len = nw.checked_mul(nh).and_then(|n| n.checked_mul(channels)).ok_or_else(|| encode_err("mip level size overflow"))?; let mut dst = alloc_zeroed(len)?; let span = |i: usize, n: usize, size: usize| 2 * i..=if i + 1 == n { size.saturating_sub(1) } else { 2 * i + 1 }; @@ -919,8 +1167,8 @@ fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), chan let interior = nw.saturating_sub(1); for ((px, a), b) in row.chunks_exact_mut(ch).zip(r0.chunks_exact(2 * ch)).zip(r1.chunks_exact(2 * ch)).take(interior) { for (c, v) in px.iter_mut().enumerate() { - let sum = u32::from(a[c]) + u32::from(a[ch + c]) + u32::from(b[c]) + u32::from(b[ch + c]); - *v = ((sum + 2) / 4) as u8; + let sum = a[c].wide() + a[ch + c].wide() + b[c].wide() + b[ch + c].wide(); + *v = T::from_average((sum + 2) / 4); } } edge = interior; @@ -932,12 +1180,12 @@ fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), chan for sy in ys.clone() { for sx in xs.clone() { if let Some(&s) = src.get((sy * w + sx) * channels + c) { - sum += u32::from(s); + sum += s.wide(); n += 1; } } } - *v = ((sum + n / 2) / n.max(1)) as u8; + *v = T::from_average((sum + n / 2) / n.max(1)); } } }); @@ -980,4 +1228,127 @@ mod tests { assert!(matches!(texture.decode(), Err(DdsError::Malformed(_)))); assert_eq!(Texture::parse(&file).and_then(|t| t.decode()), Ok(vec![7; 64])); } + + /// Samples spread over the whole 16-bit range, `channels` a pixel. + fn spread16(width: usize, height: usize, channels: usize) -> Vec { + (0..width * height * channels).map(|i| (i as u32 * 7919 % 65536) as u16).collect() + } + + #[test] + fn uncompressed_16_bit_pixels_round_trip_exactly() { + let (w, h) = (7, 5); + for (layout, dxgi, read_as, pixel_bytes) in [(Layout::Gray, 56, Layout::Gray, 2), (Layout::GrayA, 35, Layout::GrayA, 4), (Layout::Rgb, 11, Layout::Rgba, 8), (Layout::Rgba, 11, Layout::Rgba, 8)] { + let pixels = spread16(w, h, layout.channels()); + let want: Vec = if layout == Layout::Rgb { pixels.chunks(3).flat_map(|p| [p[0], p[1], p[2], 65535]).collect() } else { pixels.clone() }; + for mipmaps in [false, true] { + let file = encode16(w as u32, h as u32, layout, &pixels, DdsOptions { compression: DdsCompression::None, mipmaps }).unwrap(); + assert_eq!(le32(&file, HEADER).unwrap(), dxgi, "{layout:?}"); + // 7×5, then 3×2 and 1×1. + let samples = if mipmaps { 7 * 5 + 3 * 2 + 1 } else { 7 * 5 }; + assert_eq!(file.len(), HEADER + DX10_HEADER + samples * pixel_bytes); + let texture = Texture::parse(&file).unwrap(); + assert_eq!(texture.layout(), read_as); + assert_eq!(texture.decode16().unwrap(), want); + assert_eq!(texture.decode().unwrap(), want.iter().map(|&v| to8(v)).collect::>()); + if layout == Layout::Rgba { + let mut premultiplied = file.clone(); + premultiplied[HEADER + 16] = 2; + assert!(matches!(Texture::parse(&premultiplied), Err(DdsError::Unsupported(_)))); + } + } + } + } + + #[test] + fn mip_levels_of_16_bit_pixels_average_at_16_bits() { + // 1000 to 1003 average to 1002 (rounded), which 8 bits can't hold. + let file = encode16(2, 2, Layout::Gray, &[1000, 1001, 1002, 1003], DdsOptions { compression: DdsCompression::None, mipmaps: true }).unwrap(); + assert_eq!(le32(&file, 28).unwrap(), 2); + assert_eq!(file[file.len() - 2..], 1002u16.to_le_bytes()); + } + + #[test] + fn legacy_16_bit_textures_decode_at_16_bits() { + // D3DFMT_A16B16G16R16: the legacy header's numeric FourCC 36, the same pixels as the + // DX10 file. + let pixels = spread16(3, 2, 4); + let file = encode16(3, 2, Layout::Rgba, &pixels, DdsOptions::default()).unwrap(); + let mut legacy = file[..HEADER].to_vec(); + legacy[84..88].copy_from_slice(&36u32.to_le_bytes()); + legacy.extend_from_slice(&file[HEADER + DX10_HEADER..]); + let texture = Texture::parse(&legacy).unwrap(); + assert_eq!((texture.layout(), texture.decode16().unwrap()), (Layout::Rgba, pixels)); + // L16: a 16-bit luminance mask keeps its 16 bits through `decode16`. + let mut l16 = encode(2, 1, Layout::Gray, &[0, 0], DdsOptions::default()).unwrap(); + l16[20..24].copy_from_slice(&4u32.to_le_bytes()); + l16[88..92].copy_from_slice(&16u32.to_le_bytes()); + l16[92..96].copy_from_slice(&0xffffu32.to_le_bytes()); + l16.truncate(HEADER); + l16.extend([300u16, 65000].iter().flat_map(|v| v.to_le_bytes())); + let texture = Texture::parse(&l16).unwrap(); + assert_eq!(texture.decode16().unwrap(), [300, 65000]); + assert_eq!(texture.decode().unwrap(), [1, 253]); + } + + #[test] + fn bc4_and_bc5_from_16_bits_beat_the_8_bit_rounded_source() { + // Smooth ramps, about half an 8-bit level across a block, as in height and normal maps. + // 62×61 leaves partial blocks on the right and bottom edges. + let (w, h) = (62usize, 61usize); + let at = |i: usize| (i % w, i / w); + let height: Vec = (0..w * h).map(|i| (20_000 + at(i).0 * 29 + at(i).1 * 11) as u16).collect(); + let normal: Vec = (0..w * h).flat_map(|i| [(10_000 + at(i).0 * 23 + at(i).1 * 7) as u16, (40_000 + at(i).1 * 27 + at(i).0 * 5) as u16, 65535]).collect(); + for (compression, layout, pixels) in [(DdsCompression::Bc4, Layout::Gray, &height), (DdsCompression::Bc5, Layout::Rgb, &normal)] { + let options = DdsOptions { compression, mipmaps: true }; + let rounded: Vec = pixels.iter().map(|&v| to8(v)).collect(); + let from16 = Texture::parse(&encode16(w as u32, h as u32, layout, pixels, options).unwrap()).unwrap(); + let from8 = Texture::parse(&encode(w as u32, h as u32, layout, &rounded, options).unwrap()).unwrap(); + let channels = layout.channels(); + // Root-mean-square error of the stored channels (BC5: red and green), in 16-bit units. + let rms = |got: &[u16]| { + let stored = pixels.iter().zip(got).enumerate().filter(|(i, _)| i % channels < 2); + let (sum, n) = stored.fold((0f64, 0f64), |(sum, n), (_, (&a, &b))| (sum + (f64::from(a) - f64::from(b)).powi(2), n + 1.0)); + (sum / n).sqrt() + }; + let (ours, theirs) = (rms(&from16.decode16().unwrap()), rms(&from8.decode16().unwrap())); + eprintln!("{compression:?}: RMS error {ours:.1} from 16 bits, {theirs:.1} from the 8-bit rounded source (16-bit units)"); + assert!(ours * 2.0 < theirs, "{compression:?}: {ours} vs {theirs}"); + // `decode` and `decode16` read the same blocks: 8 bits are the 16-bit values rounded, + // give or take the 8-bit palette's own rounding. + for texture in [&from16, &from8] { + let (narrow, wide) = (texture.decode().unwrap(), texture.decode16().unwrap()); + assert!(narrow.iter().zip(&wide).all(|(&a, &b)| a.abs_diff(to8(b)) <= 1)); + } + } + } + + #[test] + fn bc1_to_bc3_from_16_bits_write_what_the_rounded_pixels_write() { + let pixels = spread16(6, 5, 4); + let rounded: Vec = pixels.iter().map(|&v| to8(v)).collect(); + for compression in [DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3] { + for mipmaps in [false, true] { + let options = DdsOptions { compression, mipmaps }; + assert_eq!(encode16(6, 5, Layout::Rgba, &pixels, options), encode(6, 5, Layout::Rgba, &rounded, options)); + } + } + // `encode16` takes the layouts `encode` takes, and checks the buffer size the same way. + for compression in DdsCompression::ALL { + for layout in [Layout::Gray, Layout::GrayA, Layout::Rgb, Layout::Rgba] { + let options = DdsOptions { compression, mipmaps: false }; + let n = layout.channels(); + assert_eq!(encode16(1, 1, layout, &[9; 4][..n], options).is_ok(), encode(1, 1, layout, &[9; 4][..n], options).is_ok(), "{compression:?} from {layout:?}"); + } + } + assert!(matches!(encode16(2, 2, Layout::Rgb, &[0; 11], DdsOptions::default()), Err(DdsError::Encode(_)))); + assert!(matches!(encode16(0, 2, Layout::Rgb, &[], DdsOptions::default()), Err(DdsError::Encode(_)))); + } + + #[test] + fn decode16_checks_its_budget_in_bytes() { + let file = encode16(4, 4, Layout::Gray, &[7; 16], DdsOptions::default()).unwrap(); + let texture = Texture::parse(&file).unwrap(); + assert!(matches!(texture.decode16_with_limit(31), Err(DdsError::LimitExceeded { required: 32, limit: 31 }))); + assert_eq!(texture.decode16_with_limit(32).unwrap(), [7; 16]); + } } From 5ef5134ccc28db3a3aaba1e1a0c189340c5d277c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 15:55:15 -0500 Subject: [PATCH 095/209] image-codecs: keep stored WebP dimensions independent of EXIF compatibility (#1921) --- crates/image-codecs/src/webp_probe.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/src/webp_probe.rs b/crates/image-codecs/src/webp_probe.rs index 07d6e0269..4a31def00 100644 --- a/crates/image-codecs/src/webp_probe.rs +++ b/crates/image-codecs/src/webp_probe.rs @@ -25,9 +25,10 @@ pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> { if !has_image { return Err(err("no image data")); } - let (w, h) = crate::probe_size(bytes)?; + // Read stored dimensions directly. Undoing probe_size with the compatibility EXIF + // orientation is incorrect when its lenient parser disagrees with the strict one. + let (w, h, _) = crate::codecs::webp::probe(bytes, &crate::Limits::default())?; let orientation = exif.map_or(1, craft_metadata::compat_orientation::orientation); - let (w, h) = if (5..=8).contains(&orientation) { (h, w) } else { (w, h) }; Ok((w, h, orientation)) } @@ -47,6 +48,20 @@ mod tests { assert_eq!(header(&bytes).unwrap(), (2, 3, 6)); } + #[test] + fn malformed_exif_does_not_swap_stored_dimensions() { + let mut image = crate::Image::from_u8(2, 3, crate::ChannelLayout::Rgb, vec![42; 18]).unwrap(); + image.meta.exif = Some(b"II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00".to_vec()); + let mut bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap(); + let exif = bytes.windows(4).position(|v| v == b"EXIF").unwrap() + 8; + bytes[exif + 18] = 6; + // Compatibility metadata preserves orientation despite a malformed count; + // the strict orientation used by probe_size rejects it. Stored size is neither. + bytes[exif + 14..exif + 18].copy_from_slice(&0u32.to_le_bytes()); + assert_eq!(crate::probe_size(&bytes).unwrap(), (2, 3)); + assert_eq!(header(&bytes).unwrap(), (2, 3, 6)); + } + #[test] fn rejects_missing_and_truncated_image_chunks() { let mut bytes = b"RIFF\x00\x00\x00\x00WEBP".to_vec(); From e4d27fc1cd030be8c1d163220d07f9a00ee0fb8d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 21:58:20 +0100 Subject: [PATCH 096/209] fcproj-format: schema v18 for camera raw Source Settings A no-op v17 -> v18 step so FilmCraft can save per-clip camera raw Source Settings (Interpretation::raw) and older builds refuse such projects instead of dropping the settings on save. Library part of storytold/filmcraft#1078 (Part of storytold/filmcraft#1065); the MXF CinemaDNG part already landed in #1145. --- crates/fcproj-format/src/lib.rs | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/fcproj-format/src/lib.rs b/crates/fcproj-format/src/lib.rs index 98e50cf59..4ac1b7243 100644 --- a/crates/fcproj-format/src/lib.rs +++ b/crates/fcproj-format/src/lib.rs @@ -13,7 +13,7 @@ pub type Migration = fn(Value) -> Result; /// edit a shipped one. pub const MIGRATIONS: &[Migration] = &[ v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13, v13_to_v14, - v14_to_v15, v15_to_v16, v16_to_v17, + v14_to_v15, v15_to_v16, v16_to_v17, v17_to_v18, ]; /// The schema version this build writes (and the newest it reads). @@ -252,6 +252,14 @@ fn v16_to_v17(doc: Value) -> Result { Ok(doc) } +/// v17 → v18: camera raw Source Settings on media items (`Interpretation::raw`: white balance, +/// exposure, highlights, demosaic quality and output colour space of DNG / CinemaDNG footage; +/// absent = as the file says). Existing data needs no change; older builds must refuse such +/// projects instead of silently dropping the settings when saving. +fn v17_to_v18(doc: Value) -> Result { + Ok(doc) +} + #[cfg(test)] mod tests { use super::*; @@ -262,10 +270,10 @@ mod tests { let bare = json!({"version": 2, "items": {}, "root": {}, "extra": [1, 2]}); assert_eq!(detect_version(&bare).unwrap(), 1); let migrated = migrate_with(MIGRATIONS, bare, 1).unwrap(); - assert_eq!(migrated["schema_version"], 17); + assert_eq!(migrated["schema_version"], 18); assert_eq!(migrated["project"]["extra"], json!([1, 2])); assert_eq!(migrated["generator"], "FilmCraft (schema v1)"); - assert_eq!(detect_bytes(serde_json::to_string(&migrated).unwrap().as_bytes()).unwrap(), 17); + assert_eq!(detect_bytes(serde_json::to_string(&migrated).unwrap().as_bytes()).unwrap(), 18); let parsed: EnvelopeIn = serde_json::from_value(migrated).unwrap(); assert!(parsed.view.is_none()); assert_eq!(parsed.project["extra"], json!([1, 2])); @@ -286,10 +294,27 @@ mod tests { assert_eq!(version, 1); assert_eq!(file.project["name"], "Cut"); } + /// Schema 18 (camera raw Source Settings): a v17 project upgrades with its data untouched, and a + /// v18 project with `interpret.raw` settings is read rather than refused as too new. + #[test] + fn v17_upgrades_and_v18_raw_settings_are_read() { + let item = json!({"Media": {"interpret": {"par": [1, 1]}}}); + let v17 = json!({"format": FORMAT_ID, "schema_version": 17, "project": {"items": {"1": {"kind": item}}, "root": {}}}); + let up = migrate_with(MIGRATIONS, v17.clone(), 17).unwrap(); + assert_eq!(up["schema_version"], 18); + assert_eq!(up["project"], v17["project"]); + let raw = json!({"white_balance": {"Custom": {"temperature": 3200.0, "tint": 12.5}}, "exposure": -0.75}); + let mut v18 = up; + v18["project"]["items"]["1"]["kind"]["Media"]["interpret"]["raw"] = raw.clone(); + let bytes = serde_json::to_vec(&v18).unwrap(); + let (file, version): (EnvelopeIn, _) = decode(&bytes).unwrap(); + assert_eq!(version, 18); + assert_eq!(file.project["items"]["1"]["kind"]["Media"]["interpret"]["raw"], raw); + } #[test] fn rejects_wrong_future_and_damaged_envelopes() { assert!(matches!(detect_bytes(br#"{"format":"other","schema_version":1}"#), Err(FormatError::NotAProject(_)))); - assert!(matches!(detect_bytes(br#"{"schema_version":18}"#), Err(FormatError::TooNew { found: 18, supported: 17 }))); + assert!(matches!(detect_bytes(br#"{"schema_version":19}"#), Err(FormatError::TooNew { found: 19, supported: 18 }))); assert!(matches!(detect_bytes(br#"{"schema_version":0}"#), Err(FormatError::TooOld { found: 0, oldest: 1 }))); assert!(matches!(detect_bytes(b"{"), Err(FormatError::Corrupt(_)))); assert!(matches!(migrate_with(MIGRATIONS, json!([]), 1), Err(FormatError::Migration { from: 1, .. }))); From 853f630d6d54eccc90a6a018fc03fabe698880fc Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 21:59:19 +0100 Subject: [PATCH 097/209] dds: review fixes for 16-bit support Reword decode16 and DdsCompression::None docs, check the 16-bit buffer length before narrowing for BC1-BC3, and test typeless DXGI codes, padded 16-bit rows, G16R16 masks and decode16 of BC1. --- crates/dds/README.md | 4 +- crates/dds/src/bc.rs | 2 +- crates/dds/src/lib.rs | 105 +++++++++++++++++++++++++++++++++++------- 3 files changed, 92 insertions(+), 19 deletions(-) diff --git a/crates/dds/README.md b/crates/dds/README.md index 7a25eb36e..2dae26bc1 100644 --- a/crates/dds/README.md +++ b/crates/dds/README.md @@ -35,7 +35,9 @@ and alpha, and RGBA. BC4 and BC5 keep 8-bit endpoints, as the formats store them and the indices against the 16-bit values, so a smooth height or normal map has less error than when its 8-bit rounding is compressed. BC1 to BC3 round the pixels to 8 bits first. `Texture::decode` rounds 16-bit formats to 8 bits; `Texture::decode16` returns them exactly, -interpolates BC4 and BC5 at 16 bits, and scales every 8-bit format by 257. +rescales channel-mask fields from their own width (a 16-bit luminance field exactly), +interpolates BC4 and BC5 at 16 bits, and scales BC1 to BC3 by 257. `craft-image-codecs` still +reads DDS at 8 bits. ```rust use craft_dds::{DdsCompression, DdsOptions, Layout, Texture}; diff --git a/crates/dds/src/bc.rs b/crates/dds/src/bc.rs index c19a393f3..7000a09e4 100644 --- a/crates/dds/src/bc.rs +++ b/crates/dds/src/bc.rs @@ -579,7 +579,7 @@ mod tests { let bits: u64 = (0..16).fold(0, |b, t| b | 2 << (3 * t)); let b = bits.to_le_bytes(); let block = [200, 50, b[0], b[1], b[2], b[3], b[4], b[5]]; - assert_eq!(decode_values16(&block), [((6 * 200 + 50) * 257 + 3) / 7; 16]); + assert_eq!(decode_values16(&block), [(((6 * 200 + 50) * 257 + 3) / 7_u32) as u16; 16]); assert_eq!(decode_values(&block), [179; 16]); // Six-value mode's indices 6 and 7 are 0 and 65535. let bits: u64 = (0..16).fold(0, |b, t| b | (6 + (t % 2)) << (3 * t)); diff --git a/crates/dds/src/lib.rs b/crates/dds/src/lib.rs index 336994285..183ec4fa1 100644 --- a/crates/dds/src/lib.rs +++ b/crates/dds/src/lib.rs @@ -11,7 +11,7 @@ //! //! Writing: a 2D texture, uncompressed (BGRA or BGRX, luminance with or without alpha), //! BC1/BC2/BC3 or BC4/BC5, optionally with its full mip chain. The legacy header describes all -//! but BC4 and BC5, which are written with the DX10 header (`BC4_UNORM`, `BC5_UNORM`): their +//! but BC4, BC5 and the 16-bit formats, which are written with the DX10 header (`BC4_UNORM`, `BC5_UNORM`): their //! legacy FourCCs are vendor names that programs read differently (ATI2's two blocks are swapped //! by some). BC6H, BC7 and sRGB formats aren't written. //! @@ -105,7 +105,7 @@ impl std::error::Error for DdsError {} #[non_exhaustive] pub enum DdsCompression { /// Uncompressed, 8 bits per channel: BGRA (BGRX without alpha), or luminance with or - /// without alpha for grayscale. Lossless. + /// without alpha for grayscale. From [`encode16`], 16 bits per channel instead. Lossless. #[default] None, /// BC1 (DXT1): 4 bits per pixel. Alpha is on or off: texels under half opacity become @@ -270,7 +270,9 @@ enum Encoding { Bc2, Bc3, /// 16 bits per channel, little-endian: gray, gray and alpha (or red and green), or RGBA. - Unorm16 { channels: usize }, + Unorm16 { + channels: usize, + }, /// One value block: gray. Bc4, /// Two value blocks: red, then green. @@ -499,9 +501,10 @@ impl<'a> Texture<'a> { } /// The top mip level's pixels in [`Texture::layout`] at 16 bits per channel, bounded by - /// [`DEFAULT_MAX_DECODE_BYTES`]. 16-bit formats come back exactly, as do channel masks wider - /// than 8 bits; BC4 and BC5 values are interpolated at 16 bits; every other format is its - /// 8-bit value times 257 (so 255 is 65535). + /// [`DEFAULT_MAX_DECODE_BYTES`]. 16-bit formats come back exactly; channel-mask fields are + /// rescaled from their own width to 0..=65535 (a 16-bit luminance field exactly, an 8-bit one + /// times 257); BC4 and BC5 values are interpolated at 16 bits; BC1 to BC3 are their 8-bit + /// values times 257 (so 255 is 65535). pub fn decode16(&self) -> Result, DdsError> { self.decode16_with_limit(DEFAULT_MAX_DECODE_BYTES) } @@ -903,6 +906,7 @@ pub fn encode16(width: u32, height: u32, layout: Layout, pixels: &[u16], options return Err(encode_err("texture dimensions must be nonzero")); } let store = Store::new16(options.compression, layout)?; + check_len((width, height), layout, pixels)?; if matches!(store, Store::Bc1 | Store::Bc2 | Store::Bc3) { let mut narrow = alloc_zeroed(pixels.len())?; for (d, &v) in narrow.iter_mut().zip(pixels) { @@ -913,16 +917,29 @@ pub fn encode16(width: u32, height: u32, layout: Layout, pixels: &[u16], options encode_levels((width, height), layout, pixels, store, options.mipmaps, write_level16) } -/// A level writer: one level of `width` × `height` pixels (in the image's layout) into `dst`. -type WriteLevel = fn(Store, &[T], (usize, usize), &mut [u8]) -> Result<(), DdsError>; - -/// The header and every level of `pixels`, each written by `write`. -fn encode_levels((width, height): (u32, u32), layout: Layout, pixels: &[T], store: Store, mipmaps: bool, write: WriteLevel) -> Result, DdsError> { +/// Refuses a pixel buffer that isn't `width` × `height` pixels of `layout`. +fn check_len((width, height): (u32, u32), layout: Layout, pixels: &[T]) -> Result<(), DdsError> { let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(layout.channels())); if expected != Some(pixels.len()) { let unit = if size_of::() == 1 { "bytes" } else { "16-bit samples" }; return Err(encode_err(format!("{} {unit} of pixels for a {width}x{height} {layout:?} texture", pixels.len()))); } + Ok(()) +} + +/// A level writer: one level of `width` × `height` pixels (in the image's layout) into `dst`. +type WriteLevel = fn(Store, &[T], (usize, usize), &mut [u8]) -> Result<(), DdsError>; + +/// The header and every level of `pixels`, each written by `write`. +fn encode_levels( + (width, height): (u32, u32), + layout: Layout, + pixels: &[T], + store: Store, + mipmaps: bool, + write: WriteLevel, +) -> Result, DdsError> { + check_len((width, height), layout, pixels)?; let levels = if mipmaps { mip_levels(width, height) } else { 1 }; let top = (width as usize, height as usize); // Every level's size, checked before anything is allocated. @@ -1237,9 +1254,15 @@ mod tests { #[test] fn uncompressed_16_bit_pixels_round_trip_exactly() { let (w, h) = (7, 5); - for (layout, dxgi, read_as, pixel_bytes) in [(Layout::Gray, 56, Layout::Gray, 2), (Layout::GrayA, 35, Layout::GrayA, 4), (Layout::Rgb, 11, Layout::Rgba, 8), (Layout::Rgba, 11, Layout::Rgba, 8)] { + for (layout, dxgi, read_as, pixel_bytes) in [ + (Layout::Gray, 56, Layout::Gray, 2), + (Layout::GrayA, 35, Layout::GrayA, 4), + (Layout::Rgb, 11, Layout::Rgba, 8), + (Layout::Rgba, 11, Layout::Rgba, 8), + ] { let pixels = spread16(w, h, layout.channels()); - let want: Vec = if layout == Layout::Rgb { pixels.chunks(3).flat_map(|p| [p[0], p[1], p[2], 65535]).collect() } else { pixels.clone() }; + let want: Vec = + if layout == Layout::Rgb { pixels.chunks(3).flat_map(|p| [p[0], p[1], p[2], 65535]).collect() } else { pixels.clone() }; for mipmaps in [false, true] { let file = encode16(w as u32, h as u32, layout, &pixels, DdsOptions { compression: DdsCompression::None, mipmaps }).unwrap(); assert_eq!(le32(&file, HEADER).unwrap(), dxgi, "{layout:?}"); @@ -1288,6 +1311,40 @@ mod tests { let texture = Texture::parse(&l16).unwrap(); assert_eq!(texture.decode16().unwrap(), [300, 65000]); assert_eq!(texture.decode().unwrap(), [1, 253]); + // G16R16 as masks (red low, green high) reads as RGB at 16 bits, blue 0. + let mut g16r16 = l16[..HEADER].to_vec(); + g16r16[80..84].copy_from_slice(&DDPF_RGB.to_le_bytes()); + g16r16[20..24].copy_from_slice(&8u32.to_le_bytes()); + g16r16[88..92].copy_from_slice(&32u32.to_le_bytes()); + g16r16[96..100].copy_from_slice(&0xffff_0000u32.to_le_bytes()); + g16r16.extend([300u16, 40000, 65535, 7].iter().flat_map(|v| v.to_le_bytes())); + let texture = Texture::parse(&g16r16).unwrap(); + assert_eq!((texture.layout(), texture.decode16().unwrap()), (Layout::Rgb, vec![300, 40000, 0, 65535, 7, 0])); + } + + #[test] + fn typeless_and_padded_16_bit_textures_are_read() { + // Rows padded to 8 bytes, as other writers may store them: the padding is skipped. + let file = encode16(3, 2, Layout::Gray, &[1, 2, 3, 4, 5, 6], DdsOptions::default()).unwrap(); + let mut padded = file[..HEADER + DX10_HEADER].to_vec(); + padded[20..24].copy_from_slice(&8u32.to_le_bytes()); + for row in file[HEADER + DX10_HEADER..].chunks(6) { + padded.extend_from_slice(row); + padded.extend_from_slice(&[0xee, 0xee]); + } + assert_eq!(Texture::parse(&padded).unwrap().decode16().unwrap(), [1, 2, 3, 4, 5, 6]); + // The typeless forms read as the UNORM ones; typeless RGBA refuses premultiplied alpha too. + for (unorm, typeless, layout) in [(56u32, 53u32, Layout::Gray), (35, 33, Layout::GrayA), (11, 9, Layout::Rgba)] { + let pixels = spread16(3, 2, layout.channels()); + let file = encode16(3, 2, layout, &pixels, DdsOptions::default()).unwrap(); + assert_eq!(le32(&file, HEADER).unwrap(), unorm); + let mut retyped = file.clone(); + retyped[HEADER..HEADER + 4].copy_from_slice(&typeless.to_le_bytes()); + let texture = Texture::parse(&retyped).unwrap(); + assert_eq!((texture.layout(), texture.decode16().unwrap()), (layout, pixels)); + retyped[HEADER + 16] = 2; + assert_eq!(Texture::parse(&retyped).is_err(), layout == Layout::Rgba); + } } #[test] @@ -1297,12 +1354,15 @@ mod tests { let (w, h) = (62usize, 61usize); let at = |i: usize| (i % w, i / w); let height: Vec = (0..w * h).map(|i| (20_000 + at(i).0 * 29 + at(i).1 * 11) as u16).collect(); - let normal: Vec = (0..w * h).flat_map(|i| [(10_000 + at(i).0 * 23 + at(i).1 * 7) as u16, (40_000 + at(i).1 * 27 + at(i).0 * 5) as u16, 65535]).collect(); + let normal: Vec = + (0..w * h).flat_map(|i| [(10_000 + at(i).0 * 23 + at(i).1 * 7) as u16, (40_000 + at(i).1 * 27 + at(i).0 * 5) as u16, 65535]).collect(); for (compression, layout, pixels) in [(DdsCompression::Bc4, Layout::Gray, &height), (DdsCompression::Bc5, Layout::Rgb, &normal)] { let options = DdsOptions { compression, mipmaps: true }; let rounded: Vec = pixels.iter().map(|&v| to8(v)).collect(); - let from16 = Texture::parse(&encode16(w as u32, h as u32, layout, pixels, options).unwrap()).unwrap(); - let from8 = Texture::parse(&encode(w as u32, h as u32, layout, &rounded, options).unwrap()).unwrap(); + let (file16, file8) = + (encode16(w as u32, h as u32, layout, pixels, options).unwrap(), encode(w as u32, h as u32, layout, &rounded, options).unwrap()); + let from16 = Texture::parse(&file16).unwrap(); + let from8 = Texture::parse(&file8).unwrap(); let channels = layout.channels(); // Root-mean-square error of the stored channels (BC5: red and green), in 16-bit units. let rms = |got: &[u16]| { @@ -1337,10 +1397,21 @@ mod tests { for layout in [Layout::Gray, Layout::GrayA, Layout::Rgb, Layout::Rgba] { let options = DdsOptions { compression, mipmaps: false }; let n = layout.channels(); - assert_eq!(encode16(1, 1, layout, &[9; 4][..n], options).is_ok(), encode(1, 1, layout, &[9; 4][..n], options).is_ok(), "{compression:?} from {layout:?}"); + assert_eq!( + encode16(1, 1, layout, &[9; 4][..n], options).is_ok(), + encode(1, 1, layout, &[9; 4][..n], options).is_ok(), + "{compression:?} from {layout:?}" + ); } } assert!(matches!(encode16(2, 2, Layout::Rgb, &[0; 11], DdsOptions::default()), Err(DdsError::Encode(_)))); + let bc1 = DdsOptions { compression: DdsCompression::Bc1, mipmaps: false }; + assert_eq!(encode16(1, 1, Layout::Rgba, &[0; 3], bc1), Err(DdsError::Encode("3 16-bit samples of pixels for a 1x1 Rgba texture".into()))); + // Reading BC1 to BC3 at 16 bits: their 8-bit values times 257. + let file = encode16(6, 5, Layout::Rgba, &pixels, bc1).unwrap(); + let texture = Texture::parse(&file).unwrap(); + let narrow = texture.decode().unwrap(); + assert_eq!(texture.decode16().unwrap(), narrow.iter().map(|&v| u16::from(v) * 257).collect::>()); assert!(matches!(encode16(0, 2, Layout::Rgb, &[], DdsOptions::default()), Err(DdsError::Encode(_)))); } From 892ef9116c432d7afb5c08aba93e93a4e0b1548f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 16:21:25 -0500 Subject: [PATCH 098/209] ci: accept moved stable rust toolchain pin (#1472) From deaee4e8e10ad3e4a2406b8fe7213562d5e538c2 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 16 Nov 2026 16:47:28 -0500 Subject: [PATCH 099/209] craft-mesh: decode a glTF data URI's percent escapes to bytes A base64 body with `%2B`, `%2F` or `%3D` escapes was refused as bad base64, and a plain body lost every escaped byte that was not UTF-8 to U+FFFD. Escapes are now decoded to bytes first, then base64 when the URI says so. `uri_decode` is unchanged for file names. Closes #1936 Co-Authored-By: Claude Opus 5.5 --- crates/mesh/src/gltf.rs | 13 ++++++++++--- crates/mesh/src/tests/gltf.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/crates/mesh/src/gltf.rs b/crates/mesh/src/gltf.rs index 9d45a4453..e92b1fcba 100644 --- a/crates/mesh/src/gltf.rs +++ b/crates/mesh/src/gltf.rs @@ -115,6 +115,11 @@ pub use crate::base64_decode; /// Percent-decode a URI. A `%` that two hex digits do not follow stays as it is. pub fn uri_decode(s: &str) -> String { + String::from_utf8_lossy(&percent_decode(s)).into_owned() +} + +/// [`uri_decode`] as bytes, so escaped bytes that are not UTF-8 survive. +fn percent_decode(s: &str) -> Vec { let b = s.as_bytes(); let mut out = Vec::with_capacity(b.len()); let mut i = 0; @@ -128,7 +133,7 @@ pub fn uri_decode(s: &str) -> String { i += 1; } } - String::from_utf8_lossy(&out).into_owned() + out } /// The media type and bytes of a `data:` URI. @@ -137,10 +142,12 @@ fn data_uri(uri: &str) -> Option)>> { Some((|| { let (meta, data) = rest.split_once(',').ok_or_else(|| malformed(KIND, "a data URI without a comma"))?; let mime = meta.split(';').next().unwrap_or("").to_string(); + // Any character may be escaped (RFC 3986 2.1), base64's `+`, `/` and `=` included. + let bytes = percent_decode(data); let bytes = if meta.ends_with(";base64") { - base64_decode(data).ok_or_else(|| malformed(KIND, "bad base64 in a data URI"))? + std::str::from_utf8(&bytes).ok().and_then(base64_decode).ok_or_else(|| malformed(KIND, "bad base64 in a data URI"))? } else { - uri_decode(data).into_bytes() + bytes }; Ok((mime, bytes)) })()) diff --git a/crates/mesh/src/tests/gltf.rs b/crates/mesh/src/tests/gltf.rs index d35b63d57..d1386dce2 100644 --- a/crates/mesh/src/tests/gltf.rs +++ b/crates/mesh/src/tests/gltf.rs @@ -152,6 +152,34 @@ fn gltf_text(buffer: &str, image: &str) -> (Vec, Vec) { (serde_json::to_vec(&doc).unwrap(), b.bin) } +/// What a `.gltf` file holding `buffer` and `image` as URIs reads as: the image's bytes, or +/// the error. +fn embedded(buffer: &str, image: &str) -> std::result::Result>, Error> { + let (doc, _) = gltf_text(buffer, image); + read(&doc, &Limits::default(), &mut |_| None).map(|s| s.images[0].data.clone()) +} + +/// A data URI may escape any character (RFC 3986 §2.1), base64's `+`, `/` and `=` included. +#[test] +fn percent_escapes_in_a_data_uri_are_decoded_to_bytes() { + let bin = gltf_text("", "").1; + let buffer = format!("data:application/octet-stream;base64,{}", base64(&bin)); + // 0xFB 0xFF 0xBF: "+/+/" in base64. + let image = [0xFB, 0xFF, 0xBF, 0xFB, 0xFF, 0xBF]; + assert_eq!(base64(&image), "+/+/+/+/"); + let escaped = |s: &str| s.replace('+', "%2B").replace('/', "%2f").replace('=', "%3D"); + let want = Some(image.to_vec()); + let plain_image = format!("data:image/png;base64,{}", base64(&image)); + let (head, body) = buffer.split_once(',').unwrap(); + let every = format!("{head},{}", body.bytes().map(|c| format!("%{c:02X}")).collect::()); + assert_eq!(embedded(&every, &plain_image).unwrap(), want, "a buffer escaped throughout"); + assert_eq!(embedded(&buffer, &escaped(&plain_image)).unwrap(), want, "an escaped image"); + // Not base64: the escaped bytes as they are, high bytes too. + assert_eq!(embedded(&buffer, "data:image/png,%FB%FF%BF%FB%ff%bf").unwrap(), want); + assert_eq!(embedded(&buffer, "data:,a%20b%").unwrap(), Some(b"a b%".to_vec())); + assert_eq!(embedded(&buffer, "data:image/png;base64,%25").unwrap(), None, "an escaped % is not base64"); +} + #[test] fn embedded_and_external_resources() { // Embedded: data URIs for the buffer and the image. From 8c1498d1f82308ea4bc5ac744e83d1f62b11dbd8 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 16 Nov 2026 16:53:23 -0500 Subject: [PATCH 100/209] craft-mesh: warn when glTF morph targets or other scenes are left out A primitive's morph targets are not read, and only one scene becomes the roots. Both now add a warning: a count of the primitives drawn in their base shape, and which scene of how many was read. Closes #1938. Co-Authored-By: Claude Opus 5.5 --- crates/mesh/src/gltf.rs | 15 ++++++++++++--- crates/mesh/src/tests/gltf.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/mesh/src/gltf.rs b/crates/mesh/src/gltf.rs index 9d45a4453..2581aeafd 100644 --- a/crates/mesh/src/gltf.rs +++ b/crates/mesh/src/gltf.rs @@ -15,8 +15,9 @@ //! same accessors), and each primitive a [`Part`]. //! //! Not read: required extensions that change how geometry is stored (Draco, meshopt) are -//! [`Error::Unsupported`]; points, lines, morph targets and -//! other extensions are passed over (skipped primitives are counted in a warning). +//! [`Error::Unsupported`]; points, lines, morph targets, scenes other than the default and +//! other extensions are passed over (skipped primitives, primitives with morph targets and the +//! other scenes are named in warnings). //! //! Buffers are needed to read anything, so an external buffer is asked of the resolver (by its //! percent-decoded URI) and is [`Error::Missing`] when declined. External images are only named @@ -345,6 +346,7 @@ impl Doc<'_> { struct Skipped { triangles: usize, primitives: usize, + morphs: usize, } fn rows_of(v: &[f64]) -> impl Iterator + '_ { @@ -439,6 +441,7 @@ fn primitive(d: &Doc, p: &Value, mesh: &mut Mesh, shared: &mut Shared, materials return Ok(()); }; charge(&d.parts_left, 1, "primitives")?; + skipped.morphs += usize::from(!arr(p, "targets").is_empty()); let key = attrs.to_string(); let ([base, normal_base, uv_base, tangent_base], nv) = match shared.get(&key) { Some(known) => *known, @@ -648,7 +651,12 @@ fn nodes(j: &Value, limits: &Limits, scene: &mut Scene) -> Result<()> { children: indices(n, "children", list.len()), }); } - scene.roots = match arr(j, "scenes").get(idx(j, "scene").unwrap_or(0)).or_else(|| arr(j, "scenes").first()) { + let scenes = arr(j, "scenes"); + let chosen = idx(j, "scene").filter(|s| *s < scenes.len()).unwrap_or(0); + if scenes.len() > 1 { + scene.warnings.push(format!("scene {chosen} of {} was read; the others were left out", scenes.len())); + } + scene.roots = match scenes.get(chosen) { Some(s) => indices(s, "nodes", list.len()), None => { // No scene: every node that is nobody's child. @@ -832,6 +840,7 @@ pub fn read(bytes: &[u8], limits: &Limits, resolve: Resolver) -> Result { animations(&d, limits, &mut scene)?; count_warning(&mut scene.warnings, skipped.primitives, "primitive(s) are points, lines or have no positions and were left out"); count_warning(&mut scene.warnings, skipped.triangles, "triangle(s) could not be read and were left out"); + count_warning(&mut scene.warnings, skipped.morphs, "primitive(s) have morph targets, which were left out (drawn in the base shape)"); // A hierarchy with a cycle, or one too deep to walk, is refused here rather than by every caller. scene.instances(limits)?; Ok(scene) diff --git a/crates/mesh/src/tests/gltf.rs b/crates/mesh/src/tests/gltf.rs index d35b63d57..0e6733b65 100644 --- a/crates/mesh/src/tests/gltf.rs +++ b/crates/mesh/src/tests/gltf.rs @@ -829,3 +829,33 @@ fn specular_glossiness_and_webp_textures() { let plain = &s.materials[1]; assert_eq!((plain.base_color, plain.metallic, plain.roughness, plain.base_color_texture), ([1.0; 4], 0.0, 0.0, None)); } + +#[test] +fn morph_targets_and_other_scenes_are_named_in_warnings() { + let mut b = B::default(); + let p = b.f32s(&QUAD[..9], "VEC3", 3); + let d = b.f32s(&[0.0; 9], "VEC3", 3); + let plain = json!({ "attributes": { "POSITION": p } }); + let morphed = json!({ "attributes": { "POSITION": p }, "targets": [{ "POSITION": d }] }); + let doc = |scene: Value, scenes: Value| { + b.glb(json!({ + "scene": scene, + "scenes": scenes, + "nodes": [{ "mesh": 0 }, { "mesh": 0 }], + "meshes": [{ "primitives": [plain, morphed, morphed] }], + })) + }; + let s = parse(&doc(json!(1), json!([{ "nodes": [0] }, { "nodes": [1] }, { "nodes": [] }]))).unwrap(); + assert_eq!(s.roots, [1]); + assert_eq!( + s.warnings, + ["scene 1 of 3 was read; the others were left out", "2 primitive(s) have morph targets, which were left out (drawn in the base shape)",] + ); + // The morphed primitives are still drawn, in their base shape. + assert_eq!(s.meshes[0].parts.len(), 3); + // One scene, or an out-of-range choice, is not a warning about scenes. + let one = parse(&doc(json!(5), json!([{ "nodes": [0] }]))).unwrap(); + assert_eq!((one.roots.as_slice(), one.warnings.len()), ([0].as_slice(), 1)); + let s = parse(&one_mesh(&b, plain.clone())).unwrap(); + assert!(s.warnings.is_empty(), "{:?}", s.warnings); +} From 511e260d23fecfd5839cad2220999f998638097a Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 16 Nov 2026 16:55:57 -0500 Subject: [PATCH 101/209] craft-mesh: test every scene choice; README names the new warnings The README's glTF "Not read" line was doubled and listed sparse accessors and tangents, which are read. Co-Authored-By: Claude Opus 5.5 --- crates/mesh/README.md | 6 +++--- crates/mesh/src/tests/gltf.rs | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/mesh/README.md b/crates/mesh/README.md index 61074a2cb..3b57208fa 100644 --- a/crates/mesh/README.md +++ b/crates/mesh/README.md @@ -110,9 +110,9 @@ FBX 6.x files (a `Model` holds its mesh, `Properties60`, objects connected by na geometry, transforms, the hierarchy and materials. Not read: FBX 5 and older, 6.x textures, animation, skin deformers, blend shapes, cameras, lights, NURBS. -Not read (glTF): required geometry extensions (Draco, meshopt) are -Not read (glTF): sparse accessors and required geometry extensions (Draco, meshopt) are -`Error::Unsupported`; points, lines, morph targets, tangents and other extensions are passed over. +Not read (glTF): required geometry extensions (Draco, meshopt) are `Error::Unsupported`. Points, +lines and morph targets are left out and counted in a warning; scenes other than the default are +named in one; other extensions are passed over. Not written: skins, animations, cameras, lights, and `.gltf` with separate files. ## Hostile input diff --git a/crates/mesh/src/tests/gltf.rs b/crates/mesh/src/tests/gltf.rs index 0e6733b65..09ae6eee3 100644 --- a/crates/mesh/src/tests/gltf.rs +++ b/crates/mesh/src/tests/gltf.rs @@ -853,9 +853,18 @@ fn morph_targets_and_other_scenes_are_named_in_warnings() { ); // The morphed primitives are still drawn, in their base shape. assert_eq!(s.meshes[0].parts.len(), 3); - // One scene, or an out-of-range choice, is not a warning about scenes. - let one = parse(&doc(json!(5), json!([{ "nodes": [0] }]))).unwrap(); - assert_eq!((one.roots.as_slice(), one.warnings.len()), ([0].as_slice(), 1)); + let morphs = "2 primitive(s) have morph targets, which were left out (drawn in the base shape)"; + // An out-of-range choice falls back to the first scene. + let s = parse(&doc(json!(5), json!([{ "nodes": [1] }, { "nodes": [0] }]))).unwrap(); + assert_eq!(s.roots, [1]); + assert_eq!(s.warnings, ["scene 0 of 2 was read; the others were left out", morphs]); + // One scene, or none, is not a warning about scenes. + let s = parse(&doc(json!(5), json!([{ "nodes": [0] }]))).unwrap(); + assert_eq!(s.roots, [0]); + assert_eq!(s.warnings, [morphs]); + let s = parse(&doc(json!(null), json!(null))).unwrap(); + assert_eq!(s.roots, [0, 1]); + assert_eq!(s.warnings, [morphs]); let s = parse(&one_mesh(&b, plain.clone())).unwrap(); assert!(s.warnings.is_empty(), "{:?}", s.warnings); } From a6b7bdce3ee3401a6148c6f3e8003b6dba109ae9 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Mon, 16 Nov 2026 14:01:25 -0800 Subject: [PATCH 102/209] ps: write numbers from their rounded integers instead of the float formatter push_num_places now writes the digits of round(v * 10^places) directly when that integer is below 2^52 in magnitude, placing the decimal point and trimming fractional zeros itself. Below 2^52 the integer is exact and dividing it by 10^places lands within half a decimal step of the exact quotient, so the text is the same as the formatter's. NaN, infinities, scaling overflow and larger rounded values take the existing formatter path unchanged. Requests for more than nine places are still clamped to nine first. Tests compare the output with the formatter path for places 0 to 12 over fixed boundary values (negative zero, ties, 1e15, 2^52, f64::MAX, non-finite values) and about 144,000 values in all (about 48,000 generated, plus the neighboring bit patterns of each). Widening the limit to 2^54 makes the comparison fail, for example -1709448349065955.5 at one place. On 1,200,000 path coordinates at four places (release build, wall clock on a Linux host running other builds at the same time, seven alternating runs), the formatter path took 362 to 373 ms and the integer path 38 to 44 ms. --- crates/ps/src/write.rs | 290 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 288 insertions(+), 2 deletions(-) diff --git a/crates/ps/src/write.rs b/crates/ps/src/write.rs index 97201771b..89c2e39df 100644 --- a/crates/ps/src/write.rs +++ b/crates/ps/src/write.rs @@ -47,6 +47,9 @@ pub fn written(v: f64) -> f64 { /// finite). pub fn push_num_places(s: &mut String, v: f64, places: u32) { let places = places.min(9); + if push_num_scaled(s, v, places) { + return; + } let scale = 10f64.powi(places as i32); // Past where the scaling overflows, values have no fractional digits to round. let v = if !v.is_finite() { @@ -70,6 +73,75 @@ pub fn push_num_places(s: &mut String, v: f64, places: u32) { } } +/// `10^places` for `places` up to 9, each exact in an `f64`. +const POW10: [f64; 10] = [1.0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9]; + +/// 2^52: below it every integer is an `f64`, and dividing one by `10^places` lands within half a +/// decimal step of the exact quotient, so the decimal digits are those of the integer. +const SCALED_LIMIT: f64 = 4_503_599_627_370_496.0; + +/// Append `v` with at most `places` decimals by writing the digits of the rounded integer +/// `v * 10^places`, the same text [`push_num_places`] writes. Returns false and appends nothing +/// when `places` is above 9, `v` is not finite, the scaling overflows or the rounded integer is +/// 2^52 or more in magnitude. +fn push_num_scaled(s: &mut String, v: f64, places: u32) -> bool { + let Some(&scale) = POW10.get(places as usize) else { + return false; + }; + let k = (v * scale).round(); + let magnitude = k.abs(); + if magnitude.is_nan() || magnitude >= SCALED_LIMIT { + return false; + } + // `magnitude` is a whole number below 2^52, so the cast is exact. + let mut m = magnitude as u64; + let mut decimals = places as usize; + while decimals > 0 && m.is_multiple_of(10) { + m /= 10; + decimals -= 1; + } + let unit = POW10_INT.get(decimals).copied().unwrap_or(1); + let (whole, fraction) = (m / unit, m % unit); + // A rounded negative zero has no sign: it is written `0`. + if k < 0.0 { + s.push('-'); + } + push_digits(s, whole, POW10_INT.iter().take_while(|&&p| p <= whole).count().max(1)); + if decimals > 0 { + s.push('.'); + push_digits(s, fraction, decimals); + } + true +} + +/// `10^n` for `n` up to 16, enough for every digit of an integer below 2^52. +const POW10_INT: [u64; 17] = [ + 1, + 10, + 100, + 1_000, + 10_000, + 100_000, + 1_000_000, + 10_000_000, + 100_000_000, + 1_000_000_000, + 10_000_000_000, + 100_000_000_000, + 1_000_000_000_000, + 10_000_000_000_000, + 100_000_000_000_000, + 1_000_000_000_000_000, + 10_000_000_000_000_000, +]; + +/// Append the last `width` decimal digits of `n`, leading zeros included. +fn push_digits(s: &mut String, n: u64, width: usize) { + for &p in POW10_INT.iter().take(width).rev() { + s.push(char::from(b'0' + ((n / p) % 10) as u8)); + } +} + /// `vs` as numbers separated by spaces, then a space. pub fn push_nums(s: &mut String, vs: &[f64]) { for v in vs { @@ -243,8 +315,8 @@ pub fn inflate(data: &[u8], max: u64) -> Option> { #[cfg(test)] mod tests { use super::{ - FillRule, PaintOps, PathOps, bytes, clip_op, deflate, fill_op, inflate, matrix, num, push_num, push_num_places, push_nums, push_path, string, - written, + FillRule, PaintOps, PathOps, bytes, clip_op, deflate, fill_op, inflate, matrix, num, push_num, push_num_places, push_num_scaled, push_nums, + push_path, string, written, }; use kurbo::{Affine, BezPath}; @@ -285,6 +357,220 @@ mod tests { assert_eq!(places(1e15 - 1.0, 0), "999999999999999"); } + /// `push_num_places` without the integer path: every value goes through the float formatter. + fn base_push_num_places(s: &mut String, v: f64, places: u32) { + use std::fmt::Write; + let places = places.min(9); + let scale = 10f64.powi(places as i32); + let v = if !v.is_finite() { + 0.0 + } else if (v * scale).is_finite() { + (v * scale).round() / scale + } else { + v + }; + if v == v.trunc() && v.abs() < 1e15 { + let _ = write!(s, "{}", v as i64); + return; + } + let start = s.len(); + let _ = write!(s, "{v:.*}", places as usize); + if s[start..].contains('.') { + let keep = s[start..].trim_end_matches('0').trim_end_matches('.').len(); + s.truncate(start + keep); + } + } + + /// A deterministic xorshift sequence. + fn next(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state + } + + /// Values that hit decimal ties, the 2^52 and 1e15 limits and their neighboring bit patterns, + /// coordinates of every magnitude, and arbitrary bit patterns. + fn number_samples() -> Vec { + let two52 = 4_503_599_627_370_496.0f64; + let mut values = vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -0.5, + 1.5, + 2.5, + -2.5, + 0.1 + 0.2, + f64::from(0.4f32), + 0.123_456_78, + 123_456.789_012_345, + -0.000_04, + -0.000_05, + 0.000_05, + 1e-10, + -1e-10, + f64::MIN_POSITIVE, + -f64::MIN_POSITIVE, + 5e-324, + 1e15, + 1e15 - 1.0, + -1e15, + 1e15 + 0.5, + two52, + two52 - 1.0, + two52 - 0.5, + -two52, + 1e16, + 1e20, + 1.5e305, + f64::MAX, + f64::MIN, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + ]; + let mut scale = 1.0; + for _ in 0..=9 { + for v in [two52 / scale, (two52 - 1.0) / scale, 1e15 / scale, 0.5 / scale, 1.5 / scale, 2.5 / scale] { + values.extend([v, -v]); + } + scale *= 10.0; + } + let mut state = 0x9e37_79b9_7f4a_7c15u64; + for _ in 0..12_000 { + let r = next(&mut state); + values.push(f64::from_bits(r)); + let exponent = (r % 24) as i32 - 8; + let mantissa = (next(&mut state) % 2_000_000_001) as f64 / 1e9 - 1.0; + values.push(mantissa * 10f64.powi(exponent)); + let places = (r % 10) as i32; + let tie = ((next(&mut state) % 20_000_000_001) as f64 - 1e10 + 0.5) / 10f64.powi(places); + values.push(tie); + values.push((next(&mut state) % 100_000_000) as f64 / 1e4 - 5_000.0); + } + let neighbors: Vec = values + .iter() + .filter(|v| v.is_finite() && **v != 0.0) + .flat_map(|v| [f64::from_bits(v.to_bits() - 1), f64::from_bits(v.to_bits() + 1)]) + .collect(); + values.extend(neighbors); + values + } + + #[test] + fn numbers_match_base_formatter() { + let mut integer_path = 0usize; + for places in 0..=12 { + for v in number_samples() { + let mut expected = String::from("x "); + base_push_num_places(&mut expected, v, places); + let mut got = String::from("x "); + push_num_places(&mut got, v, places); + assert_eq!(got, expected, "{v:e} ({:#x}) at {places} places", v.to_bits()); + if push_num_scaled(&mut String::new(), v, places.min(9)) { + integer_path += 1; + } + } + } + // The comparison covers the integer path, not only the fallback. + assert!(integer_path > 200_000, "{integer_path} values took the integer path"); + } + + #[test] + fn coordinates_are_written_through_integers() { + for (v, places) in [(0.0, 4), (-0.0, 4), (1.5, 4), (-2.25, 4), (4.3333, 4), (612.0, 0), (0.123_456_78, 7), (-0.000_05, 4), (1.23456789, 9)] { + let mut s = String::from("x "); + assert!(push_num_scaled(&mut s, v, places), "{v} at {places} places"); + let mut expected = String::from("x "); + base_push_num_places(&mut expected, v, places); + assert_eq!(s, expected); + } + let two52 = 4_503_599_627_370_496.0f64; + for (v, places) in [ + (f64::NAN, 4), + (f64::INFINITY, 4), + (f64::NEG_INFINITY, 0), + (f64::MAX, 4), + (f64::MIN, 0), + (1e305, 9), + (two52, 0), + (-two52, 0), + (two52 / 1e9, 9), + (1e20, 0), + (1.5, 10), + ] { + let mut s = String::from("x "); + assert!(!push_num_scaled(&mut s, v, places), "{v} at {places} places"); + assert_eq!(s, "x "); + } + } + + #[test] + fn number_boundaries_keep_base_text() { + let places = |v: f64, n: u32| { + let mut s = String::new(); + push_num_places(&mut s, v, n); + s + }; + let two52 = 4_503_599_627_370_496.0f64; + assert_eq!(places(1e15, 0), "1000000000000000"); + assert_eq!(places(-1e15, 0), "-1000000000000000"); + assert_eq!(places(two52 - 1.0, 0), "4503599627370495"); + assert_eq!(places(two52, 0), "4503599627370496"); + assert_eq!(places(1e20, 0), "100000000000000000000"); + assert_eq!(places(450_359_962_737_049.5, 1), "450359962737049.5"); + assert_eq!(places(-0.0, 4), "0"); + assert_eq!(places(-0.000_04, 4), "0"); + assert_eq!(places(-0.000_05, 4), "-0.0001"); + assert_eq!(places(0.5, 0), "1"); + assert_eq!(places(-1.5, 0), "-2"); + assert_eq!(places(0.000_000_001, 9), "0.000000001"); + assert_eq!(places(1.25, 20), "1.25"); + assert_eq!(places(10.5, 4), "10.5"); + assert_eq!(places(100.0, 4), "100"); + let mut s = String::from("x"); + push_nums(&mut s, &[]); + assert_eq!(s, "x"); + let mut scale = 1.0; + for (n, value) in super::POW10.iter().enumerate() { + assert_eq!(*value, scale, "10^{n}"); + assert_eq!(*value, 10f64.powi(n as i32), "10^{n}"); + scale *= 10.0; + } + } + + /// Times the base formatter against `push_num_places` on path coordinates. Run with + /// `cargo test --release -p craft-ps --features write number_writing_timing -- --ignored --nocapture`. + #[test] + #[ignore = "timing probe"] + fn number_writing_timing() { + use std::time::Instant; + let mut state = 0x2545_f491_4f6c_dd1du64; + let values: Vec = (0..1_200_000).map(|_| (next(&mut state) % 12_000_000_000) as f64 / 1e7 - 200.0).collect(); + let mut base_ms = vec![]; + let mut new_ms = vec![]; + for _ in 0..7 { + let start = Instant::now(); + let mut base = String::with_capacity(values.len() * 10); + for v in &values { + base_push_num_places(&mut base, *v, 4); + base.push(' '); + } + base_ms.push(start.elapsed().as_secs_f64() * 1e3); + let start = Instant::now(); + let mut new = String::with_capacity(values.len() * 10); + push_nums(&mut new, &values); + new_ms.push(start.elapsed().as_secs_f64() * 1e3); + assert_eq!(new, base); + } + println!("{} numbers at 4 places", values.len()); + println!("base formatter ms: {base_ms:.2?}"); + println!("integer path ms: {new_ms:.2?}"); + } + #[test] fn strings_and_bytes_are_escaped() { assert_eq!(string("ok"), "(ok)"); From 5541e248e86c923168866de7e183dee4ef4eeeda Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Mon, 16 Nov 2026 17:12:46 -0500 Subject: [PATCH 103/209] wordmodel: append to a paragraph in O(1) with push_text and push_object Building a paragraph through insert_text/insert_object at its end walked every run (and insert_object counted every object before the offset) on each call, so n appends cost O(n^2): one paragraph of 64k runs took 8.5 minutes. push_text and push_object extend the last run or start a new one, giving the same paragraph as inserting at len(). --- crates/wordmodel/src/para.rs | 31 +++++++++++++++++++++ crates/wordmodel/src/tests.rs | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/crates/wordmodel/src/para.rs b/crates/wordmodel/src/para.rs index bbfeb9cbc..901e300b6 100644 --- a/crates/wordmodel/src/para.rs +++ b/crates/wordmodel/src/para.rs @@ -296,6 +296,37 @@ impl Paragraph { Ok(()) } + /// Append plain text (U+FFFC is dropped) with `props`. Returns the appended length. The same as + /// [`Self::insert_text`] at [`Self::len`], but it doesn't walk the runs or the text, so building + /// a paragraph of `n` pieces this way is linear rather than quadratic in `n`. + pub fn push_text(&mut self, s: &str, props: &CharProps) -> usize { + let clean: std::borrow::Cow = if s.contains(OBJ) { s.chars().filter(|c| *c != OBJ).collect::().into() } else { s.into() }; + if clean.is_empty() { + return 0; + } + self.text.push_str(&clean); + self.push_run(clean.len(), props); + self.touch(); + clean.len() + } + + /// Append an object with `props`: [`Self::insert_object`] at [`Self::len`], without counting + /// the objects already there. + pub fn push_object(&mut self, obj: InlineObject, props: &CharProps) { + self.text.push(OBJ); + self.objects.push(obj); + self.push_run(OBJ.len_utf8(), props); + self.touch(); + } + + /// Extend the last run by `len` when it has `props`, or start a new one. + fn push_run(&mut self, len: usize, props: &CharProps) { + match self.runs.last_mut() { + Some(last) if last.props == *props => last.len += len, + _ => self.runs.push(Run { len, props: props.clone() }), + } + } + fn insert_run(&mut self, off: usize, len: usize, props: CharProps) { // Split the run at `off`, insert, merge neighbours. let mut start = 0; diff --git a/crates/wordmodel/src/tests.rs b/crates/wordmodel/src/tests.rs index 088bd3bd2..b4d1b17dd 100644 --- a/crates/wordmodel/src/tests.rs +++ b/crates/wordmodel/src/tests.rs @@ -644,6 +644,57 @@ fn para_of_objects(objs: Vec) -> Paragraph { p } +/// push_text and push_object build the same paragraph as inserting at the end. +#[test] +fn pushing_matches_inserting_at_the_end() { + let bold = CharProps { bold: Some(true), ..Default::default() }; + let plain = CharProps::default(); + let (mut pushed, mut inserted) = (Paragraph::new(), Paragraph::new()); + let steps: [(&str, &CharProps, bool); 7] = [ + ("Hello ", &plain, false), + ("world", &bold, false), + ("", &bold, true), + (", again", &bold, false), + ("", &bold, false), + ("x\u{FFFC}y", &plain, false), + ("", &plain, true), + ]; + for (i, (text, props, object)) in steps.into_iter().enumerate() { + if object { + let obj = InlineObject::BookmarkStart { name: format!("b{i}") }; + pushed.push_object(obj.clone(), props); + inserted.insert_object(inserted.len(), obj, props).unwrap(); + } else { + let n = pushed.push_text(text, props); + assert_eq!(n, inserted.insert_text(inserted.len(), text, props).unwrap()); + } + assert_eq!(pushed, inserted, "after step {i}"); + } + assert_eq!(pushed.text, "Hello world\u{FFFC}, againxy\u{FFFC}"); + assert_eq!(pushed.runs.len(), 3, "equal neighbours merge: {:?}", pushed.runs); + assert_eq!(pushed.objects.len(), 2); +} + +#[test] +fn pushing_a_long_paragraph_is_linear() { + // Appending through insert_text/insert_object walked every run (and counted every object) on + // each call: 64k alternating runs took minutes. Pushing takes milliseconds. + let n = 64_000; + let bold = CharProps { bold: Some(true), ..Default::default() }; + let plain = CharProps::default(); + let t = std::time::Instant::now(); + let mut p = Paragraph::new(); + for i in 0..n { + p.push_text("word ", if i % 2 == 0 { &bold } else { &plain }); + if i % 8 == 0 { + p.push_object(InlineObject::BookmarkStart { name: format!("b{i}") }, &plain); + } + } + assert_eq!(p.runs.iter().map(|r| r.len).sum::(), p.text.len()); + assert_eq!(p.objects.len(), n / 8); + assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed()); +} + #[test] fn object_walks_are_linear_in_the_paragraph() { // #3614: loops of object_offsets + object_at recounted the objects before each one, so a From d5eddf0cc2dd43e4edc47669de9f90491448f579 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Mon, 16 Nov 2026 17:16:24 -0500 Subject: [PATCH 104/209] flow: a table with no style has no header row from the default look flow_table made a table's first row a header row whenever its look had header_row set, and the look defaults to Word's (header row on). The look is an option of the table's style (w:tblLook/@w:firstRow), so a table with no style, or a style the document doesn't have, got a header row Word never shows: RTF bolded it and marked it \trhdr (repeat on each page), HTML made it th, and ODT repeated it. Only Repeat as header row (RowProps::header) makes such a row a header now; styled tables are unchanged. --- crates/flow/src/from_doc.rs | 5 ++++- crates/flow/src/lib.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/flow/src/from_doc.rs b/crates/flow/src/from_doc.rs index b7204f1bf..d85953a0a 100644 --- a/crates/flow/src/from_doc.rs +++ b/crates/flow/src/from_doc.rs @@ -1101,7 +1101,10 @@ fn flow_table(doc: &Document, t: &Table, depth: usize, tracked: FlowOpts) -> FTa let ncols = t.rows.iter().map(|r| r.first_grid() + r.cells.iter().map(|c| c.span()).sum::() + r.grid_after()).max().unwrap_or(0).max(t.grid.len()); for (ri, r) in t.rows.iter().enumerate() { - let header = r.props.header || (ri == 0 && t.props.look.header_row && t.rows.len() > 1); + // The look's header row is an option of the table's style (`w:tblLook/@w:firstRow`): with + // no table style there is nothing to apply, so the first row stays an ordinary row, as in + // Word. Only Repeat as header row makes it one then. + let header = r.props.header || (ri == 0 && style.is_some() && t.props.look.header_row && t.rows.len() > 1); let mut cells = Vec::new(); for (ci, c) in r.cells.iter().enumerate() { let styled = if bake { styled_cell(t, style.as_ref(), (ri, ci), ncols) } else { None }; diff --git a/crates/flow/src/lib.rs b/crates/flow/src/lib.rs index 774b5ffd1..7dc83c99d 100644 --- a/crates/flow/src/lib.rs +++ b/crates/flow/src/lib.rs @@ -80,6 +80,32 @@ mod tests { assert!(kept); } + /// The default look's header row is an option of the table style: a table with no style (or a + /// style the document doesn't have) has an ordinary first row, so RTF doesn't bold it or + /// repeat it on each page, and HTML doesn't make it `th`. Repeat as header row still does. + #[test] + fn look_header_row_needs_a_table_style() { + use craft_wordmodel::{Block, Document, table::Table}; + let first_row_header = |style: Option<&str>, repeat: bool| { + let mut t = Table::new(2, 2, 200.0); + t.props.style = style.map(str::to_string); + assert!(t.props.look.header_row, "the default look has a header row"); + if let Some(r) = t.rows.first_mut() { + r.props.header = repeat; + } + let mut doc = Document::new(); + doc.body = vec![std::sync::Arc::new(Block::Table(t))]; + let flow = from_doc(&doc); + let Some(FBlock::Table(ft)) = flow.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table in {:?}", flow.blocks) }; + assert!(ft.rows.get(1).is_some_and(|r| r.iter().all(|c| !c.header)), "the second row is never a header"); + ft.rows.first().is_some_and(|r| r.iter().all(|c| c.header)) + }; + assert!(first_row_header(Some("TableGrid"), false), "a table style applies the look's header row"); + assert!(!first_row_header(None, false), "no style, no header row"); + assert!(!first_row_header(Some("NoSuchStyle"), false), "a missing style applies nothing"); + assert!(first_row_header(None, true), "Repeat as header row"); + } + #[test] fn percent_decode_requires_hex_digits() { assert_eq!(percent_decode("%+A"), b"%+A".to_vec()); From 778493598fcca586745a502133fca1368b911a7c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 17:53:46 -0500 Subject: [PATCH 105/209] Migrate Flatpak updates to the OCI image and appimage pipeline (#1318) --- Cargo.lock | 8 +- crates/update/Cargo.toml | 2 + crates/update/src/detect.rs | 210 ++++++++++++++++- crates/update/src/install.rs | 381 +++++++++++++++++++++++++++++-- crates/update/src/packages.rs | 181 ++++++++++++++- crates/update/src/process.rs | 78 +++++++ crates/update/src/self_update.rs | 20 +- crates/update/src/status.rs | 17 ++ 8 files changed, 857 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8679e4eac..ef82f1afa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2887,6 +2887,10 @@ dependencies = [ name = "craft-sniff" version = "0.1.0" +[[package]] +name = "craft-ssip" +version = "0.1.0" + [[package]] name = "craft-stencilcraft-document" version = "0.1.0" @@ -2900,10 +2904,6 @@ dependencies = [ "thiserror 2.0.21", ] -[[package]] -name = "craft-ssip" -version = "0.1.0" - [[package]] name = "craft-stencilcraft-format" version = "0.1.0" diff --git a/crates/update/Cargo.toml b/crates/update/Cargo.toml index 93a0719a5..e130656e3 100644 --- a/crates/update/Cargo.toml +++ b/crates/update/Cargo.toml @@ -11,6 +11,8 @@ repository.workspace = true workspace = true [features] +# Opt-in migration of first party Flatpak bundles to the ArtCraft OCI remote. +oci_migration = [] default = [] egui = ["dep:egui", "dep:image"] local-mirror = [] diff --git a/crates/update/src/detect.rs b/crates/update/src/detect.rs index 437e315d7..53c000ebc 100644 --- a/crates/update/src/detect.rs +++ b/crates/update/src/detect.rs @@ -67,6 +67,9 @@ fn from_manifest(app: &App, manifest: &Manifest) -> Option { source: InstallSource::Launcher, location: rec.path.display().to_string(), package: None, + externally_managed: false, + origin: None, + installation: None, }) } @@ -169,7 +172,15 @@ pub mod macos { let full_key = format!("{}Version", app.name); let text = [full_key.as_str(), "CFBundleShortVersionString", "CFBundleVersion"].iter().find_map(|k| dict.get(k).and_then(plist::Value::as_string)); - Some(Installed { version: text.and_then(version::parse), source: InstallSource::MacApp, location: path.display().to_string(), package: None }) + Some(Installed { + version: text.and_then(version::parse), + source: InstallSource::MacApp, + location: path.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }) } } @@ -277,6 +288,9 @@ pub mod windows { source: InstallSource::Msi, location: m.exe.map_or_else(|| app.name.to_string(), |p| p.display().to_string()), package: Some(m.product_code), + externally_managed: false, + origin: None, + installation: None, }) } } @@ -303,7 +317,14 @@ pub mod linux { let t = Duration::from_secs(10); let mut out = Vec::new(); let mut add = |manager, list: Vec<(String, String)>| { - out.extend(list.into_iter().filter(|(n, _)| ours(n)).map(|(name, version)| Package { manager, name, version })); + out.extend(list.into_iter().filter(|(n, _)| ours(n)).map(|(name, version)| Package { + manager, + name, + version, + user: false, + origin: None, + installation: None, + })); }; if let Some(list) = capture("dpkg-query", &["-W", "-f", "${Package}|${db:Status-Status}|${Version}\n"], t) { add(InstallSource::Deb, packages::parse_dpkg(&list)); @@ -319,15 +340,28 @@ pub mod linux { add(InstallSource::Pacman, packages::parse_pacman(&list)); } let arch = format!("--arch={}", std::env::consts::ARCH); - if let Some(list) = crate::process::capture_flatpak(&["list", "--app", &arch, "--columns=application,version"], t) { - let mut flatpaks = packages::parse_flatpak_list(&list); + if let Some(list) = crate::process::capture_flatpak(&["list", "--app", &arch, "--columns=application,version,origin,installation"], t) { + let mut flatpaks = packages::parse_flatpak_list_with_origin(&list); // Bundles built from a release tarball often list no version: ask Flatpak directly. - for (id, version) in flatpaks.iter_mut().filter(|(id, v)| v.is_empty() && id.starts_with("ai.storyteller.")) { - if let Some(v) = crate::process::capture_flatpak(&["info", &arch, id], t).as_deref().and_then(packages::parse_flatpak_info) { + for (id, version, _, installation) in flatpaks.iter_mut().filter(|(id, v, _, _)| v.is_empty() && id.starts_with("ai.storyteller.")) { + let custom_scope; + let scope_arg = match installation.as_deref() { + Some("user") => vec!["--user"], + Some("system") | None => vec!["--system"], + Some(named) => { + custom_scope = format!("--installation={named}"); + vec![custom_scope.as_str()] + } + }; + let info_args = [&["info"][..], &scope_arg, &[&arch, id]].concat(); + if let Some(v) = crate::process::capture_flatpak(&info_args, t).as_deref().and_then(packages::parse_flatpak_info) { *version = v; } } - add(InstallSource::Flatpak, flatpaks); + out.extend(flatpaks.into_iter().filter(|(n, _, _, _)| ours(n)).map(|(name, version, origin, installation)| { + let is_user = installation.as_deref() == Some("user"); + Package { manager: InstallSource::Flatpak, name, version, user: is_user, origin, installation } + })); } out } @@ -402,12 +436,41 @@ pub mod linux { pub fn detect(app: &App, snapshot: &Snapshot) -> Option { if let Some((path, version)) = appimages(app).into_iter().next() { - return Some(Installed { version, source: InstallSource::AppImage, location: path.display().to_string(), package: None }); + return Some(Installed { + version, + source: InstallSource::AppImage, + location: path.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }); } let order = [InstallSource::Deb, InstallSource::Rpm, InstallSource::Pacman, InstallSource::Flatpak]; if let Some(p) = packages::find(app, &snapshot.packages, &order) { - let location = if p.manager == InstallSource::Flatpak { app.bundle_id() } else { binary_on_path(app.id) }; - return Some(Installed { version: packages::version_of(p), source: p.manager, location, package: Some(p.name.clone()) }); + let location = if p.manager == InstallSource::Flatpak { p.name.clone() } else { binary_on_path(app.id) }; + let externally_managed = p.manager == InstallSource::Flatpak + && p.origin.as_deref().is_none_or(|origin| { + if packages::is_flatpark_origin(origin) { + return true; + } + let scope = match p.installation.as_deref() { + Some("user") => "--user".to_string(), + Some("system") | None => "--system".to_string(), + Some(name) => format!("--installation={name}"), + }; + crate::process::capture_flatpak(&["remotes", &scope, "--show-disabled", "--columns=name,url"], Duration::from_secs(10)) + .is_none_or(|remotes| packages::remote_manages_flatpak(origin, &remotes)) + }); + return Some(Installed { + version: packages::version_of(p), + source: p.manager, + location, + package: Some(p.name.clone()), + externally_managed, + origin: p.origin.clone(), + installation: p.installation.clone(), + }); } detect_system_install(app) } @@ -418,7 +481,15 @@ pub mod linux { /// to whatever installed it, since it can't tell which package manager that was. fn detect_system_install(app: &App) -> Option { let bin = find_binary(app.id)?; - Some(Installed { version: binary_version(&bin, app), source: InstallSource::Binary, location: bin.display().to_string(), package: None }) + Some(Installed { + version: binary_version(&bin, app), + source: InstallSource::Binary, + location: bin.display().to_string(), + package: None, + externally_managed: true, + origin: None, + installation: None, + }) } fn find_binary(id: &str) -> Option { @@ -445,7 +516,15 @@ pub mod freebsd { if !bin.exists() { return None; } - Some(Installed { version: binary_version(&bin, app), source: InstallSource::Binary, location: bin.display().to_string(), package: None }) + Some(Installed { + version: binary_version(&bin, app), + source: InstallSource::Binary, + location: bin.display().to_string(), + package: None, + externally_managed: true, + origin: None, + installation: None, + }) } /// ` --version` prints ` 0.5.0 (sha, date)`. @@ -690,6 +769,113 @@ mod tests { assert!(strip_cli_prefix("photocraft-cli 1.0.0", "photocraft").is_none()); } + /// The launcher's own record is the first thing detection trusts, and its source is the + /// launcher rather than whatever installed the file underneath. + #[test] + fn a_manifest_record_for_a_file_that_is_there_is_a_launcher_install() { + let app = App::by_id("photocraft").unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("photocraft.AppImage"); + std::fs::write(&path, b"").unwrap(); + let mut manifest = crate::manifest::Manifest::default(); + manifest.apps.insert( + app.id.to_string(), + crate::manifest::Record { + version: "0.6.1".into(), + kind: crate::PackageKind::AppImage, + path: path.clone(), + installed_at: 0, + files: Vec::new(), + needs_root: false, + }, + ); + let found = from_manifest(app, &manifest).expect("the record names a file that is there"); + assert_eq!(found.source, InstallSource::Launcher); + assert_eq!(found.version.map(|v| v.to_string()).as_deref(), Some("0.6.1")); + assert_eq!(found.location, path.display().to_string()); + assert!(!found.externally_managed); + assert_eq!(found.origin, None); + assert_eq!(found.installation, None); + } + + /// A record whose file the user removed is not an install, so detection keeps looking. + #[test] + fn a_manifest_record_for_a_file_that_is_gone_is_not_an_install() { + let app = App::by_id("photocraft").unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut manifest = crate::manifest::Manifest::default(); + manifest.apps.insert( + app.id.to_string(), + crate::manifest::Record { + version: "0.6.0".into(), + kind: crate::PackageKind::AppImage, + path: dir.path().join("gone.AppImage"), + installed_at: 0, + files: Vec::new(), + needs_root: false, + }, + ); + assert!(from_manifest(app, &manifest).is_none()); + } + + /// A package the snapshot already found, as the launcher sees it after a package-manager + /// query: only the fields detection reads are varied. + #[cfg(target_os = "linux")] + fn injected(manager: InstallSource, installation: Option<&str>, origin: Option<&str>) -> crate::packages::Package { + crate::packages::Package { + manager, + name: match manager { + InstallSource::Flatpak => "ai.storyteller.photocraft".to_string(), + _ => "photocraft".to_string(), + }, + version: "0.6.2".into(), + user: installation == Some("user"), + origin: origin.map(str::to_string), + installation: installation.map(str::to_string), + } + } + + #[cfg(target_os = "linux")] + #[test] + fn a_deb_package_is_reported_with_its_manager_and_version() { + let app = App::by_id("photocraft").unwrap(); + let snapshot = Snapshot { packages: vec![injected(InstallSource::Deb, None, None)] }; + let found = linux::detect(app, &snapshot).expect("the injected package is installed"); + assert_eq!(found.source, InstallSource::Deb); + assert_eq!(found.version.map(|v| v.to_string()).as_deref(), Some("0.6.2")); + assert_eq!(found.package.as_deref(), Some("photocraft")); + assert!(!found.externally_managed); + assert_eq!(found.installation, None); + } + + /// A Flatpak from a remote the launcher doesn't drive stays externally managed, in every + /// scope the `flatpak remotes` query can need. + #[cfg(target_os = "linux")] + #[test] + fn a_flatpak_from_a_repository_remote_is_externally_managed() { + let app = App::by_id("photocraft").unwrap(); + for installation in [Some("user"), Some("system"), Some("custom"), None] { + let snapshot = Snapshot { packages: vec![injected(InstallSource::Flatpak, installation, Some("flathub"))] }; + let found = linux::detect(app, &snapshot).expect("the injected Flatpak is installed"); + assert_eq!(found.source, InstallSource::Flatpak); + assert!(found.externally_managed, "{installation:?}"); + assert_eq!(found.location, "ai.storyteller.photocraft"); + assert_eq!(found.origin.as_deref(), Some("flathub")); + assert_eq!(found.installation.as_deref(), installation); + } + } + + /// FlatPark's own remote is known to manage the copy without asking flatpak again. + #[cfg(target_os = "linux")] + #[test] + fn a_flatpark_flatpak_is_externally_managed_without_asking_its_remotes() { + let app = App::by_id("photocraft").unwrap(); + let snapshot = Snapshot { packages: vec![injected(InstallSource::Flatpak, Some("user"), Some("flatpark"))] }; + let found = linux::detect(app, &snapshot).expect("the injected Flatpak is installed"); + assert!(found.externally_managed); + assert_eq!(found.origin.as_deref(), Some("flatpark")); + } + #[cfg(target_os = "linux")] #[test] fn finds_appimages_in_every_folder_newest_first() { diff --git a/crates/update/src/install.rs b/crates/update/src/install.rs index 4b65e3af0..071434595 100644 --- a/crates/update/src/install.rs +++ b/crates/update/src/install.rs @@ -94,6 +94,11 @@ pub fn install_at( { not_running(app, process::running(Path::new(&inst.location)))?; } + if kind == PackageKind::Flatpak + && let Some(installed) = existing.filter(|installed| installed.source == InstallSource::Flatpak && installed.externally_managed) + { + return Err(managed_elsewhere(app, installed, false).unwrap_or_else(|| "Update this app through its Flatpak repository".into())); + } bind_artifact_version(app, release, asset)?; if !job.replace && let Some(inst) = existing @@ -104,6 +109,16 @@ pub fn install_at( } check_download_link(app, release, asset)?; log::info!("{}: starting {} to {} ({})", app.name, if existing.is_some() { "update" } else { "install" }, release.version_label(), kind.label()); + #[cfg(all(target_os = "linux", feature = "oci_migration"))] + if kind == PackageKind::Flatpak + && !job.replace + && !release.prerelease + && let Some(installed) = existing + && let Some(expected) = release.version.as_ref() + && linux::migrate_bundle_to_oci(app, installed, expected, job.stop)? + { + return Ok(None); + } let file = download(release, asset, job)?; log::info!("{}: download ready ({}), running installer...", app.name, file.display()); // Past this point the install can't be interrupted halfway. @@ -363,6 +378,9 @@ pub(crate) fn choose_kinds_with( // opens and removes them but never replaces them. InstallSource::Pacman if is_release_arch_package(app, inst) => Some(PackageKind::ArchPkg), InstallSource::Pacman => return Vec::new(), + // Repository-managed Flatpaks are externally managed: the launcher will not + // replace them with a downloaded local .flatpak bundle. + InstallSource::Flatpak if inst.externally_managed => return Vec::new(), InstallSource::Flatpak => Some(PackageKind::Flatpak), InstallSource::AppImage => Some(PackageKind::AppImage), InstallSource::Binary => Some(PackageKind::TarGz), @@ -463,6 +481,16 @@ fn managed_elsewhere_with(app: &App, installed: &Installed, uninstall: bool, rep let pkg = installed.package.clone().unwrap_or_else(|| app.id.to_string()); Some(pacman_update_hint(app, &pkg, repo_fn(&pkg).as_deref())) } + InstallSource::Flatpak if !uninstall && installed.externally_managed => { + let default_pkg = app.bundle_id(); + let pkg = installed.package.as_deref().unwrap_or(&default_pkg); + let scope = match installed.installation.as_deref() { + Some("user") => "--user".to_string(), + Some("system") | None => "--system".to_string(), + Some(name) => format!("--installation={name}"), + }; + Some(format!("{} was installed from a Flatpak repository. Update it with flatpak update {scope} {pkg}", app.name)) + } InstallSource::Binary => Some(format!("{} was installed outside the launcher. Update or remove it the way it was installed.", app.name)), _ => None, } @@ -805,8 +833,15 @@ fn cancelled(stop: &Stop) -> impl Fn() -> bool + '_ { || stop.check().is_err_and(|e| e == crate::protocol::CANCELLED) } +/// For a running helper command: only CANCELLED ends it. PAUSED lets the current command finish +/// (callers check `stop` before starting the next one). #[cfg(target_os = "linux")] -pub(crate) mod linux { +fn kill_on_cancel(stop: &Stop) -> impl Fn() -> Result<(), String> + '_ { + || if cancelled(stop)() { Err(crate::protocol::CANCELLED.into()) } else { Ok(()) } +} + +#[cfg(target_os = "linux")] +pub mod linux { use std::ffi::{OsStr, OsString}; use std::os::unix::fs::PermissionsExt; use std::time::Duration; @@ -817,7 +852,7 @@ pub(crate) mod linux { use super::{ App, INSTALLER, InstallSource, Installed, JobPhase, PackageKind, Path, PathBuf, Paths, Record, SHARED_CALL, SYSTEM_BIN_DIRS, Stop, cancelled, - find_in, pkexec, process, run_as_root, swap_into_place, tree, turn, + find_in, kill_on_cancel, pkexec, process, run_as_root, swap_into_place, tree, turn, }; /// apt waits this long for another package tool (a software centre, automatic updates) to @@ -1076,6 +1111,125 @@ pub(crate) mod linux { root.queue.alone(&cancelled(stop), &mut turn(phase, JobPhase::Installing), install).unwrap_or(Err(CANCELLED.into())) } + /// Opt-in OCI migration; only an absent ref permits bundle fallback. + #[cfg(feature = "oci_migration")] + pub fn migrate_bundle_to_oci(app: &App, installed: &Installed, expected: &super::version::Version, stop: &Stop) -> Result { + if installed.source != InstallSource::Flatpak || installed.externally_managed || !app.repo.starts_with("storytold/") { + return Ok(false); + } + // Only bundle-created origins qualify; never redirect another repository's install. + if !installed.origin.as_deref().is_some_and(|origin| origin.ends_with("-origin")) { + return Ok(false); + } + let id = installed.package.as_deref().ok_or("Flatpak installation has no package ID")?; + let scope = match installed.installation.as_deref() { + Some("user") => "--user".to_string(), + Some("system") => "--system".to_string(), + Some(name) => format!("--installation={name}"), + None => return Err("Flatpak installation scope is unknown".into()), + }; + // `cancellable` is false for steps that must not stop part-way: killing flatpak-spawn + // doesn't wait for the host transaction, and `--reinstall` removes the old ref first. + let flatpak = |args: &[&str], cancellable: bool| -> Result { + let mut cmd = if process::in_sandbox() { + let mut cmd = process::command("flatpak-spawn"); + cmd.args(["--host", "--watch-bus", "flatpak"]); + cmd + } else { + process::command("flatpak") + }; + cmd.arg(&scope).args(args); + if cancellable { + stop.check()?; // paused or cancelled: start no further command + } + let end = kill_on_cancel(stop); + let output = process::cancellable_output(cmd, || if cancellable { end() } else { Ok(()) })?; + if !output.status.success() { + return Err(format!("Flatpak OCI migration failed: {}", String::from_utf8_lossy(&output.stderr).trim())); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + }; + let run = |args: &[&str]| flatpak(args, true); + let reference = run(&["info", "--show-ref", id])?; + let reference = reference.trim(); + if !reference.starts_with(&format!("app/{id}/")) || !reference.ends_with("/stable") { + return Ok(false); + } + let remotes = run(&["remotes", "--show-disabled", "--columns=name,url"])?; + let remote = remotes.lines().find_map(|line| { + let (name, _) = line.split_once('\t')?; + crate::packages::is_artcraft_remote(name, line).then(|| name.to_string()) + }); + let remote = match remote { + Some(remote) => remote, + None => { + if remotes.lines().any(|line| line.split('\t').next() == Some("artcraft-oci")) { + return Err("The artcraft-oci remote points to a different repository".into()); + } + // Another app's migration may add the remote first; the URL is checked below. + run(&["remote-add", "--if-not-exists", "artcraft-oci", "https://getartcraft.com/artcraft-oci.flatpakrepo"])?; + let remotes = run(&["remotes", "--show-disabled", "--columns=name,url"])?; + if !crate::packages::is_artcraft_remote("artcraft-oci", &remotes) { + return Err("The ArtCraft OCI descriptor configured an unexpected repository".into()); + } + "artcraft-oci".to_string() + } + }; + // The Version column comes from the remote's AppStream catalog, which Flatpak only fetches + // on demand; without a refresh a new remote lists refs with empty versions. + run(&["update", "--appstream", &remote])?; + let refs = run(&["remote-ls", "--app", "--columns=ref,version", &remote])?; + if !crate::packages::oci_ref_available(reference, expected, &refs)? { + return Ok(false); + } + stop.check()?; + if let Err(e) = flatpak(&["install", "--reinstall", "-y", "--noninteractive", &remote, reference], false) { + // The bundle origin has no repository to restore from, so point at the bundle itself. + if flatpak(&["info", id], false).is_err() { + return Err(format!("{e}. {} is no longer installed; reinstall it from the .flatpak bundle at {}", app.name, app.releases_url())); + } + return Err(e); + } + let origin = run(&["info", "--show-origin", id])?; + if origin.trim() != remote { + return Err("Flatpak did not switch the installed app to the ArtCraft OCI remote".into()); + } + Ok(true) + } + + /// Updates an installed Flatpak from its remote repository (such as FlatPark). + pub fn update_flatpak_app(app: &App, id: &str, installation: Option<&str>, stop: &Stop, phase: &mut dyn FnMut(JobPhase)) -> Result<(), String> { + phase(JobPhase::Installing); + stop.check()?; + let scope_arg = match installation { + Some("user") => "--user".to_string(), + Some("system") | None => "--system".to_string(), + Some(named) => format!("--installation={named}"), + }; + // Let Flatpak's system helper authorize system updates through polkit. + // watch-bus also terminates the host command if the sandbox proxy exits. + let mut cmd = if process::in_sandbox() { + let mut command = process::command("flatpak-spawn"); + command.args(["--host", "--watch-bus", "flatpak"]); + command + } else { + process::command("flatpak") + }; + cmd.args(["update", &scope_arg, "-y", "--noninteractive", id]); + let output = process::cancellable_output(cmd, kill_on_cancel(stop))?; + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr); + let err = stderr.trim(); + if err.is_empty() { + Err(format!("Flatpak couldn't update {}", app.name)) + } else { + let clean = err.strip_prefix("error: ").or_else(|| err.strip_prefix("Error: ")).unwrap_or(err); + Err(format!("Flatpak couldn't update {}: {clean}", app.name)) + } + } + /// Whether a bundle install of `id` failed because the installation already has the bundle's /// commit: Flatpak's "already installed", in the C locale. Any other failure leaves the app alone. pub(crate) fn already_installed(err: &str, id: &str) -> bool { @@ -2481,6 +2635,9 @@ mod tests { source: InstallSource::Msi, location: dir.path().join("MSI/PhotoCraft/photocraft.exe").display().to_string(), package: None, + externally_managed: false, + origin: None, + installation: None, }; assert_eq!(destination(app, PackageKind::Msi, Some(&msi), Some(&old), &unavailable).unwrap(), Some(dir.path().join("MSI/PhotoCraft"))); assert!(destination(app, PackageKind::TarGz, None, None, &locations).unwrap_err().contains("aren't supported")); @@ -2492,8 +2649,24 @@ mod tests { let app = App::by_id("photocraft").unwrap(); let path = dir.path().join("old.AppImage"); let rec = record("0.5.0", PackageKind::AppImage, path.clone(), vec![], false); - let launcher = Installed { version: None, source: InstallSource::Launcher, location: path.display().to_string(), package: None }; - let users = Installed { version: None, source: InstallSource::AppImage, location: "/home/me/AppImages/p.AppImage".into(), package: None }; + let launcher = Installed { + version: None, + source: InstallSource::Launcher, + location: path.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; + let users = Installed { + version: None, + source: InstallSource::AppImage, + location: "/home/me/AppImages/p.AppImage".into(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; let chosen = dir.path().join("New apps"); std::fs::create_dir(&chosen).unwrap(); let mut locations = InstallLocations { root: Some(chosen.display().to_string()), ..Default::default() }; @@ -3333,7 +3506,15 @@ esac #[test] fn updates_keep_the_installed_format() { - let inst = |source| Installed { version: None, source, location: String::new(), package: None }; + let inst = |source| Installed { + version: None, + source, + location: String::new(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; let app = crate::App::by_id("photocraft").unwrap(); let available = available_kinds(); if let Some(first) = available.first() { @@ -3350,7 +3531,15 @@ esac #[test] fn only_the_release_arch_package_is_updated_by_the_launcher() { - let pacman = |name: &str| Installed { version: None, source: InstallSource::Pacman, location: String::new(), package: Some(name.into()) }; + let pacman = |name: &str| Installed { + version: None, + source: InstallSource::Pacman, + location: String::new(), + package: Some(name.into()), + externally_managed: false, + origin: None, + installation: None, + }; let app = crate::App::by_id("wordcraft").unwrap(); assert!(is_release_arch_package(app, &pacman("ai.storyteller.wordcraft"))); for aur in ["wordcraft", "wordcraft-bin", "wordcraft-git"] { @@ -3378,7 +3567,15 @@ esac #[test] fn aur_packages_are_not_updated_as_arch_pkg() { - let pacman = |name: &str| Installed { version: None, source: InstallSource::Pacman, location: String::new(), package: Some(name.into()) }; + let pacman = |name: &str| Installed { + version: None, + source: InstallSource::Pacman, + location: String::new(), + package: Some(name.into()), + externally_managed: false, + origin: None, + installation: None, + }; let app = crate::App::by_id("filmcraft").unwrap(); let installed = pacman("filmcraft-bin"); assert!(managed_elsewhere_with(app, &installed, false, |_| None).is_some()); @@ -3388,7 +3585,15 @@ esac #[test] fn release_arch_packages_are_updated_as_arch_pkg() { - let pacman = |name: &str| Installed { version: None, source: InstallSource::Pacman, location: String::new(), package: Some(name.into()) }; + let pacman = |name: &str| Installed { + version: None, + source: InstallSource::Pacman, + location: String::new(), + package: Some(name.into()), + externally_managed: false, + origin: None, + installation: None, + }; let app = crate::App::by_id("filmcraft").unwrap(); let installed = pacman("ai.storyteller.filmcraft"); assert_eq!(managed_elsewhere_with(app, &installed, false, |_| None), None); @@ -3396,6 +3601,102 @@ esac assert_eq!(choose_kinds_with(app, Some(&installed), None, &[], &[PackageKind::ArchPkg], false), vec![PackageKind::ArchPkg]); } + #[test] + fn flatpark_flatpaks_are_externally_managed_and_not_updated_via_bundle() { + let app = crate::App::by_id("photocraft").unwrap(); + let flatpark_installed = Installed { + version: version::parse("0.19.0"), + source: InstallSource::Flatpak, + location: "ai.storyteller.photocraft".into(), + package: Some("ai.storyteller.photocraft".into()), + externally_managed: true, + origin: Some("flatpark".into()), + installation: Some("user".into()), + }; + assert!(flatpark_installed.is_externally_managed()); + assert_eq!(choose_kinds_with(app, Some(&flatpark_installed), None, &[], &[PackageKind::Flatpak], false), Vec::::new()); + let hint = managed_elsewhere_with(app, &flatpark_installed, false, |_| None); + assert_eq!( + hint.as_deref(), + Some("PhotoCraft was installed from a Flatpak repository. Update it with flatpak update --user ai.storyteller.photocraft") + ); + // During uninstall, managed_elsewhere returns None so it can be removed + assert_eq!(managed_elsewhere_with(app, &flatpark_installed, true, |_| None), None); + + // When installed under a former bundle ID, the hint targets that detected ID + let former_installed = Installed { + version: version::parse("0.19.0"), + source: InstallSource::Flatpak, + location: "ai.storyteller.photocraft-old".into(), + package: Some("ai.storyteller.photocraft-old".into()), + externally_managed: true, + origin: Some("flatpark".into()), + installation: Some("system".into()), + }; + let hint_former = managed_elsewhere_with(app, &former_installed, false, |_| None); + assert_eq!( + hint_former.as_deref(), + Some("PhotoCraft was installed from a Flatpak repository. Update it with flatpak update --system ai.storyteller.photocraft-old") + ); + + let bundle_installed = Installed { + version: version::parse("0.19.0"), + source: InstallSource::Flatpak, + location: "ai.storyteller.photocraft".into(), + package: Some("ai.storyteller.photocraft".into()), + externally_managed: false, + origin: Some("photocraft-origin".into()), + installation: Some("user".into()), + }; + assert!(!bundle_installed.is_externally_managed()); + assert_eq!(choose_kinds_with(app, Some(&bundle_installed), None, &[], &[PackageKind::Flatpak], false), vec![PackageKind::Flatpak]); + assert_eq!(managed_elsewhere_with(app, &bundle_installed, false, |_| None), None); + } + + /// A copy in a named Flatpak installation is addressed through that installation, not the + /// default system scope. + #[test] + fn a_named_flatpak_installation_is_addressed_by_name() { + let app = crate::App::by_id("photocraft").unwrap(); + let installed = Installed { + version: version::parse("0.19.0"), + source: InstallSource::Flatpak, + location: "ai.storyteller.photocraft".into(), + package: Some("ai.storyteller.photocraft".into()), + externally_managed: true, + origin: Some("flatpark".into()), + installation: Some("work".into()), + }; + assert_eq!( + managed_elsewhere_with(app, &installed, false, |_| None).as_deref(), + Some("PhotoCraft was installed from a Flatpak repository. Update it with flatpak update --installation=work ai.storyteller.photocraft") + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn update_flatpak_app_respects_cancellation() { + let app = crate::App::by_id("photocraft").unwrap(); + let stop = Stop::default(); + stop.cancel(); + let mut phase = |_| (); + assert_eq!(linux::update_flatpak_app(app, "ai.storyteller.photocraft", Some("user"), &stop, &mut phase), Err(CANCELLED.into())); + } + + #[test] + #[cfg(target_os = "linux")] + fn pause_lets_a_running_flatpak_command_finish_and_only_cancel_ends_it() { + let stop = Stop::default(); + stop.pause(); + assert_eq!(kill_on_cancel(&stop)(), Ok(())); + let app = crate::App::by_id("photocraft").unwrap(); + let mut phase = |_| (); + // A paused job starts no further command. + assert_eq!(linux::update_flatpak_app(app, "ai.storyteller.photocraft", Some("user"), &stop, &mut phase), Err(crate::protocol::PAUSED.into())); + stop.cancel(); + assert_eq!(kill_on_cancel(&stop)(), Err(CANCELLED.into())); + } + #[test] fn unavailable_arch_package_not_selected_for_new_install() { let app = crate::App::by_id("filmcraft").unwrap(); @@ -3419,6 +3720,9 @@ esac source: InstallSource::Launcher, location: dir.path().join("PhotoCraft.AppImage").display().to_string(), package: None, + externally_managed: false, + origin: None, + installation: None, }; let stop = Stop::default(); let mut job = Job { http: &http, paths: &paths, stop: &stop, phase, install_locations: &locations, replace: false }; @@ -3595,8 +3899,15 @@ esac let old = dir.path().join("Apps").join("PhotoCraft.AppImage"); std::fs::create_dir_all(old.parent().unwrap()).unwrap(); std::fs::write(&old, b"the installed 0.4.0").unwrap(); - let installed = - Installed { version: version::parse("0.4.0"), source: InstallSource::Launcher, location: old.display().to_string(), package: None }; + let installed = Installed { + externally_managed: false, + origin: None, + installation: None, + version: version::parse("0.4.0"), + source: InstallSource::Launcher, + location: old.display().to_string(), + package: None, + }; let previous = record("0.4.0", PackageKind::AppImage, old.clone(), vec![], false); let mut phases = Vec::new(); let mut phase = |p: JobPhase| phases.push(p); @@ -3637,7 +3948,15 @@ esac #[cfg(unix)] let _probe = crate::process::tests::RunningProbe::start(&location); assert!(process::is_running(Path::new(&location)), "{location} should be seen as running"); - let installed = Installed { version: version::parse("0.4.0"), source: InstallSource::Launcher, location, package: None }; + let installed = Installed { + externally_managed: false, + origin: None, + installation: None, + version: version::parse("0.4.0"), + source: InstallSource::Launcher, + location, + package: None, + }; let stop = Stop::default(); stop.cancel(); let mut job = Job { http: &http, paths: &paths, stop: &stop, phase, install_locations: &locations, replace: false }; @@ -3881,7 +4200,15 @@ esac } fn found_at(bundle: &Path) -> Installed { - Installed { version: None, source: InstallSource::MacApp, location: bundle.display().to_string(), package: None } + Installed { + version: None, + source: InstallSource::MacApp, + location: bundle.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + } } /// What [`check`] says when `codesign` fails with `failure`. @@ -4062,7 +4389,15 @@ esac let dir = tempfile::tempdir().unwrap(); let apps = dir.path().join("Applications"); let current = bundle(&apps, "0.5.0"); - let existing = Installed { version: None, source: InstallSource::MacApp, location: current.display().to_string(), package: None }; + let existing = Installed { + version: None, + source: InstallSource::MacApp, + location: current.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; let paths = Paths::under(dir.path()); let image = dmg(dir.path(), "0.6.0"); @@ -4095,7 +4430,15 @@ esac let dir = tempfile::tempdir().unwrap(); let apps = dir.path().join("Applications"); let current = bundle(&apps, "0.5.0"); - let existing = Installed { version: None, source: InstallSource::MacApp, location: current.display().to_string(), package: None }; + let existing = Installed { + version: None, + source: InstallSource::MacApp, + location: current.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; let image = dmg(dir.path(), "0.6.0"); let release = Release { tag: "v0.6.0".into(), version: version::parse("0.6.0"), ..Release::default() }; @@ -4125,7 +4468,15 @@ esac let dir = tempfile::tempdir().unwrap(); let apps = dir.path().join("Applications"); let current = bundle(&apps, "0.5.0"); - let existing = Installed { version: None, source: InstallSource::MacApp, location: current.display().to_string(), package: None }; + let existing = Installed { + version: None, + source: InstallSource::MacApp, + location: current.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; let paths = Paths::under(dir.path()); let image = dmg(dir.path(), "0.6.0"); diff --git a/crates/update/src/packages.rs b/crates/update/src/packages.rs index 1820c2f15..72c93a4b0 100644 --- a/crates/update/src/packages.rs +++ b/crates/update/src/packages.rs @@ -13,6 +13,12 @@ pub struct Package { pub name: String, /// As the package manager reports it (`0.5.0-1`); empty when it doesn't say. pub version: String, + /// Whether this is a user installation (`true`) or system installation (`false`). + pub user: bool, + /// The origin remote (`flatpark`, `-origin`, `flathub`), when known. + pub origin: Option, + /// The installation scope or name (`user`, `system`, or a named installation), for Flatpak installs. + pub installation: Option, } /// `dpkg-query -W -f '${Package}|${db:Status-Status}|${Version}\n'`: installed packages only. @@ -46,18 +52,35 @@ pub fn parse_pacman(text: &str) -> Vec<(String, String)> { .collect() } -/// `flatpak list --app --columns=application,version`: the version column is often empty for -/// bundles built from a release tarball. -pub fn parse_flatpak_list(text: &str) -> Vec<(String, String)> { +/// `flatpak list --app --columns=application,version,origin,installation`: parses id, version, origin, and installation. +pub fn parse_flatpak_list_with_origin(text: &str) -> Vec<(String, String, Option, Option)> { text.lines() .filter_map(|line| { let mut f = line.split('\t'); let id = f.next()?.trim(); - (!id.is_empty()).then(|| (id.to_string(), f.next().unwrap_or("").trim().to_string())) + if id.is_empty() { + return None; + } + let version = f.next().unwrap_or("").trim().to_string(); + let origin = f.next().map(str::trim).filter(|o| !o.is_empty()).map(ToString::to_string); + let installation = f.next().map(str::trim).filter(|i| !i.is_empty()).map(ToString::to_string); + Some((id.to_string(), version, origin, installation)) }) .collect() } +/// `flatpak list --app --columns=application,version`: the version column is often empty for +/// bundles built from a release tarball. +pub fn parse_flatpak_list(text: &str) -> Vec<(String, String)> { + parse_flatpak_list_with_origin(text).into_iter().map(|(id, v, _, _)| (id, v)).collect() +} + +/// Whether an origin string indicates an install from FlatPark. +pub fn is_flatpark_origin(origin: &str) -> bool { + let o = origin.trim(); + o.eq_ignore_ascii_case("flatpark") || o.to_ascii_lowercase().contains("flatpark") +} + /// The `Version:` line of `flatpak info `. pub fn parse_flatpak_info(text: &str) -> Option { text.lines().find_map(|l| l.trim().strip_prefix("Version:").map(|v| v.trim().to_string())).filter(|v| !v.is_empty()) @@ -78,11 +101,14 @@ pub fn candidate_names(app: &App, manager: InstallSource) -> Vec { } } -/// The package of `app` among `installed`, preferring the manager listed first in `order`. +/// The package of `app` among `installed`, preferring the manager listed first in `order`. A user +/// install beats a system one under any of the app's ids; within a scope the current id wins. pub fn find<'a>(app: &App, installed: &'a [Package], order: &[InstallSource]) -> Option<&'a Package> { order.iter().find_map(|manager| { let names = candidate_names(app, *manager); - names.iter().find_map(|n| installed.iter().find(|p| p.manager == *manager && p.name == *n)) + [true, false] + .into_iter() + .find_map(|user| names.iter().find_map(|n| installed.iter().find(|p| p.manager == *manager && p.user == user && p.name == *n))) }) } @@ -96,7 +122,7 @@ mod tests { use super::*; fn pkgs(manager: InstallSource, list: Vec<(String, String)>) -> Vec { - list.into_iter().map(|(name, version)| Package { manager, name, version }).collect() + list.into_iter().map(|(name, version)| Package { manager, name, version, user: false, origin: None, installation: None }).collect() } #[test] @@ -111,6 +137,8 @@ mod tests { parse_flatpak_list("ai.storyteller.wordcraft\t\nai.storyteller.cadcraft\t0.3.0\n"), [("ai.storyteller.wordcraft".into(), String::new()), ("ai.storyteller.cadcraft".into(), "0.3.0".into())] ); + // A row with no id is not a package listing. + assert!(parse_flatpak_list_with_origin("\t0.2.0\tflatpark\tuser\n").is_empty()); let info = "\nWordCraft - Word processor\n\n ID: ai.storyteller.wordcraft\n Version: 0.3.0\n Branch: master\n"; assert_eq!(parse_flatpak_info(info).as_deref(), Some("0.3.0")); assert_eq!(parse_flatpak_info("Version: \n"), None); @@ -158,4 +186,143 @@ mod tests { assert_eq!(version_of(hit).unwrap().to_string(), "0.5.0"); assert_eq!(find(photo, &installed, &[InstallSource::Flatpak]).unwrap().version, "0.4.0"); } + + #[test] + fn parses_flatpak_list_with_origin_and_spots_flatpark() { + let text = "ai.storyteller.photocraft\t0.19.0\tflatpark\tuser\nai.storyteller.cadcraft\t0.16.0\tcadcraft-origin\tsystem\nai.storyteller.wordcraft\t0.14.0\t\t\n"; + let list = parse_flatpak_list_with_origin(text); + assert_eq!( + list, + [ + ("ai.storyteller.photocraft".into(), "0.19.0".into(), Some("flatpark".into()), Some("user".into())), + ("ai.storyteller.cadcraft".into(), "0.16.0".into(), Some("cadcraft-origin".into()), Some("system".into())), + ("ai.storyteller.wordcraft".into(), "0.14.0".into(), None, None), + ] + ); + assert!(is_flatpark_origin("flatpark")); + assert!(is_flatpark_origin("FlatPark")); + assert!(is_flatpark_origin("flatpark-origin")); + assert!(!is_flatpark_origin("cadcraft-origin")); + assert!(!is_flatpark_origin("flathub")); + assert!(!is_flatpark_origin("")); + } + + #[test] + fn flatpak_find_prioritizes_user_over_system() { + let photo = App::by_id("photocraft").unwrap(); + let installed = vec![ + Package { + manager: InstallSource::Flatpak, + name: "ai.storyteller.photocraft".into(), + version: "0.18.0".into(), + user: false, + origin: Some("flathub".into()), + installation: Some("system".into()), + }, + Package { + manager: InstallSource::Flatpak, + name: "ai.storyteller.photocraft".into(), + version: "0.19.0".into(), + user: true, + origin: Some("flatpark".into()), + installation: Some("user".into()), + }, + ]; + let found = find(photo, &installed, &[InstallSource::Flatpak]).unwrap(); + assert!(found.user); + assert_eq!(found.version, "0.19.0"); + assert_eq!(found.origin.as_deref(), Some("flatpark")); + assert_eq!(found.installation.as_deref(), Some("user")); + } +} + +/// Matches the OCI URL from https://getartcraft.com/artcraft-oci.flatpakrepo, +/// independently of its local remote name (normally `artcraft-oci`). +pub fn is_artcraft_remote(origin: &str, remotes: &str) -> bool { + remotes.lines().any(|line| { + let Some((name, url)) = line.split_once('\t') else { return false }; + let url = url.trim().trim_end_matches('/'); + name.trim() == origin && url == "oci+https://getartcraft.com" + }) +} + +#[cfg(test)] +mod remote_url_tests { + use super::*; + #[test] + fn recognizes_custom_names_and_rejects_unrelated_urls() { + assert!(is_artcraft_remote("artcraft-oci", "artcraft-oci\toci+https://getartcraft.com\n")); + assert!(is_artcraft_remote("custom", "custom\toci+https://getartcraft.com/\n")); + for url in [ + "oci+https://getartcraft.com.evil", + "oci+https://getartcraft.com/other", + "oci+http://getartcraft.com", + "https://getartcraft.com/linux", + "https://getartcraft.com/artcraft-oci.flatpakrepo", + "https://getartcraft.com", + ] { + assert!(!is_artcraft_remote("custom", &format!("custom\t{url}"))); + } + assert!(!is_artcraft_remote("other", "custom\toci+https://getartcraft.com")); + } +} + +/// A missing ref permits bundle fallback; an unknown or different version must not replace it. +#[cfg(all(target_os = "linux", feature = "oci_migration"))] +pub(crate) fn oci_ref_available(reference: &str, expected: &version::Version, listing: &str) -> Result { + for line in listing.lines() { + let mut fields = line.split('\t'); + if fields.next().map(str::trim) != Some(reference) { + continue; + } + let candidate = fields.next().and_then(|value| version::parse(value.trim())); + if candidate.as_ref() != Some(expected) { + return Err("The ArtCraft OCI repository does not yet provide the selected version".into()); + } + return Ok(true); + } + Ok(false) +} + +#[cfg(all(test, target_os = "linux", feature = "oci_migration"))] +mod migration_tests { + use super::*; + #[test] + fn migration_requires_the_exact_ref_and_selected_version() { + let reference = "app/ai.storyteller.photocraft/x86_64/stable"; + let expected = version::parse("1.2.3").unwrap(); + assert_eq!(oci_ref_available(reference, &expected, &format!("{reference}\t1.2.3")), Ok(true)); + assert_eq!(oci_ref_available(reference, &expected, "app/ai.storyteller.photocraft/aarch64/stable\t1.2.3"), Ok(false)); + for value in ["", "1.2.2", "1.2.4", "unknown"] { + assert!(oci_ref_available(reference, &expected, &format!("{reference}\t{value}")).is_err()); + } + } +} + +/// Only a confirmed bundle origin with no repository URL permits bundle replacement. +/// Missing rows are unknown, so preserve remote ownership conservatively. +pub fn remote_manages_flatpak(origin: &str, remotes: &str) -> bool { + remotes + .lines() + .find_map(|line| { + let mut fields = line.split('\t'); + if fields.next().map(str::trim) != Some(origin) { + return None; + } + Some(fields.next().is_some_and(|url| !url.trim().is_empty())) + }) + .unwrap_or(true) +} + +#[cfg(test)] +mod remote_ownership_tests { + use super::*; + #[test] + fn only_confirmed_empty_urls_allow_bundle_replacement() { + assert!(!remote_manages_flatpak("app-origin", "app-origin\t\n")); + assert!(remote_manages_flatpak("renamed", "renamed\thttps://example.org/repo\n")); + assert!(remote_manages_flatpak("renamed", "renamed\toci+https://getartcraft.com\n")); + assert!(remote_manages_flatpak("missing", "other\t\n")); + assert!(!remote_manages_flatpak("app-origin", "app-origin")); + } } diff --git a/crates/update/src/process.rs b/crates/update/src/process.rs index 13ed8dda4..c20c3bc45 100644 --- a/crates/update/src/process.rs +++ b/crates/update/src/process.rs @@ -839,3 +839,81 @@ mod inherited_pipe_tests { } } } + +/// Runs a helper while polling cancellation and draining its bounded output. +#[cfg(target_os = "linux")] +pub(crate) fn cancellable_output(mut cmd: Command, check: impl Fn() -> Result<(), String>) -> Result { + check()?; + let mut child = + cmd.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().map_err(|e| format!("couldn't run flatpak ({e})"))?; + let stdout = read_all(child.stdout.take()).map_err(|e| format!("couldn't read flatpak output ({e})"))?; + let stderr = read_all(child.stderr.take()).map_err(|e| format!("couldn't read flatpak output ({e})"))?; + let started = Instant::now(); + let status = loop { + if started.elapsed() >= Duration::from_secs(60 * 60) { + let _ = child.kill(); + let _ = child.wait(); + return Err("Flatpak did not finish within one hour".into()); + } + if let Err(e) = check() { + let _ = child.kill(); + let _ = child.wait(); + return Err(e); + } + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) => std::thread::sleep(Duration::from_millis(25)), + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!("couldn't wait for flatpak ({e})")); + } + } + }; + let deadline = Instant::now(); + while !stdout.is_finished() || !stderr.is_finished() { + check()?; + if deadline.elapsed() > Duration::from_secs(5) { + return Err("Flatpak output did not close after completion".into()); + } + std::thread::sleep(Duration::from_millis(25)); + } + Ok(std::process::Output { status, stdout: stdout.join().unwrap_or_default(), stderr: stderr.join().unwrap_or_default() }) +} + +#[cfg(all(test, target_os = "linux"))] +mod update_cancel_tests { + use super::*; + + #[test] + fn cancels_a_running_helper_promptly() { + let mut cmd = command("sleep"); + cmd.arg("30"); + let start = Instant::now(); + let result = cancellable_output(cmd, || if start.elapsed() >= Duration::from_millis(100) { Err("cancelled".into()) } else { Ok(()) }); + assert_eq!(result.unwrap_err(), "cancelled"); + assert!(start.elapsed() < Duration::from_secs(5)); + } + + /// A descendant that outlives the helper keeps the pipes open; the caller gets an error at + /// the deadline instead of waiting for it. + #[test] + fn output_held_open_by_a_descendant_fails_at_the_deadline() { + let mut cmd = command("sh"); + cmd.args(["-c", "sleep 8 &"]); + let start = Instant::now(); + let err = cancellable_output(cmd, || Ok(())).expect_err("a descendant holding the pipes should time out"); + assert!(err.contains("did not close after completion"), "{err}"); + assert!(start.elapsed() < Duration::from_secs(8), "{:?}", start.elapsed()); + } + + #[test] + fn retains_both_output_streams_and_exit_status() { + let mut cmd = command("sh"); + cmd.args(["-c", "printf output; printf error >&2; exit 7"]); + let output = cancellable_output(cmd, || Ok(())).unwrap(); + assert_eq!(output.status.code(), Some(7)); + assert_eq!(output.stdout, b"output"); + assert_eq!(output.stderr, b"error"); + } +} diff --git a/crates/update/src/self_update.rs b/crates/update/src/self_update.rs index a20f466ce..f96c3af63 100644 --- a/crates/update/src/self_update.rs +++ b/crates/update/src/self_update.rs @@ -878,7 +878,15 @@ pub fn prepare_update_with_options( match install { #[cfg(target_os = "macos")] Install::MacApp(bundle) => { - let existing = Installed { version: None, source: InstallSource::MacApp, location: bundle.display().to_string(), package: None }; + let existing = Installed { + version: None, + source: InstallSource::MacApp, + location: bundle.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; let wait = install::macos::CheckTurn { stop, phase: &mut *phase }; let placed = install::macos::install_dmg(app, &file, Some(&existing), None, false, paths, &install::macos::Expect::self_update(release), wait); @@ -3535,7 +3543,15 @@ mod tests { assert!(ok, "hdiutil create failed"); drop(one_at_a_time); let current = apps.join("ArtCraft Launcher.app"); - let existing = Installed { version: None, source: InstallSource::MacApp, location: current.display().to_string(), package: None }; + let existing = Installed { + version: None, + source: InstallSource::MacApp, + location: current.display().to_string(), + package: None, + externally_managed: false, + origin: None, + installation: None, + }; // An unsigned test bundle: the signature check has its own tests in `install`. let expect = install::macos::Expect { team: None, version: None, quit_first: false }; let stop = Stop::default(); diff --git a/crates/update/src/status.rs b/crates/update/src/status.rs index 1cd0cf76c..cc985df0a 100644 --- a/crates/update/src/status.rs +++ b/crates/update/src/status.rs @@ -61,4 +61,21 @@ pub struct Installed { /// The package that installed it, for package-manager installs (`photocraft-bin`). #[serde(default)] pub package: Option, + /// Whether this copy is managed externally (such as a Flatpak from FlatPark) and + /// will not be replaced with a downloaded bundle. + #[serde(default)] + pub externally_managed: bool, + /// The remote repository or origin (`flatpark`, `-origin`), when known. + #[serde(default)] + pub origin: Option, + /// The installation scope or name (`user`, `system`, or a named installation), for Flatpak installs. + #[serde(default)] + pub installation: Option, +} + +impl Installed { + /// Whether this copy is managed externally and should not be replaced by a downloaded bundle. + pub fn is_externally_managed(&self) -> bool { + self.externally_managed + } } From a9efd722b49924588448fba5c59f36d3aee3f4c6 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 17:56:38 -0500 Subject: [PATCH 106/209] Expose bounded BMP header recognition for consumer cutovers (#1929) * Expose supported BMP header recognition Port the BMP routing check from LightCraft commit 304180a500d1b3f87c8c841f269d305161d6c8b9, preserving minimum length and DIB allowlist. * Test BMP routing boundaries and unsupported DIB sizes --------- Co-authored-by: Brandon Thomas --- crates/image-codecs/src/bmp_probe.rs | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crates/image-codecs/src/bmp_probe.rs b/crates/image-codecs/src/bmp_probe.rs index 59c78ca1e..5b47721bc 100644 --- a/crates/image-codecs/src/bmp_probe.rs +++ b/crates/image-codecs/src/bmp_probe.rs @@ -1,5 +1,18 @@ //! Header-only BMP dimensions and pixel-data coverage, without allocating a canvas. +/// Recognize a BMP file header with a supported DIB header kind. +/// +/// This is a format-routing check, not full validation: dimensions and pixel data may still +/// be invalid. Unlike the broad signature detector, it requires the minimum 26-byte core +/// header and a known DIB size, preserving callers that reject arbitrary `BM` prefixes. +pub fn is_supported_header(b: &[u8]) -> bool { + if b.len() >= 26 && b.starts_with(b"BM") { + let dib = u32::from_le_bytes([b[14], b[15], b[16], b[17]]); + return matches!(dib, 12 | 16 | 40 | 52 | 56 | 64 | 108 | 124); + } + false +} + /// Coverage of the declared pixel array. Row byte counts include row padding; RLE coverage /// counts the pixels drawn by commands and may be smaller than the canvas. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -142,4 +155,23 @@ mod tests { let nibble = bmp(2, &[0, 3, 0x12, 0x30, 0, 1]); assert_eq!(coverage(&nibble, 4, 2).pixels, 3); } + #[test] + fn supported_header_recognition_preserves_length_signature_and_dib_rules() { + let mut b = [0_u8; 26]; + b[..2].copy_from_slice(b"BM"); + for dib in [12_u32, 16, 40, 52, 56, 64, 108, 124] { + b[14..18].copy_from_slice(&dib.to_le_bytes()); + assert!(is_supported_header(&b), "DIB {dib}"); + for n in 0..26 { + assert!(!is_supported_header(&b[..n]), "prefix {n}"); + } + } + for dib in [0_u32, 13, 39, 41, 125, u32::MAX] { + b[14..18].copy_from_slice(&dib.to_le_bytes()); + assert!(!is_supported_header(&b), "DIB {dib}"); + } + b[14..18].copy_from_slice(&40_u32.to_le_bytes()); + b[0] = b'X'; + assert!(!is_supported_header(&b)); + } } From be0b1e655229e34f1ad263ef3d2170e46449c39b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 18:22:56 -0500 Subject: [PATCH 107/209] cover the Flatpak repository guard and its update command (#1956) The Flatpak/OCI migration left update_flatpak_app and the refusal to replace a repository-managed Flatpak without tests. A stand-in flatpak on PATH, in child processes so the environment stays private, covers every scope flag, both command shapes, a successful update and Flatpak's error text; the guard test asserts the refusal happens before anything is downloaded. --- crates/update/src/install.rs | 111 +++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/crates/update/src/install.rs b/crates/update/src/install.rs index 071434595..3b3513294 100644 --- a/crates/update/src/install.rs +++ b/crates/update/src/install.rs @@ -3673,6 +3673,117 @@ esac ); } + /// A Flatpak the user installed from a repository is never replaced by the launcher's bundle: + /// the refusal names the repository's own update command, and nothing is downloaded. + #[test] + #[cfg(target_os = "linux")] + fn a_repository_flatpak_is_not_replaced_by_a_bundle_install() { + let app = crate::App::by_id("photocraft").unwrap(); + let dir = tempfile::tempdir().unwrap(); + let paths = Paths::under(dir.path()); + let http = Http::new(crate::TEST_AGENT).unwrap(); + let mut phases = Vec::new(); + let mut phase = |p: JobPhase| phases.push(p); + let mut job = Job { + http: &http, + paths: &paths, + stop: &Stop::default(), + phase: &mut phase, + install_locations: &InstallLocations::default(), + replace: false, + }; + let release = Release { version: version::parse("0.20.0"), tag: "v0.20.0".into(), ..Release::default() }; + let asset = Asset { name: "photocraft-0.20.0-linux-x86_64.flatpak".into(), ..Asset::default() }; + let installed = Installed { + version: version::parse("0.19.0"), + source: InstallSource::Flatpak, + location: app.bundle_id(), + package: Some(app.bundle_id()), + externally_managed: true, + origin: Some("flatpark".into()), + installation: Some("work".into()), + }; + let error = install(app, &release, &asset, PackageKind::Flatpak, Some(&installed), None, &mut job).unwrap_err(); + assert_eq!( + error, + "PhotoCraft was installed from a Flatpak repository. Update it with flatpak update --installation=work ai.storyteller.photocraft" + ); + assert!(phases.is_empty(), "the bundle installer started for a Flatpak the user manages"); + assert!(!paths.cache.exists(), "the refusal must come before anything is downloaded"); + } + + /// `update_flatpak_app` drives the user's own copy: the scope flag for the installation it sits + /// in, a successful update, and Flatpak's error text with its `error: ` prefix removed. A + /// stand-in `flatpak` on PATH reports what it was asked to run; the test runs in child + /// processes so that PATH stays private to it (as `updater`'s pretend-version test does). + #[test] + #[cfg(target_os = "linux")] + fn a_flatpak_from_a_repository_is_updated_through_its_own_installation() { + const NAME: &str = "install::tests::a_flatpak_from_a_repository_is_updated_through_its_own_installation"; + const DIR: &str = "CRAFT_UPDATE_FAKE_FLATPAK_DIR"; + if std::env::var_os(DIR).is_none() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let script = r#"#!/bin/sh +echo "$(basename "$0") $*" >> "$CRAFT_UPDATE_FAKE_FLATPAK_CALLS" +case "$*" in +*denied.app*) echo "error: Permission denied" >&2; exit 1 ;; +*diskfull.app*) echo "Error: Disk full" >&2; exit 1 ;; +*silent.app*) exit 1 ;; +esac +exit 0 +"#; + for program in ["flatpak", "flatpak-spawn"] { + let path = dir.path().join(program); + std::fs::write(&path, script).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let path = format!("{}:{}", dir.path().display(), std::env::var("PATH").unwrap_or_default()); + for sandboxed in [false, true] { + let calls = dir.path().join(format!("calls-{sandboxed}")); + let mut cmd = std::process::Command::new(std::env::current_exe().unwrap()); + cmd.args(["--exact", NAME, "--nocapture"]) + .env(DIR, dir.path()) + .env("CRAFT_UPDATE_FAKE_FLATPAK_CALLS", &calls) + .env("PATH", &path) + .env_remove("FLATPAK_ID"); + if sandboxed { + cmd.env("FLATPAK_ID", "ai.storyteller.photocraft"); + } + let status = cmd.status().unwrap(); + assert!(status.success(), "the sandboxed={sandboxed} run failed"); + } + return; + } + let calls = std::env::var_os("CRAFT_UPDATE_FAKE_FLATPAK_CALLS").map(PathBuf::from).unwrap(); + let app = crate::App::by_id("photocraft").unwrap(); + let stop = Stop::default(); + let mut phase = |_| (); + let id = app.bundle_id(); + assert_eq!(linux::update_flatpak_app(app, &id, None, &stop, &mut phase), Ok(())); + assert_eq!( + linux::update_flatpak_app(app, "denied.app", Some("user"), &stop, &mut phase), + Err("Flatpak couldn't update PhotoCraft: Permission denied".to_string()) + ); + assert_eq!( + linux::update_flatpak_app(app, "diskfull.app", Some("work"), &stop, &mut phase), + Err("Flatpak couldn't update PhotoCraft: Disk full".to_string()) + ); + assert_eq!( + linux::update_flatpak_app(app, "silent.app", Some("system"), &stop, &mut phase), + Err("Flatpak couldn't update PhotoCraft".to_string()) + ); + let prefix = if process::in_sandbox() { "flatpak-spawn --host --watch-bus flatpak" } else { "flatpak" }; + let expected: Vec = vec![ + format!("{prefix} update --system -y --noninteractive {id}"), + format!("{prefix} update --user -y --noninteractive denied.app"), + format!("{prefix} update --installation=work -y --noninteractive diskfull.app"), + format!("{prefix} update --system -y --noninteractive silent.app"), + ]; + let calls = std::fs::read_to_string(&calls).unwrap(); + assert_eq!(calls.lines().map(str::to_string).collect::>(), expected); + } + #[test] #[cfg(target_os = "linux")] fn update_flatpak_app_respects_cancellation() { From 12db39ca963d2a333bd1891b0dbba65218800fd8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 108/209] Extract score writer changes from SoundCraft 0751dd31 --- crates/score/src/lib.rs | 344 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 crates/score/src/lib.rs diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs new file mode 100644 index 000000000..30375b652 --- /dev/null +++ b/crates/score/src/lib.rs @@ -0,0 +1,344 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! MusicXML and SVG score writing over application-independent notation data. + +use std::fmt::Write as _; + +#[derive(Debug, Clone, PartialEq)] +pub struct ScoreSetup { + pub title: String, + pub composer: String, + pub bars_per_system: u32, + pub show_track_names: bool, +} + +/// One measure: its meter and its items (notes, chords or rests), in sixteenths. +#[derive(Debug, Clone, PartialEq)] +pub struct Measure { + pub numerator: u32, + pub denominator: u32, + pub items: Vec, +} + +/// A notehead group (chord) or a rest (no pitches). +#[derive(Debug, Clone, PartialEq)] +pub struct Item { + pub pitches: Vec, + /// Duration in sixteenths; always one of the supported duration values. + pub units: u32, + pub tie_start: bool, + pub tie_stop: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Part { + pub name: String, + /// True when the part reads best in bass clef. + pub bass: bool, + pub measures: Vec, +} + +fn spell(pitch: u8) -> (char, i32, i32) { + const STEPS: [(char, i32); 12] = + [('C', 0), ('C', 1), ('D', 0), ('D', 1), ('E', 0), ('F', 0), ('F', 1), ('G', 0), ('G', 1), ('A', 0), ('A', 1), ('B', 0)]; + let (step, alter) = STEPS[usize::from(pitch % 12)]; + (step, alter, i32::from(pitch / 12) - 1) +} + +fn xml_escape(s: &str) -> String { + s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """) +} + +fn note_type(units: u32) -> (&'static str, bool) { + match units { + 16 => ("whole", false), + 12 => ("half", true), + 8 => ("half", false), + 6 => ("quarter", true), + 4 => ("quarter", false), + 3 => ("eighth", true), + 2 => ("eighth", false), + _ => ("16th", false), + } +} + +/// MusicXML 4.0 (partwise) for `parts`. +pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> String { + let mut x = String::new(); + x.push_str("\n"); + x.push_str("\n"); + x.push_str("\n"); + let _ = writeln!(x, " {}", xml_escape(&setup.title)); + x.push_str(" \n"); + if !setup.composer.is_empty() { + let _ = writeln!(x, " {}", xml_escape(&setup.composer)); + } + x.push_str(" SoundCraft\n \n \n"); + for (i, p) in parts.iter().enumerate() { + let _ = writeln!(x, " {}", i + 1, xml_escape(&p.name)); + } + x.push_str(" \n"); + for (i, p) in parts.iter().enumerate() { + let _ = writeln!(x, " ", i + 1); + let mut meter = (0, 0); + for (mi, m) in p.measures.iter().enumerate() { + let _ = writeln!(x, " ", mi + 1); + if mi == 0 || meter != (m.numerator, m.denominator) { + x.push_str(" "); + if mi == 0 { + x.push_str("40"); + } + let _ = write!(x, "", m.numerator, m.denominator); + if mi == 0 { + x.push_str(if p.bass { "F4" } else { "G2" }); + } + x.push_str("\n"); + meter = (m.numerator, m.denominator); + } + for it in &m.items { + let (ty, dot) = note_type(it.units); + if it.pitches.is_empty() { + let _ = writeln!( + x, + " {}1{ty}{}", + it.units, + if dot { "" } else { "" } + ); + continue; + } + for (k, pitch) in it.pitches.iter().enumerate() { + let (step, alter, octave) = spell(*pitch); + x.push_str(" "); + if k > 0 { + x.push_str(""); + } + let _ = write!(x, "{step}"); + if alter != 0 { + let _ = write!(x, "{alter}"); + } + let _ = write!(x, "{octave}{}", it.units); + if it.tie_stop { + x.push_str(""); + } + if it.tie_start { + x.push_str(""); + } + let _ = write!(x, "1{ty}"); + if dot { + x.push_str(""); + } + if alter != 0 { + x.push_str("sharp"); + } + if it.tie_stop || it.tie_start { + x.push_str(""); + if it.tie_stop { + x.push_str(""); + } + if it.tie_start { + x.push_str(""); + } + x.push_str(""); + } + x.push_str("\n"); + } + } + x.push_str(" \n"); + } + x.push_str(" \n"); + } + x.push_str("\n"); + x +} + +/// Staff position of `pitch` in half-spaces above the bottom line (treble E4 / bass G2). +fn staff_step(pitch: u8, bass: bool) -> i32 { + let (step, _, octave) = spell(pitch); + let idx = "CDEFGAB".find(step).map_or(0, |i| i as i32); + let diatonic = octave * 7 + idx; + diatonic - if bass { 2 * 7 + 4 } else { 4 * 7 + 2 } +} + +/// An engraved score as a printable SVG page (A4 portrait width, as tall as needed). +pub fn svg(parts: &[Part], setup: &ScoreSetup) -> String { + const W: f64 = 794.0; + const MARGIN: f64 = 48.0; + const SP: f64 = 7.0; // staff space + const STAFF_GAP: f64 = 70.0; + const SYSTEM_GAP: f64 = 40.0; + let n_measures = parts.iter().map(|p| p.measures.len()).max().unwrap_or(0); + let per = setup.bars_per_system.max(1) as usize; + let systems = n_measures.div_ceil(per).max(1); + let label_w = if setup.show_track_names { 70.0 } else { 0.0 }; + let staff_x0 = MARGIN + label_w + 34.0; + let staff_x1 = W - MARGIN; + let sys_h = parts.len().max(1) as f64 * STAFF_GAP + SYSTEM_GAP; + let top = 110.0; + let h = top + systems as f64 * sys_h + MARGIN; + let mut o = String::new(); + let _ = + writeln!(o, ""); + let _ = writeln!(o, ""); + let _ = writeln!(o, "{}", W / 2.0, xml_escape(&setup.title)); + if !setup.composer.is_empty() { + let _ = writeln!(o, "{}", W - MARGIN, xml_escape(&setup.composer)); + } + let bar_w = (staff_x1 - staff_x0) / per as f64; + for sys in 0..systems { + let y_sys = top + sys as f64 * sys_h; + for (pi, p) in parts.iter().enumerate() { + let y0 = y_sys + pi as f64 * STAFF_GAP; // top line + let bottom = y0 + 4.0 * SP; + if setup.show_track_names && sys == 0 { + let _ = writeln!(o, "{}", y0 + 2.0 * SP + 4.0, xml_escape(&p.name)); + } + for l in 0..5 { + let y = y0 + f64::from(l) * SP; + let _ = writeln!( + o, + "", + staff_x0 - 30.0 + ); + } + let clef = if p.bass { "𝄢" } else { "𝄞" }; + let _ = writeln!( + o, + "{clef}", + staff_x0 - 29.0, + bottom + if p.bass { -SP } else { SP * 0.6 }, + if p.bass { 26 } else { 36 } + ); + for k in 0..per { + let mi = sys * per + k; + let x0 = staff_x0 + k as f64 * bar_w; + let _ = writeln!( + o, + "", + x0 + bar_w, + x0 + bar_w + ); + let Some(m) = p.measures.get(mi) else { continue }; + if pi == 0 { + let _ = writeln!(o, "{}", x0 + 2.0, y0 - 8.0, mi + 1); + } + let total: u32 = m.items.iter().map(|i| i.units).sum::().max(1); + let mut acc = 0u32; + for it in &m.items { + let x = x0 + 10.0 + (bar_w - 16.0) * f64::from(acc) / f64::from(total); + acc += it.units; + let (_, dot) = note_type(it.units); + if it.pitches.is_empty() { + rest(&mut o, x, y0, SP, it.units); + if dot { + let _ = writeln!(o, "", x + 7.0, y0 + 1.5 * SP); + } + continue; + } + let steps: Vec = it.pitches.iter().map(|p2| staff_step(*p2, p.bass)).collect(); + let filled = it.units < 8; + for (pitch, st) in it.pitches.iter().zip(&steps) { + let y = bottom - f64::from(*st) * SP / 2.0; + // Ledger lines. + let mut l = if *st < 0 { -2 } else { 10 }; + while (*st < 0 && l >= *st) || (*st > 8 && l <= *st) { + let ly = bottom - f64::from(l) * SP / 2.0; + let _ = writeln!( + o, + "", + x - 7.0, + x + 7.0 + ); + l += if *st < 0 { -2 } else { 2 }; + } + let fill = if filled { "black" } else { "white" }; + let _ = writeln!( + o, + "" + ); + if spell(*pitch).1 != 0 { + let _ = writeln!(o, "♯", x - 14.0, y + 4.0); + } + if dot { + let _ = writeln!(o, "", x + 8.0, y - 1.5); + } + } + if it.units < 16 { + let (lo, hi) = (steps.iter().copied().min().unwrap_or(4), steps.iter().copied().max().unwrap_or(4)); + let up = (lo + hi) < 8; + let (sx, y_from, y_to) = if up { + (x + 4.3, bottom - f64::from(lo) * SP / 2.0, bottom - f64::from(hi) * SP / 2.0 - 3.5 * SP) + } else { + (x - 4.3, bottom - f64::from(hi) * SP / 2.0, bottom - f64::from(lo) * SP / 2.0 + 3.5 * SP) + }; + let _ = writeln!( + o, + "" + ); + let flags = match it.units { + 1 => 2, + 2 | 3 => 1, + _ => 0, + }; + for f in 0..flags { + let fy = y_to + f64::from(f) * if up { 5.0 } else { -5.0 }; + let dy = if up { 9.0 } else { -9.0 }; + let _ = writeln!( + o, + "", + dy * 0.5 + ); + } + } + if it.tie_start { + let y = bottom - f64::from(steps[0]) * SP / 2.0 + 6.0; + let _ = writeln!(o, "", x + 3.0); + } + } + } + } + // System bracket. + let y_last = y_sys + (parts.len().max(1) - 1) as f64 * STAFF_GAP + 4.0 * SP; + let _ = writeln!( + o, + "", + staff_x0 - 30.0, + staff_x0 - 30.0 + ); + } + o.push_str("\n"); + o +} + +fn rest(o: &mut String, x: f64, y0: f64, sp: f64, units: u32) { + let mid = y0 + 2.0 * sp; + match units { + 16 => { + let _ = writeln!(o, "", x - 4.5, y0 + sp, sp / 2.0); + } + 8 | 12 => { + let _ = writeln!(o, "", x - 4.5, mid - sp / 2.0, sp / 2.0); + } + 4 | 6 => { + let _ = writeln!( + o, + "", + x - 2.0, + mid - 9.0 + ); + } + _ => { + let _ = writeln!(o, "", x - 2.0, mid - 3.0); + let _ = writeln!( + o, + "", + x + 2.5, + mid - 4.0, + x - 1.0, + mid + 7.0 + ); + if units == 1 { + let _ = writeln!(o, "", x - 3.5, mid + 2.0); + } + } + } +} + From 68abf77e2ddaea226a742efcb363630fa07faa3c Mon Sep 17 00:00:00 2001 From: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 109/209] Extract score writer changes from SoundCraft 7509b64c --- crates/score/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index 30375b652..3aee96472 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -45,7 +45,13 @@ fn spell(pitch: u8) -> (char, i32, i32) { } fn xml_escape(s: &str) -> String { - s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """) + s.chars() + .filter(|c| matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')) + .collect::() + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) } fn note_type(units: u32) -> (&'static str, bool) { From 9dfaaf27af936868710d44042ede6705cb9e616f Mon Sep 17 00:00:00 2001 From: Samuel Bacaro Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 110/209] Extract score writer changes from SoundCraft ccd3fe7c --- crates/score/src/lib.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index 3aee96472..d51ab8333 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -3,6 +3,14 @@ use std::fmt::Write as _; +const MAX_OUTPUT_BYTES: usize = 256 * 1024 * 1024; // 256 MiB +fn over_budget(buf: &str) -> Result<(), String> { + if buf.len() > MAX_OUTPUT_BYTES { + return Err(format!("score too large: over {} MiB", MAX_OUTPUT_BYTES / (1024 * 1024))); + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq)] pub struct ScoreSetup { pub title: String, @@ -45,13 +53,7 @@ fn spell(pitch: u8) -> (char, i32, i32) { } fn xml_escape(s: &str) -> String { - s.chars() - .filter(|c| matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')) - .collect::() - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) + s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """) } fn note_type(units: u32) -> (&'static str, bool) { @@ -68,7 +70,7 @@ fn note_type(units: u32) -> (&'static str, bool) { } /// MusicXML 4.0 (partwise) for `parts`. -pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> String { +pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> Result { let mut x = String::new(); x.push_str("\n"); x.push_str("\n"); @@ -149,11 +151,12 @@ pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> String { } } x.push_str(" \n"); + over_budget(&x)?; } x.push_str(" \n"); } x.push_str("\n"); - x + Ok(x) } /// Staff position of `pitch` in half-spaces above the bottom line (treble E4 / bass G2). @@ -165,7 +168,7 @@ fn staff_step(pitch: u8, bass: bool) -> i32 { } /// An engraved score as a printable SVG page (A4 portrait width, as tall as needed). -pub fn svg(parts: &[Part], setup: &ScoreSetup) -> String { +pub fn svg(parts: &[Part], setup: &ScoreSetup) -> Result { const W: f64 = 794.0; const MARGIN: f64 = 48.0; const SP: f64 = 7.0; // staff space @@ -299,6 +302,7 @@ pub fn svg(parts: &[Part], setup: &ScoreSetup) -> String { let _ = writeln!(o, "", x + 3.0); } } + over_budget(&o)?; } } // System bracket. @@ -311,7 +315,7 @@ pub fn svg(parts: &[Part], setup: &ScoreSetup) -> String { ); } o.push_str("\n"); - o + Ok(o) } fn rest(o: &mut String, x: f64, y0: f64, sp: f64, units: u32) { From 8c6e050f324f8e50bddf556d12fcd48e3476d505 Mon Sep 17 00:00:00 2001 From: Samuel Bacaro Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 111/209] Extract score writer changes from SoundCraft fa77e03e --- crates/score/src/lib.rs | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index d51ab8333..59a28de68 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -4,9 +4,9 @@ use std::fmt::Write as _; const MAX_OUTPUT_BYTES: usize = 256 * 1024 * 1024; // 256 MiB -fn over_budget(buf: &str) -> Result<(), String> { - if buf.len() > MAX_OUTPUT_BYTES { - return Err(format!("score too large: over {} MiB", MAX_OUTPUT_BYTES / (1024 * 1024))); +fn over_budget(buf: &str, budget: usize) -> Result<(), String> { + if buf.len() > budget { + return Err(format!("score too large: over {} bytes", budget)); } Ok(()) } @@ -71,10 +71,19 @@ fn note_type(units: u32) -> (&'static str, bool) { /// MusicXML 4.0 (partwise) for `parts`. pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> Result { + musicxml_budgeted(parts, setup, MAX_OUTPUT_BYTES) +} + +/// [`musicxml`] with an explicit byte budget (kept separate so tests can pass a tiny budget +/// instead of building a 256 MiB score). +fn musicxml_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result { let mut x = String::new(); x.push_str("\n"); x.push_str("\n"); x.push_str("\n"); + // The header (and an oversized title/composer, escaped) can itself exceed the budget even with + // no measures, so check before the per-measure loop too. + over_budget(&x, budget)?; let _ = writeln!(x, " {}", xml_escape(&setup.title)); x.push_str(" \n"); if !setup.composer.is_empty() { @@ -85,6 +94,7 @@ pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> Result { let _ = writeln!(x, " {}", i + 1, xml_escape(&p.name)); } x.push_str(" \n"); + over_budget(&x, budget)?; for (i, p) in parts.iter().enumerate() { let _ = writeln!(x, " ", i + 1); let mut meter = (0, 0); @@ -151,11 +161,12 @@ pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> Result { } } x.push_str(" \n"); - over_budget(&x)?; + over_budget(&x, budget)?; } x.push_str(" \n"); } x.push_str("\n"); + over_budget(&x, budget)?; Ok(x) } @@ -169,6 +180,12 @@ fn staff_step(pitch: u8, bass: bool) -> i32 { /// An engraved score as a printable SVG page (A4 portrait width, as tall as needed). pub fn svg(parts: &[Part], setup: &ScoreSetup) -> Result { + svg_budgeted(parts, setup, MAX_OUTPUT_BYTES) +} + +/// [`svg`] with an explicit byte budget (kept separate so tests can pass a tiny budget instead of +/// building a 256 MiB score). +fn svg_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result { const W: f64 = 794.0; const MARGIN: f64 = 48.0; const SP: f64 = 7.0; // staff space @@ -190,7 +207,9 @@ pub fn svg(parts: &[Part], setup: &ScoreSetup) -> Result { let _ = writeln!(o, "{}", W / 2.0, xml_escape(&setup.title)); if !setup.composer.is_empty() { let _ = writeln!(o, "{}", W - MARGIN, xml_escape(&setup.composer)); + over_budget(&o, budget)?; } + over_budget(&o, budget)?; let bar_w = (staff_x1 - staff_x0) / per as f64; for sys in 0..systems { let y_sys = top + sys as f64 * sys_h; @@ -302,8 +321,13 @@ pub fn svg(parts: &[Part], setup: &ScoreSetup) -> Result { let _ = writeln!(o, "", x + 3.0); } } - over_budget(&o)?; + // Check each measure, so a run of rest-only or missing measures that only draws + // bar lines cannot grow the output past the budget untouched. + over_budget(&o, budget)?; } + // The staff markup (lines, clef, track names) itself grows with the part count even + // without any items, so check once per staff too. + over_budget(&o, budget)?; } // System bracket. let y_last = y_sys + (parts.len().max(1) - 1) as f64 * STAFF_GAP + 4.0 * SP; @@ -313,8 +337,10 @@ pub fn svg(parts: &[Part], setup: &ScoreSetup) -> Result { staff_x0 - 30.0, staff_x0 - 30.0 ); + over_budget(&o, budget)?; } o.push_str("\n"); + over_budget(&o, budget)?; Ok(o) } From 1002aca27092b6fe41e3af5b3212080a354555cb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 112/209] Extract score writer changes from SoundCraft 5075da2f --- crates/score/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index 59a28de68..023f35554 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -53,7 +53,13 @@ fn spell(pitch: u8) -> (char, i32, i32) { } fn xml_escape(s: &str) -> String { - s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """) + s.chars() + .filter(|c| matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')) + .collect::() + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) } fn note_type(units: u32) -> (&'static str, bool) { From bfef2e5ce28dbc1e57e897058851024d161bd09b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 113/209] Extract score writer changes from SoundCraft fa34e0c1 --- crates/score/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index 023f35554..c64ac0054 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -198,7 +198,9 @@ fn svg_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 114/209] Extract score writer changes from SoundCraft 4bdda5ff --- crates/score/src/lib.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index c64ac0054..d8e757b21 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -53,13 +53,10 @@ fn spell(pitch: u8) -> (char, i32, i32) { } fn xml_escape(s: &str) -> String { - s.chars() - .filter(|c| matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')) - .collect::() - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) + // Keep the score export's existing literal whitespace and quote spelling. The + // shared attribute writer owns XML legality/escaping; only its generated + // whitespace references are restored (input ampersands were escaped first). + craft_xml::esc_attr(s).replace(" ", "\t").replace(" ", "\n").replace(" ", "\r") } fn note_type(units: u32) -> (&'static str, bool) { From 3d160af4d90797f39196c22e81288d9c403030d2 Mon Sep 17 00:00:00 2001 From: Vikram Bhaskaran Date: Mon, 16 Nov 2026 19:20:52 -0500 Subject: [PATCH 115/209] Extract score writer changes from SoundCraft 21cb618d --- crates/score/src/lib.rs | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index d8e757b21..cab0ad9bd 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -33,8 +33,10 @@ pub struct Item { pub pitches: Vec, /// Duration in sixteenths; always one of the supported duration values. pub units: u32, - pub tie_start: bool, - pub tie_stop: bool, + /// Pitches continuing into the next item. + pub tie_start: Vec, + /// Pitches continuing from the previous item. + pub tie_stop: Vec, } #[derive(Debug, Clone, PartialEq)] @@ -53,10 +55,13 @@ fn spell(pitch: u8) -> (char, i32, i32) { } fn xml_escape(s: &str) -> String { - // Keep the score export's existing literal whitespace and quote spelling. The - // shared attribute writer owns XML legality/escaping; only its generated - // whitespace references are restored (input ampersands were escaped first). - craft_xml::esc_attr(s).replace(" ", "\t").replace(" ", "\n").replace(" ", "\r") + s.chars() + .filter(|c| matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')) + .collect::() + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) } fn note_type(units: u32) -> (&'static str, bool) { @@ -137,10 +142,10 @@ fn musicxml_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Resul let _ = write!(x, "{alter}"); } let _ = write!(x, "{octave}{}", it.units); - if it.tie_stop { + if it.tie_stop.contains(pitch) { x.push_str(""); } - if it.tie_start { + if it.tie_start.contains(pitch) { x.push_str(""); } let _ = write!(x, "1{ty}"); @@ -150,12 +155,12 @@ fn musicxml_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Resul if alter != 0 { x.push_str("sharp"); } - if it.tie_stop || it.tie_start { + if it.tie_stop.contains(pitch) || it.tie_start.contains(pitch) { x.push_str(""); - if it.tie_stop { + if it.tie_stop.contains(pitch) { x.push_str(""); } - if it.tie_start { + if it.tie_start.contains(pitch) { x.push_str(""); } x.push_str(""); @@ -321,9 +326,16 @@ fn svg_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result", x + 3.0); + for pitch in &it.tie_start { + let above = it.pitches.first().is_some_and(|low| pitch > low); + let direction = if above { -1.0 } else { 1.0 }; + let y = bottom - f64::from(staff_step(*pitch, p.bass)) * SP / 2.0 + direction * 6.0; + let _ = writeln!( + o, + "", + x + 3.0, + direction * 5.0 + ); } } // Check each measure, so a run of rest-only or missing measures that only draws From 9b570237ef8682c3848bacf374ff1db06dc95241 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 19:23:04 -0500 Subject: [PATCH 116/209] feat(score): register reusable writers with bounded public notation input --- ATTRIBUTION.md | 4 +++ Cargo.lock | 7 +++++ Cargo.toml | 1 + crates/score/Cargo.toml | 14 +++++++++ crates/score/src/lib.rs | 65 ++++++++++++++++++++++++++++++++--------- 5 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 crates/score/Cargo.toml diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index ab167207c..70a929de8 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -103,3 +103,7 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso `crates/tga/src/metadata_tests.rs` and metadata-boundary helpers in `crates/tga/src/lib.rs` are adapted from nuclearspike's FilmCraft #1932 at `e7842f76f887005c460c64a8ab51d59d086e6f27` (MIT OR Apache-2.0). Synthetic fixtures are original contributor test data; primary Git author retained. The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper are ported from Craft Anything `8f3b775157d331dd31cfa4da911e26d8856b8c77` (same MIT OR Apache-2.0 source licenses). Brandon Thomas is the primary import author; retained hardening credits include Afloat, BFlat, Rahul, RRFZ and NetterSilver. The finite-real fix preserves gmrmad1 as author from Craft Anything #1905 (`d85e104f`); quoted-attribute/line-ending regressions preserve RRFZ. Bounds/adaptation and bounded entity scanning are subsequent Dylan M. Taylor changes. App-specific JSON projections and keyed-archive interpretation remain outside this crate. + +## Score serializers + +`crates/score` extracts SoundCraft MusicXML and SVG score writing from `crates/engine/src/score.rs` at main `648ed9db9a0e98cc55643babffac381521a556c8`, with per-pitch ties adapted from [SoundCraft #182](https://github.com/storytold/soundcraft/pull/182) at `21cb618d93ec0e8d4771ac534994de8d5996ff10`. Source is MIT OR Apache-2.0. Import commits preserve Brandon Thomas, NetterSilver, Samuel Bacaro and Vikram Bhaskaran as primary authors of their respective changes. The consumer retains session mapping and layout. Shared API wiring, validation and focused tests are subsequent Dylan M. Taylor changes. diff --git a/Cargo.lock b/Cargo.lock index ef82f1afa..41290ac89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2791,6 +2791,13 @@ dependencies = [ "proptest", ] +[[package]] +name = "craft-score" +version = "0.1.0" +dependencies = [ + "craft-xml", +] + [[package]] name = "craft-scraft-format" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c8ae36287..8c3c97517 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ rust-version = "1.95" repository = "https://github.com/storytold/craft-libs" [workspace.dependencies] +craft-score = { path = "crates/score" } craft-shx = { path = "crates/shx" } craft-http-url = { path = "crates/http-url" } craft-usd = { path = "crates/usd" } diff --git a/crates/score/Cargo.toml b/crates/score/Cargo.toml new file mode 100644 index 000000000..f283282c4 --- /dev/null +++ b/crates/score/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-score" +description = "Application-independent MusicXML and printable SVG score writers" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[dependencies] +craft-xml.workspace = true + +[lints] +workspace = true diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index cab0ad9bd..130f9e857 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -13,6 +13,8 @@ fn over_budget(buf: &str, budget: usize) -> Result<(), String> { #[derive(Debug, Clone, PartialEq)] pub struct ScoreSetup { + /// Software name recorded in the MusicXML encoding metadata. + pub software: String, pub title: String, pub composer: String, pub bars_per_system: u32, @@ -55,13 +57,10 @@ fn spell(pitch: u8) -> (char, i32, i32) { } fn xml_escape(s: &str) -> String { - s.chars() - .filter(|c| matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')) - .collect::() - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) + // Keep the score export's existing literal whitespace and quote spelling. The + // shared attribute writer owns XML legality/escaping; only its generated + // whitespace references are restored (input ampersands were escaped first). + craft_xml::esc_attr(s).replace(" ", "\t").replace(" ", "\n").replace(" ", "\r") } fn note_type(units: u32) -> (&'static str, bool) { @@ -82,9 +81,10 @@ pub fn musicxml(parts: &[Part], setup: &ScoreSetup) -> Result { musicxml_budgeted(parts, setup, MAX_OUTPUT_BYTES) } -/// [`musicxml`] with an explicit byte budget (kept separate so tests can pass a tiny budget -/// instead of building a 256 MiB score). -fn musicxml_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result { +/// MusicXML with an explicit byte budget, capped at 256 MiB. +pub fn musicxml_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result { + validate(parts, setup)?; + let budget = budget.min(MAX_OUTPUT_BYTES); let mut x = String::new(); x.push_str("\n"); x.push_str("\n"); @@ -97,7 +97,8 @@ fn musicxml_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Resul if !setup.composer.is_empty() { let _ = writeln!(x, " {}", xml_escape(&setup.composer)); } - x.push_str(" SoundCraft\n \n \n"); + let _ = writeln!(x, " {}", xml_escape(&setup.software)); + x.push_str(" \n \n"); for (i, p) in parts.iter().enumerate() { let _ = writeln!(x, " {}", i + 1, xml_escape(&p.name)); } @@ -191,9 +192,10 @@ pub fn svg(parts: &[Part], setup: &ScoreSetup) -> Result { svg_budgeted(parts, setup, MAX_OUTPUT_BYTES) } -/// [`svg`] with an explicit byte budget (kept separate so tests can pass a tiny budget instead of -/// building a 256 MiB score). -fn svg_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result { +/// SVG with an explicit byte budget, capped at 256 MiB. +pub fn svg_budgeted(parts: &[Part], setup: &ScoreSetup, budget: usize) -> Result { + validate(parts, setup)?; + let budget = budget.min(MAX_OUTPUT_BYTES); const W: f64 = 794.0; const MARGIN: f64 = 48.0; const SP: f64 = 7.0; // staff space @@ -395,3 +397,38 @@ fn rest(o: &mut String, x: f64, y0: f64, sp: f64, units: u32) { } } +// Public notation data can come from callers other than SoundCraft's bounded layout. +// Validate before writing/escaping so malicious dimensions cannot overflow sums or create +// an unbounded temporary string before the output-budget checks run. +fn validate(parts: &[Part], setup: &ScoreSetup) -> Result<(), String> { + let invalid = || "score exceeds supported notation limits".to_string(); + if parts.len() > 4096 { + return Err(invalid()); + } + let mut text = setup.title.len().checked_add(setup.composer.len()).and_then(|n| n.checked_add(setup.software.len())).ok_or_else(invalid)?; + for part in parts { + text = text.checked_add(part.name.len()).ok_or_else(invalid)?; + if text > 16 * 1024 * 1024 || part.measures.len() > 2000 { + return Err(invalid()); + } + for measure in &part.measures { + if measure.items.len() > 128 { + return Err(invalid()); + } + for item in &measure.items { + if ![1, 2, 3, 4, 6, 8, 12, 16].contains(&item.units) + || item.pitches.len() > 16 + || item.tie_start.len() > 16 + || item.tie_stop.len() > 16 + || item.pitches.iter().chain(&item.tie_start).chain(&item.tie_stop).any(|p| *p > 127) + { + return Err(invalid()); + } + } + } + } + if text > 16 * 1024 * 1024 { + return Err(invalid()); + } + Ok(()) +} From 0006665d60e6a4ee6aeb231de33cd41d38931ece Mon Sep 17 00:00:00 2001 From: Vikram Bhaskaran Date: Mon, 16 Nov 2026 19:23:23 -0500 Subject: [PATCH 117/209] test(score): retain per-pitch MusicXML and SVG tie regression from SoundCraft 182 --- crates/score/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index 130f9e857..fccb5c32f 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -432,3 +432,37 @@ fn validate(parts: &[Part], setup: &ScoreSetup) -> Result<(), String> { } Ok(()) } +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::indexing_slicing)] +mod tests { + use super::*; + fn setup() -> ScoreSetup { + ScoreSetup { software: "SoundCraft".into(), title: "A<&\u{1}".into(), composer: "B".into(), bars_per_system: 4, show_track_names: true } + } + fn parts() -> Vec { + vec![Part { + name: "Keys".into(), + bass: false, + measures: vec![Measure { + numerator: 4, + denominator: 4, + items: vec![ + Item { pitches: vec![60, 64], units: 4, tie_start: vec![64], tie_stop: vec![] }, + Item { pitches: vec![64], units: 12, tie_start: vec![], tie_stop: vec![64] }, + ], + }], + }] + } + #[test] + fn per_pitch_ties_and_escaping_survive_serialization() { + let parts = parts(); + let xml = musicxml(&parts, &setup()).unwrap(); + assert!(xml.contains("A<&")); + let notes: Vec<_> = xml.split("").skip(1).map(|n| n.split("").next().unwrap()).collect(); + assert!(notes[0].contains("4")); + assert!(!notes[0].contains("") && notes[1].contains("")); + assert!(notes[2].contains("12") && notes[2].contains("")); + assert_eq!(svg(&parts, &setup()).unwrap().matches("stroke-width=\"0.9\"").count(), 1); + } +} From d5d65b128dfc74f77ffd23ea08fd3ea428a7d0cd Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 19:23:23 -0500 Subject: [PATCH 118/209] test(score): exercise byte budgets and hostile notation input --- crates/score/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index fccb5c32f..7ec2141c5 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -465,4 +465,13 @@ mod tests { assert!(notes[2].contains("12") && notes[2].contains("")); assert_eq!(svg(&parts, &setup()).unwrap().matches("stroke-width=\"0.9\"").count(), 1); } + #[test] + fn byte_budgets_and_hostile_durations_fail() { + assert!(musicxml_budgeted(&parts(), &setup(), 32).is_err()); + assert!(svg_budgeted(&parts(), &setup(), 32).is_err()); + let mut parts = parts(); + parts[0].measures[0].items[0].units = u32::MAX; + assert!(musicxml(&parts, &setup()).is_err()); + assert!(svg(&parts, &setup()).is_err()); + } } From 9595e0ca9bff15b04c0096458c63d96d8b090b63 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 16 Nov 2026 19:25:08 -0500 Subject: [PATCH 119/209] test(score): retain original pitch spelling and staff placement regression --- crates/score/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/score/src/lib.rs b/crates/score/src/lib.rs index 7ec2141c5..7ca06261e 100644 --- a/crates/score/src/lib.rs +++ b/crates/score/src/lib.rs @@ -474,4 +474,12 @@ mod tests { assert!(musicxml(&parts, &setup()).is_err()); assert!(svg(&parts, &setup()).is_err()); } + #[test] + fn spelling_and_staff_steps() { + assert_eq!(spell(60), ('C', 0, 4)); + assert_eq!(spell(61), ('C', 1, 4)); + assert_eq!(staff_step(64, false), 0); // E4: bottom treble line + assert_eq!(staff_step(77, false), 8); // F5: top treble line + assert_eq!(staff_step(43, true), 0); // G2: bottom bass line + } } From a0cd0e2afbe5c3c1a232c21b0ba36e39e03f07a5 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 20:10:44 -0500 Subject: [PATCH 120/209] fix(flow): use CSS percentage scale for numeric HSL channels --- crates/flow/src/color.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/flow/src/color.rs b/crates/flow/src/color.rs index a80fdf60a..941b25902 100644 --- a/crates/flow/src/color.rs +++ b/crates/flow/src/color.rs @@ -131,7 +131,8 @@ pub fn parse_color(s: &str) -> Option { if let Some(p) = x.strip_suffix('%') { Some((p.trim().parse::().ok()? / 100.0).clamp(0.0, 1.0)) } else { - Some(x.parse::().ok()?.clamp(0.0, 1.0)) + // CSS Color 4: numeric saturation and lightness use the same 0–100 scale as percentages. + Some((x.parse::().ok()? / 100.0).clamp(0.0, 1.0)) } }; let s = parse_pct(parts[1])?; @@ -193,4 +194,20 @@ mod tests { assert_eq!(parse_color("rgba(0, 0, 0, inf)"), Some(Rgb(0, 0, 0))); assert_eq!(parse_color("rgba(0, 0, 0, nan%)"), Some(Rgb(0, 0, 0))); } + + #[test] + fn numeric_hsl_channels_use_the_percentage_scale() { + for (numeric, percent, expected) in [ + ("hsl(0 0 50)", "hsl(0 0% 50%)", Rgb(128, 128, 128)), + ("hsl(0 100 50)", "hsl(0 100% 50%)", Rgb(255, 0, 0)), + ("hsl(120 50 50)", "hsl(120 50% 50%)", Rgb(64, 191, 64)), + ("hsl(0 0 1)", "hsl(0 0% 1%)", Rgb(3, 3, 3)), + ("hsl(0 -10 150)", "hsl(0 -10% 150%)", Rgb(255, 255, 255)), + ("hsl(0 100 -10)", "hsl(0 100% -10%)", Rgb(0, 0, 0)), + ("hsla(0.5turn 100 50 / 50%)", "hsla(0.5turn 100% 50% / 50%)", Rgb(128, 255, 255)), + ] { + assert_eq!(parse_color(numeric), Some(expected), "{numeric}"); + assert_eq!(parse_color(numeric), parse_color(percent), "{numeric} == {percent}"); + } + } } From e2bf53f2bdba4828e21097774a568ed9c74b0004 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 20:25:09 -0500 Subject: [PATCH 121/209] craft-audio: share non-seeking canonical WAV streaming primitives (#1962) --- ATTRIBUTION.md | 4 ++ crates/audio/src/lib.rs | 4 ++ crates/audio/src/wav_stream.rs | 119 +++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 crates/audio/src/wav_stream.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index ab167207c..8840199b2 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -103,3 +103,7 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso `crates/tga/src/metadata_tests.rs` and metadata-boundary helpers in `crates/tga/src/lib.rs` are adapted from nuclearspike's FilmCraft #1932 at `e7842f76f887005c460c64a8ab51d59d086e6f27` (MIT OR Apache-2.0). Synthetic fixtures are original contributor test data; primary Git author retained. The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper are ported from Craft Anything `8f3b775157d331dd31cfa4da911e26d8856b8c77` (same MIT OR Apache-2.0 source licenses). Brandon Thomas is the primary import author; retained hardening credits include Afloat, BFlat, Rahul, RRFZ and NetterSilver. The finite-real fix preserves gmrmad1 as author from Craft Anything #1905 (`d85e104f`); quoted-attribute/line-ending regressions preserve RRFZ. Bounds/adaptation and bounded entity scanning are subsequent Dylan M. Taylor changes. App-specific JSON projections and keyed-archive interpretation remain outside this crate. + +## Non-seeking WAV primitives + +`crates/audio/src/wav_stream.rs` is extracted from AniCraft `crates/audio/src/wav.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0). Tanish Manku is retained as the primary import author, from original commit `30c121d269a49d9abe54a3834ed4aa0201542535`. Odd-data padding accounting, shared API documentation and regression tests are subsequent Dylan M. Taylor changes. diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index c7382a973..e4a1c1695 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -775,3 +775,7 @@ mod tests { /// DVD and Blu-ray LPCM packet decoding. pub mod lpcm; + +/// Non-seeking canonical WAV streaming primitives. +#[cfg(feature = "pcm-encode")] +pub mod wav_stream; diff --git a/crates/audio/src/wav_stream.rs b/crates/audio/src/wav_stream.rs new file mode 100644 index 000000000..9da2bdcc9 --- /dev/null +++ b/crates/audio/src/wav_stream.rs @@ -0,0 +1,119 @@ +//! Canonical RIFF/WAVE header and interleaved sample blocks for non-seekable sinks. +//! +//! Extracted from AniCraft's WAV writer, preserving its byte-level compatibility. +//! Unlike `FileEncoder`, these primitives do not require `Seek` and leave final lengths, +//! chunk padding and sink error handling to the caller. The caller must supply a complete +//! frame count and append exactly that many frames. For odd-length data, append a pad byte +//! (the header includes that padding in the RIFF size); current AniCraft output is always stereo (even). +//! Non-finite samples become silence, including float output. This deliberately differs +//! from the general encoder's infinity handling. Prefer `FileEncoder` for new seekable sinks. + +/// Sample formats written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Bits { + /// Signed 16-bit PCM. + Int16, + /// Signed 24-bit PCM. + Int24, + /// IEEE 754 32-bit float. + Float32, +} + +impl Bits { + fn bytes(self) -> u16 { + match self { + Bits::Int16 => 2, + Bits::Int24 => 3, + Bits::Float32 => 4, + } + } +} + +const PCM: u16 = 1; +const FLOAT: u16 = 3; + +/// A WAV header for `frames` frames of `channels` channels. +/// Returns `None` for zero rate/channels or any size/byte-rate field overflow. +pub fn header(rate: u32, channels: u16, bits: Bits, frames: u64) -> Option> { + if rate == 0 || channels == 0 { + return None; + } + let b = bits.bytes(); + let data = u32::try_from(frames.checked_mul(u64::from(b) * u64::from(channels))?).ok()?; + let riff = data.checked_add(36)?.checked_add(data & 1)?; + let block = channels.checked_mul(b)?; + let byte_rate = rate.checked_mul(u32::from(block))?; + let tag = if bits == Bits::Float32 { FLOAT } else { PCM }; + let mut h = Vec::with_capacity(44); + h.extend_from_slice(b"RIFF"); + h.extend_from_slice(&riff.to_le_bytes()); + h.extend_from_slice(b"WAVEfmt "); + h.extend_from_slice(&16u32.to_le_bytes()); + h.extend_from_slice(&tag.to_le_bytes()); + h.extend_from_slice(&channels.to_le_bytes()); + h.extend_from_slice(&rate.to_le_bytes()); + h.extend_from_slice(&byte_rate.to_le_bytes()); + h.extend_from_slice(&block.to_le_bytes()); + h.extend_from_slice(&(b * 8).to_le_bytes()); + h.extend_from_slice(b"data"); + h.extend_from_slice(&data.to_le_bytes()); + Some(h) +} + +/// Appends samples in `bits` format. Integers are rounded and clipped to range; `x · 32768` +/// for 16-bit, so 16-bit sources read by [`crate::decode`] come back bit-identical. +pub fn append(out: &mut Vec, samples: &[f32], bits: Bits) { + out.reserve(samples.len() * usize::from(bits.bytes())); + for &s in samples { + let s = if s.is_finite() { s } else { 0.0 }; + match bits { + Bits::Int16 => out.extend_from_slice(&((s * 32_768.0).round().clamp(-32_768.0, 32_767.0) as i16).to_le_bytes()), + Bits::Int24 => { + let v = (f64::from(s) * 8_388_608.0).round().clamp(-8_388_608.0, 8_388_607.0) as i32; + out.extend_from_slice(v.to_le_bytes().get(..3).unwrap_or(&[0, 0, 0])); + } + Bits::Float32 => out.extend_from_slice(&s.to_le_bytes()), + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::indexing_slicing)] +mod tests { + use super::*; + + #[test] + fn canonical_header_and_odd_padding() { + let h = header(48_000, 2, Bits::Int16, 1).unwrap(); + assert_eq!(h.len(), 44); + assert_eq!(&h[..12], b"RIFF\x28\0\0\0WAVE"); + assert_eq!(&h[28..36], &[0, 238, 2, 0, 4, 0, 16, 0]); + let h = header(8_000, 1, Bits::Int24, 1).unwrap(); + assert_eq!(&h[4..8], &40u32.to_le_bytes()); + assert_eq!(&h[40..44], &3u32.to_le_bytes()); + assert!(header(0, 2, Bits::Int16, 1).is_none()); + assert!(header(48_000, 0, Bits::Int16, 1).is_none()); + assert!(header(u32::MAX, 2, Bits::Int16, 1).is_none()); + assert!(header(48_000, 2, Bits::Float32, u64::MAX).is_none()); + assert!(header(48_000, u16::MAX, Bits::Float32, 0).is_none()); + } + + #[test] + fn samples_preserve_rounding_clipping_silence_and_chunk_boundaries() { + let samples = [-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0, f32::NAN, f32::INFINITY, f32::NEG_INFINITY]; + let mut pcm = Vec::new(); + append(&mut pcm, &samples, Bits::Int16); + let expected = [-32768i16, -32768, -16384, 0, 16384, 32767, 32767, 0, 0, 0]; + assert_eq!(pcm, expected.iter().flat_map(|s| s.to_le_bytes()).collect::>()); + for bits in [Bits::Int16, Bits::Int24, Bits::Float32] { + let mut whole = Vec::new(); + append(&mut whole, &samples, bits); + let mut blocks = Vec::new(); + for block in samples.chunks(3) { + append(&mut blocks, block, bits); + } + assert_eq!(whole, blocks); + assert_eq!(&whole[whole.len() - 3 * usize::from(bits.bytes())..], vec![0; 3 * usize::from(bits.bytes())]); + } + } +} From 8479a1f3bbabb0e0b25194c508ab2eb14aa5490f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 20:30:05 -0500 Subject: [PATCH 122/209] pkcs7: CMS signing with caller-owned key providers (#1963) --- crates/pkcs7/src/cms.rs | 116 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 10 deletions(-) diff --git a/crates/pkcs7/src/cms.rs b/crates/pkcs7/src/cms.rs index 083aa4302..2bccf5099 100644 --- a/crates/pkcs7/src/cms.rs +++ b/crates/pkcs7/src/cms.rs @@ -449,25 +449,55 @@ struct SignedDataInput<'a> { content_digest: &'a [u8], } -fn build(input: SignedDataInput<'_>) -> Result, Error> { - let SignedDataInput { key, cert, chain, alg, content_type, content, content_digest } = input; - let cert_hash = DigestAlg::Sha256.digest(&[&cert.raw]); - let general_names = der::seq(&[&der::explicit(4, &cert.issuer.raw)]); +/// Public certificate material used by the CMS writer. `serial` is the original INTEGER +/// contents, including its sign padding; `issuer_der` and `der` are complete DER encodings. +/// Callers obtain these fields from an already parsed certificate. No private key is held. +#[derive(Clone, Copy, Debug)] +pub struct SigningCertificate<'a> { + pub der: &'a [u8], + pub issuer_der: &'a [u8], + pub serial: &'a [u8], +} + +/// Inputs for a CAdES SignedData envelope. The caller computes `content_digest` using `alg`. +/// `content` is absent for detached signatures, or the bytes to encapsulate for a token. +/// Certificate-chain entries are complete DER certificates; the signer is deduplicated. +#[derive(Clone, Copy, Debug)] +pub struct SigningInput<'a> { + pub cert: SigningCertificate<'a>, + pub chain: &'a [&'a [u8]], + pub alg: DigestAlg, + pub content_type: &'a str, + pub content: Option<&'a [u8]>, + pub content_digest: &'a [u8], + /// Complete DER AlgorithmIdentifier matching the callback's signing operation. + pub signature_algorithm: &'a [u8], +} + +/// Build the same signed attributes and envelope as [`sign_detached`] / [`sign_encapsulated`], +/// signing through a caller-owned key provider. The callback receives the exact DER SET +/// of signed attributes once (not its digest). It must sign those bytes using `alg` and the +/// supplied signature AlgorithmIdentifier. Its error is returned unchanged, without retry. +/// This lets apps retain platform/keychain/token signing without exporting a private key. +pub fn sign_with(input: SigningInput<'_>, sign: impl FnOnce(&[u8]) -> Result, E>) -> Result, E> { + let SigningInput { cert, chain, alg, content_type, content, content_digest, signature_algorithm } = input; + let cert_hash = DigestAlg::Sha256.digest(&[cert.der]); + let general_names = der::seq(&[&der::explicit(4, cert.issuer_der)]); // The serial as the certificate has it: `cert.serial` is the INTEGER's own bytes, so one with // its high bit set gets no leading zero here. - let serial = der::tlv(tag::INTEGER, if cert.serial.is_empty() { &[0] } else { &cert.serial }); + let serial = der::tlv(tag::INTEGER, if cert.serial.is_empty() { &[0] } else { cert.serial }); let ess = der::seq(&[&der::seq(&[&der::seq(&[&der::octets(&cert_hash), &der::seq(&[&general_names, &serial])])])]); let attrs = [attribute(CONTENT_TYPE, &der::oid(content_type)), attribute(MESSAGE_DIGEST, &der::octets(content_digest)), attribute(SIGNING_CERT_V2, &ess)]; let refs: Vec<&[u8]> = attrs.iter().map(Vec::as_slice).collect(); let set = der::set_of(&refs); - let signature = key.sign(alg, &set)?; + let signature = sign(&set)?; let mut signed_attrs = set.clone(); signed_attrs[0] = tag::ctx(0); - let sid = der::seq(&[&cert.issuer.raw, &serial]); - let signer = der::seq(&[&der::int(1), &sid, &alg.algorithm(), &signed_attrs, &key.signature_algorithm(alg), &der::octets(&signature)]); - let mut certs: Vec<&[u8]> = vec![&cert.raw]; - certs.extend(chain.iter().filter(|c| c.raw != cert.raw).map(|c| c.raw.as_slice())); + let sid = der::seq(&[cert.issuer_der, &serial]); + let signer = der::seq(&[&der::int(1), &sid, &alg.algorithm(), &signed_attrs, signature_algorithm, &der::octets(&signature)]); + let mut certs: Vec<&[u8]> = vec![cert.der]; + certs.extend(chain.iter().copied().filter(|c| *c != cert.der)); let encap = match content { Some(c) => der::seq(&[&der::oid(content_type), &der::explicit(0, &der::octets(c))]), None => der::seq(&[&der::oid(content_type)]), @@ -477,10 +507,76 @@ fn build(input: SignedDataInput<'_>) -> Result, Error> { Ok(der::seq(&[&der::oid(SIGNED_DATA), &der::explicit(0, &signed_data)])) } +fn build(input: SignedDataInput<'_>) -> Result, Error> { + let SignedDataInput { key, cert, chain, alg, content_type, content, content_digest } = input; + let chain: Vec<&[u8]> = chain.iter().map(|c| c.raw.as_slice()).collect(); + sign_with( + SigningInput { + cert: SigningCertificate { der: &cert.raw, issuer_der: &cert.issuer.raw, serial: &cert.serial }, + chain: &chain, + alg, + content_type, + content, + content_digest, + signature_algorithm: &key.signature_algorithm(alg), + }, + |attrs| key.sign(alg, attrs), + ) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn callback_signer_matches_builtin_and_returns_provider_error_once() -> Result<(), Box> { + use crate::x509::Name; + let key = PrivateKey::generate_p256()?; + let name = Name::build("Callback signer", "", "", "", ""); + let from = Time { year: 2026, month: 1, day: 1, hour: 0, minute: 0, second: 0 }; + let cert = Certificate::self_signed(&name, &key, from, 1, &[1])?; + let alg = DigestAlg::Sha256; + let bytes = b"content"; + let digest = alg.digest(&[bytes]); + let algorithm = key.signature_algorithm(alg); + let chain = [cert.raw.as_slice()]; + for content in [None, Some(bytes.as_slice())] { + let input = SigningInput { + cert: SigningCertificate { der: &cert.raw, issuer_der: &cert.issuer.raw, serial: &cert.serial }, + chain: &chain, + alg, + content_type: DATA, + content, + content_digest: &digest, + signature_algorithm: &algorithm, + }; + let mut calls = 0; + let via_callback = sign_with(input, |attrs| { + calls += 1; + assert_eq!(Tlv::parse_all(attrs)?.tag, tag::SET); + key.sign(alg, attrs) + })?; + assert_eq!(calls, 1); + let builtin = match content { + Some(c) => sign_encapsulated(&key, &cert, std::slice::from_ref(&cert), alg, DATA, c)?, + None => sign_detached(&key, &cert, std::slice::from_ref(&cert), alg, &digest)?, + }; + assert_eq!(via_callback, builtin); + let parsed = SignedData::parse(&via_callback)?; + assert_eq!(parsed.certificates.len(), 1); + assert!(parsed.verify_signature(&cert, &digest)?); + assert_eq!(parsed.content.as_deref(), content); + let mut calls = 0; + let failed = sign_with(input, |_| { + calls += 1; + Err(42u32) + }); + assert_eq!(failed, Err(42)); + assert_eq!(calls, 1); + } + Ok(()) + } + #[test] fn strip_removes_multiple_leading_zero_bytes() { assert_eq!(strip(&[0, 0, 0, 0x42]), &[0x42]); From 472462df9f1cb7316a92d1be21cdbf90e0980300 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 20:36:13 -0500 Subject: [PATCH 123/209] craft-hdr: direct RGBA decoding and preallocation axis limits (#1961) --- crates/hdr/src/lib.rs | 40 ++++++++++++++++++++++++++++++++++------ crates/hdr/src/tests.rs | 21 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/hdr/src/lib.rs b/crates/hdr/src/lib.rs index d46232306..731737ce1 100644 --- a/crates/hdr/src/lib.rs +++ b/crates/hdr/src/lib.rs @@ -43,16 +43,19 @@ pub type Result = std::result::Result; #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub struct Limits { - /// Most pixels (width × height) a picture may have; each takes 12 bytes decoded. Default 2^26. + /// Most pixels (width × height) a picture may have; each takes 12 bytes as RGB or 16 as RGBA. Default 2^26. /// A picture over the target's largest allocation is refused at any setting. pub max_pixels: u64, /// Longest header read, in bytes (real ones are a few hundred). Default 64 KiB. pub max_header: usize, + /// Largest width or height, checked before allocating the validation scan line. + /// Defaults to `u32::MAX`, preserving the existing RGB decoder limit. + pub max_axis: u32, } impl Default for Limits { fn default() -> Self { - Self { max_pixels: 1 << 26, max_header: 64 << 10 } + Self { max_pixels: 1 << 26, max_header: 64 << 10, max_axis: u32::MAX } } } @@ -65,6 +68,15 @@ pub struct Picture { pub rgb: Vec, } +/// A decoded opaque RGBA picture, without an intermediate RGB allocation. +#[derive(Debug, Clone, PartialEq)] +pub struct RgbaPicture { + pub width: u32, + pub height: u32, + /// Scene-linear RGBA, top row first. Every alpha sample is 1.0. + pub rgba: Vec, +} + /// The largest allocation in bytes that a `Vec` or `String` can make on the target. const MAX_ALLOC_BYTES: u64 = isize::MAX as u64; @@ -187,6 +199,19 @@ pub fn decode(b: &[u8], limits: &Limits) -> Result { /// [`decode`] with `max_alloc_bytes` as the largest allocation the target allows. fn decode_within(b: &[u8], limits: &Limits, max_alloc_bytes: u64) -> Result { + let (width, height, rgb) = decode_channels::<3>(b, limits, max_alloc_bytes)?; + Ok(Picture { width, height, rgb }) +} + +/// Decodes directly to opaque linear float RGBA, with no intermediate RGB image. +/// The allocation limit includes all four channels; scan lines are validated first. +pub fn decode_rgba(b: &[u8], limits: &Limits) -> Result { + let (width, height, rgba) = decode_channels::<4>(b, limits, MAX_ALLOC_BYTES)?; + Ok(RgbaPicture { width, height, rgba }) +} + +// Only the RGB and RGBA entry points call this helper. +fn decode_channels(b: &[u8], limits: &Limits, max_alloc_bytes: u64) -> Result<(u32, u32, Vec)> { if !sniff(b) { return Err(Error::NotHdr); } @@ -235,13 +260,16 @@ fn decode_within(b: &[u8], limits: &Limits, max_alloc_bytes: u64) -> Result limits.max_axis || height > limits.max_axis { + return Err(Error::LimitExceeded("axis length")); + } let pixels = u64::from(width) * u64::from(height); if pixels > limits.max_pixels { return Err(Error::LimitExceeded("pixel count")); } // One scan line of RGBE pixels and the float picture, each within the target's largest allocation. let scan_len = alloc_len(u64::from(if x1 { height } else { width }), size_of::<[u8; 4]>(), max_alloc_bytes); - let rgb_len = pixels.checked_mul(3).and_then(|floats| alloc_len(floats, size_of::(), max_alloc_bytes)); + let rgb_len = pixels.checked_mul(CHANNELS as u64).and_then(|floats| alloc_len(floats, size_of::(), max_alloc_bytes)); let (Some(scan_len), Some(total)) = (scan_len, rgb_len) else { return Err(Error::LimitExceeded("pixel count")) }; // A scan line takes at least a few bits of file, so a header cannot ask for more than the data has. if (b.len().saturating_sub(pos) as u64).saturating_mul(4096) < pixels { @@ -255,7 +283,7 @@ fn decode_within(b: &[u8], limits: &Limits, max_alloc_bytes: u64) -> Result Result(bytes, &Limits::default(), 28).is_ok()); + assert_eq!(decode_channels::<4>(bytes, &Limits::default(), 28), Err(Error::LimitExceeded("pixel count"))); +} + +#[test] +fn axis_limit_precedes_pixel_data_and_scratch_allocation() { + let limits = Limits { max_axis: 32768, ..Limits::default() }; + for resolution in ["-Y 1 +X 32769", "-Y 32769 +X 1", "+X 1 -Y 32769"] { + let bytes = format!("#?RADIANCE\n\n{resolution}\n"); + assert_eq!(decode_rgba(bytes.as_bytes(), &limits), Err(Error::LimitExceeded("axis length"))); + assert_eq!(decode(bytes.as_bytes(), &limits), Err(Error::LimitExceeded("axis length"))); + } +} From 006ed3d965d5f3b77829b87a43eacd5ca0b4f997 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 20:46:05 -0500 Subject: [PATCH 124/209] svg: share bounded simple stylesheet parsing and matching (#1968) --- crates/svg/README.md | 15 ++ crates/svg/src/lib.rs | 3 + crates/svg/src/simple_css.rs | 346 +++++++++++++++++++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 crates/svg/README.md create mode 100644 crates/svg/src/simple_css.rs diff --git a/crates/svg/README.md b/crates/svg/README.md new file mode 100644 index 000000000..a387e58c2 --- /dev/null +++ b/crates/svg/README.md @@ -0,0 +1,15 @@ + +### Simple stylesheet compatibility subset + +`simple_css` preserves EffectCraft's compound tag/class/id stylesheet grammar and indexed +matching budget, rather than substituting the richer selector semantics of `paths`. +Property values remain text; color precision, path-prefix behavior and geometry policy +stay with the importer. Call `parse` for each sheet, then `StyleSheet::index`; discard the +sheet after a parse error and reject the import when its matching-work callback refuses. + +The parser/matcher and structural tests were ported from EffectCraft +`60e2cbbbe3d73c0a9074c25a1a615ccfec18721c`, originally by FredApps. Nested at-rule skipping +and its regression test preserve NetterSilver's +`10e173c0e49696ad335e3da8a069889c089b09a6`; both contributors retain primary Git authorship +for their respective port commits. Source and destination use MIT OR Apache-2.0. +The shared adapter adds an aggregate 64 MiB stylesheet-text limit and idempotent indexing. diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs index 7cd36692e..286bd36d5 100644 --- a/crates/svg/src/lib.rs +++ b/crates/svg/src/lib.rs @@ -30,3 +30,6 @@ pub mod svgz; #[cfg(feature = "tree")] pub mod tree; pub mod write; + +/// Compatibility CSS subset for simple SVG importers. +pub mod simple_css; diff --git a/crates/svg/src/simple_css.rs b/crates/svg/src/simple_css.rs new file mode 100644 index 000000000..767b25257 --- /dev/null +++ b/crates/svg/src/simple_css.rs @@ -0,0 +1,346 @@ +//! Compatibility stylesheet subset for simple SVG importers, extracted from EffectCraft. +//! +//! Supports compound tag/class/id selectors and comma lists. Unsupported selectors/at-rules +//! are skipped; malformed trailing rules keep the accepted prefix. This deliberately is not +//! the richer `paths` cascade or a full CSS parser. Property values are retained as text, so +//! caller colour precision and geometry policy are unaffected. `!important` is stripped as in +//! the source importer; it does not change precedence here. +//! +//! Call `parse` for each style element, then `StyleSheet::index` before matching. Discard a +//! sheet after a parse error. Matching charges the caller's aggregate budget before applying +//! any declarations. The caller must stop using results when its budget callback refuses. +use std::collections::HashMap; + +/// Aggregate stylesheet byte bound before copying or allocating declaration strings. +pub const MAX_CSS_BYTES: usize = 64 * 1024 * 1024; + +/// A stylesheet exceeds its byte, selector or declaration limit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(pub String); +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} +impl std::error::Error for Error {} + +/// The most selectors (each entry of a comma list counts) the `

x

"), &|_| None, true); } } + +/// Ensure container elements (such as `div` and `p`) with background colors do not leak +/// character shading onto their child text runs (#1786). +#[test] +fn container_background_color_does_not_leak_into_character_shading() { + let html = r#"
code
"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert!(!p.runs.is_empty(), "expected nonempty runs"); + assert_eq!(&p.text, "code"); + assert_eq!(p.props.shading, Some(craft_wordmodel::Rgb(0x1e, 0x1e, 0x1e)), "container div background is preserved as paragraph shading"); + for r in &p.runs { + assert_eq!(r.props.shading, None, "character should have no shading from container div"); + } +} + +/// Verify that table background colors cascade down to cell shading through implicit or explicit +/// row groups (`tbody`, `thead`, `tfoot`), but do not leak into cell character shading. +#[test] +fn table_background_inherits_through_row_groups() { + let html = r#"
y
cell1
+
cell2
+
head
"#; + let doc = import(html); + let tables: Vec<_> = doc + .body + .iter() + .filter_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .collect(); + assert_eq!(tables.len(), 3); + for (t, want_hex) in tables.iter().zip(["e8f0fa", "d0e0f0", "c0d0e0"]) { + let cell = &t.rows[0].cells[0]; + assert_eq!( + cell.props.shading.map(|c| c.hex().to_ascii_lowercase()), + Some(want_hex.to_string()), + "cell should inherit shading from table or row group" + ); + let p = &cell.blocks[0]; + let craft_wordmodel::Block::Para(p) = &**p else { + panic!("expected table cell block to be a paragraph"); + }; + assert!(!p.runs.is_empty(), "expected nonempty runs"); + for r in &p.runs { + assert_eq!(r.props.shading, None, "cell text should have no character shading"); + } + } +} + +/// Verify that block-level container backgrounds (`p`, `pre`, `div`) are preserved as paragraph +/// shading on paragraphs rather than character shading on individual runs (#1786). +#[test] +fn block_element_backgrounds_preserve_paragraph_shading() { + let html = r#"

text

code

inner

"#; + let doc = import(html); + assert_eq!(doc.body.len(), 3); + let craft_wordmodel::Block::Para(p1) = &*doc.body[0] else { panic!("expected Para") }; + let craft_wordmodel::Block::Para(p2) = &*doc.body[1] else { panic!("expected Para") }; + let craft_wordmodel::Block::Para(p3) = &*doc.body[2] else { panic!("expected Para") }; + assert_eq!(p1.props.shading, Some(craft_wordmodel::Rgb(0x1e, 0x1e, 0x1e))); + assert_eq!(p2.props.shading, Some(craft_wordmodel::Rgb(0x2a, 0x2a, 0x2a))); + assert_eq!(p3.props.shading, Some(craft_wordmodel::Rgb(0x3a, 0x3a, 0x3a))); + assert_eq!(&p1.text, "text"); + assert_eq!(&p2.text, "code"); + assert_eq!(&p3.text, "inner"); + assert!(!p1.runs.is_empty()); + assert!(!p2.runs.is_empty()); + assert!(!p3.runs.is_empty()); + for r in &p1.runs { + assert_eq!(r.props.shading, None, "character should have no shading from paragraph background"); + } + for r in &p2.runs { + assert_eq!(r.props.shading, None, "character should have no shading from pre background"); + } + for r in &p3.runs { + assert_eq!(r.props.shading, None, "character should have no shading from container div background"); + } +} + +/// Verify that paragraph shading created by a container background survives an HTML round trip (#1786). +#[test] +fn paragraph_shading_survives_an_html_round_trip() { + let html = r#"

code

pre
"#; + let doc = import(html); + let out = export(&doc); + assert!(out.contains("background-color:#1E1E1E"), "paragraph shading is exported: {out}"); + assert!(out.contains("background-color:#2A2A2A"), "code block shading is exported: {out}"); + + let back = import(&out); + let craft_wordmodel::Block::Para(p1) = &*back.body[0] else { panic!("expected Para") }; + let craft_wordmodel::Block::Para(p2) = &*back.body[1] else { panic!("expected Para") }; + assert_eq!(p1.props.shading, Some(craft_wordmodel::Rgb(0x1e, 0x1e, 0x1e))); + assert_eq!(p2.props.shading, Some(craft_wordmodel::Rgb(0x2a, 0x2a, 0x2a))); +} + +/// Verify that paragraph shading inside a single-paragraph table cell survives an HTML round trip (#1786). +#[test] +fn table_cell_paragraph_shading_survives_an_html_round_trip() { + let html = r#"

text

"#; + let doc = import(html); + let out = export(&doc); + assert!(out.contains("background-color:#123456"), "paragraph shading is exported: {out}"); + + let back = import(&out); + let table = back + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .expect("expected table"); + let cell = &table.rows[0].cells[0]; + assert_eq!(cell.props.shading, None, "paragraph shading must not become cell shading"); + let craft_wordmodel::Block::Para(p) = &*cell.blocks[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); +} + +/// Verify that consecutive code paragraphs with different backgrounds export their own `
`.
+#[test]
+fn consecutive_code_blocks_keep_their_own_shading() {
+    let html = r#"
a
b
"#; + let out = export(&import(html)); + assert!(out.contains("
a
"), "{out}"); + assert!(out.contains("
b
"), "{out}"); +} + +/// Verify that paragraph shading inherited from a paragraph style is exported (#1786). +#[test] +fn paragraph_shading_from_a_style_survives_html_export() { + let mut doc = Document::new(); + std::sync::Arc::make_mut(&mut doc.styles).styles.push(craft_wordmodel::Style { + id: "Shaded".into(), + name: "Shaded".into(), + para: craft_wordmodel::props::ParaProps { shading: Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56)), ..Default::default() }, + ..Default::default() + }); + let mut p = craft_wordmodel::Paragraph::with_text("text", craft_wordmodel::CharProps::default()); + p.props.style = Some("Shaded".into()); + doc.body = vec![craft_wordmodel::para_block(p)]; + + let out = export(&doc); + assert!(out.contains("background-color:#123456"), "style shading is exported: {out}"); +} + +/// A closing parenthesis without an opening one invalidates the declaration (#1786). +#[test] +fn background_shorthand_unbalanced_closing_paren_is_ignored() { + let html = r#"text"#; + let doc = import(html); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(0xff, 0x00, 0x00)), "invalid shorthand must not clear inherited shading"); +} + +/// A shaded paragraph keeps its shading on both sides of an internal page break (#1786). +#[test] +fn paragraph_shading_survives_an_internal_page_break() { + let mut p = craft_wordmodel::Paragraph::with_text("before\u{000C}after", craft_wordmodel::CharProps::default()); + p.props.shading = Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56)); + let mut doc = Document::new(); + doc.body = vec![craft_wordmodel::para_block(p)]; + let out = export(&doc); + assert_eq!(out.matches("background-color:#123456").count(), 2, "{out}"); +} + +/// Verify that CSS-escaped quote characters in background shorthand URLs remain part of the URL +/// content and do not cause the shorthand declaration to be rejected. +#[test] +fn background_shorthand_escaped_quotes_in_url() { + let doc = import(r#"

text

"#); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(&p.text, "text"); + assert!(!p.runs.is_empty()); + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); +} + +/// Verify that valid multi-token CSS background shorthands without a color (such as `none repeat`) +/// reset character shading, while invalid declarations are ignored. +#[test] +fn background_shorthand_without_color_resets_background() { + let html = r#"plain"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, None); + + let html_invalid = r#"shaded"#; + let doc_invalid = import(html_invalid); + let p_invalid = match &*doc_invalid.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_invalid.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); +} + +/// Verify that CSS `background: inherit` keeps inherited character shading instead of removing it (#1786). +#[test] +fn background_inherit_retains_inline_shading() { + let html = r#"shaded"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0)), "background: inherit should retain inherited character shading"); + + // Also verify background-color: inherit retains shading. + let html_color = r#"shaded"#; + let doc_color = import(html_color); + let p_color = match &*doc_color.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!( + p_color.runs[0].props.shading, + Some(craft_wordmodel::Rgb(255, 0, 0)), + "background-color: inherit should retain inherited character shading" + ); + + // CSS-wide keyword inherit mixed with other values is invalid and ignored. + let html_mixed = r#"green"#; + let doc_mixed = import(html_mixed); + let p_mixed = match &*doc_mixed.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!( + p_mixed.runs[0].props.shading, + Some(craft_wordmodel::Rgb(0, 255, 0)), + "invalid shorthand mixing inherit with values should be ignored" + ); +} + +/// Verify that hostile background declarations with deeply nested slashes do not cause stack overflow (#1786). +#[test] +fn hostile_background_slashes_never_stack_overflow() { + let slashes = "/".repeat(20_000); + let html = format!(r#"hostile"#); + let doc = import(&html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, None); +} + +/// Verify that non-breaking space (U+00A0) is not treated as CSS whitespace when splitting tokens (#1786). +#[test] +fn background_whitespace_preserves_nbsp() { + let html = "shaded"; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!( + p.runs[0].props.shading, + Some(craft_wordmodel::Rgb(255, 0, 0)), + "non-breaking space is not CSS whitespace and should not clear shading" + ); +} + +/// Verify that background shorthands exceeding the token limit are rejected and unapplied (#1786). +#[test] +fn background_shorthand_truncated_rejects_unapplied() { + let mut layers = String::new(); + for i in 0..64 { + layers.push_str(&format!("url(img{i}.png), ")); + } + layers.push_str("#00ff00"); + let html = format!(r#"retained"#); + let doc = import(&html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!( + p.runs[0].props.shading, + Some(craft_wordmodel::Rgb(255, 0, 0)), + "truncated shorthand should be rejected and leave inherited background untouched" + ); +} + +/// Verify that shorthands with image-set or calc functions parse and preserve colors (#1786). +#[test] +fn background_shorthand_valid_functions_preserve_color() { + let html_image_set = r#"image-set"#; + let doc = import(html_image_set); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); + + let html_calc = r#"calc"#; + let doc_calc = import(html_calc); + let p_calc = match &*doc_calc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_calc.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); +} + +/// Verify that multi-layer background shorthands validate slashes per layer and preserve final-layer color (#1786). +#[test] +fn background_shorthand_multi_layer_slashes_apply_color() { + let html = r#"multi-layer"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); + + // Color in a non-final layer is invalid CSS and must be ignored. + let html_invalid_color = r#"invalid"#; + let doc_invalid_color = import(html_invalid_color); + let p_invalid_color = match &*doc_invalid_color.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_invalid_color.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + // Multiple slashes in a single layer makes the declaration invalid. + let html_multi_slash = + r#"invalid"#; + let doc_multi_slash = import(html_multi_slash); + let p_multi_slash = match &*doc_multi_slash.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_multi_slash.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); +} + +/// Verify that background shorthands with empty layers (leading, trailing, or double commas) are rejected (#1786). +#[test] +fn background_shorthand_rejects_empty_layers() { + // Leading comma + let html_leading = r#"leading"#; + let doc_leading = import(html_leading); + let p_leading = match &*doc_leading.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_leading.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + // Trailing comma + let html_trailing = r#"trailing"#; + let doc_trailing = import(html_trailing); + let p_trailing = match &*doc_trailing.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_trailing.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + // Double comma + let html_double = r#"double"#; + let doc_double = import(html_double); + let p_double = match &*doc_double.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_double.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); +} + +/// Verify that zero-alpha colors in background shorthands or declarations remain transparent (#1786). +#[test] +fn background_zero_alpha_colors_are_transparent() { + for decl in ["rgb(0 0 0 / 0)", "rgba(0, 0, 0, 0)", "#0000", "#00000000", "#ffffff00", "hsla(120, 100%, 50%, 0)", "hsl(120 100% 50% / 0)"] { + let html = format!(r#"transparent"#); + let doc = import(&html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, None, "zero-alpha color `{decl}` must clear background rather than composite over white"); + } +} + +/// Verify that ancestor table shading is preserved beneath transparent table layers (#1786). +#[test] +fn transparent_table_layers_preserve_ancestor_table_shading() { + let html = r#" + + + + + + +
transparent_layersexplicit_cell
"#; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!( + table.rows[0].cells[0].props.shading, + Some(craft_wordmodel::Rgb(0xe8, 0xf0, 0xfa)), + "table background must show through transparent tbody, tr, and td" + ); + assert_eq!( + table.rows[0].cells[1].props.shading, + Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56)), + "cell with explicit background must override table background" + ); +} + +/// Verify that block-level container background is available to immediate inline children using explicit `inherit` (#1786). +#[test] +fn block_background_explicit_inherit() { + // When the immediate parent

has no background, an inline span inheriting background gets transparent (None): + let html_indirect = r#"

text

"#; + let doc_indirect = import(html_indirect); + let p_indirect = match &*doc_indirect.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_indirect.runs[0].props.shading, None); + + // When the span is an immediate child of the styled div, it inherits the div's computed background: + let html_direct = r#"
text
"#; + let doc_direct = import(html_direct); + let p_direct = match &*doc_direct.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_direct.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + let html_shorthand = r#"
text
"#; + let doc_shorthand = import(html_shorthand); + let p_shorthand = match &*doc_shorthand.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_shorthand.runs[0].props.shading, Some(craft_wordmodel::Rgb(0, 255, 0))); +} + +/// Verify that CSS-escaped quotes inside background URLs do not close the quote or drop declaration colors (#1786). +#[test] +fn background_escaped_quote_url() { + let html = r#"text"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); + + let html_single = r#"text"#; + let doc_single = import(html_single); + let p_single = match &*doc_single.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_single.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x65, 0x43, 0x21))); +} + +/// Verify that invalid hex color declarations like `#ggg0` do not clear inherited shading (#1786). +#[test] +fn background_invalid_hex_retains_shading() { + let html = r#"text"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); +} + +/// Verify that background size keywords (`cover`, `contain`) without preceding position and slash are invalid and retain shading (#1786). +#[test] +fn background_cover_without_slash_invalid() { + let html_cover = r#"text"#; + let doc_cover = import(html_cover); + let p_cover = match &*doc_cover.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_cover.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + let html_contain = r#"text"#; + let doc_contain = import(html_contain); + let p_contain = match &*doc_contain.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_contain.runs[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + let html_valid = r#"text"#; + let doc_valid = import(html_valid); + let p_valid = match &*doc_valid.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p_valid.runs[0].props.shading, None); +} + +/// Verify that table backgrounds declared on thead or tbody are applied to cells (#1786). +#[test] +fn table_row_group_background_inheritance() { + let html = r#" + + + + + + +
head
body
"#; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(table.rows[0].cells[0].props.shading, Some(craft_wordmodel::Rgb(0, 255, 0))); + assert_eq!(table.rows[1].cells[0].props.shading, Some(craft_wordmodel::Rgb(0, 0, 255))); +} + +/// Verify that background shorthand with slashes inside URL functions correctly identifies the position/size slash (#1786). +#[test] +fn background_url_with_slashes_and_position_size() { + let html = r#"text"#; + let doc = import(html); + let p = match &*doc.body[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); +} + +/// Verify that a table inside an inline element with background shading does not turn character shading into table/cell background (#1786). +#[test] +fn table_inside_inline_element_does_not_inherit_character_shading() { + let html = r#"
cell
"#; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(table.rows[0].cells[0].props.shading, None); +} + +/// Verify that a child with `background: inherit` inside an unstyled cell in a shaded table does not inherit the table fallback (#1786). +#[test] +fn cell_inherit_uses_cell_background_not_table_fallback() { + let html = r#"
text
"#; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(table.rows[0].cells[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + let p = match &*table.rows[0].cells[0].blocks[0] { + craft_wordmodel::Block::Para(p) => p, + _ => panic!("expected paragraph"), + }; + assert_eq!(p.runs[0].props.shading, None); +} + +/// Verify that CSS `background: transparent` overrides HTML `bgcolor` presentational attribute (#1786). +#[test] +fn bgcolor_overridden_by_css_transparent() { + let html = r##"
cell
"##; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(table.rows[0].cells[0].props.shading, None); +} + +/// Verify that nested container shading is preserved when merging list continuation paragraphs (#1786). +#[test] +fn list_continuation_preserves_nested_container_shading() { + let html = r#"
  • before
    inside
    after
"#; + let flow = parse(html); + assert_eq!(flow.blocks.len(), 1); + let FBlock::Para(p) = &flow.blocks[0] else { panic!("expected Para") }; + assert!(p.list.is_some(), "expected list item"); + let dark = Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11)); + let inside = p + .inlines + .iter() + .find_map(|i| match i { + Inline::Text(t, fmt) if t.contains("inside") => Some(fmt.background), + _ => None, + }) + .expect("inside text inline"); + assert_eq!(inside, dark, "nested container shading should be preserved on merged run"); + + let doc = import(html); + let craft_wordmodel::Block::Para(p_doc) = &*doc.body[0] else { panic!("expected Para") }; + let mut offset = 0usize; + let mut inside_shading = None; + for r in &p_doc.runs { + let text_slice = &p_doc.text[offset..offset + r.len]; + if text_slice.contains("inside") { + inside_shading = Some(r.props.shading); + } else if text_slice.contains("before") || text_slice.contains("after") { + assert_eq!(r.props.shading, None); + } + offset += r.len; + } + assert_eq!(inside_shading, Some(dark)); +} + +/// Verify that starting a new row group implicitly closes any preceding sibling row group so +/// cell shading is not inherited from an omitted-end-tag preceding group (#1786). +#[test] +fn consecutive_row_groups_with_omitted_end_tag_do_not_leak_shading() { + let html = r#"
a
b
"#; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(table.rows.len(), 2); + assert_eq!(table.rows[0].cells[0].props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + assert_eq!(table.rows[1].cells[0].props.shading, None, "second cell should not inherit shading from closed tbody"); +} + +/// Verify that a list item with a shaded container followed by unshaded text preserves separate +/// background regions so the trailing text does not inherit the container's background (#1786). +#[test] +fn shaded_container_first_in_list_item_preserves_unshaded_trailing_text() { + let html = r#"
  • inside
    after
"#; + let doc = import(html); + assert_eq!(doc.body.len(), 1); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, None, "list paragraph as a whole should have no background when regions differ"); + let dark = Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11)); + let mut offset = 0usize; + let mut inside_shading = None; + let mut after_shading = None; + for r in &p.runs { + let text_slice = &p.text[offset..offset + r.len]; + if text_slice.contains("inside") { + inside_shading = Some(r.props.shading); + } else if text_slice.contains("after") { + after_shading = Some(r.props.shading); + } + offset += r.len; + } + assert_eq!(inside_shading, Some(dark)); + assert_eq!(after_shading, Some(None), "trailing text should have no shading"); +} + +/// Verify that container backgrounds behind tables are preserved on table cells (#1786). +#[test] +fn container_background_behind_table_preserved_on_cells() { + let html = r#"
text
"#; + let doc = import(html); + let table = doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(table.rows[0].cells[0].props.shading, Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11))); + + // Nested table inside an unshaded cell stops at the cell boundary: + let nested_html = r#"
nested
"#; + let nested_doc = import(nested_html); + let outer_table = nested_doc + .body + .iter() + .find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t), + _ => None, + }) + .unwrap(); + assert_eq!(outer_table.rows[0].cells[0].props.shading, Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11))); + let inner_table = match &*outer_table.rows[0].cells[0].blocks[0] { + craft_wordmodel::Block::Table(t) => t, + _ => panic!("expected nested Table"), + }; + assert_eq!(inner_table.rows[0].cells[0].props.shading, None, "nested table in unshaded cell must stop at cell boundary"); +} + +/// Verify that page imports preserve root (body/html) backgrounds while pasted fragments do not (#1786). +#[test] +fn root_background_preserved_on_page_import_only() { + let html = r#"

text

"#; + let page_doc = import_page(html, &|_| None); + let craft_wordmodel::Block::Para(p_page) = &*page_doc.body[0] else { panic!("expected Para") }; + assert_eq!(p_page.props.shading, Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11)), "page import must preserve body background on paragraphs"); + assert_eq!( + p_page.runs[0].props.color, + Some(craft_wordmodel::TextColor::Rgb(craft_wordmodel::Rgb(0xff, 0xff, 0xff))), + "page import keeps the root foreground with its background" + ); + + let frag_doc = import(html); + let craft_wordmodel::Block::Para(p_frag) = &*frag_doc.body[0] else { panic!("expected Para") }; + assert_eq!(p_frag.props.shading, None, "fragment paste must not turn body background into paragraph shading"); + assert_eq!(p_frag.runs[0].props.color, None, "fragment paste drops the root foreground with its background"); +} + +/// Verify that an inline child with `background: inherit` does not receive character shading from a pasted body background (#1786). +#[test] +fn pasted_body_background_not_inherited_by_inline_child() { + let html = r#"text"#; + let frag_doc = import(html); + let craft_wordmodel::Block::Para(p_frag) = &*frag_doc.body[0] else { panic!("expected Para") }; + assert_eq!(p_frag.runs[0].props.shading, None, "inline child in pasted fragment must not inherit character shading from body background"); + + let page_doc = import_page(html, &|_| None); + let craft_wordmodel::Block::Para(p_page) = &*page_doc.body[0] else { panic!("expected Para") }; + assert_eq!( + p_page.runs[0].props.shading, + Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11)), + "inline child in page import inherits body background via inherit" + ); +} + +#[test] +fn root_background_on_html_and_foreground_on_body() { + let html = r#"

text

"#; + let frag_doc = import(html); + let craft_wordmodel::Block::Para(p_frag) = &*frag_doc.body[0] else { panic!("expected Para") }; + assert_eq!(p_frag.props.shading, None, "fragment paste drops html background"); + assert_eq!(p_frag.runs[0].props.color, None, "fragment paste drops body foreground when html supplied suppressed background"); + + let page_doc = import_page(html, &|_| None); + let craft_wordmodel::Block::Para(p_page) = &*page_doc.body[0] else { panic!("expected Para") }; + assert_eq!(p_page.props.shading, Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11))); + assert_eq!(p_page.runs[0].props.color, Some(craft_wordmodel::TextColor::Rgb(craft_wordmodel::Rgb(0xff, 0xff, 0xff)))); +} + +#[test] +fn background_unitless_zero_lengths() { + let html = r#"

text

"#; + let doc = import(html); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, Some(craft_wordmodel::Rgb(0x12, 0x34, 0x56))); + + let html2 = r#"

text

"#; + let doc2 = import(html2); + let craft_wordmodel::Block::Para(p2) = &*doc2.body[0] else { panic!("expected Para") }; + assert_eq!(p2.props.shading, Some(craft_wordmodel::Rgb(0x65, 0x43, 0x21))); +} + +#[test] +fn background_comments_skipped_outside_strings() { + let html = r#"

text

"#; + let doc = import(html); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11))); + + let html2 = r#"

text

"#; + let doc2 = import(html2); + let craft_wordmodel::Block::Para(p2) = &*doc2.body[0] else { panic!("expected Para") }; + assert_eq!(p2.props.shading, Some(craft_wordmodel::Rgb(0x22, 0x22, 0x22))); +} + +#[test] +fn invalid_color_does_not_clear_preceding_background() { + let html = r#"

text

"#; + let doc = import(html); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); + + let html2 = r#"

text

"#; + let doc2 = import(html2); + let craft_wordmodel::Block::Para(p2) = &*doc2.body[0] else { panic!("expected Para") }; + assert_eq!(p2.props.shading, Some(craft_wordmodel::Rgb(255, 0, 0))); +} + +#[test] +fn code_block_export_preserves_foreground_color() { + let html = r#"
const x = 1;
"#; + let doc = import_page(html, &|_| None); + let exported = export(&doc); + assert!(exported.contains("style=\"background-color:#000000\""), "exported pre must keep background: {}", exported); + assert!(exported.contains("color:#FFFFFF"), "exported code inlines must keep foreground: {}", exported); + assert!(exported.contains("const x = 1;"), "exported code text preserved: {}", exported); +} + +#[test] +fn background_comments_inside_functions() { + let html = r#"

text

"#; + let doc = import(html); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, Some(craft_wordmodel::Rgb(0x11, 0x11, 0x11))); + assert_eq!(p.runs[0].props.color, Some(craft_wordmodel::TextColor::Rgb(craft_wordmodel::Rgb(0xff, 0xff, 0xff)))); +} + +#[test] +fn zero_alpha_colors_with_none_channels() { + let html = r#"

text

"#; + let doc = import(html); + let craft_wordmodel::Block::Para(p) = &*doc.body[0] else { panic!("expected Para") }; + assert_eq!(p.props.shading, None); + + let html2 = r#"

text

"#; + let doc2 = import(html2); + let craft_wordmodel::Block::Para(p2) = &*doc2.body[0] else { panic!("expected Para") }; + assert_eq!(p2.props.shading, None); +} From cfad8f0bcca6900ddd88361220fb5ed432ce817e Mon Sep 17 00:00:00 2001 From: el-j Date: Tue, 17 Nov 2026 03:30:57 +0100 Subject: [PATCH 152/209] craft-captions: PJS, RealText and JACOsub subtitle formats (#1557) (#1788) --- crates/captions/src/jacosub.rs | 242 +++++++++++++++++ crates/captions/src/lib.rs | 50 +++- crates/captions/src/pjs.rs | 102 ++++++++ crates/captions/src/realtext.rs | 247 ++++++++++++++++++ crates/captions/src/text.rs | 46 ++++ crates/captions/tests/it/main.rs | 1 + .../captions/tests/it/pjs_realtext_jacosub.rs | 174 ++++++++++++ 7 files changed, 860 insertions(+), 2 deletions(-) create mode 100644 crates/captions/src/jacosub.rs create mode 100644 crates/captions/src/pjs.rs create mode 100644 crates/captions/src/realtext.rs create mode 100644 crates/captions/tests/it/pjs_realtext_jacosub.rs diff --git a/crates/captions/src/jacosub.rs b/crates/captions/src/jacosub.rs new file mode 100644 index 000000000..30233b6f1 --- /dev/null +++ b/crates/captions/src/jacosub.rs @@ -0,0 +1,242 @@ +//! JACOsub (`.jss`, `.js`): multi-feature scriptable subtitle format. +//! +//! ```text +//! # JACOsub file +//! #TimeRes 100 +//! 0:00:01.00 0:00:03.00 D0 First line\nsecond line +//! ``` +//! +//! Lines beginning with `#` are comments or header directives. Timed caption lines begin with +//! `START STOP [DIRECTIVE] Text`, where timestamps are `H:MM:SS.FF` (or `@units`) relative to +//! the declared `#TimeRes` (default 30). Inside the text, `\n` represents a newline. + +use crate::text::{escape_markup, ms_to_tick, plain_text, tick_to_ms}; +use crate::{Cue, Document, Error, Result}; + +/// Parse a JACOsub timestamp `H:MM:SS.FF` or `@units` into milliseconds. +pub(crate) fn parse_time(s: &str, time_res: i64) -> Option { + let s = s.trim(); + if s.is_empty() { + return None; + } + + if let Some(units_str) = s.strip_prefix('@') { + let units: i64 = units_str.trim().parse().ok()?; + let time_res = if time_res > 0 { time_res } else { 30 }; + return Some(units.saturating_mul(1000) / time_res); + } + + let (h_str, rest) = s.split_once(':')?; + let (m_str, rest) = rest.split_once(':')?; + let (s_str, f_str) = rest.split_once('.')?; + + let h: i64 = h_str.trim().parse().ok()?; + let m: i64 = m_str.trim().parse().ok()?; + let sec: i64 = s_str.trim().parse().ok()?; + let frac: i64 = f_str.trim().parse().ok()?; + + let time_res = if time_res > 0 { time_res } else { 30 }; + let frac_ms = frac.saturating_mul(1000) / time_res; + + let total_secs = h.saturating_mul(3600).saturating_add(m.saturating_mul(60)).saturating_add(sec); + Some(total_secs.saturating_mul(1000).saturating_add(frac_ms)) +} + +/// Format milliseconds into JACOsub timestamp string `H:MM:SS.cc` (at 100 centiseconds/sec). +pub(crate) fn format_time(ms: i64) -> String { + let ms = ms.max(0); + let total_secs = ms / 1000; + let centis = (ms % 1000) / 10; + let hours = total_secs / 3600; + let mins = (total_secs % 3600) / 60; + let secs = total_secs % 60; + + format!("{hours}:{mins:02}:{secs:02}.{centis:02}") +} + +/// Check if a line is a JACOsub timed caption line: `H:MM:SS.FF H:MM:SS.FF ...` or `@units @units ...`. +pub(crate) fn is_timed_line(line: &str) -> bool { + let line = line.trim(); + if line.starts_with('#') { + return false; + } + let mut parts = line.split_whitespace(); + let Some(t0) = parts.next() else { return false }; + let Some(t1) = parts.next() else { return false }; + (t0.contains(':') && t0.contains('.') && t1.contains(':') && t1.contains('.')) || (t0.starts_with('@') && t1.starts_with('@')) +} + +pub fn parse(text: &str) -> Result { + let mut doc = Document::default(); + let mut time_res: i64 = 30; // standard default + let mut shift_ms: i64 = 0; + let mut found = false; + + for (n, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + + if let Some(cmd) = line.strip_prefix('#') { + let cmd_lower = cmd.to_ascii_lowercase(); + if cmd_lower.starts_with("timeres") || cmd_lower.starts_with('t') { + let parts: Vec<&str> = cmd.split_whitespace().collect(); + if parts.len() >= 2 + && let Ok(res) = parts[1].parse::() + && res > 0 + { + time_res = res; + } + } else if cmd_lower.starts_with("shift") || cmd_lower.starts_with('s') { + let parts: Vec<&str> = cmd.split_whitespace().collect(); + if parts.len() >= 2 { + let s_val = parts[1].trim(); + let (neg, s_raw) = if let Some(stripped) = s_val.strip_prefix('-') { + (true, stripped) + } else if let Some(stripped) = s_val.strip_prefix('+') { + (false, stripped) + } else { + (false, s_val) + }; + if let Some(ms) = parse_time(s_raw, time_res) { + shift_ms = if neg { -ms } else { ms }; + } + } + } + continue; + } + + if !is_timed_line(line) { + continue; + } + + let mut parts = line.split_whitespace(); + let Some(start_str) = parts.next() else { continue }; + let Some(end_str) = parts.next() else { continue }; + + let start_ms = match parse_time(start_str, time_res) { + Some(ms) => ms.saturating_add(shift_ms).max(0), + None => { + doc.warnings.push(format!("line {}: invalid start timestamp, skipped", n + 1)); + continue; + } + }; + let end_ms = match parse_time(end_str, time_res) { + Some(ms) => ms.saturating_add(shift_ms).max(0), + None => { + doc.warnings.push(format!("line {}: invalid end timestamp, skipped", n + 1)); + continue; + } + }; + + found = true; + + // Skip to after the end timestamp in the original line to preserve exact spaces + let after_end_pos = match line.find(end_str) { + Some(idx) => idx + end_str.len(), + None => continue, + }; + let remainder = &line[after_end_pos..]; + let trimmed_rem = remainder.trim_start(); + + // Check if there is a directive prefix: either [Directive] or a single non-space word starting with D + let text_part = if let Some(stripped) = trimmed_rem.strip_prefix('[') { + match stripped.find(']') { + Some(end_bracket) => stripped[end_bracket + 1..].trim_start(), + None => trimmed_rem, + } + } else if let Some(space_idx) = trimmed_rem.find(' ') { + let first_token = &trimmed_rem[..space_idx]; + if first_token.starts_with('D') && first_token[1..].chars().all(|c| c.is_ascii_digit()) { + trimmed_rem[space_idx + 1..].trim_start() + } else { + trimmed_rem + } + } else { + trimmed_rem + }; + + // Text parsing: `\n` represents newline + let raw_lines: Vec<&str> = text_part.split(r"\n").map(str::trim).filter(|l| !l.is_empty()).collect(); + let lines: Vec = raw_lines.into_iter().map(escape_markup).collect(); + + let start = ms_to_tick(start_ms); + let end = ms_to_tick(end_ms); + if end <= start || lines.is_empty() { + doc.warnings.push(format!("line {}: empty or ends before it starts; skipped", n + 1)); + continue; + } + + doc.cues.push(Cue { start, end, text: lines.join("\n"), ..Default::default() }); + } + + if !found && !text.trim().is_empty() { + return Err(Error::NotFormat("JACOsub")); + } + + Ok(doc) +} + +pub fn write(doc: &Document) -> String { + let mut out = String::new(); + out.push_str("# JACOsub file\n"); + out.push_str("#TimeRes 100\n\n"); + + for c in &doc.cues { + let start_ms = tick_to_ms(c.start); + let end_ms = tick_to_ms(c.end).max(start_ms + 10); + let raw_lines = plain_text(&c.text); + let lines: Vec<&str> = raw_lines.lines().map(str::trim).filter(|l| !l.is_empty()).collect(); + + if !lines.is_empty() { + let text = lines.join(r"\n"); + out.push_str(&format!("{} {} D0 {}\n", format_time(start_ms), format_time(end_ms), text)); + } + } + + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_and_write_roundtrip() { + let jss = "# JACOsub file\n#TimeRes 100\n\n0:00:01.00 0:00:03.00 D0 First line\\nsecond line\n0:00:04.50 0:00:06.00 D0 Second cue\n"; + let doc = parse(jss).expect("parse"); + assert_eq!(doc.cues.len(), 2); + assert_eq!(doc.cues[0].start, ms_to_tick(1000)); + assert_eq!(doc.cues[0].end, ms_to_tick(3000)); + assert_eq!(doc.cues[0].text, "First line\nsecond line"); + assert_eq!(doc.cues[1].start, ms_to_tick(4500)); + assert_eq!(doc.cues[1].end, ms_to_tick(6000)); + assert_eq!(doc.cues[1].text, "Second cue"); + + let written = write(&doc); + assert_eq!(written, jss); + } + + #[test] + fn parse_bracketed_directive() { + let jss = "#TimeRes 100\n0:00:01.00 0:00:03.00 [Style1] Bracketed style text\n"; + let doc = parse(jss).expect("parse"); + assert_eq!(doc.cues.len(), 1); + assert_eq!(doc.cues[0].text, "Bracketed style text"); + } + + #[test] + fn negative_shift_under_one_hour() { + let jss = "#TimeRes 100\n#Shift -0:00:01.00\n0:00:02.00 0:00:04.00 D0 Shifted line\n"; + let doc = parse(jss).expect("parse"); + assert_eq!(doc.cues.len(), 1); + assert_eq!(doc.cues[0].start, ms_to_tick(1000)); + assert_eq!(doc.cues[0].end, ms_to_tick(3000)); + } + + #[test] + fn rejects_non_jacosub() { + assert!(parse("1\n00:00:01,000 --> 00:00:02,000\nHello\n").is_err()); + } +} diff --git a/crates/captions/src/lib.rs b/crates/captions/src/lib.rs index e8a492722..ca99dba43 100644 --- a/crates/captions/src/lib.rs +++ b/crates/captions/src/lib.rs @@ -34,7 +34,10 @@ struct ReadmeExamples; pub mod ass; pub mod cea608; +pub mod jacosub; pub mod mcc; +pub mod pjs; +pub mod realtext; pub mod scc; pub mod srt; pub mod stl; @@ -72,10 +75,28 @@ pub enum Format { Dfxp, /// Advanced SubStation Alpha (also reads SubStation Alpha). Ass, + /// Phoenix Japanimation Society (PJS) subtitles. + Pjs, + /// RealText streaming timed text markup. + RealText, + /// JACOsub scriptable subtitles. + JacoSub, } impl Format { - pub const ALL: [Format; 8] = [Format::Srt, Format::WebVtt, Format::Scc, Format::Mcc, Format::Stl, Format::Ttml, Format::Dfxp, Format::Ass]; + pub const ALL: [Format; 11] = [ + Format::Srt, + Format::WebVtt, + Format::Scc, + Format::Mcc, + Format::Stl, + Format::Ttml, + Format::Dfxp, + Format::Ass, + Format::Pjs, + Format::RealText, + Format::JacoSub, + ]; pub fn name(self) -> &'static str { match self { Format::Srt => "SubRip (SRT)", @@ -86,6 +107,9 @@ impl Format { Format::Ttml => "TTML (IMSC1)", Format::Dfxp => "DFXP (TTML1)", Format::Ass => "Advanced SubStation Alpha (ASS)", + Format::Pjs => "PJS", + Format::RealText => "RealText", + Format::JacoSub => "JACOsub", } } pub fn extension(self) -> &'static str { @@ -98,6 +122,9 @@ impl Format { Format::Ttml => "ttml", Format::Dfxp => "dfxp", Format::Ass => "ass", + Format::Pjs => "pjs", + Format::RealText => "rt", + Format::JacoSub => "jss", } } pub fn from_name(s: &str) -> Option { @@ -110,6 +137,9 @@ impl Format { "ttml" | "imsc" | "imsc1" | "xml" => Some(Format::Ttml), "dfxp" => Some(Format::Dfxp), "ass" | "ssa" => Some(Format::Ass), + "pjs" => Some(Format::Pjs), + "rt" | "realtext" => Some(Format::RealText), + "jss" | "js" | "jacosub" => Some(Format::JacoSub), _ => None, } } @@ -260,6 +290,16 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option { if head.starts_with("[Script Info]") { return Some(Format::Ass); } + if realtext::is_realtext(head) { + return Some(Format::RealText); + } + let first_lines: Vec<&str> = head.lines().map(str::trim).filter(|l| !l.is_empty()).take(3).collect(); + if first_lines.first().is_some_and(|l| pjs::split_cue(l).is_some()) { + return Some(Format::Pjs); + } + if first_lines.iter().any(|l| jacosub::is_timed_line(l)) || (head.starts_with('#') && (head.contains("TimeRes") || head.contains("#T"))) { + return Some(Format::JacoSub); + } let ext = ext.map(|e| e.trim_start_matches('.').to_ascii_lowercase()); if (head.starts_with('<') && (head.contains("") || head.contains(":tt "))) && (head.contains("ns/ttml") || head.contains("ttaf1")) @@ -271,7 +311,7 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option { match ext.as_deref() { Some("srt") if has_arrow_timing || head.is_empty() => Some(Format::Srt), Some("vtt") if has_arrow_timing => Some(Format::WebVtt), - Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml" | "ass" | "ssa") => None, + Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml" | "ass" | "ssa" | "pjs" | "rt" | "jss" | "js") => None, _ if has_arrow_timing => Some(Format::Srt), _ => None, } @@ -306,6 +346,9 @@ fn parse_unbounded(bytes: &[u8], format: Format) -> Result { Format::Mcc => mcc::parse(&text), Format::Ttml | Format::Dfxp => ttml::parse(&text), Format::Ass => ass::parse(&text), + Format::Pjs => pjs::parse(&text), + Format::RealText => realtext::parse(&text), + Format::JacoSub => jacosub::parse(&text), Format::Stl => stl::parse(bytes), } } @@ -344,6 +387,9 @@ pub fn write(doc: &Document, format: Format, opts: WriteOptions) -> Vec { Format::Ttml => ttml::write(doc, ttml::Flavor::Imsc1, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(), Format::Dfxp => ttml::write(doc, ttml::Flavor::Dfxp, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(), Format::Ass => ass::write(doc).into_bytes(), + Format::Pjs => pjs::write(doc).into_bytes(), + Format::RealText => realtext::write(doc).into_bytes(), + Format::JacoSub => jacosub::write(doc).into_bytes(), } } diff --git a/crates/captions/src/pjs.rs b/crates/captions/src/pjs.rs new file mode 100644 index 000000000..91874941b --- /dev/null +++ b/crates/captions/src/pjs.rs @@ -0,0 +1,102 @@ +//! Phoenix Japanimation Society (PJS) subtitles (`.pjs`): times in tenths of a second. +//! +//! ```text +//! 10,50,"First line" +//! 60,120,"Second line|with break" +//! ``` +//! +//! Lines are comma-separated `start,end,"text"` with times in deciseconds (tenths of a second). +//! Inside the quotes, `|` breaks a line. + +use crate::text::{escape_markup, ms_to_tick, number, plain_lines, tick_to_ms}; +use crate::{Cue, Document, Error, Result}; + +/// `start,end,"text"` as its parts. +pub(crate) fn split_cue(line: &str) -> Option<(i64, i64, &str)> { + let (start_str, rest) = line.split_once(',')?; + let (end_str, text_str) = rest.split_once(',')?; + let start = number(start_str.trim())?; + let end = number(end_str.trim())?; + let text = text_str.trim(); + let text = if let Some(stripped) = text.strip_prefix('"').and_then(|t| t.strip_suffix('"')) { stripped } else { text }; + Some((start, end, text)) +} + +/// Parse a PJS subtitle document. +pub fn parse(text: &str) -> Result { + let mut doc = Document::default(); + let mut found = false; + for (n, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let Some((start_ds, end_ds, raw)) = split_cue(line) else { + doc.warnings.push(format!("line {}: not a PJS cue, skipped", n + 1)); + continue; + }; + found = true; + let mut lines = Vec::new(); + for l in raw.split('|') { + let trimmed = l.trim(); + if !trimmed.is_empty() { + let unquoted = trimmed.replace("\"\"", "\""); + lines.push(escape_markup(&unquoted)); + } + } + let start = ms_to_tick(start_ds.saturating_mul(100)); + let end = ms_to_tick(end_ds.saturating_mul(100)); + if end <= start || lines.is_empty() { + doc.warnings.push(format!("line {}: empty or ends before it starts; skipped", n + 1)); + continue; + } + doc.cues.push(Cue { start, end, text: lines.join("\n"), ..Default::default() }); + } + if !found && !text.trim().is_empty() { + return Err(Error::NotFormat("PJS")); + } + Ok(doc) +} + +/// Serialize a document into PJS format. +pub fn write(doc: &Document) -> String { + let mut out = String::new(); + let ds = |t| (tick_to_ms(t) + 50) / 100; + for c in &doc.cues { + let start = ds(c.start); + let end = ds(c.end).max(start + 1); + let lines: Vec = + plain_lines(&c.text).into_iter().map(|l| l.replace('"', "\"\"").replace('|', "|")).filter(|l| !l.is_empty()).collect(); + if !lines.is_empty() { + out.push_str(&format!("{start},{end},\"{}\"\n", lines.join("|"))); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_and_write_roundtrip() { + let pjs = "10,50,\"First line\"\n60,120,\"Second line|with break\"\n130,150,\"Say \"\"Hi\"\"\"\n"; + let doc = parse(pjs).expect("parse"); + assert_eq!(doc.cues.len(), 3); + assert_eq!(doc.cues[0].start, ms_to_tick(1000)); + assert_eq!(doc.cues[0].end, ms_to_tick(5000)); + assert_eq!(doc.cues[0].text, "First line"); + assert_eq!(doc.cues[1].start, ms_to_tick(6000)); + assert_eq!(doc.cues[1].end, ms_to_tick(12000)); + assert_eq!(doc.cues[1].text, "Second line\nwith break"); + assert_eq!(doc.cues[2].text, "Say \"Hi\""); + + let written = write(&doc); + assert_eq!(written, pjs); + } + + #[test] + fn rejects_non_pjs() { + assert!(parse("1\n00:00:01,000 --> 00:00:02,000\nHello\n").is_err()); + } +} diff --git a/crates/captions/src/realtext.rs b/crates/captions/src/realtext.rs new file mode 100644 index 000000000..1c4328a3b --- /dev/null +++ b/crates/captions/src/realtext.rs @@ -0,0 +1,247 @@ +//! RealText (`.rt`): streaming HTML-like timed text markup. +//! +//! ```text +//! +//! +//! ``` +//! +//! RealText documents encapsulate timed text inside a `` container with `