From e7914038f339325f9e51807b6f2cc7fae4585eff Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 09:55:35 -0500 Subject: [PATCH 01/27] craft-xlsx: read the defined names of a binary workbook Each BrtName becomes a DefinedName as Excel writes it in .xlsx: built-in names as _xlnm.Print_Area, sheet scope as localSheetId, the formula decoded with relative references as offsets from A1, the comment, hidden, and a macro's function, vbProcedure or xlm, functionGroupId, shortcutKey, publishToServer and workbookParameter. The hidden names for newer functions and LET parameters are left out, as .xlsx does. Cell formulas using a name that was read now decode instead of keeping their saved results, PtgNameX through the workbook's own link is 'Other Sheet'!Local, and a deleted 3-D reference is Sheet2!#REF!, as in .xlsx. A name whose formula can't be read is left out with a warning, and formulas using it keep their saved results. Closes #1250. --- crates/xlsx/README.md | 9 +- crates/xlsx/src/xlsb.rs | 153 +++++++++++++++++++++++++++++--- crates/xlsx/src/xlsb/expr.rs | 3 + crates/xlsx/src/xlsb/formula.rs | 105 ++++++++++++++++------ crates/xlsx/tests/xlsb.rs | 135 +++++++++++++++++++++++++++- 5 files changed, 360 insertions(+), 45 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 1dc30574d..4e90ca830 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -84,12 +84,17 @@ The parts this crate writes use the transitional namespaces; a strict file's kep they were. A binary workbook is not written. A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not -read (each cell keeps its format index). Its cell formulas are decoded from their tokens into the +read (each cell keeps its format index). Its defined names are those of `definedNames` in the +`.xlsx` Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their +`localSheetId`, relative references as the cell they reach from A1, macro names with their +`function`, `xlm`, `functionGroupId` and `shortcutKey`. Excel's hidden names for newer functions +and LET parameters are left out, as in `.xlsx`; so is a name whose formula can't be read, with a +warning. Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, no implicit `@`), with `FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: user defined names, table references, +keeps only its saved result, with a warning saying why: a defined name left out, table references, other workbooks, Python code stored outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 2fdde59be..1941ea61e 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,5 +1,5 @@ -//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from -//! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an +//! Binary workbooks ([MS-XLSB]): the workbook part with its defined names, shared strings and +//! worksheet cells, from their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an //! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why. //! Styles are not read. A record that does not parse fails the read. @@ -15,7 +15,7 @@ use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS, Range, Visibility}; use crate::sheet::{Cell, CellFormula, CellValue, FormulaKind, Row, SheetPart}; use crate::strings::StringItem; -use crate::workbook::{SheetEntry, SheetKind, Workbook}; +use crate::workbook::{DefinedName, SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; @@ -266,6 +266,9 @@ pub(crate) fn workbook( records.first(131, warnings)?; let mut wb = Workbook::default(); let mut links = Some(Links::default()); + let mut pending = vec![]; + // Each BrtBundleSh's place in `wb.sheets`, `None` for a skipped one. + let mut bundle_sheets: Vec> = vec![]; let mut bundle = false; let mut bundle_seen = false; while let Some((id, payload)) = records.next(warnings)? { @@ -279,14 +282,15 @@ pub(crate) fn workbook( if wb.sheets.is_empty() { return Err(invalid("no supported worksheets")); } - if links.is_none() { - warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"); + match links.as_mut() { + Some(l) => wb.defined_names = defined_names(pending, &bundle_sheets, l, limits, budget, warnings), + None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), } return Ok((wb, links)); } 39 | 355 | 357 | 358 | 362 | 667 => { if let Some(l) = links.as_mut() - && link_record(id, payload, l).is_err() + && link_record(id, payload, l, &mut pending).is_err() { links = None; } @@ -341,6 +345,7 @@ pub(crate) fn workbook( if let Some(l) = links.as_mut() { l.sheets.push(None); } + bundle_sheets.push(None); continue; }; let rel = rels.by_id(&rel_id).ok_or_else(|| invalid("worksheet relationship is missing"))?; @@ -355,6 +360,7 @@ pub(crate) fn workbook( if let Some(l) = links.as_mut() { l.sheets.push((kind == SheetKind::Worksheet).then(|| name.clone())); } + bundle_sheets.push(u32::try_from(wb.sheets.len()).ok()); wb.sheets.push(SheetEntry { name, sheet_id, visibility, rel_id, kind, part: Some(rel.target.clone()) }); } _ => {} @@ -363,9 +369,110 @@ pub(crate) fn workbook( Err(invalid("missing workbook end record")) } +/// A `BrtName`'s parts that only its [`DefinedName`] needs, until the workbook part is read. +struct PendingName { + /// Its place in [`Links`]'s names. + index: usize, + flags: u32, + /// `chKey`: a macro's shortcut key. + key: u8, + rgce: Vec, + rgcb: Vec, + comment: Option, + menu: Option, + description: Option, +} + +/// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the +/// hidden ones for newer functions and LET parameters, which Excel doesn't write there. `sheets` +/// maps a name's `itab` (its sheet's place in `BrtBundleSh` order) to [`Workbook::sheets`]. +/// A name whose formula can't be read, or that belongs to a sheet the reader skips, is left out +/// with a warning, and a cell formula using it keeps its saved result. +fn defined_names( + pending: Vec, + sheets: &[Option], + links: &mut Links, + limits: &Limits, + budget: &Budget, + warnings: &mut Warnings, +) -> Vec { + let mut out = vec![]; + for p in pending { + let Some(name) = links.names.get(p.index) else { continue }; + if name.future || formula::is_param(&name.text) { + continue; + } + let local_sheet_id = match name.scope { + None => None, + Some(itab) => match usize::try_from(itab).ok().and_then(|i| sheets.get(i)) { + Some(Some(i)) => Some(*i), + _ => { + warnings.warn("XLSB: some defined names were left out (names of sheets the reader skips)"); + continue; + } + }, + }; + let formula = match formula::decode(&p.rgce, &p.rgcb, links, Mode::Name) { + Ok(e) => expr::print(&e), + Err(formula::Unsupported(why)) => { + warnings.warn(format!("XLSB: some defined names were left out ({why})")); + continue; + } + }; + if !budget.try_formula_text(formula.len(), limits) { + warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); + continue; + } + let mut attrs = vec![]; + let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); + // In `CT_DefinedName`'s order. + if let Some(m) = p.menu { + put("customMenu", m); + } + if let Some(d) = p.description { + put("description", d); + } + let (func, vba, proc) = (p.flags & 0x02 != 0, p.flags & 0x04 != 0, p.flags & 0x08 != 0); + if func { + put("function", "1".into()); + } + if vba { + put("vbProcedure", "1".into()); + } else if proc { + put("xlm", "1".into()); + } + let group = (p.flags >> 6) & 0x1FF; + if proc && group != 0 { + put("functionGroupId", group.to_string()); + } + if p.key != 0 { + put("shortcutKey", char::from(p.key).to_string()); + } + if p.flags & (1 << 15) != 0 { + put("publishToServer", "1".into()); + } + if p.flags & (1 << 16) != 0 { + put("workbookParameter", "1".into()); + } + out.push(DefinedName { + name: name.text.clone(), + local_sheet_id, + formula, + comment: p.comment.unwrap_or_default(), + hidden: p.flags & 1 != 0, + attrs, + }); + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + } + out +} + /// A supporting link (`BrtSupSelf`, `BrtSupSame`, `BrtSupAddin`, `BrtSupBookSrc`), the sheet links -/// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`. -fn link_record(id: u16, payload: &[u8], links: &mut Links) -> Result<(), Error> { +/// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`; a name's other parts go to +/// `pending`. +fn link_record(id: u16, payload: &[u8], links: &mut Links, pending: &mut Vec) -> Result<(), Error> { let mut b = Bytes::new(payload); match id { 357 | 358 | 667 => { @@ -391,15 +498,37 @@ fn link_record(id: u16, payload: &[u8], links: &mut Links) -> Result<(), Error> b.finish()?; } _ => { - // BrtName ([MS-XLSB] 2.4.718): flags, chKey, itab, then the name; the formula and the - // rest aren't needed to call a function or tell a name from one. + // BrtName ([MS-XLSB] 2.4.718). Its formula is decoded once the workbook part is read. if links.names.len() >= MAX_NAMES { return Err(invalid("too many defined names")); } let flags = b.u32()?; - b.take(5)?; + let key = b.u8()?; + let itab = b.u32()?; let text = b.text(255)?; - links.names.push(Name { text, future: flags & (1 << 17) != 0 }); + let cce = b.u32()? as usize; + let rgce = b.take(cce)?.to_vec(); + let cb = b.u32()? as usize; + let rgcb = b.take(cb)?.to_vec(); + let comment = b.string(255, true)?; + // A macro's name (fProc) has four more strings: one unused, its custom menu, its + // description and another unused. + let (menu, description) = if flags & 0x08 != 0 { + b.string(32767, true)?; + let menu = b.string(32767, true)?; + let description = b.string(32767, true)?; + b.string(32767, true)?; + (menu, description) + } else { + (None, None) + }; + b.finish()?; + let scope = (itab != u32::MAX).then_some(itab); + // A built-in name (fBuiltin) is stored bare; `.xlsx` spells it `_xlnm.Print_Area`, in + // `definedNames` and in formulas alike. + let text = if flags & 0x20 != 0 { format!("_xlnm.{text}") } else { text }; + pending.push(PendingName { index: links.names.len(), flags, key, rgce, rgcb, comment, menu, description }); + links.names.push(Name { text, future: flags & (1 << 17) != 0, scope, read: false }); } } Ok(()) diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index 3dd643a6a..17f5086ae 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -24,6 +24,8 @@ pub(crate) enum RefKind { Rows(u32, bool, u32, bool), /// Whole columns `A:C`. Cols(u32, bool, u32, bool), + /// A deleted cell or range on a named sheet: `Sheet2!#REF!`. + Deleted, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -173,6 +175,7 @@ fn push_reference(s: &mut String, r: &Reference) { s.push_str(dollar(*a1)); push_col(s, *c1); } + RefKind::Deleted => s.push_str(CellError::Ref.as_str()), } } diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 327ffcc25..2e7f7cc96 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -4,8 +4,9 @@ //! called through hidden names), so this is its own decoder, with its own function table //! ([`super::ftab`]) and its own copies of craft-xls's depth guards and limits. Anything outside the //! token set it knows is reported as unsupported, and the caller keeps the cell's cached value. -//! So is a formula that needs something the reader doesn't read: a user's defined name (XLSB -//! names are not read yet) or Python code stored outside the cell. +//! So is a formula that needs something the reader doesn't read: a defined name whose own +//! formula could not be read, or Python code stored outside the cell. Defined names' formulas +//! are decoded here too ([`Mode::Name`]). //! //! Names keep their file spelling (`_xlfn.CONCAT`, `_xlpm.x`), as `.xlsx` formula text has them. @@ -92,13 +93,26 @@ pub(super) struct Xti { pub last: i32, } -/// A `BrtName` as far as cell formulas need it. +/// A `BrtName` as far as formulas need it. #[derive(Clone, Debug)] pub(super) struct Name { - /// The name as written (`_xlfn.CONCAT`, `_xlpm.x`, `Rate`). + /// The name as `.xlsx` spells it (`_xlfn.CONCAT`, `_xlpm.x`, `_xlnm.Print_Area`, `Rate`). pub text: String, /// `fFutureFunction`: a hidden name standing for a function newer than the file format. pub future: bool, + /// `itab`: the sheet a sheet-scoped name belongs to, by its place in `BrtBundleSh` order. + pub scope: Option, + /// Its own formula was read, so it is one of the workbook's defined names. + pub read: bool, +} + +impl Name { + /// Whether a formula may use the name: one the formula text itself defines (a newer + /// function, a LET parameter), or a defined name the reader gives. A name's formula may use + /// any other name: its text is the same either way. + fn usable(&self, mode: Mode) -> bool { + self.future || is_param(&self.text) || self.read || mode == Mode::Name + } } /// Workbook-level tables formulas refer to. @@ -114,20 +128,37 @@ pub(crate) struct Links { } /// Whether a name stands for a LET parameter (`_xlpm.x`), which only the formula itself defines. -fn is_param(name: &str) -> bool { +pub(super) fn is_param(name: &str) -> bool { name.get(..6).is_some_and(|p| p.eq_ignore_ascii_case("_xlpm.")) } impl Links { - /// The sheets a 3-D reference names, or `None` for a deleted sheet (the reference is `#REF!`). - fn sheet_sel(&self, ixti: u16) -> Result, Unsupported> { + /// `PtgName`'s and `PtgNameX`'s name: a 1-based index into every `BrtName`. A cell formula + /// using a defined name the reader doesn't give would calculate to `#NAME?`, so it keeps its + /// saved result instead. + fn name(&self, index: u32, mode: Mode) -> Result<&Name, Unsupported> { + let i = usize::try_from(index).ok().and_then(|i| i.checked_sub(1)); + let name = i.and_then(|i| self.names.get(i)).ok_or(Unsupported("reference to a missing defined name"))?; + if !name.usable(mode) { + return Err(Unsupported("defined names whose formula could not be read")); + } + Ok(name) + } + + /// Sheet link `ixti`, which must be through this workbook's own supporting link. + fn this_book(&self, ixti: u16) -> Result<&Xti, Unsupported> { let x = self.xti.get(usize::from(ixti)).ok_or(Unsupported("reference to a missing sheet link"))?; match usize::try_from(x.book).ok().and_then(|b| self.books.get(b)) { - Some(Book::This) => {} - Some(Book::External) => return Err(Unsupported("references to other workbooks (external links)")), - Some(_) => return Err(Unsupported("references through add-in or same-sheet links")), - None => return Err(Unsupported("reference to a missing workbook link")), + Some(Book::This) => Ok(x), + Some(Book::External) => Err(Unsupported("references to other workbooks (external links)")), + Some(_) => Err(Unsupported("references through add-in or same-sheet links")), + None => Err(Unsupported("reference to a missing workbook link")), } + } + + /// The sheets a 3-D reference names, or `None` for a deleted sheet (the reference is `#REF!`). + fn sheet_sel(&self, ixti: u16) -> Result, Unsupported> { + let x = self.this_book(ixti)?; // -1: "the first sheet in the reference cannot be found" (a deleted sheet). if x.first == -1 || x.last == -1 { return Ok(None); @@ -194,10 +225,15 @@ const LAST_COL: u32 = MAX_COLS - 1; /// Where a formula lives: a cell formula's references hold coordinates; a shared formula's /// relative tokens (`PtgRefN`, `PtgAreaN`) hold offsets from the shared formula's first cell. +/// +/// A defined name's relative references are offsets from A1 that wrap around the grid +/// (`RgceLocRel`), in its 3-D references too: `.xlsx` spells them as the cell they reach from A1 +/// (one column left is `Data!XFD1`). #[derive(Clone, Copy, PartialEq)] pub(super) enum Mode { Cell, Shared(CellRef), + Name, } /// RgceLoc ([MS-XLSB] 2.5.98.91): a row and a `ColRelShort` (14-bit column, then the @@ -225,9 +261,15 @@ fn loc(row: u32, col: u16, mode: Mode) -> Result { match mode { Mode::Cell => anchor(row, col), Mode::Shared(base) => relative_anchor(row, col, base), + Mode::Name => relative_anchor(row, col, CellRef::new(0, 0)), } } +/// A 3-D reference's cell: a coordinate, except in a defined name. +fn loc3d(row: u32, col: u16, mode: Mode) -> Result { + if mode == Mode::Name { loc(row, col, mode) } else { anchor(row, col) } +} + /// RgceArea: a range spanning every row (or column) is a whole-column (row) reference. fn area(r0: u32, r1: u32, c0: u16, c1: u16) -> Result { area_of(anchor(r0, c0)?, anchor(r1, c1)?) @@ -380,14 +422,7 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res } 0x23 => { // PtgName: a 1-based index into every BrtName, hidden ones included. - let i = usize::try_from(t.u32()?).ok().and_then(|i| i.checked_sub(1)); - let name = i.and_then(|i| links.names.get(i)).ok_or(Unsupported("reference to a missing defined name"))?; - // Defined names are not imported from XLSB yet: a cell using one would calculate - // to #NAME?, so it keeps its saved result. Newer functions and LET parameters are - // hidden names that the formula text itself defines. - if !name.future && !is_param(&name.text) { - return Err(Unsupported("defined names")); - } + let name = links.name(t.u32()?, mode)?; push(&mut stack, kids, Expr::Name(name.text.clone()))?; } 0x24 => { @@ -420,36 +455,52 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res t.take(12)?; push(&mut stack, kids, Expr::Error(CellError::Ref))?; } - 0x39 => return Err(Unsupported("names in other workbooks or on other sheets")), + 0x39 => { + // PtgNameX: a name through a supporting link. Through this workbook's own link + // it is a sheet-scoped name used from another sheet, which `.xlsx` spells + // `'Other Sheet'!Local`: Excel's files show it, [MS-XLSB] doesn't say. + let ixti = t.u16()?; + let index = t.u32()?; + links.this_book(ixti)?; + let name = links.name(index, mode)?; + let sheet = name.scope.and_then(|s| links.sheets.get(usize::try_from(s).ok()?).cloned().flatten()); + let sheet = sheet.ok_or(Unsupported("names in other workbooks or on other sheets"))?; + push(&mut stack, kids, Expr::Name(format!("{sheet}!{}", name.text)))?; + } 0x3A => { let ixti = t.u16()?; let (r, c) = (t.u32()?, t.u16()?); - let kind = RefKind::Cell(anchor(r, c)?); + let kind = RefKind::Cell(loc3d(r, c, mode)?); push(&mut stack, kids, reference(links.sheet_sel(ixti)?, kind))?; } 0x3B => { let ixti = t.u16()?; let (r0, r1, c0, c1) = (t.u32()?, t.u32()?, t.u16()?, t.u16()?); - let kind = area(r0, r1, c0, c1)?; + let kind = area_of(loc3d(r0, c0, mode)?, loc3d(r1, c1, mode)?)?; push(&mut stack, kids, reference(links.sheet_sel(ixti)?, kind))?; } 0x3C | 0x3D => { + // PtgRefErr3d, PtgAreaErr3d: a deleted cell or range, which `.xlsx` spells + // `Sheet2!#REF!` (`#REF!` when the sheet is gone too). The sheet link must still + // be one the reader follows: an external #REF! stays cached. let ixti = t.u16()?; t.take(if id == 0x3C { 6 } else { 12 })?; - // The sheet link must still be one the reader follows: an external #REF! stays cached. - links.sheet_sel(ixti)?; - push(&mut stack, kids, Expr::Error(CellError::Ref))?; + push(&mut stack, kids, reference(links.sheet_sel(ixti)?, RefKind::Deleted))?; } 0x01 => return Err(Unsupported("the shared or array formula form")), 0x02 => return Err(Unsupported("what-if data tables")), 0x18 => return Err(Unsupported("structured table references")), 0x2C => { - let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; + if mode == Mode::Cell { + return Err(Unsupported("relative references in a cell formula")); + } let (r, c) = (t.u32()?, t.u16()?); push(&mut stack, kids, Expr::Ref(Reference { sheet: SheetSel::Current, kind: RefKind::Cell(loc(r, c, mode)?) }))?; } 0x2D => { - let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; + if mode == Mode::Cell { + return Err(Unsupported("relative references in a cell formula")); + } let (r0, r1, c0, c1) = (t.u32()?, t.u32()?, t.u16()?, t.u16()?); let kind = area_of(loc(r0, c0, mode)?, loc(r1, c1, mode)?)?; push(&mut stack, kids, Expr::Ref(Reference { sheet: SheetSel::Current, kind }))?; diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index ee9ac3afd..73bd23218 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -5,7 +5,7 @@ mod common; use common::{read, read_with}; use craft_xlsx::testing::zip; -use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb}; +use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Package, Range, SheetKind, Value, Visibility, is_xlsb}; const ROOT: &str = r#""#; const RELS: &str = r#""#; @@ -463,6 +463,9 @@ fn name(flags: u32, text: &str) -> Vec { data.extend(wide(text)); data.extend([2, 0, 0, 0, 0x1c, 0x1d, 0, 0, 0, 0]); // Formula `#NAME?`, no extra data. data.extend(u32::MAX.to_le_bytes()); // No comment. + if flags & 0x08 != 0 { + data.extend(u32::MAX.to_le_bytes().repeat(4)); // A macro's four strings, all null. + } record(39, &data) } @@ -543,8 +546,8 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { ), (9, ref3d(3, 0, 0), vec![]), (10, [&[0x25u8][..], &0u32.to_le_bytes(), &1u32.to_le_bytes(), &REL.to_le_bytes(), &REL.to_le_bytes(), &[0x41, 20, 0]].concat(), vec![]), - // Kept as their saved results: (11, vec![0x43, 4, 0, 0, 0, 0x1E, 2, 0, 0x05], vec![]), + // Kept as their saved results: (12, ref3d(4, 0, REL), vec![]), (13, vec![0x01, 0, 0, 0, 0], vec![0, 0, 0, 0]), // Part of a shared formula at A1 that isn't one. ]; @@ -572,7 +575,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { Some("_xlfn.LET(_xlpm.x,1,_xlpm.x+1)"), Some("#REF!"), Some("SQRT(A1:A2)"), - None, + Some("Rate*2"), None, None, // A BrtCellMeta before a cell gives it cell metadata, as `cm` does in .xlsx. @@ -584,7 +587,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&CellRef::new(0, 14), &1)]); // Every formula cell keeps its saved result as its value. assert!((1..=15).all(|c| sheet.value(CellRef::new(0, c)) == Value::Number(0.0))); - for why in ["defined names", "other workbooks", "missing from the file", "structured table references"] { + for why in ["other workbooks", "missing from the file", "structured table references"] { assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); } } @@ -930,3 +933,127 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&at("C1"), &1)]); assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings); } + +// Defined names ([MS-XLSB] 2.4.718): `BrtName` records, and the `definedNames` Excel writes for +// the same names when it saves the workbook as `.xlsx`. + +/// `BrtName`. A macro's name (`fProc`, 0x08) has four more strings, null here. +fn brt_name(flags: u32, key: u8, itab: u32, text: &str, rgce: &[u8], comment: Option<&str>) -> Vec { + let mut data = flags.to_le_bytes().to_vec(); + data.push(key); + data.extend(itab.to_le_bytes()); + data.extend(wide(text)); + data.extend((rgce.len() as u32).to_le_bytes()); + data.extend(rgce); + data.extend(0u32.to_le_bytes()); + data.extend(comment.map_or_else(|| u32::MAX.to_le_bytes().to_vec(), wide)); + if flags & 0x08 != 0 { + data.extend(u32::MAX.to_le_bytes().repeat(4)); + } + record(39, &data) +} +fn ref3d_name(ixti: u16, row: u32, col: u16) -> Vec { + [&[0x3Au8][..], &ixti.to_le_bytes(), &loc(row, col)].concat() +} +const WORKBOOK: u32 = u32::MAX; + +/// `book(false)` with the supporting links of `book_with_links` (sheet links Data, Hidden, +/// Data:Hidden, a deleted sheet, the other workbook, and this workbook as a whole, which +/// `PtgNameX` uses), and `names`. +fn book_with_names(names: &[Vec]) -> Vec { + let plain = book(false); + let end = record(132, &[]); + let mut xti = 6u32.to_le_bytes().to_vec(); + for (book, first, last) in [(0u32, 0i32, 0i32), (0, 1, 1), (0, 0, 1), (0, -1, -1), (1, 0, 0), (0, -2, -2)] { + xti.extend([book.to_le_bytes(), first.to_le_bytes(), last.to_le_bytes()].concat()); + } + let links = [record(353, &[]), record(357, &[]), record(355, &wide("rId9")), record(362, &xti), record(354, &[])]; + [&plain[..plain.len() - end.len()], &links.concat(), &names.concat(), &end].concat() +} + +/// The names below, in record order, as `PtgName` numbers them from 1. +fn names() -> Vec> { + vec![ + // 1: a newer function's hidden name: not a defined name in `.xlsx`. + brt_name(0x0002_000b, 0, WORKBOOK, "_xlfn.CONCAT", &[0x1C, 0x1D], None), + // 2: built-in and hidden, on Data. + brt_name(0x21, 0, 0, "_FilterDatabase", &[&[0x3Bu8, 0, 0][..], &0u32.to_le_bytes(), &9u32.to_le_bytes(), &[0, 0, 2, 0]].concat(), None), + // 3: refers to another workbook: left out. + brt_name(0, 0, WORKBOOK, "External", &ref3d_name(4, 0, 0), None), + // 4: a deleted cell. + brt_name(0, 0, WORKBOOK, "Gone", &[0x3C, 0, 0, 0, 0, 0, 0, 0, 0], None), + brt_name(1, 0, WORKBOOK, "Hid", &[0x1E, 42, 0], None), + // 6, 7: the same name on two sheets. + brt_name(0, 0, 0, "Local", &ref3d_name(0, 1, 1), None), + brt_name(0, 0, 1, "Local", &ref3d_name(1, 0, 1), None), + // 8: an XLM macro with a shortcut key, in the "User Defined" group (14). + brt_name(0x388, b'k', WORKBOOK, "MacroName", &ref3d_name(0, 0, 0), None), + // 9 + brt_name(0, 0, WORKBOOK, "Rate", &[&[0x1Fu8][..], &0.05f64.to_le_bytes()].concat(), Some("a note")), + // 10, 11: relative references are offsets from A1, wrapping around the grid. + brt_name(0, 0, WORKBOOK, "Rel", &ref3d_name(0, 0, 0xFFFF), None), + brt_name(0, 0, WORKBOOK, "RelN", &[&[0x2Cu8][..], &loc(u32::MAX, REL)].concat(), None), + // 12: on a sheet the workbook doesn't have: left out. + brt_name(0, 0, 5, "Skipped", &[0x1E, 1, 0], None), + // 13: a name of a name. + brt_name(0, 0, WORKBOOK, "Twice", &[0x23, 9, 0, 0, 0, 0x1E, 2, 0, 0x05], None), + ] +} + +const NAMES_XML: &str = r#"Data!#REF!Data!$B$2Hidden!$B$1Data!$A$10.05Data!XFD1A1048576Rate*2"#; + +fn xml_names(xml: &str) -> Vec { + let types = r#""#; + let root = r#""#; + let bytes = zip(&[("[Content_Types].xml", types.as_bytes()), ("_rels/.rels", root.as_bytes()), ("xl/workbook.xml", xml.as_bytes())]); + Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap().defined_names +} + +#[test] +fn defined_names_are_read_as_their_xlsx_spelling() { + let cells: Vec> = vec![ + vec![0x23, 9, 0, 0, 0, 0x1E, 100, 0, 0x05], + vec![0x23, 6, 0, 0, 0], + // PtgNameX through this workbook's own link: a sheet-scoped name used from another sheet. + vec![0x39, 5, 0, 7, 0, 0, 0], + vec![0x23, 2, 0, 0, 0], + vec![0x3C, 1, 0, 0, 0, 0, 0, 0, 0], + // Kept as their saved results: a name left out, a name in another workbook, and a + // workbook name through PtgNameX. + vec![0x23, 3, 0, 0, 0], + vec![0x39, 4, 0, 9, 0, 0, 0], + vec![0x39, 5, 0, 9, 0, 0, 0], + ]; + let mut data = row(0); + for (i, rgce) in cells.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, rgce, &[])); + } + let bytes = package(&book_with_names(&names()), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert_eq!(wb.defined_names, xml_names(NAMES_XML)); + let book = read(&bytes).unwrap(); + let got: Vec<_> = (1..=8).map(|c| book.sheets[0].formula(CellRef::new(0, c))).collect(); + let want = [Some("Rate*100"), Some("Local"), Some("Hidden!Local"), Some("_xlnm._FilterDatabase"), Some("Hidden!#REF!"), None, None, None]; + assert_eq!(got, want); + for why in ["defined names whose formula could not be read", "other workbooks", "names in other workbooks or on other sheets"] { + assert!(warned(&book, why), "{why}: {:?}", book.warnings); + } + for why in ["(references to other workbooks (external links))", "(names of sheets the reader skips)"] { + assert!(book.warnings.iter().any(|w| w.contains("defined names were left out") && w.contains(why)), "{why}: {:?}", book.warnings); + } +} + +#[test] +fn a_damaged_name_leaves_names_and_formulas_as_values() { + // A truncated BrtName: the workbook opens with no names, and every formula keeps its value. + let mut list = names(); + let full = brt_name(0, 0, WORKBOOK, "Rate", &[0x1E, 1, 0], None); + list[8] = record(39, &full[2..full.len() - 2]); // The payload without its record header and last two bytes. + let data = [row(0), formula_cell(1, &[0x1E, 1, 0], &[])].concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert!(wb.defined_names.is_empty()); + let book = read(&bytes).unwrap(); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 1)), None); + assert!(book.warnings.iter().any(|w| w.contains("names could not be read")), "{:?}", book.warnings); +} From 2a786e27a2826e9a67cfe0af167410409717b0d3 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 11:30:32 -0500 Subject: [PATCH 02/27] craft-xlsx: names on macro sheets, spans over chart sheets, formula-less built-ins A second Excel probe showed three cases where the .xlsb reader left out what .xlsx has: - Macro names point at an XLM macro sheet (Macro1!$A$1), which the sheet links treated as a sheet with no cells, so every such name was left out. Macro sheets have cells; references to them are now named. - A 3-D span over a chart sheet ('Data:After Sheet'!$A$1) was refused. A span is spelled by its ends; a sheet with no cells between them adds nothing, and .xlsx spells it the same. - Excel writes a workbook-level built-in with no formula (Database) beside a sheet's own. .xlsx leaves it out of definedNames, but formulas using it spell _xlnm.Database; they now do instead of keeping their saved results. --- crates/xlsx/README.md | 7 +++-- crates/xlsx/src/xlsb.rs | 13 ++++++++-- crates/xlsx/src/xlsb/formula.rs | 13 ++++------ crates/xlsx/tests/xlsb.rs | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 4e90ca830..3b636a2fa 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -88,8 +88,11 @@ read (each cell keeps its format index). Its defined names are those of `defined `.xlsx` Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their `localSheetId`, relative references as the cell they reach from A1, macro names with their `function`, `xlm`, `functionGroupId` and `shortcutKey`. Excel's hidden names for newer functions -and LET parameters are left out, as in `.xlsx`; so is a name whose formula can't be read, with a -warning. Its cell formulas are decoded from their tokens into the +and LET parameters are left out, as in `.xlsx`, and so is the formula-less workbook-level built-in +Excel writes beside a sheet's (`Database`), which formulas still spell `_xlnm.Database`; a name +whose formula can't be read is left out with a warning. A reference may name an XLM macro sheet +(`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends whatever chart +sheets lie between them. Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, no implicit `@`), with `FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 1941ea61e..dd52a55b4 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -356,9 +356,10 @@ pub(crate) fn workbook( if let SheetKind::Other(_) = kind { return Err(invalid("unknown sheet relationship type")); } - // Only a worksheet has cells a formula can refer to. + // Only worksheets and macro sheets have cells a formula can refer to (a macro's + // name refers to its macro sheet: `Macro1!$A$1`). if let Some(l) = links.as_mut() { - l.sheets.push((kind == SheetKind::Worksheet).then(|| name.clone())); + l.sheets.push(matches!(kind, SheetKind::Worksheet | SheetKind::Macrosheet).then(|| name.clone())); } bundle_sheets.push(u32::try_from(wb.sheets.len()).ok()); wb.sheets.push(SheetEntry { name, sheet_id, visibility, rel_id, kind, part: Some(rel.target.clone()) }); @@ -412,6 +413,14 @@ fn defined_names( } }, }; + // Excel writes a workbook-level built-in with no formula (`Database`) beside a sheet's + // own; `.xlsx` leaves it out of `definedNames`, and formulas using it spell its name. + if p.rgce.is_empty() { + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + continue; + } let formula = match formula::decode(&p.rgce, &p.rgcb, links, Mode::Name) { Ok(e) => expr::print(&e), Err(formula::Unsupported(why)) => { diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 2e7f7cc96..be6f47bfb 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -118,8 +118,8 @@ impl Name { /// Workbook-level tables formulas refer to. #[derive(Default)] pub(crate) struct Links { - /// Every sheet's name in `BrtBundleSh` order, `None` for sheets the reader skips or that hold - /// no cells (chart, dialog and macro sheets). + /// Every sheet's name in `BrtBundleSh` order, `None` for the sheets that hold no cells (chart, + /// dialog and module sheets). pub(super) sheets: Vec>, /// Every `BrtName`, in order: `PtgName` indexes them from 1. pub(super) names: Vec, @@ -167,14 +167,11 @@ impl Links { let (Ok(first), Ok(last)) = (usize::try_from(x.first), usize::try_from(x.last)) else { return Err(Unsupported("reference to a missing sheet")); }; - // A span takes every sheet between its ends, so a skipped one anywhere in it changes - // what the reference means. + // A span is spelled by its ends. A sheet between them with no cells (a chart sheet) adds + // nothing to it, and `.xlsx` spells the span the same; an end with no cells can't be named. let span = self.sheets.get(first..=last).filter(|s| !s.is_empty()).ok_or(Unsupported("reference to a missing sheet"))?; - if span.iter().any(Option::is_none) { - return Err(Unsupported("references to sheets the reader skips")); - } let (Some(Some(a)), Some(Some(b))) = (span.first(), span.last()) else { - return Err(Unsupported("reference to a missing sheet")); + return Err(Unsupported("references to sheets with no cells")); }; Ok(Some(if first == last { SheetSel::Named(a.clone()) } else { SheetSel::Span(a.clone(), b.clone()) })) } diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 73bd23218..78e2d89b1 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1057,3 +1057,49 @@ fn a_damaged_name_leaves_names_and_formulas_as_values() { assert_eq!(book.sheets[0].formula(CellRef::new(0, 1)), None); assert!(book.warnings.iter().any(|w| w.contains("names could not be read")), "{:?}", book.warnings); } + +#[test] +fn names_on_macro_sheets_spans_over_chart_sheets_and_empty_built_ins() { + // Sheets Data, a chart sheet, an XLM macro sheet, Hidden; sheet links Macro1, Data:Hidden + // (over both) and Data. + let mut book = record(131, &[]); + book.extend(record(153, &[[0u8; 8].as_slice(), &wide("")].concat())); + book.extend(record(143, &[])); + for (id, rel, name) in [(1u32, "one", "Data"), (3, "c", "Chart1"), (4, "m", "Macro1"), (2, "two", "Hidden")] { + book.extend(record(156, &[&0u32.to_le_bytes()[..], &id.to_le_bytes(), &wide(rel), &wide(name)].concat())); + } + book.extend(record(144, &[])); + let mut xti = 3u32.to_le_bytes().to_vec(); + for (first, last) in [(2i32, 2i32), (0, 3), (0, 0)] { + xti.extend([0u32.to_le_bytes(), first.to_le_bytes(), last.to_le_bytes()].concat()); + } + book.extend([record(353, &[]), record(357, &[]), record(362, &xti), record(354, &[])].concat()); + let area = [&[0x3Bu8][..], &2u16.to_le_bytes(), &0u32.to_le_bytes(), &1u32.to_le_bytes(), &0u16.to_le_bytes(), &1u16.to_le_bytes()].concat(); + // A sheet's built-in Database, then the workbook-level one Excel writes with no formula. + book.extend(brt_name(0x20, 0, 0, "Database", &area, None)); + book.extend(brt_name(0x20, 0, WORKBOOK, "Database", &[], None)); + book.extend(brt_name(0x388, b'k', WORKBOOK, "Cmd", &ref3d_name(0, 0, 0), None)); + book.extend(brt_name(0, 0, WORKBOOK, "Span", &ref3d_name(1, 0, 0), None)); + book.extend(record(132, &[])); + let rels = RELS.replace( + "", + r#""#, + ); + let ref3d_cell = |ixti: u16| [&[0x5Au8][..], &ixti.to_le_bytes(), &loc(0, 0)].concat(); + let data = [ + row(0), + formula_cell(1, &[0x23, 2, 0, 0, 0], &[]), + formula_cell(2, &[0x23, 4, 0, 0, 0], &[]), + formula_cell(3, &ref3d_cell(0), &[]), + formula_cell(4, &ref3d_cell(1), &[]), + ] + .concat(); + let bytes = package(&book, &worksheet(&data, &[]), &sst(), ROOT, &rels, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"Data!$A$1:$B$2Macro1!$A$1Data:Hidden!$A$1"#; + assert_eq!(wb.defined_names, xml_names(xml)); + let read = read(&bytes).unwrap(); + let got: Vec<_> = (1..=4).map(|c| read.sheets[0].formula(CellRef::new(0, c))).collect(); + assert_eq!(got, [Some("_xlnm.Database"), Some("Span"), Some("Macro1!$A$1"), Some("Data:Hidden!$A$1")]); + assert!(read.warnings.iter().all(|w| !w.contains("left out") && !w.contains("saved results")), "{:?}", read.warnings); +} From b5eda42a33893ec17143532cef6fc1d97accb9d5 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 12:21:06 -0500 Subject: [PATCH 03/27] craft-xlsx: VBA function names and macro descriptions in binary workbooks An Excel probe with VBA enabled showed two things the names reader got wrong: - Excel writes a VBA function a cell calls as a BrtName with fFunc, fOB and fProc, no formula, and none of the four strings fProc names otherwise carry. The reader refused it, which dropped every name and left every formula as its saved result. Such a name now ends after its comment; it stays out of the names, as .xlsm lists none, and cells calling it spell it. - The two middle strings of a macro's name are its description and help topic, as .xlsm writes them (description="Function A" help="!0"), not the custom menu and description the specification's labels say. --- crates/xlsx/README.md | 39 ++++++++++++++++----------------- crates/xlsx/src/xlsb.rs | 23 +++++++++++--------- crates/xlsx/tests/xlsb.rs | 45 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index b873c87f6..e79fdbd55 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -83,25 +83,26 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1 The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. -A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not -read (each cell keeps its format index). Its defined names are those of `definedNames` in the -`.xlsx` Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their -`localSheetId`, relative references as the cell they reach from A1, macro names with their -`function`, `xlm`, `functionGroupId` and `shortcutKey`. Excel's hidden names for newer functions -and LET parameters are left out, as in `.xlsx`, and so is the formula-less workbook-level built-in -Excel writes beside a sheet's (`Database`), which formulas still spell `_xlnm.Database`; a name -whose formula can't be read is left out with a warning. A reference may name an XLM macro sheet -(`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends whatever chart -sheets lie between them. Its cell formulas are decoded from their tokens into the text an `.xlsx` -file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC 29500 functions -`.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the formula, numbers as -Excel spells them), with -`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell -metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the -file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: a defined name left out, table references, -other workbooks, Python code stored outside the cell, data tables. The XLSB function table -(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. +A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not read +(each cell keeps its format index). Its defined names are those of `definedNames` in the `.xlsx` +Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their `localSheetId`, +relative references as the cell they reach from A1, macro names with their `function`, `xlm` or +`vbProcedure`, `functionGroupId`, `shortcutKey`, `description` and `help`. Excel's hidden names for +newer functions and LET parameters are left out, as in `.xlsx`, and so is the formula-less +workbook-level built-in Excel writes beside a sheet's (`Database`), which formulas still spell +`_xlnm.Database`, and the formula-less name of a VBA function a cell calls (`.xlsm` lists none; the +cell spells it); a name whose formula can't be read is left out with a warning. A reference may name +an XLM macro sheet (`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends +whatever chart sheets lie between them. Its cell formulas are decoded from their tokens into the +text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC +29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the +formula, numbers as Excel spells them), with `FormulaKind::Shared` or `FormulaKind::Array` and their +ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own +text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs +something the reader does not read keeps only its saved result, with a warning saying why: a defined +name left out, table references, other workbooks, Python code stored outside the cell, data tables. +The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from +`craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index dd52a55b4..e6246f4be 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -380,8 +380,8 @@ struct PendingName { rgce: Vec, rgcb: Vec, comment: Option, - menu: Option, description: Option, + help: Option, } /// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the @@ -435,12 +435,12 @@ fn defined_names( let mut attrs = vec![]; let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); // In `CT_DefinedName`'s order. - if let Some(m) = p.menu { - put("customMenu", m); - } if let Some(d) = p.description { put("description", d); } + if let Some(h) = p.help { + put("help", h); + } let (func, vba, proc) = (p.flags & 0x02 != 0, p.flags & 0x04 != 0, p.flags & 0x08 != 0); if func { put("function", "1".into()); @@ -520,14 +520,17 @@ fn link_record(id: u16, payload: &[u8], links: &mut Links, pending: &mut VecData!$A$1"#; + assert_eq!(wb.defined_names, xml_names(xml)); +} From 0f944f62d5330c58bfe2cfdeac3d74881ec67748 Mon Sep 17 00:00:00 2001 From: kiwi4boy Date: Sat, 14 Nov 2026 12:44:03 -0500 Subject: [PATCH 04/27] craft-xlsx: review fixes: names using a left-out name are left out; quoted sheet-scoped calls - A name whose formula uses a name that is left out (another workbook, a skipped sheet, an unreadable formula) was still published, so a cell using it decoded and would calculate to #NAME?. Each name's formula now records the names it uses, and every name reached from a left-out one is left out too, whatever their order: once each, so a long chain costs no more than its uses. A cell using one keeps its saved result. - A sheet-scoped name called as a function through PtgNameX prints its sheet quoted where needed ('My Sheet'!Fn(1)), as a sheet-qualified name does. --- crates/xlsx/README.md | 24 +++++++++---------- crates/xlsx/src/xlsb.rs | 42 +++++++++++++++++++++++++++++++-- crates/xlsx/src/xlsb/expr.rs | 24 ++++++++++++++++++- crates/xlsx/src/xlsb/formula.rs | 10 +++++++- crates/xlsx/tests/xlsb.rs | 22 +++++++++++++++++ 5 files changed, 106 insertions(+), 16 deletions(-) diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 75dd8602a..6e6291934 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -91,18 +91,18 @@ relative references as the cell they reach from A1, macro names with their `func newer functions and LET parameters are left out, as in `.xlsx`, and so is the formula-less workbook-level built-in Excel writes beside a sheet's (`Database`), which formulas still spell `_xlnm.Database`, and the formula-less name of a VBA function a cell calls (`.xlsm` lists none; the -cell spells it); a name whose formula can't be read is left out with a warning. A reference may name -an XLM macro sheet (`Macro1!$A$1`, where macro names point), and a 3-D span is spelled by its ends -whatever chart sheets lie between them. Its cell formulas are decoded from their tokens into the -text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four ISO/IEC -29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in the -formula, numbers as Excel spells them), with `FormulaKind::Shared` or `FormulaKind::Array` and their -ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own -text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs -something the reader does not read keeps only its saved result, with a warning saying why: a defined -name left out, table references, other workbooks, Python code stored outside the cell, data tables. -The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from -`craft-xls`'s. +cell spells it); a name whose formula can't be read is left out with a warning, and so is every name +whose formula uses a left-out name. A reference may name an XLM macro sheet (`Macro1!$A$1`, where +macro names point), and a 3-D span is spelled by its ends whatever chart sheets lie between them. +Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and +`_xlpm.` names as written, apart from the four ISO/IEC 29500 functions `.xlsx` names bare, no +implicit `@`, the spaces and line breaks typed in the formula, numbers as Excel spells them), with +`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell +metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the +file keeps the tokens once. A formula whose decoding needs something the reader does not read keeps +only its saved result, with a warning saying why: a defined name left out, table references, other +workbooks, Python code stored outside the cell, data tables. The XLSB function table +(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 1eb7cddc1..22976d826 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -397,7 +397,10 @@ fn defined_names( budget: &Budget, warnings: &mut Warnings, ) -> Vec { - let mut out = vec![]; + // Each name's formula, and the names it uses. A name whose formula uses a name that is left + // out is left out too, whatever their order, so a cell using it keeps its saved result. + let mut omitted = vec![false; links.names.len()]; + let mut kept = vec![]; for p in pending { let Some(name) = links.names.get(p.index) else { continue }; if name.future || formula::is_param(&name.text) { @@ -409,6 +412,7 @@ fn defined_names( Some(Some(i)) => Some(*i), _ => { warnings.warn("XLSB: some defined names were left out (names of sheets the reader skips)"); + omit(&mut omitted, p.index); continue; } }, @@ -421,17 +425,46 @@ fn defined_names( } continue; } - let formula = match formula::decode(&p.rgce, &p.rgcb, links, Mode::Name) { + let mut used = vec![]; + let formula = match formula::decode_using(&p.rgce, &p.rgcb, links, Mode::Name, &mut used) { Ok(e) => expr::print(&e), Err(formula::Unsupported(why)) => { warnings.warn(format!("XLSB: some defined names were left out ({why})")); + omit(&mut omitted, p.index); continue; } }; if !budget.try_formula_text(formula.len(), limits) { warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); + omit(&mut omitted, p.index); continue; } + kept.push((p, local_sheet_id, formula, used)); + } + // Who uses each name, then every name reached from a left-out one: once each, so a chain + // of any length costs what its uses do. + let mut users: HashMap> = HashMap::new(); + for (k, (.., used)) in kept.iter().enumerate() { + for &u in used { + users.entry(u).or_default().push(k); + } + } + let mut queue: Vec = (0..omitted.len()).filter(|&i| omitted[i]).collect(); + while let Some(i) = queue.pop() { + for &k in users.get(&i).map_or(&[][..], Vec::as_slice) { + let Some((p, ..)) = kept.get(k) else { continue }; + if omit(&mut omitted, p.index) { + warnings.warn("XLSB: some defined names were left out (names using a name that was left out)"); + queue.push(p.index); + } + } + } + let mut out = vec![]; + for (p, local_sheet_id, formula, _) in kept { + if omitted.get(p.index).copied().unwrap_or(true) { + continue; + } + let Some(name) = links.names.get(p.index) else { continue }; let mut attrs = vec![]; let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); // In `CT_DefinedName`'s order. @@ -478,6 +511,11 @@ fn defined_names( out } +/// Marks name `i` left out; whether it wasn't already. +fn omit(omitted: &mut [bool], i: usize) -> bool { + omitted.get_mut(i).is_some_and(|o| !std::mem::replace(o, true)) +} + /// A supporting link (`BrtSupSelf`, `BrtSupSame`, `BrtSupAddin`, `BrtSupBookSrc`), the sheet links /// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`; a name's other parts go to /// `pending`. diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index eee43a557..4c5efe4f7 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -283,6 +283,20 @@ fn write_args(s: &mut String, args: &[Expr], sep: char) { } } +/// A function's name; a sheet-scoped one (`My Sheet!Fn`) is quoted as a sheet-qualified name is. +/// Kept out of [`write_expr`], whose frame every nesting level of a formula repeats. +#[inline(never)] +fn push_function_name(s: &mut String, name: &str) { + match sheet_split(name) { + Some((sheet, name)) => { + s.push_str("e_sheet(sheet)); + s.push('!'); + s.push_str(name); + } + None => s.push_str(name), + } +} + fn write_expr(s: &mut String, e: &Expr) { match e { Expr::Number(n) => s.push_str(&number(*n)), @@ -331,7 +345,7 @@ fn write_expr(s: &mut String, e: &Expr) { write_expr(s, b); } Expr::Call(name, args) => { - s.push_str(name); + push_function_name(s, name); s.push('('); write_args(s, args, ','); s.push(')'); @@ -377,4 +391,12 @@ mod tests { assert_eq!(number_to_text(1e100), "1E+100"); assert!(number(f64::MAX).parse::().is_ok_and(f64::is_finite)); } + + #[test] + fn a_sheet_scoped_function_name_is_quoted() { + let call = |name: &str| print(&Expr::Call(name.into(), vec![Expr::Number(1.0)])); + assert_eq!(call("My Sheet!Fn"), "'My Sheet'!Fn(1)"); + assert_eq!(call("Data!Fn"), "Data!Fn(1)"); + assert_eq!(call("SUM"), "SUM(1)"); + } } diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 2c5712d33..46f7b5acd 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -338,6 +338,11 @@ fn reference(sheet: Option, kind: RefKind) -> Expr { /// represented. A shared formula's tokens are decoded once for each of its cells /// (`Mode::Shared` with that cell), whose relative references are offsets from it. pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Result { + decode_using(rgce, rgcb, links, mode, &mut Vec::new()) +} + +/// [`decode`], adding to `used` the 0-based index of every defined name the formula uses. +pub(super) fn decode_using(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode, used: &mut Vec) -> Result { if rgce.len() > MAX_RGCE { return Err(Unsupported("formula too long")); } @@ -512,7 +517,9 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res } 0x23 => { // PtgName: a 1-based index into every BrtName, hidden ones included. - let name = links.name(t.u32()?, mode)?; + let index = t.u32()?; + let name = links.name(index, mode)?; + used.extend(usize::try_from(index).ok().and_then(|i| i.checked_sub(1))); push(&mut stack, kids, Expr::Name(name.text.clone()))?; } 0x24 => { @@ -553,6 +560,7 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res let index = t.u32()?; links.this_book(ixti)?; let name = links.name(index, mode)?; + used.extend(usize::try_from(index).ok().and_then(|i| i.checked_sub(1))); let sheet = name.scope.and_then(|s| links.sheets.get(usize::try_from(s).ok()?).cloned().flatten()); let sheet = sheet.ok_or(Unsupported("names in other workbooks or on other sheets"))?; push(&mut stack, kids, Expr::Name(format!("{sheet}!{}", name.text)))?; diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 6502c1e5d..4d8f12d75 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1280,3 +1280,25 @@ fn a_spent_formula_allowance_skips_decoding() { assert!(warned(&wb, "more formula text than the limits allow"), "{:?}", wb.warnings); assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings); } + +#[test] +fn a_name_using_a_left_out_name_is_left_out_too() { + // 14 Early = Later (a later name), 15 Alias = External (left out: another workbook), + // 16 Later = Alias, 17 Fine = Rate. Early, Alias and Later are left out whatever their order, + // and cells using them keep their saved results; Fine is read. + let mut list = names(); + for (text, index) in [("Early", 16u32), ("Alias", 3), ("Later", 15), ("Fine", 9)] { + list.push(brt_name(0, 0, WORKBOOK, text, &[&[0x23u8][..], &index.to_le_bytes()].concat(), None)); + } + let cell = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + let data = + [row(0), formula_cell(1, &cell(14), &[]), formula_cell(2, &cell(15), &[]), formula_cell(3, &cell(16), &[]), formula_cell(4, &cell(17), &[])] + .concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = NAMES_XML.replace("", r#"Rate"#); + assert_eq!(wb.defined_names, xml_names(&xml)); + let book = read(&bytes).unwrap(); + assert_eq!((1..=4).map(|c| book.sheets[0].formula(CellRef::new(0, c))).collect::>(), [None, None, None, Some("Fine")]); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} From 6fde933469b8e86fc9d6d0f034fc7b680bb819d0 Mon Sep 17 00:00:00 2001 From: Diego Guerini Date: Sun, 15 Nov 2026 18:01:57 +0100 Subject: [PATCH 05/27] craft-docx: read a DrawingML locked canvas as a group of its members Word 2007 writes shapes, pictures and WordArt in a drawing as an lc:lockedCanvas (ECMA-376 Part 1 20.3.2.1) in compatibility mode. Only its first picture was read; its shapes and their text were dropped. The canvas is now rewritten as the wpg group the reader already handles, its DrawingML text as text-box paragraphs with their run formatting. Moved from storytold/wordcraft#5067 (wordcraft#5060), whose DOCX reader now lives here. --- crates/docx/src/read/story.rs | 1 + crates/docx/src/read/story/drawing.rs | 4 + crates/docx/src/read/story/locked_canvas.rs | 254 ++++++++++++++++++++ crates/docx/src/xml.rs | 4 +- crates/docx/tests/it/locked_canvas.rs | 143 +++++++++++ crates/docx/tests/it/main.rs | 1 + 6 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 crates/docx/src/read/story/locked_canvas.rs create mode 100644 crates/docx/tests/it/locked_canvas.rs diff --git a/crates/docx/src/read/story.rs b/crates/docx/src/read/story.rs index 770375365..86ceb1651 100644 --- a/crates/docx/src/read/story.rs +++ b/crates/docx/src/read/story.rs @@ -22,6 +22,7 @@ use craft_wordmodel::wordart::TextWarp; mod blocks; mod drawing; mod fields; +mod locked_canvas; mod runs; mod vml; diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..0b7f7e78a 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -82,6 +82,10 @@ impl Reader<'_> { self.read_canvas(sc, c, rels, w, h, float) } else if let Some(g) = gd.child("wpg:wgp") { self.read_group(sc, g, rels, w, h, float)? + } else if let Some(lc) = gd.child("lc:lockedCanvas") { + // Word 2007's shapes, pictures and text in a drawing: read as the group they are. + let g = super::locked_canvas::as_group(lc, &self.doc.settings.theme_colors); + self.read_group(sc, &g, rels, w, h, float)? } else if direct_shape.is_none() && gd.find("a:blip").is_some() { self.read_pic(gd, rels, w, h, alt.clone(), float)? } else { diff --git a/crates/docx/src/read/story/locked_canvas.rs b/crates/docx/src/read/story/locked_canvas.rs new file mode 100644 index 000000000..01e8f2872 --- /dev/null +++ b/crates/docx/src/read/story/locked_canvas.rs @@ -0,0 +1,254 @@ +//! A DrawingML locked canvas (`lc:lockedCanvas`, ECMA-376 Part 1 §20.3.2.1): the shapes, +//! pictures and text Word 2007 writes in a drawing in compatibility mode (wordcraft#5060). It is a group +//! shape in the DrawingML namespace (`a:sp`, `a:pic`, `a:grpSp`, `a:cxnSp`, text in `a:txSp`), so +//! it is read as the `wpg:wgp` group the reader already knows: each member renamed into its +//! WordprocessingML drawing form, and its DrawingML text into `w:txbxContent` paragraphs. + +use super::{El, Node, Rgb}; + +/// Groups nested deeper than this are left out, as `group_members` does. +const MAX_NEST: usize = 8; +/// DrawingML's text size when neither the run nor the list style gives one (18 pt, in +/// hundredths of a point). +const DEFAULT_SZ: i64 = 1800; + +/// The locked canvas `lc` as a `wpg:wgp` group. `theme` resolves its text colours. +pub(super) fn as_group(lc: &El, theme: &[Rgb]) -> El { + el("wpg:wgp", Vec::new(), members(lc, theme, 0)) +} + +/// The members of DrawingML group `g`, in their WordprocessingML drawing form. +fn members(g: &El, theme: &[Rgb], depth: usize) -> Vec { + g.els() + .filter_map(|e| match e.name.as_str() { + "a:grpSpPr" => Some(renamed(e, "wpg:grpSpPr")), + "a:grpSp" if depth < MAX_NEST => Some(el("wpg:grpSp", Vec::new(), members(e, theme, depth + 1))), + "a:sp" => Some(shape(e, theme)), + "a:cxnSp" => Some(connector(e)), + "a:pic" => Some(picture(e)), + "a:txSp" => Some(text_shape(e, theme)), + _ => None, + }) + .map(Node::El) + .collect() +} + +/// `a:sp` as `wps:wsp`: its name, properties and style, and its text as a text box's. +fn shape(sp: &El, theme: &[Rgb]) -> El { + let nv = sp.child("a:nvSpPr"); + let mut kids: Vec = [ + nv.and_then(|n| n.child("a:cNvPr")).map(|e| renamed(e, "wps:cNvPr")), + nv.and_then(|n| n.child("a:cNvSpPr")).map(|e| renamed(e, "wps:cNvSpPr")), + sp.child("a:spPr").map(|e| renamed(e, "wps:spPr")), + sp.child("a:style").map(|e| renamed(e, "wps:style")), + ] + .into_iter() + .flatten() + .map(Node::El) + .collect(); + // The text sits in the shape's own rectangle (`a:useSpRect`); a text rectangle of its own + // (`a:txSp/a:xfrm`) is not kept apart. + if let Some(body) = sp.child("a:txSp").and_then(|t| t.child("a:txBody")) { + kids.extend(text_box(body, theme)); + } + el("wps:wsp", Vec::new(), kids) +} + +/// A group's text shape (`a:txSp` as a member, text with no shape of its own) as a `wps:wsp` text +/// box without fill or outline at its own rectangle (`a:xfrm`). +fn text_shape(tx: &El, theme: &[Rgb]) -> El { + let none = || Node::El(el("a:noFill", Vec::new(), Vec::new())); + let geometry = el("a:prstGeom", attrs(&[("prst", "rect")]), vec![Node::El(el("a:avLst", Vec::new(), Vec::new()))]); + let mut sp_pr: Vec = tx.child("a:xfrm").map(|x| Node::El(x.clone())).into_iter().collect(); + sp_pr.extend([Node::El(geometry), none(), Node::El(el("a:ln", Vec::new(), vec![none()]))]); + let mut kids = vec![Node::El(el("wps:cNvSpPr", attrs(&[("txBox", "1")]), Vec::new())), Node::El(el("wps:spPr", Vec::new(), sp_pr))]; + if let Some(body) = tx.child("a:txBody") { + kids.extend(text_box(body, theme)); + } + el("wps:wsp", Vec::new(), kids) +} + +/// DrawingML text body `body` as a text box's content and body properties. +fn text_box(body: &El, theme: &[Rgb]) -> Vec { + let content = el("w:txbxContent", Vec::new(), paragraphs(body, theme)); + let mut kids = vec![Node::El(el("wps:txbx", Vec::new(), vec![Node::El(content)]))]; + if let Some(bp) = body.child("a:bodyPr") { + kids.push(Node::El(renamed(bp, "wps:bodyPr"))); + } + kids +} + +/// `a:cxnSp` as a `wps:wsp` line. Its glued ends name shapes by the canvas's own ids, which +/// aren't the document's, so they are left out. +fn connector(cx: &El) -> El { + let kids = [ + cx.child("a:nvCxnSpPr").and_then(|n| n.child("a:cNvPr")).map(|e| renamed(e, "wps:cNvPr")), + cx.child("a:spPr").map(|e| renamed(e, "wps:spPr")), + cx.child("a:style").map(|e| renamed(e, "wps:style")), + ]; + el("wps:wsp", Vec::new(), kids.into_iter().flatten().map(Node::El).collect()) +} + +/// `a:pic` as `pic:pic`. +fn picture(pic: &El) -> El { + let nv = pic.child("a:nvPicPr").and_then(|n| n.child("a:cNvPr")).map(|e| el("pic:nvPicPr", Vec::new(), vec![Node::El(renamed(e, "pic:cNvPr"))])); + let kids = [nv, pic.child("a:blipFill").map(|e| renamed(e, "pic:blipFill")), pic.child("a:spPr").map(|e| renamed(e, "pic:spPr"))]; + el("pic:pic", Vec::new(), kids.into_iter().flatten().map(Node::El).collect()) +} + +/// The paragraphs of DrawingML text body `body` as `w:p`s. DrawingML text has no space before +/// or after a paragraph and single lines, so each says so rather than taking the document's +/// defaults. +fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { + body.children("a:p") + .map(|p| { + // A run's properties default to its paragraph's own (`a:pPr/a:defRPr`), then to the + // list style's for the paragraph's level (`lvl`, 0–8), property by property. + let ppr_el = p.child("a:pPr"); + let level = ppr_el.and_then(|pp| pp.attr("lvl")).and_then(|v| v.trim().parse::().ok()).filter(|l| *l <= 8).unwrap_or(0); + let level_style = body.child("a:lstStyle").and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))); + let defaults: Vec<&El> = + [ppr_el.and_then(|pp| pp.child("a:defRPr")), level_style.and_then(|l| l.child("a:defRPr"))].into_iter().flatten().collect(); + let defaults = defaults.as_slice(); + let mut ppr = + vec![Node::El(el("w:spacing", attrs(&[("w:before", "0"), ("w:after", "0"), ("w:line", "240"), ("w:lineRule", "auto")]), Vec::new()))]; + let jc = match ppr_el.and_then(|pp| pp.attr("algn")) { + Some("ctr") => Some("center"), + Some("r") => Some("right"), + Some("just") => Some("both"), + Some("dist") => Some("distribute"), + Some("l") => Some("left"), + _ => None, + }; + if let Some(jc) = jc { + ppr.push(Node::El(el("w:jc", attrs(&[("w:val", jc)]), Vec::new()))); + } + // The paragraph mark's own size and font (`a:endParaRPr`), which an empty one's + // height follows. + ppr.push(Node::El(run_props(p.child("a:endParaRPr"), defaults, theme))); + let mut kids = vec![Node::El(el("w:pPr", Vec::new(), ppr))]; + for r in p.els() { + let text = match r.name.as_str() { + "a:r" | "a:fld" => Node::El(el( + "w:t", + attrs(&[("xml:space", "preserve")]), + vec![Node::Text(r.child("a:t").map(|t| t.text()).unwrap_or_default())], + )), + "a:br" => Node::El(el("w:br", Vec::new(), Vec::new())), + _ => continue, + }; + let rpr = run_props(r.child("a:rPr"), defaults, theme); + kids.push(Node::El(el("w:r", Vec::new(), vec![Node::El(rpr), text]))); + } + Node::El(el("w:p", Vec::new(), kids)) + }) + .collect() +} + +/// A DrawingML run's properties (`a:rPr`, each falling back on the first of `defaults`, the +/// `a:defRPr`s in order, that gives it) as `w:rPr`: font, size, bold, italic, underline, +/// strikethrough, caps, baseline and colour. +fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { + let get = |n: &str| rpr.and_then(|r| r.attr(n)).or_else(|| defaults.iter().find_map(|d| d.attr(n))); + let child = |n: &str| rpr.and_then(|r| r.child(n)).or_else(|| defaults.iter().find_map(|d| d.child(n))); + let on = |n: &str| get(n).is_some_and(|v| v == "1" || v == "true"); + let mut kids = Vec::new(); + let mut push = |name: &str, a: &[(&str, &str)]| kids.push(Node::El(el(name, attrs(a), Vec::new()))); + // Theme fonts (`+mn-lt`) are left to the document's. + let face = |n: &str| child(n).and_then(|f| f.attr("typeface")).filter(|t| !t.is_empty() && !t.starts_with('+')); + let mut fonts = Vec::new(); + if let Some(f) = face("a:latin") { + fonts.extend([("w:ascii", f), ("w:hAnsi", f)]); + } + if let Some(f) = face("a:ea") { + fonts.push(("w:eastAsia", f)); + } + if let Some(f) = face("a:cs") { + fonts.push(("w:cs", f)); + } + if !fonts.is_empty() { + push("w:rFonts", &fonts); + } + if on("b") { + push("w:b", &[]); + } + if on("i") { + push("w:i", &[]); + } + match get("cap") { + Some("all") => push("w:caps", &[]), + Some("small") => push("w:smallCaps", &[]), + _ => {} + } + match get("strike") { + Some("sngStrike") => push("w:strike", &[]), + Some("dblStrike") => push("w:dstrike", &[]), + _ => {} + } + match get("u") { + None | Some("none") => {} + Some("dbl") => push("w:u", &[("w:val", "double")]), + Some(_) => push("w:u", &[("w:val", "single")]), + } + match get("baseline").and_then(|v| v.trim().parse::().ok()) { + Some(b) if b > 0 => push("w:vertAlign", &[("w:val", "superscript")]), + Some(b) if b < 0 => push("w:vertAlign", &[("w:val", "subscript")]), + _ => {} + } + if let Some(rgb) = child("a:solidFill").and_then(|f| crate::read::drawing_color::color(f, theme)) { + push("w:color", &[("w:val", &rgb.hex())]); + } + // Hundredths of a point into half-points, within Word's 1–1638 pt. + let sz = get("sz").and_then(|v| v.trim().parse::().ok()).unwrap_or(DEFAULT_SZ).clamp(100, 163_800) / 50; + let sz = sz.to_string(); + push("w:sz", &[("w:val", &sz)]); + push("w:szCs", &[("w:val", &sz)]); + // WordArt's outline, shadow, glow and reflection, as Word 2010 writes them in `w:rPr`. + kids.extend(text_effects(child("a:ln"), child("a:effectLst"), theme).into_iter().map(Node::El)); + el("w:rPr", Vec::new(), kids) +} + +/// A DrawingML run's text outline (`a:ln`) and effects (`a:effectLst`: outer shadow, glow, +/// reflection) as the `w14:` run properties ([MS-DOCX] §2.6.1) the run reader takes for WordArt. +/// Colours are resolved here, theme and modifiers included, into a `w14:srgbClr` with its +/// transparency. Lengths (EMUs), angles and percentages carry over as they are. +fn text_effects(ln: Option<&El>, fx: Option<&El>, theme: &[Rgb]) -> Vec { + let color = |e: &El| -> Option { + let (rgb, transparency) = crate::read::drawing_color::color_alpha(e, theme, None)?; + let alpha = ((transparency.clamp(0.0, 100.0) * 1000.0).round() as i64).to_string(); + let kids = if transparency > 0.0 { vec![Node::El(el("w14:alpha", attrs(&[("w14:val", &alpha)]), Vec::new()))] } else { Vec::new() }; + Some(Node::El(el("w14:srgbClr", attrs(&[("w14:val", &rgb.hex())]), kids))) + }; + // The attributes `names` of `e`, under the `w14:` prefix. + let copied = |e: &El, names: &[&str]| -> Vec<(String, String)> { + names.iter().filter_map(|n| e.attr(n).map(|v| (format!("w14:{n}"), v.to_string()))).collect() + }; + let mut out = Vec::new(); + if let Some(ln) = ln + && let Some(c) = ln.child("a:solidFill").and_then(color) + { + out.push(el("w14:textOutline", copied(ln, &["w"]), vec![Node::El(el("w14:solidFill", Vec::new(), vec![c]))])); + } + for e in fx.into_iter().flat_map(|f| f.els()) { + match e.name.as_str() { + "a:outerShdw" => out.push(el("w14:shadow", copied(e, &["blurRad", "dist", "dir"]), color(e).into_iter().collect())), + "a:glow" => out.push(el("w14:glow", copied(e, &["rad"]), color(e).into_iter().collect())), + "a:reflection" => out.push(el("w14:reflection", copied(e, &["blurRad", "stA", "endPos", "dist"]), Vec::new())), + _ => {} + } + } + out +} + +fn el(name: &str, attrs: Vec<(String, String)>, kids: Vec) -> El { + El { name: name.to_string(), attrs, kids } +} + +fn attrs(a: &[(&str, &str)]) -> Vec<(String, String)> { + a.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() +} + +/// `e` under another name, with its attributes and children. +fn renamed(e: &El, name: &str) -> El { + El { name: name.to_string(), ..e.clone() } +} diff --git a/crates/docx/src/xml.rs b/crates/docx/src/xml.rs index bb86755d9..fa50d217a 100644 --- a/crates/docx/src/xml.rs +++ b/crates/docx/src/xml.rs @@ -52,8 +52,10 @@ pub const NAMESPACES: &[(&str, &str)] = &[ pub const DECORATIVE_EXT: &str = "{C183D7F6-B498-43B3-948B-1728B52AA6E4}"; pub const DECORATIVE_NS: &str = "http://schemas.microsoft.com/office/drawing/2017/decorative"; -/// Namespaces that only appear in non-body parts. +/// Namespaces that only appear in non-body parts, or that the reader reads but never writes. const OTHER_NAMESPACES: &[(&str, &str)] = &[ + // A locked canvas, read into a drawing group (wordcraft#5060). + ("lc", "http://schemas.openxmlformats.org/drawingml/2006/lockedCanvas"), ("b", "http://schemas.openxmlformats.org/officeDocument/2006/bibliography"), ("rel", "http://schemas.openxmlformats.org/package/2006/relationships"), ("ct", "http://schemas.openxmlformats.org/package/2006/content-types"), diff --git a/crates/docx/tests/it/locked_canvas.rs b/crates/docx/tests/it/locked_canvas.rs new file mode 100644 index 000000000..8fa7a0a20 --- /dev/null +++ b/crates/docx/tests/it/locked_canvas.rs @@ -0,0 +1,143 @@ +//! A DrawingML locked canvas (`lc:lockedCanvas`, ECMA-376 Part 1 §20.3.2.1), as Word 2007 writes +//! shapes, pictures and WordArt in a drawing in compatibility mode (wordcraft#5060). Fixtures are +//! hand-written from ECMA-376 Part 1 §20.1 (DrawingML main) and §20.3 (locked canvas). + +use std::io::Write; + +use craft_wordmodel::para::ShapeKind; +use craft_wordmodel::props::{Rgb, TextColor}; +use craft_wordmodel::{Document, InlineObject, StoryRef}; + +const NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main""#; + +fn docx(body: &str) -> Vec { + let doc = format!(r#"{body}"#); + let types = + r#""#; + let root = r#""#; + let rels = r#""#; + let entries: [(&str, &[u8]); 5] = [ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", root.as_bytes()), + ("word/document.xml", doc.as_bytes()), + ("word/_rels/document.xml.rels", rels.as_bytes()), + ("word/media/image1.png", "\u{89}PNG stand-in".as_bytes()), + ]; + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, bytes) in entries { + zw.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +/// A floating drawing, 200 × 100 pt, holding `canvas` (the locked canvas's members) in a canvas +/// whose members' space starts at 10 pt, 10 pt. +fn drawing(canvas: &str) -> String { + format!( + r#"00{canvas}"# + ) +} + +const OVAL: &str = r#""#; + +/// WordArt in a nested group: bold, all-caps red text with a green outline and a reflection, +/// 32 pt, centred, bent by a chevron warp; a second paragraph takes the list style's size, and an +/// empty third one its mark's. +const WORD_ART: &str = r#"Youngkeepers"#; + +const PICTURE: &str = r#""#; + +/// A line glued to the oval by the canvas's own id, which isn't a document shape's. +const LINE: &str = r#""#; + +fn group(d: &Document) -> (f32, f32, Vec) { + let p = d.body.iter().find_map(|b| b.as_para()).unwrap(); + match p.objects.first() { + Some(InlineObject::Group { w, h, children, canvas: None, .. }) => (*w, *h, children.clone()), + o => panic!("{o:?}"), + } +} + +/// wordcraft#5060: the locked canvas is a group of its shapes, text, picture and line, each where the +/// canvas (and the group inside it) puts it, and nothing is lost on save. +#[test] +fn a_locked_canvas_reads_as_a_group_of_its_members() { + let d = craft_docx::read(&docx(&drawing(&[OVAL, WORD_ART, PICTURE, LINE].concat()))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (w, h, children) = group(d); + assert_eq!((w, h), (200.0, 100.0)); + let at: Vec<(f32, f32)> = children.iter().map(|c| (c.x, c.y)).collect(); + assert_eq!(at, [(0.0, 0.0), (110.0, 60.0), (0.0, 50.0), (0.0, 80.0)]); + let [oval, art, picture, line] = &children[..] else { panic!("{children:?}") }; + let InlineObject::Shape { kind: ShapeKind::Ellipse, w: 100.0, h: 50.0, fill: Some(fill), stroke: Some(stroke), .. } = &oval.obj else { + panic!("{:?}", oval.obj) + }; + assert_eq!((fill.main_color(), *stroke), (Rgb(0xFF, 0xFF, 0x66), Rgb(0, 0, 0))); + let InlineObject::Shape { kind: ShapeKind::TextBox, w: 20.0, h: 10.0, story: Some(id), extra, .. } = &art.obj else { + panic!("{:?}", art.obj) + }; + assert_eq!(extra.warp.as_ref().map(|w| w.preset.as_str()), Some("textChevron")); + assert_eq!(d.plain_text(StoryRef::Part(*id)).trim_end(), "Young\nkeepers"); + let blocks = &d.parts.get(id).unwrap().blocks; + let first = blocks[0].as_para().unwrap(); + assert_eq!(first.props.align, Some(craft_wordmodel::props::Align::Center)); + assert_eq!((first.props.space_before, first.props.space_after), (Some(0.0), Some(0.0)), "DrawingML text has no paragraph spacing"); + let run = &first.runs[0].props; + assert_eq!((run.size, run.bold, run.caps, run.color), (Some(32.0), Some(true), Some(true), Some(TextColor::Rgb(Rgb(0xFF, 0, 0))))); + assert_eq!(run.font.as_deref(), Some("Corsiva Stand-in")); + let fx = run.text_effects.as_deref().unwrap(); + let outline = fx.outline.unwrap(); + assert_eq!((outline.color, outline.width), (Some(Rgb(0, 0xFF, 0)), 0.75)); + assert!(fx.reflection.is_some(), "{fx:?}"); + assert_eq!(blocks[1].as_para().unwrap().runs[0].props.size, Some(14.0), "the list style's size"); + assert_eq!(blocks[2].as_para().unwrap().mark.size, Some(10.0), "an empty paragraph's mark"); + let InlineObject::Image { w: 50.0, h: 50.0, alt, .. } = &picture.obj else { panic!("{:?}", picture.obj) }; + assert_eq!(alt, "A logo"); + let InlineObject::Shape { kind: ShapeKind::Line, stroke: Some(blue), extra, .. } = &line.obj else { panic!("{:?}", line.obj) }; + assert_eq!((*blue, extra.start), (Rgb(0, 0, 0xFF), None), "no glue to a canvas id"); + } +} + +/// A text shape as a member of the canvas (`a:txSp`, text with no shape of its own) is a text box +/// at its own rectangle. Runs take their paragraph's own defaults (`a:pPr/a:defRPr`) before the +/// list style's for the paragraph's level. +#[test] +fn a_locked_canvas_text_shape_and_paragraph_defaults() { + let text = r#"OwnLevelFirst"#; + let d = craft_docx::read(&docx(&drawing(text))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (_, _, children) = group(d); + let [shape] = &children[..] else { panic!("{children:?}") }; + assert_eq!((shape.x, shape.y), (40.0, 20.0)); + let InlineObject::Shape { kind: ShapeKind::TextBox, w: 100.0, h: 40.0, fill: None, stroke: None, story: Some(id), .. } = &shape.obj else { + panic!("{:?}", shape.obj) + }; + assert_eq!(d.plain_text(StoryRef::Part(*id)).trim_end(), "Own\nLevel\nFirst"); + let blocks = &d.parts.get(id).unwrap().blocks; + let run = |i: usize| blocks[i].as_para().unwrap().runs[0].props.clone(); + assert_eq!((run(0).size, run(0).italic), (Some(32.0), Some(true)), "the paragraph's own size, the level's italic"); + assert_eq!((run(1).size, run(1).italic), (Some(20.0), None), "level 2's"); + assert_eq!((run(2).size, run(2).italic), (Some(14.0), Some(true)), "level 1's"); + } +} + +/// Before wordcraft#5060 only the canvas's first picture showed: the shapes and their text were dropped. +#[test] +fn a_locked_canvas_is_not_read_as_its_picture_alone() { + let d = craft_docx::read(&docx(&drawing(&[OVAL, PICTURE].concat()))).unwrap(); + let (_, _, children) = group(&d); + assert_eq!(children.len(), 2); +} + +/// Hostile canvases never panic: no members, groups nested past the limit, text with no runs, +/// huge and broken sizes. +#[test] +fn hostile_locked_canvases_never_panic() { + let deep = format!("{}{}{}", r#""#.repeat(40), OVAL, "".repeat(40)); + let odd = r#"f"#; + for canvas in ["", deep.as_str(), odd] { + let d = craft_docx::read(&docx(&drawing(canvas))).unwrap(); + let _ = craft_docx::write(&d).unwrap(); + } +} diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index b0bab5833..7d706e3d8 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -23,6 +23,7 @@ mod heading_levels; mod legacy_forms; mod ligatures; mod line_numbering; +mod locked_canvas; mod malformed; mod missing_pictures; mod nested_block_controls; From ce8ab5cc6d5ba4df57a29da6a43f3ccbd1c7205d Mon Sep 17 00:00:00 2001 From: Diego Guerini Date: Mon, 16 Nov 2026 08:52:44 +0100 Subject: [PATCH 06/27] craft-docx: a locked canvas's paragraph falls back on its level's alignment and the list style's defaults; bold or italic set off stays off --- crates/docx/src/read/story/locked_canvas.rs | 35 ++++++++++++++------- crates/docx/tests/it/locked_canvas.rs | 20 ++++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/crates/docx/src/read/story/locked_canvas.rs b/crates/docx/src/read/story/locked_canvas.rs index 01e8f2872..59a52cea2 100644 --- a/crates/docx/src/read/story/locked_canvas.rs +++ b/crates/docx/src/read/story/locked_canvas.rs @@ -102,17 +102,22 @@ fn picture(pic: &El) -> El { fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { body.children("a:p") .map(|p| { - // A run's properties default to its paragraph's own (`a:pPr/a:defRPr`), then to the - // list style's for the paragraph's level (`lvl`, 0–8), property by property. + // A paragraph's properties default to the list style's for its level (`lvl`, 0–8), + // then to the list style's defaults (`a:defPPr`), property by property; a run's to its + // paragraph's own (`a:pPr/a:defRPr`), then to those two's. let ppr_el = p.child("a:pPr"); let level = ppr_el.and_then(|pp| pp.attr("lvl")).and_then(|v| v.trim().parse::().ok()).filter(|l| *l <= 8).unwrap_or(0); - let level_style = body.child("a:lstStyle").and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))); - let defaults: Vec<&El> = - [ppr_el.and_then(|pp| pp.child("a:defRPr")), level_style.and_then(|l| l.child("a:defRPr"))].into_iter().flatten().collect(); + let list_style = body.child("a:lstStyle"); + let paragraph_styles: Vec<&El> = + [ppr_el, list_style.and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))), list_style.and_then(|l| l.child("a:defPPr"))] + .into_iter() + .flatten() + .collect(); + let defaults: Vec<&El> = paragraph_styles.iter().filter_map(|pp| pp.child("a:defRPr")).collect(); let defaults = defaults.as_slice(); let mut ppr = vec![Node::El(el("w:spacing", attrs(&[("w:before", "0"), ("w:after", "0"), ("w:line", "240"), ("w:lineRule", "auto")]), Vec::new()))]; - let jc = match ppr_el.and_then(|pp| pp.attr("algn")) { + let jc = match paragraph_styles.iter().find_map(|pp| pp.attr("algn")) { Some("ctr") => Some("center"), Some("r") => Some("right"), Some("just") => Some("both"), @@ -151,7 +156,12 @@ fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { let get = |n: &str| rpr.and_then(|r| r.attr(n)).or_else(|| defaults.iter().find_map(|d| d.attr(n))); let child = |n: &str| rpr.and_then(|r| r.child(n)).or_else(|| defaults.iter().find_map(|d| d.child(n))); - let on = |n: &str| get(n).is_some_and(|v| v == "1" || v == "true"); + // On, off (kept, so the document's style doesn't turn it back on), or not given. + let toggle = |n: &str| match get(n) { + Some("1" | "true") => Some(true), + Some("0" | "false") => Some(false), + _ => None, + }; let mut kids = Vec::new(); let mut push = |name: &str, a: &[(&str, &str)]| kids.push(Node::El(el(name, attrs(a), Vec::new()))); // Theme fonts (`+mn-lt`) are left to the document's. @@ -169,11 +179,12 @@ fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { if !fonts.is_empty() { push("w:rFonts", &fonts); } - if on("b") { - push("w:b", &[]); - } - if on("i") { - push("w:i", &[]); + for (drawingml, word) in [("b", "w:b"), ("i", "w:i")] { + match toggle(drawingml) { + Some(true) => push(word, &[]), + Some(false) => push(word, &[("w:val", "0")]), + None => {} + } } match get("cap") { Some("all") => push("w:caps", &[]), diff --git a/crates/docx/tests/it/locked_canvas.rs b/crates/docx/tests/it/locked_canvas.rs index 8fa7a0a20..fb01406a4 100644 --- a/crates/docx/tests/it/locked_canvas.rs +++ b/crates/docx/tests/it/locked_canvas.rs @@ -122,6 +122,26 @@ fn a_locked_canvas_text_shape_and_paragraph_defaults() { } } +/// A paragraph's alignment and its runs' properties fall back on its level's style, then on the +/// list style's defaults (`a:defPPr`); a run's bold set off stays off. +#[test] +fn a_locked_canvas_paragraph_takes_its_level_and_the_list_defaults() { + use craft_wordmodel::props::Align; + let text = r#"PlainCentredOwn"#; + let d = craft_docx::read(&docx(&drawing(text))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (_, _, children) = group(d); + let [shape] = &children[..] else { panic!("{children:?}") }; + let InlineObject::Shape { story: Some(id), .. } = &shape.obj else { panic!("{:?}", shape.obj) }; + let blocks = &d.parts.get(id).unwrap().blocks; + let para = |i: usize| blocks[i].as_para().unwrap(); + let aligns: Vec> = (0..3).map(|i| para(i).props.align).collect(); + assert_eq!(aligns, [Some(Align::Right), Some(Align::Center), Some(Align::Left)], "the list defaults', the level's, its own"); + let runs: Vec<(Option, Option)> = (0..3).map(|i| (para(i).runs[0].props.size, para(i).runs[0].props.bold)).collect(); + assert_eq!(runs, [(Some(16.0), Some(false)), (Some(16.0), Some(true)), (Some(16.0), Some(true))]); + } +} + /// Before wordcraft#5060 only the canvas's first picture showed: the shapes and their text were dropped. #[test] fn a_locked_canvas_is_not_read_as_its_picture_alone() { From ecc89eccffe8a3a99b9e611c55274469f4b2d35c Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:40:58 +0100 Subject: [PATCH 07/27] Add craft-unzip: Craft Anything's hardened ZIP reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves Craft Anything's crates/formats/src/zip.rs, zip_file.rs and the entry decoders from decompress.rs into a pure-Rust crate, keeping the R3.12 hardening: directory and extra-field bounds, ZIP64 end-record recovery after a prepended stub, multi-volume refusal, CRC-checked Info-ZIP Unicode Path names, decoded-size and CRC-32 checks. bzip2, LZMA/XZ and Zstandard entry methods sit behind default features; Serialize sits behind an optional serde feature. Part of #778. Co-authored-by: Afloat Co-authored-by: BFlat Co-authored-by: Brandon Thomas Co-authored-by: GGWA <65288322+rech4210@users.noreply.github.com> Co-authored-by: Max Mittelstädt <62167861+TheGalitube@users.noreply.github.com> Co-authored-by: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Co-authored-by: RRFZ <247457381+Plastic-time@users.noreply.github.com> Co-authored-by: Roman Serikoff --- Cargo.lock | 48 ++ Cargo.toml | 1 + README.md | 1 + crates/unzip/Cargo.toml | 34 ++ crates/unzip/src/bytes.rs | 41 ++ crates/unzip/src/decode.rs | 352 +++++++++++ crates/unzip/src/file.rs | 262 +++++++++ crates/unzip/src/lib.rs | 549 ++++++++++++++++++ crates/unzip/src/testkit.rs | 80 +++ crates/unzip/src/tests.rs | 393 +++++++++++++ crates/unzip/tests/common/mod.rs | 41 ++ .../tests/zip64_end_record_compatibility.rs | 169 ++++++ crates/unzip/tests/zip64_extra_boundaries.rs | 64 ++ crates/unzip/tests/zip_decoded_size.rs | 43 ++ crates/unzip/tests/zip_directory_bounds.rs | 74 +++ crates/unzip/tests/zip_entry_detection.rs | 43 ++ crates/unzip/tests/zip_split_volumes.rs | 165 ++++++ crates/unzip/tests/zip_stored_cap.rs | 21 + 18 files changed, 2381 insertions(+) create mode 100644 crates/unzip/Cargo.toml create mode 100644 crates/unzip/src/bytes.rs create mode 100644 crates/unzip/src/decode.rs create mode 100644 crates/unzip/src/file.rs create mode 100644 crates/unzip/src/lib.rs create mode 100644 crates/unzip/src/testkit.rs create mode 100644 crates/unzip/src/tests.rs create mode 100644 crates/unzip/tests/common/mod.rs create mode 100644 crates/unzip/tests/zip64_end_record_compatibility.rs create mode 100644 crates/unzip/tests/zip64_extra_boundaries.rs create mode 100644 crates/unzip/tests/zip_decoded_size.rs create mode 100644 crates/unzip/tests/zip_directory_bounds.rs create mode 100644 crates/unzip/tests/zip_entry_detection.rs create mode 100644 crates/unzip/tests/zip_split_volumes.rs create mode 100644 crates/unzip/tests/zip_stored_cap.rs diff --git a/Cargo.lock b/Cargo.lock index 480e000e5..7a00d57d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -921,6 +921,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bzip2-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beeb59e7e4c811ab37cc73680c798c7a5da77fc9989c62b09138e31ee740f735" +dependencies = [ + "crc32fast", + "tinyvec", +] + [[package]] name = "calloop" version = "0.13.0" @@ -2888,6 +2898,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "craft-unzip" +version = "0.1.0" +dependencies = [ + "bzip2-rs", + "lzma-rs", + "miniz_oxide 0.8.9", + "ruzstd", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-update" version = "0.1.0" @@ -3096,6 +3119,21 @@ dependencies = [ "zip", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc24" version = "0.1.6" @@ -6149,6 +6187,16 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "maybe-owned" version = "0.3.4" diff --git a/Cargo.toml b/Cargo.toml index bd08fa725..34ee6095e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,6 +174,7 @@ craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } +craft-unzip = { path = "crates/unzip" } craft-epub = { path = "crates/epub" } craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } diff --git a/README.md b/README.md index 8150dfea9..8a4c3d327 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit - [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. - [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. - [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. +- [`craft-unzip`](crates/unzip/src/lib.rs): ZIP reading over bytes already in memory (or one named entry from a seekable file), from Craft Anything: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind default features) checked against the declared size and CRC-32. Pure Rust with no `zip` crate underneath; for inspectors that must describe damaged archives rather than refuse them. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/crates/unzip/Cargo.toml b/crates/unzip/Cargo.toml new file mode 100644 index 000000000..8a414277b --- /dev/null +++ b/crates/unzip/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "craft-unzip" +description = "ZIP archives (APK, JAR, IPA, MSIX…) from hostile bytes, from Craft Anything: a central-directory listing with ZIP64, prepended-stub and multi-volume checks, and bounded entry reads checked against the declared size and CRC-32. Pure Rust, no I/O beyond an optional seekable reader." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +thiserror = { workspace = true } +miniz_oxide = { workspace = true } +# Entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. +bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 +lzma-rs = { version = "0.3", optional = true } # MIT +ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT +# `serde`: `Serialize` on the listing types, for apps that report listings as JSON. +serde = { workspace = true, optional = true } + +[features] +default = ["bzip2", "lzma", "zstd"] +# Method 12. +bzip2 = ["dep:bzip2-rs"] +# Methods 14 (ZIP LZMA) and 95 (XZ). +lzma = ["dep:lzma-rs"] +# Method 93. +zstd = ["dep:ruzstd"] +serde = ["dep:serde"] + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/crates/unzip/src/bytes.rs b/crates/unzip/src/bytes.rs new file mode 100644 index 000000000..be7aa3ebe --- /dev/null +++ b/crates/unzip/src/bytes.rs @@ -0,0 +1,41 @@ +//! Checked little-endian reads over untrusted bytes. Every read returns `None` when out of range. + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Reader<'a> { + data: &'a [u8], +} + +impl<'a> Reader<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Reader { data } + } + + pub(crate) fn data(&self) -> &'a [u8] { + self.data + } + + pub(crate) fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + let end = off.checked_add(len)?; + self.data.get(off..end) + } + + pub(crate) fn u8(&self, off: usize) -> Option { + self.data.get(off).copied() + } + + fn arr(&self, off: usize) -> Option<[u8; N]> { + self.slice(off, N)?.try_into().ok() + } + + pub(crate) fn u16_le(&self, off: usize) -> Option { + self.arr(off).map(u16::from_le_bytes) + } + + pub(crate) fn u32_le(&self, off: usize) -> Option { + self.arr(off).map(u32::from_le_bytes) + } + + pub(crate) fn u64_le(&self, off: usize) -> Option { + self.arr(off).map(u64::from_le_bytes) + } +} diff --git a/crates/unzip/src/decode.rs b/crates/unzip/src/decode.rs new file mode 100644 index 000000000..1d41d4a55 --- /dev/null +++ b/crates/unzip/src/decode.rs @@ -0,0 +1,352 @@ +//! Bounded decoders for the bzip2, ZIP LZMA, XZ and Zstandard entry methods, from Craft +//! Anything's `decompress.rs`. Each caps its output at `max_out` bytes (`TooLarge` rather than +//! allocating without limit) and never panics on a corrupt or truncated stream. + +use std::io; +#[cfg(any(feature = "bzip2", feature = "lzma"))] +use std::io::Read; +#[cfg(any(feature = "lzma", feature = "zstd"))] +use std::io::Write; + +/// Why a decode stopped. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub(crate) enum DecompressError { + #[error("the compressed stream is damaged")] + Corrupt, + #[error("the compressed stream ends early")] + #[cfg(any(feature = "lzma", feature = "zstd"))] + Truncated, + #[error("the output would exceed the size limit")] + TooLarge, +} + +/// A `Write` sink that collects at most `cap` bytes, then refuses the rest. The `exceeded` flag lets +/// the caller tell a size-limit stop from a genuine decode error. +#[cfg(any(feature = "lzma", feature = "zstd"))] +struct CapWrite { + buf: Vec, + cap: usize, + exceeded: bool, +} + +#[cfg(any(feature = "lzma", feature = "zstd"))] +impl CapWrite { + fn new(cap: usize) -> Self { + CapWrite { buf: Vec::new(), cap, exceeded: false } + } +} + +#[cfg(any(feature = "lzma", feature = "zstd"))] +impl Write for CapWrite { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.buf.len().saturating_add(data.len()) > self.cap { + self.exceeded = true; + return Err(io::Error::other("output size limit reached")); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn cap_of(max_out: u64) -> usize { + usize::try_from(max_out).unwrap_or(usize::MAX) +} + +/// Read a `Read` source into a `Vec`, capped at `cap` bytes. +#[cfg(feature = "bzip2")] +fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressError> { + let mut out = Vec::new(); + let mut chunk = [0u8; 64 * 1024]; + loop { + match src.read(&mut chunk) { + Ok(0) => return Ok(out), + Ok(n) => { + let slice = chunk.get(..n).unwrap_or(&[]); + if out.len().saturating_add(slice.len()) > cap { + return Err(DecompressError::TooLarge); + } + out.extend_from_slice(slice); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return Err(DecompressError::Corrupt), + } + } +} + +#[cfg(feature = "lzma")] +/// Decompress every xz stream, including standard stream padding, within one output budget. +pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> { + let ranges = xz_stream_ranges(input, max_out)?; + let mut out = CapWrite::new(cap_of(max_out)); + for range in ranges.into_iter().rev() { + let stream = input.get(range).ok_or(DecompressError::Truncated)?; + let mut cursor = io::Cursor::new(stream); + match lzma_rs::xz_decompress(&mut cursor, &mut out) { + Ok(()) => {} + Err(_) if out.exceeded => return Err(DecompressError::TooLarge), + Err(lzma_rs::error::Error::HeaderTooShort(_)) => return Err(DecompressError::Truncated), + Err(_) => return Err(DecompressError::Corrupt), + } + } + Ok(out.buf) +} + +#[cfg(feature = "lzma")] +/// Find stream boundaries backwards through their checked footer/index sizes, without +/// repeatedly trying the decoder at candidate magic bytes inside compressed data. +fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result>, DecompressError> { + let mut ranges = Vec::new(); + let mut end = input.len(); + let mut declared = 0u64; + while end > 0 { + let padded_end = end; + while end > 0 && input.get(end - 1) == Some(&0) { + end -= 1; + } + if end == 0 || !(padded_end - end).is_multiple_of(4) { + return Err(DecompressError::Corrupt); + } + let footer_at = end.checked_sub(12).ok_or(DecompressError::Truncated)?; + let footer = input.get(footer_at..end).ok_or(DecompressError::Truncated)?; + let read = crate::bytes::Reader::new(footer); + let footer_fields = footer.get(4..10).ok_or(DecompressError::Truncated)?; + if footer.get(10..) != Some(b"YZ") || read.u32_le(0) != Some(crate::crc32(footer_fields)) { + return Err(DecompressError::Corrupt); + } + let backward = u64::from(read.u32_le(4).ok_or(DecompressError::Truncated)?); + let index_len = usize::try_from((backward + 1) * 4).map_err(|_| DecompressError::Corrupt)?; + let index_at = footer_at.checked_sub(index_len).ok_or(DecompressError::Truncated)?; + let index = input.get(index_at..footer_at).ok_or(DecompressError::Truncated)?; + let crc_at = index.len().checked_sub(4).ok_or(DecompressError::Truncated)?; + let index_read = crate::bytes::Reader::new(index); + let index_fields = index.get(..crc_at).ok_or(DecompressError::Truncated)?; + if index.first() != Some(&0) || index_read.u32_le(crc_at) != Some(crate::crc32(index_fields)) { + return Err(DecompressError::Corrupt); + } + let mut records = index.get(1..crc_at).ok_or(DecompressError::Corrupt)?; + let count = xz_vli(&mut records)?; + if count > (records.len() / 2) as u64 { + return Err(DecompressError::Corrupt); + } + let mut block_bytes = 0usize; + for _ in 0..count { + let unpadded = xz_vli(&mut records)?; + if unpadded == 0 { + return Err(DecompressError::Corrupt); + } + let padded = usize::try_from(unpadded.checked_add(3).ok_or(DecompressError::Corrupt)? & !3).map_err(|_| DecompressError::Corrupt)?; + block_bytes = block_bytes.checked_add(padded).filter(|bytes| *bytes <= index_at).ok_or(DecompressError::Corrupt)?; + declared = declared.checked_add(xz_vli(&mut records)?).filter(|size| *size <= max_out).ok_or(DecompressError::TooLarge)?; + } + if records.len() > 3 || records.iter().any(|&byte| byte != 0) { + return Err(DecompressError::Corrupt); + } + let start = index_at.checked_sub(block_bytes).and_then(|at| at.checked_sub(12)).ok_or(DecompressError::Corrupt)?; + let magic_end = start.checked_add(6).ok_or(DecompressError::Corrupt)?; + if input.get(start..magic_end) != Some(b"\xfd7zXZ\0") { + return Err(DecompressError::Corrupt); + } + ranges.push(start..end); + end = start; + } + if ranges.is_empty() { + return Err(DecompressError::Corrupt); + } + Ok(ranges) +} + +#[cfg(feature = "lzma")] +/// XZ variable-length integers have at most nine bytes and no redundant final zero. +fn xz_vli(input: &mut &[u8]) -> Result { + let mut value = 0u64; + for shift in (0..63).step_by(7) { + let (&byte, rest) = input.split_first().ok_or(DecompressError::Truncated)?; + *input = rest; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return if shift != 0 && byte == 0 { Err(DecompressError::Corrupt) } else { Ok(value) }; + } + } + Err(DecompressError::Corrupt) +} + +#[cfg(feature = "lzma")] +/// Decode the property bytes and raw stream of a ZIP method-14 member. +pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, max_out: u64) -> Result, DecompressError> { + let cap = cap_of(max_out).min(isize::MAX as usize); + if expected_size > cap as u64 { + return Err(DecompressError::TooLarge); + } + let (properties, stream) = input.split_at_checked(5).ok_or(DecompressError::Truncated)?; + let [property, d0, d1, d2, d3] = <[u8; 5]>::try_from(properties).map_err(|_| DecompressError::Truncated)?; + let window_limit = u32::try_from(cap).unwrap_or(u32::MAX).max(4096); + let dictionary = u32::from_le_bytes([d0, d1, d2, d3]).max(4096).min(window_limit); + let [d0, d1, d2, d3] = dictionary.to_le_bytes(); + let properties = [property, d0, d1, d2, d3]; + let size = if has_end_marker { None } else { Some(expected_size) }; + let options = lzma_rs::decompress::Options { + unpacked_size: lzma_rs::decompress::UnpackedSize::UseProvided(size), + memlimit: Some(dictionary as usize), + ..Default::default() + }; + let mut out = CapWrite::new(cap); + let mut cursor = io::Cursor::new(properties).chain(io::Cursor::new(stream)); + match lzma_rs::lzma_decompress_with_options(&mut cursor, &mut out, &options) { + Ok(()) if cursor.get_ref().1.position() == stream.len() as u64 => Ok(out.buf), + Ok(()) => Err(DecompressError::Corrupt), + Err(_) if out.exceeded => Err(DecompressError::TooLarge), + Err(lzma_rs::error::Error::HeaderTooShort(_)) => Err(DecompressError::Truncated), + Err(_) => Err(DecompressError::Corrupt), + } +} + +#[cfg(feature = "bzip2")] +/// Decompress a bzip2 stream, producing at most `max_out` bytes. +pub(crate) fn bzip2(input: &[u8], max_out: u64) -> Result, DecompressError> { + read_capped(bzip2_rs::DecoderReader::new(input), cap_of(max_out)) +} + +#[cfg(feature = "zstd")] +/// Decompress a Zstandard stream, producing at most `max_out` bytes. +pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, DecompressError> { + if input.is_empty() { + return Err(DecompressError::Corrupt); + } + let mut out = CapWrite::new(cap_of(max_out)); + while !input.is_empty() { + if matches!(input.get(..4), Some([0x50..=0x5f, 0x2a, 0x4d, 0x18])) { + let header = input.get(4..8).and_then(|s| s.try_into().ok()).ok_or(DecompressError::Truncated)?; + let len = u32::from_le_bytes(header) as usize; + input = input.get(8..).and_then(|body| body.get(len..)).ok_or(DecompressError::Truncated)?; + continue; + } + let mut decoder = ruzstd::decoding::StreamingDecoder::new(&mut input).map_err(|_| DecompressError::Corrupt)?; + if io::copy(&mut decoder, &mut out).is_err() { + return Err(if out.exceeded { DecompressError::TooLarge } else { DecompressError::Corrupt }); + } + } + Ok(out.buf) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testkit as fx; + + #[cfg(feature = "lzma")] + #[test] + fn xz_round_trip() { + let c = fx::XZ_FIXTURE; + assert_eq!(xz(c, 1 << 20).unwrap(), fx::XZ_PLAIN); + // A tiny cap reports TooLarge, never a partial lie. + assert_eq!(xz(c, 4), Err(DecompressError::TooLarge)); + // Truncation never panics. + for len in 0..c.len() { + let _ = xz(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "lzma")] + #[test] + fn xz_concatenated_streams_and_padding_share_one_budget() { + let input = [fx::XZ_FIXTURE, &[0; 4], fx::XZ_FIXTURE, &[0; 8]].concat(); + let expected = [fx::XZ_PLAIN, fx::XZ_PLAIN].concat(); + assert_eq!(xz(&input, expected.len() as u64).unwrap(), expected); + assert_eq!(xz(&input, expected.len() as u64 - 1), Err(DecompressError::TooLarge)); + let mut empty = Vec::new(); + lzma_rs::xz_compress(&mut io::Cursor::new(b""), &mut empty).unwrap(); + let with_empty = [&empty[..], fx::XZ_FIXTURE, &empty[..]].concat(); + assert_eq!(xz(&with_empty, fx::XZ_PLAIN.len() as u64).unwrap(), fx::XZ_PLAIN); + assert!(xz(&empty, 0).unwrap().is_empty()); + } + + #[cfg(feature = "lzma")] + #[test] + fn xz_rejects_incomplete_members_and_nonstandard_padding() { + for padding in [1, 2, 3, 5] { + let input = [fx::XZ_FIXTURE, &vec![0; padding]].concat(); + assert!(xz(&input, 1000).is_err()); + } + assert!(xz(&[&[0; 4][..], fx::XZ_FIXTURE].concat(), 1000).is_err()); + assert!(xz(&[fx::XZ_FIXTURE, b"junk"].concat(), 1000).is_err()); + for missing in 1..fx::XZ_FIXTURE.len() { + let input = [fx::XZ_FIXTURE, &fx::XZ_FIXTURE[..fx::XZ_FIXTURE.len() - missing]].concat(); + assert!(xz(&input, 1000).is_err(), "missing {missing} bytes"); + } + for at in [fx::XZ_FIXTURE.len() - 13, fx::XZ_FIXTURE.len() - 12] { + let mut damaged = fx::XZ_FIXTURE.to_vec(); + damaged[at] ^= 1; + assert!(xz(&damaged, 1000).is_err()); + } + } + + #[cfg(feature = "lzma")] + #[test] + fn xz_index_sizes_and_integer_encodings_are_checked() { + assert_eq!(xz_vli(&mut &[0x80, 0][..]), Err(DecompressError::Corrupt)); + assert_eq!(xz_vli(&mut &[0xff; 10][..]), Err(DecompressError::Corrupt)); + assert_eq!(xz_vli(&mut &[0x81, 1][..]), Ok(129)); + let mut damaged = fx::XZ_FIXTURE.to_vec(); + let footer = damaged.len() - 12; + let backward = u32::from_le_bytes(damaged[footer + 4..footer + 8].try_into().unwrap()); + let index = footer - (backward as usize + 1) * 4; + damaged[index + 2] = 0; + let checksum = crate::crc32(&damaged[index..footer - 4]); + damaged[footer - 4..footer].copy_from_slice(&checksum.to_le_bytes()); + assert!(xz(&damaged, 1000).is_err()); + } + + #[cfg(feature = "bzip2")] + #[test] + fn bzip2_round_trip() { + let c = fx::BZIP2_FIXTURE; + assert_eq!(bzip2(c, 1 << 20).unwrap(), fx::BZIP2_PLAIN); + assert_eq!(bzip2(c, 2), Err(DecompressError::TooLarge)); + for len in 0..c.len() { + let _ = bzip2(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "zstd")] + #[test] + fn zstd_round_trip() { + let c = fx::ZSTD_FIXTURE; + assert_eq!(zstd(c, 1 << 20).unwrap(), fx::ZSTD_PLAIN); + assert_eq!(zstd(c, 2), Err(DecompressError::TooLarge)); + for len in 0..c.len() { + let _ = zstd(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "zstd")] + #[test] + fn zstandard_members_can_contain_multiple_frames() { + let frame = fx::ZSTD_FIXTURE; + let plain = fx::ZSTD_PLAIN; + let skip = [0x50, 0x2a, 0x4d, 0x18, 3, 0, 0, 0, 1, 2, 3]; + let input = [frame, &skip, frame].concat(); + let expected = [plain, plain].concat(); + assert_eq!(zstd(&input, expected.len() as u64).unwrap(), expected); + assert_eq!(zstd(&input, plain.len() as u64), Err(DecompressError::TooLarge)); + assert!(zstd(&[frame, &skip[..9]].concat(), u64::MAX).is_err()); + assert!(zstd(&[frame, &frame[..5]].concat(), u64::MAX).is_err()); + } + + #[test] + fn garbage_never_panics() { + for seed in 0..64u64 { + let n = fx::noise(seed, 400); + #[cfg(feature = "lzma")] + let _ = xz(&n, 1 << 20); + #[cfg(feature = "bzip2")] + let _ = bzip2(&n, 1 << 20); + #[cfg(feature = "zstd")] + let _ = zstd(&n, 1 << 20); + let _ = &n; + } + } +} diff --git a/crates/unzip/src/file.rs b/crates/unzip/src/file.rs new file mode 100644 index 000000000..1bb4a3662 --- /dev/null +++ b/crates/unzip/src/file.rs @@ -0,0 +1,262 @@ +//! One entry read straight from a ZIP file without loading the file: only its end records, its +//! central directory and that entry's bytes are read. For archives too big to hold in memory just +//! to look at one small entry (a JAR's manifest). + +use crate::bytes::Reader; +use crate::{self as zip, ZipError}; +use std::io::{Read, Seek, SeekFrom}; + +/// Most central directory read (some hundreds of thousands of entries); a bigger one is refused. +const MAX_DIRECTORY: u64 = 16 << 20; +/// The ZIP64 end record as `zip::parse` reads it, without any extensible data after it. +const ZIP64_RECORD: u64 = 56; + +/// The entry named `name` (not a folder), decompressed to at most `max_bytes` and checked against +/// its CRC-32. `None`: the archive has no such entry. +pub fn read_named_entry(file: &mut (impl Read + Seek), name: &str, max_bytes: u64) -> Result>, ZipError> { + let len = file.seek(SeekFrom::End(0)).map_err(io)?; + let tail_start = len.saturating_sub(zip::EOCD_SEARCH as u64); + let tail = read_at(file, tail_start, len - tail_start)?; + let eocd = zip::find_eocd(&tail).ok_or(ZipError::NotZip)?; + let mut end = tail.get(eocd..).ok_or(ZipError::Truncated("end record"))?.to_vec(); + let r = Reader::new(&end); + let field = |at| r.u16_le(at).ok_or(ZipError::Truncated("end record")); + let (size, classic_offset) = (r.u32_le(12).ok_or(ZipError::Truncated("end record"))?, r.u32_le(16).ok_or(ZipError::Truncated("end record"))?); + // Any of these saturated makes `zip::parse` read the ZIP64 records, as below. + let saturated = [field(4)?, field(6)?, field(8)?, field(10)?].contains(&0xffff) || size == 0xffff_ffff || classic_offset == 0xffff_ffff; + let end_at = tail_start + eocd as u64; + // A ZIP64 archive puts its own end record and a locator between the directory and the classic + // end record, whether or not the classic fields are saturated. + let locator_at = end_at.checked_sub(20); + let locator = match locator_at { + Some(at) => read_at(file, at, 20)?, + None => Vec::new(), + }; + // Its stated offset doesn't count a prepended stub; it then sits right before the locator. A + // locator signature with no record at either place is other data (an entry's comment). + let record_at = match (locator_at, locator.starts_with(b"PK\x06\x07")) { + (Some(at), true) => { + let declared = Reader::new(&locator).u64_le(8).ok_or(ZipError::Truncated("zip64 locator"))?; + [declared, at.saturating_sub(ZIP64_RECORD)].into_iter().find(|&at| read_at(file, at, 4).is_ok_and(|sig| sig == b"PK\x06\x06")) + } + _ => None, + }; + let zip64 = match record_at { + Some(record_at) => { + let record = read_at(file, record_at, ZIP64_RECORD)?; + let rr = Reader::new(&record); + let size = rr.u64_le(40).ok_or(ZipError::Truncated("zip64 record"))?; + let offset = rr.u64_le(48).ok_or(ZipError::Truncated("zip64 record"))?; + Some((record_at, size, offset, record)) + } + None => None, + }; + // Unsaturated classic fields must agree with a real ZIP64 record, as `zip::parse` requires; + // one that disagrees is other data shaped like it. + let zip64 = zip64.filter(|&(_, z_size, z_offset, _)| saturated || (z_size == u64::from(size) && z_offset == u64::from(classic_offset))); + let (directory_end, size, offset, zip64) = match zip64 { + Some((record_at, size, offset, record)) => (record_at, size, offset, saturated.then_some((record, locator))), + None => (end_at, u64::from(size), u64::from(classic_offset), None), + }; + if size > MAX_DIRECTORY { + return Err(ZipError::TooLarge(MAX_DIRECTORY)); + } + // The directory ends where the end records start; anything before its stated offset is a + // prepended stub, as `zip::parse` allows. + let start = directory_end.checked_sub(size).ok_or(ZipError::Corrupt("directory starts before the file".into()))?; + let prefix = start.checked_sub(offset).ok_or(ZipError::Corrupt("directory sits before its stated offset".into()))?; + // The directory and its end records alone, their offsets rewritten for that buffer. The + // ZIP64 records go along only when the classic fields need them, as `zip::parse` reads them. + let mut directory = read_at(file, start, size)?; + if let Some((mut record, mut locator)) = zip64 { + put(&mut record, 4, &(ZIP64_RECORD - 12).to_le_bytes())?; + put(&mut record, 48, &0u64.to_le_bytes())?; + put(&mut locator, 8, &size.to_le_bytes())?; + directory.extend_from_slice(&record); + directory.extend_from_slice(&locator); + } + // A saturated classic offset stays so: `zip::parse` then takes the rewritten ZIP64 one. + if classic_offset != 0xffff_ffff { + put(&mut end, 16, &[0; 4])?; + } + directory.extend_from_slice(&end); + // `MAX_DIRECTORY` already bounds how many entries there can be. + let archive = zip::parse(&directory, usize::MAX)?; + let Some(mut entry) = archive.entries.into_iter().find(|e| e.name == name && !e.is_dir) else { return Ok(None) }; + if entry.compressed_size > max_bytes.saturating_add(1 << 16) { + return Err(ZipError::TooLarge(max_bytes)); + } + let at = entry.local_header_offset.checked_add(prefix).ok_or(ZipError::Corrupt("local header offset".into()))?; + let header = read_at(file, at, 30)?; + let fields = Reader::new(&header); + let name_len = u64::from(fields.u16_le(26).ok_or(ZipError::Truncated("local header"))?); + let extra_len = u64::from(fields.u16_le(28).ok_or(ZipError::Truncated("local header"))?); + let local_len = (30 + name_len + extra_len).checked_add(entry.compressed_size).ok_or(ZipError::Corrupt("entry size".into()))?; + let local = read_at(file, at, local_len)?; + entry.local_header_offset = 0; + zip::read_entry(&local, &entry, max_bytes).map(Some) +} + +/// Overwrite `bytes` at `at` in a record read from the file. +fn put(record: &mut [u8], at: usize, bytes: &[u8]) -> Result<(), ZipError> { + record.get_mut(at..at + bytes.len()).ok_or(ZipError::Truncated("end record"))?.copy_from_slice(bytes); + Ok(()) +} + +/// Exactly `len` bytes from `at`; a file that ends sooner is truncated. +fn read_at(file: &mut (impl Read + Seek), at: u64, len: u64) -> Result, ZipError> { + file.seek(SeekFrom::Start(at)).map_err(io)?; + let mut out = Vec::new(); + file.take(len).read_to_end(&mut out).map_err(io)?; + if (out.len() as u64) < len { + return Err(ZipError::Truncated("file")); + } + Ok(out) +} + +fn io(e: std::io::Error) -> ZipError { + ZipError::Corrupt(format!("couldn't read it: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + const MANIFEST: &[u8] = b"Manifest-Version: 1.0\r\nMain-Class: app.Main\r\n\r\n"; + + fn read(bytes: &[u8], name: &str) -> Result>, ZipError> { + read_named_entry(&mut Cursor::new(bytes.to_vec()), name, 1 << 20) + } + + /// The same archive with a ZIP64 end record and locator before its classic end record, whose + /// fields at these offsets are saturated (entry counts and disk numbers 2 bytes, others 4). + fn as_zip64(archive: &[u8], saturate: &[usize]) -> Vec { + let end = archive.len() - 22; + let r = Reader::new(archive); + let (entries, size, offset) = + (u64::from(r.u16_le(end + 10).unwrap()), u64::from(r.u32_le(end + 12).unwrap()), u64::from(r.u32_le(end + 16).unwrap())); + let mut out = archive[..end].to_vec(); + let record_at = out.len() as u64; + out.extend_from_slice(b"PK\x06\x06"); + out.extend_from_slice(&44u64.to_le_bytes()); + out.extend_from_slice(&[45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + for v in [entries, entries, size, offset] { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(b"PK\x06\x07\0\0\0\0"); + out.extend_from_slice(&record_at.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + let mut classic = archive[end..].to_vec(); + for &at in saturate { + let width = if at < 12 { 2 } else { 4 }; + classic[at..at + width].fill(0xff); + } + out.extend_from_slice(&classic); + out + } + + #[test] + fn an_entry_is_read_from_the_directory_without_the_whole_file() { + let filler = vec![7u8; 200_000]; + let archive = crate::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + assert_eq!(read(&archive, "META-INF/MISSING").unwrap(), None); + assert_eq!(read(&archive, "META-INF/").unwrap(), None, "a folder isn't an entry to read"); + + let mut stubbed = b"#!/bin/sh\nexec java -jar \"$0\" \"$@\"\n".to_vec(); + stubbed.extend_from_slice(&archive); + assert_eq!(read(&stubbed, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "a launcher stub before the archive"); + + assert_eq!(read(b"not a zip", "META-INF/MANIFEST.MF"), Err(ZipError::NotZip)); + assert!(read(&archive[..archive.len() - 30], "META-INF/MANIFEST.MF").is_err(), "a cut-off file"); + assert_eq!(read_named_entry(&mut Cursor::new(archive), "app/Big.class", 1000), Err(ZipError::TooLarge(1000))); + } + + #[test] + fn a_directory_bigger_than_the_searched_tail_is_read() { + let names: Vec = (0..3000).map(|i| format!("com/example/generated/Class{i:05}.class")).collect(); + let mut files: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b"x"[..])).collect(); + files.push(("META-INF/MANIFEST.MF", MANIFEST)); + let archive = crate::testkit::zip_stored(&files); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// A one-entry archive whose entry is deflated, as `jar` writes a manifest. + fn zip_deflated(name: &str, body: &[u8]) -> Vec { + let packed = miniz_oxide::deflate::compress_to_vec(body, 6); + let fixed = |out: &mut Vec| { + out.extend_from_slice(&[20, 0, 0, 0, 8, 0, 0, 0, 0, 0]); + out.extend_from_slice(&zip::crc32(body).to_le_bytes()); + out.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&[0, 0]); + }; + let mut out = b"PK\x03\x04".to_vec(); + fixed(&mut out); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&packed); + let directory_at = out.len() as u32; + out.extend_from_slice(b"PK\x01\x02\x14\0"); + fixed(&mut out); + out.extend_from_slice(&[0; 14]); + out.extend_from_slice(name.as_bytes()); + let directory_len = out.len() as u32 - directory_at; + out.extend_from_slice(b"PK\x05\x06\0\0\0\0\x01\0\x01\0"); + out.extend_from_slice(&directory_len.to_le_bytes()); + out.extend_from_slice(&directory_at.to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out + } + + #[test] + fn a_deflated_entry_and_a_comment_that_looks_like_an_end_record() { + let deflated = zip_deflated("META-INF/MANIFEST.MF", MANIFEST); + assert_eq!(read(&deflated, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + + let mut commented = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let comment = b"built by PK\x05\x06 tools"; + let at = commented.len() - 2; + commented[at..].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + commented.extend_from_slice(comment); + assert_eq!(read(&commented, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// The last entry's comment ends like a ZIP64 record and locator, saying the directory is huge; + /// the classic fields disagree with it and win, as they do for `zip::parse`. + #[test] + fn a_comment_shaped_like_zip64_records_is_not_read_as_them() { + let mut archive = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let end = archive.len() - 22; + let last = archive.windows(4).rposition(|w| w == b"PK\x01\x02").unwrap(); + let mut fake = b"PK\x06\x06".to_vec(); + fake.extend_from_slice(&44u64.to_le_bytes()); + fake.extend_from_slice(&[0; 28]); + fake.extend_from_slice(&(MAX_DIRECTORY + 1).to_le_bytes()); + fake.extend_from_slice(&0u64.to_le_bytes()); + fake.extend_from_slice(b"PK\x06\x07\0\0\0\0"); + fake.extend_from_slice(&0u64.to_le_bytes()); + fake.extend_from_slice(&1u32.to_le_bytes()); + archive[last + 32..last + 34].copy_from_slice(&(fake.len() as u16).to_le_bytes()); + let size = Reader::new(&archive).u32_le(end + 12).unwrap() + fake.len() as u32; + archive[end + 12..end + 16].copy_from_slice(&size.to_le_bytes()); + archive.splice(end..end, fake); + assert!(zip::parse(&archive, 10).is_ok(), "precondition: a valid archive"); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// ZIP64 end records are read with any classic fields saturated or none (`zip -fz` saturates + /// only the offset), after a stub too. + #[test] + fn zip64_archives_are_read_in_parts() { + let archive = crate::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); + for saturate in [&[][..], &[8, 10, 12, 16], &[16], &[12], &[10], &[4]] { + let zip64 = as_zip64(&archive, saturate); + assert_eq!(read(&zip64, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}"); + let mut stubbed = b"#!/bin/sh\n".to_vec(); + stubbed.extend_from_slice(&zip64); + assert_eq!(read(&stubbed, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}, after a stub"); + } + } +} diff --git a/crates/unzip/src/lib.rs b/crates/unzip/src/lib.rs new file mode 100644 index 000000000..35dfbbfcf --- /dev/null +++ b/crates/unzip/src/lib.rs @@ -0,0 +1,549 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…): central directory listing and bounded entry reads. +//! +//! Only the central directory is trusted for the listing. Entry data is located through the local +//! header, decompressed with a hard output cap, and checked against the recorded CRC-32. Stored, +//! DEFLATE, bzip2, ZIP LZMA, XZ and Zstandard entries are supported; other methods and encrypted entries are listed. +//! +//! The bzip2, LZMA/XZ and Zstandard methods sit behind the default `bzip2`, `lzma` and `zstd` +//! features; without one, entries using it are listed and reading them is +//! [`ZipError::Unsupported`]. [`file::read_named_entry`] reads one entry from a seekable file +//! without loading the archive. With the `serde` feature the listing types are `Serialize`. +//! +//! From Craft Anything (`crates/formats/src/zip.rs` and `zip_file.rs`), with its hardening: +//! directory and extra-field bounds, ZIP64 end-record recovery after a prepended stub, refusal of +//! multi-volume archives, Info-ZIP Unicode Path names checked by CRC, decoded sizes checked +//! against the directory, and CRC-32 verification. + +mod bytes; +#[cfg(any(feature = "bzip2", feature = "lzma", feature = "zstd"))] +mod decode; +pub mod file; + +use crate::bytes::Reader; + +/// Biggest tail we search for the end-of-central-directory record (64 KiB comment + record). +pub const EOCD_SEARCH: usize = 22 + 0xffff; +const ZIP64_FIXED_SIZE: usize = 56; +/// Bound the search for a record shifted by a prepended stub, including extensible data. +const ZIP64_RECOVERY_SEARCH: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ZipError { + #[error("not a ZIP archive (no end-of-central-directory record)")] + NotZip, + #[error("ZIP is truncated: {0}")] + Truncated(&'static str), + #[error("ZIP is corrupt: {0}")] + Corrupt(String), + #[error("unsupported ZIP feature: {0}")] + Unsupported(String), + #[error("entry is larger than the {0}-byte limit")] + TooLarge(u64), + #[error("CRC-32 mismatch (expected {expected:08x}, found {found:08x})")] + CrcMismatch { expected: u32, found: u32 }, +} + +/// One central-directory entry. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct ZipEntry { + /// Entry name as stored (forward slashes). Bit 11 selects lossy UTF-8; otherwise + /// a valid Unicode Path extra field takes precedence over the UTF-8/CP437 fallback. + pub name: String, + pub method: u16, + pub flags: u16, + pub crc32: u32, + pub compressed_size: u64, + pub uncompressed_size: u64, + /// Offset of the local header, already adjusted for any prepended data. + pub local_header_offset: u64, + pub is_dir: bool, + pub encrypted: bool, + /// Unix mode from the external attributes (when stored by Unix or macOS). + #[cfg_attr(feature = "serde", serde(skip))] + pub unix_mode: Option, +} + +impl ZipEntry { + /// Why this entry is not a regular file or directory (symlink, device, FIFO, socket), if so. + pub fn special_kind(&self) -> Option<&'static str> { + match self.unix_mode? & 0o170000 { + 0o120000 => Some("symlink"), + 0o020000 => Some("character device"), + 0o060000 => Some("block device"), + 0o010000 => Some("FIFO"), + 0o140000 => Some("socket"), + _ => None, + } + } + + /// Human name of the compression method. + pub fn method_name(&self) -> &'static str { + match self.method { + 0 => "stored", + 8 => "deflate", + 9 => "deflate64", + 12 => "bzip2", + 14 => "lzma", + 93 => "zstd", + 95 => "xz", + 99 => "aes", + _ => "other", + } + } +} + +/// The parsed central directory. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct ZipArchive { + pub entries: Vec, + /// Entries the directory declares (may exceed `entries.len()` when truncated). + pub declared_entries: u64, + /// True when `max_entries` stopped the listing early. + pub truncated: bool, + pub zip64: bool, + /// Bytes before the archive proper (self-extracting stubs). + pub prefix_bytes: u64, + /// Inconsistent end-record metadata retained without hiding a readable directory. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Vec::is_empty"))] + pub warnings: Vec, +} + +/// True when the bytes start like a ZIP (local header or empty archive). +pub fn looks_like_zip(data: &[u8]) -> bool { + data.starts_with(b"PK\x03\x04") || data.starts_with(b"PK\x05\x06") || data.starts_with(b"PK\x07\x08") +} + +/// Recover a shifted end record only when its fixed fields still describe the classic +/// directory. Longer records must frame their extension exactly; the adjacent fixed +/// record can still report an inconsistent size through the normal warning path. +fn recover_zip64_record(data: &[u8], loc: usize, disk_entries: u64, total: u64, cd_size: u64, cd_offset: u64) -> Option { + let r = Reader::new(data); + let last = loc.checked_sub(ZIP64_FIXED_SIZE)?; + let first = loc.saturating_sub(ZIP64_RECOVERY_SEARCH); + let candidates = data.get(first..last.checked_add(4)?)?; + candidates.windows(4).enumerate().rev().find_map(|(relative, signature)| { + if signature != b"PK\x06\x06" { + return None; + } + let rec = first.checked_add(relative)?; + let fixed = Reader::new(r.slice(rec, ZIP64_FIXED_SIZE)?); + let payload_size = fixed.u64_le(4)?; + if payload_size < 44 { + return None; + } + let declared_end = (rec as u64).checked_add(12)?.checked_add(payload_size)?; + if rec != last && declared_end != loc as u64 { + return None; + } + let record_disk_entries = fixed.u64_le(24)?; + let record_total = fixed.u64_le(32)?; + let record_cd_size = fixed.u64_le(40)?; + let record_cd_offset = fixed.u64_le(48)?; + if fixed.u32_le(16)? != 0 + || fixed.u32_le(20)? != 0 + || record_disk_entries != record_total + || (disk_entries != 0xffff && disk_entries != record_disk_entries) + || (total != 0xffff && total != record_total) + || (cd_size != 0xffff_ffff && cd_size != record_cd_size) + || (cd_offset != 0xffff_ffff && cd_offset != record_cd_offset) + || record_cd_offset.checked_add(record_cd_size)? > rec as u64 + { + return None; + } + if record_total > 0 { + let directory = rec.checked_sub(usize::try_from(record_cd_size).ok()?)?; + if record_cd_size < 46 || r.slice(directory, 4) != Some(b"PK\x01\x02") { + return None; + } + } + Some(rec) + }) +} + +/// Parse the central directory, keeping at most `max_entries`. +pub fn parse(data: &[u8], max_entries: usize) -> Result { + parse_directory(data, max_entries, None) +} + +/// Search the validated directory while retaining only the requested entry. +pub fn has_entry(data: &[u8], name: &str) -> bool { + parse_directory(data, 1, Some(name)).is_ok_and(|archive| !archive.entries.is_empty()) +} + +/// List directory entries, optionally retaining only a matching decoded name. +fn parse_directory(data: &[u8], max_entries: usize, name_filter: Option<&str>) -> Result { + let r = Reader::new(data); + let eocd = find_eocd(data).ok_or(ZipError::NotZip)?; + let mut disk = u32::from(r.u16_le(eocd + 4).ok_or(ZipError::Truncated("end record"))?); + let mut directory_disk = u32::from(r.u16_le(eocd + 6).ok_or(ZipError::Truncated("end record"))?); + let mut disk_entries = u64::from(r.u16_le(eocd + 8).ok_or(ZipError::Truncated("end record"))?); + if !matches!(disk, 0 | 0xffff) || !matches!(directory_disk, 0 | 0xffff) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let mut total = u64::from(r.u16_le(eocd + 10).ok_or(ZipError::Truncated("end record"))?); + let mut cd_size = u64::from(r.u32_le(eocd + 12).ok_or(ZipError::Truncated("end record"))?); + let mut cd_offset = u64::from(r.u32_le(eocd + 16).ok_or(ZipError::Truncated("end record"))?); + let mut zip64 = false; + let mut warnings = Vec::new(); + let mut eocd_record_start = eocd as u64; + if total == 0xffff || disk_entries == 0xffff || disk == 0xffff || directory_disk == 0xffff || cd_size == 0xffff_ffff || cd_offset == 0xffff_ffff { + // ZIP64 locator sits right before the classic record. + if let Some(loc) = eocd.checked_sub(20) + && r.slice(loc, 4) == Some(b"PK\x06\x07") + { + if r.u32_le(loc + 4) != Some(0) || r.u32_le(loc + 16) != Some(1) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let rec = r.u64_le(loc + 8).ok_or(ZipError::Truncated("zip64 locator"))?; + let rec = usize::try_from(rec).ok().filter(|&rec| r.slice(rec, 4) == Some(b"PK\x06\x06")); + let rec = rec.or_else(|| { + let recovered = recover_zip64_record(data, loc, disk_entries, total, cd_size, cd_offset); + if recovered.is_some() { + warnings.push("ZIP64 locator offset does not identify its end record; recovered the record before the locator".into()); + } + recovered + }); + if let Some(rec) = rec { + let payload_size = r.u64_le(rec + 4).ok_or(ZipError::Truncated("zip64 record size"))?; + if payload_size < 44 { + return Err(ZipError::Corrupt("ZIP64 end record is smaller than its fixed fields".into())); + } + let declared_end = (rec as u64) + .checked_add(12) + .and_then(|start| start.checked_add(payload_size)) + .ok_or_else(|| ZipError::Corrupt("ZIP64 end record extent overflows".into()))?; + // Fixed fields must be present before the locator. A bad declared extension + // length cannot make the reader borrow locator or classic end-record bytes. + if rec.checked_add(ZIP64_FIXED_SIZE).is_none_or(|end| end > loc) { + return Err(ZipError::Truncated("zip64 record fixed fields")); + } + if declared_end != loc as u64 { + warnings.push("ZIP64 end record's declared extent does not end at its locator; using bounded fixed fields".into()); + } + disk = r.u32_le(rec + 16).ok_or(ZipError::Truncated("zip64 record"))?; + directory_disk = r.u32_le(rec + 20).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_disk_entries = r.u64_le(rec + 24).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_total = r.u64_le(rec + 32).ok_or(ZipError::Truncated("zip64 record"))?; + if (disk_entries != 0xffff && disk_entries != zip64_disk_entries) || (total != 0xffff && total != zip64_total) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + disk_entries = zip64_disk_entries; + total = zip64_total; + let zip64_cd_size = r.u64_le(rec + 40).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_cd_offset = r.u64_le(rec + 48).ok_or(ZipError::Truncated("zip64 record"))?; + if (cd_size != 0xffff_ffff && cd_size != zip64_cd_size) || (cd_offset != 0xffff_ffff && cd_offset != zip64_cd_offset) { + return Err(ZipError::Corrupt("ZIP64 and classic directory bounds disagree".into())); + } + cd_size = zip64_cd_size; + cd_offset = zip64_cd_offset; + zip64 = true; + eocd_record_start = rec as u64; + } else { + // A prepended stub can invalidate the locator's absolute offset. Keep the + // classic-directory fallback, including its existing prefix adjustment. + warnings.push("ZIP64 locator does not point to an end record; using the classic end record".into()); + } + } + } + if disk != 0 || directory_disk != 0 || disk_entries != total { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + // Self-extracting archives: the directory sits later than it claims by the stub length. + let expected_end = cd_offset.checked_add(cd_size).ok_or_else(|| ZipError::Corrupt("directory bounds overflow".into()))?; + let prefix = eocd_record_start.saturating_sub(expected_end); + let start = usize::try_from(cd_offset.saturating_add(prefix)).map_err(|_| ZipError::Corrupt("directory offset".into()))?; + let directory_end = (start as u64) + .checked_add(cd_size) + .filter(|&end| end <= eocd_record_start) + .and_then(|end| usize::try_from(end).ok()) + .ok_or_else(|| ZipError::Corrupt("directory extends into its end record".into()))?; + let mut entries = Vec::new(); + let mut pos = start; + let mut truncated = false; + let mut seen = 0u64; + while seen < total { + if entries.len() >= max_entries { + truncated = true; + break; + } + if pos.checked_add(46).is_none_or(|end| end > directory_end) { + return Err(ZipError::Truncated("directory entry")); + } + if r.slice(pos, 4) != Some(b"PK\x01\x02") { + if entries.is_empty() { + return Err(ZipError::Corrupt(format!("no central directory header at offset {pos}"))); + } + truncated = true; + break; + } + let flags = r.u16_le(pos + 8).ok_or(ZipError::Truncated("directory entry"))?; + let method = r.u16_le(pos + 10).ok_or(ZipError::Truncated("directory entry"))?; + let crc32 = r.u32_le(pos + 16).ok_or(ZipError::Truncated("directory entry"))?; + let mut csize = u64::from(r.u32_le(pos + 20).ok_or(ZipError::Truncated("directory entry"))?); + let mut usize_ = u64::from(r.u32_le(pos + 24).ok_or(ZipError::Truncated("directory entry"))?); + let name_len = usize::from(r.u16_le(pos + 28).ok_or(ZipError::Truncated("directory entry"))?); + let extra_len = usize::from(r.u16_le(pos + 30).ok_or(ZipError::Truncated("directory entry"))?); + let comment_len = usize::from(r.u16_le(pos + 32).ok_or(ZipError::Truncated("directory entry"))?); + let entry_end = pos + .checked_add(46 + name_len + extra_len + comment_len) + .filter(|&end| end <= directory_end) + .ok_or(ZipError::Truncated("directory entry fields"))?; + let host = r.slice(pos + 5, 1).and_then(|b| b.first().copied()).unwrap_or(0); + let attrs = r.u32_le(pos + 38).ok_or(ZipError::Truncated("entry attributes"))?; + let unix_mode = matches!(host, 3 | 19).then_some(attrs >> 16); + let mut local = u64::from(r.u32_le(pos + 42).ok_or(ZipError::Truncated("directory entry"))?); + let mut entry_disk = u32::from(r.u16_le(pos + 34).ok_or(ZipError::Truncated("directory entry"))?); + let name_bytes = r.slice(pos + 46, name_len).ok_or(ZipError::Truncated("entry name"))?; + let extra = r.slice(pos + 46 + name_len, extra_len).ok_or(ZipError::Truncated("entry extra fields"))?; + // Each extra field owns its payload: ZIP64 values cannot borrow a following field's bytes. + let needs_zip64 = usize_ == 0xffff_ffff || csize == 0xffff_ffff || local == 0xffff_ffff || entry_disk == 0xffff; + let mut found_zip64 = false; + let mut unicode_name = None; + let mut name_crc = None; + let mut x = 0usize; + let er = Reader::new(extra); + // Trailing bytes too short for a field header are padding, not truncation. + while x.saturating_add(4) <= extra.len() { + let id = er.u16_le(x).ok_or(ZipError::Truncated("extra field header"))?; + let len = usize::from(er.u16_le(x + 2).ok_or(ZipError::Truncated("extra field header"))?); + let body = x + 4; + let field = er.slice(body, len).ok_or(ZipError::Truncated("extra field payload"))?; + if id == 0x0001 && !found_zip64 { + let values = Reader::new(field); + let mut f = 0; + if usize_ == 0xffff_ffff { + usize_ = values.u64_le(f).ok_or(ZipError::Truncated("zip64 uncompressed size"))?; + f += 8; + } + if csize == 0xffff_ffff { + csize = values.u64_le(f).ok_or(ZipError::Truncated("zip64 compressed size"))?; + f += 8; + } + if local == 0xffff_ffff { + local = values.u64_le(f).ok_or(ZipError::Truncated("zip64 local header offset"))?; + f += 8; + } + if entry_disk == 0xffff { + entry_disk = values.u32_le(f).ok_or(ZipError::Truncated("zip64 entry disk"))?; + } + found_zip64 = true; + } else if id == 0x7075 && flags & (1 << 11) == 0 && unicode_name.is_none() { + let field = Reader::new(field); + if field.u8(0) == Some(1) && field.u32_le(1) == Some(*name_crc.get_or_insert_with(|| self::crc32(name_bytes))) { + unicode_name = field.data().get(5..).and_then(|bytes| std::str::from_utf8(bytes).ok()).filter(|s| !s.is_empty()); + } + } + x = body + len; + } + if needs_zip64 && !found_zip64 { + return Err(ZipError::Corrupt("missing ZIP64 extended information".into())); + } + if entry_disk != 0 { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let name = unicode_name.map(str::to_owned).unwrap_or_else(|| decode_name(name_bytes, flags)).replace('\\', "/"); + if name_filter.is_none_or(|wanted| wanted == name) { + entries.push(ZipEntry { + is_dir: name.ends_with('/') + || unix_mode.is_some_and(|m| m & 0o170000 == 0o040000) + || (matches!(host, 0 | 10 | 14) && attrs & 0x10 != 0), + name, + method, + flags, + crc32, + compressed_size: csize, + uncompressed_size: usize_, + local_header_offset: local.saturating_add(prefix), + encrypted: flags & 1 != 0, + unix_mode, + }); + } + seen += 1; + pos = entry_end; + } + Ok(ZipArchive { entries, declared_entries: total, truncated, zip64, prefix_bytes: prefix, warnings }) +} + +fn decode_name(bytes: &[u8], flags: u16) -> String { + if flags & 0x0800 != 0 || bytes.is_ascii() { + return String::from_utf8_lossy(bytes).into_owned(); + } + if let Ok(name) = std::str::from_utf8(bytes) { + return name.to_owned(); + } + const CP437: [char; 128] = [ + 'Ç', 'ü', 'é', 'â', 'ä', 'à', 'å', 'ç', 'ê', 'ë', 'è', 'ï', 'î', 'ì', 'Ä', 'Å', 'É', 'æ', 'Æ', 'ô', 'ö', 'ò', 'û', 'ù', 'ÿ', 'Ö', 'Ü', '¢', + '£', '¥', '₧', 'ƒ', 'á', 'í', 'ó', 'ú', 'ñ', 'Ñ', 'ª', 'º', '¿', '⌐', '¬', '½', '¼', '¡', '«', '»', '░', '▒', '▓', '│', '┤', '╡', '╢', '╖', + '╕', '╣', '║', '╗', '╝', '╜', '╛', '┐', '└', '┴', '┬', '├', '─', '┼', '╞', '╟', '╚', '╔', '╩', '╦', '╠', '═', '╬', '╧', '╨', '╤', '╥', '╙', + '╘', '╒', '╓', '╫', '╪', '┘', '┌', '█', '▄', '▌', '▐', '▀', 'α', 'ß', 'Γ', 'π', 'Σ', 'σ', 'µ', 'τ', 'Φ', 'Θ', 'Ω', 'δ', '∞', 'φ', 'ε', '∩', + '≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∙', '·', '√', 'ⁿ', '²', '■', '\u{00a0}', + ]; + bytes + .iter() + .map(|&byte| if byte.is_ascii() { char::from(byte) } else { CP437.get(usize::from(byte - 0x80)).copied().unwrap_or('\u{fffd}') }) + .collect() +} + +/// Where the end-of-central-directory record starts, searching backwards through the last +/// [`EOCD_SEARCH`] bytes. +pub fn find_eocd(data: &[u8]) -> Option { + if data.len() < 22 { + return None; + } + let from = data.len().saturating_sub(EOCD_SEARCH); + let tail = data.get(from..)?; + // Search backwards so a stray signature in the comment can't shadow the real record. + let mut i = tail.len().checked_sub(22)?; + let mut found = None; + let mut last_signature = None; + loop { + if tail.get(i..i + 4) == Some(b"PK\x05\x06") { + last_signature.get_or_insert(from + i); + if let Some(comment_len) = tail.get(i + 20..i + 22).and_then(|bytes| <[u8; 2]>::try_from(bytes).ok()).map(u16::from_le_bytes) + && i + 22 + usize::from(comment_len) == tail.len() + { + found = Some(from + i); + } + } + if i == 0 { + return found.or(last_signature); + } + i -= 1; + } +} + +/// The raw (possibly compressed) bytes of an entry. +pub fn raw_entry_data<'a>(data: &'a [u8], e: &ZipEntry) -> Result<&'a [u8], ZipError> { + let r = Reader::new(data); + let off = usize::try_from(e.local_header_offset).map_err(|_| ZipError::Corrupt("local header offset".into()))?; + if r.slice(off, 4) != Some(b"PK\x03\x04") { + return Err(ZipError::Corrupt(format!("no local header for {}", e.name))); + } + let n = usize::from(r.u16_le(off + 26).ok_or(ZipError::Truncated("local header"))?); + let x = usize::from(r.u16_le(off + 28).ok_or(ZipError::Truncated("local header"))?); + let start = off.checked_add(30 + n + x).ok_or(ZipError::Truncated("local header"))?; + let len = usize::try_from(e.compressed_size).map_err(|_| ZipError::TooLarge(e.compressed_size))?; + r.slice(start, len).ok_or(ZipError::Truncated("entry data")) +} + +/// Read and decompress one entry, refusing to produce more than `max_bytes`. +pub fn read_entry(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let out = read_entry_unverified(data, e, max_bytes)?; + let found = crc32(&out); + if found != e.crc32 { + return Err(ZipError::CrcMismatch { expected: e.crc32, found }); + } + Ok(out) +} + +/// Like [`read_entry`] but without comparing the stored CRC-32, so a caller that accepts a mismatch can still get the bytes. +pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + if e.encrypted { + return Err(ZipError::Unsupported(format!("{} is encrypted", e.name))); + } + if e.uncompressed_size > max_bytes { + return Err(ZipError::TooLarge(max_bytes)); + } + let raw = raw_entry_data(data, e)?; + let cap = usize::try_from(max_bytes).unwrap_or(usize::MAX); + let out = match e.method { + 0 if raw.len() as u64 > max_bytes => return Err(ZipError::TooLarge(max_bytes)), + 0 => raw.to_vec(), + 8 => miniz_oxide::inflate::decompress_to_vec_with_limit(raw, cap).map_err(|err| match err.status { + miniz_oxide::inflate::TINFLStatus::HasMoreOutput => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("deflate stream for {} is damaged", e.name)), + })?, + 12 | 95 => decode_bzip2_or_xz(raw, e, max_bytes)?, + 14 => decode_lzma(raw, e, max_bytes)?, + 93 => decode_zstd(raw, e, max_bytes)?, + m => return Err(ZipError::Unsupported(format!("compression method {m} ({}) for {}", e.method_name(), e.name))), + }; + if out.len() as u64 != e.uncompressed_size { + return Err(ZipError::Corrupt(format!( + "decoded size for {} is {}, but its directory entry declares {}", + e.name, + out.len(), + e.uncompressed_size + ))); + } + Ok(out) +} + +#[cfg(any(feature = "bzip2", feature = "lzma"))] +fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let decoded = match e.method { + #[cfg(feature = "bzip2")] + 12 => decode::bzip2(raw, max_bytes), + #[cfg(feature = "lzma")] + 95 => decode::xz(raw, max_bytes), + _ => return Err(not_built(e)), + }; + decoded.map_err(|error| match error { + decode::DecompressError::TooLarge => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("{} stream for {} is damaged: {error}", e.method_name(), e.name)), + }) +} + +#[cfg(not(any(feature = "bzip2", feature = "lzma")))] +fn decode_bzip2_or_xz(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +#[cfg(feature = "lzma")] +fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let header = Reader::new(raw); + let properties_size = header.u16_le(2).ok_or(ZipError::Truncated("LZMA properties header"))?; + if properties_size != 5 { + return Err(ZipError::Unsupported(format!("LZMA properties size {properties_size} for {}", e.name))); + } + let stream = raw.get(4..).ok_or(ZipError::Truncated("LZMA properties"))?; + decode::lzma_zip(stream, e.uncompressed_size, e.flags & 2 != 0, max_bytes).map_err(|error| match error { + decode::DecompressError::TooLarge => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("LZMA stream for {} is damaged: {error}", e.name)), + }) +} + +#[cfg(not(feature = "lzma"))] +fn decode_lzma(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +#[cfg(feature = "zstd")] +fn decode_zstd(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + decode::zstd(raw, max_bytes).map_err(|err| ZipError::Corrupt(format!("zstd stream for {} is damaged: {err}", e.name))) +} + +#[cfg(not(feature = "zstd"))] +fn decode_zstd(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +/// An entry whose method this build leaves out (see the crate features). +#[cfg_attr(all(feature = "bzip2", feature = "lzma", feature = "zstd"), allow(dead_code))] +fn not_built(e: &ZipEntry) -> ZipError { + ZipError::Unsupported(format!("compression method {} ({}) for {} is not built in", e.method, e.method_name(), e.name)) +} + +/// CRC-32 (IEEE, as used by ZIP and PNG). +pub fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + for &b in data { + crc ^= u32::from(b); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc +} + +#[cfg(test)] +mod testkit; + +#[cfg(test)] +mod tests; diff --git a/crates/unzip/src/testkit.rs b/crates/unzip/src/testkit.rs new file mode 100644 index 000000000..ae55cc052 --- /dev/null +++ b/crates/unzip/src/testkit.rs @@ -0,0 +1,80 @@ +//! Test fixtures from Craft Anything's testkit: known-good compressed streams built with the +//! reference encoders (`xz`, `bzip2`, `zstd`), a stored-only ZIP writer and a noise generator. + +#![allow(clippy::cast_possible_truncation, dead_code)] + +use crate::crc32; + +/// Plaintext of [`XZ_FIXTURE`] and [`LZMA_FIXTURE`]. +pub const XZ_PLAIN: &[u8] = b"Hello, xz compression world!\n"; +/// An xz stream of [`XZ_PLAIN`]. +pub const XZ_FIXTURE: &[u8] = &[ + 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, + 0x01, 0x00, 0x1c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x78, 0x7a, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x5f, 0x47, 0xab, 0x2a, 0x4c, 0xd2, 0xb2, 0x00, 0x01, 0x35, 0x1d, + 0x44, 0x1b, 0xb1, 0xe1, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x04, 0x59, 0x5a, +]; +/// Plaintext of [`BZIP2_FIXTURE`]. +pub const BZIP2_PLAIN: &[u8] = b"Hello, bzip2!\n"; +/// A bzip2 stream of [`BZIP2_PLAIN`]. +pub const BZIP2_FIXTURE: &[u8] = &[ + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0x04, 0xad, 0x5a, 0x6b, 0x00, 0x00, 0x02, 0xdd, 0x80, 0x00, 0x10, 0x60, 0x04, 0x10, + 0x00, 0x00, 0x40, 0x12, 0x24, 0xc0, 0x10, 0x20, 0x00, 0x22, 0x00, 0x03, 0x42, 0x01, 0xa0, 0x05, 0x67, 0xde, 0x89, 0xa1, 0xa0, 0x80, 0xf1, 0x77, + 0x24, 0x53, 0x85, 0x09, 0x00, 0x4a, 0xd5, 0xa6, 0xb0, +]; +/// Plaintext of [`ZSTD_FIXTURE`]. +pub const ZSTD_PLAIN: &[u8] = b"Hello, zstd!\n"; +/// A Zstandard frame of [`ZSTD_PLAIN`]. +pub const ZSTD_FIXTURE: &[u8] = &[ + 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x0d, 0x69, 0x00, 0x00, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x7a, 0x73, 0x74, 0x64, 0x21, 0x0a, 0x41, 0xb4, + 0x53, 0xb2, +]; + +/// A stored-only ZIP of `files`. +pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + let len = body.len() as u32; + out.extend_from_slice(b"PK\x03\x04\x14\0\0\0\0\0\0\0\0\0"); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02\x14\0\x14\0\0\0\0\0\0\0\0\0"); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops. +pub fn noise(seed: u64, len: usize) -> Vec { + let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() +} diff --git a/crates/unzip/src/tests.rs b/crates/unzip/src/tests.rs new file mode 100644 index 000000000..3ea92afe4 --- /dev/null +++ b/crates/unzip/src/tests.rs @@ -0,0 +1,393 @@ +//! The reader's tests, from Craft Anything's `zip.rs`. + +use super::*; + +/// Minimal stored-only ZIP writer for tests. +pub(crate) fn build(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + out.extend_from_slice(b"PK\x03\x04"); + out.extend_from_slice(&[20, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02"); + cd.extend_from_slice(&[20, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&(body.len() as u32).to_le_bytes()); + cd.extend_from_slice(&(body.len() as u32).to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +#[cfg(all(feature = "bzip2", feature = "lzma"))] +fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { + let mut file = build(&[("entry", compressed)]); + let central = 30 + "entry".len() + compressed.len(); + let version: u16 = if method == 12 { 46 } else { 63 }; + file[4..6].copy_from_slice(&version.to_le_bytes()); + file[8..10].copy_from_slice(&method.to_le_bytes()); + file[14..18].copy_from_slice(&crc32(plain).to_le_bytes()); + file[22..26].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + file[central + 6..central + 8].copy_from_slice(&version.to_le_bytes()); + file[central + 10..central + 12].copy_from_slice(&method.to_le_bytes()); + file[central + 16..central + 20].copy_from_slice(&crc32(plain).to_le_bytes()); + file[central + 24..central + 28].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + file +} + +#[cfg(all(feature = "bzip2", feature = "lzma"))] +#[test] +fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { + use crate::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; + for (method, compressed, plain) in [(12, BZIP2_FIXTURE, BZIP2_PLAIN), (95, XZ_FIXTURE, XZ_PLAIN)] { + let file = build_codec_entry(method, compressed, plain); + let original = parse(&file, 1).unwrap().entries.remove(0); + assert_eq!(read_entry(&file, &original, plain.len() as u64).unwrap(), plain); + let mut entry = original.clone(); + entry.crc32 ^= 1; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::CrcMismatch { .. }))); + assert_eq!(read_entry_unverified(&file, &entry, 100).unwrap(), plain); + entry = original.clone(); + entry.uncompressed_size = 0; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::Corrupt(_)))); + assert!(matches!(read_entry(&file, &entry, plain.len() as u64 - 1), Err(ZipError::TooLarge(_)))); + let mut damaged = file.clone(); + damaged[30 + "entry".len()] ^= 0xff; + assert!(matches!(read_entry(&damaged, &original, 100), Err(ZipError::Corrupt(_)))); + entry = original; + entry.encrypted = true; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::Unsupported(_)))); + } +} + +#[cfg(feature = "lzma")] +fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { + let mut encoded = Vec::new(); + let size = if end_marker { None } else { Some(plain.len() as u64) }; + let options = lzma_rs::compress::Options { unpacked_size: lzma_rs::compress::UnpackedSize::WriteToHeader(size) }; + lzma_rs::lzma_compress_with_options(&mut std::io::Cursor::new(plain), &mut encoded, &options).unwrap(); + let mut compressed = vec![9, 4, 5, 0]; + compressed.extend_from_slice(&encoded[..5]); + compressed.extend_from_slice(&encoded[13..]); + let mut file = build(&[("entry", &compressed)]); + let central = 30 + "entry".len() + compressed.len(); + let flags: u16 = if end_marker { 2 } else { 0 }; + for (version_at, flags_at, method_at, crc_at, size_at) in + [(4, 6, 8, 14, 22), (central + 6, central + 8, central + 10, central + 16, central + 24)] + { + file[version_at..version_at + 2].copy_from_slice(&63u16.to_le_bytes()); + file[flags_at..flags_at + 2].copy_from_slice(&flags.to_le_bytes()); + file[method_at..method_at + 2].copy_from_slice(&14u16.to_le_bytes()); + file[crc_at..crc_at + 4].copy_from_slice(&crc32(plain).to_le_bytes()); + file[size_at..size_at + 4].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + } + file +} + +#[cfg(feature = "lzma")] +#[test] +fn zip_lzma_reads_size_delimited_and_end_marked_streams() { + for end_marker in [false, true] { + for plain in [&b"ZIP LZMA contents"[..], &[0; 23][..]] { + let file = zip_lzma_member(plain, end_marker); + let original = parse(&file, 1).unwrap().entries.remove(0); + assert_eq!(read_entry(&file, &original, plain.len() as u64).unwrap(), plain); + assert!(matches!(read_entry(&file, &original, plain.len() as u64 - 1), Err(ZipError::TooLarge(_)))); + let mut entry = original.clone(); + entry.crc32 ^= 1; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::CrcMismatch { .. }))); + entry = original.clone(); + entry.uncompressed_size -= 1; + assert!(read_entry(&file, &entry, 100).is_err()); + entry = original.clone(); + entry.compressed_size -= 1; + assert!(read_entry(&file, &entry, 100).is_err()); + entry = original; + entry.compressed_size += 1; + assert!(read_entry(&file, &entry, 100).is_err()); + } + } +} + +#[cfg(feature = "lzma")] +#[test] +fn zip_lzma_rejects_bad_properties_without_unbounded_dictionaries() { + let mut file = zip_lzma_member(b"small", true); + let entry = parse(&file, 1).unwrap().entries.remove(0); + let body = 30 + "entry".len(); + file[body + 5..body + 9].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(read_entry(&file, &entry, 5).unwrap(), b"small"); + file[body + 4] = 255; + assert!(read_entry(&file, &entry, 5).is_err()); + file[body + 2..body + 4].copy_from_slice(&6u16.to_le_bytes()); + assert!(matches!(read_entry(&file, &entry, 5), Err(ZipError::Unsupported(_)))); +} + +fn build_named_entry(name: &[u8], flags: u16) -> Vec { + let mut zip = build(&[(&"x".repeat(name.len()), b"")]); + let central = 30 + name.len(); + zip[6..8].copy_from_slice(&flags.to_le_bytes()); + zip[30..central].copy_from_slice(name); + zip[central + 8..central + 10].copy_from_slice(&flags.to_le_bytes()); + zip[central + 46..central + 46 + name.len()].copy_from_slice(name); + zip +} + +#[test] +fn verified_unicode_path_overrides_a_legacy_zip_name() { + let raw = b"notes.txt"; + let unicode = "笔记.txt"; + for (flags, version, checksum, expected) in + [(0, 1, crc32(raw), unicode), (0, 1, crc32(b"old-name"), "notes.txt"), (0, 2, crc32(raw), "notes.txt"), (1 << 11, 1, crc32(raw), "notes.txt")] + { + let mut zip = build_named_entry(raw, flags); + let central = 30 + raw.len(); + let end = central + 46 + raw.len(); + let mut extra = 0x7075_u16.to_le_bytes().to_vec(); + extra.extend_from_slice(&((5 + unicode.len()) as u16).to_le_bytes()); + extra.push(version); + extra.extend_from_slice(&checksum.to_le_bytes()); + extra.extend_from_slice(unicode.as_bytes()); + zip[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + zip[end + 12..end + 16].copy_from_slice(&((end - central + extra.len()) as u32).to_le_bytes()); + zip.splice(end..end, extra); + assert_eq!(parse(&zip, 1).unwrap().entries[0].name, expected); + } +} + +#[test] +fn directory_markers_do_not_require_a_trailing_slash() { + let sample = build(&[("folder", b"")]); + let central = 30 + "folder".len(); + for (host, attrs, directory, special) in [ + (0, 0x10u32, true, None), + (10, 0x10, true, None), + (3, 0o040755 << 16, true, None), + (3, 0o120777 << 16, false, Some("symlink")), + (0, 0x20, false, None), + (14, 0x10, true, None), + (14, 0x20, false, None), + (19, 0o040755 << 16, true, None), + (19, 0o100644 << 16, false, None), + (19, 0o120777 << 16, false, Some("symlink")), + (42, 0x10, false, None), + ] { + let mut file = sample.clone(); + file[central + 5] = host; + file[central + 38..central + 42].copy_from_slice(&attrs.to_le_bytes()); + let archive = parse(&file, 1).unwrap(); + let entry = &archive.entries[0]; + assert_eq!((entry.is_dir, entry.special_kind()), (directory, special), "host {host}, attributes {attrs:#x}"); + } +} + +#[test] +fn a_cp437_name_without_the_utf8_flag_is_decoded() { + let zip = build_named_entry(b"caf\x82.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "café.txt"); +} + +#[test] +fn a_cp437_box_drawing_name_without_the_utf8_flag_is_decoded() { + let zip = build_named_entry(b"\xc9.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "╔.txt"); +} + +#[test] +fn a_utf8_name_with_the_utf8_flag_is_preserved() { + let zip = build_named_entry("한글/é.txt".as_bytes(), 0x0800); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "한글/é.txt"); +} + +#[test] +fn a_valid_utf8_name_without_the_utf8_flag_is_preserved() { + let zip = build_named_entry("é.txt".as_bytes(), 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "é.txt"); +} + +#[test] +fn an_ascii_name_is_preserved() { + let zip = build_named_entry(b"dir/file.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "dir/file.txt"); +} + +#[test] +fn valid_entry_survives_eocd_signature_in_comment() { + let mut zip = build(&[("a.txt", b"hello")]); + let comment = b"PK\x05\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + let eocd = zip.len() - 22; + zip[eocd + 20..eocd + 22].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + zip.extend_from_slice(comment); + + assert_eq!(parse(&zip, 10).unwrap().entries.len(), 1); +} + +#[test] +fn valid_entry_survives_bytes_after_eocd_comment() { + let mut zip = build(&[("a.txt", b"hello")]); + zip.extend_from_slice(b"tail"); + + assert_eq!(parse(&zip, 10).unwrap().entries.len(), 1); +} + +#[test] +fn checks_the_zip64_record_extent_including_extension_bytes() { + let mut record = vec![0; 56]; + record[..4].copy_from_slice(b"PK\x06\x06"); + record[4..12].copy_from_slice(&46u64.to_le_bytes()); + record.extend([1, 2]); // extensible data + record.extend(b"PK\x06\x07"); + record.extend([0; 12]); + record.extend(1u32.to_le_bytes()); + record.extend(b"PK\x05\x06"); + record.extend([0; 4]); + record.extend([0xff; 4]); + record.extend([0; 10]); + assert!(parse(&record, 1).unwrap().zip64); + for len in [44u64, 47] { + record[4..12].copy_from_slice(&len.to_le_bytes()); + let archive = parse(&record, 1).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.warnings.len(), 1); + } + for len in [43u64, u64::MAX] { + record[4..12].copy_from_slice(&len.to_le_bytes()); + assert!(parse(&record, 1).is_err(), "{len}"); + } +} + +#[test] +fn lists_and_reads_stored_entries() { + let z = build(&[("a.txt", b"hello"), ("dir/b.js", b"console.log(1)")]); + let a = parse(&z, 100).unwrap(); + assert_eq!(a.entries.len(), 2); + assert_eq!(a.entries[1].name, "dir/b.js"); + assert_eq!(read_entry(&z, &a.entries[0], 100).unwrap(), b"hello"); + assert!(matches!(read_entry(&z, &a.entries[1], 3), Err(ZipError::TooLarge(3)))); +} + +#[cfg(feature = "zstd")] +#[test] +fn zstandard_members_are_read_with_their_limits_and_checks() { + // A raw Zstandard frame holding "hello zip" (single segment, one raw block). + let body = b"hello zip"; + let mut frame = vec![0x28, 0xb5, 0x2f, 0xfd, 0x20, body.len() as u8]; + let block_header = ((body.len() as u32) << 3) | 1; + frame.extend(&block_header.to_le_bytes()[..3]); + frame.extend(body); + let mut file = vec![0u8; 30]; + file[..4].copy_from_slice(b"PK\x03\x04"); + file.extend(&frame); + let mut e = ZipEntry { + name: "x".into(), + method: 93, + flags: 0, + crc32: crc32(body), + compressed_size: frame.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + assert_eq!(read_entry(&file, &e, 100).unwrap(), body); + assert!(matches!(read_entry(&file, &e, 3), Err(ZipError::TooLarge(3)))); + e.crc32 ^= 1; + assert!(matches!(read_entry(&file, &e, 100), Err(ZipError::CrcMismatch { .. }))); +} + +#[test] +fn deflate_round_trip_and_crc_check() { + let body = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".repeat(20); + let comp = miniz_oxide::deflate::compress_to_vec(&body, 6); + let e = ZipEntry { + name: "x".into(), + method: 8, + flags: 0, + crc32: crc32(&body), + compressed_size: comp.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + let mut z = b"PK\x03\x04".to_vec(); + z.extend_from_slice(&[0; 26]); + z.extend_from_slice(&comp); + assert_eq!(read_entry(&z, &e, 1 << 20).unwrap(), body); + // A lying uncompressed size can't make us exceed the cap. + let liar = ZipEntry { uncompressed_size: 10, ..e.clone() }; + assert!(matches!(read_entry(&z, &liar, 10), Err(ZipError::TooLarge(10)))); + let bad = ZipEntry { crc32: 1, ..e }; + assert!(matches!(read_entry(&z, &bad, 1 << 20), Err(ZipError::CrcMismatch { .. }))); +} + +#[test] +fn hostile_inputs_do_not_panic() { + let z = build(&[("a", b"1"), ("b", b"22")]); + for cut in 0..z.len() { + if let Ok(a) = parse(&z[..cut], 10) { + for e in &a.entries { + let _ = read_entry(&z[..cut], e, 100); + } + } + } + for i in 0..z.len() { + let mut m = z.clone(); + m[i] ^= 0xff; + if let Ok(a) = parse(&m, 10) { + for e in &a.entries { + let _ = read_entry(&m, e, 100); + } + } + } + assert_eq!(parse(b"", 1), Err(ZipError::NotZip)); +} + +#[test] +fn crc_known_value() { + assert_eq!(crc32(b"123456789"), 0xcbf4_3926); +} + +#[cfg(feature = "serde")] +#[test] +fn listings_serialize_without_empty_warnings_or_unix_mode() { + let data = crate::testkit::zip_stored(&[("a.txt", b"hi")]); + let archive = parse(&data, 10).unwrap(); + let value = serde_json::to_value(&archive).unwrap(); + assert!(value.get("warnings").is_none()); + assert_eq!(value["entries"][0]["name"], "a.txt"); + assert!(value["entries"][0].get("unix_mode").is_none()); +} + +#[cfg(not(feature = "zstd"))] +#[test] +fn a_method_left_out_of_the_build_is_unsupported() { + let mut data = crate::testkit::zip_stored(&[("a", b"abc")]); + let archive = parse(&data, 10).unwrap(); + let mut entry = archive.entries[0].clone(); + entry.method = 93; + data[8..10].copy_from_slice(&93u16.to_le_bytes()); + assert!(matches!(read_entry_unverified(&data, &entry, 100), Err(ZipError::Unsupported(m)) if m.contains("not built in"))); +} diff --git a/crates/unzip/tests/common/mod.rs b/crates/unzip/tests/common/mod.rs new file mode 100644 index 000000000..ba0f3d778 --- /dev/null +++ b/crates/unzip/tests/common/mod.rs @@ -0,0 +1,41 @@ +//! The stored-only ZIP writer the integration tests share, as in Craft Anything's testkit. + +#![allow(dead_code, clippy::cast_possible_truncation)] + +use craft_unzip::crc32; + +/// A stored-only ZIP of `files`. +pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + let len = body.len() as u32; + out.extend_from_slice(b"PK\x03\x04\x14\0\0\0\0\0\0\0\0\0"); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02\x14\0\x14\0\0\0\0\0\0\0\0\0"); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} diff --git a/crates/unzip/tests/zip64_end_record_compatibility.rs b/crates/unzip/tests/zip64_end_record_compatibility.rs new file mode 100644 index 000000000..37f63cf76 --- /dev/null +++ b/crates/unzip/tests/zip64_end_record_compatibility.rs @@ -0,0 +1,169 @@ +// From Craft Anything crates/formats/tests/zip64_end_record_compatibility.rs. +use craft_unzip::{self as zip, ZipArchive, ZipError}; +mod common; +use common::zip_stored; + +fn warning_contains(archive: &ZipArchive, text: &str) -> bool { + archive.warnings.iter().any(|warning| warning.contains(text)) +} + +fn zip64_with_extension(extension: &[u8]) -> (Vec, usize) { + let mut data = zip_stored(&[("a.txt", b"retained payload")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let offset = u32::from_le_bytes(data[end + 16..end + 20].try_into().unwrap()); + let mut record = b"PK\x06\x06".to_vec(); + record.extend((44 + extension.len() as u64).to_le_bytes()); + record.extend([45, 0, 45, 0]); + record.extend([0; 8]); + record.extend(1u64.to_le_bytes()); + record.extend(1u64.to_le_bytes()); + record.extend(u64::from(size).to_le_bytes()); + record.extend(u64::from(offset).to_le_bytes()); + record.extend(extension); + record.extend(b"PK\x06\x07"); + record.extend(0u32.to_le_bytes()); + record.extend((end as u64).to_le_bytes()); + record.extend(1u32.to_le_bytes()); + data[end + 8..end + 12].fill(0xff); + data.splice(end..end, record); + (data, end) +} + +#[test] +fn readable_fixed_fields_survive_inconsistent_declared_extents() { + let (original, record) = zip64_with_extension(&[1, 2, 3, 4]); + for declared in [44u64, 47, 49, 4096] { + let mut data = original.clone(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.declared_entries, 1); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.prefix_bytes, 0); + assert!(warning_contains(&archive, "extent"), "declared payload {declared}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn correctly_sized_extensions_need_no_warning() { + for extension in [&[][..], &[1, 2, 3, 4][..]] { + let (data, _) = zip64_with_extension(extension); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert!(archive.warnings.is_empty()); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn too_small_and_overflowing_record_extents_remain_untrusted() { + let (original, record) = zip64_with_extension(&[]); + for declared in [0u64, 43, u64::MAX - 12, u64::MAX] { + let mut data = original.clone(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Corrupt(_))), "declared payload {declared}"); + } +} + +#[test] +fn fixed_fields_cannot_borrow_bytes_from_the_locator() { + let (mut data, record) = zip64_with_extension(&[]); + data.drain(record + 48..record + 56); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn prepended_stubs_keep_real_zip64_directories_and_entry_offsets() { + for extension in [&[][..], &[1, 2, 3, 4][..], &[0; 1024][..]] { + let (original, _) = zip64_with_extension(extension); + for prefix in [&b"MZ"[..], &b"MZ synthetic self-extracting prefix"[..]] { + for saturated_bounds in [false, true] { + let mut data = prefix.to_vec(); + data.extend(&original); + if saturated_bounds { + let end = data.len() - 22; + data[end + 12..end + 20].fill(0xff); + } + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.declared_entries, 1); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert_eq!(archive.entries[0].local_header_offset, prefix.len() as u64); + assert!(warning_contains(&archive, "recovered")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } + } + } +} + +#[test] +fn shifted_fixed_records_keep_extent_warnings() { + let (original, record) = zip64_with_extension(&[]); + let prefix = b"MZ synthetic self-extracting prefix"; + for declared in [45u64, 4096] { + let mut data = prefix.to_vec(); + data.extend(&original); + let record = record + prefix.len(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert!(warning_contains(&archive, "recovered")); + assert!(warning_contains(&archive, "extent")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn shifted_record_recovery_has_a_bounded_lookbehind() { + let (original, _) = zip64_with_extension(&[0; 65_536]); + let mut data = b"MZ synthetic self-extracting prefix".to_vec(); + data.extend(&original); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn an_unusable_locator_keeps_the_classic_directory_and_prefix_adjustment() { + // 65,535 is a valid classic count as well as a ZIP64 sentinel. A final entry comment + // can look like a locator without actually pointing to a ZIP64 end record. + let files = vec![("a.txt", &b"retained payload"[..]); 65_535]; + let original = zip_stored(&files); + let end = original.len() - 22; + let size = u32::from_le_bytes(original[end + 12..end + 16].try_into().unwrap()); + let last_entry = end - 46 - "a.txt".len(); + let mut locator = b"PK\x06\x07".to_vec(); + locator.extend(0u32.to_le_bytes()); + locator.extend(1u64.to_le_bytes()); + locator.extend(1u32.to_le_bytes()); + // A nearby record-shaped comment is still not a ZIP64 directory when its bounds + // disagree with the explicit classic fields. + let mut fake_record = [0; 56]; + fake_record[..4].copy_from_slice(b"PK\x06\x06"); + fake_record[4..12].copy_from_slice(&44u64.to_le_bytes()); + for record_shaped_comment in [&[][..], &fake_record[..]] { + let mut comment = record_shaped_comment.to_vec(); + comment.extend(&locator); + let mut archive_data = original.clone(); + archive_data[last_entry + 32..last_entry + 34].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + archive_data[end + 12..end + 16].copy_from_slice(&(size + comment.len() as u32).to_le_bytes()); + archive_data.splice(end..end, comment); + for prefix in [&[][..], &b"MZ synthetic self-extracting prefix"[..]] { + for offset in [1u64, u64::MAX] { + let mut data = prefix.to_vec(); + data.extend(&archive_data); + let locator = data.len() - 22 - 20; + data[locator + 8..locator + 16].copy_from_slice(&offset.to_le_bytes()); + let archive = zip::parse(&data, 2).unwrap(); + assert!(!archive.zip64); + assert_eq!(archive.declared_entries, 65_535); + assert_eq!(archive.entries.len(), 2); + assert!(archive.truncated); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert!(warning_contains(&archive, "classic end record")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } + } + } +} diff --git a/crates/unzip/tests/zip64_extra_boundaries.rs b/crates/unzip/tests/zip64_extra_boundaries.rs new file mode 100644 index 000000000..a8c673d08 --- /dev/null +++ b/crates/unzip/tests/zip64_extra_boundaries.rs @@ -0,0 +1,64 @@ +// From Craft Anything crates/formats/tests/zip64_extra_boundaries.rs. + +mod common; +use craft_unzip as zip; + +fn archive(extra: &[u8], sentinel: usize) -> Vec { + let mut data = common::zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + data[central + sentinel..central + sentinel + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + data[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + data.splice(end..end, extra.iter().copied()); + let end = data.len() - 22; + data[end + 12..end + 16].copy_from_slice(&((47 + extra.len()) as u32).to_le_bytes()); + data +} + +#[test] +fn a_short_zip64_field_cannot_borrow_values_from_its_neighbor() { + for field in [20, 24, 42] { + let mut extra = vec![1, 0, 0, 0, 2, 0, 8, 0]; + extra.extend_from_slice(&1u64.to_le_bytes()); + assert!(zip::parse(&archive(&extra, field), 10).is_err(), "field {field}"); + } +} + +#[test] +fn zip64_sentinels_require_complete_values() { + for field in [20, 24, 42] { + assert!(zip::parse(&archive(&[], field), 10).is_err()); + assert!(zip::parse(&archive(&[1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0], field), 10).is_err()); + } +} + +#[test] +fn a_complete_zip64_field_preserves_the_value() { + for field in [20, 24, 42] { + let value = if field == 42 { 0u64 } else { 1u64 }; + let mut extra = vec![1, 0, 8, 0]; + extra.extend_from_slice(&value.to_le_bytes()); + let data = archive(&extra, field); + let parsed = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &parsed.entries[0], 1).unwrap(), b"x"); + } +} + +#[test] +fn a_legitimate_64_bit_value_equal_to_the_32_bit_sentinel_is_preserved() { + let mut extra = vec![1, 0, 8, 0]; + extra.extend_from_slice(&u64::from(u32::MAX).to_le_bytes()); + let archive = zip::parse(&archive(&extra, 24), 10).unwrap(); + assert_eq!(archive.entries[0].uncompressed_size, u64::from(u32::MAX)); +} + +#[test] +fn trailing_bytes_too_short_for_a_field_header_are_padding() { + for stray in 1..=3usize { + let mut extra = vec![0xaa, 0xbb, 0, 0]; + extra.extend(std::iter::repeat_n(0u8, stray)); + let data = archive(&extra, 36); + let parsed = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &parsed.entries[0], 1).unwrap(), b"x", "stray {stray}"); + } +} diff --git a/crates/unzip/tests/zip_decoded_size.rs b/crates/unzip/tests/zip_decoded_size.rs new file mode 100644 index 000000000..5bf15ea94 --- /dev/null +++ b/crates/unzip/tests/zip_decoded_size.rs @@ -0,0 +1,43 @@ +// From Craft Anything crates/formats/tests/zip_decoded_size.rs. + +mod common; +use craft_unzip::{self as zip, ZipEntry, ZipError}; + +fn deflated(body: &[u8]) -> (Vec, ZipEntry) { + let compressed = miniz_oxide::deflate::compress_to_vec(body, 6); + let mut data = vec![0; 30]; + data[..4].copy_from_slice(b"PK\x03\x04"); + data.extend_from_slice(&compressed); + let entry = ZipEntry { + name: "payload".into(), + method: 8, + flags: 0, + crc32: zip::crc32(body), + compressed_size: compressed.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + (data, entry) +} + +#[test] +fn decoded_length_must_match_the_directory_even_when_the_crc_matches() { + let (data, good) = deflated(b"complete body"); + for declared in [1, good.uncompressed_size + 1] { + let entry = ZipEntry { uncompressed_size: declared, ..good.clone() }; + assert!(matches!(zip::read_entry(&data, &entry, 100), Err(ZipError::Corrupt(_)))); + assert!(matches!(zip::read_entry_unverified(&data, &entry, 100), Err(ZipError::Corrupt(_)))); + } + assert_eq!(zip::read_entry(&data, &good, 100).unwrap(), b"complete body"); +} + +#[test] +fn stored_length_must_also_match_the_directory() { + let data = common::zip_stored(&[("payload", b"hello")]); + let mut entry = zip::parse(&data, 10).unwrap().entries.remove(0); + entry.uncompressed_size = 4; + assert!(matches!(zip::read_entry(&data, &entry, 100), Err(ZipError::Corrupt(_)))); +} diff --git a/crates/unzip/tests/zip_directory_bounds.rs b/crates/unzip/tests/zip_directory_bounds.rs new file mode 100644 index 000000000..3f594dd74 --- /dev/null +++ b/crates/unzip/tests/zip_directory_bounds.rs @@ -0,0 +1,74 @@ +// From Craft Anything crates/formats/tests/zip_directory_bounds.rs. +use craft_unzip::{self as zip, ZipError}; +mod common; +use common::zip_stored; + +#[test] +fn a_short_declared_entry_is_rejected_before_its_signature_is_checked() { + for remaining in 0..46 { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + data[end + 8..end + 12].copy_from_slice(&[2, 0, 2, 0]); + data[end + 12..end + 16].copy_from_slice(&(size + remaining).to_le_bytes()); + data.splice(end..end, vec![0; remaining as usize]); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Truncated("directory entry"))), "remaining {remaining} bytes"); + } +} + +#[test] +fn directory_names_and_comments_cannot_borrow_end_record_bytes() { + for field in [28, 32] { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let length = if field == 28 { 2u16 } else { 1 }; + data[central + field..central + field + 2].copy_from_slice(&length.to_le_bytes()); + assert!(zip::parse(&data, 10).is_err(), "field {field}"); + } +} + +#[test] +fn directory_extra_fields_cannot_borrow_end_record_bytes() { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + data[central + 30..central + 32].copy_from_slice(&2u16.to_le_bytes()); + assert_eq!(zip::parse(&data, 10), Err(ZipError::Truncated("directory entry fields"))); +} + +#[test] +fn a_directory_cannot_extend_into_its_end_record() { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + data[end + 12..end + 16].copy_from_slice(&(size + 1).to_le_bytes()); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn self_extracting_archives_keep_both_offset_conventions() { + for adjusted in [false, true] { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + let stub = b"executable stub"; + if adjusted { + data[central + 42..central + 46].copy_from_slice(&(stub.len() as u32).to_le_bytes()); + data[end + 16..end + 20].copy_from_slice(&((central + stub.len()) as u32).to_le_bytes()); + } + let mut file = stub.to_vec(); + file.extend(data); + let archive = zip::parse(&file, 10).unwrap(); + assert_eq!(zip::read_entry(&file, &archive.entries[0], 10).unwrap(), b"x"); + } +} + +#[test] +fn a_directory_signature_after_the_entries_is_not_an_extra_file() { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let signature = b"PK\x05\x05\x01\x00x"; + data[end + 12..end + 16].copy_from_slice(&(size + signature.len() as u32).to_le_bytes()); + data.splice(end..end, signature.iter().copied()); + assert_eq!(zip::parse(&data, 10).unwrap().entries.len(), 1); +} diff --git a/crates/unzip/tests/zip_entry_detection.rs b/crates/unzip/tests/zip_entry_detection.rs new file mode 100644 index 000000000..8a74ecf63 --- /dev/null +++ b/crates/unzip/tests/zip_entry_detection.rs @@ -0,0 +1,43 @@ +// From Craft Anything crates/formats/tests/zip_entry_detection.rs. +use craft_unzip::has_entry as zip_has_entry; +mod common; +use common::zip_stored; + +/// A directory-shaped payload is ordinary file content, not an archive member. +#[test] +fn embedded_archive_does_not_supply_outer_entries() { + let inner = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + let outer = zip_stored(&[("backup.zip", &inner)]); + assert!(!zip_has_entry(&outer, "AndroidManifest.xml")); + assert!(zip_has_entry(&outer, "backup.zip")); +} + +/// A ZIP comment can contain another directory without changing the archive's entries. +#[test] +fn comment_does_not_supply_entries() { + let mut outer = zip_stored(&[("readme.txt", b"notes")]); + let inner = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + let len = outer.len(); + outer[len - 2..].copy_from_slice(&(inner.len() as u16).to_le_bytes()); + outer.extend(inner); + assert!(!zip_has_entry(&outer, "AndroidManifest.xml")); + assert!(zip_has_entry(&outer, "readme.txt")); +} + +/// Actual manifests are found regardless of member order, including prefixed archives. +#[test] +fn actual_entries_are_found() { + let data = zip_stored(&[("classes.dex", b"dex"), ("AndroidManifest.xml", b"manifest")]); + assert!(zip_has_entry(&data, "AndroidManifest.xml")); + let mut prefixed = b"stub".to_vec(); + prefixed.extend(data); + assert!(zip_has_entry(&prefixed, "AndroidManifest.xml")); + assert!(!zip_has_entry(&prefixed, "missing")); +} + +/// A stray central-directory signature without an archive is not an entry. +#[test] +fn incomplete_archive_is_not_a_manifest() { + let data = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + assert!(!zip_has_entry(&data[..data.len() - 22], "AndroidManifest.xml")); +} diff --git a/crates/unzip/tests/zip_split_volumes.rs b/crates/unzip/tests/zip_split_volumes.rs new file mode 100644 index 000000000..c5412b72e --- /dev/null +++ b/crates/unzip/tests/zip_split_volumes.rs @@ -0,0 +1,165 @@ +// From Craft Anything crates/formats/tests/zip_split_volumes.rs. +use craft_unzip::{self as zip, ZipError}; +mod common; +use common::zip_stored; + +fn assert_split(data: &[u8]) { + assert!(matches!(zip::parse(data, 10), Err(ZipError::Unsupported(_)))); +} + +#[test] +fn classic_end_records_must_describe_one_complete_volume() { + for field in [4, 6, 8] { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let value = if field == 8 { 0u16 } else { 1 }; + data[end + field..end + field + 2].copy_from_slice(&value.to_le_bytes()); + assert_split(&data); + } +} + +#[test] +fn central_entries_cannot_point_to_another_volume() { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + data[central + 34..central + 36].copy_from_slice(&1u16.to_le_bytes()); + assert_split(&data); +} + +fn zip64_entry_disk(disk: u32, with_sizes_and_offset: bool) -> Vec { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + data[central + 34..central + 36].copy_from_slice(&u16::MAX.to_le_bytes()); + let mut extra = vec![1, 0, 4, 0]; + if with_sizes_and_offset { + data[central + 20..central + 28].fill(0xff); + data[central + 42..central + 46].fill(0xff); + extra[2..4].copy_from_slice(&28u16.to_le_bytes()); + // Uncompressed size, compressed size, and local header offset precede the disk. + for value in [1u64, 1, 0] { + extra.extend(value.to_le_bytes()); + } + } + extra.extend(disk.to_le_bytes()); + data[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + let directory_size = (end - central + extra.len()) as u32; + data.splice(end..end, extra); + let end = data.len() - 22; + data[end + 12..end + 16].copy_from_slice(&directory_size.to_le_bytes()); + data +} + +#[test] +fn a_zip64_disk_sentinel_is_resolved_before_rejecting_the_entry() { + let data = zip64_entry_disk(0, false); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + assert_split(&zip64_entry_disk(1, false)); +} + +#[test] +fn a_zip64_disk_sentinel_follows_sizes_and_the_local_offset() { + let data = zip64_entry_disk(0, true); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.entries[0].uncompressed_size, 1); + assert_eq!(archive.entries[0].compressed_size, 1); + assert_eq!(archive.entries[0].local_header_offset, 0); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + assert_split(&zip64_entry_disk(1, true)); +} + +fn zip64_archive(record_disk: u32, central_disk: u32, locator_disk: u32, disks: u32, entries_on_disk: u64) -> Vec { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let central = u32::from_le_bytes(data[end + 16..end + 20].try_into().unwrap()); + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let mut record = b"PK\x06\x06".to_vec(); + record.extend(44u64.to_le_bytes()); + record.extend([45, 0, 45, 0]); + record.extend(record_disk.to_le_bytes()); + record.extend(central_disk.to_le_bytes()); + record.extend(entries_on_disk.to_le_bytes()); + record.extend(1u64.to_le_bytes()); + record.extend(u64::from(size).to_le_bytes()); + record.extend(u64::from(central).to_le_bytes()); + record.extend(b"PK\x06\x07"); + record.extend(locator_disk.to_le_bytes()); + record.extend((end as u64).to_le_bytes()); + record.extend(disks.to_le_bytes()); + data[end + 8..end + 12].fill(0xff); + data.splice(end..end, record); + data +} + +#[test] +fn zip64_end_records_and_locators_must_also_describe_one_volume() { + for fields in [(1, 0, 0, 1, 1), (0, 1, 0, 1, 1), (0, 0, 1, 1, 1), (0, 0, 0, 2, 1), (0, 0, 0, 1, 0)] { + assert_split(&zip64_archive(fields.0, fields.1, fields.2, fields.3, fields.4)); + } + assert_eq!(zip::parse(&zip64_archive(0, 0, 0, 1, 1), 10).unwrap().entries.len(), 1); +} + +#[test] +fn zip64_does_not_erase_explicit_classic_volume_markers() { + for field in [4, 6] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&1u16.to_le_bytes()); + assert_split(&data); + data[end + field..end + field + 2].fill(0xff); + assert_eq!(zip::parse(&data, 10).unwrap().entries.len(), 1); + } +} + +#[test] +fn zip64_does_not_erase_explicit_classic_entry_counts() { + for field in [8, 10] { + for count in [0u16, 2] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&count.to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Unsupported(_))), "field {field}, count {count}"); + } + } +} + +#[test] +fn mixed_classic_and_zip64_entry_counts_can_describe_one_volume() { + for field in [8, 10] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&1u16.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.entries.len(), 1, "field {field}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + } +} + +#[test] +fn zip64_cannot_replace_explicit_classic_directory_bounds() { + for field in [12, 16] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + let value = u32::from_le_bytes(data[end + field..end + field + 4].try_into().unwrap()); + data[end + field..end + field + 4].copy_from_slice(&(value + 1).to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Corrupt(_))), "field {field}"); + } +} + +#[test] +fn each_zip64_field_can_resolve_its_own_classic_sentinel() { + for (field, width) in [(4, 2), (6, 2), (8, 2), (10, 2), (12, 4), (16, 4)] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + 8..end + 10].copy_from_slice(&1u16.to_le_bytes()); + data[end + 10..end + 12].copy_from_slice(&1u16.to_le_bytes()); + data[end + field..end + field + width].fill(0xff); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64, "field {field}"); + assert_eq!(archive.declared_entries, 1, "field {field}"); + assert_eq!(archive.entries.len(), 1, "field {field}"); + assert_eq!(archive.prefix_bytes, 0, "field {field}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x", "field {field}"); + } +} diff --git a/crates/unzip/tests/zip_stored_cap.rs b/crates/unzip/tests/zip_stored_cap.rs new file mode 100644 index 000000000..61bb06d50 --- /dev/null +++ b/crates/unzip/tests/zip_stored_cap.rs @@ -0,0 +1,21 @@ +// From Craft Anything crates/formats/tests/zip_stored_cap.rs. + +mod common; +use craft_unzip::{self as zip, ZipError}; + +#[test] +fn a_stored_entry_cannot_understate_its_size_to_bypass_the_output_limit() { + let data = common::zip_stored(&[("payload", &[0x5a; 64])]); + let mut entry = zip::parse(&data, 10).unwrap().entries.remove(0); + entry.uncompressed_size = 1; + assert!(matches!(zip::read_entry(&data, &entry, 8), Err(ZipError::TooLarge(8)))); + assert!(matches!(zip::read_entry_unverified(&data, &entry, 8), Err(ZipError::TooLarge(8)))); +} + +#[test] +fn stored_entries_at_the_limit_and_empty_entries_still_read() { + let data = common::zip_stored(&[("payload", b"hello"), ("empty", b"")]); + let entries = zip::parse(&data, 10).unwrap().entries; + assert_eq!(zip::read_entry(&data, &entries[0], 5).unwrap(), b"hello"); + assert_eq!(zip::read_entry(&data, &entries[1], 0).unwrap(), b""); +} From 712feab58680652c22e28391ec2e610c17bb374b Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:25:00 +0100 Subject: [PATCH 08/27] craft-zip: the lenient reader as craft_zip::lenient instead of a second ZIP crate craft-unzip's raw-bytes reader moves unchanged into craft-zip as the `lenient` module, behind the `lenient` feature (DEFLATE) and `lenient-bzip2`, `lenient-lzma` and `lenient-zstd` for the other methods, with `serde` for the listing types. craft-zip's default build is unchanged; the module doesn't use the `zip` crate. Its unit and integration tests move with it and run in craft-zip's test build. --- Cargo.lock | 19 ++---- Cargo.toml | 1 - README.md | 3 +- crates/unzip/Cargo.toml | 34 ----------- crates/zip/Cargo.toml | 20 ++++++- crates/zip/README.md | 10 ++++ .../{unzip/src/lib.rs => zip/src/lenient.rs} | 33 ++++++----- .../{unzip/src => zip/src/lenient}/bytes.rs | 0 .../{unzip/src => zip/src/lenient}/decode.rs | 58 +++++++++---------- crates/{unzip/src => zip/src/lenient}/file.rs | 14 ++--- .../{unzip/src => zip/src/lenient}/testkit.rs | 2 +- .../{unzip/src => zip/src/lenient}/tests.rs | 20 +++---- crates/zip/src/lib.rs | 5 ++ .../tests/lenient_common}/mod.rs | 2 +- ...lenient_zip64_end_record_compatibility.rs} | 4 +- .../tests/lenient_zip64_extra_boundaries.rs} | 4 +- .../tests/lenient_zip_decoded_size.rs} | 4 +- .../tests/lenient_zip_directory_bounds.rs} | 4 +- .../tests/lenient_zip_entry_detection.rs} | 4 +- .../tests/lenient_zip_split_volumes.rs} | 4 +- .../tests/lenient_zip_stored_cap.rs} | 4 +- 21 files changed, 128 insertions(+), 121 deletions(-) delete mode 100644 crates/unzip/Cargo.toml rename crates/{unzip/src/lib.rs => zip/src/lenient.rs} (95%) rename crates/{unzip/src => zip/src/lenient}/bytes.rs (100%) rename crates/{unzip/src => zip/src/lenient}/decode.rs (91%) rename crates/{unzip/src => zip/src/lenient}/file.rs (95%) rename crates/{unzip/src => zip/src/lenient}/testkit.rs (99%) rename crates/{unzip/src => zip/src/lenient}/tests.rs (96%) rename crates/{unzip/tests/common => zip/tests/lenient_common}/mod.rs (98%) rename crates/{unzip/tests/zip64_end_record_compatibility.rs => zip/tests/lenient_zip64_end_record_compatibility.rs} (98%) rename crates/{unzip/tests/zip64_extra_boundaries.rs => zip/tests/lenient_zip64_extra_boundaries.rs} (96%) rename crates/{unzip/tests/zip_decoded_size.rs => zip/tests/lenient_zip_decoded_size.rs} (92%) rename crates/{unzip/tests/zip_directory_bounds.rs => zip/tests/lenient_zip_directory_bounds.rs} (96%) rename crates/{unzip/tests/zip_entry_detection.rs => zip/tests/lenient_zip_entry_detection.rs} (93%) rename crates/{unzip/tests/zip_split_volumes.rs => zip/tests/lenient_zip_split_volumes.rs} (98%) rename crates/{unzip/tests/zip_stored_cap.rs => zip/tests/lenient_zip_stored_cap.rs} (88%) diff --git a/Cargo.lock b/Cargo.lock index 7a00d57d7..27a34f299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2898,19 +2898,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "craft-unzip" -version = "0.1.0" -dependencies = [ - "bzip2-rs", - "lzma-rs", - "miniz_oxide 0.8.9", - "ruzstd", - "serde", - "serde_json", - "thiserror 2.0.21", -] - [[package]] name = "craft-update" version = "0.1.0" @@ -3113,8 +3100,14 @@ dependencies = [ name = "craft-zip" version = "0.1.0" dependencies = [ + "bzip2-rs", "craft-zip", "flate2", + "lzma-rs", + "miniz_oxide 0.8.9", + "ruzstd", + "serde", + "serde_json", "thiserror 2.0.21", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 34ee6095e..bd08fa725 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,7 +174,6 @@ craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } -craft-unzip = { path = "crates/unzip" } craft-epub = { path = "crates/epub" } craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } diff --git a/README.md b/README.md index 8a4c3d327..63303e3f0 100644 --- a/README.md +++ b/README.md @@ -392,8 +392,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit - [`craft-kra`](crates/kra/README.md): Krita documents (`.kra`, `.krz`), read: the ZIP archive, the `maindoc.xml` node tree as written, and the tiled pixel files with their LZF compression, a tile at a time. From PhotoCraft. - [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. - [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. -- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. -- [`craft-unzip`](crates/unzip/src/lib.rs): ZIP reading over bytes already in memory (or one named entry from a seekable file), from Craft Anything: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind default features) checked against the declared size and CRC-32. Pure Rust with no `zip` crate underneath; for inspectors that must describe damaged archives rather than refuse them. +- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. With the `lenient` feature, also a reader of the raw bytes without the `zip` crate, from Craft Anything, for inspectors that must describe damaged archives rather than refuse them: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind `lenient-*` features) checked against the declared size and CRC-32. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/crates/unzip/Cargo.toml b/crates/unzip/Cargo.toml deleted file mode 100644 index 8a414277b..000000000 --- a/crates/unzip/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "craft-unzip" -description = "ZIP archives (APK, JAR, IPA, MSIX…) from hostile bytes, from Craft Anything: a central-directory listing with ZIP64, prepended-stub and multi-volume checks, and bounded entry reads checked against the declared size and CRC-32. Pure Rust, no I/O beyond an optional seekable reader." -version.workspace = true -edition.workspace = true -license.workspace = true -rust-version.workspace = true -repository.workspace = true - -[lints] -workspace = true - -[dependencies] -thiserror = { workspace = true } -miniz_oxide = { workspace = true } -# Entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. -bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 -lzma-rs = { version = "0.3", optional = true } # MIT -ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT -# `serde`: `Serialize` on the listing types, for apps that report listings as JSON. -serde = { workspace = true, optional = true } - -[features] -default = ["bzip2", "lzma", "zstd"] -# Method 12. -bzip2 = ["dep:bzip2-rs"] -# Methods 14 (ZIP LZMA) and 95 (XZ). -lzma = ["dep:lzma-rs"] -# Method 93. -zstd = ["dep:ruzstd"] -serde = ["dep:serde"] - -[dev-dependencies] -serde_json = { workspace = true } diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 6d50b5352..89cb2d1a7 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -17,10 +17,28 @@ thiserror = { workspace = true } zip = { version = "9", default-features = false, features = ["deflate-flate2"] } # The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +# `lenient`: DEFLATE for the raw-bytes reader. +miniz_oxide = { workspace = true, optional = true } +# `lenient` entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. +bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 +lzma-rs = { version = "0.3", optional = true } # MIT +ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT +# `serde`: `Serialize` on the `lenient` listing types, for apps that report listings as JSON. +serde = { workspace = true, optional = true } [features] # `zip64_entry_count_sample`, the regression input for a reader's own entry-count test. test-support = [] +# `craft_zip::lenient`: the raw-bytes reader for inspectors (from Craft Anything), without `zip`. +lenient = ["dep:miniz_oxide"] +# Method 12. +lenient-bzip2 = ["lenient", "dep:bzip2-rs"] +# Methods 14 (ZIP LZMA) and 95 (XZ). +lenient-lzma = ["lenient", "dep:lzma-rs"] +# Method 93. +lenient-zstd = ["lenient", "dep:ruzstd"] +serde = ["dep:serde"] [dev-dependencies] -craft-zip = { path = ".", features = ["test-support"] } +craft-zip = { path = ".", features = ["test-support", "lenient-bzip2", "lenient-lzma", "lenient-zstd", "serde"] } +serde_json = { workspace = true } diff --git a/crates/zip/README.md b/crates/zip/README.md index 6cffa7786..071d2124d 100644 --- a/crates/zip/README.md +++ b/crates/zip/README.md @@ -79,3 +79,13 @@ bytes and charges repeated reads against twice that number. Call `Indexing::fini returned handle after the ZIP index has been parsed to permit normal streaming payload reads. The caller must still enforce entry-count, declared-size and overlap limits: the wrapper only bounds metadata reads and does not validate the archive. + +## Lenient reading (feature `lenient`) + +`craft_zip::lenient` reads an archive from its raw bytes without the `zip` crate, for inspectors +(Craft Anything) that must list and describe damaged, prepended, split or oddly encoded archives +instead of refusing them. Only the central directory is trusted for the listing; entry reads are +bounded, checked against the declared size and CRC-32, and decode stored and DEFLATE entries, plus +bzip2 (`lenient-bzip2`), ZIP LZMA and XZ (`lenient-lzma`) and Zstandard (`lenient-zstd`). +`lenient::file::read_named_entry` reads one entry from a seekable file without loading the +archive. With `serde`, the listing types are `Serialize`. diff --git a/crates/unzip/src/lib.rs b/crates/zip/src/lenient.rs similarity index 95% rename from crates/unzip/src/lib.rs rename to crates/zip/src/lenient.rs index 35dfbbfcf..c9bed3764 100644 --- a/crates/unzip/src/lib.rs +++ b/crates/zip/src/lenient.rs @@ -1,14 +1,17 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -#![forbid(unsafe_code)] -//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…): central directory listing and bounded entry reads. +//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…) read leniently from the raw bytes, for +//! inspectors that must list and describe damaged, prepended, split or oddly encoded archives +//! rather than refuse them (feature `lenient`). The rest of this crate is the strict side: it +//! refuses what fails [`crate::preflight`] and reads through the `zip` crate; nothing here uses +//! `zip`. //! //! Only the central directory is trusted for the listing. Entry data is located through the local //! header, decompressed with a hard output cap, and checked against the recorded CRC-32. Stored, //! DEFLATE, bzip2, ZIP LZMA, XZ and Zstandard entries are supported; other methods and encrypted entries are listed. //! -//! The bzip2, LZMA/XZ and Zstandard methods sit behind the default `bzip2`, `lzma` and `zstd` -//! features; without one, entries using it are listed and reading them is +//! The bzip2, LZMA/XZ and Zstandard methods sit behind the `lenient-bzip2`, `lenient-lzma` and +//! `lenient-zstd` features; without one, entries using it are listed and reading them is //! [`ZipError::Unsupported`]. [`file::read_named_entry`] reads one entry from a seekable file //! without loading the archive. With the `serde` feature the listing types are `Serialize`. //! @@ -18,11 +21,11 @@ //! against the directory, and CRC-32 verification. mod bytes; -#[cfg(any(feature = "bzip2", feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma", feature = "lenient-zstd"))] mod decode; pub mod file; -use crate::bytes::Reader; +use crate::lenient::bytes::Reader; /// Biggest tail we search for the end-of-central-directory record (64 KiB comment + record). pub const EOCD_SEARCH: usize = 22 + 0xffff; @@ -474,12 +477,12 @@ pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Resul Ok(out) } -#[cfg(any(feature = "bzip2", feature = "lzma"))] +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma"))] fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { let decoded = match e.method { - #[cfg(feature = "bzip2")] + #[cfg(feature = "lenient-bzip2")] 12 => decode::bzip2(raw, max_bytes), - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] 95 => decode::xz(raw, max_bytes), _ => return Err(not_built(e)), }; @@ -489,12 +492,12 @@ fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result Result, ZipError> { Err(not_built(e)) } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { let header = Reader::new(raw); let properties_size = header.u16_le(2).ok_or(ZipError::Truncated("LZMA properties header"))?; @@ -508,23 +511,23 @@ fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipE }) } -#[cfg(not(feature = "lzma"))] +#[cfg(not(feature = "lenient-lzma"))] fn decode_lzma(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { Err(not_built(e)) } -#[cfg(feature = "zstd")] +#[cfg(feature = "lenient-zstd")] fn decode_zstd(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { decode::zstd(raw, max_bytes).map_err(|err| ZipError::Corrupt(format!("zstd stream for {} is damaged: {err}", e.name))) } -#[cfg(not(feature = "zstd"))] +#[cfg(not(feature = "lenient-zstd"))] fn decode_zstd(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { Err(not_built(e)) } /// An entry whose method this build leaves out (see the crate features). -#[cfg_attr(all(feature = "bzip2", feature = "lzma", feature = "zstd"), allow(dead_code))] +#[cfg_attr(all(feature = "lenient-bzip2", feature = "lenient-lzma", feature = "lenient-zstd"), allow(dead_code))] fn not_built(e: &ZipEntry) -> ZipError { ZipError::Unsupported(format!("compression method {} ({}) for {} is not built in", e.method, e.method_name(), e.name)) } diff --git a/crates/unzip/src/bytes.rs b/crates/zip/src/lenient/bytes.rs similarity index 100% rename from crates/unzip/src/bytes.rs rename to crates/zip/src/lenient/bytes.rs diff --git a/crates/unzip/src/decode.rs b/crates/zip/src/lenient/decode.rs similarity index 91% rename from crates/unzip/src/decode.rs rename to crates/zip/src/lenient/decode.rs index 1d41d4a55..1189451b4 100644 --- a/crates/unzip/src/decode.rs +++ b/crates/zip/src/lenient/decode.rs @@ -3,9 +3,9 @@ //! allocating without limit) and never panics on a corrupt or truncated stream. use std::io; -#[cfg(any(feature = "bzip2", feature = "lzma"))] +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma"))] use std::io::Read; -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] use std::io::Write; /// Why a decode stopped. @@ -14,7 +14,7 @@ pub(crate) enum DecompressError { #[error("the compressed stream is damaged")] Corrupt, #[error("the compressed stream ends early")] - #[cfg(any(feature = "lzma", feature = "zstd"))] + #[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] Truncated, #[error("the output would exceed the size limit")] TooLarge, @@ -22,21 +22,21 @@ pub(crate) enum DecompressError { /// A `Write` sink that collects at most `cap` bytes, then refuses the rest. The `exceeded` flag lets /// the caller tell a size-limit stop from a genuine decode error. -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] struct CapWrite { buf: Vec, cap: usize, exceeded: bool, } -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] impl CapWrite { fn new(cap: usize) -> Self { CapWrite { buf: Vec::new(), cap, exceeded: false } } } -#[cfg(any(feature = "lzma", feature = "zstd"))] +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] impl Write for CapWrite { fn write(&mut self, data: &[u8]) -> io::Result { if self.buf.len().saturating_add(data.len()) > self.cap { @@ -57,7 +57,7 @@ fn cap_of(max_out: u64) -> usize { } /// Read a `Read` source into a `Vec`, capped at `cap` bytes. -#[cfg(feature = "bzip2")] +#[cfg(feature = "lenient-bzip2")] fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressError> { let mut out = Vec::new(); let mut chunk = [0u8; 64 * 1024]; @@ -77,7 +77,7 @@ fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressErro } } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] /// Decompress every xz stream, including standard stream padding, within one output budget. pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> { let ranges = xz_stream_ranges(input, max_out)?; @@ -95,7 +95,7 @@ pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> Ok(out.buf) } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] /// Find stream boundaries backwards through their checked footer/index sizes, without /// repeatedly trying the decoder at candidate magic bytes inside compressed data. fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result>, DecompressError> { @@ -112,9 +112,9 @@ fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result Result Result Result { let mut value = 0u64; @@ -174,7 +174,7 @@ fn xz_vli(input: &mut &[u8]) -> Result { Err(DecompressError::Corrupt) } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] /// Decode the property bytes and raw stream of a ZIP method-14 member. pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, max_out: u64) -> Result, DecompressError> { let cap = cap_of(max_out).min(isize::MAX as usize); @@ -204,13 +204,13 @@ pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, m } } -#[cfg(feature = "bzip2")] +#[cfg(feature = "lenient-bzip2")] /// Decompress a bzip2 stream, producing at most `max_out` bytes. pub(crate) fn bzip2(input: &[u8], max_out: u64) -> Result, DecompressError> { read_capped(bzip2_rs::DecoderReader::new(input), cap_of(max_out)) } -#[cfg(feature = "zstd")] +#[cfg(feature = "lenient-zstd")] /// Decompress a Zstandard stream, producing at most `max_out` bytes. pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, DecompressError> { if input.is_empty() { @@ -235,9 +235,9 @@ pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, Decompress #[cfg(test)] mod tests { use super::*; - use crate::testkit as fx; + use crate::lenient::testkit as fx; - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_round_trip() { let c = fx::XZ_FIXTURE; @@ -250,7 +250,7 @@ mod tests { } } - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_concatenated_streams_and_padding_share_one_budget() { let input = [fx::XZ_FIXTURE, &[0; 4], fx::XZ_FIXTURE, &[0; 8]].concat(); @@ -264,7 +264,7 @@ mod tests { assert!(xz(&empty, 0).unwrap().is_empty()); } - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_rejects_incomplete_members_and_nonstandard_padding() { for padding in [1, 2, 3, 5] { @@ -284,7 +284,7 @@ mod tests { } } - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] #[test] fn xz_index_sizes_and_integer_encodings_are_checked() { assert_eq!(xz_vli(&mut &[0x80, 0][..]), Err(DecompressError::Corrupt)); @@ -295,12 +295,12 @@ mod tests { let backward = u32::from_le_bytes(damaged[footer + 4..footer + 8].try_into().unwrap()); let index = footer - (backward as usize + 1) * 4; damaged[index + 2] = 0; - let checksum = crate::crc32(&damaged[index..footer - 4]); + let checksum = crate::lenient::crc32(&damaged[index..footer - 4]); damaged[footer - 4..footer].copy_from_slice(&checksum.to_le_bytes()); assert!(xz(&damaged, 1000).is_err()); } - #[cfg(feature = "bzip2")] + #[cfg(feature = "lenient-bzip2")] #[test] fn bzip2_round_trip() { let c = fx::BZIP2_FIXTURE; @@ -311,7 +311,7 @@ mod tests { } } - #[cfg(feature = "zstd")] + #[cfg(feature = "lenient-zstd")] #[test] fn zstd_round_trip() { let c = fx::ZSTD_FIXTURE; @@ -322,7 +322,7 @@ mod tests { } } - #[cfg(feature = "zstd")] + #[cfg(feature = "lenient-zstd")] #[test] fn zstandard_members_can_contain_multiple_frames() { let frame = fx::ZSTD_FIXTURE; @@ -340,11 +340,11 @@ mod tests { fn garbage_never_panics() { for seed in 0..64u64 { let n = fx::noise(seed, 400); - #[cfg(feature = "lzma")] + #[cfg(feature = "lenient-lzma")] let _ = xz(&n, 1 << 20); - #[cfg(feature = "bzip2")] + #[cfg(feature = "lenient-bzip2")] let _ = bzip2(&n, 1 << 20); - #[cfg(feature = "zstd")] + #[cfg(feature = "lenient-zstd")] let _ = zstd(&n, 1 << 20); let _ = &n; } diff --git a/crates/unzip/src/file.rs b/crates/zip/src/lenient/file.rs similarity index 95% rename from crates/unzip/src/file.rs rename to crates/zip/src/lenient/file.rs index 1bb4a3662..fc50dfd26 100644 --- a/crates/unzip/src/file.rs +++ b/crates/zip/src/lenient/file.rs @@ -2,8 +2,8 @@ //! central directory and that entry's bytes are read. For archives too big to hold in memory just //! to look at one small entry (a JAR's manifest). -use crate::bytes::Reader; -use crate::{self as zip, ZipError}; +use crate::lenient::bytes::Reader; +use crate::lenient::{self as zip, ZipError}; use std::io::{Read, Seek, SeekFrom}; /// Most central directory read (some hundreds of thousands of entries); a bigger one is refused. @@ -159,7 +159,7 @@ mod tests { #[test] fn an_entry_is_read_from_the_directory_without_the_whole_file() { let filler = vec![7u8; 200_000]; - let archive = crate::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); + let archive = crate::lenient::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); assert_eq!(read(&archive, "META-INF/MISSING").unwrap(), None); assert_eq!(read(&archive, "META-INF/").unwrap(), None, "a folder isn't an entry to read"); @@ -178,7 +178,7 @@ mod tests { let names: Vec = (0..3000).map(|i| format!("com/example/generated/Class{i:05}.class")).collect(); let mut files: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b"x"[..])).collect(); files.push(("META-INF/MANIFEST.MF", MANIFEST)); - let archive = crate::testkit::zip_stored(&files); + let archive = crate::lenient::testkit::zip_stored(&files); assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); } @@ -215,7 +215,7 @@ mod tests { let deflated = zip_deflated("META-INF/MANIFEST.MF", MANIFEST); assert_eq!(read(&deflated, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); - let mut commented = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let mut commented = crate::lenient::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); let comment = b"built by PK\x05\x06 tools"; let at = commented.len() - 2; commented[at..].copy_from_slice(&(comment.len() as u16).to_le_bytes()); @@ -227,7 +227,7 @@ mod tests { /// the classic fields disagree with it and win, as they do for `zip::parse`. #[test] fn a_comment_shaped_like_zip64_records_is_not_read_as_them() { - let mut archive = crate::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let mut archive = crate::lenient::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); let end = archive.len() - 22; let last = archive.windows(4).rposition(|w| w == b"PK\x01\x02").unwrap(); let mut fake = b"PK\x06\x06".to_vec(); @@ -250,7 +250,7 @@ mod tests { /// only the offset), after a stub too. #[test] fn zip64_archives_are_read_in_parts() { - let archive = crate::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); + let archive = crate::lenient::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); for saturate in [&[][..], &[8, 10, 12, 16], &[16], &[12], &[10], &[4]] { let zip64 = as_zip64(&archive, saturate); assert_eq!(read(&zip64, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}"); diff --git a/crates/unzip/src/testkit.rs b/crates/zip/src/lenient/testkit.rs similarity index 99% rename from crates/unzip/src/testkit.rs rename to crates/zip/src/lenient/testkit.rs index ae55cc052..ca87510e9 100644 --- a/crates/unzip/src/testkit.rs +++ b/crates/zip/src/lenient/testkit.rs @@ -3,7 +3,7 @@ #![allow(clippy::cast_possible_truncation, dead_code)] -use crate::crc32; +use crate::lenient::crc32; /// Plaintext of [`XZ_FIXTURE`] and [`LZMA_FIXTURE`]. pub const XZ_PLAIN: &[u8] = b"Hello, xz compression world!\n"; diff --git a/crates/unzip/src/tests.rs b/crates/zip/src/lenient/tests.rs similarity index 96% rename from crates/unzip/src/tests.rs rename to crates/zip/src/lenient/tests.rs index 3ea92afe4..c2ca97394 100644 --- a/crates/unzip/src/tests.rs +++ b/crates/zip/src/lenient/tests.rs @@ -39,7 +39,7 @@ pub(crate) fn build(files: &[(&str, &[u8])]) -> Vec { out } -#[cfg(all(feature = "bzip2", feature = "lzma"))] +#[cfg(all(feature = "lenient-bzip2", feature = "lenient-lzma"))] fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { let mut file = build(&[("entry", compressed)]); let central = 30 + "entry".len() + compressed.len(); @@ -55,10 +55,10 @@ fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { file } -#[cfg(all(feature = "bzip2", feature = "lzma"))] +#[cfg(all(feature = "lenient-bzip2", feature = "lenient-lzma"))] #[test] fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { - use crate::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; + use crate::lenient::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; for (method, compressed, plain) in [(12, BZIP2_FIXTURE, BZIP2_PLAIN), (95, XZ_FIXTURE, XZ_PLAIN)] { let file = build_codec_entry(method, compressed, plain); let original = parse(&file, 1).unwrap().entries.remove(0); @@ -80,7 +80,7 @@ fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { } } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { let mut encoded = Vec::new(); let size = if end_marker { None } else { Some(plain.len() as u64) }; @@ -104,7 +104,7 @@ fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { file } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] #[test] fn zip_lzma_reads_size_delimited_and_end_marked_streams() { for end_marker in [false, true] { @@ -129,7 +129,7 @@ fn zip_lzma_reads_size_delimited_and_end_marked_streams() { } } -#[cfg(feature = "lzma")] +#[cfg(feature = "lenient-lzma")] #[test] fn zip_lzma_rejects_bad_properties_without_unbounded_dictionaries() { let mut file = zip_lzma_member(b"small", true); @@ -286,7 +286,7 @@ fn lists_and_reads_stored_entries() { assert!(matches!(read_entry(&z, &a.entries[1], 3), Err(ZipError::TooLarge(3)))); } -#[cfg(feature = "zstd")] +#[cfg(feature = "lenient-zstd")] #[test] fn zstandard_members_are_read_with_their_limits_and_checks() { // A raw Zstandard frame holding "hello zip" (single segment, one raw block). @@ -373,7 +373,7 @@ fn crc_known_value() { #[cfg(feature = "serde")] #[test] fn listings_serialize_without_empty_warnings_or_unix_mode() { - let data = crate::testkit::zip_stored(&[("a.txt", b"hi")]); + let data = crate::lenient::testkit::zip_stored(&[("a.txt", b"hi")]); let archive = parse(&data, 10).unwrap(); let value = serde_json::to_value(&archive).unwrap(); assert!(value.get("warnings").is_none()); @@ -381,10 +381,10 @@ fn listings_serialize_without_empty_warnings_or_unix_mode() { assert!(value["entries"][0].get("unix_mode").is_none()); } -#[cfg(not(feature = "zstd"))] +#[cfg(not(feature = "lenient-zstd"))] #[test] fn a_method_left_out_of_the_build_is_unsupported() { - let mut data = crate::testkit::zip_stored(&[("a", b"abc")]); + let mut data = crate::lenient::testkit::zip_stored(&[("a", b"abc")]); let archive = parse(&data, 10).unwrap(); let mut entry = archive.entries[0].clone(); entry.method = 93; diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index 0a348dc1f..ff8ce17a6 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -13,6 +13,9 @@ //! timestamp on every entry so equal input gives equal bytes. //! - [`preflight`] is a stricter check of the whole end-record and central-directory structure //! on the raw bytes, for a caller that wants nothing ambiguous to reach a ZIP reader at all. +//! - `lenient` (feature `lenient`) is the other way round: a reader of the raw bytes without +//! `zip`, for inspectors that list and describe damaged or unusual archives instead of refusing +//! them, with bounded, CRC-checked entry reads. //! //! Every limit is the caller's number. Entry-name rules (which names are safe, which spellings //! are one part) stay with the format that knows them. @@ -23,6 +26,8 @@ struct ReadmeExamples; mod archive; mod guard; +#[cfg(feature = "lenient")] +pub mod lenient; mod metadata_reader; pub mod preflight; mod seekable; diff --git a/crates/unzip/tests/common/mod.rs b/crates/zip/tests/lenient_common/mod.rs similarity index 98% rename from crates/unzip/tests/common/mod.rs rename to crates/zip/tests/lenient_common/mod.rs index ba0f3d778..05d8c5fff 100644 --- a/crates/unzip/tests/common/mod.rs +++ b/crates/zip/tests/lenient_common/mod.rs @@ -2,7 +2,7 @@ #![allow(dead_code, clippy::cast_possible_truncation)] -use craft_unzip::crc32; +use craft_zip::lenient::crc32; /// A stored-only ZIP of `files`. pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { diff --git a/crates/unzip/tests/zip64_end_record_compatibility.rs b/crates/zip/tests/lenient_zip64_end_record_compatibility.rs similarity index 98% rename from crates/unzip/tests/zip64_end_record_compatibility.rs rename to crates/zip/tests/lenient_zip64_end_record_compatibility.rs index 37f63cf76..ed599fbda 100644 --- a/crates/unzip/tests/zip64_end_record_compatibility.rs +++ b/crates/zip/tests/lenient_zip64_end_record_compatibility.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip64_end_record_compatibility.rs. -use craft_unzip::{self as zip, ZipArchive, ZipError}; +use craft_zip::lenient::{self as zip, ZipArchive, ZipError}; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip64_extra_boundaries.rs b/crates/zip/tests/lenient_zip64_extra_boundaries.rs similarity index 96% rename from crates/unzip/tests/zip64_extra_boundaries.rs rename to crates/zip/tests/lenient_zip64_extra_boundaries.rs index a8c673d08..fae4fd5f9 100644 --- a/crates/unzip/tests/zip64_extra_boundaries.rs +++ b/crates/zip/tests/lenient_zip64_extra_boundaries.rs @@ -1,7 +1,9 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip64_extra_boundaries.rs. +#[path = "lenient_common/mod.rs"] mod common; -use craft_unzip as zip; +use craft_zip::lenient as zip; fn archive(extra: &[u8], sentinel: usize) -> Vec { let mut data = common::zip_stored(&[("a", b"x")]); diff --git a/crates/unzip/tests/zip_decoded_size.rs b/crates/zip/tests/lenient_zip_decoded_size.rs similarity index 92% rename from crates/unzip/tests/zip_decoded_size.rs rename to crates/zip/tests/lenient_zip_decoded_size.rs index 5bf15ea94..9db3ec80e 100644 --- a/crates/unzip/tests/zip_decoded_size.rs +++ b/crates/zip/tests/lenient_zip_decoded_size.rs @@ -1,7 +1,9 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_decoded_size.rs. +#[path = "lenient_common/mod.rs"] mod common; -use craft_unzip::{self as zip, ZipEntry, ZipError}; +use craft_zip::lenient::{self as zip, ZipEntry, ZipError}; fn deflated(body: &[u8]) -> (Vec, ZipEntry) { let compressed = miniz_oxide::deflate::compress_to_vec(body, 6); diff --git a/crates/unzip/tests/zip_directory_bounds.rs b/crates/zip/tests/lenient_zip_directory_bounds.rs similarity index 96% rename from crates/unzip/tests/zip_directory_bounds.rs rename to crates/zip/tests/lenient_zip_directory_bounds.rs index 3f594dd74..8e912fc3b 100644 --- a/crates/unzip/tests/zip_directory_bounds.rs +++ b/crates/zip/tests/lenient_zip_directory_bounds.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_directory_bounds.rs. -use craft_unzip::{self as zip, ZipError}; +use craft_zip::lenient::{self as zip, ZipError}; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip_entry_detection.rs b/crates/zip/tests/lenient_zip_entry_detection.rs similarity index 93% rename from crates/unzip/tests/zip_entry_detection.rs rename to crates/zip/tests/lenient_zip_entry_detection.rs index 8a74ecf63..8708b6400 100644 --- a/crates/unzip/tests/zip_entry_detection.rs +++ b/crates/zip/tests/lenient_zip_entry_detection.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_entry_detection.rs. -use craft_unzip::has_entry as zip_has_entry; +use craft_zip::lenient::has_entry as zip_has_entry; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip_split_volumes.rs b/crates/zip/tests/lenient_zip_split_volumes.rs similarity index 98% rename from crates/unzip/tests/zip_split_volumes.rs rename to crates/zip/tests/lenient_zip_split_volumes.rs index c5412b72e..9675177ce 100644 --- a/crates/unzip/tests/zip_split_volumes.rs +++ b/crates/zip/tests/lenient_zip_split_volumes.rs @@ -1,5 +1,7 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_split_volumes.rs. -use craft_unzip::{self as zip, ZipError}; +use craft_zip::lenient::{self as zip, ZipError}; +#[path = "lenient_common/mod.rs"] mod common; use common::zip_stored; diff --git a/crates/unzip/tests/zip_stored_cap.rs b/crates/zip/tests/lenient_zip_stored_cap.rs similarity index 88% rename from crates/unzip/tests/zip_stored_cap.rs rename to crates/zip/tests/lenient_zip_stored_cap.rs index 61bb06d50..a5424605f 100644 --- a/crates/unzip/tests/zip_stored_cap.rs +++ b/crates/zip/tests/lenient_zip_stored_cap.rs @@ -1,7 +1,9 @@ +#![cfg(feature = "lenient")] // From Craft Anything crates/formats/tests/zip_stored_cap.rs. +#[path = "lenient_common/mod.rs"] mod common; -use craft_unzip::{self as zip, ZipError}; +use craft_zip::lenient::{self as zip, ZipError}; #[test] fn a_stored_entry_cannot_understate_its_size_to_bypass_the_output_limit() { From b367c04c9deb2a1ac7dc391b270572ec4ab44b40 Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 12:25:17 +0100 Subject: [PATCH 09/27] Keep cached results for malformed empty XLSB names --- crates/xlsx/src/xlsb.rs | 16 ++++++++++++---- crates/xlsx/tests/xlsb.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index c1a2f8962..ff54ffcaf 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -449,11 +449,19 @@ fn defined_names( } }, }; - // Excel writes a workbook-level built-in with no formula (`Database`) beside a sheet's - // own; `.xlsx` leaves it out of `definedNames`, and formulas using it spell its name. + // Excel omits the formula for a workbook-level Database built-in and a VBA function + // (fFunc, fOB, fProc). They remain usable by formulas without a definedNames entry. + // Other empty names are malformed: omitting them must also omit their dependants. if p.rgce.is_empty() { - if let Some(n) = links.names.get_mut(p.index) { - n.read = true; + let database = p.flags & 0x20 != 0 && name.scope.is_none() && name.text == "_xlnm.Database"; + let vba_function = p.flags & 0x0E == 0x0E; + if database || vba_function { + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + } else { + warnings.warn("XLSB: some defined names were left out (empty formula)"); + omit(&mut omitted, p.index); } continue; } diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 27bcc18c1..6b403bfe8 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1737,3 +1737,34 @@ fn module_sheets_and_supporting_links_are_bounded() { assert_eq!(wb.sheets[0].formula(CellRef::new(0, 0)), None); assert!(wb.warnings.iter().any(|w| w.contains("sheet links or names could not be read")), "{:?}", wb.warnings); } + +#[test] +fn empty_ordinary_names_and_their_dependants_keep_cached_values() { + // Forward and transitive references to an empty ordinary name must not become dangling + // formulas. An empty sheet-scoped Database or another built-in is not the observed + // workbook-level Database exception either. + let mut list = names(); + let cell = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + list.push(brt_name(0, 0, WORKBOOK, "Early", &cell(16), None)); // 14 + list.push(brt_name(0, 0, WORKBOOK, "Empty", &[], None)); // 15 + list.push(brt_name(0, 0, WORKBOOK, "Alias", &cell(15), None)); // 16 + list.push(brt_name(0x20, 0, 0, "Database", &[], None)); // 17 + list.push(brt_name(0x20, 0, WORKBOOK, "Print_Area", &[], None)); // 18 + list.push(brt_name(0, 0, WORKBOOK, "Fine", &cell(9), None)); // 19 + let mut data = row(0); + for i in 14..=19 { + data.extend(formula_cell(i - 13, &cell(i), &[])); + } + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = NAMES_XML.replace("", r#"Rate"#); + assert_eq!(wb.defined_names, xml_names(&xml)); + let book = read(&bytes).unwrap(); + for col in 1..=5 { + assert_eq!(book.sheets[0].formula(CellRef::new(0, col)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, col)), Value::Number(0.0)); + } + assert_eq!(book.sheets[0].formula(CellRef::new(0, 6)), Some("Fine")); + assert!(book.warnings.iter().any(|w| w.contains("empty formula")), "{:?}", book.warnings); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} From 99e0f2be891c3d2acfa8f462d90b2bf5fab7bf4b Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:34:01 +0100 Subject: [PATCH 10/27] render-parity: compare two renders against a stated tolerance (#587) --- Cargo.lock | 7 ++ Cargo.toml | 1 + README.md | 3 + crates/render-parity/Cargo.toml | 15 +++ crates/render-parity/README.md | 37 ++++++ crates/render-parity/src/lib.rs | 199 ++++++++++++++++++++++++++++++ crates/render-parity/src/tests.rs | 89 +++++++++++++ 7 files changed, 351 insertions(+) create mode 100644 crates/render-parity/Cargo.toml create mode 100644 crates/render-parity/README.md create mode 100644 crates/render-parity/src/lib.rs create mode 100644 crates/render-parity/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 13362ace1..8c34794cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2630,6 +2630,13 @@ dependencies = [ "zune-jpegxl", ] +[[package]] +name = "craft-render-parity" +version = "0.1.0" +dependencies = [ + "craft-cms", +] + [[package]] name = "craft-rtf" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bd08fa725..02757f870 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ craft-usd = { path = "crates/usd" } craft-smime = { path = "crates/smime" } craft-ber = { path = "crates/ber" } craft-proto = { path = "crates/proto" } +craft-render-parity = { path = "crates/render-parity" } acadrust = "=0.6.6" log = "0.4" # internal: add each crate here as `craft- = { path = "crates/" }` diff --git a/README.md b/README.md index 8150dfea9..ce3a3cdcd 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through `craft-crw`) so far; demosaicing and everything after stay with the develop pipeline. +- [`craft-render-parity`](crates/render-parity/README.md): test support for the shared develop + pipeline (#587): how far two renders of the same photo are apart (per-channel error, CIEDE2000, + the worst pixel) and whether that is within a stated tolerance. - [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft (writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on diff --git a/crates/render-parity/Cargo.toml b/crates/render-parity/Cargo.toml new file mode 100644 index 000000000..087ff748b --- /dev/null +++ b/crates/render-parity/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-render-parity" +description = "Compare two renders of the same photo (per-channel error and CIEDE2000) against a stated tolerance: the parity check for the shared develop pipeline. Test support only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-cms = { workspace = true } diff --git a/crates/render-parity/README.md b/crates/render-parity/README.md new file mode 100644 index 000000000..0924d891a --- /dev/null +++ b/crates/render-parity/README.md @@ -0,0 +1,37 @@ +# craft-render-parity + +Test support for the shared develop pipeline ([#587](https://github.com/storytold/craft-libs/issues/587)): +how far two renders of the same photo are apart, and whether that is within a stated tolerance. +A develop stage that moves from an app, or an app that switches to the shared pipeline, renders +the same fixture both ways and compares the two here. + +```toml +[dev-dependencies] +craft-render-parity = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +```rust +use craft_render_parity::{Image, Tolerance, compare}; + +let expected = Image::new(width, height, &app_render)?; +let actual = Image::new(width, height, &shared_render)?; +let report = compare(&expected, &actual)?; +assert!(report.within(&Tolerance::BASELINE), "{report}: {:?}", report.exceeded(&Tolerance::BASELINE)); +``` + +## Input + +Scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, three interleaved `f32` samples +per pixel, row by row, with 1.0 as diffuse white. Values above 1.0 are compared as they are. A NaN +or infinite sample is an error, never a pass. + +## Measures + +| Measure | Meaning | +|---|---| +| `max_delta_e`, `mean_delta_e` | CIEDE2000 (kL = kC = kH = 1) in CIELAB relative to D65, from `craft_cms::math` | +| `max_channel`, `mean_channel` | Absolute difference of the linear samples, overall and per channel | +| `worst` | The first pixel with the largest CIEDE2000 difference, or the first that differs at all | + +`Tolerance::BASELINE`, the acceptance bar of #587, is CIEDE2000 at most 0.5 at any pixel and 0.05 +on average (1.0 is about the smallest visible difference), and at most 1/1024 in any linear sample. diff --git a/crates/render-parity/src/lib.rs b/crates/render-parity/src/lib.rs new file mode 100644 index 000000000..f2e541801 --- /dev/null +++ b/crates/render-parity/src/lib.rs @@ -0,0 +1,199 @@ +//! Render parity: how far two renders of the same photo are apart, and whether that is within a +//! stated [`Tolerance`]. +//! +//! The shared develop pipeline (craft-libs#587) has to give LightCraft and PhotoCraft the same +//! picture. A stage moved from an app, or an app switched to the shared crate, is checked by +//! rendering the same fixture both ways and comparing them here. The [`Report`] names the worst +//! pixel, so a failure points at where to look. +//! +//! Renders are scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, interleaved, with +//! 1.0 as diffuse white. Colour difference is CIEDE2000 in CIELAB relative to D65 (from +//! [`craft_cms::math`]); values above 1.0 are allowed and compared as they are. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::fmt; + +use craft_cms::math::{delta_e2000, xyz_to_lab}; + +/// Linear Rec. 709 RGB → CIE XYZ (D65), IEC 61966-2-1. +const RGB_TO_XYZ: [[f64; 3]; 3] = [[0.4124, 0.3576, 0.1805], [0.2126, 0.7152, 0.0722], [0.0193, 0.1192, 0.9505]]; +/// The D65 white in XYZ, from the matrix above applied to RGB (1, 1, 1). +const D65: [f64; 3] = [0.9505, 1.0, 1.089]; + +/// A render to compare: `width` × `height` interleaved RGB samples, row by row. +#[derive(Clone, Copy, Debug)] +pub struct Image<'a> { + width: u32, + height: u32, + rgb: &'a [f32], +} + +impl<'a> Image<'a> { + /// A render of `width` × `height` pixels whose `rgb` holds exactly three samples per pixel. + pub fn new(width: u32, height: u32, rgb: &'a [f32]) -> Result { + let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(3)); + if expected != Some(rgb.len()) { + return Err(ParityError::Length { width, height, samples: rgb.len() }); + } + Ok(Self { width, height, rgb }) + } + + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } +} + +/// Why two renders could not be compared. +#[derive(Clone, Debug, PartialEq)] +pub enum ParityError { + /// The sample count is not three per pixel. + Length { width: u32, height: u32, samples: usize }, + /// The renders differ in size, so a pixel of one has no counterpart in the other. + Size { expected: (u32, u32), actual: (u32, u32) }, + /// A sample is NaN or infinite: no difference can be stated for it. + NotFinite { x: u32, y: u32 }, +} + +impl fmt::Display for ParityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Length { width, height, samples } => write!(f, "{samples} samples is not RGB for {width} × {height} pixels"), + Self::Size { expected, actual } => write!(f, "render is {} × {}, expected {} × {}", actual.0, actual.1, expected.0, expected.1), + Self::NotFinite { x, y } => write!(f, "pixel ({x}, {y}) is not a finite colour"), + } + } +} + +impl std::error::Error for ParityError {} + +/// How far two renders may be apart and still count as the same picture. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Tolerance { + /// The largest CIEDE2000 difference at any pixel. + pub max_delta_e: f64, + /// The largest mean CIEDE2000 difference over the picture. + pub mean_delta_e: f64, + /// The largest difference in any one linear channel sample. + pub max_channel: f32, +} + +impl Tolerance { + /// The baseline parity of craft-libs#587: well below a visible difference (CIEDE2000 1.0), and + /// within a 10-bit code value in any channel. + pub const BASELINE: Self = Self { max_delta_e: 0.5, mean_delta_e: 0.05, max_channel: 1.0 / 1024.0 }; +} + +impl Default for Tolerance { + fn default() -> Self { + Self::BASELINE + } +} + +/// The difference between two renders of the same size. +#[derive(Clone, Debug, PartialEq)] +pub struct Report { + /// Pixels compared. + pub pixels: u64, + /// The largest CIEDE2000 difference, at [`Self::worst`]. + pub max_delta_e: f64, + /// The mean CIEDE2000 difference. + pub mean_delta_e: f64, + /// The largest absolute difference in any channel sample. + pub max_channel: f32, + /// The mean absolute difference per channel (R, G, B). + pub mean_channel: [f64; 3], + /// The first pixel with the largest CIEDE2000 difference, when any pixel differs. + pub worst: Option<(u32, u32)>, +} + +impl Report { + /// Whether every measure is within `tolerance`. + pub fn within(&self, tolerance: &Tolerance) -> bool { + self.exceeded(tolerance).is_empty() + } + + /// The measures over `tolerance`, each with its value and limit, for a test failure message. + pub fn exceeded(&self, tolerance: &Tolerance) -> Vec { + let mut over = Vec::new(); + if self.max_delta_e > tolerance.max_delta_e { + over.push(format!("max ΔE2000 {:.4} > {}", self.max_delta_e, tolerance.max_delta_e)); + } + if self.mean_delta_e > tolerance.mean_delta_e { + over.push(format!("mean ΔE2000 {:.4} > {}", self.mean_delta_e, tolerance.mean_delta_e)); + } + if self.max_channel > tolerance.max_channel { + over.push(format!("max channel difference {:.6} > {}", self.max_channel, tolerance.max_channel)); + } + over + } +} + +impl fmt::Display for Report { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} px: ΔE2000 max {:.4} mean {:.4}, channel max {:.6}", self.pixels, self.max_delta_e, self.mean_delta_e, self.max_channel)?; + match self.worst { + Some((x, y)) => write!(f, ", worst at ({x}, {y})"), + None => write!(f, ", identical"), + } + } +} + +/// Compare `actual` with `expected`, pixel by pixel. +pub fn compare(expected: &Image<'_>, actual: &Image<'_>) -> Result { + if (expected.width, expected.height) != (actual.width, actual.height) { + return Err(ParityError::Size { expected: (expected.width, expected.height), actual: (actual.width, actual.height) }); + } + let width = expected.width.max(1) as usize; + let mut report = Report { pixels: 0, max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0, mean_channel: [0.0; 3], worst: None }; + let mut sum_delta_e = 0.0; + for (i, (e, a)) in expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).enumerate() { + let (x, y) = ((i % width) as u32, (i / width) as u32); + let (Some(e), Some(a)) = (finite(*e), finite(*a)) else { return Err(ParityError::NotFinite { x, y }) }; + for c in 0..3 { + let d = (e[c] - a[c]).abs(); + report.max_channel = report.max_channel.max(d); + report.mean_channel[c] += f64::from(d); + } + let delta_e = delta_e2000(lab(e), lab(a)); + sum_delta_e += delta_e; + if delta_e > report.max_delta_e { + report.max_delta_e = delta_e; + report.worst = Some((x, y)); + } + report.pixels += 1; + } + if report.pixels > 0 { + let n = report.pixels as f64; + report.mean_delta_e = sum_delta_e / n; + report.mean_channel = report.mean_channel.map(|s| s / n); + } + if report.worst.is_none() && report.max_channel > 0.0 { + // channel differences too small to move CIELAB still name a pixel + report.worst = first_difference(expected, actual); + } + Ok(report) +} + +fn finite(px: [f32; 3]) -> Option<[f32; 3]> { + px.iter().all(|v| v.is_finite()).then_some(px) +} + +fn first_difference(expected: &Image<'_>, actual: &Image<'_>) -> Option<(u32, u32)> { + let width = expected.width.max(1) as usize; + let i = expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).position(|(e, a)| e != a)?; + Some(((i % width) as u32, (i / width) as u32)) +} + +fn lab(rgb: [f32; 3]) -> [f64; 3] { + let rgb = rgb.map(f64::from); + let xyz = RGB_TO_XYZ.map(|row| row[0] * rgb[0] + row[1] * rgb[1] + row[2] * rgb[2]); + xyz_to_lab(xyz, D65) +} + +#[cfg(test)] +mod tests; diff --git a/crates/render-parity/src/tests.rs b/crates/render-parity/src/tests.rs new file mode 100644 index 000000000..3649c37d3 --- /dev/null +++ b/crates/render-parity/src/tests.rs @@ -0,0 +1,89 @@ +use super::*; + +fn gray_ramp(width: u32, height: u32) -> Vec { + (0..width * height).flat_map(|i| [i as f32 / (width * height) as f32; 3]).collect() +} + +#[test] +fn identical_renders_are_within_any_tolerance() { + let rgb = gray_ramp(4, 3); + let image = Image::new(4, 3, &rgb).unwrap(); + let report = compare(&image, &image).unwrap(); + assert_eq!((report.pixels, report.max_delta_e, report.max_channel, report.worst), (12, 0.0, 0.0, None)); + assert!(report.within(&Tolerance { max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0 })); + assert_eq!(report.to_string(), "12 px: ΔE2000 max 0.0000 mean 0.0000, channel max 0.000000, identical"); +} + +#[test] +fn white_in_the_matrix_is_neutral() { + // RGB (1, 1, 1) is the D65 white: L* 100, a* = b* = 0 + let [l, a, b] = lab([1.0; 3]); + assert!((l - 100.0).abs() < 1e-9 && a.abs() < 1e-9 && b.abs() < 1e-9, "{l} {a} {b}"); +} + +#[test] +fn the_worst_pixel_and_the_means_are_reported() { + let expected = gray_ramp(3, 2); + let mut actual = expected.clone(); + // pixel (1, 1): red up by 0.1, a visible difference + actual[4 * 3] += 0.1; + let report = compare(&Image::new(3, 2, &expected).unwrap(), &Image::new(3, 2, &actual).unwrap()).unwrap(); + assert_eq!(report.worst, Some((1, 1))); + assert!(report.max_delta_e > 1.0, "{report}"); + assert!((report.mean_delta_e - report.max_delta_e / 6.0).abs() < 1e-9); + assert!((report.max_channel - 0.1).abs() < 1e-6); + assert!((report.mean_channel[0] - 0.1 / 6.0).abs() < 1e-6 && report.mean_channel[1] == 0.0); + let over = report.exceeded(&Tolerance::BASELINE); + assert_eq!(over.len(), 3, "{over:?}"); + assert!(over[0].starts_with("max ΔE2000") && over[1].starts_with("mean ΔE2000") && over[2].starts_with("max channel")); + assert!(!report.within(&Tolerance::default())); + assert!(report.to_string().ends_with("worst at (1, 1)")); +} + +#[test] +fn a_tenth_of_a_10_bit_code_value_passes_the_baseline() { + let expected = vec![0.18f32; 2 * 2 * 3]; + let actual: Vec = expected.iter().map(|v| v + 0.1 / 1024.0).collect(); + let report = compare(&Image::new(2, 2, &expected).unwrap(), &Image::new(2, 2, &actual).unwrap()).unwrap(); + assert!(report.within(&Tolerance::BASELINE), "{report}"); + assert!(report.worst.is_some()); +} + +#[test] +fn a_channel_difference_too_small_for_cielab_still_names_a_pixel() { + let expected = vec![0.5f32; 6]; + let mut actual = expected.clone(); + actual[5] = f32::from_bits(0.5f32.to_bits() + 1); + let report = compare(&Image::new(2, 1, &expected).unwrap(), &Image::new(2, 1, &actual).unwrap()).unwrap(); + assert!(report.max_channel > 0.0); + assert_eq!(report.worst, Some((1, 0))); +} + +#[test] +fn an_empty_render_compares_as_identical() { + let image = Image::new(0, 5, &[]).unwrap(); + let report = compare(&image, &image).unwrap(); + assert_eq!((report.pixels, report.mean_delta_e, report.worst), (0, 0.0, None)); + assert_eq!((image.width(), image.height()), (0, 5)); +} + +#[test] +fn bad_inputs_are_errors_not_panics() { + let e = Image::new(2, 2, &[0.0; 11]).unwrap_err(); + assert_eq!(e, ParityError::Length { width: 2, height: 2, samples: 11 }); + assert_eq!(e.to_string(), "11 samples is not RGB for 2 × 2 pixels"); + assert!(Image::new(u32::MAX, u32::MAX, &[]).is_err()); + + let a = vec![0.0f32; 6]; + let b = vec![0.0f32; 6]; + let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(1, 2, &b).unwrap()).unwrap_err(); + assert_eq!(e.to_string(), "render is 1 × 2, expected 2 × 1"); + + let mut nan = a.clone(); + nan[4] = f32::NAN; + let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(2, 1, &nan).unwrap()).unwrap_err(); + assert_eq!(e, ParityError::NotFinite { x: 1, y: 0 }); + assert_eq!(e.to_string(), "pixel (1, 0) is not a finite colour"); + let e: Box = Box::new(e); + assert!(e.source().is_none()); +} From 69d26616e0881d90ad5132b225e76d6faa86b35a Mon Sep 17 00:00:00 2001 From: tsw2k Date: Mon, 16 Nov 2026 11:34:11 +0000 Subject: [PATCH 11/27] craft-rtf: tables keep their column grid on odd rows Moves storytold/wordcraft#4595 into craft-rtf, where WordCraft's RTF reader now lives (#778). column_grid dropped the whole grid, so every spanning cell fell back to one of N equal columns, when a table had more than 63 distinct cell edges, a zero-width cell, a cell without \cellx, or two edges under a point apart (which the document model rejects as a column width). Forms exported from spreadsheets, with many narrow columns, hit this: a report title wrapped a few words per line in a quarter of the page and the document grew to hundreds of pages. Now the narrowest column merges into the next, one at a time, while there are over 63 columns, a column of a point or less, or a column under 6 pt with more columns left than the widest row has cells; each cell edge snaps to the nearest grid edge, and a zero-width or undefined cell takes a column in its own row only. --- crates/rtf/src/doc/parse.rs | 140 +++++++++++++++++++++++++++++++----- crates/rtf/src/doc/tests.rs | 135 ++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 19 deletions(-) diff --git a/crates/rtf/src/doc/parse.rs b/crates/rtf/src/doc/parse.rs index 60a65740a..f44d666ce 100644 --- a/crates/rtf/src/doc/parse.rs +++ b/crates/rtf/src/doc/parse.rs @@ -1,3 +1,6 @@ +use std::cmp::Reverse; +use std::collections::BinaryHeap; + use super::reader::Reader; use super::state::{CommentData, Dest, HeaderFooter, HfPages, MARK_CLOSE, MARK_OPEN, MAX_PICT, NoteData, PageGeom}; use super::{Arc, Block, Blocks, Cell, Document, FBlock, Flow, HashMap, InlineObject, Para, Paragraph, model}; @@ -262,43 +265,142 @@ fn finalize(mut r: Reader) -> Parsed { Parsed { flow: Flow { blocks, meta: r.meta }, page: r.page, last_section, default_font, headers_footers, notes: r.notes, comments: r.comments } } +/// Most grid columns a table gets (the document model's own cap). +const MAX_GRID_COLS: usize = 63; +/// Grid columns under this width, in twips (6 pt), merge while the rows leave room: import widens +/// any narrower column to 6 pt, which grows the table, and files exported from spreadsheets put +/// edges a few twips apart. +const MIN_GRID_GAP: i64 = 120; +/// Grid columns of this width or less, in twips (1 pt), always merge: the document model refuses +/// them, and would then give the table equal columns. +const NO_GRID_GAP: i64 = 20; + /// The column grid of a table from its rows' cell right edges: every distinct edge starts a grid /// column, and each cell spans the grid columns between its left and right edges, as Word lays /// out RTF rows whose cells don't line up. Returns the widths in points, or none (equal columns) -/// when the edges don't make a grid. +/// when no grid fits, which only a row of more cells than grid columns can cause. +/// +/// One odd row never costs the whole table its grid (a form exported from a spreadsheet, with +/// many narrow columns, otherwise had its wide title squeezed into one of four equal columns): +/// - Over [`MAX_GRID_COLS`] edges, or edges closer than [`MIN_GRID_GAP`], merge, leaving a column +/// for each cell of the widest row ([`coarse_edges`]); each cell edge moves to the nearest +/// grid edge. +/// - A cell that ends at or before the previous one (a zero-width cell, or a cell with no +/// `\cellx` of its own) still gets a grid column of its own; cells pushed past the table's +/// right edge move back. pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec { + if rights.len() != rows.len() { + return Vec::new(); + } let mut edges: Vec = rights.iter().flatten().copied().filter(|&x| x > 0).collect(); edges.sort_unstable(); edges.dedup(); - let ok = rights.len() == rows.len() - && rows - .iter() - .zip(rights) - .all(|(r, x)| r.len() == x.len() && x.windows(2).all(|w| w.first() < w.get(1)) && x.first().is_some_and(|&v| v > 0)) - && !edges.is_empty() - && edges.len() <= 63; - if !ok { + if edges.is_empty() { return Vec::new(); } - for (row, x) in rows.iter_mut().zip(rights) { + let grid = coarse_edges(&edges, rows.iter().map(Vec::len).max().unwrap_or(0)); + // Each cell's right edge as a grid edge number (1 = the first grid column's right edge). + let mut ends: Vec> = Vec::with_capacity(rows.len()); + for (row, x) in rows.iter().zip(rights) { + if row.len() > grid.len() { + return Vec::new(); + } + let mut at: Vec = Vec::with_capacity(row.len()); + let mut prev = 0; + for i in 0..row.len() { + let end = x.get(i).map_or(0, |&r| nearest_edge(&grid, r)).max(prev + 1); + at.push(end); + prev = end; + } + // Cells pushed past the right edge move back, each keeping a column of its own. + let mut limit = grid.len(); + for end in at.iter_mut().rev() { + *end = (*end).min(limit); + limit = end.saturating_sub(1); + } + ends.push(at); + } + for (row, at) in rows.iter_mut().zip(&ends) { let mut left = 0; - for (c, &right) in row.iter_mut().zip(x) { - let span = edges.iter().filter(|&&e| e > left && e <= right).count().max(1); - c.colspan = span as u32; - left = right; + for (c, &end) in row.iter_mut().zip(at) { + c.colspan = u32::try_from(end.saturating_sub(left)).unwrap_or(1).max(1); + left = end; } } - let mut prev = 0; - edges - .iter() + let mut prev = 0i64; + grid.iter() .map(|&e| { - let w = (e - prev) as f32 / 20.0; - prev = e; + let w = (i64::from(e) - prev) as f32 / 20.0; + prev = i64::from(e); w }) .collect() } +/// The grid edges kept from the sorted, distinct, positive `edges`. The narrowest column merges +/// into the next, one at a time, so an edge moves only when it has to: while there are over +/// [`MAX_GRID_COLS`] columns, or columns of [`NO_GRID_GAP`] or less, and while a column is under +/// [`MIN_GRID_GAP`] and more columns remain than the `widest_row`'s cells. The table's right edge +/// is always kept. +fn coarse_edges(edges: &[i32], widest_row: usize) -> Vec { + let n = edges.len(); + // Each edge's neighbours among the edges still kept. + let mut left: Vec> = (0..n).map(|i| i.checked_sub(1)).collect(); + let mut right: Vec> = (1..=n).map(|i| (i < n).then_some(i)).collect(); + let mut kept = vec![true; n]; + let at = |i: Option| i.and_then(|i| edges.get(i)).map_or(0, |&e| i64::from(e)); + let width = |i: usize, left: &[Option]| at(Some(i)) - at(left.get(i).copied().flatten()); + let mut narrowest: BinaryHeap> = (0..n).map(|i| Reverse((width(i, &left), i))).collect(); + let mut count = n; + while count > 1 { + let Some(Reverse((w, i))) = narrowest.pop() else { + break; + }; + // A column that has since grown, or gone, has a fresher entry or none. + if !kept.get(i).copied().unwrap_or(false) || w != width(i, &left) { + continue; + } + if count <= MAX_GRID_COLS && w > NO_GRID_GAP && (w >= MIN_GRID_GAP || count <= widest_row) { + break; + } + // The column's right edge goes, unless it is the table's: then its left edge does. + let last = right.get(i).copied().flatten().is_none(); + let Some(gone) = (if last { left.get(i).copied().flatten() } else { Some(i) }) else { + break; + }; + let before = left.get(gone).copied().flatten(); + let after = right.get(gone).copied().flatten(); + if let Some(slot) = before.and_then(|b| right.get_mut(b)) { + *slot = after; + } + if let Some(a) = after { + if let Some(slot) = left.get_mut(a) { + *slot = before; + } + narrowest.push(Reverse((width(a, &left), a))); + } + if let Some(slot) = kept.get_mut(gone) { + *slot = false; + } + count -= 1; + } + edges.iter().zip(&kept).filter(|&(_, &k)| k).map(|(&e, _)| e).collect() +} + +/// The number of the grid edge nearest `x` (1-based; 0 is the table's left edge). +fn nearest_edge(grid: &[i32], x: i32) -> usize { + if x <= 0 { + return 0; + } + let p = grid.partition_point(|&e| e < x); + let below = if p == 0 { 0 } else { grid.get(p - 1).copied().unwrap_or(0) }; + match grid.get(p) { + Some(&above) if i64::from(above) - i64::from(x) <= i64::from(x) - i64::from(below) => p + 1, + Some(_) => p, + None => grid.len(), + } +} + /// Parse RTF bytes into a document. pub fn import(bytes: &[u8]) -> Result { model::begin_import(); diff --git a/crates/rtf/src/doc/tests.rs b/crates/rtf/src/doc/tests.rs index 2a4103eb1..fda05f3b4 100644 --- a/crates/rtf/src/doc/tests.rs +++ b/crates/rtf/src/doc/tests.rs @@ -460,6 +460,141 @@ fn table_grid_from_cell_edges_and_borders() { assert_eq!(t.grid, [240.9, 241.0]); } +/// The table of `rtf`'s flow and every row's column spans; checks the spans fit the grid. +fn grid_of(rtf: &[u8]) -> (Vec, Vec>) { + let f = parse(rtf).unwrap(); + let FBlock::Table(t) = &f.blocks[0] else { panic!("no table") }; + let spans: Vec> = t.rows.iter().map(|r| r.iter().map(|c| c.colspan).collect()).collect(); + assert!(!t.widths.is_empty(), "the table kept no grid"); + assert_eq!(t.cols(), t.widths.len(), "spans {spans:?} don't fit {} grid columns", t.widths.len()); + assert!(t.widths.iter().all(|&w| w > 1.0), "a grid column under a point: {:?}", t.widths); + (t.widths.clone(), spans) +} + +/// Width in points of the first `n` grid columns. +fn first_cols(widths: &[f32], n: u32) -> f32 { + widths.iter().take(n as usize).sum() +} + +#[test] +fn table_grid_survives_more_than_63_edges() { + // A form exported from a spreadsheet: a wide title over rows whose middle edge moves by + // 100 twips, 80 distinct edges in all. The grid used to be dropped, giving equal columns. + let mut rtf = String::from(r"{\rtf1\ansi\trowd\cellx6000\cellx8000\pard\intbl Title\cell Codes\cell\row"); + for k in 1..80 { + rtf.push_str(&format!(r"\trowd\cellx{}\cellx8000\pard\intbl a\cell b\cell\row", k * 100)); + } + rtf.push_str(r"\pard\par}"); + let (widths, spans) = grid_of(rtf.as_bytes()); + assert!(widths.len() <= 63, "{} grid columns", widths.len()); + assert!((widths.iter().sum::() - 400.0).abs() < 0.01, "the table keeps its 400 pt"); + // The title keeps its 300 pt, within a grid column's merge. + let title = first_cols(&widths, spans[0][0]); + assert!((title - 300.0).abs() <= 5.0, "title {title} pt"); + // The staircase still climbs: the 60th row's first cell is about 300 pt wide too. + assert!((first_cols(&widths, spans[60][0]) - 300.0).abs() <= 5.0); + + // The same through import: the document table has the grid, not equal columns. + let d = import(rtf.as_bytes()).unwrap(); + let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); + assert_eq!(t.grid.len(), widths.len()); + let w = t.rows[0].cells[0].props.width.unwrap(); + assert!((w - 300.0).abs() <= 5.0, "title cell {w} pt"); +} + +#[test] +fn table_grid_survives_a_zero_width_cell_and_a_missing_cellx() { + // A zero-width cell (two edges at 3000) in one row: the other rows keep their grid. + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx6000\cellx9000 \pard\intbl Title\cell b\cell\row +\trowd\cellx2000\cellx2000\cellx9000 \pard\intbl a\cell\cell c\cell\row +\pard\par}", + ); + assert_eq!(first_cols(&widths, spans[0][0]), 300.0); + assert_eq!(spans[1].len(), 3, "the zero-width cell is kept: {spans:?}"); + assert_eq!(first_cols(&widths, spans[1][0]), 100.0, "the cell before it keeps its width"); + + // A row with a third cell but only two `\cellx`. + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx6000\cellx9000 \pard\intbl Title\cell b\cell\row +\trowd\cellx2000\cellx6000 \pard\intbl a\cell b\cell extra\cell\row +\pard\par}", + ); + assert_eq!(first_cols(&widths, spans[0][0]), 300.0); + assert_eq!(spans[1].len(), 3); + assert_eq!(first_cols(&widths, spans[1][0]), 100.0); +} + +#[test] +fn table_grid_merges_edges_a_few_twips_apart() { + // Two rows that mean the same 3000-twip edge, 5 twips apart: one grid column, not one of + // a quarter point (which the document model refuses, so the table got equal columns). + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx3000\cellx9000 \pard\intbl a\cell b\cell\row +\trowd\cellx3005\cellx9000 \pard\intbl c\cell d\cell\row +\pard\par}", + ); + assert_eq!(widths.len(), 2, "{widths:?}"); + assert_eq!(spans, [[1, 1], [1, 1]]); + assert_eq!(widths.iter().sum::(), 450.0); +} + +/// `edges` as one table row, a cell for each. +fn rtf_row(edges: &[usize]) -> String { + let cells: String = edges.iter().map(|e| format!(r"\cellx{e}")).collect(); + format!(r"\trowd{cells}\pard\intbl {}\row", r"x\cell ".repeat(edges.len())) +} + +#[test] +fn table_grid_keeps_a_column_for_each_cell_of_the_widest_row() { + // 64 evenly spaced edges: merging by a common gap halves them to 32 columns, too few for a + // row of 33 or more cells, and the table lost its grid. + let edges: Vec = (1..=64).map(|k| k * 100).collect(); + for cells in [33, 40, 50, 62] { + let wide: Vec = edges.iter().copied().take(cells).chain([6400]).collect(); + let rest: Vec = edges.iter().copied().skip(cells - 1).collect(); + let rtf = format!(r"{{\rtf1\ansi {}{}\pard\par}}", rtf_row(&wide), rtf_row(&rest)); + let (widths, spans) = grid_of(rtf.as_bytes()); + assert!((cells + 1..=63).contains(&widths.len()), "{cells} cells, {} grid columns", widths.len()); + assert_eq!(spans[0].len(), cells + 1, "every cell is kept"); + assert_eq!(widths.iter().sum::(), 320.0, "the table keeps its 320 pt"); + } +} + +#[test] +fn table_grid_merges_columns_import_would_widen() { + // Columns under 6 pt that no row needs: import raises each one to 6 pt, so this table of + // 3 pt columns doubled in width. Merged, it keeps its 60 pt. + let mut rtf = String::from(r"{\rtf1\ansi "); + for k in 1..=20 { + rtf.push_str(&rtf_row(&[k * 60, 1200])); + } + rtf.push_str(r"\pard\par}"); + let (widths, _) = grid_of(rtf.as_bytes()); + assert!(widths.iter().all(|&w| w >= 6.0), "{widths:?}"); + let d = import(rtf.as_bytes()).unwrap(); + let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); + assert_eq!(t.grid.iter().sum::(), 60.0, "{:?}", t.grid); + + // A row that needs them keeps them, narrow as they are. + let (widths, spans) = grid_of(format!(r"{{\rtf1\ansi {}\pard\par}}", rtf_row(&[60, 120, 180])).as_bytes()); + assert_eq!(widths, [3.0, 3.0, 3.0]); + assert_eq!(spans, [[1, 1, 1]]); +} + +#[test] +fn table_grid_never_keeps_a_column_the_document_model_refuses() { + // Three cells within a point of each other: too narrow to keep even for a row that needs + // them. The row can't fit, so the table has no grid, and nothing panics. + let f = parse(format!(r"{{\rtf1\ansi {}\pard\par}}", rtf_row(&[5, 10, 15])).as_bytes()).unwrap(); + let FBlock::Table(t) = &f.blocks[0] else { panic!("no table") }; + assert!(t.widths.is_empty(), "{:?}", t.widths); + assert_eq!(t.rows[0].len(), 3); +} + #[test] fn code_pages_from_ansicpg_and_font_charsets() { // The document's code page: Russian RTF as accounting systems write it. From fe68ce8c51d770a9586cfe6939ee2a28f2ecbc17 Mon Sep 17 00:00:00 2001 From: tsw2k Date: Mon, 16 Nov 2026 11:40:14 +0000 Subject: [PATCH 12/27] craft-flow: narrow table columns keep their width on import Moves the import part of storytold/wordcraft#4964 into craft-flow, where WordCraft's RTF and ODT import now builds its tables (#778). Import raised every column under 6 pt to 6 pt, so a form with many narrow columns came out wider than its source. Columns over 1 pt now keep the width the file gives; one of a point or less, none or not a number still gives the table equal columns, so nothing below a point, negative or NaN reaches layout. craft-rtf merged grid columns under 6 pt that no row needed, because import would widen them. With import keeping them, that merge only moved cell edges, so it goes: grid columns merge only while there are over 63 or one is a point or less. Stacked on the craft-rtf column grid change. --- crates/flow/src/to_doc.rs | 3 ++- crates/odf/src/text/tests.rs | 35 +++++++++++++++++++++++++++++++++++ crates/rtf/src/doc/parse.rs | 21 ++++++++------------- crates/rtf/src/doc/tests.rs | 28 ++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 18 deletions(-) diff --git a/crates/flow/src/to_doc.rs b/crates/flow/src/to_doc.rs index ce29a4448..0375c448c 100644 --- a/crates/flow/src/to_doc.rs +++ b/crates/flow/src/to_doc.rs @@ -841,7 +841,8 @@ impl<'a> Builder<'a> { tb.props.style = None; } if t.widths.len() == cols && t.widths.iter().all(|w| w.is_finite() && *w > 1.0) { - tb.grid = t.widths.iter().map(|w| w.clamp(6.0, 1584.0)).collect(); + // Narrow columns stay as they are: raising them would widen the table past its source. + tb.grid = t.widths.iter().map(|w| w.min(1584.0)).collect(); } tb.rows.clear(); if let Some(x) = &t.extra { diff --git a/crates/odf/src/text/tests.rs b/crates/odf/src/text/tests.rs index 8d7af18dc..da00b350c 100644 --- a/crates/odf/src/text/tests.rs +++ b/crates/odf/src/text/tests.rs @@ -442,6 +442,41 @@ fn table_column_widths_survive_a_round_trip() { assert_eq!(grid(&back), widths); } +#[test] +fn narrow_table_columns_keep_their_width() { + // Columns of a few points stay as the file gives them, so the table stays as wide as its source. + let content = format!( + r##"AB"## + ); + let doc = import(&package(&content, None)).unwrap(); + let grid_of = |doc: &craft_wordmodel::Document| { + doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }) + }; + assert_eq!(grid_of(&doc), Some(vec![3.0, 100.0])); + // Saved and opened again, the narrow column is still the file's width. + let back = import(&export(&doc).unwrap()).unwrap(); + let grid = grid_of(&back).unwrap(); + assert_eq!(grid.len(), 2, "{grid:?}"); + assert!((grid[0] - 3.0).abs() < 0.01 && (grid[1] - 100.0).abs() < 0.01, "{grid:?}"); +} + +#[test] +fn table_columns_of_a_point_or_less_give_equal_columns() { + // A column of a point or less, nothing or NaN (a damaged or hostile file) is no width to + // lay out: the whole grid falls back to equal columns, and nothing reaches layout below a + // point, negative or not a number. + for bad in ["1pt", "0.5pt", "0pt", "-3pt", "NaNpt"] { + let content = format!( + r##"AB"## + ); + let doc = import(&package(&content, None)).unwrap(); + let grid = doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }).unwrap(); + assert_eq!(grid.len(), 2, "{bad}: {grid:?}"); + assert!(grid.iter().all(|w| w.is_finite() && *w > 1.0), "{bad}: {grid:?}"); + assert_eq!(grid[0], grid[1], "{bad}: equal columns, {grid:?}"); + } +} + #[test] fn table_without_column_widths_keeps_the_equal_split() { let content = format!( diff --git a/crates/rtf/src/doc/parse.rs b/crates/rtf/src/doc/parse.rs index f44d666ce..01978da1e 100644 --- a/crates/rtf/src/doc/parse.rs +++ b/crates/rtf/src/doc/parse.rs @@ -267,10 +267,6 @@ fn finalize(mut r: Reader) -> Parsed { /// Most grid columns a table gets (the document model's own cap). const MAX_GRID_COLS: usize = 63; -/// Grid columns under this width, in twips (6 pt), merge while the rows leave room: import widens -/// any narrower column to 6 pt, which grows the table, and files exported from spreadsheets put -/// edges a few twips apart. -const MIN_GRID_GAP: i64 = 120; /// Grid columns of this width or less, in twips (1 pt), always merge: the document model refuses /// them, and would then give the table equal columns. const NO_GRID_GAP: i64 = 20; @@ -282,9 +278,9 @@ const NO_GRID_GAP: i64 = 20; /// /// One odd row never costs the whole table its grid (a form exported from a spreadsheet, with /// many narrow columns, otherwise had its wide title squeezed into one of four equal columns): -/// - Over [`MAX_GRID_COLS`] edges, or edges closer than [`MIN_GRID_GAP`], merge, leaving a column -/// for each cell of the widest row ([`coarse_edges`]); each cell edge moves to the nearest -/// grid edge. +/// - Over [`MAX_GRID_COLS`] edges, or edges [`NO_GRID_GAP`] or less apart, merge +/// ([`coarse_edges`]); each cell edge moves to the nearest grid edge. Narrow columns over a +/// point stay: import keeps them as narrow as the file gives them. /// - A cell that ends at or before the previous one (a zero-width cell, or a cell with no /// `\cellx` of its own) still gets a grid column of its own; cells pushed past the table's /// right edge move back. @@ -298,7 +294,7 @@ pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec> = Vec::with_capacity(rows.len()); for (row, x) in rows.iter().zip(rights) { @@ -339,10 +335,9 @@ pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec Vec { +/// [`MAX_GRID_COLS`] columns, or columns of [`NO_GRID_GAP`] or less. The table's right edge is +/// always kept. +fn coarse_edges(edges: &[i32]) -> Vec { let n = edges.len(); // Each edge's neighbours among the edges still kept. let mut left: Vec> = (0..n).map(|i| i.checked_sub(1)).collect(); @@ -360,7 +355,7 @@ fn coarse_edges(edges: &[i32], widest_row: usize) -> Vec { if !kept.get(i).copied().unwrap_or(false) || w != width(i, &left) { continue; } - if count <= MAX_GRID_COLS && w > NO_GRID_GAP && (w >= MIN_GRID_GAP || count <= widest_row) { + if count <= MAX_GRID_COLS && w > NO_GRID_GAP { break; } // The column's right edge goes, unless it is the table's: then its left edge does. diff --git a/crates/rtf/src/doc/tests.rs b/crates/rtf/src/doc/tests.rs index fda05f3b4..fddd198da 100644 --- a/crates/rtf/src/doc/tests.rs +++ b/crates/rtf/src/doc/tests.rs @@ -389,6 +389,26 @@ fn page_size_and_margins() { assert_eq!(d.last_section, craft_wordmodel::SectionProps::default()); } +#[test] +fn narrow_grid_columns_keep_their_width() { + // Rows' edges a few points apart, as in a form exported from a spreadsheet. The table stays as + // wide as its source, through an export and back too. + let rtf = br"{\rtf1\ansi +\trowd\cellx400\cellx800 \pard\intbl a\cell b\cell\row +\trowd\cellx80\cellx480\cellx800 \pard\intbl c\cell d\cell e\cell\row +\pard after\par}"; + let table = |doc: &Document| match doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.clone()) } else { None }) + { + Some(t) => t, + None => panic!("no table"), + }; + let doc = import(rtf).unwrap(); + let t = table(&doc); + assert_eq!(t.grid, [4.0, 16.0, 4.0, 16.0]); + assert_eq!(t.rows[0].cells.iter().map(|c| c.props.width).collect::>(), [Some(20.0), Some(20.0)]); + assert_eq!(table(&import(export(&doc).as_bytes()).unwrap()).grid, t.grid); +} + #[test] fn table_grid_from_cell_edges_and_borders() { // Rows whose cells don't line up share one grid: edges 2000, 3000, 9000. @@ -565,16 +585,16 @@ fn table_grid_keeps_a_column_for_each_cell_of_the_widest_row() { } #[test] -fn table_grid_merges_columns_import_would_widen() { - // Columns under 6 pt that no row needs: import raises each one to 6 pt, so this table of - // 3 pt columns doubled in width. Merged, it keeps its 60 pt. +fn table_grid_keeps_narrow_columns_at_their_width() { + // Columns under 6 pt that no row needs keep their 3 pt: import no longer widens them, so the + // table keeps its 60 pt. let mut rtf = String::from(r"{\rtf1\ansi "); for k in 1..=20 { rtf.push_str(&rtf_row(&[k * 60, 1200])); } rtf.push_str(r"\pard\par}"); let (widths, _) = grid_of(rtf.as_bytes()); - assert!(widths.iter().all(|&w| w >= 6.0), "{widths:?}"); + assert_eq!(widths, [3.0; 20], "{widths:?}"); let d = import(rtf.as_bytes()).unwrap(); let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); assert_eq!(t.grid.iter().sum::(), 60.0, "{:?}", t.grid); From 8b876291a1547ebe57b282b8f86b35644f731496 Mon Sep 17 00:00:00 2001 From: tsw2k Date: Mon, 16 Nov 2026 11:46:35 +0000 Subject: [PATCH 13/27] craft-docx: a frame preset shape is a border with a hole Moves the format part of storytold/wordcraft#6162 into craft-wordmodel, craft-docx, craft-flow and craft-odf, where WordCraft's DOCX and ODT I/O now live (#778). A DrawingML `prstGeom frame` (a letterhead's page border) was read as a rectangle, so its fill covered the whole page behind the text. It is now ShapeKind::Frame, its border width read from `adj1` (capped, as a rounded rectangle's corner is) and written back as the same preset. connector::frame_width gives the border in points. Formats with no frame shape (ODT and the other flow formats) get the freeform it draws: its outline and, wound the other way, its hole, so the hole stays empty under either fill rule. --- crates/docx/src/read/shape_style.rs | 1 + crates/docx/src/read/story/drawing.rs | 13 ++++--- crates/docx/src/write/story/drawing.rs | 12 +++++-- crates/docx/tests/it/fixtures.rs | 33 ++++++++++++++++++ crates/flow/src/from_doc.rs | 20 ++++++++++- crates/odf/src/text/shapes.rs | 3 +- crates/odf/src/text/tests_more/shapes.rs | 41 +++++++++++++++++++++++ crates/wordmodel/src/connector.rs | 12 +++++++ crates/wordmodel/src/para/inline_kinds.rs | 3 ++ crates/wordmodel/src/wrap.rs | 1 + 10 files changed, 130 insertions(+), 9 deletions(-) diff --git a/crates/docx/src/read/shape_style.rs b/crates/docx/src/read/shape_style.rs index 9f28ef2ff..983218f9e 100644 --- a/crates/docx/src/read/shape_style.rs +++ b/crates/docx/src/read/shape_style.rs @@ -47,6 +47,7 @@ pub(crate) fn preset_kind(prst: &str) -> ShapeKind { "downArrow" => ShapeKind::ArrowDown, "star4" | "star5" | "star6" | "star7" | "star8" | "star10" | "star12" | "star16" | "star24" | "star32" => ShapeKind::Star, "heart" => ShapeKind::Heart, + "frame" => ShapeKind::Frame, _ => ShapeKind::Rectangle, } } diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..6bc697a3b 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -526,12 +526,17 @@ impl Reader<'_> { outline_fill: paint.outline_fill, ..ShapeExtra::default() }; - // A rounded rectangle's own corner (`adj`, ECMA-376 §20.1.10.56); other presets drawn as - // rounded rectangles keep the default. - if prst == "roundRect" { + // A rounded rectangle's own corner (`adj`, ECMA-376 §20.1.10.56), a frame's own border + // width (`adj1`); other presets drawn as rounded rectangles keep the default. + let adj = match prst { + "roundRect" => Some("adj"), + "frame" => Some("adj1"), + _ => None, + }; + if let Some(adj) = adj { extra.corner = sppr .and_then(|s| s.child("a:prstGeom")) - .and_then(|g| preset_adj(g, "adj")) + .and_then(|g| preset_adj(g, adj)) .and_then(|v| u32::try_from(v.clamp(0, i64::from(craft_wordmodel::connector::MAX_CORNER))).ok()); } // Arrowheads: any end type but `none` (ECMA-376 §20.1.10.33) is drawn as a triangle of its diff --git a/crates/docx/src/write/story/drawing.rs b/crates/docx/src/write/story/drawing.rs index 7c03cca6e..1479d2bd8 100644 --- a/crates/docx/src/write/story/drawing.rs +++ b/crates/docx/src/write/story/drawing.rs @@ -117,16 +117,22 @@ impl Writer<'_> { ShapeKind::ArrowDown => "downArrow", ShapeKind::Star => "star5", ShapeKind::Heart => "heart", + ShapeKind::Frame => "frame", }; match geom { Some(f) => cust_geom(w, f, *sw, *sh), None => { w.open("a:prstGeom", &[("prst", prst)]); - match extra.corner.filter(|_| *kind == ShapeKind::RoundedRectangle) { - Some(c) => { + let adj = match kind { + ShapeKind::RoundedRectangle => Some("adj"), + ShapeKind::Frame => Some("adj1"), + _ => None, + }; + match adj.zip(extra.corner) { + Some((name, c)) => { w.open("a:avLst", &[]); let fmla = format!("val {}", c.min(craft_wordmodel::connector::MAX_CORNER)); - w.empty("a:gd", &[("name", "adj"), ("fmla", &fmla)]); + w.empty("a:gd", &[("name", name), ("fmla", &fmla)]); w.close("a:avLst"); } None => w.empty("a:avLst", &[]), diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 348d6239e..47663008d 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -2729,6 +2729,39 @@ fn rounded_rectangle_corners_are_read_and_saved() { assert_eq!(corners(&craft_docx::read(&bytes).unwrap()), expected); } +/// A `frame` preset (a letterhead's page border) is a frame, not a filled rectangle: its border +/// width is its `adj1`, read, capped and written back as the same preset. +#[test] +fn frame_presets_keep_their_hole_and_border_width() { + let geom = + |av: &str| format!(r#"{av}"#); + let gd = |fmla: &str| format!(r#""#); + let body = [geom(&gd("val 2604")), geom(""), geom(&gd("val 90000")), geom(r#""#)] + .iter() + .map(|sppr| anchored_wsp("Frame", 2540000, 1270000, sppr, "", "")) + .collect::(); + let frames = |d: &Document| -> Vec<(ShapeKind, Option)> { + paras(d)[0] + .objects + .iter() + .filter_map(|o| match o { + InlineObject::Shape { kind, extra, .. } => Some((*kind, extra.corner)), + _ => None, + }) + .collect() + }; + let f = ShapeKind::Frame; + let expected = vec![(f, Some(2604)), (f, None), (f, Some(50_000)), (f, None)]; + let d = read_body(&format!("{body}")); + assert_eq!(frames(&d), expected); + let bytes = craft_docx::write(&d).unwrap(); + let mut xml = String::new(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes.as_slice())).unwrap(); + std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap(); + assert!(xml.contains(r#""#), "{xml}"); + assert_eq!(frames(&craft_docx::read(&bytes).unwrap()), expected); +} + /// #611: legacy VML lines and rectangles (documents without DrawingML). #[test] fn vml_lines_and_rectangles() { diff --git a/crates/flow/src/from_doc.rs b/crates/flow/src/from_doc.rs index 810e67d55..b7204f1bf 100644 --- a/crates/flow/src/from_doc.rs +++ b/crates/flow/src/from_doc.rs @@ -342,6 +342,20 @@ fn rich_inlines(doc: &Document, o: &InlineObject, f: &Fmt, opts: FlowOpts) -> Op }) } +/// The freeform a `w` × `h` frame shape with border `corner` (see +/// [`craft_wordmodel::connector::frame_width`]) draws: its outline clockwise and its hole the +/// other way, so the hole stays empty under either fill rule. +fn frame_freeform(corner: Option, w: f32, h: f32) -> craft_wordmodel::freeform::Freeform { + use craft_wordmodel::freeform::{FreePath, Freeform}; + let (w, h) = (craft_wordmodel::geom::finite(w).max(0.0), craft_wordmodel::geom::finite(h).max(0.0)); + let t = craft_wordmodel::connector::frame_width(corner, w, h); + let mut paths = vec![FreePath { pts: vec![[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]], closed: true }]; + if w - 2.0 * t > 0.0 && h - 2.0 * t > 0.0 { + paths.push(FreePath { pts: vec![[t, t], [t, h - t], [w - t, h - t], [w - t, t]], closed: true }); + } + Freeform { w, h, paths, ..Default::default() } +} + /// Comment `c` (a reply to `parent`) of `doc` as a flow comment: its text holds no comments or /// notes of its own. /// The flow inline for picture, shape or group `o` of `doc` (a member of groups `depth` deep), @@ -358,8 +372,12 @@ fn drawing_inline(doc: &Document, o: &InlineObject, opts: FlowOpts, depth: usize { // A connector stays one in formats that glue them (ODT, with boxes); elsewhere it is a // straight line. + // A frame (a border with a hole), which the other formats have no shape for, is the + // freeform it draws: its outline and, wound the other way, its hole. + let frame = (*kind == ShapeKind::Frame).then(|| std::sync::Arc::new(frame_freeform(extra.corner, *w, *h))); let kind = match kind { ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector if !opts.boxes => ShapeKind::Line, + ShapeKind::Frame => ShapeKind::Freeform, k => *k, }; let inner = FlowOpts { notes: false, comments: false, boxes: false, ..opts }; @@ -383,7 +401,7 @@ fn drawing_inline(doc: &Document, o: &InlineObject, opts: FlowOpts, depth: usize Box::new(crate::Chain { name: format!("Frame{id}"), next }) }), }; - let path = freeform.clone().filter(|_| kind == ShapeKind::Freeform); + let path = frame.or_else(|| freeform.clone()).filter(|_| kind == ShapeKind::Freeform); Some(Inline::Shape { kind, w: w * sx, h: h * sy, float: kept_float(float), blocks, look, path }) } InlineObject::Image { media, w, h, alt, float, ole, crop, .. } => { diff --git a/crates/odf/src/text/shapes.rs b/crates/odf/src/text/shapes.rs index 7c08151aa..eb184f5c9 100644 --- a/crates/odf/src/text/shapes.rs +++ b/crates/odf/src/text/shapes.rs @@ -124,7 +124,8 @@ pub(super) fn element_of(kind: ShapeKind) -> (&'static str, Option<&'static str> ShapeKind::ArrowDown => custom("down-arrow"), ShapeKind::Star => custom("star5"), ShapeKind::Heart => custom("heart"), - ShapeKind::Freeform => ("draw:path", None), + // A frame reaches the writer as the freeform it draws (see `model::from_doc`). + ShapeKind::Freeform | ShapeKind::Frame => ("draw:path", None), ShapeKind::Rectangle | ShapeKind::TextBox => ("draw:rect", None), } } diff --git a/crates/odf/src/text/tests_more/shapes.rs b/crates/odf/src/text/tests_more/shapes.rs index f6bbe9ae8..8d1a9d5fd 100644 --- a/crates/odf/src/text/tests_more/shapes.rs +++ b/crates/odf/src/text/tests_more/shapes.rs @@ -188,3 +188,44 @@ fn a_shape_with_only_an_empty_paragraph_has_no_text() { let stories: Vec = d.body.iter().filter_map(|b| b.as_para()).flat_map(|p| p.objects.iter()).map(|o| o.text_box().is_some()).collect(); assert_eq!(stories, [false, false, true, false, false, false]); } + +/// A frame shape (a letterhead's page border, from DOCX `frame`), which ODT has no shape for, +/// is written as the outline it draws: its outer edge and its hole, so LibreOffice doesn't fill +/// the whole box; read back, it is that freeform, hole and all. +#[test] +fn a_frame_shape_is_written_as_a_border_with_a_hole() { + let mut p = Paragraph::new(); + p.text.push(OBJ); + p.objects.push(InlineObject::Shape { + kind: ShapeKind::Frame, + w: 200.0, + h: 100.0, + fill: Some(craft_wordmodel::fill::Fill::solid(Rgb(1, 2, 3))), + stroke: None, + stroke_width: 0.0, + float: Default::default(), + story: None, + freeform: None, + effects: Default::default(), + // A border a tenth of the shorter side: 10 pt. + extra: craft_wordmodel::connector::ShapeExtra { corner: Some(10_000), ..Default::default() }, + }); + p.runs = vec![craft_wordmodel::para::Run { len: p.text.len(), props: Default::default() }]; + let mut d = Document::new(); + d.body = vec![para_block(p)]; + let odt = export(&d).unwrap(); + let content = content_of(&odt); + assert!(content.contains("= 9.5 * sx && *x <= 190.5 * sx && *y >= 9.5 * sy && *y <= 90.5 * sy), "{hole:?}"); +} diff --git a/crates/wordmodel/src/connector.rs b/crates/wordmodel/src/connector.rs index 366679070..2233b67f6 100644 --- a/crates/wordmodel/src/connector.rs +++ b/crates/wordmodel/src/connector.rs @@ -201,6 +201,8 @@ pub struct ShapeExtra { pub end_style: ArrowStyle, /// A rounded rectangle's corner radius (`roundRect`'s `adj`, ECMA-376 §20.1.10.56): in /// 100000ths of the shorter side, at most [`MAX_CORNER`]; `None` draws [`DEFAULT_CORNER`]. + /// A [`ShapeKind::Frame`]'s border width (`frame`'s `adj1`), in the same units; `None` + /// draws [`DEFAULT_FRAME`]. #[serde(skip_serializing_if = "Option::is_none")] pub corner: Option, /// The outline dash pattern; absent in older saved documents means solid. @@ -273,6 +275,16 @@ pub const MAX_CORNER: u32 = 50_000; /// A rounded rectangle's corner radius without its own: 16% of the shorter side. pub const DEFAULT_CORNER: f32 = 0.16; +/// A frame's border width without its own: `frame`'s default `adj1`, 12.5% of the shorter side. +pub const DEFAULT_FRAME: f32 = 0.125; + +/// The border width, in points, of a `w` × `h` [`ShapeKind::Frame`] with `corner` (see +/// [`ShapeExtra::corner`]): at most half the shorter side, which leaves no hole. +pub fn frame_width(corner: Option, w: f32, h: f32) -> f32 { + let side = crate::geom::finite(w).min(crate::geom::finite(h)).max(0.0); + side * corner.map_or(DEFAULT_FRAME, |c| c.min(MAX_CORNER) as f32 / 100_000.0) +} + /// The corner radius, in points, of a `w` × `h` rounded rectangle with `corner` (see /// [`ShapeExtra::corner`]). pub fn corner_radius(corner: Option, w: f32, h: f32) -> f32 { diff --git a/crates/wordmodel/src/para/inline_kinds.rs b/crates/wordmodel/src/para/inline_kinds.rs index 686089f7f..c36bd8fe6 100644 --- a/crates/wordmodel/src/para/inline_kinds.rs +++ b/crates/wordmodel/src/para/inline_kinds.rs @@ -24,6 +24,9 @@ pub enum ShapeKind { ArrowDown, Star, Heart, + /// A picture frame (DrawingML's `frame`): a rectangle with a rectangular hole, its border as + /// wide as [`crate::connector::frame_width`] says. + Frame, /// A text box (rectangle with a text story). TextBox, /// A shape drawn point by point, such as ink (its geometry is the shape's `freeform`). diff --git a/crates/wordmodel/src/wrap.rs b/crates/wordmodel/src/wrap.rs index 1c235d73e..9b305fafe 100644 --- a/crates/wordmodel/src/wrap.rs +++ b/crates/wordmodel/src/wrap.rs @@ -70,6 +70,7 @@ pub fn shape_outline(kind: ShapeKind, w: f32, h: f32, corner: Option) -> Op | ShapeKind::TextBox | ShapeKind::Line | ShapeKind::Freeform + | ShapeKind::Frame | ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector => return None, From dd80bb5bc5f09afae4972bf4128d24647841e29e Mon Sep 17 00:00:00 2001 From: FredApps Date: Mon, 16 Nov 2026 13:21:01 +0100 Subject: [PATCH 14/27] Keep cached results for shadowed XLSB name references --- crates/xlsx/src/xlsb.rs | 22 ++++++++++++++++++++- crates/xlsx/tests/xlsb.rs | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index ff54ffcaf..02087ed4a 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -10,7 +10,7 @@ mod formula; mod ftab; mod styles; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use craft_opc::{ContentTypes, Rels}; @@ -433,6 +433,14 @@ fn defined_names( // out is left out too, whatever their order, so a cell using it keeps its saved result. let mut omitted = vec![false; links.names.len()]; let mut kept = vec![]; + // PtgName identifies a record, but a bare workbook name in a sheet-scoped definition + // would instead bind to a same-spelled local name. Without a workbook qualifier, keep + // the cached results rather than silently changing that identity. Index once so hostile + // files with many names and references cannot turn this check into a quadratic scan. + // Ordinary shadows still count if their own formulas are omitted: do not infer a new + // binding from a partially emitted name environment. Synthetic names are not definitions. + let local_names: HashSet<_> = + links.names.iter().filter(|n| !n.future && !formula::is_param(&n.text)).filter_map(|n| n.scope.map(|s| (s, n.text.to_lowercase()))).collect(); for p in pending { let Some(name) = links.names.get(p.index) else { continue }; if name.future || formula::is_param(&name.text) { @@ -474,6 +482,18 @@ fn defined_names( continue; } }; + if name.scope.is_some_and(|scope| { + used.iter().filter_map(|&i| links.names.get(i)).any(|target| { + target.scope.is_none() + && !target.future + && !formula::is_param(&target.text) + && local_names.contains(&(scope, target.text.to_lowercase())) + }) + }) { + warnings.warn("XLSB: some defined names were left out (workbook names shadowed by local names)"); + omit(&mut omitted, p.index); + continue; + } if !budget.try_formula_text(formula.len(), limits) { warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); omit(&mut omitted, p.index); diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 6b403bfe8..96c7cff5d 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -1768,3 +1768,44 @@ fn empty_ordinary_names_and_their_dependants_keep_cached_values() { assert!(book.warnings.iter().any(|w| w.contains("empty formula")), "{:?}", book.warnings); assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); } + +#[test] +fn scoped_names_do_not_rebind_shadowed_workbook_names() { + let token = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + let list = vec![ + brt_name(0, 0, WORKBOOK, "Rate", &[0x1E, 1, 0], None), // 1 + brt_name(0, 0, 1, "rAtE", &[0x1E, 2, 0], None), // 2: Hidden shadows Rate case-insensitively. + brt_name(0, 0, 1, "Alias", &token(1), None), // 3: must not become Hidden!rAtE. + brt_name(0, 0, 1, "Forward", &token(5), None), // 4 + brt_name(0, 0, 1, "Dependent", &token(3), None), // 5 + brt_name(0, 0, 0, "Fine", &token(1), None), // 6: no shadow on Data. + brt_name(0, 0, 1, "LocalAlias", &token(2), None), // 7: intentionally uses Hidden!rAtE. + brt_name(0, 0, WORKBOOK, "Other", &[0x1E, 3, 0], None), // 8 + brt_name(0, 0, 1, "Other", &[], None), // 9: malformed, but still an ordinary source shadow. + brt_name(0, 0, 1, "Conservative", &token(8), None), // 10 + brt_name(0, 0, WORKBOOK, "Synthetic", &[0x1E, 4, 0], None), // 11 + brt_name(0x0002_0000, 0, 1, "Synthetic", &[0x1C, 0x1D], None), // 12: future-function metadata. + brt_name(0, 0, 1, "SyntheticControl", &token(11), None), // 13: synthetic name is not a local definition. + ]; + let mut data = row(0); + for (col, index) in [3u32, 4, 5, 6, 10, 13].into_iter().enumerate() { + // PtgNameX qualifies these names on their own sheets. + let formula = [&[0x39u8, 0, 0][..], &index.to_le_bytes()].concat(); + data.extend(formula_cell(col as u32 + 1, &formula, &[])); + } + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"12RaterAtE34Synthetic"#; + assert_eq!(wb.defined_names, xml_names(xml)); + let book = read(&bytes).unwrap(); + for col in 1..=3 { + assert_eq!(book.sheets[0].formula(CellRef::new(0, col)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, col)), Value::Number(0.0)); + } + assert_eq!(book.sheets[0].formula(CellRef::new(0, 4)), Some("Data!Fine")); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 5)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, 5)), Value::Number(0.0)); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 6)), Some("Hidden!SyntheticControl")); + assert!(book.warnings.iter().any(|w| w.contains("workbook names shadowed by local names")), "{:?}", book.warnings); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} From 7ed3163f35ea3a60f809889df6e50cc44ef9170c Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:45:33 +0100 Subject: [PATCH 15/27] render-parity: a NaN tolerance fails its check instead of switching it off --- crates/render-parity/src/lib.rs | 14 ++++++++++---- crates/render-parity/src/tests.rs | 11 +++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/render-parity/src/lib.rs b/crates/render-parity/src/lib.rs index f2e541801..aedac7f0f 100644 --- a/crates/render-parity/src/lib.rs +++ b/crates/render-parity/src/lib.rs @@ -112,7 +112,7 @@ pub struct Report { } impl Report { - /// Whether every measure is within `tolerance`. + /// Whether every measure is within `tolerance`. A NaN limit is never met. pub fn within(&self, tolerance: &Tolerance) -> bool { self.exceeded(tolerance).is_empty() } @@ -120,13 +120,13 @@ impl Report { /// The measures over `tolerance`, each with its value and limit, for a test failure message. pub fn exceeded(&self, tolerance: &Tolerance) -> Vec { let mut over = Vec::new(); - if self.max_delta_e > tolerance.max_delta_e { + if exceeds(self.max_delta_e, tolerance.max_delta_e) { over.push(format!("max ΔE2000 {:.4} > {}", self.max_delta_e, tolerance.max_delta_e)); } - if self.mean_delta_e > tolerance.mean_delta_e { + if exceeds(self.mean_delta_e, tolerance.mean_delta_e) { over.push(format!("mean ΔE2000 {:.4} > {}", self.mean_delta_e, tolerance.mean_delta_e)); } - if self.max_channel > tolerance.max_channel { + if exceeds(f64::from(self.max_channel), f64::from(tolerance.max_channel)) { over.push(format!("max channel difference {:.6} > {}", self.max_channel, tolerance.max_channel)); } over @@ -179,6 +179,12 @@ pub fn compare(expected: &Image<'_>, actual: &Image<'_>) -> Result bool { + !matches!(value.partial_cmp(&limit), Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal)) +} + fn finite(px: [f32; 3]) -> Option<[f32; 3]> { px.iter().all(|v| v.is_finite()).then_some(px) } diff --git a/crates/render-parity/src/tests.rs b/crates/render-parity/src/tests.rs index 3649c37d3..a5c81f24f 100644 --- a/crates/render-parity/src/tests.rs +++ b/crates/render-parity/src/tests.rs @@ -87,3 +87,14 @@ fn bad_inputs_are_errors_not_panics() { let e: Box = Box::new(e); assert!(e.source().is_none()); } + +#[test] +fn a_nan_tolerance_fails_every_check() { + let rgb = gray_ramp(2, 2); + let image = Image::new(2, 2, &rgb).unwrap(); + let report = compare(&image, &image).unwrap(); + let nan = Tolerance { max_delta_e: f64::NAN, mean_delta_e: f64::NAN, max_channel: f32::NAN }; + assert_eq!(report.exceeded(&nan).len(), 3); + assert!(!report.within(&nan)); + assert!(report.within(&Tolerance { max_delta_e: f64::INFINITY, ..Tolerance::BASELINE })); +} From 7888c863dffc9d55d349c76e4f0006149fe33e2d Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:51:42 +0100 Subject: [PATCH 16/27] svg: bound entity declaration scans and reset smooth controls after Z #778, #789. Comments and CDATA are skipped, and a declaration name is only scanned up to MAX_NAME+1, so a comment full of ` HashMap<&str, Vec<&str>> { let mut map: HashMap<&str, Vec<&str>> = HashMap::new(); - for (at, _) in svg.match_indices("") { + Some(n) => at + 1 + 3 + n + 3, + None => svg.len(), + }; continue; } - let Some(end) = rest.find(char::is_whitespace) else { continue }; - let (name, rest) = rest.split_at(end); - let rest = rest.trim_start(); - let Some(quote) = rest.chars().next().filter(|q| matches!(q, '"' | '\'')) else { continue }; - let Some(value) = rest.get(1..).and_then(|r| r.split_once(quote)).map(|(v, _)| v) else { continue }; - map.entry(name).or_default().push(value); + if let Some(body) = after.strip_prefix("![CDATA[") { + i = match body.find("]]>") { + Some(n) => at + 1 + 8 + n + 3, + None => svg.len(), + }; + continue; + } + if let Some(rest) = after.strip_prefix("!ENTITY") { + let rest = rest.trim_start(); + // A parameter entity (`%`) isn't referenced as `&name;`. + if rest.starts_with('%') { + i = at + 1; + continue; + } + // Bound the name scan: a comment full of ` MAX_NAME { + // A name longer than MAX_NAME cannot be referenced, but it must still be + // recorded so `entities_expand_past` can refuse the document. + map.entry(window).or_default().push(""); + } + i = at + 1; + continue; + } + i = at + 1; } map } @@ -99,7 +133,8 @@ pub fn entities_expand_past(svg: &str, most: usize) -> bool { /// an entity becomes elements where it is referenced, so what the text shows of the tree is not /// what a reader builds; drawing apps declare entities for namespace names only. A caller may /// refuse these outright instead of counting them ([`prescan`] counts them ten times). Text that -/// only looks like a declaration (in a comment) counts too. `<` is text, not markup. +/// only looks like a declaration in another entity's value counts too; comments are skipped. +/// `<` is text, not markup. pub fn entity_holds_markup(svg: &str) -> bool { declared(svg).values().flatten().any(|value| value.contains('<')) } @@ -396,6 +431,16 @@ mod tests { assert!(expansion_too_large(&long_name)); } + #[test] + fn entity_markers_in_a_comment_are_scanned_in_linear_time() { + // 100,000 `"); + assert!(!entities_expand_past(&svg, MAX_ENTITY_EXPANSION)); + } + fn svg(body: &str) -> String { format!(r#"{body}"#) } diff --git a/crates/svg/src/path_data.rs b/crates/svg/src/path_data.rs index 96d0db771..fc31fbaf2 100644 --- a/crates/svg/src/path_data.rs +++ b/crates/svg/src/path_data.rs @@ -214,6 +214,10 @@ pub fn parse_path(d: &str) -> Result> { } cur = start; cmd = 0; + // Z is not a cubic or quadratic, so a following S or T starts from the + // current point rather than reflecting the pre-close control. + prev_cubic = None; + prev_quad = None; continue; } cmd = c; @@ -434,6 +438,15 @@ mod tests { assert_eq!(paths[0].segs[1], Segment::Cubic(Point::new(8.0, -2.0), Point::new(10.0, -2.0), Point::new(12.0, 0.0))); } + #[test] + fn close_resets_smooth_control_history() { + // After Z, S must not reflect the preceding C's control (20,20) → (0,0). + let paths = parse_path("M10 10 C10 20 20 20 20 10 Z S30 0 30 10").unwrap(); + assert_eq!(paths.len(), 2); + assert!(paths[0].closed); + assert_eq!(paths[1].segs[0], Segment::Cubic(Point::new(10.0, 10.0), Point::new(30.0, 0.0), Point::new(30.0, 10.0))); + } + #[test] fn arc_centers_keep_radius_correction_and_sweep() { let (center, delta) = arc_center_circular(Point::new(0.0, 0.0), Point::new(10.0, 0.0), 2.0, false, true).unwrap(); From 21ceafea04a551781bd3124a823073b9ea310746 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 07:59:10 -0500 Subject: [PATCH 17/27] fix(svg): validate declaration encodings and internal subset boundaries --- crates/svg/src/write.rs | 70 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/crates/svg/src/write.rs b/crates/svg/src/write.rs index 658454ab2..af3eec992 100644 --- a/crates/svg/src/write.rs +++ b/crates/svg/src/write.rs @@ -258,6 +258,42 @@ pub fn path_data_fixed(path: &BezPath, decimals: usize) -> String { #[derive(Clone, Debug, Default)] pub struct SvgWriter(craft_xml::Writer); +// A subset ends with `] S? >`; quoted entity values, comments and PIs may contain that text. +// Scan once without allocations; malformed unterminated constructs remain the caller's error. +fn closes_internal_subset(internal: &str) -> bool { + let bytes = internal.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i..].starts_with(b"") } else { (2, b"?>") }; + let Some(offset) = bytes[i + start..].windows(end.len()).position(|part| part == end) else { + return false; + }; + i += start + offset + end.len(); + } else if matches!(bytes[i], b'\'' | b'"') { + let quote = bytes[i]; + i += 1; + while i < bytes.len() && bytes[i] != quote { + i += 1; + } + if i < bytes.len() { + i += 1; + } + } else if bytes[i] == b']' { + i += 1; + while i < bytes.len() && matches!(bytes[i], b' ' | b'\t' | b'\r' | b'\n') { + i += 1; + } + if bytes.get(i) == Some(&b'>') { + return true; + } + } else { + i += 1; + } + } + false +} + impl SvgWriter { /// A document: starts with the XML declaration. pub fn new() -> SvgWriter { @@ -270,25 +306,28 @@ impl SvgWriter { SvgWriter::default() } /// A document that starts with an XML declaration naming `encoding` (for example `UTF-8` or - /// `ISO-8859-1`) and a newline. Nothing is written if `encoding` is not an XML name. + /// `ISO-8859-1`) and a newline. Nothing is written for an invalid XML encoding name. pub fn with_encoding(encoding: &str) -> SvgWriter { let mut w = SvgWriter::fragment(); w.decl(encoding, "\n"); w } /// Writes `` followed by `newline`. Nothing is - /// written if `encoding` is not an XML name. + /// written unless `encoding` starts with an ASCII letter and continues with ASCII letters, + /// digits, `.`, `_`, or `-` (XML EncName). pub fn decl(&mut self, encoding: &str, newline: &str) { - if craft_xml::is_name(encoding) { + let mut bytes = encoding.bytes(); + if bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { self.0.raw(&format!("{newline}")); } } /// Writes `` + `newline`. When /// `internal` is empty, the inner newline is omitted, so a minified document can be /// `` with no extra breaks. Nothing is written if `name` is not an XML name - /// or `internal` contains `]]>`. + /// or `internal` closes the subset outside a quoted value, comment or processing instruction. + /// The caller still supplies valid DTD declarations; this is not a DTD validator. pub fn doctype_internal(&mut self, name: &str, internal: &str, newline: &str) { - if !craft_xml::is_name(name) || internal.contains("]]>") { + if !craft_xml::is_name(name) || closes_internal_subset(internal) { return; } let mut out = format!("\n"); assert!(SvgWriter::with_encoding("UTF-8\"").is_empty()); assert!(SvgWriter::with_encoding("").is_empty()); + for valid in ["A", "a0._-"] { + assert_eq!(SvgWriter::with_encoding(valid).finish(), format!("\n")); + } + for invalid in [":UTF-8", "_UTF8", "1UTF8", "é", "UTF:8", "UTFé", "UTF 8"] { + assert!(SvgWriter::with_encoding(invalid).is_empty(), "{invalid}"); + } let mut compact = SvgWriter::fragment(); compact.decl("ISO-8859-1", ""); compact.root().open(); @@ -479,7 +524,22 @@ mod tests { assert_eq!(empty.finish(), "\n"); let mut refused = SvgWriter::fragment(); refused.doctype_internal("svg", "]]>", "\n"); + refused.doctype_internal("svg", "]>", "?>"] { + let mut writer = SvgWriter::fragment(); + writer.doctype_internal("svg", internal, ""); + assert_eq!(writer.finish(), format!("")); + } + for internal in ["]>", "] >", "]>"] { + let mut writer = SvgWriter::fragment(); + writer.doctype_internal("svg", internal, ""); + assert!(writer.is_empty()); + } + } } From 9ac3245966eb2b88a37930b1d820523124e0a02f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:13 -0500 Subject: [PATCH 18/27] settings, atomic: give a replaced file's group bits only to its group (#558) * settings, atomic: give a replaced file's group bits only to its group write_backup copied the config's mode onto a new backup inode without giving it the config's group, and craft-atomic's stage() did the same for every replaced file: on a setgid directory or with another primary group, the group bits opened the contents to a group never granted them. Both now chown the new inode to the replaced file's group first and drop the group bits when that fails. * address review: skip fchown when the stage already has the replaced file's group * atomic: route the stage's group change through AtomicFs so the fallback is tested --- crates/atomic/src/lib.rs | 90 +++++++++++++++++++++++++++++++-- crates/settings/src/recovery.rs | 17 ++++--- crates/settings/tests/store.rs | 4 ++ 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/crates/atomic/src/lib.rs b/crates/atomic/src/lib.rs index 1590372ad..08b881e17 100644 --- a/crates/atomic/src/lib.rs +++ b/crates/atomic/src/lib.rs @@ -70,6 +70,9 @@ pub trait AtomicFs { fn sync_dir(&self, dir: &Path) -> io::Result<()>; /// Give the open temporary file its final permissions. fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()>; + /// Put the open temporary file in group `gid` (Unix only). + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()>; } /// The production file system implementation. @@ -126,6 +129,11 @@ impl AtomicFs for RealFs { fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> { file.set_permissions(perms) } + + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()> { + std::os::unix::fs::fchown(file, None, Some(gid)) + } } /// Flush `file` to stable storage (`sync_all`). @@ -342,10 +350,29 @@ fn split(target: &Path, path: &Path) -> io::Result<(PathBuf, String)> { Ok((dir, name.to_string_lossy().into_owned())) } -/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions of the file it -/// replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when the mode -/// already matches after `fill`. A leftover file with the same name is skipped. When any step -/// fails, the temporary file is removed. +/// Give `perms` the replaced file's group, or drop its group bits when the stage cannot join +/// that group. A stage already in the group needs no fchown, which some filesystems reject even +/// as a no-op, and dropping the bits there would narrow a mode the replaced file granted. +#[cfg(unix)] +fn adopt_group(fs: &dyn AtomicFs, file: &File, stage_gid: u32, replaced_gid: u32, perms: std::fs::Permissions) -> io::Result { + use std::os::unix::fs::PermissionsExt; + if perms.mode() & 0o070 == 0 || stage_gid == replaced_gid { + return Ok(perms); + } + if fs.set_group(file, replaced_gid).is_ok() { + return Ok(perms); + } + let mut narrowed = perms; + narrowed.set_mode(narrowed.mode() & !0o070); + Ok(narrowed) +} + +/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions and group of +/// the file it replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when +/// the mode already matches after `fill`; the group bits are dropped when the file cannot be put +/// in the replaced file's group, so they never open the file to a group that was not granted +/// them. A leftover file with the same name is skipped. When any step fails, the temporary file +/// is removed. fn stage( fs: &dyn AtomicFs, dir: &Path, @@ -383,6 +410,14 @@ fn stage( if (file.metadata()?.permissions().mode() & 0o7777) == (p.mode() & 0o7777) { return Ok(()); } + // The group bits belong to the replaced file's group, which a new + // inode need not be in (setgid directory, another primary group). + if let Some(replaced) = replaced { + let stage_gid = std::os::unix::fs::MetadataExt::gid(&file.metadata()?); + let replaced_gid = std::os::unix::fs::MetadataExt::gid(replaced); + let perms = adopt_group(fs, &file, stage_gid, replaced_gid, p)?; + return fs.set_permissions(&file, perms); + } } fs.set_permissions(&file, p) }) @@ -678,6 +713,9 @@ mod tests { fixed_mode: Option, fail_sync_dir: bool, fail_normal_create: bool, + /// Simulate a filesystem that refuses the stage's group change. + #[cfg(unix)] + fail_set_group: bool, normal_collisions: Cell, /// Another process creates the target just before the hard link. created_meanwhile: Option<&'static [u8]>, @@ -750,6 +788,15 @@ mod tests { } /// Record permission restoration and payload length, then inject a configured error /// or delegate the mode change to the real filesystem. + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()> { + self.events.borrow_mut().push("set_group"); + if self.fail_set_group { + return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected: set_group failed")); + } + RealFs.set_group(file, gid) + } + fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> { self.events.borrow_mut().push("set_permissions"); #[cfg(unix)] @@ -1090,6 +1137,36 @@ mod tests { Ok(()) } + #[cfg(unix)] + #[test] + fn group_bits_are_dropped_when_the_stage_cannot_join_the_replaced_group() -> io::Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let d = TempDir::new("group-fallback"); + let file = File::create(d.0.join("stage"))?; + let gid = file.metadata()?.gid(); + let elsewhere = gid.wrapping_add(1); + + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o640))?.mode(); + assert_eq!(mode & 0o777, 0o600, "group bits are dropped when the group cannot be joined"); + assert!(fs.events.borrow().contains(&"set_group")); + + // No group bits: the group change is never attempted. + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o600))?.mode(); + assert_eq!(mode & 0o777, 0o600); + assert!(!fs.events.borrow().contains(&"set_group")); + + // A stage already in the replaced file's group keeps its bits untouched. + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, gid, std::fs::Permissions::from_mode(0o640))?.mode(); + assert_eq!(mode & 0o777, 0o640); + assert!(!fs.events.borrow().contains(&"set_group")); + + assert_no_temp(&d.0); + Ok(()) + } + #[cfg(unix)] #[test] fn normal_policy_preserves_existing_modes_with_private_staging() -> io::Result<()> { @@ -1102,6 +1179,11 @@ mod tests { let fs = Faulty { fail_normal_create: true, ..Default::default() }; replace_with_options(&fs, NO_RETRY, &p, NewFilePermissions::PlatformDefault, |f| f.write_all(b"replacement"))?; assert_eq!(std::fs::metadata(&p)?.permissions().mode() & 0o777, mode); + assert_eq!( + std::os::unix::fs::MetadataExt::gid(&std::fs::metadata(&p)?), + std::os::unix::fs::MetadataExt::gid(&d.0.metadata()?), + "kept in the directory's group" + ); assert!(!fs.events.borrow().contains(&"create_new_normal")); assert_eq!(*fs.created_modes.borrow(), [0o600]); } diff --git a/crates/settings/src/recovery.rs b/crates/settings/src/recovery.rs index 074a1b4a8..1435797e0 100644 --- a/crates/settings/src/recovery.rs +++ b/crates/settings/src/recovery.rs @@ -37,23 +37,28 @@ pub fn backup_path(path: &Path) -> PathBuf { /// Writes `text`, the last version that parsed, to the backup file atomically. /// /// The caller decides what is safe to keep: a blank or unparsable file must never replace the -/// previous backup. On Unix the backup always gets the same mode as `path` (`0600` when there is -/// none), so it is never readable by more people than the settings themselves. +/// previous backup. On Unix the backup always gets the same mode and group as `path` (`0600` +/// when there is none), so it is never readable by more people than the settings themselves; +/// when it cannot be put in the config's group, it keeps the owner bits only. pub fn write_backup(path: &Path, text: &str) -> io::Result<()> { let backup = backup_path(path); #[cfg(unix)] { - use std::os::unix::fs::PermissionsExt; - let mode = std::fs::metadata(path).map(|m| m.permissions().mode() & 0o777).unwrap_or(0o600); + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let config = std::fs::metadata(path).ok(); + let mode = config.as_ref().map(|m| m.permissions().mode() & 0o777).unwrap_or(0o600); // An atomic replace keeps the existing file's mode, so narrow an old, broader backup // before new contents go into it, then match the config once written. if backup.exists() { - std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode))?; + std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode & 0o700))?; } craft_atomic::WriteOptions::default() .with_new_file_permissions(craft_atomic::NewFilePermissions::Private) .atomic_write(&backup, text.as_bytes())?; - std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode)) + // The group bits were granted to the config's group; the new file may have been created + // in another (a setgid directory, a different primary group). + let grouped = config.as_ref().is_some_and(|m| std::os::unix::fs::chown(&backup, None, Some(m.gid())).is_ok()); + std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(if grouped { mode } else { mode & 0o700 })) } #[cfg(not(unix))] { diff --git a/crates/settings/tests/store.rs b/crates/settings/tests/store.rs index bc55d97cb..a9c4b9553 100644 --- a/crates/settings/tests/store.rs +++ b/crates/settings/tests/store.rs @@ -289,4 +289,8 @@ fn backup_follows_a_mode_the_user_chose_for_the_config() { store.update(|p| p.general.language = "fr".into()); store.flush().unwrap(); assert_eq!(mode(&tmp.path().join("config.toml.bak")), 0o640); + // The group bits authorize the config's group, which the backup is in. + use std::os::unix::fs::MetadataExt; + let gid = |p: &std::path::Path| std::fs::metadata(p).unwrap().gid(); + assert_eq!(gid(&tmp.path().join("config.toml.bak")), gid(&config_path(tmp.path()))); } From eeae2bb7ca5bdc201c608c321d6bcf9330858fa3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:17 -0500 Subject: [PATCH 19/27] feat(ssip): extract bounded Speech Dispatcher wire helpers (#1500) Adapted from WordCraft SSIP implementation; preserves contributor framing behavior while rejecting overlong complete lines. Co-authored-by: Skaywalker <115165402+Skaywalker@users.noreply.github.com> --- Cargo.lock | 4 ++ Cargo.toml | 1 + crates/ssip/Cargo.toml | 11 ++++ crates/ssip/README.md | 19 ++++++ crates/ssip/src/lib.rs | 140 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+) create mode 100644 crates/ssip/Cargo.toml create mode 100644 crates/ssip/README.md create mode 100644 crates/ssip/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 83670ba58..3385eab1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2798,6 +2798,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-ssip" +version = "0.1.0" + [[package]] name = "craft-stencilcraft-format" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 07aebdc37..801dc9ba8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ craft-shx = { path = "crates/shx" } craft-http-url = { path = "crates/http-url" } craft-usd = { path = "crates/usd" } craft-smime = { path = "crates/smime" } +craft-ssip = { path = "crates/ssip" } craft-ber = { path = "crates/ber" } craft-proto = { path = "crates/proto" } craft-render-parity = { path = "crates/render-parity" } diff --git a/crates/ssip/Cargo.toml b/crates/ssip/Cargo.toml new file mode 100644 index 000000000..5ff35544f --- /dev/null +++ b/crates/ssip/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-ssip" +description = "Bounded Speech Dispatcher SSIP wire framing and message-data encoding" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true diff --git a/crates/ssip/README.md b/crates/ssip/README.md new file mode 100644 index 000000000..7fc98924e --- /dev/null +++ b/crates/ssip/README.md @@ -0,0 +1,19 @@ +# craft-ssip + +A focused Speech Dispatcher SSIP wire layer: sentence-as-data encoding, final response codes, +and a bounded incremental reply reader. No sockets, process launch, cancellation commands, +voice discovery, retry or playback scheduling: those policies remain with the application. + +Derived from Skaywalker's WordCraft SSIP implementation in +`crates/engine/src/speech/ssip.rs` at `e9750c5cb75d577a4669059d7bf6f9874e945839` +(MIT OR Apache-2.0). WordCraft #2937 (`65d7999eddebf120ca9fde523b01156b3e1f5fce`) +adds voice selection on this same core; this extraction does not duplicate those application changes. + +Unlike the original reader, a complete line that becomes overlong in the final read is refused. +The 4096-byte limit counts bytes before LF, including an optional CR. Reply buffering holds +at most that limit plus one 512-byte read. Invalid UTF-8 is replaced, preserving WordCraft behavior. +Message input is bounded by the caller; ASCII three-digit response codes are enforced. +Caller-provided `Read` implementations must obey Rust's Read contract; callers remain responsible +for transport timeouts, cancellation and an overall deadline between `next_line` calls. + +This is not a complete SSIP client and does not establish WordCraft consumer cutover. diff --git a/crates/ssip/src/lib.rs b/crates/ssip/src/lib.rs new file mode 100644 index 000000000..8bb4cce37 --- /dev/null +++ b/crates/ssip/src/lib.rs @@ -0,0 +1,140 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Speech Dispatcher SSIP wire helpers extracted from WordCraft's contributor implementation. +//! Socket discovery, deadlines, voice selection, retry and cancellation policy stay in the app. +use std::io::{self, ErrorKind, Read}; + +/// Maximum reply bytes before LF, including an optional CR. +pub const MAX_LINE: usize = 4096; + +/// Encode one sentence as data, never as protocol commands. CR/LF become spaces; a leading +/// dot is doubled and the message terminator is appended. The caller bounds input bytes. +pub fn message_data(text: &str, max_input_bytes: usize) -> io::Result { + if text.len() > max_input_bytes { + return Err(ErrorKind::InvalidInput.into()); + } + let size = text.len().checked_add(6).ok_or(ErrorKind::InvalidInput)?; + let mut out = String::new(); + out.try_reserve_exact(size).map_err(|_| io::Error::other("SSIP message allocation failed"))?; + if text.starts_with('.') { + out.push('.'); + } + out.extend(text.chars().map(|c| if c == '\r' || c == '\n' { ' ' } else { c })); + out.push_str("\r\n.\r\n"); + Ok(out) +} + +/// The three ASCII digits of a final response; continuation (`225-...`) is not final. +pub fn final_code(line: &str) -> Option { + let digits = line.as_bytes().get(..3)?; + if !digits.iter().all(u8::is_ascii_digit) || !(line.len() == 3 || line.as_bytes().get(3) == Some(&b' ')) { + return None; + } + line.get(..3)?.parse().ok() +} + +/// Incremental replies. At most one read per call lets the app enforce its own deadline. +/// Invalid UTF-8 retains WordCraft's replacement-character behavior. Overlong replies fail, +/// including those whose terminating newline arrives in the same read that exceeds the cap. +pub struct Lines { + reader: R, + pending: Vec, +} + +impl Lines { + pub fn new(reader: R) -> Self { + Self { reader, pending: Vec::new() } + } + + /// Complete next line without CR/LF, or None after a partial read or transient timeout. + /// The caller must set transport timeouts and enforce cancellation/deadlines. + pub fn next_line(&mut self) -> io::Result> { + if let Some(line) = self.complete_line()? { + return Ok(Some(line)); + } + let mut buf = [0u8; 512]; + match self.reader.read(&mut buf) { + Ok(0) => Err(ErrorKind::UnexpectedEof.into()), + Ok(n) => { + let bytes = buf.get(..n).ok_or(ErrorKind::InvalidData)?; + // Invariant: pending <= MAX_LINE before this read, so storage <= MAX_LINE+512. + self.pending.try_reserve(bytes.len()).map_err(|_| io::Error::other("SSIP reply allocation failed"))?; + self.pending.extend_from_slice(bytes); + self.complete_line() + } + Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted) => Ok(None), + Err(e) => Err(e), + } + } + + fn complete_line(&mut self) -> io::Result> { + let end = self.pending.iter().position(|&b| b == b'\n'); + if end.unwrap_or(self.pending.len()) > MAX_LINE { + return Err(ErrorKind::InvalidData.into()); + } + let Some(end) = end else { return Ok(None) }; + let bytes = self.pending.get(..end).ok_or(ErrorKind::InvalidData)?; + let line = String::from_utf8_lossy(bytes).trim_end_matches('\r').to_string(); + self.pending.drain(..=end); + Ok(Some(line)) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + use std::collections::VecDeque; + struct Chunks(VecDeque>>); + impl Read for Chunks { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let bytes = self.0.pop_front().unwrap_or(Ok(Vec::new()))?; + out[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + #[test] + fn message_text_cannot_inject_commands() { + assert_eq!(message_data(".\r\nCANCEL ALL\n!-!", 64).unwrap(), ".. CANCEL ALL !-!\r\n.\r\n"); + assert_eq!(message_data("你好", 6).unwrap(), "你好\r\n.\r\n"); + assert!(message_data("你好", 5).is_err()); + } + #[test] + fn final_codes_are_ascii_and_not_continuations() { + assert_eq!(final_code("225 OK"), Some(225)); + assert_eq!(final_code("701"), Some(701)); + for bad in ["225-12", "+22 OK", "22 OK", "225 OK", "225x", ""] { + assert_eq!(final_code(bad), None); + } + } + #[test] + fn fragmented_utf8_timeout_and_buffered_lines() { + let mut lines = Lines::new(Chunks(VecDeque::from([ + Ok(vec![b'2', b'2', b'5', b' ', 0xc3]), + Err(ErrorKind::TimedOut.into()), + Ok(vec![0xa9, b'\r', b'\n', b'7', b'0', b'2', b'\n']), + ]))); + assert!(lines.next_line().unwrap().is_none()); + assert!(lines.next_line().unwrap().is_none()); + assert_eq!(lines.next_line().unwrap().as_deref(), Some("225 é")); + assert_eq!(lines.next_line().unwrap().as_deref(), Some("702")); + assert_eq!(lines.next_line().unwrap_err().kind(), ErrorKind::UnexpectedEof); + } + #[test] + fn completed_overlong_line_is_rejected_at_boundary() { + for extra in [0, 1, 511] { + let mut chunks: VecDeque<_> = (0..8).map(|_| Ok(vec![b'x'; 512])).collect(); + let mut last = vec![b'x'; extra]; + last.push(b'\n'); + chunks.push_back(Ok(last)); + let mut lines = Lines::new(Chunks(chunks)); + for _ in 0..8 { + assert!(lines.next_line().unwrap().is_none()); + } + if extra == 0 { + assert_eq!(lines.next_line().unwrap().unwrap().len(), MAX_LINE); + } else { + assert_eq!(lines.next_line().unwrap_err().kind(), ErrorKind::InvalidData); + } + } + } +} From 094537374cc9878e141f829748fb91d329b472b7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:22 -0500 Subject: [PATCH 20/27] Integrate shared IPC with full executable ancestry validation (#1571) * craft-ipc: local-socket server, client and discovery for Craft apps Adds craft-ipc, the transport for a Craft app that talks to other Craft apps on one machine. It is native only: interprocess has no wasm32 build. - server and client: one JSON object per line, push events, a bulk connection for frames, token auth on every connection, connection and time limits. - discovery: one record per running app in a per-user runtime directory, with a lock file that shows whether the owner is alive. - launch, trust, cache: find an app binary, start it only after an executable check, keep caches in a private per-user directory. - activate: focus hand-off between apps. - moniker, record, payload, asset, refresh, fetch, watch: asset names, stored link records, open payloads, linked-file polling and a frame cache. - PROTOCOL.md describes the wire protocol. The record schemas and fixtures are in the crate. - The crate compiles and passes clippy -D warnings on x86_64-pc-windows-gnu and aarch64-apple-darwin. This is a compile and lint fix only. No one ran the Windows or macOS code paths, so this does not claim they work there. Part of #414. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: app.activate no longer claims a request it cannot make ActivationBackend gains can_apply_token (default false). On Linux a token only counts when the backend can apply it to a live window. Stock winit cannot, so the answer is false plus a request for user attention. Wire values are unchanged. PROTOCOL.md says what each value means. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: keep a partial reply line across client read timeouts pull_line dropped the bytes it had read when a read timed out, so next_event returned Ok(None) and the next read saw only the tail of the line. The client now keeps the partial line and resumes it on the next read. A test splits an event line across a timeout. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: scope the macOS fallback runtime directory to the user With TMPDIR unset or relative, the fallback was the shared path /tmp/craft. The first user to create it owned it, and every other user then failed the owner check. The fallback is now /tmp/craft-, as on Linux. The path choice is a pure function with a test. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: fetch rejects non-finite time, bounds its queue, keeps waited-for asks Three changes to FrameFetcher::fetch. A non-finite time_s was queued and failed in request_reply before any socket I/O. The worker read that as a dead connection and exited, so enough bad asks took every worker down. fetch now returns Fetched::Failed and queues nothing. The queue had no length limit, so stalled workers let it grow with the request rate. It now holds at most QUEUE_MAX asks (256, a new public constant). A miss past the cap drops the oldest queued ask and counts it in stats.skipped. next_job dropped any job older than max_age, even when the caller had waited longer than max_age and was still there. The caller then re-queued at the tail. Each job now records the latest deadline of its waiting callers and is skipped for age only when no caller is left. Tests cover each case. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: reject file:// media URLs that do not percent-decode local_path returned None for a file:// URL with a bad percent escape, and missing_media then skipped it, so the server accepted a local target without the media probe. Such a URL now gets payload_invalid. URLs that name no local file (remote host, other scheme, relative path) are still left to the app. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: open watched files without waiting on a FIFO Tracker::poll, Tracker::new and fingerprint opened the path with File::open before stat checked the file type. A FIFO with no writer made that open wait, and poll_entry held the entry lock for the whole call, so one such path stalled every link on the watcher and the watcher drop. The open now sets O_NONBLOCK (new crate fsopen), and stat rejects the FIFO on the open handle with LinkError::Invalid. A check before the open would not close the window between check and open. craft-ipc gains a unix-only dependency on libc for the open flags. libc 0.2.190 is already in Cargo.lock. The lock file gains one dependency line for craft-ipc. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: read_private checks and reads one open handle read_private checked the path with symlink_metadata and then read it with fs::read, so the entry could change between the two calls. On Unix it now opens once with O_NOFOLLOW and O_NONBLOCK, checks type, owner and mode on that handle, and reads from the same handle. A symlink is still Refused (found by looking at the path after a failed open, because the errno differs between Unix systems), a FIFO is Refused and cannot stall the open, and a missing file is still an I/O error. Other platforms keep the old checks. Tests cover the symlink and FIFO cases. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: Tracker reports an epoch it cannot raise instead of a false bump apply saturated the epoch at u64::MAX while poll and relink still returned bumped: true, so Change::bumped (documented as one added to the epoch) was false and a cached frame kept passing for changed content. poll and relink now return LinkError::Invalid and leave the tracker as it was when the bump would pass u64::MAX. A test covers both calls. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: document write_atomic errors that follow the rename The directory flush and the fingerprint run after the rename. An error from either can be returned while the path already holds the new bytes. The doc comment now says so. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: say that check_executable accepts group-writable paths The module and function docs described the check as protection from other local users. A group-writable file or directory passes, and the other members of its group can replace it. The docs now state that and keep the private-group reason. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: refuse an auth line that completes after the deadline authenticate returned a completed line before it looked at the deadline, so a line that finished after auth_deadline still authenticated. The overshoot was up to one 250 ms read poll. The deadline is now checked first, with the same auth failure reply as the other expiry paths. A test sends the line between the deadline and the next poll. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: do not append an error record after a partial frame write send_frame wrote an error record whenever write_frame failed. When the failure came from the stream (for example a send timeout in the middle of a record), the first record was cut short, and a peer that drained the buffer then read the error record as the rest of it. send_frame now checks that the record can be built before it writes. A record that cannot be built gets the error record as before. A write failure is returned alone and the bulk loop closes the connection. wire gains check_cfrm, the pre-write part of write_cfrm. write_cfrm behaves as before. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: LineReader applies the line limit to a last line without a newline At end of stream LineReader::read returned a buffered line with no length check, so a peer that sent one byte over the limit and closed its write side got that line dispatched, while the same bytes with a newline were refused. The end of stream path now returns TooLarge for a line over the limit. A test covers both paths. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: answer a control reply over the line limit with payload_too_large write_value passed every reply to write_line, which has no size check. A sink result larger than MAX_LINE went out as one line that the client refuses, and the rest of the line stayed unread on the connection. Replies now go through wire::write_reply_line. A reply over the limit is replaced by a payload_too_large error that keeps the request id. When that does not fit either, or the value has no id, nothing is written and the request loop ends. Event lines are unchanged. wire gains write_reply_line. write_line behaves as before. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: accept a shallow or deep poll in the native ticker test The test expected the first notice to come from a shallow poll. With a 50 ms deep interval the poll that first sees the edit can be a deep one, so the assertion failed on a slow or loaded machine. It now takes the deep flag from the notice and checks every other field. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: refuse a frame stride that cannot hold one row FrameLayout::check only tested len == stride * h, so w = 4, stride = 1, h = 3, len = 3 was a valid rgba8 layout. A reader that splits the payload by stride cannot decode it. frames.open, a rendered frame and the client reply parser now require stride >= w * bytes per pixel (4 for rgba8_premul, 8 for rgba16f_premul). open checks before it mints a ticket, so a refused layout leaves no ticket. The bulk test source used stride 16 for rgba16f with w = 4, which needs 32. It now uses a stride that fits the format. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: reject calendar days that do not exist in a record's created field valid_rfc3339 checked the day only against 1 to 31, so 2026-02-31T09:30:00Z passed LinkRecord::validate while the schema's date-time format refused it. The year, month and day now go through chrono::NaiveDate::from_ymd_opt, as the workspace rules require for calendar logic. The accepted syntax is unchanged. The schema agreement table gets rows for impossible days and for leap days. craft-ipc gains the workspace chrono dependency. chrono is already in Cargo.lock; the lock file gains one dependency line for craft-ipc. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: make the link record moniker pattern match Moniker::parse The asset.moniker pattern accepted any id without / ? #, so a record the schema passed could fail LinkRecord::validate (bad or truncated escape, raw space, escape that decodes to invalid UTF-8, id over 512 bytes). The pattern is now two patterns under allOf. One checks the shape: unreserved characters and two-digit escapes only, 1 to 512 of them, which is 1 to 512 decoded bytes. The other checks that the decoded bytes are valid UTF-8, written as the well-formed UTF-8 byte sequences. Neither uses lookahead, so any ECMA 262 engine can run them. The schema agreement test gets rows for each case in the list above and for the length limits, and a sweep of lead, second and tail bytes that compares the schema verdict with the Rust verdict. This tightens the published schema: documents it accepted that Rust refused are now refused. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: document that Assets methods must return in bounded time Server::shutdown and drop join every connection thread, and an Assets call runs on one with no deadline and no view of the stop flag. The trait docs now say so and ask implementations to bound their own I/O. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: name the writer type in the listener tests Clippy's type_complexity lint failed on the test helper's return type. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: run Assets and Activate calls on detached helper threads with a deadline Server::shutdown and drop join every connection thread, and an Assets or Activate call ran on one with no deadline and no view of the stop flag, so a provider that never returned made shutdown hang. These calls now run on a helper thread (server/provider.rs). The connection thread waits for the answer in 100 ms slices, for at most Limits::provider_deadline (default REPLY_DEADLINE_SECS) and not past the stop flag. On timeout the caller gets a timeout reply and the helper is detached, so shutdown does not join it. At most 32 helper threads are alive at once, stuck ones included, and a call past that gets a busy error without starting a thread. A panic in a provider becomes an io_error reply. Frames stays on the connection thread because it is one call per frame. Its trait docs now say that the stop flag is checked before each request and that a running call is waited for. The module docs describe the shutdown guarantee for each provider kind. Public API: new Limits::provider_deadline field. Tests use a provider that blocks until released, for Assets and for Activate. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: give a caller whose queued ask was dropped a Failed outcome When the fetch queue was full, the oldest ask was dropped but its waiting callers got no answer. They woke, found the key neither cached nor in flight, and queued it again, which dropped another ask and repeated the cycle. A dropped ask now records a failure for its key before the waiters are woken, so each waiter returns Fetched::Failed and does not queue the key again. The drop-oldest policy and stats.skipped are unchanged. Failure records are shared with the worker path and kept to a fixed number, oldest first. No type or variant changes. The Fetched::Failed doc now covers this case. A test overflows the queue while a caller waits on the oldest key, and checks that the caller returns at once and that no further asks are dropped. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: require two hex digits in a file URL percent escape percent_decode used u8::from_str_radix, which accepts a leading plus, so file:///tmp/a%+fb decoded to the byte 0x0f and went on to the media probe instead of getting payload_invalid. The two bytes after a percent sign must now both be ASCII hex digits. The decode test gets a plus, a minus, a space and a short escape. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: make the euid probe create only its own file, and gate the macOS path test On Unix systems other than Linux, current_euid read the owner of a file named .craft-uid- in the temp directory, made with create and truncate. A file or symlink planted at that name was opened and truncated. The probe now lives in probe_euid: random name, create_new (which refuses an existing path), mode 0600, and removal before it returns. The std library has no safe euid call and this crate forbids unsafe code, so the probe stays. A test plants a file and a symlink at the old and a similar name and checks that both are untouched and that the probe file is gone. macos_fallback_is_per_user compares Unix paths, so it and the macos_runtime_path helper it needs are now compiled for tests on Unix only. That removes the unused function warning in Windows test builds. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * craft-ipc: an event larger than one line becomes an events.overflow marker Server::emit wrote the serialized event unchecked. A line over MAX_LINE made Client::next_event return TooLarge and left the stream out of sync. enqueue now checks the serialized size and queues the existing overflow marker instead. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NFCDiYoKS325v2rtv2F7D8 * ipc: validate executable path ancestry * ipc: preserve queued replies after macOS peer shutdown * ipc: validate native transports and use portable platform fixtures * Make IPC integration fixtures portable and collision-free * Use native paths and unique directories across IPC fixtures * Bound Windows pipe I/O and stabilize IPC tests * Retry empty nonblocking Windows pipes * #433: Serialize hub tests sharing a Windows pipe * #433: Use native endpoint in schema capability tests * ipc: report a non-regular watch target as Invalid on every platform * ipc: format the new error classification --------- Co-authored-by: User Co-authored-by: Claude Sonnet 5.5 --- Cargo.lock | 286 ++++++ Cargo.toml | 1 + README.md | 1 + crates/ipc/Cargo.toml | 34 + crates/ipc/PROTOCOL.md | 297 ++++++ crates/ipc/README.md | 38 + .../ipc/schemas/instance-record.schema.json | 63 ++ crates/ipc/schemas/link-record.schema.json | 63 ++ crates/ipc/src/activate.rs | 362 ++++++++ crates/ipc/src/asset.rs | 70 ++ crates/ipc/src/auth.rs | 107 +++ crates/ipc/src/bulk.rs | 487 ++++++++++ crates/ipc/src/cache.rs | 278 ++++++ crates/ipc/src/client.rs | 853 ++++++++++++++++++ crates/ipc/src/discovery/dir.rs | 354 ++++++++ crates/ipc/src/discovery/mod.rs | 11 + crates/ipc/src/discovery/record.rs | 575 ++++++++++++ crates/ipc/src/error.rs | 134 +++ crates/ipc/src/fetch.rs | 370 ++++++++ crates/ipc/src/fsopen.rs | 25 + crates/ipc/src/ids.rs | 41 + crates/ipc/src/launch.rs | 777 ++++++++++++++++ crates/ipc/src/lib.rs | 39 + crates/ipc/src/moniker.rs | 313 +++++++ crates/ipc/src/payload.rs | 396 ++++++++ crates/ipc/src/record.rs | 237 +++++ crates/ipc/src/refresh.rs | 572 ++++++++++++ crates/ipc/src/server/dispatch.rs | 316 +++++++ crates/ipc/src/server/events.rs | 214 +++++ crates/ipc/src/server/listener.rs | 633 +++++++++++++ crates/ipc/src/server/mod.rs | 376 ++++++++ crates/ipc/src/server/provider.rs | 71 ++ crates/ipc/src/server/tcp.rs | 32 + crates/ipc/src/trust.rs | 215 +++++ crates/ipc/src/watch.rs | 397 ++++++++ crates/ipc/src/wire.rs | 561 ++++++++++++ crates/ipc/tests/bulk.rs | 334 +++++++ crates/ipc/tests/connection_limits.rs | 280 ++++++ crates/ipc/tests/discovery.rs | 97 ++ crates/ipc/tests/fetch.rs | 366 ++++++++ .../link-record-filmcraft-effectcraft.json | 13 + .../ipc/tests/fixtures/link-record-still.json | 15 + crates/ipc/tests/hub.rs | 334 +++++++ crates/ipc/tests/native_roundtrip.rs | 93 ++ crates/ipc/tests/open_payload.rs | 169 ++++ crates/ipc/tests/protocol.rs | 361 ++++++++ crates/ipc/tests/provider_stall.rs | 248 +++++ crates/ipc/tests/schemas.rs | 318 +++++++ crates/ipc/tests/security.rs | 290 ++++++ crates/ipc/tests/watch.rs | 361 ++++++++ 50 files changed, 12878 insertions(+) create mode 100644 crates/ipc/Cargo.toml create mode 100644 crates/ipc/PROTOCOL.md create mode 100644 crates/ipc/README.md create mode 100644 crates/ipc/schemas/instance-record.schema.json create mode 100644 crates/ipc/schemas/link-record.schema.json create mode 100644 crates/ipc/src/activate.rs create mode 100644 crates/ipc/src/asset.rs create mode 100644 crates/ipc/src/auth.rs create mode 100644 crates/ipc/src/bulk.rs create mode 100644 crates/ipc/src/cache.rs create mode 100644 crates/ipc/src/client.rs create mode 100644 crates/ipc/src/discovery/dir.rs create mode 100644 crates/ipc/src/discovery/mod.rs create mode 100644 crates/ipc/src/discovery/record.rs create mode 100644 crates/ipc/src/error.rs create mode 100644 crates/ipc/src/fetch.rs create mode 100644 crates/ipc/src/fsopen.rs create mode 100644 crates/ipc/src/ids.rs create mode 100644 crates/ipc/src/launch.rs create mode 100644 crates/ipc/src/lib.rs create mode 100644 crates/ipc/src/moniker.rs create mode 100644 crates/ipc/src/payload.rs create mode 100644 crates/ipc/src/record.rs create mode 100644 crates/ipc/src/refresh.rs create mode 100644 crates/ipc/src/server/dispatch.rs create mode 100644 crates/ipc/src/server/events.rs create mode 100644 crates/ipc/src/server/listener.rs create mode 100644 crates/ipc/src/server/mod.rs create mode 100644 crates/ipc/src/server/provider.rs create mode 100644 crates/ipc/src/server/tcp.rs create mode 100644 crates/ipc/src/trust.rs create mode 100644 crates/ipc/src/watch.rs create mode 100644 crates/ipc/src/wire.rs create mode 100644 crates/ipc/tests/bulk.rs create mode 100644 crates/ipc/tests/connection_limits.rs create mode 100644 crates/ipc/tests/discovery.rs create mode 100644 crates/ipc/tests/fetch.rs create mode 100644 crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json create mode 100644 crates/ipc/tests/fixtures/link-record-still.json create mode 100644 crates/ipc/tests/hub.rs create mode 100644 crates/ipc/tests/native_roundtrip.rs create mode 100644 crates/ipc/tests/open_payload.rs create mode 100644 crates/ipc/tests/protocol.rs create mode 100644 crates/ipc/tests/provider_stall.rs create mode 100644 crates/ipc/tests/schemas.rs create mode 100644 crates/ipc/tests/security.rs create mode 100644 crates/ipc/tests/watch.rs diff --git a/Cargo.lock b/Cargo.lock index 3385eab1f..5f28d1878 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,12 +116,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "serde", "version_check", "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned" version = "0.4.3" @@ -576,6 +586,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + [[package]] name = "bit-set" version = "0.10.0" @@ -594,6 +613,12 @@ dependencies = [ "bit-vec 0.10.1", ] +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -772,6 +797,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "borsh" version = "1.8.1" @@ -883,6 +914,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.2" @@ -2031,6 +2068,21 @@ dependencies = [ "indd", ] +[[package]] +name = "craft-ipc" +version = "0.1.0" +dependencies = [ + "chrono", + "getrandom 0.2.17", + "interprocess", + "jsonschema", + "libc", + "serde", + "serde_json", + "sha2 0.10.9", + "widestring", +] + [[package]] name = "craft-jmap" version = "0.1.0" @@ -3703,6 +3755,12 @@ dependencies = [ "libloading", ] +[[package]] +name = "doctest-file" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359" + [[package]] name = "document-features" version = "0.2.12" @@ -4051,6 +4109,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "emath" version = "0.37.0" @@ -4263,6 +4330,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -4376,6 +4454,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fmt-derive" version = "0.1.2" @@ -4528,6 +4617,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs-set-times" version = "0.20.3" @@ -4959,6 +5058,8 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -5607,6 +5708,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "interprocess" +version = "2.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb02ffbf25e8ef0f4a95bff054eef75f6d1bf16919474cf2caa63b21997478a" +dependencies = [ + "doctest-file", + "libc", + "recvmsg", + "widestring", + "windows-sys 0.61.2", +] + [[package]] name = "io-extras" version = "0.19.0" @@ -5784,6 +5898,58 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonschema" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "jsonschema-regex", + "jsonschema-value", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "strum", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "jsonschema-value" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" +dependencies = [ + "ahash", + "bytecount", + "fraction", + "num-cmp", + "num-traits", + "serde_json", +] + [[package]] name = "jxl-bitstream" version = "1.1.0" @@ -6308,6 +6474,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + [[package]] name = "mime" version = "0.3.17" @@ -6552,6 +6724,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -6579,6 +6765,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.6" @@ -7094,6 +7286,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -8219,6 +8417,12 @@ dependencies = [ "yoke", ] +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + [[package]] name = "redox_syscall" version = "0.4.1" @@ -8256,6 +8460,66 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "referencing" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown 0.17.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-lite" version = "0.1.9" @@ -10525,6 +10789,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "v_frame" version = "0.3.9" @@ -10847,6 +11121,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -11316,6 +11596,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi-util" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index 801dc9ba8..0ed80bda2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,7 @@ craft-heif = { path = "crates/heif" } craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } +craft-ipc = { path = "crates/ipc" } craft-idml = { path = "crates/idml" } craft-indd = { path = "crates/indd" } craft-jmap = { path = "crates/jmap" } diff --git a/README.md b/README.md index a64889992..47b223d48 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit primitives (`.abr`, `.asl`, `.aco`, `.act`, `.ase`, `.grd`, `.pat`, `.atn`, `.csh`, `.shc`) and the Photoshop blocks of a layered TIFF. Bounded and panic-free on hostile input. Compositing and the mapping onto a document stay in each app. +- [`craft-ipc`](crates/ipc/README.md): Craft apps talking on one machine: per-user local sockets (Unix socket, Windows named pipe), token auth, instance discovery, launch on demand, focus hand-off, frame transfer and linked-file watching. Native only: it does not build for `wasm32`. Protocol in [PROTOCOL.md](crates/ipc/PROTOCOL.md). - [`craft-jmap`](crates/jmap/README.md): the JMAP protocol core (RFC 8620), from MailCraft, without sockets: session, requests of method calls with back-references, responses, ids, address templates, credentials and the judgement of HTTP answers, over any HTTP implementation. Bounded diff --git a/crates/ipc/Cargo.toml b/crates/ipc/Cargo.toml new file mode 100644 index 000000000..5d4b0ff06 --- /dev/null +++ b/crates/ipc/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "craft-ipc" +description = "Local-socket control server, client and discovery for Craft apps on one machine: token auth, launch on demand, focus hand-off. Native only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +# Native only: `interprocess` has no wasm32 build, so this crate does not build for +# wasm32-unknown-unknown. Apps gate it by target. interprocess is pinned: the transport behavior is +# what the tests were run against. +[dependencies] +# Calendar checks only (AGENTS.md: all date logic goes through chrono). +chrono.workspace = true +getrandom = "0.2.15" +interprocess = "=2.4.4" +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10.9" + +[target.'cfg(unix)'.dependencies] +# O_NONBLOCK and O_NOFOLLOW for opens that must not wait on a FIFO or follow a symlink. Constants only. +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +widestring = "1" + +[dev-dependencies] +# Default features off: the tests read local schemas and need no network resolver. +jsonschema = { version = "0.49.9", default-features = false } + +[lints] +workspace = true diff --git a/crates/ipc/PROTOCOL.md b/crates/ipc/PROTOCOL.md new file mode 100644 index 000000000..27945f021 --- /dev/null +++ b/crates/ipc/PROTOCOL.md @@ -0,0 +1,297 @@ +# Craft Link protocol, version 1 + +Extends the existing Craft control channel (the control protocol document of each app). Every +existing method keeps its exact behaviour. New methods are additive. `craft-ipc` implements this +protocol. + +## 1. Transport and framing + +- **Endpoint**: from the instance record (`schemas/instance-record.schema.json`). Unix domain socket file + (mode 0600, directory 0700) on Linux and macOS; named pipe with an owner-only SDDL descriptor on + Windows; loopback TCP only when the user opts in (`--control `, legacy). +- **Control framing**: one UTF-8 JSON object per line, max 4 MiB per line. A request has `method` + (string), optional `id` (number or string), optional `params` (object). A reply has the request's + `id`, `ok` (bool), and `result` or `error`. Malformed input gets one error line and the connection is + closed (existing behaviour). +- **Push events** (new): lines with `event` and `seq` and no `id`: + `{"event": "", "seq": , "data": {...}}`. Sent only on connections that called + `events.subscribe`. Per-subscriber queue of 256; on overflow the server sends + `{"event":"events.overflow","seq":n,"data":{"since":}}` and drops events until the client + catches up. +- **Bulk connection** (new): a second connection to the same endpoint, used only for frames (section 6). +- **Connection limit**: 16 concurrent connections per app (existing). Bulk connections count. At most + 4 of the 16 may be unauthenticated at one time; the other 12 slots go only to peers that sent the + right token. A peer past either limit gets one line `{"ok":false,"error":"too many control + connections"}` or `"too many unauthenticated connections"` and a close. +- **Time limits** (server defaults): a connection must send a valid `auth` line within 2 s of the + accept; a request line that has started must end within 10 s; an authenticated control + connection with no input for 1 h, no event subscription and no bulk role is closed with + `{"id":null,"ok":false,"error":"idle connection closed"}`. A client that keeps a quiet connection + longer must call `events.subscribe` or reconnect. A request may arrive in any number of pieces + with pauses between them; the server keeps the bytes of an unfinished line. +- **Launch on demand** (library rule, not wire): `craft_ipc::launch` runs an app binary only if the + path is absolute and the file is regular and executable, and the file and its directory belong to + the user or root and are not writable by every user. Empty and relative `PATH` entries are + skipped. A fork that starts an app by another route must call `craft_ipc::trust::check_executable`. + Caches use `craft_ipc::cache::private_cache_dir` (`0700`, owner checked, no symlink, no + fallback to a shared temp directory); `CRAFT_LINK_CACHE_DIR` must be absolute. + +## 2. Authentication (required on every transport: sockets, pipes, loopback TCP) + +The first line on every connection must be: + +```json +{"id": 0, "method": "auth", "params": {"token": "<64 hex from the instance record>"}} +``` + +Reply `{"id":0,"ok":true,"result":{"authenticated":true}}`, or +`{"id":0,"ok":false,"error":"authentication required"}` followed by close. The server also checks +the peer's user id (Linux/macOS euid; Windows token user) and refuses other users. Any other first +line gets the same refusal. (Shape copied from PhotoCraft's existing handshake.) + +Loopback TCP (`--control `, legacy) runs the same handshake, the same unauthenticated cap and +the same 2 s auth deadline. It binds 127.0.0.1 only. There is no uid check on TCP, so the token is +the only proof. The instance record (0600) publishes the port as `tcp_port` next to the `token`; +a TCP client reads both from the record (`Client::connect_tcp`). Never print the token or put it +in a config file or on a command line. A TCP client without a token gets the auth refusal. + +## 3. Session methods (answered on the connection thread, no UI frame needed) + +### `link.hello` +```json +{"id":1,"method":"link.hello","params":{"client":{"app":"filmcraft","app_version":"0.9.1","pid":4242,"instance":""},"protocol":{"min":1,"max":1}}} +-> {"id":1,"ok":true,"result":{"app":"effectcraft","app_version":"0.8.3","pid":5151,"instance":"","protocol":1,"caps":["events","frames","activate","link.receive"]}} +``` +Errors: `protocol_unsupported{server_min, server_max}`. The client must check `pid` and `instance` +equal the instance record. Legacy clients that never call `link.hello` keep today's behaviour. + +`caps` is the fixed base set (`events`, `activate`, `link.receive`, `frames`) plus the `caps` array of +the instance record (`schemas/instance-record.schema.json`), without duplicates. Roles and optional verbs use +this array. Defined names: + +| Cap | Meaning | +|---|---| +| `hub` | The peer is a browser or launcher hub. It owns no document, it only reads and opens assets (section 9). | +| `assets` | The peer answers `link.resolve` and `asset.describe` itself, on the connection thread (section 4). | + +A client that does not know a cap ignores it. + +### `events.subscribe` / `events.unsubscribe` +```json +{"id":2,"method":"events.subscribe","params":{"events":["doc.changed","link.changed","app.closing"]}} +-> {"id":2,"ok":true,"result":{"seq":0}} +``` + +### `app.activate` +```json +{"id":3,"method":"app.activate","params":{"token":"","from_pid":4242,"open":{"link_id":""}}} +-> {"id":3,"ok":true,"result":{"activated":"requested"}} +``` +`activated` has three values: + +| Value | Meaning | +|---|---| +| `true` | The window has focus now. | +| `"requested"` | The app asked the OS to focus the window. The OS decides later. Nobody confirmed it, and the reply does not say it will happen. | +| `false` | Focus was not taken. The app asked for user attention instead (taskbar flash, dock bounce). | + +On Linux, `"requested"` needs a token that the app could apply to its existing window. An app whose +toolkit can mint a token but cannot apply one (stock winit 0.30 on Wayland) answers `false`. The +sender can still pass `token`. Old senders and receivers keep working: the three values did not change. +`activated` is distinct from the existing `ui.focus` method, which never steals focus by design. + +`open` is an object that names what to show. It carries `link_id` (open a link), `asset` (a moniker, +section 9: open that asset in its document) or both. A hub or launcher uses `asset` to send the user +to the app and document that owns the asset, with no link in between. The server rejects an `asset` +that is not a valid moniker with `bad_params`. An `open` with neither key is passed on unchanged +(existing behaviour). The receiving app decides what "show" means for the asset. + +### `link.bye` +Clean close. The server drops subscriptions and bulk tickets of this connection. + +## 4. Link methods (run on the UI thread through the app's existing request channel) + +### `link.open` (FilmCraft -> EffectCraft) +```json +{"id":10,"method":"link.open","params":{ + "link_id":"", + "payload":{"format":"otio-json","data":""}, + "place":{"seconds":12.48,"selection":[{"start_s":10.0,"end_s":14.0,"track_index":0}],"view":{"zoom":1.0}}, + "result_path":"/abs/path/Project Links/.mov", + "source":{"app":"filmcraft","doc_id":"","item":""}}} +-> {"id":10,"ok":true,"result":{"comp":"","project":"/abs/path/Project Links/.ecproj", + "format":{"w":1920,"h":1080,"rate_num":24,"rate_den":1,"duration_ticks":...,"alpha":true}, + "unsupported":[{"what":"effect","name":"Lumetri Color","kept_as":"none"}]}} +``` +Errors: `payload_invalid`, `media_missing`, `payload_too_large` (OTIO string > 4 MiB), `io_error`. +A request line longer than 4 MiB (`MAX_LINE`) is not an error reply: the server answers once in text +and closes the connection. `unsupported` lists parts that +did not survive the hand-off. + +**Payload formats.** `payload.format` selects the shape of the rest of `payload`: + +| `format` | Other fields | Use | +|---|---|---| +| `otio-json` | `data`: OTIO JSON string | Video: a nested sequence (above). | +| `image-doc` | `asset`: moniker (section 9), `path`: absolute path of the producer document or image file | Stills, vectors and other single images. `result_path` is the last-known flattened image. `place`, `source` and `format` in the reply stay as above. `rate_num`, `rate_den` and `duration_ticks` are absent from `format` in the reply. | + +An unknown `format` gets `payload_invalid`. A new `format` value is additive and needs no protocol +bump. The server checks the shape of `payload` with `craft_ipc::payload::OpenPayload` before it +forwards `link.open` to the app. A missing `payload`, a `payload` that is not an object, a missing or +unknown `format`, a missing field, a relative `path` or a bad `asset` moniker gets `payload_invalid` +and the app never sees the request. `params` that is not an object stays `bad_params`. The server +checks shape, and for `otio-json` it also opens every `ExternalReference` whose `target_url` is a +`file://` URL or an absolute path. A file that does not open gets `media_missing` with +`data.missing` (the paths) and the app never sees the request. A `file://` URL whose path has a bad +percent escape, or does not decode to UTF-8, gets `payload_invalid`. Relative targets are left to the app. + +Path rules. `result_path`, `payload.path` and every absolute media `target_url` must +be absolute, hold no NUL byte and no `..` component, or the request gets `payload_invalid`. The +server does not canonicalise and does not confine paths to a root: footage lives anywhere and +may be a symlink. The media check is the only file probe. It looks only at paths the payload +names, at most 4096 distinct ones, and reports a path as missing when it is not a regular file +(after symlinks) or does not open for reading. A missing file, a directory, a FIFO, a device and a +file without read access all look the same, and the probe never opens a FIFO. An OTIO document +with more than 4096 distinct files, more than 2^20 JSON values or nesting over 128 gets +`payload_invalid`. The app must apply its own checks to `result_path` before it writes there; +the server check is a first filter. +An app still answers `payload_invalid` when the content is bad (for example OTIO that does not +parse). + +### `link.attach` (FilmCraft -> EffectCraft, reopen an existing link) +`{"link_id","project","comp"}` -> `{"format", "epoch"}`. Errors: `comp_missing` (link becomes `Broken`). +The receiver makes the comp of the link the active one. It reads `project` (the sidecar written by +`link.open`) when it holds no such link and has no unsaved edits, otherwise `busy`. +FilmCraft sends this when `link.editIn` runs on a clip that plays a linked item already. It nests +nothing and creates no item. On `comp_missing` or `io_error` (no sidecar, damaged sidecar) it sends +`link.open` again with the same `link_id` and `result_path`, built from the nest in `nested_sequence`. + +### `link.focus` (either direction, used by Back) +`{"link_id","place"}` -> `{}`. The receiver moves to `place` and is then activated with `app.activate`. + +### `link.close` +`{"link_id"}` -> `{}`. EffectCraft drops the binding; the sidecar project stays on disk. + +### `link.refresh` (consumer side, any app that holds a link) +`{"link_id"}` -> `{"link_id","epoch","state":"current"|"modified"|"missing","result_path","hash"}`. +The receiver compares the last-known result file with the disk (rules in section 10), bumps the epoch +when the content changed, and answers the state. `hash` is `sha256:<64 hex>` and is absent when the +state is `missing`. A link that exists in the receiver but whose file is gone is not an error. The +answer is `state:"missing"`. Errors: `link_unknown`, `bad_params`, `io_error`. + +### `link.resolve` (producer or hub, any app that owns assets) +`{"asset":"craft://..."}` -> `{"asset","path","epoch","size","hash","mtime_ms"}`. `path` is an +absolute path of the last-known flattened result that a consumer can read now. `hash` is +`sha256:<64 hex>` of that file. `epoch` is the producer epoch. Errors: `link_unknown` (no such asset, +or the asset has no result file yet), `bad_params` (not a moniker), `io_error`. + +### `asset.describe` (producer or hub) +`{"asset":"craft://..."}` -> `{"asset","name","kind","epoch","w","h","hash","thumb_path","modified_ms"}`. +`kind` is `still`, `vector`, `sequence` or `comp`. `w`, `h`, `hash` and `thumb_path` are `null` when +unknown. `thumb_path` is an absolute path of a PNG, at most 512 px on the long side, that the +producer keeps in a per-document sidecar. A hub browses assets with this call and never opens the +document. Errors as `link.resolve`. + +`link.resolve` and `asset.describe` have no side effects and never need a UI frame. A server that was +started with an asset provider (`Server::start_with` and `Extensions`) answers both on the connection thread +and lists `assets` in `caps`. `link.refresh` is answered by the provider the same way. A server +started without a provider forwards all three to the app like any other link method (existing +behaviour). + +### Events +- `link.changed`: `{"link_id","epoch"}`. Sent by EffectCraft when the bound comp changes (any + `bump()` whose change touches the comp). +- `link.result`: `{"link_id","epoch","path"}`. A fresh last-known result was written. +- `doc.changed`: `{"doc_id","revision","cause"}`. Generic change feed (from `craft-command`). +- `app.closing`: `{}`. Sent before a clean exit. + +## 5. Command passthrough (existing, unchanged) + +`engine.execute {command, params}`, `engine.commands`, `ui.*`, `perf.stats` keep their current +contracts. `engine.commands` now also lists plugin commands and carries a `schema` per command +(`x-craft-source`: `doc` | `derived` | `opaque`). + +## 6. Bulk frames + +1. Control connection: `frames.open {"link_id","format":"rgba8_premul"|"rgba16f_premul","scale":1.0}` + -> `{"ticket":"<32 hex>","w","h","stride","len"}`. +2. New connection: `auth`, then `{"method":"bulk.attach","params":{"ticket":"..."}}` -> `{"ok":true}`. + The connection is now binary in both directions. +3. Client request record (binary): `'CFRQ' | u32 header_len | header JSON` with header + `{"req":,"time_s":,"epoch":,"region":[x,y,w,h]|null}`. +4. Server reply record: `'CFRM' | u32 header_len | u64 payload_len | header JSON | payload`, header + `{"req","epoch","w","h","stride","format","render_ms"}`. `epoch` is the epoch the frame was + rendered at; a client that asked for an older epoch may get a newer one. `payload_len` is + `stride * h`, and `stride` is at least one row: `w` times 4 bytes for `rgba8_premul`, `w` times 8 + bytes for `rgba16f_premul`. A layout that breaks this is refused at `frames.open`, and a reply + that breaks it is refused by the server and by the client. +5. Errors: header `{"req","error":"..."}` with `payload_len` 0. +All integers little endian. Tickets expire when the control connection closes. + +## 7. Error codes (in `error` objects for new methods) + +`{"code": "", "message": "", "data": {...}}`. Codes: `auth_required`, +`protocol_unsupported`, `unknown_method`, `bad_params`, `not_ready`, `payload_invalid`, +`media_missing`, `payload_too_large`, `comp_missing`, `link_unknown`, `io_error`, `busy`, `internal`. +`link_unknown` also covers an unknown asset moniker. +Existing methods keep their current string `error` field. + +## 8. Versioning + +- `protocol` is an integer. A server serves any client version in `[min, max]`. New methods and new + optional fields do not bump the version. Removing or changing the meaning of a field does. +- Method names are never reused with a different meaning. + +## 9. Asset identity: the moniker + +An asset is addressed by a moniker, a string that stays valid across restarts: + +``` +craft:///// +``` + +| Part | Rule | +|---|---| +| `app` | Same pattern as the instance record `app`: `^[a-z][a-z0-9-]{1,31}$`. The app that owns the document. | +| `doc` | The document id, a UUID in 8-4-4-4-12 hex form. Parsers accept either hex case and keep the text as written. | +| `kind` | `^[a-z][a-z0-9-]{0,31}$`. Known values: `comp`, `image`, `vector`, `sequence`, `layer`. An unknown value is valid and is kept. | +| `id` | Non-empty, at most 512 bytes after decoding, valid UTF-8. Characters other than `A-Z a-z 0-9 - . _ ~` are percent-encoded as `%XX` (uppercase hex) by the formatter. The parser accepts either hex case. A raw `/`, `?`, `#` or `%` not followed by two hex digits is invalid. | + +The `asset.moniker` pattern in `schemas/link-record.schema.json` states the same `id` rules, so a +record that the schema accepts also parses in Rust. + +A moniker has no query and no fragment. It names an asset, not a state: the state is the epoch. +A moniker can outlive its asset. Resolving a dead moniker gives `link_unknown`. Parsing then +formatting a moniker gives the canonical text (uppercase `%XX`). Formatting then parsing gives back +an equal moniker. `craft_ipc::moniker::Moniker` implements both. The module uses only `serde` and +`std`, so an app can copy it to a wasm build. The `craft-ipc` crate as a whole still does not build +for wasm32 (local sockets). + +## 10. Refresh and watch rules for linked files + +These rules apply to any linked asset with a `result_path` (a still, a rendered movie). + +1. **Producer writes are atomic.** The producer writes the file to a temporary name in the same + directory (`....tmp`), flushes it, renames it over `result_path`, and then flushes + the directory. A reader sees the old file or the new file, never a half file. A reader ignores + names that start with `.` and end with `.tmp`. +2. **Epoch.** `result_epoch` is a counter of result versions. The producer adds 1 on every saved + change and emits `link.changed` and `link.result`. A consumer that detects a change by polling + (rule 4) adds 1 to its own stored epoch. When the producer is reachable again, the consumer keeps + the larger of the two values. An epoch never decreases. +3. **Fingerprint.** `(size, mtime_ms, sha256)` of the file. The hash is of the full content. +4. **Polling when the producer is offline.** The consumer calls `poll` about once per second + (`craft_ipc::watch::Watcher` runs this schedule with a thread and an injectable ticker). A poll + reads size and mtime. It hashes the file only when size or mtime differ from the last fingerprint. + Some file systems have coarse mtime, so the consumer also calls a deep poll (always hash) about + once every 30 seconds. A touch that leaves the hash equal does not bump the epoch. +5. **States.** `current` (disk equals what the consumer loaded), `modified` (disk differs from what + the consumer loaded; the epoch was bumped), `missing` (the file does not exist). The consumer + moves `modified` to `current` when it has loaded the new content. A file that returns with the + hash it had when loaded goes back to `current` with no epoch bump. A file that returns with a + different hash is `modified` and bumps the epoch. +6. **Other read errors** (permission denied, I/O error) are errors, not `missing`. The state does not + change and the next poll tries again. +7. **Relink.** A consumer can point a link at a new path. The new path is polled at once. The state + is `current` when its hash equals what the consumer loaded and `modified` (epoch bump) otherwise. + A missing new path is rejected and the old path stays. diff --git a/crates/ipc/README.md b/crates/ipc/README.md new file mode 100644 index 000000000..8bcdbea66 --- /dev/null +++ b/crates/ipc/README.md @@ -0,0 +1,38 @@ +# craft-ipc + +`craft-ipc` is how a Craft app talks to another Craft app on the same machine. It has a server, a client, instance discovery, launch on demand and focus hand-off. It has no async runtime. The wire protocol is in [PROTOCOL.md](PROTOCOL.md). The crate depends on no app and on no other Craft crate. + +**Native only.** `craft-ipc` does not build for `wasm32-unknown-unknown`, because `interprocess` (local sockets and named pipes) has no wasm32 build. An app gates the dependency by target. The pure data modules (`moniker`, `record`, `payload`, `asset`, `ids`) use only `std` and `serde`. + +## Boundaries + +- Never crashes: no panics, no `unsafe`. +- Every connection must present a 64 hex token within 2 seconds. The server also checks the peer user id (Unix) or the token user (Windows). At most 16 connections are open, and at most 4 of them are unauthenticated. +- One UTF-8 JSON object per line, at most 4 MiB per line. A request can arrive in pieces with pauses between them. `wire::LineReader` keeps the partial line. Do not read control lines with a stateless `read_line` on a stream that has a receive timeout. +- On Unix, the socket file is mode `0600` inside a `0700` directory. The Windows code paths (named pipe, token user check) have not been run on Windows. + +## Modules + +| Module | Role | +|---|---| +| `server`, `client`, `wire`, `bulk`, `auth`, `launch`, `activate`, `discovery` | transport, session, frames, focus hand-off, instance records | +| `moniker`, `record`, `payload`, `asset`, `ids` | pure data types | +| `refresh` | `Tracker` (poll, epoch bump, relink) and `write_atomic` for linked files | +| `fetch` | `FrameFetcher`: ticket, a few `BulkClient` workers, a RAM cache keyed `FrameKey` with a byte budget and LRU eviction. `fetch` never waits past its `wait` argument. A late frame still lands in the cache | +| `watch` | `Watcher` thread that polls `Tracker`s: shallow every second, deep every 30 seconds. `IntervalTicker` runs on real time and `ManualTicker` is for tests | +| `trust`, `cache` | `check_executable` guards every launch (absolute path, regular file, executable, owner and mode checks). `private_cache_dir`, `write_private` and `read_private` make a `0700` cache with no `/tmp` fallback | + +- `server::Limits` holds the connection limits: auth deadline 2 s, started line 10 s, idle 1 h for unsubscribed control connections, 4 unauthenticated of 16. +- `payload::check_peer_path` is the one path filter: absolute, no NUL, no `..`. The media probe only touches regular files and never opens a FIFO. +- `Server::start_with(config, Extensions { assets, caps })` adds an asset provider and extra caps. `InstanceRecord` keeps `caps` in `extra`. Use `caps()` and `add_cap()`. +- Discovery: each app writes one JSON record, mode `0600`, into a per-user runtime directory. A lock file held for the life of the app shows whether the owner is alive. +- Launch: `launch::resolve` finds an app binary (environment, a read-only parse of the launcher's install record, `PATH`, usual install locations). `launch` starts it only when `trust::check_executable` accepts the file. +- Loopback TCP is available only when the app opts in. It binds 127.0.0.1 and runs the same handshake. A TCP client reads the port and token from the instance record (`Client::connect_tcp`). + +## Schemas and fixtures + +`schemas/instance-record.schema.json` and `schemas/link-record.schema.json` describe the stored records. `tests/schemas.rs` checks that each schema and its Rust type agree, and that the stored records in `tests/fixtures/` stay valid. + +## Tests + +A forked child holds a copy of every open file descriptor until exec, so a `flock` released by `drop` can look held. Unit tests that start a process take `crate::SPAWN_GUARD.write()`. Unit tests that publish or scan a discovery lock take `.read()`. diff --git a/crates/ipc/schemas/instance-record.schema.json b/crates/ipc/schemas/instance-record.schema.json new file mode 100644 index 000000000..6cc9b4f15 --- /dev/null +++ b/crates/ipc/schemas/instance-record.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "craft://schemas/link/instance-record/1", + "title": "Craft Link instance record", + "description": "Written by each running Craft app to /craft/-.json (mode 0600). See PROTOCOL.md.", + "type": "object", + "required": ["v", "app", "app_version", "pid", "instance", "endpoint", "token", "started", "protocol", "docs"], + "additionalProperties": true, + "properties": { + "v": { "const": 1 }, + "app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$" }, + "app_version": { "type": "string" }, + "pid": { "type": "integer", "minimum": 1 }, + "instance": { "type": "string", "format": "uuid" }, + "endpoint": { + "oneOf": [ + { + "type": "object", + "required": ["kind", "path"], + "properties": { "kind": { "const": "unix" }, "path": { "type": "string", "maxLength": 100 } } + }, + { + "type": "object", + "required": ["kind", "name"], + "properties": { "kind": { "const": "pipe" }, "name": { "type": "string", "pattern": "^craft-" } } + }, + { + "type": "object", + "required": ["kind", "port"], + "properties": { "kind": { "const": "tcp" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 } } + } + ] + }, + "token": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "started": { "type": "integer", "description": "unix seconds" }, + "protocol": { + "type": "object", + "required": ["link"], + "properties": { + "link": { "type": "integer", "minimum": 1 }, + "commands_schema": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } + } + }, + "caps": { + "type": "array", + "description": "Roles and optional verbs this app offers. Merged into the link.hello caps (link-protocol.md section 3). 'hub' marks a hub peer, 'assets' marks an app that answers link.resolve and asset.describe on the connection thread. Absent means none.", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z][a-z0-9._-]{0,63}$" } + }, + "docs": { + "type": "array", + "items": { + "type": "object", + "required": ["doc_id", "title"], + "properties": { + "doc_id": { "type": "string" }, + "title": { "type": "string" }, + "path": { "type": "string" } + } + } + } + } +} diff --git a/crates/ipc/schemas/link-record.schema.json b/crates/ipc/schemas/link-record.schema.json new file mode 100644 index 000000000..0a1183085 --- /dev/null +++ b/crates/ipc/schemas/link-record.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "craft://schemas/link/link-record/1", + "title": "craft.link project item metadata", + "description": "Stored in the consumer's project item metadata[\"craft.link\"] on the linked media item. Unknown fields must be preserved on save. Records written before the asset fields existed (FilmCraft to EffectCraft) stay valid.", + "type": "object", + "required": ["v", "link_id", "target_app", "target_project", "result_path", "result_epoch", "format", "created"], + "anyOf": [ + { "required": ["target_comp"] }, + { "required": ["asset"] } + ], + "additionalProperties": true, + "properties": { + "v": { "const": 1 }, + "link_id": { "type": "string", "format": "uuid" }, + "target_app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$", "description": "app id of the producer, same pattern as the instance record app" }, + "target_project": { "type": "string", "description": "path of the producer document, relative to the consumer project file when possible" }, + "target_comp": { "type": "string", "description": "comp id inside the producer document. Required unless asset is present." }, + "nested_sequence": { "type": "string", "description": "FilmCraft item id of the nest created by link.editIn. Absent for stills." }, + "asset": { + "type": "object", + "required": ["kind", "moniker"], + "additionalProperties": true, + "properties": { + "kind": { "enum": ["still", "vector", "sequence", "comp"] }, + "moniker": { + "type": "string", + "description": "craft://app/doc/kind/id. The id is 1 to 512 bytes. Each byte is an unreserved character (A-Z a-z 0-9 . _ ~ -) or a percent escape of two hex digits, and the bytes the escapes decode to must be valid UTF-8. The first pattern checks the shape and the byte count (one character or one escape is one byte). The second checks the UTF-8.", + "allOf": [ + { "pattern": "^craft://[a-z][a-z0-9-]{1,31}/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/[a-z][a-z0-9-]{0,31}/(?:[A-Za-z0-9._~-]|%[0-9A-Fa-f]{2}){1,512}$" }, + { "pattern": "^craft://[^/]+/[^/]+/[^/]+/(?:[A-Za-z0-9._~-]|%[0-7][0-9A-Fa-f]|%(?:[Cc][2-9A-Fa-f]|[Dd][0-9A-Fa-f])%[89ABab][0-9A-Fa-f]|%[Ee]0%[ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ee](?:[1-9A-Ca-c]|[EeFf])%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ee][Dd]%[89][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff]0%(?:9|[ABab])[0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff][1-3]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff]4%8[0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f])+$" } + ] + } + } + }, + "producer": { + "type": "object", + "required": ["app"], + "additionalProperties": true, + "properties": { + "app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$" }, + "app_version": { "type": "string" }, + "doc_id": { "type": "string" } + } + }, + "result_path": { "type": "string" }, + "result_epoch": { "type": "integer", "minimum": 0 }, + "format": { + "type": "object", + "required": ["w", "h", "alpha"], + "dependentRequired": { "rate_num": ["rate_den"], "rate_den": ["rate_num"] }, + "properties": { + "w": { "type": "integer", "minimum": 1 }, + "h": { "type": "integer", "minimum": 1 }, + "rate_num": { "type": "integer", "minimum": 1 }, + "rate_den": { "type": "integer", "minimum": 1 }, + "duration_ticks": { "type": "integer", "minimum": 0 }, + "alpha": { "type": "boolean" } + } + }, + "created": { "type": "string", "format": "date-time" } + } +} diff --git a/crates/ipc/src/activate.rs b/crates/ipc/src/activate.rs new file mode 100644 index 000000000..476634cdc --- /dev/null +++ b/crates/ipc/src/activate.rs @@ -0,0 +1,362 @@ +//! Focus handoff between Craft apps. +//! +//! The platform calls live behind [`ActivationBackend`]. An app adapter +//! implements it with the real winit and egui calls, so this crate depends on +//! neither. The decision logic here is plain Rust and runs against a fake +//! backend in tests. +//! +//! | Platform | Sender | Receiver | +//! |---|---|---| +//! | Linux | mint a token from the focused window | apply the token if the backend can, then `Focus`; else ask for attention | +//! | Windows | none | `Focus` (winit's foreground path) | +//! | macOS | yield activation to the target bundle | `activate`, then `Focus` | +//! +//! The three answers mean: +//! +//! - [`Activated::Yes`]: the window has focus. +//! - [`Activated::Requested`]: the app asked the OS to focus the window. The OS +//! decides later. Nobody confirmed it. +//! - [`Activated::No`]: focus was not taken. The app asked for user attention +//! instead (taskbar flash, dock bounce). +//! +//! On Linux, a token only counts when the backend can apply it to an existing +//! window ([`ActivationBackend::can_apply_token`]). Stock winit 0.30 cannot on +//! Wayland: the app can mint a token but has no call to use one. In that case +//! the answer is [`Activated::No`], never `Requested`. + +use crate::server::Activation; + +/// The answer to `app.activate`. Same three values as the wire reply. +pub type Activated = Activation; + +/// What a focus request did, as far as the adapter can tell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FocusResult { + /// The window has focus now. + Focused, + /// The request was sent. The compositor or window manager decides later. + Pending, + /// The platform refused or the window is gone. + Refused, +} + +/// Platform calls an app adapter provides. +/// +/// Every method is called on the thread that owns the UI. The adapter maps +/// them to winit and egui: +/// +/// - `mint_token`: egui's `Frame::request_activation_token` on an earlier +/// frame, then `Frame::take_activation_token`. Return the token or `None`. +/// - `can_apply_token`: true only if the winit in use can activate an existing window with a token. +/// - `set_pending_token`: the winit call that stores the token for that window. +/// - `focus`: `ViewportCommand::Focus`. +/// - `request_user_attention`: `ViewportCommand::RequestUserAttention`. +/// - `yield_activation_to` and `activate_application`: the `objc2-app-kit` +/// `NSApplication` and `NSRunningApplication` calls, macOS only. +pub trait ActivationBackend { + /// A fresh activation token, or `None` when the platform cannot mint one. + fn mint_token(&self) -> Option; + /// True when `set_pending_token` followed by `focus` really hands the token + /// to the compositor for an existing window. The default is false: no + /// default success. An adapter returns true only when its winit has a call + /// that applies a token to a live window. + fn can_apply_token(&self) -> bool { + false + } + /// Store `token` so the next `focus` call applies it. Called only when + /// `can_apply_token` is true. + fn set_pending_token(&self, token: &str); + /// Ask the window to take focus. + fn focus(&self) -> FocusResult; + /// Flash the taskbar or dock entry. Used when focus is refused. + fn request_user_attention(&self); + /// macOS sender. Hand activation to the app with `bundle_id`. Returns false on failure. + fn yield_activation_to(&self, bundle_id: &str) -> bool; + /// macOS receiver. Activate this app. Returns false on failure. + fn activate_application(&self) -> bool; +} + +/// The platform whose rules apply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Platform { + Linux, + Windows, + MacOs, + /// Anything else. No handoff is possible. + Other, +} + +impl Platform { + pub const fn current() -> Self { + if cfg!(target_os = "linux") { + Self::Linux + } else if cfg!(target_os = "windows") { + Self::Windows + } else if cfg!(target_os = "macos") { + Self::MacOs + } else { + Self::Other + } + } +} + +/// Sender side. A token to put in `app.activate`, or `None`. +/// +/// Only Linux has tokens. Windows and macOS return `None`. +pub fn mint_token(backend: &dyn ActivationBackend) -> Option { + mint_token_on(Platform::current(), backend) +} + +pub fn mint_token_on(platform: Platform, backend: &dyn ActivationBackend) -> Option { + match platform { + Platform::Linux => backend.mint_token().filter(|token| !token.is_empty()), + Platform::Windows | Platform::MacOs | Platform::Other => None, + } +} + +/// Sender side, before `app.activate`. macOS yields activation to `bundle_id`. +/// Other platforms do nothing and return true. A false return means the yield failed. +pub fn yield_to(backend: &dyn ActivationBackend, bundle_id: &str) -> bool { + yield_to_on(Platform::current(), backend, bundle_id) +} + +pub fn yield_to_on(platform: Platform, backend: &dyn ActivationBackend, bundle_id: &str) -> bool { + match platform { + Platform::MacOs => backend.yield_activation_to(bundle_id), + Platform::Linux | Platform::Windows | Platform::Other => true, + } +} + +/// Receiver side. Apply `app.activate`. `token` is the sender's token or `None`. +pub fn apply_activation(backend: &dyn ActivationBackend, token: Option<&str>) -> Activated { + apply_activation_on(Platform::current(), backend, token) +} + +pub fn apply_activation_on(platform: Platform, backend: &dyn ActivationBackend, token: Option<&str>) -> Activated { + match platform { + Platform::Linux => { + let token = token.filter(|text| !text.is_empty()); + let applied = match token { + Some(token) if backend.can_apply_token() => { + backend.set_pending_token(token); + true + } + _ => false, + }; + let result = backend.focus(); + // Pending means "the compositor may act". Without an applied token it will not. + let result = match (result, applied) { + (FocusResult::Pending, false) => FocusResult::Refused, + (other, _) => other, + }; + settle(backend, result) + } + Platform::Windows => settle(backend, backend.focus()), + Platform::MacOs => { + if !backend.activate_application() { + backend.request_user_attention(); + return Activated::No; + } + settle(backend, backend.focus()) + } + Platform::Other => { + backend.request_user_attention(); + Activated::No + } + } +} + +fn settle(backend: &dyn ActivationBackend, result: FocusResult) -> Activated { + match result { + FocusResult::Focused => Activated::Yes, + FocusResult::Pending => Activated::Requested, + FocusResult::Refused => { + backend.request_user_attention(); + Activated::No + } + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + + use super::*; + + #[derive(Default)] + struct Fake { + minted: Option, + focus: Option, + yield_ok: bool, + activate_ok: bool, + can_apply: bool, + calls: RefCell>, + } + + impl Fake { + fn calls(&self) -> Vec { + self.calls.borrow().clone() + } + fn log(&self, text: impl Into) { + self.calls.borrow_mut().push(text.into()); + } + } + + impl ActivationBackend for Fake { + fn mint_token(&self) -> Option { + self.log("mint"); + self.minted.clone() + } + fn can_apply_token(&self) -> bool { + self.can_apply + } + fn set_pending_token(&self, token: &str) { + self.log(format!("pending:{token}")); + } + fn focus(&self) -> FocusResult { + self.log("focus"); + self.focus.unwrap_or(FocusResult::Refused) + } + fn request_user_attention(&self) { + self.log("attention"); + } + fn yield_activation_to(&self, bundle_id: &str) -> bool { + self.log(format!("yield:{bundle_id}")); + self.yield_ok + } + fn activate_application(&self) -> bool { + self.log("activate"); + self.activate_ok + } + } + + #[test] + fn linux_sets_the_token_before_focus_and_reports_requested() { + let fake = Fake { focus: Some(FocusResult::Pending), can_apply: true, ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("tok")); + assert_eq!(got, Activated::Requested); + assert_eq!(fake.calls(), ["pending:tok", "focus"]); + } + + #[test] + fn linux_token_the_backend_cannot_apply_is_not_a_request() { + // Stock winit on Wayland: a token can be minted but not applied to an existing window. + let fake = Fake { focus: Some(FocusResult::Pending), can_apply: false, ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("tok")); + assert_eq!(got, Activated::No); + assert_eq!(fake.calls(), ["focus", "attention"]); + } + + #[test] + fn linux_token_the_backend_cannot_apply_still_reports_real_focus() { + let fake = Fake { focus: Some(FocusResult::Focused), can_apply: false, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Linux, &fake, Some("tok")), Activated::Yes); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn can_apply_token_defaults_to_false() { + struct Bare; + impl ActivationBackend for Bare { + fn mint_token(&self) -> Option { + None + } + fn set_pending_token(&self, _: &str) {} + fn focus(&self) -> FocusResult { + FocusResult::Pending + } + fn request_user_attention(&self) {} + fn yield_activation_to(&self, _: &str) -> bool { + true + } + fn activate_application(&self) -> bool { + true + } + } + assert!(!Bare.can_apply_token()); + } + + #[test] + fn linux_without_a_token_cannot_claim_requested() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, None); + assert_eq!(got, Activated::No); + assert_eq!(fake.calls(), ["focus", "attention"]); + } + + #[test] + fn linux_empty_token_counts_as_none() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("")); + assert_eq!(got, Activated::No); + assert!(!fake.calls().iter().any(|call| call.starts_with("pending"))); + } + + #[test] + fn linux_focused_window_is_yes_without_attention() { + let fake = Fake { focus: Some(FocusResult::Focused), ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Linux, &fake, None), Activated::Yes); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn refusal_requests_attention_on_every_platform() { + for platform in [Platform::Linux, Platform::Windows, Platform::MacOs] { + let fake = Fake { focus: Some(FocusResult::Refused), activate_ok: true, ..Fake::default() }; + assert_eq!(apply_activation_on(platform, &fake, Some("t")), Activated::No, "{platform:?}"); + assert_eq!(fake.calls().last().map(String::as_str), Some("attention"), "{platform:?}"); + } + } + + #[test] + fn windows_ignores_the_token() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Windows, &fake, Some("t")), Activated::Requested); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn macos_activates_then_focuses() { + let fake = Fake { focus: Some(FocusResult::Focused), activate_ok: true, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::MacOs, &fake, None), Activated::Yes); + assert_eq!(fake.calls(), ["activate", "focus"]); + } + + #[test] + fn macos_activation_failure_is_no_with_attention() { + let fake = Fake { focus: Some(FocusResult::Focused), activate_ok: false, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::MacOs, &fake, None), Activated::No); + assert_eq!(fake.calls(), ["activate", "attention"]); + } + + #[test] + fn other_platforms_only_ask_for_attention() { + let fake = Fake::default(); + assert_eq!(apply_activation_on(Platform::Other, &fake, Some("t")), Activated::No); + assert_eq!(fake.calls(), ["attention"]); + } + + #[test] + fn only_linux_mints_tokens() { + let fake = Fake { minted: Some("abc".into()), ..Fake::default() }; + assert_eq!(mint_token_on(Platform::Linux, &fake).as_deref(), Some("abc")); + assert_eq!(mint_token_on(Platform::Windows, &fake), None); + assert_eq!(mint_token_on(Platform::MacOs, &fake), None); + let empty = Fake { minted: Some(String::new()), ..Fake::default() }; + assert_eq!(mint_token_on(Platform::Linux, &empty), None); + } + + #[test] + fn only_macos_yields() { + let fake = Fake { yield_ok: false, ..Fake::default() }; + assert!(yield_to_on(Platform::Linux, &fake, "com.x")); + assert!(fake.calls().is_empty()); + assert!(!yield_to_on(Platform::MacOs, &fake, "com.x")); + assert_eq!(fake.calls(), ["yield:com.x"]); + } + + #[test] + fn current_platform_matches_the_build() { + assert_eq!(Platform::current() == Platform::Linux, cfg!(target_os = "linux")); + } +} diff --git a/crates/ipc/src/asset.rs b/crates/ipc/src/asset.rs new file mode 100644 index 000000000..9ffd6f429 --- /dev/null +++ b/crates/ipc/src/asset.rs @@ -0,0 +1,70 @@ +//! Reply bodies for `link.resolve`, `asset.describe` and `link.refresh` +//! (`PROTOCOL.md` section 4). +//! +//! `std`, `serde` and the sibling pure modules only. + +use serde::Serialize; + +use crate::moniker::Moniker; +use crate::record::AssetKind; +use crate::refresh::{FileState, Fingerprint}; + +/// Answer to `link.resolve`: where the last-known result is and which version it is. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Resolved { + pub asset: Moniker, + pub path: String, + pub epoch: u64, + pub size: u64, + pub hash: String, + pub mtime_ms: u64, +} + +impl Resolved { + /// Build the answer from a file fingerprint. + pub fn from_fingerprint(asset: Moniker, path: &str, epoch: u64, fp: &Fingerprint) -> Self { + Self { asset, path: path.to_string(), epoch, size: fp.size, hash: fp.hash.clone(), mtime_ms: fp.mtime_ms } + } +} + +/// Answer to `asset.describe`: enough for a hub to list an asset without opening its document. +/// Unknown values serialize as `null`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Described { + pub asset: Moniker, + pub name: String, + pub kind: AssetKind, + pub epoch: u64, + pub w: Option, + pub h: Option, + pub hash: Option, + pub thumb_path: Option, + pub modified_ms: Option, +} + +/// Answer to `link.refresh`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Refreshed { + pub link_id: String, + pub epoch: u64, + pub state: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub result_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hash: Option, +} + +impl Refreshed { + /// Build the answer from a tracker state. `hash` and `result_path` are + /// dropped when the file is missing. + pub fn new(link_id: &str, epoch: u64, state: FileState, result_path: &str, seen: Option<&Fingerprint>) -> Self { + let present = state != FileState::Missing; + Self { + link_id: link_id.to_string(), + epoch, + state: state.as_str(), + result_path: present.then(|| result_path.to_string()), + hash: seen.filter(|_| present).map(|fp| fp.hash.clone()), + } + } +} diff --git a/crates/ipc/src/auth.rs b/crates/ipc/src/auth.rs new file mode 100644 index 000000000..851ec7d8d --- /dev/null +++ b/crates/ipc/src/auth.rs @@ -0,0 +1,107 @@ +//! Session tokens and the peer user check. +//! +//! On Unix the peer check is the effective uid from `interprocess`. A missing +//! credential is a refusal. On Windows, `interprocess` 2.4.4 `PeerCreds` only +//! exposes a pid. The pipe SDDL is the user check there. A pid is not a user. + +use interprocess::local_socket::Stream; +#[cfg(unix)] +use interprocess::local_socket::traits::StreamCommon; + +#[cfg(unix)] +use crate::discovery::current_euid; +use crate::error::LinkError; + +const HEX: &[u8; 16] = b"0123456789abcdef"; + +pub fn encode_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} + +pub fn is_lower_hex(text: &str) -> bool { + text.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// 32 random bytes as 64 lowercase hex characters. +pub fn generate_token() -> Result { + let mut bytes = [0u8; 32]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + Ok(encode_hex(&bytes)) +} + +/// UUID version 4. The instance record stores this string. +pub fn random_uuid() -> Result { + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + let hex = encode_hex(&bytes); + Ok(format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32])) +} + +/// Compare two tokens without stopping at the first different byte. +/// Length is public (always 64). A different length is not equal. +pub fn tokens_equal(expected: &str, presented: &str) -> bool { + if expected.len() != 64 || presented.len() != 64 { + return false; + } + let mut diff = 0u8; + for (left, right) in expected.bytes().zip(presented.bytes()) { + diff |= left ^ right; + } + diff == 0 +} + +/// `Ok` when this Unix peer is the current effective user. +/// Missing credentials are `Err(Auth)`, not a pass. +/// +/// Windows returns `Ok`. The named-pipe SDDL admits only the owner and SYSTEM. +/// This function cannot see a Windows user id through interprocess 2.4.4. +pub fn ensure_peer_user(stream: &Stream) -> Result<(), LinkError> { + #[cfg(unix)] + { + let creds = stream.peer_creds().map_err(|_| LinkError::Auth)?; + let Some(peer) = creds.euid() else { + return Err(LinkError::Auth); + }; + let me = current_euid().map_err(|_| LinkError::Auth)?; + if peer == me { Ok(()) } else { Err(LinkError::Auth) } + } + #[cfg(not(unix))] + { + let _ = stream; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_is_64_lower_hex_and_compare_is_exact() { + let token = generate_token().unwrap(); + assert_eq!(token.len(), 64); + assert!(is_lower_hex(&token)); + assert!(tokens_equal(&token, &token)); + let mut other = token.clone(); + other.replace_range(0..1, if token.starts_with('a') { "b" } else { "a" }); + assert!(!tokens_equal(&token, &other)); + assert!(!tokens_equal(&token, "abcd")); + assert!(!tokens_equal(&token, &token.to_uppercase())); + } + + #[test] + fn uuid_v4_shape() { + let id = random_uuid().unwrap(); + assert_eq!(id.len(), 36); + assert_eq!(id.as_bytes()[14], b'4'); + let variant = id.as_bytes()[19]; + assert!(matches!(variant, b'8' | b'9' | b'a' | b'b')); + } +} diff --git a/crates/ipc/src/bulk.rs b/crates/ipc/src/bulk.rs new file mode 100644 index 000000000..488288778 --- /dev/null +++ b/crates/ipc/src/bulk.rs @@ -0,0 +1,487 @@ +//! Bulk frames (contract section 6). +//! +//! Server side. An app implements [`FrameSource`]. [`FrameTickets`] adapts it +//! to the server's [`Frames`] trait: it mints the 32 hex tickets, checks every +//! `CFRQ` header, and checks every reply before it reaches the wire. The server +//! drops a ticket when its control connection closes, which calls +//! [`Frames::close_ticket`] and ends the source's session for that ticket. +//! +//! Client side. [`Client::frames_open`] asks for a ticket. [`BulkClient`] opens +//! the second connection, attaches the ticket, and requests frames. + +use std::collections::HashMap; +use std::io::BufReader; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::{Map, Value, json}; + +use crate::InstanceRecord as Record; +use crate::auth::{encode_hex, is_lower_hex}; +use crate::client::{Client, Transport, map_reply, open_reader, read_json, write_json}; +use crate::error::LinkError; +use crate::server::{FrameOpen, Frames}; +use crate::wire; + +/// Largest `scale` a client may ask for in `frames.open`. +pub const SCALE_MAX: f64 = 16.0; + +/// Pixel format of a frame. Both are premultiplied alpha. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameFormat { + Rgba8Premul, + Rgba16fPremul, +} + +impl FrameFormat { + pub const fn as_str(self) -> &'static str { + match self { + Self::Rgba8Premul => "rgba8_premul", + Self::Rgba16fPremul => "rgba16f_premul", + } + } + + /// Bytes one pixel takes in a row. + pub(crate) const fn bytes_per_pixel(self) -> u32 { + match self { + Self::Rgba8Premul => 4, + Self::Rgba16fPremul => 8, + } + } + + pub fn parse(text: &str) -> Result { + match text { + "rgba8_premul" => Ok(Self::Rgba8Premul), + "rgba16f_premul" => Ok(Self::Rgba16fPremul), + other => Err(LinkError::BadParams(format!("unknown frame format {other}"))), + } + } +} + +/// What a client asked for in `frames.open`. +#[derive(Debug, Clone, PartialEq)] +pub struct FrameSpec { + pub link_id: String, + pub format: FrameFormat, + pub scale: f64, +} + +impl FrameSpec { + pub fn from_params(params: &Map) -> Result { + let link_id = params + .get("link_id") + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) + .ok_or_else(|| LinkError::BadParams("link_id must be a non-empty string".into()))? + .to_string(); + let format = + FrameFormat::parse(params.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::BadParams("format is required".into()))?)?; + let scale = match params.get("scale") { + None => 1.0, + Some(value) => value.as_f64().ok_or_else(|| LinkError::BadParams("scale must be a number".into()))?, + }; + if !scale.is_finite() || scale <= 0.0 || scale > SCALE_MAX { + return Err(LinkError::BadParams(format!("scale must be above 0 and at most {SCALE_MAX}"))); + } + Ok(Self { link_id, format, scale }) + } +} + +/// Size of the frames a ticket yields. `len` is `stride * h`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameLayout { + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +/// A row of `w` pixels in `format` must fit in `stride` bytes, or a reader that splits the payload +/// into rows of `stride` bytes cannot decode the frame. +fn check_row(w: u32, stride: u32, format: FrameFormat) -> Result<(), LinkError> { + if u64::from(stride) < u64::from(w) * u64::from(format.bytes_per_pixel()) { + return Err(LinkError::Invalid("frame layout: stride is smaller than one row of w pixels".into())); + } + Ok(()) +} + +impl FrameLayout { + fn check(&self) -> Result<(), LinkError> { + if u64::from(self.stride) * u64::from(self.h) != self.len { + return Err(LinkError::Invalid("frame layout: len is not stride * h".into())); + } + if self.len > wire::FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + Ok(()) + } +} + +/// One `CFRQ` header. +#[derive(Debug, Clone, PartialEq)] +pub struct FrameRequest { + pub req: u64, + pub time_s: f64, + /// The epoch the client last saw. The source may answer with a newer one. + pub epoch: u64, + /// `[x, y, w, h]` in pixels, or `None` for the whole frame. + pub region: Option<[u32; 4]>, +} + +impl FrameRequest { + pub fn from_header(header: &Value) -> Result { + let req = header.get("req").and_then(Value::as_u64).ok_or_else(|| LinkError::BadParams("req must be an unsigned integer".into()))?; + let time_s = header + .get("time_s") + .and_then(Value::as_f64) + .filter(|t| t.is_finite()) + .ok_or_else(|| LinkError::BadParams("time_s must be a finite number".into()))?; + let epoch = header.get("epoch").and_then(Value::as_u64).ok_or_else(|| LinkError::BadParams("epoch must be an unsigned integer".into()))?; + let region = match header.get("region") { + None | Some(Value::Null) => None, + Some(Value::Array(items)) if items.len() == 4 => { + let mut out = [0u32; 4]; + for (slot, item) in out.iter_mut().zip(items) { + let number = item.as_u64().ok_or_else(|| LinkError::BadParams("region entries must be unsigned integers".into()))?; + *slot = u32::try_from(number).map_err(|_| LinkError::BadParams("region entry is out of range".into()))?; + } + Some(out) + } + Some(_) => return Err(LinkError::BadParams("region must be null or [x, y, w, h]".into())), + }; + Ok(Self { req, time_s, epoch, region }) + } + + pub fn to_header(&self) -> Value { + json!({"req": self.req, "time_s": self.time_s, "epoch": self.epoch, "region": self.region}) + } +} + +/// A rendered frame from the source. +#[derive(Debug, Clone, PartialEq)] +pub struct RenderedFrame { + /// The epoch this frame was rendered at. + pub epoch: u64, + pub w: u32, + pub h: u32, + pub stride: u32, + pub format: FrameFormat, + pub render_ms: f64, + pub pixels: Vec, +} + +/// The app side of bulk frames. Calls may come from several connection threads. +pub trait FrameSource: Send + Sync { + /// Validate `spec` and report the frame layout. Errors become `frames.open` errors. + fn open(&self, spec: &FrameSpec) -> Result; + /// Render one frame. The returned epoch must be at least `request.epoch`. + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result; + /// The ticket for `spec` is gone. Release anything held for it. + fn close(&self, spec: &FrameSpec); +} + +/// [`Frames`] over a [`FrameSource`]. Pass it as `ServerConfig::frames`. +pub struct FrameTickets { + source: Arc, + open: Mutex>, +} + +impl FrameTickets { + pub fn new(source: Arc) -> Self { + Self { source, open: Mutex::new(HashMap::new()) } + } + + /// Count of tickets that are still valid. + pub fn open_tickets(&self) -> usize { + crate::server::lock(&self.open).len() + } + + fn spec(&self, ticket: &str) -> Result { + crate::server::lock(&self.open).get(ticket).cloned().ok_or_else(|| LinkError::BadParams("unknown or closed ticket".into())) + } +} + +fn new_ticket() -> Result { + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + Ok(encode_hex(&bytes)) +} + +impl Frames for FrameTickets { + fn open(&self, params: &Map) -> Result { + let spec = FrameSpec::from_params(params)?; + let layout = self.source.open(&spec)?; + layout.check()?; + check_row(layout.w, layout.stride, spec.format)?; + let ticket = new_ticket()?; + crate::server::lock(&self.open).insert(ticket.clone(), spec); + Ok(FrameOpen { ticket, w: layout.w, h: layout.h, stride: layout.stride, len: layout.len }) + } + + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError> { + let spec = self.spec(ticket)?; + let request = FrameRequest::from_header(header)?; + let frame = self.source.render(&spec, &request)?; + if frame.epoch < request.epoch { + return Err(LinkError::NotReady(format!("source is at epoch {}, request wants {}", frame.epoch, request.epoch))); + } + check_row(frame.w, frame.stride, frame.format)?; + let expect = u64::from(frame.stride) * u64::from(frame.h); + if u64::try_from(frame.pixels.len()).ok() != Some(expect) { + return Err(LinkError::Invalid("rendered payload is not stride * h bytes".into())); + } + let reply = json!({ + "req": request.req, + "epoch": frame.epoch, + "w": frame.w, + "h": frame.h, + "stride": frame.stride, + "format": frame.format.as_str(), + "render_ms": frame.render_ms, + }); + Ok((reply, frame.pixels)) + } + + fn close_ticket(&self, ticket: &str) { + let removed = crate::server::lock(&self.open).remove(ticket); + if let Some(spec) = removed { + self.source.close(&spec); + } + } +} + +/// The `frames.open` reply. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameTicket { + pub ticket: String, + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +impl Client { + /// `frames.open`. The ticket lives until this connection closes. + pub fn frames_open(&mut self, link_id: &str, format: FrameFormat, scale: f64) -> Result { + let result = self.call("frames.open", json!({"link_id": link_id, "format": format.as_str(), "scale": scale}))?; + let ticket = result + .get("ticket") + .and_then(Value::as_str) + .filter(|t| t.len() == 32 && is_lower_hex(t)) + .ok_or_else(|| LinkError::Invalid("frames.open ticket is not 32 hex characters".into()))?; + Ok(FrameTicket { + ticket: ticket.to_string(), + w: result_u32(&result, "w")?, + h: result_u32(&result, "h")?, + stride: result_u32(&result, "stride")?, + len: result.get("len").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("frames.open result has no len".into()))?, + }) + } +} + +fn result_u32(result: &Value, key: &str) -> Result { + result + .get(key) + .and_then(Value::as_u64) + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| LinkError::Invalid(format!("frames.open result has no {key}"))) +} + +/// One `CFRM` reply that carried pixels. +#[derive(Debug, Clone, PartialEq)] +pub struct Frame { + pub req: u64, + /// The epoch the frame was rendered at. It can be newer than the epoch asked for. + pub epoch: u64, + pub w: u32, + pub h: u32, + pub stride: u32, + pub format: FrameFormat, + pub render_ms: f64, + pub pixels: Vec, +} + +/// The second connection of contract section 6. It is binary after the attach. +pub struct BulkClient { + reader: BufReader, + next_req: u64, + io_timeout: Duration, +} + +impl BulkClient { + /// Connect, authenticate, and `bulk.attach` with `ticket`. + pub fn attach(record: &Record, ticket: &str) -> Result { + let mut reader = open_reader(record)?; + write_json(&mut reader, &json!({"id": 1, "method": "bulk.attach", "params": {"ticket": ticket}}))?; + let deadline = std::time::Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + let reply = read_json(&mut reader, &mut Vec::new(), deadline)?.ok_or(LinkError::Closed)?; + map_reply(&reply)?; + Ok(Self { reader, next_req: 1, io_timeout: Duration::from_secs(wire::REPLY_DEADLINE_SECS) }) + } + + /// How long one request may wait for the socket to take the request or give the reply. + /// The default is [`wire::REPLY_DEADLINE_SECS`]. A request that runs out of time is + /// `Err(Timeout)`, and the connection must then be dropped, because a reply that arrives + /// late cannot be told apart from the next one. + pub fn set_io_timeout(&mut self, timeout: Duration) { + self.io_timeout = timeout; + } + + /// Ask for one frame and wait for its reply. + /// + /// A reply with an `error` header is `Err(Invalid(message))`. The + /// connection stays usable after it. A short or malformed record is an + /// error and the connection must be dropped. [`BulkClient::request_reply`] + /// tells the two apart. + pub fn request(&mut self, time_s: f64, epoch: u64, region: Option<[u32; 4]>) -> Result { + self.request_reply(time_s, epoch, region)?.map_err(LinkError::Invalid) + } + + /// Like [`BulkClient::request`], with the two failures apart. The outer `Err` is the + /// connection: drop it. The inner `Err` is the text of an `error` header from the server: + /// the connection is still good. + pub fn request_reply(&mut self, time_s: f64, epoch: u64, region: Option<[u32; 4]>) -> Result, LinkError> { + if !time_s.is_finite() { + return Err(LinkError::BadParams("time_s must be finite".into())); + } + let req = self.next_req; + self.next_req = req.checked_add(1).ok_or_else(|| LinkError::Invalid("request id overflow".into()))?; + let header = FrameRequest { req, time_s, epoch, region }.to_header(); + let transport = self.reader.get_ref(); + transport.arm_send_timeout(Some(self.io_timeout))?; + transport.arm_recv_timeout(Some(self.io_timeout))?; + wire::write_cfrq(self.reader.get_mut(), &header)?; + let (reply, pixels) = wire::read_cfrm(&mut self.reader)?; + if reply.get("req").and_then(Value::as_u64) != Some(req) { + return Err(LinkError::Invalid("reply req does not match the request".into())); + } + if let Some(message) = reply.get("error") { + return Ok(Err(message.as_str().unwrap_or("frame error").to_string())); + } + parse_reply(req, &reply, pixels).map(Ok) + } +} + +fn parse_reply(req: u64, reply: &Value, pixels: Vec) -> Result { + let number = |key: &str| reply.get(key).and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid(format!("frame header has no {key}"))); + let small = |key: &str| number(key).and_then(|n| u32::try_from(n).map_err(|_| LinkError::Invalid(format!("frame header {key} is out of range")))); + let (w, h, stride) = (small("w")?, small("h")?, small("stride")?); + let format = + FrameFormat::parse(reply.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::Invalid("frame header has no format".into()))?)?; + check_row(w, stride, format)?; + if u64::try_from(pixels.len()).ok() != Some(u64::from(stride) * u64::from(h)) { + return Err(LinkError::Invalid("payload is not stride * h bytes".into())); + } + let render_ms = reply.get("render_ms").and_then(Value::as_f64).ok_or_else(|| LinkError::Invalid("frame header has no render_ms".into()))?; + Ok(Frame { req, epoch: number("epoch")?, w, h, stride, format, render_ms, pixels }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_header_round_trips() { + let request = FrameRequest { req: 9, time_s: 2.5, epoch: 4, region: Some([1, 2, 3, 4]) }; + assert_eq!(FrameRequest::from_header(&request.to_header()).unwrap(), request); + let whole = FrameRequest { req: 1, time_s: 0.0, epoch: 0, region: None }; + assert_eq!(FrameRequest::from_header(&whole.to_header()).unwrap(), whole); + } + + #[test] + fn bad_request_headers_are_rejected() { + for header in [ + json!({"time_s": 1.0, "epoch": 0}), + json!({"req": 1, "epoch": 0}), + json!({"req": 1, "time_s": 1.0}), + json!({"req": 1, "time_s": 1.0, "epoch": 0, "region": [1, 2, 3]}), + json!({"req": 1, "time_s": 1.0, "epoch": 0, "region": [1, 2, 3, -4]}), + json!({"req": -1, "time_s": 1.0, "epoch": 0}), + ] { + assert!(FrameRequest::from_header(&header).is_err(), "{header}"); + } + } + + #[test] + fn spec_rejects_bad_scale_and_format() { + let ok = json!({"link_id": "a", "format": "rgba8_premul", "scale": 2.0}); + assert!(FrameSpec::from_params(ok.as_object().unwrap()).is_ok()); + for bad in [ + json!({"link_id": "a", "format": "rgba8_premul", "scale": 0.0}), + json!({"link_id": "a", "format": "rgba8_premul", "scale": 17.0}), + json!({"link_id": "a", "format": "bgra"}), + json!({"format": "rgba8_premul"}), + ] { + assert!(FrameSpec::from_params(bad.as_object().unwrap()).is_err(), "{bad}"); + } + } + + struct OneLayout { + w: u32, + stride: u32, + h: u32, + } + + impl FrameSource for OneLayout { + fn open(&self, _spec: &FrameSpec) -> Result { + Ok(FrameLayout { w: self.w, h: self.h, stride: self.stride, len: u64::from(self.stride) * u64::from(self.h) }) + } + + fn render(&self, spec: &FrameSpec, _request: &FrameRequest) -> Result { + let len = usize::try_from(u64::from(self.stride) * u64::from(self.h)).unwrap(); + Ok(RenderedFrame { epoch: 1, w: self.w, h: self.h, stride: self.stride, format: spec.format, render_ms: 0.0, pixels: vec![0; len] }) + } + + fn close(&self, _spec: &FrameSpec) {} + } + + fn open_params(format: &str) -> Map { + json!({"link_id": "a", "format": format}).as_object().unwrap().clone() + } + + #[test] + fn a_stride_that_cannot_hold_one_row_is_refused_before_a_ticket_exists() { + // (w, stride, format, accepted): one row is w * 4 bytes for rgba8 and w * 8 for rgba16f. + for (w, stride, format, accepted) in [ + (4, 16, "rgba8_premul", true), + (4, 17, "rgba8_premul", true), + (4, 15, "rgba8_premul", false), + (4, 1, "rgba8_premul", false), + (4, 32, "rgba16f_premul", true), + (4, 16, "rgba16f_premul", false), + (0, 0, "rgba8_premul", true), + (u32::MAX, u32::MAX, "rgba16f_premul", false), + ] { + let tickets = FrameTickets::new(Arc::new(OneLayout { w, stride, h: 3 })); + let got = tickets.open(&open_params(format)); + assert_eq!(got.is_ok(), accepted, "w {w} stride {stride} {format}: {got:?}"); + assert_eq!(tickets.open_tickets(), usize::from(accepted), "a refused layout leaves no ticket"); + } + } + + #[test] + fn a_rendered_frame_with_a_short_stride_is_refused_and_the_client_refuses_it_too() { + let render = json!({"req": 1, "time_s": 0.0, "epoch": 0}); + let tickets = FrameTickets::new(Arc::new(OneLayout { w: 4, stride: 16, h: 3 })); + let ticket = tickets.open(&open_params("rgba8_premul")).unwrap().ticket; + assert!(tickets.render(&ticket, &render).is_ok()); + // A source that reports a narrower stride than it promised at open. + let lying = FrameTickets::new(Arc::new(OneLayout { w: 4, stride: 2, h: 3 })); + lying.open.lock().unwrap().insert(ticket.clone(), FrameSpec { link_id: "a".into(), format: FrameFormat::Rgba8Premul, scale: 1.0 }); + assert!(matches!(lying.render(&ticket, &render), Err(LinkError::Invalid(_)))); + + let reply = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 1, "format": "rgba8_premul", "render_ms": 1.0}); + assert!(matches!(parse_reply(1, &reply, vec![0; 3]), Err(LinkError::Invalid(_)))); + let fine = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + assert!(parse_reply(1, &fine, vec![0; 48]).is_ok()); + } + + #[test] + fn tickets_are_32_lower_hex_and_distinct() { + let a = new_ticket().unwrap(); + let b = new_ticket().unwrap(); + assert_eq!(a.len(), 32); + assert!(is_lower_hex(&a)); + assert_ne!(a, b); + } +} diff --git a/crates/ipc/src/cache.rs b/crates/ipc/src/cache.rs new file mode 100644 index 000000000..6cc77545d --- /dev/null +++ b/crates/ipc/src/cache.rs @@ -0,0 +1,278 @@ +//! A private per-user cache directory for Craft Link. +//! +//! Prepared media and other link caches must not sit in a directory that another user can create +//! first. This module picks the directory, creates it `0700`, refuses a symlink, a wrong owner or a +//! loose mode, and writes files `0600` through a temp name and a rename. It never falls back to +//! `$TMPDIR` or `/tmp`: with no home and no override it returns an error. +//! +//! Order: `CRAFT_LINK_CACHE_DIR` (absolute), `$XDG_CACHE_HOME/artcraft/link`, then +//! `$HOME/.cache/artcraft/link`. A leaf name `sub` is added under the chosen directory. + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::error::LinkError; + +/// Environment variable that overrides the cache root. +pub const CACHE_DIR_ENV: &str = "CRAFT_LINK_CACHE_DIR"; + +/// [`cache_dir_with`] for the process environment. +pub fn private_cache_dir(sub: &str) -> Result { + let env = std::env::vars().collect::>(); + cache_dir_with(&env, sub) +} + +/// Pick, create and check the cache directory `sub` for `env`. +pub fn cache_dir_with(env: &BTreeMap, sub: &str) -> Result { + if sub.is_empty() || sub.starts_with('.') || !sub.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { + return Err(refused(Path::new(sub), "cache name must be letters, digits, dot, dash or underscore")); + } + let (base, rest) = base_and_rest(env)?; + let mut path = base.clone(); + create_base(&path)?; + for part in rest.iter().map(String::as_str).chain([sub]) { + path.push(part); + crate::discovery::ensure_private_dir(&path)?; + } + Ok(path) +} + +fn base_and_rest(env: &BTreeMap) -> Result<(PathBuf, Vec), LinkError> { + let absolute = |key: &str| env.get(key).filter(|value| !value.is_empty()).map(PathBuf::from); + if let Some(root) = absolute(CACHE_DIR_ENV) { + if !root.is_absolute() { + return Err(refused(&root, "CRAFT_LINK_CACHE_DIR is not absolute")); + } + let (Some(parent), Some(leaf)) = (root.parent(), root.file_name().and_then(|name| name.to_str())) else { + return Err(refused(&root, "CRAFT_LINK_CACHE_DIR has no parent directory")); + }; + return Ok((parent.to_path_buf(), vec![leaf.to_string()])); + } + if let Some(xdg) = absolute("XDG_CACHE_HOME").filter(|path| path.is_absolute()) { + return Ok((xdg, vec!["artcraft".into(), "link".into()])); + } + if let Some(home) = absolute("HOME").filter(|path| path.is_absolute()) { + return Ok((home.join(".cache"), vec!["artcraft".into(), "link".into()])); + } + Err(refused(Path::new(""), "no cache directory: set CRAFT_LINK_CACHE_DIR, XDG_CACHE_HOME or HOME (a shared temp directory is not used)")) +} + +/// Create `path` and missing parents `0700`. An existing directory is left as it is: it is the +/// user's own `~/.cache` or the parent of the override. +fn create_base(path: &Path) -> Result<(), LinkError> { + let mut builder = std::fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path)?; + Ok(()) +} + +/// Write `bytes` to `/` with mode `0600`. The file is made under a temp name with +/// `create_new` (which does not follow a symlink or reuse a file) and renamed into place, so a +/// reader never sees half a file and a planted symlink at `name` is replaced, not followed. +pub fn write_private(dir: &Path, name: &str, bytes: &[u8]) -> Result<(), LinkError> { + if name.is_empty() || name.contains(['/', '\\', '\0']) || name.starts_with('.') { + return Err(refused(&dir.join(name), "file name must be one plain component")); + } + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let tmp = dir.join(format!(".{name}.{}.tmp", crate::auth::encode_hex(&rand))); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&tmp)?; + let written = file.write_all(bytes).and_then(|()| file.sync_all()); + drop(file); + if let Err(err) = written.and_then(|()| std::fs::rename(&tmp, dir.join(name))) { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(()) +} + +/// Read `/` when it is a regular file (not a symlink) that this user owns and no one +/// else can access. Anything else is `Refused`. A missing file is an I/O error. +/// +/// On Unix the open refuses a symlink and does not wait on a FIFO, and the checks and the read use +/// that one handle, so the entry cannot change between the check and the read. +pub fn read_private(dir: &Path, name: &str) -> Result, LinkError> { + let path = dir.join(name); + #[cfg(unix)] + { + use std::io::Read; + use std::os::unix::fs::MetadataExt; + let mut file = match crate::fsopen::open_read(&path, false) { + Ok(file) => file, + // The errno for a symlink in the last component differs between Unix systems. + Err(_) if std::fs::symlink_metadata(&path).is_ok_and(|meta| meta.file_type().is_symlink()) => { + return Err(refused(&path, "not a regular file (a symlink is not followed)")); + } + Err(err) => return Err(err.into()), + }; + let meta = file.metadata()?; + if !meta.file_type().is_file() { + return Err(refused(&path, "not a regular file")); + } + let euid = crate::discovery::current_euid()?; + if meta.uid() != euid { + return Err(refused(&path, &format!("owner is {}, euid is {euid}", meta.uid()))); + } + if meta.mode() & 0o077 != 0 { + return Err(refused(&path, &format!("mode is {:o}, group and others must have no access", meta.mode() & 0o777))); + } + let mut bytes = Vec::with_capacity(usize::try_from(meta.len()).unwrap_or(0)); + file.read_to_end(&mut bytes)?; + Ok(bytes) + } + #[cfg(not(unix))] + { + let meta = std::fs::symlink_metadata(&path)?; + if !meta.file_type().is_file() { + return Err(refused(&path, "not a regular file (a symlink is not followed)")); + } + Ok(std::fs::read(&path)?) + } +} + +fn refused(path: &Path, reason: &str) -> LinkError { + LinkError::Refused { path: path.display().to_string(), reason: reason.to_string() } +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; + + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-cache-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + path + } + + fn env(pairs: &[(&str, &Path)]) -> BTreeMap { + pairs.iter().map(|(key, value)| ((*key).to_string(), value.display().to_string())).collect() + } + + fn mode(path: &Path) -> u32 { + std::fs::symlink_metadata(path).unwrap().mode() & 0o777 + } + + #[test] + fn home_and_xdg_give_a_0700_chain() { + let home = scratch("home"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + assert_eq!(dir, home.join(".cache/artcraft/link/prepared")); + for part in [".cache/artcraft", ".cache/artcraft/link", ".cache/artcraft/link/prepared"] { + assert_eq!(mode(&home.join(part)), 0o700, "{part}"); + } + let xdg = scratch("xdg"); + let dir = cache_dir_with(&env(&[("HOME", &home), ("XDG_CACHE_HOME", &xdg)]), "prepared").unwrap(); + assert_eq!(dir, xdg.join("artcraft/link/prepared")); + assert_eq!(mode(&dir), 0o700); + // Same call twice is fine. + assert_eq!(cache_dir_with(&env(&[("XDG_CACHE_HOME", &xdg)]), "prepared").unwrap(), dir); + std::fs::remove_dir_all(&home).unwrap(); + std::fs::remove_dir_all(&xdg).unwrap(); + } + + #[test] + fn no_home_means_an_error_not_tmp() { + let err = cache_dir_with(&BTreeMap::new(), "prepared").unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err:?}"); + let relative: BTreeMap<_, _> = + [("HOME".to_string(), "relative".to_string()), ("XDG_CACHE_HOME".to_string(), "also/relative".to_string())].into(); + assert!(cache_dir_with(&relative, "prepared").is_err()); + let rel_override: BTreeMap<_, _> = [(CACHE_DIR_ENV.to_string(), "cache".to_string())].into(); + assert!(cache_dir_with(&rel_override, "prepared").is_err()); + } + + #[test] + fn the_override_is_used_and_checked() { + let root = scratch("override"); + let chosen = root.join("mine"); + let dir = cache_dir_with(&env(&[(CACHE_DIR_ENV, &chosen), ("HOME", Path::new("/nonexistent"))]), "prepared").unwrap(); + assert_eq!(dir, chosen.join("prepared")); + assert_eq!(mode(&chosen), 0o700); + // A loose existing directory is refused, not tightened behind the user's back. + std::fs::set_permissions(&chosen, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(cache_dir_with(&env(&[(CACHE_DIR_ENV, &chosen)]), "prepared").is_err()); + // A symlink is refused. + let real = root.join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = root.join("link"); + symlink(&real, &link).unwrap(); + assert!(cache_dir_with(&env(&[(CACHE_DIR_ENV, &link)]), "prepared").is_err()); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn a_fifo_or_a_missing_file_is_not_read_and_does_not_block() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let home = scratch("fifo"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + let made = std::process::Command::new("mkfifo").arg("-m").arg("600").arg(dir.join("pipe.bin")).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + let (tx, rx) = std::sync::mpsc::channel(); + let reading = dir.clone(); + std::thread::spawn(move || { + let _ = tx.send((read_private(&reading, "pipe.bin"), read_private(&reading, "absent.bin"))); + }); + let (fifo, absent) = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("a FIFO blocked the read"); + assert!(matches!(fifo, Err(LinkError::Refused { .. })), "{fifo:?}"); + assert!(matches!(absent, Err(LinkError::Io(_))), "{absent:?}"); + std::fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn bad_names_are_refused() { + let home = scratch("names"); + std::fs::create_dir_all(&home).unwrap(); + for bad in ["", ".hidden", "a/b", "..", "a b", "x\0y"] { + assert!(cache_dir_with(&env(&[("HOME", &home)]), bad).is_err(), "{bad:?}"); + } + std::fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn files_are_0600_and_symlinks_are_replaced_not_followed() { + let home = scratch("files"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + write_private(&dir, "a.bin", b"one").unwrap(); + assert_eq!(mode(&dir.join("a.bin")), 0o600); + assert_eq!(read_private(&dir, "a.bin").unwrap(), b"one"); + // A symlink planted at the name is replaced. The target keeps its content. + let victim = home.join("victim"); + std::fs::write(&victim, b"keep").unwrap(); + let planted = dir.join("b.bin"); + symlink(&victim, &planted).unwrap(); + assert!(matches!(read_private(&dir, "b.bin"), Err(LinkError::Refused { .. })), "a symlink is refused, not read"); + write_private(&dir, "b.bin", b"two").unwrap(); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep"); + assert_eq!(read_private(&dir, "b.bin").unwrap(), b"two"); + // A loose file is refused on read. + std::fs::set_permissions(dir.join("a.bin"), std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(read_private(&dir, "a.bin").is_err()); + for bad in ["", "../x", "a/b", ".x"] { + assert!(write_private(&dir, bad, b"x").is_err(), "{bad:?}"); + } + std::fs::remove_dir_all(&home).unwrap(); + } +} diff --git a/crates/ipc/src/client.rs b/crates/ipc/src/client.rs new file mode 100644 index 000000000..8090be1d3 --- /dev/null +++ b/crates/ipc/src/client.rs @@ -0,0 +1,853 @@ +//! Craft Link client. +//! +//! Connects to one instance record, authenticates on local sockets and pipes, +//! and exchanges JSON lines. Push events that arrive before a reply are kept +//! until [`Client::next_event`]. + +use std::collections::VecDeque; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +use interprocess::local_socket::prelude::*; +use interprocess::local_socket::{ConnectOptions, GenericFilePath, Stream}; +use serde_json::{Map, Value, json}; + +use crate::error::LinkError; +use crate::wire; +use crate::{Endpoint, InstanceRecord}; + +/// Identity sent in `link.hello`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HelloClient { + pub app: String, + pub app_version: String, + pub pid: u32, + pub instance: String, + pub protocol_min: u32, + pub protocol_max: u32, +} + +/// One push line: `event`, `seq`, and `data`. +#[derive(Debug, Clone, PartialEq)] +pub struct Event { + pub event: String, + pub seq: u64, + pub data: Value, +} + +/// A connected Craft Link session. +pub struct Client { + record: InstanceRecord, + reader: BufReader, + next_id: u64, + events: VecDeque, + hello: Option, + /// Bytes of a reply line read before a timeout, kept so the next read resumes the line. + partial: Vec, +} + +impl Client { + /// Connect to `record.endpoint`. Every transport sends auth id 0 with the record token first. + pub fn connect(record: &InstanceRecord) -> Result { + let reader = open_reader(record)?; + Ok(Self { record: record.clone(), reader, next_id: 1, events: VecDeque::new(), hello: None, partial: Vec::new() }) + } + + /// Connect to the loopback TCP control port published in `record` (`record.tcp_port()`), + /// with the same token handshake. `Err(Invalid)` when the app opened no TCP port. + pub fn connect_tcp(record: &InstanceRecord) -> Result { + let Some(port) = record.tcp_port() else { + return Err(LinkError::Invalid("the instance record has no tcp_port; start the app with --control".into())); + }; + let mut tcp = record.clone(); + tcp.endpoint = Endpoint::Tcp { port }; + Self::connect(&tcp) + } + + /// Send one request. `id` starts at 1 and increases by one each call. + /// Lines with `event` and a different id are buffered. + pub fn call(&mut self, method: &str, params: Value) -> Result { + let id = self.next_id; + self.next_id = self.next_id.checked_add(1).ok_or_else(|| LinkError::Invalid("request id overflow".into()))?; + let msg = json!({"id": id, "method": method, "params": params}); + write_json(&mut self.reader, &msg)?; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + loop { + let value = match read_json(&mut self.reader, &mut self.partial, deadline)? { + Some(value) => value, + None => return Err(LinkError::Closed), + }; + if !value.is_object() { + return Err(LinkError::Invalid("reply is not an object".into())); + } + if id_is(&value, id) { + return map_reply(&value); + } + if value.get("event").is_some() { + self.events.push_back(parse_event(&value)?); + continue; + } + return Err(LinkError::Invalid("reply id does not match the request".into())); + } + } + + /// `link.hello`. Checks `pid` and `instance` against the connected record. + /// A successful hello is stored and sent again by [`Client::reconnect`]. + pub fn hello(&mut self, identity: &HelloClient) -> Result { + let result = self.call("link.hello", hello_params(identity))?; + check_identity(&result, &self.record)?; + self.hello = Some(identity.clone()); + Ok(result) + } + + /// `events.subscribe`. Returns `result.seq`. + pub fn subscribe(&mut self, events: &[&str]) -> Result { + let result = self.call("events.subscribe", json!({"events": events}))?; + result.get("seq").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("events.subscribe result has no seq".into())) + } + + /// `events.unsubscribe`. + pub fn unsubscribe(&mut self) -> Result<(), LinkError> { + let _result = self.call("events.unsubscribe", json!({}))?; + Ok(()) + } + + /// The next buffered event, or the next event read before `timeout`. + /// `None` waits up to [`wire::REPLY_DEADLINE_SECS`]. A zero duration does not read. + pub fn next_event(&mut self, timeout: Option) -> Result, LinkError> { + if let Some(event) = self.events.pop_front() { + return Ok(Some(event)); + } + if timeout.is_some_and(|wait| wait.is_zero()) { + return Ok(None); + } + let wait = timeout.unwrap_or(Duration::from_secs(wire::REPLY_DEADLINE_SECS)); + let deadline = Instant::now() + wait; + if Instant::now() >= deadline { + return Ok(None); + } + match read_json(&mut self.reader, &mut self.partial, deadline) { + Ok(None) => Err(LinkError::Closed), + Ok(Some(value)) => { + if value.get("event").is_some() { + Ok(Some(parse_event(&value)?)) + } else { + Err(LinkError::Invalid("expected an event line".into())) + } + } + Err(LinkError::Timeout) => Ok(None), + Err(err) => Err(err), + } + } + + /// Connect to the stored record again. Does not scan discovery. + /// Sends auth again. Sends the stored hello when one succeeded. + pub fn reconnect(&mut self) -> Result<(), LinkError> { + let reader = open_reader(&self.record)?; + self.reader = reader; + self.partial.clear(); + self.events.clear(); + self.next_id = 1; + if let Some(identity) = self.hello.clone() { + self.hello(&identity)?; + } + Ok(()) + } +} + +pub(crate) enum Transport { + Local(Stream), + Tcp(TcpStream), + #[cfg(windows)] + Pipe(PipeTransport), +} + +/// Windows named pipes have no socket-style timeouts. Poll their nonblocking I/O so +/// authentication and frame requests retain the same bounded waits as other transports. +#[cfg(windows)] +pub(crate) struct PipeTransport { + stream: Stream, + recv_timeout: std::cell::Cell>, + send_timeout: std::cell::Cell>, +} + +#[cfg(windows)] +impl PipeTransport { + pub(crate) fn new(stream: Stream) -> std::io::Result { + stream.set_nonblocking(true)?; + Ok(Self { stream, recv_timeout: std::cell::Cell::new(None), send_timeout: std::cell::Cell::new(None) }) + } + + pub(crate) fn set_timeouts(&self, recv: Duration, send: Duration) { + self.recv_timeout.set(Some(recv)); + self.send_timeout.set(Some(send)); + } + + fn poll(timeout: Option, mut operation: impl FnMut() -> std::io::Result) -> std::io::Result { + let start = Instant::now(); + loop { + match operation() { + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + if timeout.is_some_and(|limit| start.elapsed() >= limit) { + return Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "named pipe I/O timed out")); + } + std::thread::sleep(Duration::from_millis(1)); + } + result => return result, + } + } + } +} + +#[cfg(windows)] +impl Read for PipeTransport { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + Self::poll(self.recv_timeout.get(), || match self.stream.read(buf) { + // interprocess converts Windows ERROR_NO_DATA on an empty nonblocking pipe + // to EOF. A disconnected pipe has no peer; otherwise retry until the deadline. + Ok(0) if !buf.is_empty() && self.stream.peer_creds().is_ok() => Err(std::io::ErrorKind::WouldBlock.into()), + result => result, + }) + } +} + +#[cfg(windows)] +impl Write for PipeTransport { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Self::poll(self.send_timeout.get(), || match self.stream.write(buf) { + Ok(0) if !buf.is_empty() => Err(std::io::ErrorKind::WouldBlock.into()), + result => result, + }) + } + fn flush(&mut self) -> std::io::Result<()> { + self.stream.flush() + } +} + +impl Transport { + pub(crate) fn arm_recv_timeout(&self, timeout: Option) -> std::io::Result<()> { + match self { + Self::Local(stream) => { + let result = stream.set_recv_timeout(timeout); + // Darwin rejects socket options after full shutdown (sosetoptlock), even when + // unread reply bytes remain. Reading those bytes or EOF cannot block, so let + // the read report the actual protocol result instead of this timeout error. + #[cfg(target_os = "macos")] + if result.as_ref().is_err_and(|err| err.raw_os_error() == Some(libc::EINVAL)) { + return Ok(()); + } + result + } + Self::Tcp(stream) => stream.set_read_timeout(timeout), + #[cfg(windows)] + Self::Pipe(stream) => { + stream.recv_timeout.set(timeout); + Ok(()) + } + } + } + + pub(crate) fn arm_send_timeout(&self, timeout: Option) -> std::io::Result<()> { + match self { + Self::Local(stream) => stream.set_send_timeout(timeout), + Self::Tcp(stream) => stream.set_write_timeout(timeout), + #[cfg(windows)] + Self::Pipe(stream) => { + stream.send_timeout.set(timeout); + Ok(()) + } + } + } +} + +impl Read for Transport { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + Self::Local(stream) => stream.read(buf), + Self::Tcp(stream) => stream.read(buf), + #[cfg(windows)] + Self::Pipe(stream) => stream.read(buf), + } + } +} + +impl Write for Transport { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + Self::Local(stream) => stream.write(buf), + Self::Tcp(stream) => stream.write(buf), + #[cfg(windows)] + Self::Pipe(stream) => stream.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Local(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + #[cfg(windows)] + Self::Pipe(stream) => stream.flush(), + } + } +} + +pub(crate) fn open_reader(record: &InstanceRecord) -> Result, LinkError> { + let (transport, needs_auth) = open_transport(&record.endpoint)?; + let mut reader = BufReader::new(transport); + if needs_auth { + authenticate(&mut reader, &record.token)?; + } + Ok(reader) +} + +fn open_transport(endpoint: &Endpoint) -> Result<(Transport, bool), LinkError> { + match endpoint { + Endpoint::Unix { path } => Ok((Transport::Local(connect_unix(path)?), true)), + Endpoint::Pipe { name } => { + #[cfg(windows)] + { + Ok((Transport::Pipe(PipeTransport::new(connect_pipe(name)?)?), true)) + } + #[cfg(not(windows))] + { + Ok((Transport::Local(connect_pipe(name)?), true)) + } + } + Endpoint::Tcp { port } => { + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, *port)); + let wait = Duration::from_secs(wire::REPLY_DEADLINE_SECS); + match TcpStream::connect_timeout(&addr, wait) { + Ok(stream) => Ok((Transport::Tcp(stream), true)), + Err(err) if is_timeout(&err) => Err(LinkError::Timeout), + Err(err) => Err(err.into()), + } + } + } +} + +fn connect_unix(path: &str) -> Result { + let name = path.to_fs_name::().map_err(LinkError::io)?; + ConnectOptions::new().name(name).connect_sync().map_err(LinkError::io) +} + +/// Named pipes are the Windows endpoint. Other platforms cannot open one. +/// This returns [`LinkError::Invalid`] instead of pretending the pipe connected. +#[cfg(not(windows))] +fn connect_pipe(name: &str) -> Result { + let _ = name; + Err(LinkError::Invalid("named pipes are a Windows endpoint and are not connected on this platform".into())) +} + +#[cfg(windows)] +fn connect_pipe(name: &str) -> Result { + use interprocess::local_socket::GenericNamespaced; + let socket_name = name.to_ns_name::().map_err(LinkError::io)?; + ConnectOptions::new().name(socket_name).connect_sync().map_err(LinkError::io) +} + +fn authenticate(reader: &mut BufReader, token: &str) -> Result<(), LinkError> { + let request = json!({"id": 0, "method": "auth", "params": {"token": token}}); + write_json(reader, &request)?; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + let expected = wire::auth_success(&json!(0)); + let mut partial = Vec::new(); + loop { + let line = match pull_line(reader, &mut partial, deadline) { + Ok(Some(bytes)) => bytes, + Ok(None) => return Err(LinkError::Auth), + Err(LinkError::Timeout) => return Err(LinkError::Timeout), + Err(LinkError::TooLarge) => return Err(LinkError::TooLarge), + Err(LinkError::Io(text)) => return Err(LinkError::Io(text)), + Err(_) => return Err(LinkError::Auth), + }; + let Ok(text) = std::str::from_utf8(&line) else { + return Err(LinkError::Auth); + }; + let text = text.trim(); + if text.is_empty() { + continue; + } + let Ok(value) = serde_json::from_str::(text) else { + return Err(LinkError::Auth); + }; + if value == expected { + return Ok(()); + } + return Err(LinkError::Auth); + } +} + +pub(crate) fn write_json(reader: &mut BufReader, value: &Value) -> Result<(), LinkError> { + reader.get_ref().arm_send_timeout(Some(Duration::from_secs(wire::REPLY_DEADLINE_SECS)))?; + wire::write_line(reader.get_mut(), value) +} + +/// Reads one JSON line. `partial` holds the bytes of a line that a timeout cut short, so a caller +/// that keeps the connection after `Err(Timeout)` passes the same buffer again and gets the whole line. +pub(crate) fn read_json(reader: &mut BufReader, partial: &mut Vec, deadline: Instant) -> Result, LinkError> { + loop { + let Some(bytes) = pull_line(reader, partial, deadline)? else { + return Ok(None); + }; + let text = std::str::from_utf8(&bytes).map_err(|err| LinkError::Invalid(err.to_string()))?; + let text = text.trim(); + if text.is_empty() { + continue; + } + return Ok(Some(serde_json::from_str(text)?)); + } +} + +/// Same line limit as [`wire::read_line`]. The io error kind is kept so a +/// socket timeout becomes [`LinkError::Timeout`] instead of a generic io string. +/// +/// Bytes read before a timeout stay in `partial` and the next call continues that line. +fn pull_line(reader: &mut BufReader, partial: &mut Vec, deadline: Instant) -> Result>, LinkError> { + let max = wire::MAX_LINE; + let limit = u64::try_from(max).map_err(|_| LinkError::TooLarge)?.saturating_add(1); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(LinkError::Timeout); + } + reader.get_ref().arm_recv_timeout(Some(remaining))?; + let left = limit.saturating_sub(u64::try_from(partial.len()).unwrap_or(u64::MAX)); + match reader.take(left).read_until(b'\n', partial) { + Ok(0) if partial.is_empty() => return Ok(None), + Ok(_) => return finish_line(std::mem::take(partial), max), + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) if is_timeout(&err) => return Err(LinkError::Timeout), + Err(err) => return Err(err.into()), + } + } +} + +fn finish_line(mut buf: Vec, max: usize) -> Result>, LinkError> { + if buf.last() == Some(&b'\n') { + buf.pop(); + if buf.last() == Some(&b'\r') { + buf.pop(); + } + } else if buf.len() > max { + return Err(LinkError::TooLarge); + } + if buf.len() > max { + return Err(LinkError::TooLarge); + } + Ok(Some(buf)) +} + +fn is_timeout(err: &std::io::Error) -> bool { + if matches!(err.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) { + return true; + } + // Linux EAGAIN 11, ETIMEDOUT 110. macOS EAGAIN 35, ETIMEDOUT 60. + // Windows ERROR_SEM_TIMEOUT 121, WAIT_TIMEOUT 258. + matches!(err.raw_os_error(), Some(11 | 35 | 60 | 110 | 121 | 258)) +} + +fn hello_params(identity: &HelloClient) -> Value { + json!({ + "client": { + "app": identity.app, + "app_version": identity.app_version, + "pid": identity.pid, + "instance": identity.instance, + }, + "protocol": {"min": identity.protocol_min, "max": identity.protocol_max}, + }) +} + +fn check_identity(result: &Value, record: &InstanceRecord) -> Result<(), LinkError> { + let got_pid = result.get("pid"); + if got_pid.and_then(Value::as_u64) != Some(u64::from(record.pid)) { + return Err(LinkError::Mismatch { + field: "pid".into(), + expected: record.pid.to_string(), + got: got_pid.map(json_text).unwrap_or_else(|| "missing".into()), + }); + } + let got_instance = result.get("instance"); + if got_instance.and_then(Value::as_str) != Some(record.instance.as_str()) { + return Err(LinkError::Mismatch { + field: "instance".into(), + expected: record.instance.clone(), + got: got_instance.map(json_text).unwrap_or_else(|| "missing".into()), + }); + } + Ok(()) +} + +fn json_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Number(number) => number.to_string(), + Value::Bool(flag) => flag.to_string(), + Value::Null => "null".into(), + other => other.to_string(), + } +} + +fn id_is(value: &Value, id: u64) -> bool { + value.get("id").and_then(Value::as_u64) == Some(id) +} + +fn parse_event(value: &Value) -> Result { + let event = value.get("event").and_then(Value::as_str).ok_or_else(|| LinkError::Invalid("event line has no event name".into()))?; + let seq = value.get("seq").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("event line has no seq".into()))?; + let data = value.get("data").cloned().unwrap_or(Value::Null); + Ok(Event { event: event.to_string(), seq, data }) +} + +pub(crate) fn map_reply(value: &Value) -> Result { + let Some(ok) = value.get("ok").and_then(Value::as_bool) else { + return Err(LinkError::Invalid("reply has no ok flag".into())); + }; + if ok { + return value.get("result").cloned().ok_or_else(|| LinkError::Invalid("reply has no result".into())); + } + match value.get("error") { + Some(Value::String(text)) => Err(LinkError::Invalid(text.clone())), + Some(Value::Object(map)) => map_error_object(map), + Some(_) => Err(LinkError::Invalid("error is not a string or an object".into())), + None => Err(LinkError::Invalid("reply has no error".into())), + } +} + +fn map_error_object(map: &Map) -> Result { + let code = map.get("code").and_then(Value::as_str).unwrap_or(""); + let message = map.get("message").and_then(Value::as_str).filter(|text| !text.is_empty()).unwrap_or(code).to_string(); + match code { + "protocol_unsupported" => { + let data = map.get("data").and_then(Value::as_object); + let server_min = protocol_bound(data, "server_min")?; + let server_max = protocol_bound(data, "server_max")?; + Err(LinkError::Protocol { server_min, server_max }) + } + "auth_required" => Err(LinkError::Auth), + "bad_params" => Err(LinkError::BadParams(message)), + "payload_too_large" => Err(LinkError::TooLarge), + "payload_invalid" => Err(LinkError::PayloadInvalid(message)), + "media_missing" => { + let paths = map + .get("data") + .and_then(Value::as_object) + .and_then(|data| data.get("missing")) + .and_then(Value::as_array) + .map(|list| list.iter().filter_map(Value::as_str).map(str::to_string).collect()) + .unwrap_or_default(); + Err(LinkError::MediaMissing { paths }) + } + "not_ready" => Err(LinkError::NotReady(message)), + "link_unknown" => Err(LinkError::Unknown(message)), + "busy" => Err(LinkError::Busy(message)), + _ => Err(LinkError::Invalid(if message.is_empty() { "unknown error".into() } else { message })), + } +} + +fn protocol_bound(data: Option<&Map>, key: &str) -> Result { + let Some(value) = data.and_then(|map| map.get(key)).and_then(Value::as_u64) else { + return Err(LinkError::Invalid(format!("protocol_unsupported is missing {key}"))); + }; + u32::try_from(value).map_err(|_| LinkError::Invalid(format!("{key} does not fit in u32"))) +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::io::{BufReader, Write}; + use std::os::unix::net::{UnixListener, UnixStream}; + use std::path::PathBuf; + use std::thread::JoinHandle; + use std::time::{Duration, Instant}; + + use serde_json::{Value, json}; + + use super::*; + + struct Peer { + path: PathBuf, + record: InstanceRecord, + join: Option>, + } + + impl Drop for Peer { + fn drop(&mut self) { + if let Some(join) = self.join.take() { + let _ = join.join(); + } + let _ = std::fs::remove_file(&self.path); + } + } + + fn sock_path(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))) + } + + fn peer(label: &str, script: F) -> Peer + where + F: FnOnce(UnixStream, InstanceRecord) + Send + 'static, + { + let path = sock_path(label); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let endpoint = Endpoint::Unix { path: path.to_str().unwrap().to_string() }; + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let moved = record.clone(); + let join = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(10))).unwrap(); + script(stream, moved); + }); + Peer { path, record, join: Some(join) } + } + + fn read_req(reader: &mut BufReader) -> Value { + let bytes = crate::wire::read_line(reader, crate::wire::MAX_LINE).unwrap().expect("peer eof"); + serde_json::from_slice(&bytes).unwrap() + } + + fn write_val(reader: &mut BufReader, value: &Value) { + crate::wire::write_line(reader.get_mut(), value).unwrap(); + } + + fn accept_auth(reader: &mut BufReader, token: &str) { + let req = read_req(reader); + assert_eq!(req["method"], "auth"); + assert_eq!(req["id"], json!(0)); + assert_eq!(req["params"]["token"], token); + // Key order differs from serde_json's BTreeMap on purpose. + reader.get_mut().write_all(b"{\"result\":{\"authenticated\":true},\"ok\":true,\"id\":0}\n").unwrap(); + reader.get_mut().flush().unwrap(); + } + + #[test] + fn auth_then_call_parses_the_result() { + let server = peer("call", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + assert_eq!(req["method"], "engine.execute"); + assert_eq!(req["id"], json!(1)); + assert_eq!(req["params"]["command"], "ping"); + let id = req["id"].clone(); + write_val(&mut reader, &crate::wire::reply_ok(&id, json!({"pong": true}))); + }); + let mut client = Client::connect(&server.record).unwrap(); + let result = client.call("engine.execute", json!({"command": "ping"})).unwrap(); + assert_eq!(result, json!({"pong": true})); + } + + #[test] + fn event_line_before_the_reply_is_returned_by_next_event() { + let server = peer("ev", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + let id = req["id"].clone(); + write_val(&mut reader, &json!({"event": "doc.changed", "seq": 4, "data": {"doc_id": "abc"}})); + write_val(&mut reader, &crate::wire::reply_ok(&id, json!({"n": 1}))); + }); + let mut client = Client::connect(&server.record).unwrap(); + assert!(client.next_event(Some(Duration::ZERO)).unwrap().is_none()); + let result = client.call("link.focus", json!({})).unwrap(); + assert_eq!(result, json!({"n": 1})); + let event = client.next_event(Some(Duration::ZERO)).unwrap().expect("buffered event"); + assert_eq!(event.event, "doc.changed"); + assert_eq!(event.seq, 4); + assert_eq!(event.data, json!({"doc_id": "abc"})); + assert!(client.next_event(Some(Duration::ZERO)).unwrap().is_none()); + } + + #[test] + fn queued_events_are_read_after_the_peer_has_closed() { + let mut server = peer("closed", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + for seq in [1, 2] { + write_val(&mut reader, &json!({"event": "doc.changed", "seq": seq, "data": {}})); + } + }); + let mut client = Client::connect(&server.record).unwrap(); + server.join.take().unwrap().join().unwrap(); + for seq in [1, 2] { + let event = client.next_event(Some(Duration::from_secs(1))).unwrap().expect("queued event after peer close"); + assert_eq!(event.seq, seq); + } + assert!(matches!(client.next_event(Some(Duration::from_secs(1))), Err(LinkError::Closed))); + } + + #[test] + fn reconnect_sends_auth_again() { + let path = sock_path("re"); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let endpoint = Endpoint::Unix { path: path.to_str().unwrap().to_string() }; + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let token = record.token.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let join = std::thread::spawn(move || { + for _ in 0..2 { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(stream); + let req = read_req(&mut reader); + tx.send(req).unwrap(); + write_val(&mut reader, &crate::wire::auth_success(&json!(0))); + } + }); + let mut client = Client::connect(&record).unwrap(); + client.reconnect().unwrap(); + drop(client); + let first = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let second = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(first["method"], "auth"); + assert_eq!(second["method"], "auth"); + assert_eq!(first["params"]["token"], token); + assert_eq!(second["params"]["token"], token); + join.join().unwrap(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn failed_auth_reply_is_auth() { + let server = peer("bad", |stream, _record| { + let mut reader = BufReader::new(stream); + let _req = read_req(&mut reader); + write_val(&mut reader, &crate::wire::auth_failure()); + }); + let err = match Client::connect(&server.record) { + Err(err) => err, + Ok(_client) => panic!("connect should fail auth"), + }; + assert!(matches!(err, LinkError::Auth), "{err}"); + } + + #[test] + fn closed_before_auth_reply_is_auth() { + let server = peer("eof", |stream, _record| { + let mut reader = BufReader::new(stream); + let _req = read_req(&mut reader); + drop(reader); + }); + let err = match Client::connect(&server.record) { + Err(err) => err, + Ok(_client) => panic!("connect should fail when the peer closes"), + }; + assert!(matches!(err, LinkError::Auth), "{err}"); + } + + #[test] + fn hello_pid_mismatch_is_mismatch() { + let server = peer("hi", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + assert_eq!(req["method"], "link.hello"); + let id = req["id"].clone(); + let result = json!({ + "app": "effectcraft", + "app_version": "0.8.3", + "pid": 4243, + "instance": record.instance, + "protocol": 1, + }); + write_val(&mut reader, &crate::wire::reply_ok(&id, result)); + }); + let mut client = Client::connect(&server.record).unwrap(); + let identity = HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "11111111-1111-4111-8111-111111111111".into(), + protocol_min: wire::PROTOCOL_MIN, + protocol_max: wire::PROTOCOL_MAX, + }; + let err = client.hello(&identity).unwrap_err(); + match err { + LinkError::Mismatch { field, expected, got } => { + assert_eq!(field, "pid"); + assert_eq!(expected, "4242"); + assert_eq!(got, "4243"); + } + other => panic!("expected mismatch, got {other}"), + } + } + + #[test] + fn error_object_and_string_map_onto_link_error() { + let server = peer("err", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + for _ in 0..3 { + let req = read_req(&mut reader); + let id = req["id"].clone(); + let method = req["method"].as_str().unwrap().to_string(); + let error = match method.as_str() { + "proto" => json!({"code": "protocol_unsupported", "message": "no", "data": {"server_min": 2, "server_max": 5}}), + "text" => json!("no such method"), + "params" => json!({"code": "bad_params", "message": "missing command", "data": {}}), + other => panic!("unexpected method {other}"), + }; + write_val(&mut reader, &json!({"id": id, "ok": false, "error": error})); + } + }); + let mut client = Client::connect(&server.record).unwrap(); + match client.call("proto", json!({})).unwrap_err() { + LinkError::Protocol { server_min, server_max } => { + assert_eq!(server_min, 2); + assert_eq!(server_max, 5); + } + other => panic!("{other}"), + } + match client.call("text", json!({})).unwrap_err() { + LinkError::Invalid(text) => assert_eq!(text, "no such method"), + other => panic!("{other}"), + } + match client.call("params", json!({})).unwrap_err() { + LinkError::BadParams(text) => assert_eq!(text, "missing command"), + other => panic!("{other}"), + } + } + + #[test] + fn next_event_resumes_a_line_cut_short_by_a_timeout() { + let server = peer("split", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let out = reader.get_mut(); + out.write_all(b"{\"event\":\"doc.changed\",\"seq\":9,").unwrap(); + out.flush().unwrap(); + std::thread::sleep(Duration::from_millis(600)); + out.write_all(b"\"data\":{\"doc_id\":\"abc\"}}\n").unwrap(); + out.flush().unwrap(); + std::thread::sleep(Duration::from_millis(300)); + }); + let mut client = Client::connect(&server.record).unwrap(); + assert!(client.next_event(Some(Duration::from_millis(200))).unwrap().is_none()); + let event = client.next_event(Some(Duration::from_secs(5))).unwrap().expect("the whole event"); + assert_eq!(event.event, "doc.changed"); + assert_eq!(event.seq, 9); + assert_eq!(event.data, json!({"doc_id": "abc"})); + } + + #[test] + fn next_event_times_out_when_the_peer_is_quiet() { + let server = peer("wait", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + std::thread::sleep(Duration::from_millis(800)); + }); + let mut client = Client::connect(&server.record).unwrap(); + let started = Instant::now(); + let got = client.next_event(Some(Duration::from_millis(300))).unwrap(); + let elapsed = started.elapsed(); + assert!(got.is_none()); + assert!(elapsed >= Duration::from_millis(200), "returned too fast: {elapsed:?}"); + assert!(elapsed < Duration::from_secs(5), "waited too long: {elapsed:?}"); + } +} diff --git a/crates/ipc/src/discovery/dir.rs b/crates/ipc/src/discovery/dir.rs new file mode 100644 index 000000000..976c316bb --- /dev/null +++ b/crates/ipc/src/discovery/dir.rs @@ -0,0 +1,354 @@ +//! Per-user runtime directory. Hand-rolled environment lookups. No `dirs` crate. + +use std::path::{Path, PathBuf}; + +use crate::error::LinkError; + +/// The directory that holds instance records, locks, and socket files. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeDir { + path: PathBuf, +} + +impl RuntimeDir { + /// Resolve the directory for this process and create it when it is missing. + pub fn resolve() -> Result { + Self::from_resolved_path(runtime_path()?) + } + + fn from_resolved_path(path: PathBuf) -> Result { + // LOCALAPPDATA exists on a fresh Windows install, but its Craft namespace does not. + // Create and validate that single namespace first; do not recursively follow arbitrary + // missing ancestors. RuntimeDir::at keeps its existing caller-provided-path contract. + #[cfg(windows)] + if let Some(parent) = path.parent() { + ensure_private_dir(parent)?; + } + Self::at(path) + } + + /// Use `path` as the runtime directory. Tests use this so they do not touch + /// the real `$XDG_RUNTIME_DIR/craft`. + pub fn at(path: impl Into) -> Result { + let path = path.into(); + ensure_private_dir(&path)?; + Ok(Self { path }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn record_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.json")) + } + + pub fn lock_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.lock")) + } + + pub fn socket_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.sock")) + } + + /// Unix socket endpoint. The path must fit the instance-record limit of 100 bytes. + pub fn unix_endpoint(&self, app: &str, pid: u32) -> Result { + let path = self.socket_file(app, pid); + let text = path.to_str().ok_or_else(|| LinkError::Invalid("socket path is not utf-8".into()))?.to_string(); + if text.len() > 100 { + return Err(LinkError::Invalid(format!("socket path is {} bytes, the record allows 100", text.len()))); + } + Ok(super::Endpoint::Unix { path: text }) + } +} + +/// Set `path` to `mode` (for example `0o600`) and refuse a different result. +/// Unix socket mode cannot be set through `ListenerOptions::set_mode` (it is +/// `pub(crate)` in interprocess 2.4.4). Call this after the socket file exists. +#[cfg(unix)] +pub fn enforce_mode(path: &Path, mode: u32) -> Result<(), LinkError> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode))?; + let got = std::fs::symlink_metadata(path)?.mode() & 0o777; + if got != mode { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("mode is {got:o}, want {mode:o}") }); + } + Ok(()) +} + +#[cfg(unix)] +pub(crate) fn current_euid() -> Result { + #[cfg(target_os = "linux")] + { + let text = std::fs::read_to_string("/proc/self/status")?; + for line in text.lines() { + let Some(rest) = line.strip_prefix("Uid:") else { continue }; + let mut fields = rest.split_whitespace(); + let _real = fields.next(); + let Some(eff) = fields.next() else { + return Err(LinkError::Io("Uid line is short".into())); + }; + return eff.parse().map_err(LinkError::io); + } + Err(LinkError::Io("no Uid line in /proc/self/status".into())) + } + #[cfg(all(unix, not(target_os = "linux")))] + { + probe_euid(&std::env::temp_dir()) + } +} + +/// The effective uid, read from the owner of a file this process creates in `dir`. The std library +/// has no safe call for the euid and this crate forbids `unsafe`. The file has a random name and is +/// made with `create_new`, which fails on an existing path (a planted file or symlink) instead of +/// opening it, so the probe only ever touches a file it made. The file is removed before the +/// function returns. +#[cfg(any(all(unix, not(target_os = "linux")), all(test, unix)))] +fn probe_euid(dir: &Path) -> Result { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let path = dir.join(format!(".craft-uid-{}-{}", std::process::id(), crate::auth::encode_hex(&rand))); + let file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&path)?; + let uid = file.metadata().map(|meta| meta.uid()); + drop(file); + let _ = std::fs::remove_file(&path); + Ok(uid?) +} + +fn runtime_path() -> Result { + #[cfg(target_os = "linux")] + { + let xdg = std::env::var_os("XDG_RUNTIME_DIR"); + Ok(linux_runtime_path(xdg.as_deref(), current_euid()?)) + } + #[cfg(target_os = "macos")] + { + let tmpdir = std::env::var_os("TMPDIR"); + Ok(macos_runtime_path(tmpdir.as_deref(), current_euid()?)) + } + #[cfg(windows)] + { + let value = std::env::var_os("LOCALAPPDATA").filter(|v| !v.is_empty()); + let Some(value) = value else { + return Err(LinkError::Refused { path: "%LOCALAPPDATA%".into(), reason: "LOCALAPPDATA is unset".into() }); + }; + let base = PathBuf::from(value); + if !base.is_absolute() { + return Err(LinkError::Refused { path: base.display().to_string(), reason: "LOCALAPPDATA is not absolute".into() }); + } + Ok(base.join("Craft").join("run")) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + Err(LinkError::Refused { path: String::new(), reason: "no runtime directory for this platform".into() }) + } +} + +#[cfg(target_os = "linux")] +fn linux_runtime_path(xdg: Option<&std::ffi::OsStr>, euid: u32) -> PathBuf { + if let Some(xdg) = xdg { + let path = PathBuf::from(xdg); + if !xdg.is_empty() && path.is_absolute() { + return path.join("craft"); + } + } + PathBuf::from(format!("/tmp/craft-{euid}")) +} + +/// `$TMPDIR/craft` when `TMPDIR` is absolute. Otherwise `/tmp/craft-`, because a shared +/// `/tmp/craft` belongs to the first user who creates it and refuses everyone else. +#[cfg(any(target_os = "macos", all(test, unix)))] +fn macos_runtime_path(tmpdir: Option<&std::ffi::OsStr>, euid: u32) -> PathBuf { + if let Some(tmpdir) = tmpdir { + let path = PathBuf::from(tmpdir); + if !tmpdir.is_empty() && path.is_absolute() { + return path.join("craft"); + } + } + PathBuf::from(format!("/tmp/craft-{euid}")) +} + +pub(crate) fn ensure_private_dir(path: &Path) -> Result<(), LinkError> { + if is_symlink(path) { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "runtime directory is a symlink".into() }); + } + let created = match create_dir(path) { + Ok(()) => true, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => false, + Err(err) => return Err(LinkError::io(err)), + }; + if is_symlink(path) { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "runtime directory is a symlink".into() }); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if created { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))?; + } + verify_unix_dir(path)?; + } + #[cfg(not(unix))] + { + let _ = created; + let meta = std::fs::metadata(path)?; + if !meta.is_dir() { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "not a directory".into() }); + } + } + Ok(()) +} + +fn create_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path) + } + #[cfg(not(unix))] + { + std::fs::create_dir(path) + } +} + +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) +} + +#[cfg(unix)] +fn verify_unix_dir(path: &Path) -> Result<(), LinkError> { + use std::os::unix::fs::MetadataExt; + let meta = std::fs::symlink_metadata(path)?; + if !meta.is_dir() { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "not a directory".into() }); + } + let uid = current_euid()?; + if meta.uid() != uid { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("owner is {}, euid is {uid}", meta.uid()) }); + } + let mode = meta.mode() & 0o777; + if mode != 0o700 { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("mode is {mode:o}, want 700") }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + path + } + + #[test] + fn creates_0700_and_refuses_a_loose_dir() { + let path = scratch("dir"); + let dir = RuntimeDir::at(&path).unwrap(); + assert_eq!(dir.path(), path.as_path()); + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + assert_eq!(std::fs::symlink_metadata(&path).unwrap().mode() & 0o777, 0o700); + let loose = scratch("loose"); + std::fs::create_dir(&loose).unwrap(); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o755)).unwrap(); + let err = RuntimeDir::at(&loose).unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err}"); + let _ = std::fs::remove_dir_all(&loose); + } + let _ = std::fs::remove_dir_all(&path); + } + + #[cfg(unix)] + #[test] + fn refuses_a_symlink() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let real = scratch("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = scratch("link"); + symlink(&real, &link).unwrap(); + assert!(RuntimeDir::at(&link).is_err()); + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_dir_all(&real); + } + + #[cfg(target_os = "linux")] + #[test] + fn xdg_fallback_is_pure() { + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("/run/user/1000")), 5), PathBuf::from("/run/user/1000/craft")); + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("relative")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(linux_runtime_path(None, 9), PathBuf::from("/tmp/craft-9")); + } + + #[cfg(unix)] + #[test] + fn macos_fallback_is_per_user() { + use std::ffi::OsStr; + assert_eq!(macos_runtime_path(Some(OsStr::new("/var/folders/ab/T")), 5), PathBuf::from("/var/folders/ab/T/craft")); + assert_eq!(macos_runtime_path(Some(OsStr::new("")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(macos_runtime_path(Some(OsStr::new("relative")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(macos_runtime_path(None, 9), PathBuf::from("/tmp/craft-9")); + assert_ne!(macos_runtime_path(None, 5), macos_runtime_path(None, 6)); + } + + #[cfg(unix)] + #[test] + fn the_uid_probe_leaves_a_planted_file_alone_and_cleans_up() { + use std::os::unix::fs::symlink; + let dir = scratch("probe"); + std::fs::create_dir_all(&dir).unwrap(); + // The names the probe used to pick: a planted file and a planted symlink must not be opened. + let planted = dir.join(format!(".craft-uid-{}", std::process::id())); + std::fs::write(&planted, b"keep me").unwrap(); + let victim = dir.join("victim"); + std::fs::write(&victim, b"keep me too").unwrap(); + let link = dir.join(format!(".craft-uid-{}-0000000000000000", std::process::id())); + symlink(&victim, &link).unwrap(); + let uid = probe_euid(&dir).unwrap(); + assert_eq!(uid, current_euid().unwrap()); + assert_eq!(std::fs::read(&planted).unwrap(), b"keep me"); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep me too"); + let mut names: Vec = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + names.sort(); + assert_eq!(names.len(), 3, "the probe file is gone: {names:?}"); + // Two probes do not collide. + assert_eq!(probe_euid(&dir).unwrap(), uid); + // A directory that cannot be written is an error, not a guess. + assert!(probe_euid(&dir.join("missing")).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(windows)] + #[test] + fn a_fresh_windows_runtime_creates_its_namespace() { + let base = scratch("fresh-runtime"); + std::fs::create_dir_all(&base).unwrap(); + let path = base.join("Craft").join("run"); + let dir = RuntimeDir::from_resolved_path(path.clone()).unwrap(); + assert_eq!(dir.path(), path); + assert!(base.join("Craft").is_dir()); + std::fs::remove_dir_all(base).unwrap(); + } + + #[test] + fn resolve_points_at_a_private_craft_dir() { + let dir = RuntimeDir::resolve().unwrap(); + #[cfg(windows)] + assert!(dir.path().ends_with(Path::new("Craft").join("run")), "{}", dir.path().display()); + #[cfg(unix)] + assert!(dir.path().ends_with("craft") || dir.path().ends_with(format!("craft-{}", current_euid().unwrap())), "{}", dir.path().display()); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!(std::fs::symlink_metadata(dir.path()).unwrap().mode() & 0o777, 0o700); + } + } +} diff --git a/crates/ipc/src/discovery/mod.rs b/crates/ipc/src/discovery/mod.rs new file mode 100644 index 000000000..f81731970 --- /dev/null +++ b/crates/ipc/src/discovery/mod.rs @@ -0,0 +1,11 @@ +//! Discovery records for Craft apps on this machine. + +mod dir; +mod record; + +pub use dir::RuntimeDir; +pub use record::{Discovery, DocInfo, Endpoint, InstanceRecord, Protocol, valid_app_name}; + +pub(crate) use dir::ensure_private_dir; +#[cfg(unix)] +pub(crate) use dir::{current_euid, enforce_mode}; diff --git a/crates/ipc/src/discovery/record.rs b/crates/ipc/src/discovery/record.rs new file mode 100644 index 000000000..290ac0bb5 --- /dev/null +++ b/crates/ipc/src/discovery/record.rs @@ -0,0 +1,575 @@ +//! Instance records. A live process holds `-.lock`. `scan` deletes +//! the record, the socket, and the lock when `try_lock` acquires that file. + +use std::collections::BTreeSet; +use std::fs::{File, OpenOptions, TryLockError}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Deserialize; +use serde::Serialize; +use serde_json::{Map, Value}; + +use crate::auth::{self, is_lower_hex}; +use crate::error::LinkError; +use crate::wire::PROTOCOL_MAX; + +use super::RuntimeDir; + +/// Where clients connect. Serialized with an internal `kind` tag. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Endpoint { + Unix { path: String }, + Pipe { name: String }, + Tcp { port: u16 }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Protocol { + pub link: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commands_schema: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DocInfo { + pub doc_id: String, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// One running app. Unknown JSON fields are kept in `extra`. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct InstanceRecord { + pub v: u32, + pub app: String, + pub app_version: String, + pub pid: u32, + pub instance: String, + pub endpoint: Endpoint, + pub token: String, + pub started: u64, + pub protocol: Protocol, + pub docs: Vec, + #[serde(default, flatten)] + pub extra: Map, +} + +impl std::fmt::Debug for InstanceRecord { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InstanceRecord") + .field("v", &self.v) + .field("app", &self.app) + .field("app_version", &self.app_version) + .field("pid", &self.pid) + .field("instance", &self.instance) + .field("endpoint", &self.endpoint) + .field("token", &"redacted") + .field("started", &self.started) + .field("protocol", &self.protocol) + .field("docs", &self.docs) + .finish_non_exhaustive() + } +} + +impl InstanceRecord { + pub fn new(app: &str, app_version: &str, pid: u32, endpoint: Endpoint) -> Result { + let record = Self { + v: 1, + app: app.to_string(), + app_version: app_version.to_string(), + pid, + instance: auth::random_uuid()?, + endpoint, + token: auth::generate_token()?, + started: unix_now()?, + protocol: Protocol { link: PROTOCOL_MAX, commands_schema: None, extra: Map::new() }, + docs: Vec::new(), + extra: Map::new(), + }; + record.validate()?; + Ok(record) + } + + pub fn validate(&self) -> Result<(), LinkError> { + if self.v != 1 { + return Err(LinkError::Invalid(format!("record v is {}, want 1", self.v))); + } + if !valid_app_name(&self.app) { + return Err(LinkError::Invalid(format!("app {:?} does not match the record pattern", self.app))); + } + if self.app_version.is_empty() { + return Err(LinkError::Invalid("app_version is empty".into())); + } + if self.pid < 1 { + return Err(LinkError::Invalid("pid must be >= 1".into())); + } + if !valid_uuid(&self.instance) { + return Err(LinkError::Invalid("instance is not a uuid".into())); + } + if self.token.len() != 64 || !is_lower_hex(&self.token) { + return Err(LinkError::Invalid("token must be 64 lowercase hex characters".into())); + } + if self.protocol.link < 1 { + return Err(LinkError::Invalid("protocol.link must be >= 1".into())); + } + if let Some(schema) = &self.protocol.commands_schema { + let Some(hex) = schema.strip_prefix("sha256:") else { + return Err(LinkError::Invalid("commands_schema must start with sha256:".into())); + }; + if hex.len() != 64 || !is_lower_hex(hex) { + return Err(LinkError::Invalid("commands_schema digest is not 64 lowercase hex characters".into())); + } + } + match &self.endpoint { + Endpoint::Unix { path } => { + if path.len() > 100 || path.is_empty() || path.contains('\0') || !Path::new(path).is_absolute() { + return Err(LinkError::Invalid("unix endpoint path must be absolute and at most 100 bytes".into())); + } + } + Endpoint::Pipe { name } => { + if !name.starts_with("craft-") || name.contains('/') || name.contains('\\') { + return Err(LinkError::Invalid("pipe name must start with craft-".into())); + } + } + Endpoint::Tcp { port } => { + if *port < 1 { + return Err(LinkError::Invalid("tcp port must be >= 1".into())); + } + } + } + for doc in &self.docs { + if doc.doc_id.is_empty() || doc.title.is_empty() { + return Err(LinkError::Invalid("doc_id and title are required".into())); + } + } + self.check_caps()?; + self.check_tcp_port()?; + Ok(()) + } + + /// Roles and optional verbs from the optional `caps` field (stored in + /// `extra`, so the struct shape and old struct literals stay valid). + /// Empty when the field is absent or malformed. [`Self::validate`] rejects + /// a malformed field. + pub fn caps(&self) -> Vec { + self.extra + .get("caps") + .and_then(Value::as_array) + .map(|list| list.iter().filter_map(|c| c.as_str().map(str::to_string)).collect()) + .unwrap_or_default() + } + + /// Add `cap` to `caps` when it is not there yet. + pub fn add_cap(&mut self, cap: &str) -> Result<(), LinkError> { + if !valid_cap(cap) { + return Err(LinkError::Invalid(format!("cap {cap:?} must match [a-z][a-z0-9._-]{{0,63}}"))); + } + let mut caps = self.caps(); + if !caps.iter().any(|c| c == cap) { + caps.push(cap.to_string()); + } + self.extra.insert("caps".to_string(), Value::from(caps)); + Ok(()) + } + + /// Loopback TCP control port, when the app opened one (`--control `). Stored in + /// `extra`. The record file is 0600, so only the owner can read this port and the token + /// that the TCP listener requires. + pub fn tcp_port(&self) -> Option { + self.extra.get("tcp_port").and_then(Value::as_u64).and_then(|port| u16::try_from(port).ok()).filter(|port| *port >= 1) + } + + /// Publish the TCP control port in the record. + pub fn set_tcp_port(&mut self, port: u16) { + self.extra.insert("tcp_port".to_string(), Value::from(port)); + } + + fn check_tcp_port(&self) -> Result<(), LinkError> { + match self.extra.get("tcp_port") { + None => Ok(()), + Some(value) if value.as_u64().is_some_and(|port| (1..=65535).contains(&port)) => Ok(()), + Some(_) => Err(LinkError::Invalid("tcp_port must be an integer from 1 to 65535".into())), + } + } + + fn check_caps(&self) -> Result<(), LinkError> { + let Some(value) = self.extra.get("caps") else { + return Ok(()); + }; + let Some(list) = value.as_array() else { + return Err(LinkError::Invalid("caps must be an array of strings".into())); + }; + let mut seen = BTreeSet::new(); + for item in list { + let Some(cap) = item.as_str().filter(|c| valid_cap(c)) else { + return Err(LinkError::Invalid("each cap must match [a-z][a-z0-9._-]{0,63}".into())); + }; + if !seen.insert(cap) { + return Err(LinkError::Invalid(format!("cap {cap:?} is listed twice"))); + } + } + Ok(()) + } +} + +fn valid_cap(cap: &str) -> bool { + let bytes = cap.as_bytes(); + (1..=64).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_lowercase) + && bytes.iter().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-')) +} + +/// Holds the lock file for this process. Drop releases the lock and leaves the +/// files in place. [`Discovery::remove`] deletes them. [`Discovery::scan`] +/// deletes them when the holder is gone. +pub struct Discovery { + dir: PathBuf, + stem: String, + record: InstanceRecord, + lock: File, +} + +impl std::fmt::Debug for Discovery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Discovery").field("stem", &self.stem).field("record", &self.record).finish_non_exhaustive() + } +} + +impl Discovery { + /// Write the record and hold `-.lock` until drop. + /// + /// `try_lock` is the acquire. A successful `try_lock` is the same exclusive + /// lock as `File::lock`, and it returns [`LinkError::Busy`] when a live + /// process already holds the file instead of waiting. + pub fn publish(dir: &RuntimeDir, record: InstanceRecord) -> Result { + record.validate()?; + let stem = format!("{}-{}", record.app, record.pid); + let lock_path = dir.path().join(format!("{stem}.lock")); + let mut opts = OpenOptions::new(); + opts.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let lock = opts.open(&lock_path)?; + #[cfg(unix)] + crate::discovery::enforce_mode(&lock_path, 0o600)?; + match lock.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(LinkError::Busy(format!("{stem} is already live"))); + } + Err(TryLockError::Error(err)) => return Err(err.into()), + } + let keep = unix_path(&record); + clear_payload(dir.path(), &stem, keep.as_deref())?; + let bytes = serde_json::to_vec(&record)?; + write_atomic(dir.path(), &stem, &bytes)?; + Ok(Self { dir: dir.path().to_path_buf(), stem, record, lock }) + } + + pub fn record(&self) -> &InstanceRecord { + &self.record + } + + /// Delete this process's record, socket, and lock. + pub fn remove(self) -> Result<(), LinkError> { + reap_dead(&self.dir, &self.stem, &self.lock) + } + + /// Live records in `dir`. A lock that `try_lock` acquires belongs to a dead + /// process: this deletes its record, socket, and lock. + pub fn scan(dir: &RuntimeDir) -> Result, LinkError> { + let mut stems = BTreeSet::new(); + let entries = std::fs::read_dir(dir.path())?; + for entry in entries { + let entry = entry?; + let Some(name) = entry.file_name().into_string().ok() else { continue }; + if let Some(stem) = stem_from_entry(&name) { + stems.insert(stem); + } + } + let mut live = Vec::new(); + for stem in stems { + if let Some(record) = settle(dir.path(), &stem)? { + live.push(record); + } + } + Ok(live) + } +} + +pub fn valid_app_name(app: &str) -> bool { + crate::ids::valid_app_id(app) +} + +fn valid_uuid(text: &str) -> bool { + crate::ids::valid_uuid(text) +} + +fn unix_now() -> Result { + Ok(SystemTime::now().duration_since(UNIX_EPOCH).map_err(LinkError::io)?.as_secs()) +} + +fn stem_from_entry(name: &str) -> Option { + if let Some(stem) = name.strip_suffix(".json").or_else(|| name.strip_suffix(".lock")).or_else(|| name.strip_suffix(".sock")) { + return accept_stem(stem); + } + let rest = name.strip_prefix('.')?.strip_suffix(".tmp")?; + let (stem, hex) = rest.rsplit_once('.')?; + if hex.len() < 2 || !is_lower_hex(hex) { + return None; + } + accept_stem(stem) +} + +fn accept_stem(stem: &str) -> Option { + let (app, pid) = stem.rsplit_once('-')?; + let pid: u32 = pid.parse().ok()?; + if pid < 1 || !valid_app_name(app) { + return None; + } + Some(stem.to_string()) +} + +fn settle(dir: &Path, stem: &str) -> Result, LinkError> { + let lock_path = dir.join(format!("{stem}.lock")); + let json_path = dir.join(format!("{stem}.json")); + if lock_path.is_file() { + let lock = OpenOptions::new().read(true).write(true).open(&lock_path)?; + match lock.try_lock() { + Err(TryLockError::WouldBlock) => return read_live(&json_path), + Err(TryLockError::Error(err)) => return Err(err.into()), + Ok(()) => { + reap_dead(dir, stem, &lock)?; + return Ok(None); + } + } + } + if json_path.exists() || dir.join(format!("{stem}.sock")).exists() { + clear_payload(dir, stem, None)?; + } else { + remove_tmps(dir, stem)?; + } + Ok(None) +} + +fn read_live(json_path: &Path) -> Result, LinkError> { + if !json_path.is_file() { + return Ok(None); + } + let bytes = std::fs::read(json_path)?; + let record: InstanceRecord = match serde_json::from_slice(&bytes) { + Ok(record) => record, + Err(_) => return Ok(None), + }; + if record.validate().is_err() { + return Ok(None); + } + Ok(Some(record)) +} + +fn reap_dead(dir: &Path, stem: &str, lock: &File) -> Result<(), LinkError> { + clear_payload(dir, stem, None)?; + let lock_path = dir.join(format!("{stem}.lock")); + match std::fs::remove_file(&lock_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + lock.unlock()?; + Ok(()) +} + +fn unix_path(record: &InstanceRecord) -> Option { + match &record.endpoint { + Endpoint::Unix { path } => Some(PathBuf::from(path)), + Endpoint::Pipe { .. } | Endpoint::Tcp { .. } => None, + } +} + +/// Remove the json file, temp files, and socket files. `keep` is the socket the +/// caller just bound and is about to publish. Deleting that name would unlist +/// a live listener. +fn clear_payload(dir: &Path, stem: &str, keep: Option<&Path>) -> Result<(), LinkError> { + let json_path = dir.join(format!("{stem}.json")); + if let Ok(bytes) = std::fs::read(&json_path) + && let Ok(record) = serde_json::from_slice::(&bytes) + && let Some(old) = unix_path(&record) + && keep.is_none_or(|path| path != old.as_path()) + { + remove_socket_inside(dir, &old); + } + let conventional = dir.join(format!("{stem}.sock")); + if keep.is_none_or(|path| path != conventional.as_path()) { + remove_socket_inside(dir, &conventional); + } + match std::fs::remove_file(&json_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + remove_tmps(dir, stem) +} + +fn remove_tmps(dir: &Path, stem: &str) -> Result<(), LinkError> { + let prefix = format!(".{stem}."); + let entries = std::fs::read_dir(dir)?; + for entry in entries { + let entry = entry?; + let Some(name) = entry.file_name().into_string().ok() else { continue }; + if name.starts_with(&prefix) && name.ends_with(".tmp") { + std::fs::remove_file(entry.path())?; + } + } + Ok(()) +} + +#[cfg(unix)] +fn remove_socket_inside(runtime: &Path, candidate: &Path) { + use std::os::unix::fs::FileTypeExt; + let Ok(meta) = std::fs::symlink_metadata(candidate) else { return }; + if !meta.file_type().is_socket() { + return; + } + let Some(parent) = candidate.parent() else { return }; + let Ok(parent_real) = std::fs::canonicalize(parent) else { return }; + let Ok(runtime_real) = std::fs::canonicalize(runtime) else { return }; + if parent_real == runtime_real { + let _ = std::fs::remove_file(candidate); + } +} + +/// Only Unix endpoints leave socket files behind. A named pipe has no file to remove. +#[cfg(not(unix))] +fn remove_socket_inside(runtime: &Path, candidate: &Path) { + let _ = (runtime, candidate); +} + +/// `.-..tmp`, then rename onto `-.json`. +fn write_atomic(dir: &Path, stem: &str, bytes: &[u8]) -> Result<(), LinkError> { + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let tmp = dir.join(format!(".{stem}.{}.tmp", auth::encode_hex(&rand))); + let final_path = dir.join(format!("{stem}.json")); + let mut opts = OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut file = opts.open(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + if let Err(err) = std::fs::rename(&tmp, &final_path) { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(label: &str) -> (PathBuf, RuntimeDir) { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + let dir = RuntimeDir::at(&path).unwrap(); + (path, dir) + } + + #[test] + fn schema_accepts_a_record_and_rejects_a_bad_token() { + let (_path, dir) = scratch("schema"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let mut value = serde_json::to_value(&record).unwrap(); + value["note"] = Value::String("kept".into()); + let restored: InstanceRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(restored.extra.get("note").and_then(Value::as_str), Some("kept")); + let schema_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schemas/instance-record.schema.json"); + let schema: Value = serde_json::from_slice(&std::fs::read(&schema_path).unwrap()).unwrap(); + let validator = jsonschema::options().should_validate_formats(true).build(&schema).unwrap(); + let errors: Vec<_> = validator.iter_errors(&value).map(|err| err.to_string()).collect(); + assert!(errors.is_empty(), "{errors:?}"); + value["token"] = Value::String("nope".into()); + assert!(!validator.is_valid(&value)); + let _ = std::fs::remove_dir_all(dir.path()); + } + + #[test] + fn scan_keeps_a_held_record_and_drops_it_after_unlock() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let (path, dir) = scratch("scan"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let decoy = dir.socket_file("effectcraft", 4242); + std::fs::write(&decoy, b"not a socket").unwrap(); + let held = Discovery::publish(&dir, record.clone()).unwrap(); + let live = Discovery::scan(&dir).unwrap(); + assert_eq!(live.len(), 1); + assert_eq!(live[0].instance, record.instance); + assert_eq!(live[0].token, record.token); + drop(held); + assert!(Discovery::scan(&dir).unwrap().is_empty()); + assert!(!dir.record_file("effectcraft", 4242).exists()); + assert!(!dir.lock_file("effectcraft", 4242).exists()); + assert!(decoy.exists(), "scan must not delete a regular file"); + let _ = std::fs::remove_dir_all(&path); + } + + #[cfg(unix)] + #[test] + fn scan_unlinks_a_dead_socket() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + use interprocess::local_socket::prelude::*; + use interprocess::local_socket::{GenericFilePath, ListenerOptions}; + + let (path, dir) = scratch("sock"); + let endpoint = dir.unix_endpoint("filmcraft", 5151).unwrap(); + let sock = match &endpoint { + Endpoint::Unix { path } => path.clone(), + Endpoint::Pipe { .. } | Endpoint::Tcp { .. } => panic!("unix endpoint"), + }; + let name = Path::new(&sock).to_fs_name::().unwrap(); + let listener = ListenerOptions::new().name(name).reclaim_name(false).create_sync().unwrap(); + crate::discovery::enforce_mode(Path::new(&sock), 0o600).unwrap(); + let record = InstanceRecord::new("filmcraft", "0.9.1", 5151, endpoint).unwrap(); + let held = Discovery::publish(&dir, record).unwrap(); + assert!(Path::new(&sock).exists(), "publish must leave the bound socket in place"); + drop(held); + assert!(Path::new(&sock).exists(), "dropping the lock must not unlink the socket"); + assert!(Discovery::scan(&dir).unwrap().is_empty()); + assert!(!Path::new(&sock).exists(), "scan removes the socket once the lock is free"); + drop(listener); + let _ = std::fs::remove_dir_all(&path); + } + + #[test] + fn second_publish_of_a_live_stem_is_busy() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let (path, dir) = scratch("busy"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let first = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint.clone()).unwrap(); + let held = Discovery::publish(&dir, first).unwrap(); + let second = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let err = Discovery::publish(&dir, second).unwrap_err(); + assert!(matches!(err, LinkError::Busy(_)), "{err}"); + drop(held); + let _ = std::fs::remove_dir_all(&path); + } +} diff --git a/crates/ipc/src/error.rs b/crates/ipc/src/error.rs new file mode 100644 index 000000000..65259fd58 --- /dev/null +++ b/crates/ipc/src/error.rs @@ -0,0 +1,134 @@ +//! Failures a Craft Link caller can show or turn into a protocol error object. + +use serde::Serialize; +use serde_json::{Map, Value}; + +/// Wire error object for new methods. Existing methods keep a string `error`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ErrorObject { + pub code: String, + pub message: String, + pub data: Map, +} + +/// A link failure. The first-line auth refusal is still the string +/// `authentication required`. [`LinkError::to_object`] is for new methods. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LinkError { + Io(String), + /// The runtime directory or socket mode is not acceptable. + Refused { + path: String, + reason: String, + }, + Invalid(String), + Busy(String), + Auth, + Protocol { + server_min: u32, + server_max: u32, + }, + BadParams(String), + NotInstalled { + app: String, + }, + Mismatch { + field: String, + expected: String, + got: String, + }, + Closed, + Timeout, + TooLarge, + NotReady(String), + /// An asset moniker, link id or file that the peer does not know. + Unknown(String), + /// A `link.open` payload with a bad shape or an unknown `format`. + PayloadInvalid(String), + /// A `link.open` payload names media files that do not exist or cannot be opened. The + /// sender renders or relinks them and sends the call again. + MediaMissing { + paths: Vec, + }, +} + +impl LinkError { + pub fn io(err: impl std::fmt::Display) -> Self { + Self::Io(err.to_string()) + } + + /// Contract section 7 code. + pub fn code(&self) -> &'static str { + match self { + Self::Auth => "auth_required", + Self::Protocol { .. } => "protocol_unsupported", + Self::BadParams(_) | Self::Invalid(_) => "bad_params", + Self::NotReady(_) => "not_ready", + Self::Unknown(_) => "link_unknown", + Self::PayloadInvalid(_) => "payload_invalid", + Self::MediaMissing { .. } => "media_missing", + Self::TooLarge => "payload_too_large", + Self::Io(_) => "io_error", + Self::Busy(_) => "busy", + Self::NotInstalled { .. } => "not_ready", + Self::Refused { .. } | Self::Mismatch { .. } | Self::Closed | Self::Timeout => "internal", + } + } + + pub fn to_object(&self) -> ErrorObject { + let mut data = Map::new(); + if let Self::Protocol { server_min, server_max } = self { + data.insert("server_min".into(), Value::from(*server_min)); + data.insert("server_max".into(), Value::from(*server_max)); + } + if let Self::MediaMissing { paths } = self { + data.insert("missing".into(), Value::from(paths.clone())); + } + ErrorObject { code: self.code().to_string(), message: self.to_string(), data } + } +} + +impl std::fmt::Display for LinkError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(msg) => write!(f, "{msg}"), + Self::Refused { path, reason } => write!(f, "{path}: {reason}"), + Self::Invalid(msg) => write!(f, "{msg}"), + Self::Busy(msg) => write!(f, "{msg}"), + Self::Auth => write!(f, "authentication required"), + Self::Protocol { server_min, server_max } => { + write!(f, "protocol not supported (server {server_min} to {server_max})") + } + Self::BadParams(msg) => write!(f, "{msg}"), + Self::NotInstalled { app } => write!(f, "{app} is not installed"), + Self::Mismatch { field, expected, got } => { + write!(f, "{field} is {got}, record says {expected}") + } + Self::Closed => write!(f, "connection closed"), + Self::Timeout => write!(f, "timeout"), + Self::TooLarge => write!(f, "payload too large"), + Self::NotReady(msg) => write!(f, "{msg}"), + Self::Unknown(msg) | Self::PayloadInvalid(msg) => write!(f, "{msg}"), + Self::MediaMissing { paths } => write!(f, "the payload names media that cannot be opened: {}", paths.join(", ")), + } + } +} + +impl std::error::Error for LinkError {} + +impl From for LinkError { + /// A socket read timeout is `WouldBlock` on Linux ("Resource temporarily unavailable") and + /// `TimedOut` elsewhere. Both mean the peer was quiet, not that the connection broke. + fn from(err: std::io::Error) -> Self { + match err.kind() { + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut => Self::Timeout, + _ => Self::Io(err.to_string()), + } + } +} + +impl From for LinkError { + fn from(err: serde_json::Error) -> Self { + Self::Invalid(err.to_string()) + } +} diff --git a/crates/ipc/src/fetch.rs b/crates/ipc/src/fetch.rs new file mode 100644 index 000000000..ca3f4d194 --- /dev/null +++ b/crates/ipc/src/fetch.rs @@ -0,0 +1,370 @@ +//! Pull frames from a peer without blocking the caller past a deadline. +//! +//! [`FrameFetcher`] owns the control connection that holds the `frames.open` ticket and a few +//! bulk connections ([`crate::bulk::BulkClient`]), each on its own worker thread. A caller asks +//! with [`FrameFetcher::fetch`] and a time limit. +//! +//! - A frame in the RAM cache answers at once. The cache keeps the newest epoch of each +//! [`FrameKey`] inside a byte budget and drops the least recently used frame first. +//! - A miss goes to a worker. Asks for one key share one render. +//! - A caller that runs out of time gets [`Fetched::TimedOut`]. The render goes on, and its frame +//! lands in the cache for the next ask. The caller never waits for the socket. +//! - A queued ask older than [`FetchOptions::max_age`] is dropped when a worker reaches it, unless +//! a caller is still waiting for it. The queue holds at most [`QUEUE_MAX`] asks: a new miss past +//! that drops the oldest. +//! - A connection error ends that worker. When the last worker ends the fetcher is down and every +//! ask that misses the cache returns [`Fetched::Down`] at once. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use crate::InstanceRecord; +use crate::bulk::{BulkClient, Frame, FrameFormat, FrameTicket}; +use crate::client::Client; +use crate::error::LinkError; + +/// Most bulk connections one fetcher opens. +pub const CONNECTIONS_MAX: usize = 8; + +/// Most asks that wait for a worker at once. A new miss past this drops the oldest queued ask. +pub const QUEUE_MAX: usize = 256; + +/// What identifies a frame for the cache: the caller's frame number and scale (in thousandths). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrameKey { + pub frame: i64, + pub scale_milli: u32, +} + +/// Tuning of a [`FrameFetcher`]. +#[derive(Debug, Clone)] +pub struct FetchOptions { + /// Bulk connections, and so renders in flight. Clamped to `1..=`[`CONNECTIONS_MAX`]. + pub connections: usize, + /// How long one socket read or write may take before the connection is dropped. + pub io_timeout: Duration, + /// Cache size in bytes of pixel data. + pub cache_bytes: usize, + /// A queued ask older than this is skipped, unless a caller is still waiting for it. + pub max_age: Duration, +} + +impl Default for FetchOptions { + fn default() -> Self { + Self { connections: 3, io_timeout: Duration::from_secs(5), cache_bytes: 256 * 1024 * 1024, max_age: Duration::from_secs(1) } + } +} + +/// The answer to one [`FrameFetcher::fetch`]. +#[derive(Debug, Clone)] +pub enum Fetched { + /// A frame whose epoch is at least the one asked for. + Frame(Arc), + /// The time limit passed. The render may still finish and fill the cache. + TimedOut, + /// The peer answered this frame with an error. The fetcher still works. + /// + /// Also the answer to a caller whose ask was dropped from a full queue (see [`QUEUE_MAX`]). That + /// caller is not re-queued: it asks again if it still wants the frame. + Failed(String), + /// The connections are gone and the cache has no answer. + Down, +} + +/// Counters since [`FrameFetcher::connect`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FetchStats { + /// Calls to [`FrameFetcher::fetch`]. + pub requests: u64, + /// Calls answered by the cache on the first look. + pub hits: u64, + /// Frames a worker received. + pub renders: u64, + pub timeouts: u64, + pub failures: u64, + /// Queued asks dropped for age or for a full queue. + pub skipped: u64, +} + +struct Job { + key: FrameKey, + time_s: f64, + epoch: u64, + queued: Instant, + /// The latest deadline of a caller that is waiting for this job. + wanted_until: Instant, +} + +struct Cached { + frame: Arc, + used: u64, +} + +#[derive(Default)] +struct Cache { + frames: HashMap, + bytes: usize, + tick: u64, +} + +impl Cache { + fn get(&mut self, key: FrameKey, min_epoch: u64) -> Option> { + self.tick += 1; + let tick = self.tick; + let entry = self.frames.get_mut(&key).filter(|entry| entry.frame.epoch >= min_epoch)?; + entry.used = tick; + Some(Arc::clone(&entry.frame)) + } + + fn insert(&mut self, key: FrameKey, frame: Arc, budget: usize) { + if self.frames.get(&key).is_some_and(|old| old.frame.epoch > frame.epoch) { + return; + } + self.tick += 1; + self.bytes += frame.pixels.len(); + if let Some(old) = self.frames.insert(key, Cached { frame, used: self.tick }) { + self.bytes -= old.frame.pixels.len(); + } + while self.bytes > budget && self.frames.len() > 1 { + let Some(oldest) = self.frames.iter().min_by_key(|(_, entry)| entry.used).map(|(key, _)| *key) else { break }; + if let Some(gone) = self.frames.remove(&oldest) { + self.bytes -= gone.frame.pixels.len(); + } + } + } +} + +#[derive(Default)] +struct State { + queue: VecDeque, + inflight: HashSet, + cache: Cache, + failed: HashMap, + fail_seq: u64, + stats: FetchStats, + stop: bool, +} + +/// Most failure records kept. Callers only look for a record newer than their own ask, so old +/// ones go first. +const FAILED_MAX: usize = 4 * QUEUE_MAX; + +impl State { + /// Record a failure of `key` that callers waiting for it will see. + fn note_failed(&mut self, key: FrameKey, message: String) { + self.fail_seq += 1; + self.failed.insert(key, (self.fail_seq, message)); + if self.failed.len() > FAILED_MAX { + let keep_from = self.fail_seq.saturating_sub(u64::try_from(FAILED_MAX).unwrap_or(u64::MAX)); + self.failed.retain(|_, (seq, _)| *seq > keep_from); + } + } +} + +struct Shared { + state: Mutex, + /// Workers wait here for a job. + work: Condvar, + /// Callers wait here for a frame, a failure or the end. + done: Condvar, + alive: AtomicUsize, + options: FetchOptions, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +/// Frames pulled from one link of a peer. See the module docs. +pub struct FrameFetcher { + shared: Arc, + ticket: FrameTicket, + /// Holds the ticket: the peer drops it when this connection closes. + control: Mutex, +} + +impl FrameFetcher { + /// Open the ticket and the bulk connections. Any failure is returned here, so a caller knows + /// the peer can serve frames before it relies on them. + pub fn connect(record: &InstanceRecord, link_id: &str, format: FrameFormat, scale: f64, options: FetchOptions) -> Result { + let mut control = Client::connect(record)?; + let ticket = control.frames_open(link_id, format, scale)?; + let connections = options.connections.clamp(1, CONNECTIONS_MAX); + let mut clients = Vec::with_capacity(connections); + for _ in 0..connections { + let mut client = BulkClient::attach(record, &ticket.ticket)?; + client.set_io_timeout(options.io_timeout); + clients.push(client); + } + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + work: Condvar::new(), + done: Condvar::new(), + alive: AtomicUsize::new(clients.len()), + options, + }); + for client in clients { + let worker = Arc::clone(&shared); + // Workers are not joined: one may sit in a socket read for up to `io_timeout`, and a + // drop on a UI thread must not wait for that. A worker sees `stop` and leaves. + if let Err(err) = std::thread::Builder::new().name("craft-ipc-fetch".into()).spawn(move || run_worker(&worker, client)) { + shared_stop(&shared); + return Err(err.into()); + } + } + Ok(Self { shared, ticket, control: Mutex::new(control) }) + } + + /// Size of the frames this fetcher yields. + pub fn layout(&self) -> FrameTicket { + self.ticket.clone() + } + + /// The cached frame for `key` if its epoch is at least `min_epoch`. Never asks the peer. + pub fn cached(&self, key: FrameKey, min_epoch: u64) -> Option> { + self.shared.lock().cache.get(key, min_epoch) + } + + /// The frame for `key` at an epoch of at least `min_epoch`, waiting at most `wait`. + /// `time_s` is the comp time the peer renders. + /// + /// A `time_s` that is not finite is answered with [`Fetched::Failed`] and is never queued: the + /// request would fail before it reaches the socket, and a worker reads that as a dead connection. + pub fn fetch(&self, key: FrameKey, time_s: f64, min_epoch: u64, wait: Duration) -> Fetched { + if !time_s.is_finite() { + return Fetched::Failed("time_s must be finite".into()); + } + let deadline = Instant::now() + wait; + let shared = &self.shared; + let mut state = shared.lock(); + state.stats.requests += 1; + let started_at = state.fail_seq; + let mut first = true; + loop { + if let Some(frame) = state.cache.get(key, min_epoch) { + if first { + state.stats.hits += 1; + } + return Fetched::Frame(frame); + } + first = false; + if let Some((seq, message)) = state.failed.get(&key) + && *seq > started_at + { + return Fetched::Failed(message.clone()); + } + if shared.alive.load(Ordering::SeqCst) == 0 || state.stop { + return Fetched::Down; + } + if state.inflight.contains(&key) { + // Still waiting for a worker: this caller keeps the job alive until its own deadline. + if let Some(job) = state.queue.iter_mut().find(|job| job.key == key) { + job.wanted_until = job.wanted_until.max(deadline); + } + } else { + if state.queue.len() >= QUEUE_MAX + && let Some(oldest) = state.queue.pop_front() + { + state.inflight.remove(&oldest.key); + state.stats.skipped += 1; + // Callers waiting for the dropped key see this before they could queue it again. + state.note_failed(oldest.key, "dropped: the fetch queue is full".into()); + shared.done.notify_all(); + } + state.inflight.insert(key); + state.queue.push_back(Job { key, time_s, epoch: min_epoch, queued: Instant::now(), wanted_until: deadline }); + shared.work.notify_one(); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + state.stats.timeouts += 1; + return Fetched::TimedOut; + } + state = shared.done.wait_timeout(state, left).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } + + /// True once every connection has ended. + pub fn is_down(&self) -> bool { + self.shared.alive.load(Ordering::SeqCst) == 0 + } + + pub fn stats(&self) -> FetchStats { + self.shared.lock().stats + } + + /// Ask the peer to close the ticket now instead of at drop. + pub fn close(&self) { + shared_stop(&self.shared); + let mut control = self.control.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _ = control.call("link.bye", serde_json::json!({})); + } +} + +impl Drop for FrameFetcher { + fn drop(&mut self) { + shared_stop(&self.shared); + } +} + +fn shared_stop(shared: &Shared) { + shared.lock().stop = true; + shared.work.notify_all(); + shared.done.notify_all(); +} + +fn run_worker(shared: &Shared, mut client: BulkClient) { + while let Some(job) = next_job(shared) { + let reply = client.request_reply(job.time_s, job.epoch, None); + let mut state = shared.lock(); + state.inflight.remove(&job.key); + match reply { + Ok(Ok(frame)) => { + state.stats.renders += 1; + state.failed.remove(&job.key); + let budget = shared.options.cache_bytes; + state.cache.insert(job.key, Arc::new(frame), budget); + } + Ok(Err(message)) => { + state.stats.failures += 1; + state.note_failed(job.key, message); + } + Err(_) => { + drop(state); + break; + } + } + drop(state); + shared.done.notify_all(); + } + if shared.alive.fetch_sub(1, Ordering::SeqCst) == 1 { + let mut state = shared.lock(); + state.queue.clear(); + state.inflight.clear(); + } + shared.done.notify_all(); +} + +/// The oldest queued job that is still worth rendering. `None` when the fetcher stops. +fn next_job(shared: &Shared) -> Option { + let mut state = shared.lock(); + loop { + if state.stop { + return None; + } + while let Some(job) = state.queue.pop_front() { + if job.queued.elapsed() > shared.options.max_age && job.wanted_until <= Instant::now() { + state.inflight.remove(&job.key); + state.stats.skipped += 1; + shared.done.notify_all(); + continue; + } + return Some(job); + } + state = shared.work.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } +} diff --git a/crates/ipc/src/fsopen.rs b/crates/ipc/src/fsopen.rs new file mode 100644 index 000000000..ec1316f68 --- /dev/null +++ b/crates/ipc/src/fsopen.rs @@ -0,0 +1,25 @@ +//! Read-only opens that no file type at the path can stall. +//! +//! `File::open` on a FIFO with no writer blocks until a writer appears. A path that was checked +//! and then replaced gives the same wait, so a type check before the open does not help. These +//! opens ask the kernel not to wait, and the caller checks the type on the open handle. + +use std::fs::File; +use std::path::Path; + +/// Open `path` for reading without waiting on a FIFO. On Unix `follow` false also refuses a +/// symlink in the last component (the error is not specific, so a caller that must tell a symlink +/// apart looks at `symlink_metadata` after a failed open). Other platforms open as `File::open`. +pub(crate) fn open_read(path: &Path, follow: bool) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + let flags = if follow { libc::O_NONBLOCK } else { libc::O_NONBLOCK | libc::O_NOFOLLOW }; + std::fs::OpenOptions::new().read(true).custom_flags(flags).open(path) + } + #[cfg(not(unix))] + { + let _ = follow; + File::open(path) + } +} diff --git a/crates/ipc/src/ids.rs b/crates/ipc/src/ids.rs new file mode 100644 index 000000000..c2ed1abd4 --- /dev/null +++ b/crates/ipc/src/ids.rs @@ -0,0 +1,41 @@ +//! Id checks shared by the instance record, the moniker and the link record. +//! `std` only, so the pure modules can use it. + +/// App id: `^[a-z][a-z0-9-]{1,31}$`. +pub fn valid_app_id(app: &str) -> bool { + let bytes = app.as_bytes(); + (2..=32).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_lowercase) + && bytes.iter().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') +} + +/// UUID text in 8-4-4-4-12 form. Either hex case. +pub fn valid_uuid(text: &str) -> bool { + let bytes = text.as_bytes(); + bytes.len() == 36 + && bytes.iter().enumerate().all(|(index, byte)| if matches!(index, 8 | 13 | 18 | 23) { *byte == b'-' } else { byte.is_ascii_hexdigit() }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn app_ids() { + for ok in ["ec", "filmcraft", "a-b-1", &"a".repeat(32)] { + assert!(valid_app_id(ok), "{ok}"); + } + for bad in ["", "a", "A1", "1a", "a_b", "Film", &"a".repeat(33)] { + assert!(!valid_app_id(bad), "{bad}"); + } + } + + #[test] + fn uuids() { + assert!(valid_uuid("550e8400-e29b-41d4-a716-446655440000")); + assert!(valid_uuid("550E8400-E29B-41D4-A716-446655440000")); + assert!(!valid_uuid("550e8400e29b41d4a716446655440000")); + assert!(!valid_uuid("550e8400-e29b-41d4-a716-44665544000g")); + assert!(!valid_uuid("")); + } +} diff --git a/crates/ipc/src/launch.rs b/crates/ipc/src/launch.rs new file mode 100644 index 000000000..73365d30f --- /dev/null +++ b/crates/ipc/src/launch.rs @@ -0,0 +1,777 @@ +//! Find and start a Craft app. +//! +//! `resolve` reads the process environment. The decision itself is +//! `resolve_with`, which takes an owned env map, a home directory, and the +//! candidate directories. Tests call that function and do not change the +//! process environment. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use crate::client::{Client, HelloClient}; +use crate::error::LinkError; +use crate::trust; +use crate::{Discovery, InstanceRecord, RuntimeDir}; + +const RECORD_WAIT: Duration = Duration::from_secs(15); +const RECORD_POLL: Duration = Duration::from_millis(50); + +/// Progress reported by [`launch_at`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + Spawned, + RecordSeen, + HelloOk, +} + +/// Find the binary for `app` from the process environment. +pub fn resolve(app: &str) -> Result { + let env = std::env::vars().collect::>(); + let home = std::env::var_os("HOME").map(PathBuf::from); + let exe_dir = std::env::current_exe().ok().and_then(|path| path.parent().map(Path::to_path_buf)); + let search = default_search_dirs(home.as_deref()); + resolve_with(app, &env, home.as_deref(), exe_dir.as_deref(), &search) +} + +/// [`resolve`] then [`launch_at`] with no args and the real runtime directory. +pub fn launch(app: &str, extra_env: &[(&str, &str)]) -> Result { + let bin = resolve(app)?; + launch_at(app, &bin, &[], extra_env, None, &mut |_| {}) +} + +/// Spawn `bin` and wait until its discovery record answers `link.hello`. +/// +/// `extra_env` is applied with [`Command::env`]. This function does not mint +/// an activation token. `XDG_ACTIVATION_TOKEN` and `DESKTOP_STARTUP_ID` are +/// passed through when the caller puts them in `extra_env`. +/// +/// `runtime` `None` scans [`RuntimeDir::resolve`]. The returned record is one +/// this function observed for the spawned pid. +pub fn launch_at( + app: &str, + bin: &Path, + args: &[OsString], + extra_env: &[(&str, &str)], + runtime: Option<&RuntimeDir>, + on_phase: &mut dyn FnMut(Phase), +) -> Result { + // `resolve` already checked `bin`. The caller may pass any path, and time has passed. + trust::check_executable(bin)?; + let mut cmd = Command::new(bin); + cmd.args(args); + for (key, value) in extra_env { + cmd.env(key, value); + } + // A new process group. The child is not in this process's group. + // The waiter thread reaps it. kill_on_drop stays off. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let mut child = cmd.spawn().map_err(LinkError::io)?; + let pid = child.id(); + std::thread::Builder::new() + .name("craft-ipc-wait".into()) + .spawn(move || { + let _ = child.wait(); + }) + .map_err(LinkError::io)?; + on_phase(Phase::Spawned); + if let Some(dir) = runtime { + finish_launch(app, pid, dir, on_phase) + } else { + let owned = RuntimeDir::resolve()?; + finish_launch(app, pid, &owned, on_phase) + } +} + +fn finish_launch(app: &str, pid: u32, dir: &RuntimeDir, on_phase: &mut dyn FnMut(Phase)) -> Result { + let record = wait_for_record(dir, app, pid)?; + on_phase(Phase::RecordSeen); + greet(&record, app)?; + on_phase(Phase::HelloOk); + Ok(record) +} + +fn wait_for_record(dir: &RuntimeDir, app: &str, pid: u32) -> Result { + let deadline = Instant::now() + RECORD_WAIT; + loop { + let records = Discovery::scan(dir)?; + if let Some(record) = records.into_iter().find(|record| record.app == app && record.pid == pid) { + return Ok(record); + } + let now = Instant::now(); + if now >= deadline { + return Err(LinkError::Timeout); + } + std::thread::sleep(RECORD_POLL.min(deadline.saturating_duration_since(now))); + } +} + +fn greet(record: &InstanceRecord, app: &str) -> Result<(), LinkError> { + let mut client = Client::connect(record).map_err(as_not_ready)?; + let identity = HelloClient { + app: app.to_string(), + app_version: "launch".into(), + pid: std::process::id(), + instance: crate::random_uuid().map_err(as_not_ready)?, + protocol_min: crate::wire::PROTOCOL_MIN, + protocol_max: crate::wire::PROTOCOL_MAX, + }; + client.hello(&identity).map_err(as_not_ready)?; + Ok(()) +} + +fn as_not_ready(err: LinkError) -> LinkError { + match err { + LinkError::Timeout => LinkError::Timeout, + LinkError::NotReady(_) => err, + other => LinkError::NotReady(other.to_string()), + } +} + +fn resolve_with( + app: &str, + env: &BTreeMap, + home: Option<&Path>, + exe_dir: Option<&Path>, + search_dirs: &[PathBuf], +) -> Result { + if let Some(found) = env_override(app, env) { + return found; + } + if let Some(manifest) = manifest_path(env, home, exe_dir) + && let Some(bin) = from_manifest(app, &manifest) + { + return Ok(bin); + } + if let Some(bin) = from_path_env(app, env) { + return Ok(bin); + } + for dir in search_dirs { + if let Some(bin) = search_dir(app, dir) { + return Ok(bin); + } + } + if let Some(bin) = platform_bundle(app, home) { + return Ok(bin); + } + Err(LinkError::NotInstalled { app: app.to_string() }) +} + +/// `CRAFT__BIN`. Set means authoritative, including a missing file. +fn env_override(app: &str, env: &BTreeMap) -> Option> { + let value = env.get(&bin_env_key(app))?; + let path = PathBuf::from(value); + if !path.is_file() { + return Some(Err(LinkError::NotInstalled { app: app.to_string() })); + } + Some(trust::check_executable(&path).map(|()| path)) +} + +fn bin_env_key(app: &str) -> String { + let mut key = String::from("CRAFT_"); + for byte in app.bytes() { + if byte.is_ascii_alphanumeric() { + key.push(byte.to_ascii_uppercase() as char); + } else { + key.push('_'); + } + } + key.push_str("_BIN"); + key +} + +fn manifest_path(env: &BTreeMap, home: Option<&Path>, exe_dir: Option<&Path>) -> Option { + if let Some(dir) = exe_dir + && dir.join("portable.txt").is_file() + { + return Some(dir.join("ArtCraftLauncherData").join("data").join("installed.json")); + } + Some(data_dir(env, home)?.join(launcher_dir_name()).join("installed.json")) +} + +fn launcher_dir_name() -> &'static str { + if cfg!(any(target_os = "macos", windows)) { "ArtCraft Launcher" } else { "artcraft-launcher" } +} + +fn data_dir(env: &BTreeMap, home: Option<&Path>) -> Option { + #[cfg(target_os = "linux")] + { + linux_data_dir(env, home) + } + #[cfg(target_os = "macos")] + { + let _ = env; + home.map(|dir| dir.join("Library").join("Application Support")) + } + #[cfg(windows)] + { + let _ = home; + env.get("LOCALAPPDATA").map(PathBuf::from).filter(|path| path.is_absolute()) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + let _ = (env, home); + None + } +} + +#[cfg(target_os = "linux")] +fn linux_data_dir(env: &BTreeMap, home: Option<&Path>) -> Option { + if let Some(xdg) = env.get("XDG_DATA_HOME") { + let path = PathBuf::from(xdg); + if path.is_absolute() { + return Some(path); + } + } + home.map(|dir| dir.join(".local").join("share")) +} + +/// Only `path` is required. An unknown `kind` is ignored. A path that is not a file is skipped. +fn from_manifest(app: &str, path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + let text = value.get("apps")?.get(app)?.get("path")?.as_str()?; + let bin = PathBuf::from(text); + trust::check_executable(&bin).is_ok().then_some(bin) +} + +/// A `PATH` entry that is empty or relative names the current directory, so it is skipped. +fn from_path_env(app: &str, env: &BTreeMap) -> Option { + let path = env.get("PATH")?; + for dir in std::env::split_paths(path) { + if !dir.is_absolute() { + continue; + } + let candidate = dir.join(app); + if trust::check_executable(&candidate).is_ok() { + return Some(candidate); + } + } + None +} + +fn default_search_dirs(home: Option<&Path>) -> Vec { + #[cfg(target_os = "linux")] + { + let mut dirs = Vec::new(); + if let Some(home) = home { + dirs.push(home.join("Applications")); + dirs.push(home.join("AppImages")); + dirs.push(home.join(".local").join("bin")); + } + dirs.push(PathBuf::from("/usr/bin")); + dirs.push(PathBuf::from("/usr/local/bin")); + dirs + } + #[cfg(not(target_os = "linux"))] + { + let _ = home; + Vec::new() + } +} + +fn search_dir(app: &str, dir: &Path) -> Option { + let exact = dir.join(app); + if trust::check_executable(&exact).is_ok() { + return Some(exact); + } + let entries = std::fs::read_dir(dir).ok()?; + let mut names = Vec::new(); + for entry in entries { + let Ok(entry) = entry else { continue }; + let Ok(name) = entry.file_name().into_string() else { continue }; + if is_appimage(app, &name) { + names.push(name); + } + } + names.sort(); + for name in names { + let path = dir.join(name); + if trust::check_executable(&path).is_ok() { + return Some(path); + } + } + None +} + +/// `{app}.appimage` in any ASCII case, or a name that starts with `{app}-` and ends with `.AppImage`. +fn is_appimage(app: &str, name: &str) -> bool { + const EXT: &str = ".appimage"; + let Some(split) = name.len().checked_sub(EXT.len()) else { + return false; + }; + let Some(ext) = name.get(split..) else { + return false; + }; + let Some(base) = name.get(..split) else { + return false; + }; + if !ext.eq_ignore_ascii_case(EXT) { + return false; + } + if base.eq_ignore_ascii_case(app) { + return true; + } + let mut prefix = String::with_capacity(app.len() + 1); + prefix.push_str(app); + prefix.push('-'); + name.starts_with(&prefix) +} + +/// An id that ends with `craft` becomes `PrefixCraft` (`filmcraft` to `FilmCraft`). +/// Used by the macOS bundle search. Linux resolves AppImages without this name. +#[cfg_attr(not(any(test, target_os = "macos")), allow(dead_code))] +fn display_name(app: &str) -> String { + if let Some(prefix) = app.strip_suffix("craft") { + let mut name = String::new(); + let mut chars = prefix.chars(); + if let Some(first) = chars.next() { + name.push(first.to_ascii_uppercase()); + name.push_str(chars.as_str()); + } + name.push_str("Craft"); + return name; + } + let mut chars = app.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + let mut name = String::new(); + name.push(first.to_ascii_uppercase()); + name.push_str(chars.as_str()); + name +} + +/// macOS bundles. Windows uninstall-registry lookup is not implemented (no winreg). +/// PATH, `CRAFT__BIN`, and `installed.json` still apply on Windows. +#[cfg(target_os = "macos")] +fn platform_bundle(app: &str, home: Option<&Path>) -> Option { + let bundle = format!("{}.app", display_name(app)); + let mut candidates = vec![PathBuf::from("/Applications").join(&bundle)]; + if let Some(home) = home { + candidates.push(home.join("Applications").join(&bundle)); + } + for bundle_path in candidates { + if !bundle_path.is_dir() { + continue; + } + let exe = bundle_path.join("Contents").join("MacOS").join(app); + if trust::check_executable(&exe).is_ok() { + return Some(exe); + } + } + None +} + +#[cfg(not(target_os = "macos"))] +fn platform_bundle(_app: &str, _home: Option<&Path>) -> Option { + None +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::collections::BTreeMap; + use std::path::{Path, PathBuf}; + + use super::*; + + struct Tmp { + path: PathBuf, + } + + impl Tmp { + fn new(label: &str) -> Self { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } + } + + fn touch(path: &Path) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, b"bin").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + + /// Mode of the directory that holds `path`, for the trust tests. + #[cfg(unix)] + fn chmod(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + #[test] + fn relative_and_empty_path_entries_are_skipped() { + let root = Tmp::new("relpath"); + let cwd = std::env::current_dir().unwrap(); + // A program in the current directory must not be found through "" or ".". + let here = cwd.join(format!("effectcraft-relpath-{}", std::process::id())); + touch(&here); + let name = here.file_name().unwrap().to_str().unwrap().to_string(); + let good_dir = root.path().join("bin"); + for path in ["", ".", "./", "bin:", ":bin"] { + let env = env_map(&[("PATH", path)]); + let err = super::resolve_with(&name, &env, None, None, &[]); + assert!(err.is_err(), "PATH={path:?} found {err:?}"); + } + // An absolute entry later in the list still works. + touch(&good_dir.join(&name)); + let path = format!(":.:{}", good_dir.display()); + let env = env_map(&[("PATH", path.as_str())]); + assert_eq!(super::resolve_with(&name, &env, None, None, &[]).unwrap(), good_dir.join(&name)); + std::fs::remove_file(&here).unwrap(); + } + + #[cfg(unix)] + #[test] + fn loose_programs_and_directories_are_not_run() { + let root = Tmp::new("loose"); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let env = env_map(&[("PATH", bin_dir.to_str().unwrap())]); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_ok()); + chmod(&bin_dir, 0o777); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "world-writable directory"); + // The same holds for the override, the manifest and the fixed search directories. + let by_env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", bin.to_str().unwrap())]); + assert!(matches!(super::resolve_with("effectcraft", &by_env, None, None, &[]), Err(LinkError::Refused { .. }))); + assert!(super::resolve_with("effectcraft", &env_map(&[]), None, None, std::slice::from_ref(&bin_dir)).is_err()); + let data = root.path().join("xdg"); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&data)), "effectcraft", &bin); + let by_manifest = data_env(&data); + chmod(&bin_dir, 0o755); + assert_eq!(super::resolve_with("effectcraft", &by_manifest, Some(&home), None, &[]).unwrap(), bin); + chmod(&bin_dir, 0o777); + assert!(super::resolve_with("effectcraft", &by_manifest, Some(&home), None, &[]).is_err()); + chmod(&bin_dir, 0o755); + chmod(&bin, 0o666); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "not executable and world-writable"); + chmod(&bin, 0o644); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "not executable"); + // A relative override is refused, not run from the current directory. + let relative = env_map(&[("CRAFT_EFFECTCRAFT_BIN", "Cargo.toml")]); + assert!(super::resolve_with("effectcraft", &relative, None, None, &[]).is_err()); + // launch_at checks the path too. + let err = launch_at("effectcraft", &bin, &[], &[], None, &mut |_| {}).unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err:?}"); + } + + fn env_map(pairs: &[(&str, &str)]) -> BTreeMap { + pairs.iter().map(|(key, value)| ((*key).to_string(), (*value).to_string())).collect() + } + + // Independent fixture paths for the launcher's documented native install record. + fn installed_manifest(home: &Path, xdg: Option<&Path>) -> PathBuf { + if cfg!(target_os = "macos") { + home.join("Library/Application Support/ArtCraft Launcher/installed.json") + } else if cfg!(windows) { + xdg.unwrap_or(home).join("ArtCraft Launcher/installed.json") + } else { + xdg.map_or_else(|| home.join(".local/share"), Path::to_path_buf).join("artcraft-launcher/installed.json") + } + } + + fn data_env(data: &Path) -> BTreeMap { + let key = if cfg!(windows) { "LOCALAPPDATA" } else { "XDG_DATA_HOME" }; + env_map(&[(key, data.to_str().unwrap())]) + } + + fn write_manifest(path: &Path, app: &str, bin: &Path) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + let body = serde_json::json!({ + "apps": { + app: {"kind": "mystery", "path": bin} + } + }); + std::fs::write(path, serde_json::to_vec(&body).unwrap()).unwrap(); + } + + #[test] + fn bin_env_key_uppercases_and_replaces_other_bytes() { + assert_eq!(super::bin_env_key("effectcraft"), "CRAFT_EFFECTCRAFT_BIN"); + assert_eq!(super::bin_env_key("film-craft"), "CRAFT_FILM_CRAFT_BIN"); + } + + #[test] + fn display_name_for_craft_apps() { + assert_eq!(super::display_name("filmcraft"), "FilmCraft"); + assert_eq!(super::display_name("effectcraft"), "EffectCraft"); + assert_eq!(super::display_name("pdfcraft"), "PdfCraft"); + } + + #[test] + fn appimage_names() { + assert!(super::is_appimage("effectcraft", "effectcraft.AppImage")); + assert!(super::is_appimage("effectcraft", "EffectCraft.appimage")); + assert!(super::is_appimage("effectcraft", "effectcraft-0.8.3-linux-x86_64.AppImage")); + assert!(!super::is_appimage("effectcraft", "effectcraft-0.8.3.deb")); + assert!(!super::is_appimage("effectcraft", "not-effectcraft.AppImage")); + assert!(!super::is_appimage("effectcraft", "effectcraft.AppImage.zsync")); + } + + #[test] + fn env_override_missing_file_does_not_fall_through() { + let root = Tmp::new("miss"); + let missing = root.path().join("nope"); + let real_dir = root.path().join("bin"); + let real = real_dir.join("effectcraft"); + touch(&real); + let env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", missing.to_str().unwrap()), ("PATH", real_dir.to_str().unwrap())]); + let err = super::resolve_with("effectcraft", &env, None, None, &[real_dir]).unwrap_err(); + match err { + LinkError::NotInstalled { app } => assert_eq!(app, "effectcraft"), + other => panic!("expected NotInstalled, got {other}"), + } + } + + #[test] + fn env_override_existing_file_is_that_path() { + let root = Tmp::new("envok"); + let bin = root.path().join("custom"); + touch(&bin); + let env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", bin.to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn installed_json_supplies_the_path() { + let root = Tmp::new("man"); + let data = root.path().join("xdg"); + let bin = root.path().join("from-manifest"); + touch(&bin); + let home = root.path().join("home"); + let manifest = installed_manifest(&home, Some(&data)); + write_manifest(&manifest, "effectcraft", &bin); + let env = data_env(&data); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[cfg(target_os = "linux")] + #[test] + fn relative_xdg_data_home_uses_home() { + let root = Tmp::new("rel"); + let home = root.path().join("home"); + let bin = root.path().join("home-bin"); + touch(&bin); + let manifest = installed_manifest(&home, None); + write_manifest(&manifest, "effectcraft", &bin); + let env = env_map(&[("XDG_DATA_HOME", "relative/data")]); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[cfg(windows)] + #[test] + fn relative_localappdata_is_not_used() { + let env = env_map(&[("LOCALAPPDATA", "relative/data")]); + assert!(matches!(super::resolve_with("effectcraft", &env, None, None, &[]), Err(LinkError::NotInstalled { .. }))); + } + + #[test] + fn portable_manifest_wins_over_native_data_directory() { + let root = Tmp::new("port"); + let exe = root.path().join("exe-dir"); + std::fs::create_dir_all(&exe).unwrap(); + std::fs::write(exe.join("portable.txt"), b"1").unwrap(); + let bin = root.path().join("portable-bin"); + touch(&bin); + write_manifest(&exe.join("ArtCraftLauncherData").join("data").join("installed.json"), "effectcraft", &bin); + let xdg = root.path().join("xdg"); + let decoy = root.path().join("decoy-bin"); + touch(&decoy); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&xdg)), "effectcraft", &decoy); + let env = data_env(&xdg); + let got = super::resolve_with("effectcraft", &env, Some(&home), Some(&exe), &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn manifest_directory_is_skipped_for_path() { + let root = Tmp::new("skip"); + let data = root.path().join("xdg"); + let folder = root.path().join("not-a-file"); + std::fs::create_dir_all(&folder).unwrap(); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&data)), "effectcraft", &folder); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let mut env = data_env(&data); + env.insert("PATH".into(), bin_dir.to_str().unwrap().into()); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn path_supplies_the_path() { + let root = Tmp::new("path"); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let env = env_map(&[("PATH", bin_dir.to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn linux_location_supplies_the_path() { + let root = Tmp::new("fs"); + let apps = root.path().join("Applications"); + let image = apps.join("effectcraft-1.0.0.AppImage"); + touch(&image); + std::fs::write(apps.join("not-effectcraft.AppImage"), b"no").unwrap(); + let later = root.path().join("usr-bin"); + let exact = later.join("effectcraft"); + touch(&exact); + let env = env_map(&[("PATH", root.path().join("empty").to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[apps.clone(), later]).unwrap(); + assert_eq!(got, image); + } + + #[cfg(unix)] + #[test] + fn launch_at_reaches_hello() { + // The child branch returns at once when CRAFT_LAUNCH_DIR is unset. + // A normal cargo test takes the parent path and does not park. + if let Ok(dir) = std::env::var("CRAFT_LAUNCH_DIR") { + child_launch(&dir); + } + parent_launch(); + } + + #[cfg(unix)] + fn child_launch(dir: &str) -> ! { + use std::io::BufReader; + use std::os::unix::net::UnixListener; + + let runtime = RuntimeDir::at(dir).unwrap(); + let app = "effectcraft"; + let pid = std::process::id(); + std::fs::write(runtime.path().join("child.pid"), pid.to_string()).unwrap(); + let sock = runtime.socket_file(app, pid); + let _ = std::fs::remove_file(&sock); + let listener = UnixListener::bind(&sock).unwrap(); + crate::discovery::enforce_mode(&sock, 0o600).unwrap(); + let endpoint = runtime.unix_endpoint(app, pid).unwrap(); + let record = InstanceRecord::new(app, "0.8.3", pid, endpoint).unwrap(); + let published = Discovery::publish(&runtime, record.clone()).unwrap(); + std::fs::write(runtime.path().join("ready"), pid.to_string()).unwrap(); + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(20))).unwrap(); + let mut reader = BufReader::new(stream); + let auth = crate::wire::read_line(&mut reader, crate::wire::MAX_LINE).unwrap().unwrap(); + let auth_value: Value = serde_json::from_slice(&auth).unwrap(); + assert_eq!(auth_value["method"], "auth"); + assert_eq!(auth_value["params"]["token"], record.token); + crate::wire::write_line(reader.get_mut(), &crate::wire::auth_success(&serde_json::json!(0))).unwrap(); + let hello = crate::wire::read_line(&mut reader, crate::wire::MAX_LINE).unwrap().unwrap(); + let hello_value: Value = serde_json::from_slice(&hello).unwrap(); + assert_eq!(hello_value["method"], "link.hello"); + let id = hello_value.get("id").cloned().unwrap_or(serde_json::json!(1)); + let result = serde_json::json!({ + "app": app, + "app_version": record.app_version, + "pid": record.pid, + "instance": record.instance, + "protocol": 1, + }); + crate::wire::write_line(reader.get_mut(), &crate::wire::reply_ok(&id, result)).unwrap(); + let _keep = (published, listener, reader); + // park can wake without an unpark. Loop so Drop does not run until SIGKILL. + loop { + std::thread::park(); + } + } + + #[cfg(unix)] + fn parent_launch() { + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-launch-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let _cleanup = LaunchCleanup { dir: path.clone() }; + let exe = std::env::current_exe().unwrap(); + let args = vec![OsString::from("--exact"), OsString::from("launch::tests::launch_at_reaches_hello"), OsString::from("--test-threads=1")]; + let dir_text = path.to_str().unwrap(); + let mut phases = Vec::new(); + // The test binary can sit on a file system that marks every file 0777, and `launch_at` + // refuses such a program. A private launcher script that execs it keeps the pid. + let script = path.join("launcher"); + std::fs::write(&script, format!("#!/bin/sh\nexec '{}' \"$@\"\n", exe.display())).unwrap(); + chmod(&script, 0o755); + let result = launch_at("effectcraft", &script, &args, &[("CRAFT_LAUNCH_DIR", dir_text)], Some(&runtime), &mut |phase| phases.push(phase)); + let record = result.unwrap_or_else(|err| panic!("launch_at failed: {err}; phases={phases:?}")); + assert_eq!(phases, vec![Phase::Spawned, Phase::RecordSeen, Phase::HelloOk]); + let ready = std::fs::read_to_string(path.join("ready")).unwrap(); + let child_pid: u32 = ready.trim().parse().unwrap(); + assert_eq!(record.pid, child_pid); + assert_eq!(record.app, "effectcraft"); + } + + #[cfg(unix)] + struct LaunchCleanup { + dir: PathBuf, + } + + #[cfg(unix)] + impl Drop for LaunchCleanup { + fn drop(&mut self) { + kill_recorded(&self.dir); + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + #[cfg(unix)] + fn kill_recorded(dir: &Path) { + for name in ["child.pid", "ready"] { + let Ok(text) = std::fs::read_to_string(dir.join(name)) else { continue }; + let Ok(pid) = text.trim().parse::() else { continue }; + let _ = Command::new("kill").args(["-KILL", &pid.to_string()]).status(); + let proc = PathBuf::from(format!("/proc/{pid}")); + let deadline = Instant::now() + Duration::from_secs(2); + while proc.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + return; + } + } +} diff --git a/crates/ipc/src/lib.rs b/crates/ipc/src/lib.rs new file mode 100644 index 000000000..53b071fa6 --- /dev/null +++ b/crates/ipc/src/lib.rs @@ -0,0 +1,39 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +//! Craft Link transport. +//! +//! Local sockets, discovery files, and the JSON-lines session. The server and +//! the client are separate modules. There is no tokio runtime in this crate. + +pub mod activate; +pub mod asset; +pub mod auth; +pub mod bulk; +pub mod cache; +pub mod client; +pub mod discovery; +pub mod error; +pub mod fetch; +mod fsopen; +pub mod ids; +pub mod launch; +pub mod moniker; +pub mod payload; +pub mod record; +pub mod refresh; +pub mod server; +pub mod trust; +pub mod watch; +pub mod wire; + +/// Unit tests that hold a discovery lock take `read`. Unit tests that start a +/// child process take `write`. A forked child owns a copy of every open file +/// until it calls exec, and a `flock` lock lives until the last copy closes. +/// Without this guard a lock released by `drop` can look held for a few +/// milliseconds, and `scan` or `publish` in another test then fails at random. +#[cfg(test)] +pub(crate) static SPAWN_GUARD: std::sync::RwLock<()> = std::sync::RwLock::new(()); + +pub use auth::{ensure_peer_user, generate_token, random_uuid, tokens_equal}; +pub use discovery::{Discovery, DocInfo, Endpoint, InstanceRecord, Protocol, RuntimeDir}; +pub use error::{ErrorObject, LinkError}; diff --git a/crates/ipc/src/moniker.rs b/crates/ipc/src/moniker.rs new file mode 100644 index 000000000..5bee3e77c --- /dev/null +++ b/crates/ipc/src/moniker.rs @@ -0,0 +1,313 @@ +//! Asset monikers: `craft://///`. +//! +//! The grammar is in `PROTOCOL.md` section 9. This module uses +//! only `std` and `serde`, so an app can copy it to a wasm build. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::error::LinkError; +use crate::ids::{valid_app_id, valid_uuid}; + +const SCHEME: &str = "craft://"; +/// Longest decoded `id`, in bytes. +pub const MAX_ID_BYTES: usize = 512; + +/// What an asset is. Unknown words are valid and kept in [`MonikerKind::Other`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum MonikerKind { + Comp, + Image, + Vector, + Sequence, + Layer, + Other(String), +} + +impl MonikerKind { + pub fn as_str(&self) -> &str { + match self { + Self::Comp => "comp", + Self::Image => "image", + Self::Vector => "vector", + Self::Sequence => "sequence", + Self::Layer => "layer", + Self::Other(word) => word, + } + } + + fn parse(word: &str) -> Result { + if !valid_word(word, 32) { + return Err(bad(format!("kind {word:?} must match [a-z][a-z0-9-]{{0,31}}"))); + } + Ok(match word { + "comp" => Self::Comp, + "image" => Self::Image, + "vector" => Self::Vector, + "sequence" => Self::Sequence, + "layer" => Self::Layer, + other => Self::Other(other.to_string()), + }) + } +} + +/// A parsed moniker. `doc` keeps the hex case it was written with. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Moniker { + app: String, + doc: String, + kind: MonikerKind, + id: String, +} + +impl Moniker { + /// Build a moniker from parts. Every part is checked, so a value of this + /// type always formats to text that parses back to an equal value. + pub fn new(app: &str, doc: &str, kind: MonikerKind, id: &str) -> Result { + if !valid_app_id(app) { + return Err(bad(format!("app {app:?} must match [a-z][a-z0-9-]{{1,31}}"))); + } + if !valid_uuid(doc) { + return Err(bad(format!("doc {doc:?} is not a uuid"))); + } + if let MonikerKind::Other(word) = &kind + && (!valid_word(word, 32) || is_known_word(word)) + { + return Err(bad(format!("kind {word:?} is not a valid unknown kind"))); + } + if id.is_empty() || id.len() > MAX_ID_BYTES { + return Err(bad(format!("id must be 1 to {MAX_ID_BYTES} bytes"))); + } + Ok(Self { app: app.to_string(), doc: doc.to_string(), kind, id: id.to_string() }) + } + + pub fn parse(text: &str) -> Result { + let Some(rest) = text.strip_prefix(SCHEME) else { + return Err(bad(format!("moniker must start with {SCHEME}"))); + }; + let mut parts = rest.split('/'); + let (Some(app), Some(doc), Some(kind), Some(id), None) = (parts.next(), parts.next(), parts.next(), parts.next(), parts.next()) else { + return Err(bad("moniker must have exactly app, doc, kind and id".to_string())); + }; + if id.contains(['?', '#']) { + return Err(bad("moniker has no query or fragment".to_string())); + } + let kind = MonikerKind::parse(kind)?; + let id = decode_id(id)?; + Self::new(app, doc, kind, &id) + } + + pub fn app(&self) -> &str { + &self.app + } + + pub fn doc(&self) -> &str { + &self.doc + } + + pub fn kind(&self) -> &MonikerKind { + &self.kind + } + + /// The decoded id. + pub fn id(&self) -> &str { + &self.id + } +} + +impl fmt::Display for Moniker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{SCHEME}{}/{}/{}/", self.app, self.doc, self.kind.as_str())?; + for byte in self.id.bytes() { + if is_unreserved(byte) { + write!(f, "{}", char::from(byte))?; + } else { + write!(f, "%{byte:02X}")?; + } + } + Ok(()) + } +} + +impl FromStr for Moniker { + type Err = LinkError; + + fn from_str(text: &str) -> Result { + Self::parse(text) + } +} + +impl Serialize for Moniker { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Moniker { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + Self::parse(&text).map_err(serde::de::Error::custom) + } +} + +fn bad(message: String) -> LinkError { + LinkError::BadParams(message) +} + +fn is_known_word(word: &str) -> bool { + matches!(word, "comp" | "image" | "vector" | "sequence" | "layer") +} + +fn is_unreserved(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') +} + +fn valid_word(word: &str, max: usize) -> bool { + let mut bytes = word.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + word.len() <= max && first.is_ascii_lowercase() && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +fn decode_id(text: &str) -> Result { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + let pair = bytes.get(index + 1..index + 3).ok_or_else(|| bad("id has a truncated %XX escape".to_string()))?; + let hi = hex_value(pair.first().copied()); + let lo = hex_value(pair.get(1).copied()); + let (Some(hi), Some(lo)) = (hi, lo) else { + return Err(bad("id has a bad %XX escape".to_string())); + }; + out.push(hi * 16 + lo); + index += 3; + } else if is_unreserved(byte) { + out.push(byte); + index += 1; + } else { + return Err(bad(format!("id has the raw byte 0x{byte:02X}, which must be percent-encoded"))); + } + } + String::from_utf8(out).map_err(|_| bad("id is not valid UTF-8".to_string())) +} + +fn hex_value(byte: Option) -> Option { + match byte? { + b @ b'0'..=b'9' => Some(b - b'0'), + b @ b'a'..=b'f' => Some(b - b'a' + 10), + b @ b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + const DOC: &str = "550e8400-e29b-41d4-a716-446655440000"; + + fn sample(id: &str) -> Moniker { + Moniker::new("photocraft", DOC, MonikerKind::Image, id).unwrap() + } + + #[test] + fn plain_id_formats_and_parses() { + let text = format!("craft://photocraft/{DOC}/image/layer-3"); + assert_eq!(sample("layer-3").to_string(), text); + assert_eq!(Moniker::parse(&text).unwrap(), sample("layer-3")); + } + + #[test] + fn round_trip_keeps_awkward_ids() { + for id in ["a b", "a/b", "100%", "x?y#z", "caf\u{e9}", "\u{1F600}", "~._-", "%2F", "a%20b", &"x".repeat(MAX_ID_BYTES)] { + let m = sample(id); + let text = m.to_string(); + let back = Moniker::parse(&text).unwrap(); + assert_eq!(back, m, "{id:?} via {text}"); + assert_eq!(back.id(), id); + assert_eq!(back.to_string(), text); + } + } + + #[test] + fn encoded_slash_stays_inside_the_id() { + let m = sample("a/b"); + assert_eq!(m.to_string(), format!("craft://photocraft/{DOC}/image/a%2Fb")); + assert_eq!(Moniker::parse(&m.to_string()).unwrap().id(), "a/b"); + } + + #[test] + fn lowercase_escapes_parse_and_format_uppercase() { + let m = Moniker::parse(&format!("craft://photocraft/{DOC}/image/a%2fb")).unwrap(); + assert_eq!(m.id(), "a/b"); + assert_eq!(m.to_string(), format!("craft://photocraft/{DOC}/image/a%2Fb")); + } + + #[test] + fn doc_case_is_kept() { + let upper = DOC.to_uppercase(); + let m = Moniker::parse(&format!("craft://photocraft/{upper}/comp/c1")).unwrap(); + assert_eq!(m.doc(), upper); + assert_eq!(m.to_string(), format!("craft://photocraft/{upper}/comp/c1")); + } + + #[test] + fn kinds_known_and_unknown() { + for (word, kind) in [ + ("comp", MonikerKind::Comp), + ("image", MonikerKind::Image), + ("vector", MonikerKind::Vector), + ("sequence", MonikerKind::Sequence), + ("layer", MonikerKind::Layer), + ("palette", MonikerKind::Other("palette".into())), + ] { + let m = Moniker::parse(&format!("craft://designcraft/{DOC}/{word}/x")).unwrap(); + assert_eq!(m.kind(), &kind); + assert_eq!(m.to_string(), format!("craft://designcraft/{DOC}/{word}/x")); + } + assert!(Moniker::new("designcraft", DOC, MonikerKind::Other("comp".into()), "x").is_err(), "known word must use its variant"); + } + + #[test] + fn rejects_bad_text() { + let bad_texts = [ + "".to_string(), + "craft:/photocraft".to_string(), + "http://photocraft/x/image/a".to_string(), + format!("craft://photocraft/{DOC}/image"), + format!("craft://photocraft/{DOC}/image/a/b"), + format!("craft://photocraft/{DOC}/image/"), + format!("craft://photocraft/{DOC}/image/a?x=1"), + format!("craft://photocraft/{DOC}/image/a#frag"), + format!("craft://photocraft/{DOC}/image/a b"), + format!("craft://photocraft/{DOC}/image/%"), + format!("craft://photocraft/{DOC}/image/%4"), + format!("craft://photocraft/{DOC}/image/%GG"), + format!("craft://photocraft/{DOC}/image/%FF"), + format!("craft://Photo/{DOC}/image/a"), + format!("craft://p/{DOC}/image/a"), + "craft://photocraft/not-a-uuid/image/a".to_string(), + format!("craft://photocraft/{DOC}/Image/a"), + format!("craft://photocraft/{DOC}/1image/a"), + format!("craft://photocraft/{DOC}/image/{}", "x".repeat(MAX_ID_BYTES + 1)), + ]; + for text in bad_texts { + assert!(Moniker::parse(&text).is_err(), "{text:?} should not parse"); + } + } + + #[test] + fn serde_uses_the_string_form() { + let m = sample("a b"); + let value = serde_json::to_value(&m).unwrap(); + assert_eq!(value, serde_json::json!(format!("craft://photocraft/{DOC}/image/a%20b"))); + assert_eq!(serde_json::from_value::(value).unwrap(), m); + assert!(serde_json::from_value::(serde_json::json!("craft://nope")).is_err()); + } +} diff --git a/crates/ipc/src/payload.rs b/crates/ipc/src/payload.rs new file mode 100644 index 000000000..ddbd7d2dd --- /dev/null +++ b/crates/ipc/src/payload.rs @@ -0,0 +1,396 @@ +//! The `payload` object of `link.open`. +//! +//! Formats are listed in `PROTOCOL.md` section 4. The server +//! calls [`OpenPayload::from_value`] on every `link.open` before it forwards the +//! request, and answers `payload_invalid` on an error. An app may call it again. +//! +//! This module uses only `std`, `serde_json`, [`crate::moniker`] and the size constants in +//! [`crate::wire`]. + +use serde_json::Value; + +use crate::error::LinkError; +use crate::moniker::Moniker; +use crate::wire::MAX_LINK_PAYLOAD; + +/// A parsed `link.open` payload. +#[derive(Debug, Clone, PartialEq)] +pub enum OpenPayload { + /// Video: an OTIO JSON string. + OtioJson { data: String }, + /// A still or vector: the producer document path and the asset moniker. + ImageDoc { path: String, asset: Moniker }, +} + +impl OpenPayload { + pub fn from_value(payload: &Value) -> Result { + let obj = payload.as_object().ok_or_else(|| LinkError::PayloadInvalid("payload must be an object".into()))?; + let format = obj.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::PayloadInvalid("payload.format must be a string".into()))?; + match format { + "otio-json" => { + let data = text_field(obj, "data")?; + if data.len() as u64 > MAX_LINK_PAYLOAD { + return Err(LinkError::TooLarge); + } + let missing = missing_media(&data)?; + if !missing.is_empty() { + return Err(LinkError::MediaMissing { paths: missing }); + } + Ok(Self::OtioJson { data }) + } + "image-doc" => { + let path = text_field(obj, "path")?; + check_peer_path(&path, "payload.path")?; + let asset = Moniker::parse(&text_field(obj, "asset")?).map_err(|err| LinkError::PayloadInvalid(format!("payload.asset: {err}")))?; + Ok(Self::ImageDoc { path, asset }) + } + other => Err(LinkError::PayloadInvalid(format!("payload.format {other:?} is not supported"))), + } + } +} + +/// The most distinct files one OTIO document may name. A timeline with more is not an editor's +/// timeline, and each one costs a file system call. +pub const MAX_MEDIA_REFS: usize = 4096; +/// The most JSON values one OTIO document may hold. The line cap (4 MiB) allows about two million. +pub const MAX_NODES: usize = 1 << 20; +/// Deepest nesting walked. `serde_json` stops parsing at 128, so a deeper document never gets here. +pub const MAX_DEPTH: usize = 128; + +/// Check a path that a peer names (`payload.path`, a media `target_url`). It must be absolute, hold +/// no NUL byte and no `..` component. The server does not canonicalise it: the app opens it, and a +/// symlink to a file elsewhere is a normal way to keep footage. This check only refuses a path +/// that is written to climb out of the directory it starts in. +pub fn check_peer_path(path: &str, what: &str) -> Result<(), LinkError> { + let parsed = std::path::Path::new(path); + if !parsed.is_absolute() { + return Err(LinkError::PayloadInvalid(format!("{what} must be absolute"))); + } + if path.contains('\0') { + return Err(LinkError::PayloadInvalid(format!("{what} must not hold a NUL byte"))); + } + if parsed.components().any(|part| matches!(part, std::path::Component::ParentDir)) { + return Err(LinkError::PayloadInvalid(format!("{what} must not hold a `..` component"))); + } + Ok(()) +} + +/// The files that `ExternalReference`s of an OTIO document name and that cannot be used. Each path +/// appears once, in document order. +/// +/// A file is usable when it is a regular file (after symlinks) and opens for reading. A missing +/// file, a directory, a FIFO, a device and a file without read access all give the same answer, so +/// the reply says nothing beyond "this path from your payload does not work". The probe never opens +/// anything that is not a regular file, so a FIFO cannot hold the connection thread. Only paths the +/// payload itself names are probed, and at most [`MAX_MEDIA_REFS`] of them. +/// +/// Empty when every file works, when `data` is not JSON (the app reports that), and for references +/// that are not an absolute path or a `file://` URL: they have no base here to resolve against. +/// `Err` when the document is too deep or too big, or names an absolute path with `..` or NUL. +pub fn missing_media(data: &str) -> Result, LinkError> { + let Ok(document) = serde_json::from_str::(data) else { return Ok(Vec::new()) }; + let mut walk = Walk { seen: Vec::new(), nodes: 0 }; + walk.collect(&document, 0)?; + walk.seen.retain(|path| !media_usable(path)); + Ok(walk.seen) +} + +fn media_usable(path: &str) -> bool { + let path = std::path::Path::new(path); + std::fs::metadata(path).is_ok_and(|meta| meta.is_file()) && std::fs::File::open(path).is_ok() +} + +struct Walk { + seen: Vec, + nodes: usize, +} + +impl Walk { + fn collect(&mut self, value: &Value, depth: usize) -> Result<(), LinkError> { + if depth > MAX_DEPTH { + return Err(LinkError::PayloadInvalid(format!("the OTIO document nests deeper than {MAX_DEPTH}"))); + } + self.nodes += 1; + if self.nodes > MAX_NODES { + return Err(LinkError::PayloadInvalid(format!("the OTIO document holds more than {MAX_NODES} values"))); + } + match value { + Value::Object(map) => { + let external = map.get("OTIO_SCHEMA").and_then(Value::as_str).is_some_and(|schema| schema.starts_with("ExternalReference")); + let local = match map.get("target_url").and_then(Value::as_str) { + Some(url) if external => local_path(url)?, + _ => None, + }; + if let Some(path) = local { + check_peer_path(&path, "target_url")?; + if !self.seen.contains(&path) { + if self.seen.len() >= MAX_MEDIA_REFS { + return Err(LinkError::PayloadInvalid(format!("the OTIO document names more than {MAX_MEDIA_REFS} media files"))); + } + self.seen.push(path); + } + } + for child in map.values() { + self.collect(child, depth + 1)?; + } + } + Value::Array(items) => { + for child in items { + self.collect(child, depth + 1)?; + } + } + _ => {} + } + Ok(()) + } +} + +/// The local path a `target_url` names: a `file://` URL (percent-decoded, host empty or +/// `localhost`) or an absolute path. `Ok(None)` for anything else. `Err` for a `file://` URL +/// whose path does not percent-decode to UTF-8: it names a local file, so it cannot be skipped. +fn local_path(url: &str) -> Result, LinkError> { + let path = match url.strip_prefix("file://") { + Some(rest) => { + let rest = rest.strip_prefix("localhost").unwrap_or(rest); + if !rest.starts_with('/') { + return Ok(None); + } + percent_decode(rest).ok_or_else(|| LinkError::PayloadInvalid("target_url is a file:// URL with a bad percent escape".into()))? + } + None => url.to_string(), + }; + // `/C:/dir/a.mov` is how a Windows path sits in a URL. + let path = match path.as_bytes() { + [b'/', drive, b':', ..] if drive.is_ascii_alphabetic() && cfg!(windows) => path[1..].to_string(), + _ => path, + }; + Ok(std::path::Path::new(&path).is_absolute().then_some(path)) +} + +fn percent_decode(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + let pair = bytes.get(index + 1..index + 3)?; + // `from_str_radix` also takes a leading `+`, so the two digits are checked first. + if !pair.iter().all(u8::is_ascii_hexdigit) { + return None; + } + out.push(u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok()?); + index += 3; + } else { + out.push(byte); + index += 1; + } + } + String::from_utf8(out).ok() +} + +fn text_field(obj: &serde_json::Map, key: &str) -> Result { + match obj.get(key) { + Some(Value::String(text)) if !text.is_empty() => Ok(text.clone()), + _ => Err(LinkError::PayloadInvalid(format!("payload.{key} must be a non-empty string"))), + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use serde_json::json; + + use super::*; + + const ASSET: &str = "craft://photocraft/550e8400-e29b-41d4-a716-446655440000/image/flat"; + + fn absolute(path: &str) -> String { + if cfg!(windows) { format!("C:{path}") } else { path.to_owned() } + } + + fn file_url(path: &str) -> String { + let path = path.replace('\\', "/").replace(' ', "%20"); + format!("file://{}{path}", if path.starts_with('/') { "" } else { "/" }) + } + + #[test] + fn parses_otio() { + let got = OpenPayload::from_value(&json!({"format": "otio-json", "data": "{}"})).unwrap(); + assert_eq!(got, OpenPayload::OtioJson { data: "{}".into() }); + } + + fn otio_with(url: &str) -> String { + json!({ + "OTIO_SCHEMA": "Timeline.1", + "tracks": {"OTIO_SCHEMA": "Stack.1", "children": [{"OTIO_SCHEMA": "Track.1", "children": [{ + "OTIO_SCHEMA": "Clip.2", + "media_references": {"DEFAULT_MEDIA": {"OTIO_SCHEMA": "ExternalReference.1", "target_url": url}}, + "active_media_reference_key": "DEFAULT_MEDIA" + }]}]} + }) + .to_string() + } + + #[test] + fn rejects_media_that_does_not_exist_with_the_paths() { + let missing = absolute("/nowhere/My Clip.mov"); + let payload = json!({"format": "otio-json", "data": otio_with(&file_url(&missing))}); + let err = OpenPayload::from_value(&payload).unwrap_err(); + assert_eq!(err, LinkError::MediaMissing { paths: vec![missing.clone()] }); + assert_eq!(err.code(), "media_missing"); + assert_eq!(err.to_object().data.get("missing"), Some(&json!([missing]))); + assert!(err.to_string().contains(&missing), "{err}"); + // A bare absolute path counts the same. + let bare = json!({"format": "otio-json", "data": otio_with(&absolute("/nowhere/clip.mov"))}); + assert!(matches!(OpenPayload::from_value(&bare), Err(LinkError::MediaMissing { .. }))); + } + + #[test] + fn accepts_media_that_opens_and_references_it_cannot_resolve() { + let dir = std::env::temp_dir().join(format!("craft-payload-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("My Clip.mov"); + std::fs::write(&file, b"x").unwrap(); + let url = file_url(file.to_str().unwrap()); + assert_eq!(std::path::PathBuf::from(local_path(&url).unwrap().unwrap()), file); + let good = json!({"format": "otio-json", "data": otio_with(&url)}); + assert!(OpenPayload::from_value(&good).is_ok()); + // A relative target has no base here. The app resolves it. + let relative = json!({"format": "otio-json", "data": otio_with("clips/a.mov")}); + assert!(OpenPayload::from_value(&relative).is_ok()); + // A generator or a missing reference names no file. + let none = json!({"format": "otio-json", "data": "{\"OTIO_SCHEMA\":\"GeneratorReference.1\"}"}); + assert!(OpenPayload::from_value(&none).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn parses_image_doc() { + let expected = absolute("/tmp/a.photocraft"); + let got = OpenPayload::from_value(&json!({"format": "image-doc", "path": expected, "asset": ASSET})).unwrap(); + let OpenPayload::ImageDoc { path, asset } = got else { + panic!("wrong variant"); + }; + assert_eq!(path, expected); + assert_eq!(asset.to_string(), ASSET); + } + + fn payload_with(url: &str) -> Value { + json!({"format": "otio-json", "data": otio_with(url)}) + } + + #[test] + fn traversal_in_a_named_path_is_payload_invalid() { + for bad in [ + absolute("/tmp/../etc/passwd"), + file_url(&absolute("/tmp/a/../../etc/passwd")), + file_url(&absolute("/tmp/%2e%2e/etc/passwd")), + absolute("/tmp/a\u{0}b"), + file_url(&absolute("/tmp/a%00b")), + ] { + let err = OpenPayload::from_value(&payload_with(&bad)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad:?}: {err:?}"); + } + let err = OpenPayload::from_value(&json!({"format": "image-doc", "path": absolute("/tmp/../etc/x.png"), "asset": ASSET})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // A relative target is the app's business, `..` or not. + assert!(OpenPayload::from_value(&payload_with("clips/../a.mov")).is_ok()); + } + + #[test] + fn a_file_url_that_does_not_decode_is_payload_invalid() { + for bad in [ + "file:///tmp/clip%GG.mov", + "file:///tmp/clip%4", + "file:///tmp/clip%", + "file://localhost/tmp/clip%zz.mov", + "file:///tmp/clip%FF.mov", + // A sign or a space is not a hex digit, although `from_str_radix` takes `+`. + "file:///tmp/a%+fb", + "file:///tmp/a%-fb", + "file:///tmp/a% fb", + "file:///tmp/a%f", + ] { + let err = OpenPayload::from_value(&payload_with(bad)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad:?}: {err:?}"); + assert!(matches!(missing_media(&otio_with(bad)), Err(LinkError::PayloadInvalid(_))), "{bad:?}"); + } + // A reference that names no local file stays the app's business. + for skipped in ["file://remote-host/tmp/clip%GG.mov", "http://example.com/clip%GG.mov", "clips/clip%GG.mov"] { + assert!(OpenPayload::from_value(&payload_with(skipped)).is_ok(), "{skipped:?}"); + } + } + + #[cfg(unix)] + #[test] + fn only_regular_files_count_as_present_and_a_fifo_does_not_block() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("craft-payload-fifo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("pipe.mov"); + let made = std::process::Command::new("mkfifo").arg(&fifo).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + let real = dir.join("real.mov"); + std::fs::write(&real, b"x").unwrap(); + let link = dir.join("link.mov"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let start = std::time::Instant::now(); + let data = json!({"OTIO_SCHEMA": "Timeline.1", "refs": [ + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": fifo.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": dir.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": link.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": real.to_str().unwrap()}, + ]}) + .to_string(); + let missing = missing_media(&data).unwrap(); + assert!(start.elapsed() < std::time::Duration::from_secs(2), "the probe waited on a FIFO"); + assert_eq!(missing, vec![fifo.to_str().unwrap().to_string(), dir.to_str().unwrap().to_string()]); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn size_depth_and_count_are_bounded() { + // Too many distinct media files. + let refs: Vec = (0..=MAX_MEDIA_REFS) + .map(|n| json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": absolute(&format!("/nowhere/{n}.mov"))})) + .collect(); + let many = json!({"OTIO_SCHEMA": "Timeline.1", "refs": refs}).to_string(); + let err = OpenPayload::from_value(&json!({"format": "otio-json", "data": many})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // The same file named many times counts once. + let same: Vec = + (0..MAX_MEDIA_REFS * 2).map(|_| json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": absolute("/nowhere/a.mov")})).collect(); + let doc = json!({"OTIO_SCHEMA": "Timeline.1", "refs": same}).to_string(); + assert_eq!(missing_media(&doc).unwrap(), vec![absolute("/nowhere/a.mov")]); + // Too deep. serde_json refuses at 128, so the document reads as "not JSON" and the app reports it. + let deep = format!("{}{}", "[".repeat(300), "]".repeat(300)); + assert_eq!(missing_media(&deep).unwrap(), Vec::::new()); + // Just inside serde_json's limit the walker still finishes. + let nested = format!("{}{}", "[".repeat(MAX_DEPTH - 1), "]".repeat(MAX_DEPTH - 1)); + assert_eq!(missing_media(&nested).unwrap(), Vec::::new()); + // Too big. + let big = format!("{{\"pad\":\"{}\"}}", "a".repeat(usize::try_from(MAX_LINK_PAYLOAD).unwrap())); + let err = OpenPayload::from_value(&json!({"format": "otio-json", "data": big})).unwrap_err(); + assert_eq!(err, LinkError::TooLarge); + assert_eq!(err.code(), "payload_too_large"); + } + + #[test] + fn rejects_bad_payloads() { + for bad in [ + json!("x"), + json!({}), + json!({"format": "otio-json"}), + json!({"format": "otio-json", "data": ""}), + json!({"format": "image-doc", "path": "rel.png", "asset": ASSET}), + json!({"format": "image-doc", "path": "/a.png", "asset": "craft://bad"}), + json!({"format": "image-doc", "asset": ASSET}), + json!({"format": "svg", "data": "x"}), + ] { + let err = OpenPayload::from_value(&bad).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad}: {err:?}"); + assert_eq!(err.code(), "payload_invalid"); + } + } +} diff --git a/crates/ipc/src/record.rs b/crates/ipc/src/record.rs new file mode 100644 index 000000000..6b8b9f09f --- /dev/null +++ b/crates/ipc/src/record.rs @@ -0,0 +1,237 @@ +//! The `craft.link` project item record. +//! +//! Schema: `schemas/link-record.schema.json`. A consumer stores one of these +//! on the linked media item. Records written before the asset fields existed +//! (FilmCraft to EffectCraft) parse and validate unchanged. Unknown JSON fields +//! stay in the `extra` maps and come back out on save. +//! +//! This module uses only `std` and `serde`, so an app can copy it to a wasm build. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::error::LinkError; +use crate::ids::{valid_app_id, valid_uuid}; +use crate::moniker::Moniker; + +/// What the linked asset is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AssetKind { + Still, + Vector, + Sequence, + Comp, +} + +impl AssetKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Still => "still", + Self::Vector => "vector", + Self::Sequence => "sequence", + Self::Comp => "comp", + } + } +} + +/// The linked asset: its kind and its durable id. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AssetRef { + pub kind: AssetKind, + /// A moniker string. [`LinkRecord::validate`] parses it. + pub moniker: String, + #[serde(default, flatten)] + pub extra: Map, +} + +/// Who wrote the result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Producer { + pub app: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub doc_id: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// Size of the linked result. The video fields are absent for stills. +/// +/// `rate_num` and `rate_den` come as a pair. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Format { + pub w: u64, + pub h: u64, + pub alpha: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_num: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_den: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ticks: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// One link from a consumer media item to a producer asset. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LinkRecord { + pub v: u32, + pub link_id: String, + pub target_app: String, + pub target_project: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_comp: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nested_sequence: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub producer: Option, + pub result_path: String, + pub result_epoch: u64, + pub format: Format, + pub created: String, + #[serde(default, flatten)] + pub extra: Map, +} + +impl LinkRecord { + /// Check the record rules. Deserialization can accept a record this rejects. + pub fn validate(&self) -> Result<(), LinkError> { + if self.v != 1 { + return Err(invalid(format!("v is {}, want 1", self.v))); + } + if !valid_uuid(&self.link_id) { + return Err(invalid("link_id is not a uuid".into())); + } + validate_app("target_app", &self.target_app)?; + require_nonempty("target_project", &self.target_project)?; + require_nonempty("result_path", &self.result_path)?; + if let Some(comp) = &self.target_comp { + require_nonempty("target_comp", comp)?; + } + if let Some(nest) = &self.nested_sequence { + require_nonempty("nested_sequence", nest)?; + } + if self.target_comp.is_none() && self.asset.is_none() { + return Err(invalid("a record needs target_comp or asset".into())); + } + if let Some(asset) = &self.asset { + Moniker::parse(&asset.moniker).map_err(|err| invalid(format!("asset.moniker: {err}")))?; + } + if let Some(producer) = &self.producer { + validate_app("producer.app", &producer.app)?; + } + self.format.validate()?; + if !valid_rfc3339(&self.created) { + return Err(invalid("created is not an RFC 3339 date-time".into())); + } + Ok(()) + } +} + +impl Format { + fn validate(&self) -> Result<(), LinkError> { + require_at_least_one("format.w", self.w)?; + require_at_least_one("format.h", self.h)?; + match (self.rate_num, self.rate_den) { + (Some(num), Some(den)) => { + require_at_least_one("format.rate_num", num)?; + require_at_least_one("format.rate_den", den) + } + (None, None) => Ok(()), + _ => Err(invalid("format.rate_num and format.rate_den come as a pair".into())), + } + } +} + +fn invalid(message: String) -> LinkError { + LinkError::Invalid(message) +} + +fn validate_app(name: &str, app: &str) -> Result<(), LinkError> { + if valid_app_id(app) { Ok(()) } else { Err(invalid(format!("{name} {app:?} must match [a-z][a-z0-9-]{{1,31}}"))) } +} + +fn require_nonempty(name: &str, value: &str) -> Result<(), LinkError> { + if value.is_empty() { Err(invalid(format!("{name} is empty"))) } else { Ok(()) } +} + +fn require_at_least_one(name: &str, value: u64) -> Result<(), LinkError> { + if value >= 1 { Ok(()) } else { Err(invalid(format!("{name} must be >= 1"))) } +} + +/// `YYYY-MM-DDTHH:MM:SS` with an optional fraction and a `Z` or numeric offset. +fn valid_rfc3339(text: &str) -> bool { + let bytes = text.as_bytes(); + if bytes.len() < 20 { + return false; + } + let (Some(month), Some(day), Some(hour), Some(minute), Some(second)) = + (two_digits(bytes, 5), two_digits(bytes, 8), two_digits(bytes, 11), two_digits(bytes, 14), two_digits(bytes, 17)) + else { + return false; + }; + if !four_digits(bytes, 0) + || byte_at(bytes, 4) != Some(b'-') + || byte_at(bytes, 7) != Some(b'-') + || !matches!(byte_at(bytes, 10), Some(b'T' | b't')) + || byte_at(bytes, 13) != Some(b':') + || byte_at(bytes, 16) != Some(b':') + { + return false; + } + if !(1..=12).contains(&month) || !(1..=31).contains(&day) || hour > 23 || minute > 59 || second > 60 { + return false; + } + let (Some(century), Some(year_in_century)) = (two_digits(bytes, 0), two_digits(bytes, 2)) else { + return false; + }; + if chrono::NaiveDate::from_ymd_opt(i32::from(century * 100 + year_in_century), u32::from(month), u32::from(day)).is_none() { + return false; + } + let mut index = 19; + if byte_at(bytes, index) == Some(b'.') { + index += 1; + let start = index; + while byte_at(bytes, index).is_some_and(|byte| byte.is_ascii_digit()) { + index += 1; + } + if index == start { + return false; + } + } + match byte_at(bytes, index) { + Some(b'Z' | b'z') => index + 1 == bytes.len(), + Some(b'+' | b'-') => numeric_offset(bytes, index + 1), + _ => false, + } +} + +fn numeric_offset(bytes: &[u8], index: usize) -> bool { + if bytes.len() != index + 5 || byte_at(bytes, index + 2) != Some(b':') { + return false; + } + matches!((two_digits(bytes, index), two_digits(bytes, index + 3)), (Some(hour), Some(minute)) if hour <= 23 && minute <= 59) +} + +fn four_digits(bytes: &[u8], index: usize) -> bool { + bytes.get(index..index + 4).is_some_and(|slice| slice.iter().all(u8::is_ascii_digit)) +} + +fn two_digits(bytes: &[u8], index: usize) -> Option { + let slice = bytes.get(index..index + 2)?; + let hi = slice.first().copied()?; + let lo = slice.get(1).copied()?; + if !hi.is_ascii_digit() || !lo.is_ascii_digit() { + return None; + } + Some(u16::from(hi - b'0') * 10 + u16::from(lo - b'0')) +} + +fn byte_at(bytes: &[u8], index: usize) -> Option { + bytes.get(index).copied() +} diff --git a/crates/ipc/src/refresh.rs b/crates/ipc/src/refresh.rs new file mode 100644 index 000000000..f4b48a72e --- /dev/null +++ b/crates/ipc/src/refresh.rs @@ -0,0 +1,572 @@ +//! Watch and refresh rules for linked files (`PROTOCOL.md` section 10). +//! +//! A producer writes `result_path` with [`write_atomic`]. A consumer keeps one +//! [`Tracker`] per linked file and calls [`Tracker::poll`] on a timer. The +//! tracker needs no connection to the producer, so it also works while the +//! producer is offline. `std` only. +//! +//! | State | Meaning | +//! |---|---| +//! | `Current` | the disk has the content the consumer loaded | +//! | `Modified` | the disk differs from what the consumer loaded; the epoch went up | +//! | `Missing` | the file does not exist | + +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::UNIX_EPOCH; + +use sha2::{Digest, Sha256}; + +use crate::error::LinkError; + +/// Size, modification time and content hash of a file at one moment. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fingerprint { + pub size: u64, + pub mtime_ms: u64, + /// `sha256:` followed by 64 lowercase hex characters. + pub hash: String, +} + +/// What a consumer shows in its Links panel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileState { + Current, + Modified, + Missing, +} + +impl FileState { + /// The wire word used by `link.refresh`. + pub fn as_str(self) -> &'static str { + match self { + Self::Current => "current", + Self::Modified => "modified", + Self::Missing => "missing", + } + } +} + +/// The result of one [`Tracker::poll`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Change { + pub state: FileState, + pub epoch: u64, + /// True when this poll added 1 to the epoch. + pub bumped: bool, +} + +/// Watches one file for a consumer. +#[derive(Debug)] +pub struct Tracker { + path: PathBuf, + epoch: u64, + /// What the consumer has loaded. + loaded: Option, + /// What the last poll saw on disk. + seen: Option, + state: FileState, +} + +impl Tracker { + /// Start watching `path` at `epoch`. A file that exists now counts as loaded. + pub fn new(path: impl Into, epoch: u64) -> Result { + let path = path.into(); + let seen = fingerprint_if_present(&path)?; + let state = if seen.is_some() { FileState::Current } else { FileState::Missing }; + Ok(Self { path, epoch, loaded: seen.clone(), seen, state }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn epoch(&self) -> u64 { + self.epoch + } + + pub fn state(&self) -> FileState { + self.state + } + + /// The fingerprint from the last poll, or `None` while the file is missing. + pub fn seen(&self) -> Option<&Fingerprint> { + self.seen.as_ref() + } + + /// Raise the epoch to `epoch` when it is larger. Used when the producer is + /// reachable again and reports its own epoch. An epoch never decreases. + pub fn adopt_epoch(&mut self, epoch: u64) { + self.epoch = self.epoch.max(epoch); + } + + /// Look at the disk once. + /// + /// A shallow poll (`deep` false) reads size and mtime and hashes the file + /// only when either differs from the last poll. A deep poll always hashes. + /// An edit that keeps size and mtime is invisible to a shallow poll. + /// + /// A missing file gives `Missing` and no epoch bump. Any other read error + /// is returned and leaves the state as it was. So does a change that needs + /// a bump at epoch `u64::MAX`: that is `Err(Invalid)`, because an epoch that + /// cannot go up would let a cached frame pass for the new content. + pub fn poll(&mut self, deep: bool) -> Result { + let Some(mut file) = open_if_present(&self.path)? else { + self.seen = None; + self.state = FileState::Missing; + return Ok(self.change(false)); + }; + let (size, mtime_ms) = stat(&file, &self.path)?; + if !deep + && let Some(prev) = &self.seen + && prev.size == size + && prev.mtime_ms == mtime_ms + { + return Ok(self.change(false)); + } + let now = Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? }; + let changed = self.seen.as_ref().is_none_or(|prev| prev.hash != now.hash); + let bumped = changed && self.differs_from_loaded_or_was_seen(&now); + self.apply(now, bumped)?; + Ok(self.change(bumped)) + } + + /// The consumer loaded the content it last saw. `Modified` becomes `Current`. + pub fn accept(&mut self) { + self.loaded = self.seen.clone(); + self.state = if self.seen.is_some() { FileState::Current } else { FileState::Missing }; + } + + /// Point the link at `new_path`. The new file is read at once. Its hash + /// equal to what the consumer loaded gives `Current` with no bump. A + /// different hash gives `Modified` and one bump. A missing `new_path` is an + /// error and the old path stays. So it does when the bump would pass `u64::MAX`. + pub fn relink(&mut self, new_path: impl Into) -> Result { + let new_path = new_path.into(); + let Some(now) = fingerprint_if_present(&new_path)? else { + return Err(LinkError::Unknown(format!("{} does not exist", new_path.display()))); + }; + let same = self.loaded.as_ref().is_some_and(|loaded| loaded.hash == now.hash); + self.apply(now, !same)?; + self.path = new_path; + Ok(self.change(!same)) + } + + /// Whether a changed hash counts as a new version. The first sighting after + /// `Missing` that equals the loaded content is not a new version. + fn differs_from_loaded_or_was_seen(&self, now: &Fingerprint) -> bool { + match (&self.seen, &self.loaded) { + (Some(_), _) => true, + (None, Some(loaded)) => loaded.hash != now.hash, + (None, None) => true, + } + } + + /// Record `now`. A bump adds 1 to the epoch, or fails with the tracker untouched. + fn apply(&mut self, now: Fingerprint, bumped: bool) -> Result<(), LinkError> { + if bumped { + self.epoch = self.epoch.checked_add(1).ok_or_else(|| LinkError::Invalid("the epoch is at its maximum and cannot go up".into()))?; + } + let current = self.loaded.as_ref().is_some_and(|loaded| loaded.hash == now.hash); + self.seen = Some(now); + self.state = if current { FileState::Current } else { FileState::Modified }; + Ok(()) + } + + fn change(&self, bumped: bool) -> Change { + Change { state: self.state, epoch: self.epoch, bumped } + } +} + +/// Fingerprint `path`. A missing file is an error. +pub fn fingerprint(path: &Path) -> Result { + let mut file = crate::fsopen::open_read(path, true)?; + let (size, mtime_ms) = stat(&file, path)?; + Ok(Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? }) +} + +/// `None` when the file does not exist. +pub fn fingerprint_if_present(path: &Path) -> Result, LinkError> { + match open_if_present(path)? { + None => Ok(None), + Some(mut file) => { + let (size, mtime_ms) = stat(&file, path)?; + Ok(Some(Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? })) + } + } +} + +fn open_if_present(path: &Path) -> Result, LinkError> { + // `stat` checks the type on this handle. An open that waits for a FIFO writer would stall the caller. + match crate::fsopen::open_read(path, true) { + Ok(file) => Ok(Some(file)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + // Windows cannot open a directory for reading while Unix can, so `stat` would never + // run there and callers would see an OS-specific Io instead. Classify a path that + // exists but is not a regular file here, so both platforms report Invalid. + Err(err) => match std::fs::metadata(path) { + Ok(meta) if !meta.is_file() => Err(LinkError::Invalid(format!("{} is not a regular file", path.display()))), + _ => Err(LinkError::Io(format!("{}: {err}", path.display()))), + }, + } +} + +fn stat(file: &File, path: &Path) -> Result<(u64, u64), LinkError> { + let meta = file.metadata().map_err(|err| LinkError::Io(format!("{}: {err}", path.display())))?; + if !meta.is_file() { + return Err(LinkError::Invalid(format!("{} is not a regular file", path.display()))); + } + let modified = meta.modified().map_err(|err| LinkError::Io(format!("{}: {err}", path.display())))?; + let since = modified.duration_since(UNIX_EPOCH).map_err(|_| LinkError::Io(format!("{}: mtime is before 1970", path.display())))?; + let ms = u64::try_from(since.as_millis()).map_err(|_| LinkError::Io(format!("{}: mtime is out of range", path.display())))?; + Ok((meta.len(), ms)) +} + +fn hash_reader(reader: &mut impl Read) -> Result { + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(buf.get(..n).unwrap_or_default()); + } + Ok(format!("sha256:{}", crate::auth::encode_hex(&hasher.finalize()))) +} + +/// True for the temporary names [`write_atomic`] uses. Readers skip them. +pub fn is_temp_name(name: &str) -> bool { + name.starts_with('.') && name.ends_with(".tmp") +} + +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Write `bytes` to `path` so a reader sees the old file or the new file. +/// +/// The bytes go to `....tmp` in the same directory, are flushed, +/// and are renamed over `path`. The directory is flushed on Unix. +/// +/// An error before the rename removes the temporary file and leaves `path` as it was. The +/// directory flush and the final fingerprint run after the rename, so an error from either can +/// come back while `path` already holds the new bytes. Such an error means the write is not +/// confirmed durable or not fingerprinted, not that `path` is unchanged. +pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result { + let name = path.file_name().and_then(|n| n.to_str()).ok_or_else(|| LinkError::Invalid(format!("{} has no file name", path.display())))?; + let dir = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let tmp = dir.join(format!(".{name}.{}.{}.tmp", std::process::id(), TEMP_COUNTER.fetch_add(1, Ordering::Relaxed))); + let written = write_and_rename(&tmp, path, bytes); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written?; + #[cfg(unix)] + File::open(dir)?.sync_all()?; + fingerprint(path) +} + +fn write_and_rename(tmp: &Path, path: &Path, bytes: &[u8]) -> Result<(), LinkError> { + let mut file = OpenOptions::new().write(true).create_new(true).open(tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(tmp, path)?; + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::time::Duration; + + use super::*; + + struct Tmp(PathBuf); + + impl Tmp { + fn new(label: &str) -> Self { + let nanos = std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let dir = std::env::temp_dir().join(format!("cl-refresh-{label}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + Self(dir) + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn sha_of(bytes: &[u8]) -> String { + format!("sha256:{}", crate::auth::encode_hex(&Sha256::digest(bytes))) + } + + #[test] + fn hash_is_sha256_of_the_content() { + let tmp = Tmp::new("hash"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"abc").unwrap(); + let fp = fingerprint(&path).unwrap(); + assert_eq!(fp.hash, "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert_eq!(fp.size, 3); + assert!(fp.mtime_ms > 0); + } + + #[test] + fn unchanged_file_stays_current_with_no_bump() { + let tmp = Tmp::new("same"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 4).unwrap(); + assert_eq!(tracker.state(), FileState::Current); + for deep in [false, true] { + let change = tracker.poll(deep).unwrap(); + assert_eq!(change, Change { state: FileState::Current, epoch: 4, bumped: false }); + } + } + + #[test] + fn changed_content_is_modified_and_bumps_once() { + let tmp = Tmp::new("mod"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 4).unwrap(); + write_atomic(&path, b"two!").unwrap(); + let change = tracker.poll(false).unwrap(); + assert_eq!(change, Change { state: FileState::Modified, epoch: 5, bumped: true }); + // Polling again with no new edit does not bump again. + let again = tracker.poll(true).unwrap(); + assert_eq!(again, Change { state: FileState::Modified, epoch: 5, bumped: false }); + // A second edit bumps again. + write_atomic(&path, b"three").unwrap(); + assert_eq!(tracker.poll(false).unwrap().epoch, 6); + // Loading the new content clears the state. + tracker.accept(); + assert_eq!(tracker.state(), FileState::Current); + assert_eq!(tracker.poll(true).unwrap(), Change { state: FileState::Current, epoch: 6, bumped: false }); + } + + #[test] + fn touch_without_content_change_does_not_bump() { + let tmp = Tmp::new("touch"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let file = OpenOptions::new().write(true).open(&path).unwrap(); + file.set_modified(std::time::SystemTime::now() + Duration::from_secs(60)).unwrap(); + drop(file); + let change = tracker.poll(false).unwrap(); + assert_eq!(change, Change { state: FileState::Current, epoch: 0, bumped: false }); + assert!(tracker.seen().unwrap().mtime_ms > 0); + } + + #[test] + fn shallow_poll_catches_a_same_size_edit_when_mtime_moves() { + let tmp = Tmp::new("mtime"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"aaaa").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let before = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"bbbb").unwrap(); + OpenOptions::new().write(true).open(&path).unwrap().set_modified(before + Duration::from_secs(10)).unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn shallow_poll_misses_an_edit_that_keeps_size_and_mtime_and_deep_poll_finds_it() { + let tmp = Tmp::new("deep"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"aaaa").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"bbbb").unwrap(); + OpenOptions::new().write(true).open(&path).unwrap().set_modified(mtime).unwrap(); + assert_eq!(tracker.poll(false).unwrap().state, FileState::Current, "documented limit of a shallow poll"); + let change = tracker.poll(true).unwrap(); + assert_eq!(change, Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn missing_then_same_content_returns_to_current_without_bump() { + let tmp = Tmp::new("missing"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 2).unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Missing, epoch: 2, bumped: false }); + assert!(tracker.seen().is_none()); + std::fs::write(&path, b"one").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Current, epoch: 2, bumped: false }); + } + + #[test] + fn missing_then_new_content_is_modified_with_a_bump() { + let tmp = Tmp::new("missing2"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 2).unwrap(); + std::fs::remove_file(&path).unwrap(); + tracker.poll(false).unwrap(); + std::fs::write(&path, b"different").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 3, bumped: true }); + } + + #[test] + fn tracker_for_a_file_that_does_not_exist_yet() { + let tmp = Tmp::new("pending"); + let path = tmp.0.join("later.png"); + let mut tracker = Tracker::new(&path, 0).unwrap(); + assert_eq!(tracker.state(), FileState::Missing); + write_atomic(&path, b"first").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn unreadable_path_is_an_error_not_missing() { + let tmp = Tmp::new("dir"); + let path = tmp.0.join("a.png"); + std::fs::create_dir(&path).unwrap(); + assert!(Tracker::new(&path, 0).is_err(), "a directory is not a result file"); + } + + #[cfg(unix)] + #[test] + fn a_fifo_at_the_path_is_an_error_and_never_blocks() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let tmp = Tmp::new("fifo"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 3).unwrap(); + std::fs::remove_file(&path).unwrap(); + let made = std::process::Command::new("mkfifo").arg(&path).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + // The open of a FIFO with no writer waits for one. Run each call where a hang fails the test instead of the run. + let (tx, rx) = std::sync::mpsc::channel(); + let watched = path.clone(); + std::thread::spawn(move || { + let polled = tracker.poll(true); + let kept = (tracker.state(), tracker.epoch()); + let _ = tx.send((polled, kept, Tracker::new(&watched, 0).map(|_| ()), fingerprint(&watched).map(|_| ()))); + }); + let (polled, kept, created, printed) = rx.recv_timeout(Duration::from_secs(5)).expect("a FIFO blocked the open"); + assert!(matches!(polled, Err(LinkError::Invalid(_))), "{polled:?}"); + assert_eq!(kept, (FileState::Current, 3), "a failed poll leaves the state as it was"); + assert!(matches!(created, Err(LinkError::Invalid(_))), "{created:?}"); + assert!(matches!(printed, Err(LinkError::Invalid(_))), "{printed:?}"); + } + + #[test] + fn relink_compares_hashes_and_rejects_a_missing_target() { + let tmp = Tmp::new("relink"); + let old = tmp.0.join("old.png"); + let same = tmp.0.join("same.png"); + let other = tmp.0.join("other.png"); + std::fs::write(&old, b"one").unwrap(); + std::fs::write(&same, b"one").unwrap(); + std::fs::write(&other, b"two").unwrap(); + let mut tracker = Tracker::new(&old, 7).unwrap(); + + let err = tracker.relink(tmp.0.join("nope.png")).unwrap_err(); + assert!(matches!(err, LinkError::Unknown(_)), "{err:?}"); + assert_eq!(tracker.path(), old.as_path(), "old path stays"); + + assert_eq!(tracker.relink(&same).unwrap(), Change { state: FileState::Current, epoch: 7, bumped: false }); + assert_eq!(tracker.path(), same.as_path()); + assert_eq!(tracker.relink(&other).unwrap(), Change { state: FileState::Modified, epoch: 8, bumped: true }); + } + + #[test] + fn a_bump_at_the_largest_epoch_is_an_error_and_changes_nothing() { + let tmp = Tmp::new("maxepoch"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let other = tmp.0.join("other.png"); + std::fs::write(&other, b"two").unwrap(); + let mut tracker = Tracker::new(&path, u64::MAX).unwrap(); + // No bump needed, no error. + assert_eq!(tracker.poll(true).unwrap(), Change { state: FileState::Current, epoch: u64::MAX, bumped: false }); + write_atomic(&path, b"changed").unwrap(); + for _ in 0..2 { + assert!(matches!(tracker.poll(true), Err(LinkError::Invalid(_)))); + assert_eq!((tracker.state(), tracker.epoch()), (FileState::Current, u64::MAX)); + } + assert!(matches!(tracker.relink(&other), Err(LinkError::Invalid(_)))); + assert_eq!(tracker.path(), path.as_path(), "the old path stays"); + assert_eq!((tracker.state(), tracker.epoch()), (FileState::Current, u64::MAX)); + } + + #[test] + fn adopt_epoch_never_decreases() { + let tmp = Tmp::new("adopt"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"x").unwrap(); + let mut tracker = Tracker::new(&path, 5).unwrap(); + tracker.adopt_epoch(3); + assert_eq!(tracker.epoch(), 5); + tracker.adopt_epoch(9); + assert_eq!(tracker.epoch(), 9); + } + + #[test] + fn write_atomic_replaces_whole_and_leaves_no_temp() { + let tmp = Tmp::new("atomic"); + let path = tmp.0.join("out.png"); + let first = write_atomic(&path, b"first").unwrap(); + assert_eq!(first.hash, sha_of(b"first")); + let second = write_atomic(&path, b"second version").unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"second version"); + assert_eq!(second.hash, sha_of(b"second version")); + let names: Vec = std::fs::read_dir(&tmp.0).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert_eq!(names, vec!["out.png".to_string()], "no temp file left"); + } + + #[test] + fn write_atomic_failure_keeps_the_old_file_and_removes_the_temp() { + let tmp = Tmp::new("atomic-fail"); + let path = tmp.0.join("out.png"); + write_atomic(&path, b"keep").unwrap(); + // A directory at the destination makes rename fail after the temp file is written. + let blocked = tmp.0.join("blocked"); + std::fs::create_dir(&blocked).unwrap(); + std::fs::write(blocked.join("child"), b"x").unwrap(); + assert!(write_atomic(&blocked, b"nope").is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"keep"); + let temps = std::fs::read_dir(&tmp.0).unwrap().filter(|e| is_temp_name(e.as_ref().unwrap().file_name().to_str().unwrap())).count(); + assert_eq!(temps, 0); + } + + #[test] + fn reader_never_sees_a_half_file_while_a_writer_replaces_it() { + let tmp = Tmp::new("race"); + let path = tmp.0.join("out.bin"); + let a = vec![b'a'; 512 * 1024]; + let b = vec![b'b'; 768 * 1024]; + write_atomic(&path, &a).unwrap(); + let writer_path = path.clone(); + let (wa, wb) = (a.clone(), b.clone()); + let writer = std::thread::spawn(move || { + for i in 0..60 { + write_atomic(&writer_path, if i % 2 == 0 { &wb } else { &wa }).unwrap(); + } + }); + while !writer.is_finished() { + let got = std::fs::read(&path).unwrap(); + assert!(got == a || got == b, "saw a torn file of {} bytes", got.len()); + } + writer.join().unwrap(); + } + + #[test] + fn temp_names() { + assert!(is_temp_name(".out.png.12.0.tmp")); + assert!(!is_temp_name("out.png")); + assert!(!is_temp_name("notes.tmp")); + } +} diff --git a/crates/ipc/src/server/dispatch.rs b/crates/ipc/src/server/dispatch.rs new file mode 100644 index 000000000..5c7e24d33 --- /dev/null +++ b/crates/ipc/src/server/dispatch.rs @@ -0,0 +1,316 @@ +//! Methods answered on the connection thread, plus the sink forward. + +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use super::events::{self, Subscription}; +use super::provider; +use super::{Activation, Assets, Hub, lock}; +use crate::error::LinkError; +use crate::moniker::Moniker; +use crate::payload::OpenPayload; +use crate::wire::{self, Request}; + +pub(super) enum Outcome { + Continue(Value), + Close(Option), + Bulk { reply: Value, ticket: String }, +} + +pub(super) fn dispatch(hub: &Hub, conn_id: u64, sub: &Subscription, first: bool, req: &Request) -> Outcome { + match req.method.as_str() { + "link.hello" => continue_result(&req.id, hello(hub, &req.params)), + "events.subscribe" => continue_result(&req.id, subscribe(sub, &req.params)), + "events.unsubscribe" => { + events::unsubscribe(sub); + Outcome::Continue(wire::reply_ok(&req.id, json!({}))) + } + "link.bye" => { + events::unsubscribe(sub); + release_tickets(hub, conn_id); + Outcome::Close(Some(wire::reply_ok(&req.id, json!({})))) + } + "app.activate" => continue_result(&req.id, activate(hub, &req.params)), + "link.resolve" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "link.resolve", &req.params, resolve)), + "asset.describe" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "asset.describe", &req.params, describe)), + "link.refresh" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "link.refresh", &req.params, refresh)), + "link.open" => match validate_open(&req.params) { + Ok(()) => forward(hub, &req.method, &req.id, req.params.clone()), + Err(err) => Outcome::Continue(wire::reply_err(&req.id, &err)), + }, + "frames.open" => continue_result(&req.id, frames_open(hub, conn_id, &req.params)), + "bulk.attach" => bulk_attach(hub, first, req), + method => forward(hub, method, &req.id, req.params.clone()), + } +} + +pub(super) fn release_tickets(hub: &Hub, conn_id: u64) { + let owned = { + let mut tickets = lock(&hub.tickets); + let owned: Vec = tickets.iter().filter(|(_, id)| **id == conn_id).map(|(ticket, _)| ticket.clone()).collect(); + for ticket in &owned { + tickets.remove(ticket); + } + owned + }; + for ticket in owned { + hub.frames.close_ticket(&ticket); + } +} + +fn continue_result(id: &Value, result: Result) -> Outcome { + match result { + Ok(result) => Outcome::Continue(wire::reply_ok(id, result)), + Err(err) => Outcome::Continue(wire::reply_err(id, &err)), + } +} + +fn hello(hub: &Hub, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let protocol = obj.get("protocol").and_then(Value::as_object).ok_or_else(|| LinkError::BadParams("protocol required".into()))?; + let min = required_u32(protocol.get("min"), "protocol.min")?; + let max = required_u32(protocol.get("max"), "protocol.max")?; + if !wire::protocols_overlap(min, max) { + return Err(LinkError::Protocol { server_min: wire::PROTOCOL_MIN, server_max: wire::PROTOCOL_MAX }); + } + let record = &hub.record; + let mut caps: Vec = ["events", "activate", "link.receive", "frames"].iter().map(|c| (*c).to_string()).collect(); + for cap in record.caps() { + if !caps.contains(&cap) { + caps.push(cap); + } + } + Ok(json!({ + "app": record.app, + "app_version": record.app_version, + "pid": record.pid, + "instance": record.instance, + "protocol": wire::PROTOCOL_MAX, + "caps": caps, + })) +} + +fn subscribe(sub: &Subscription, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let list = obj.get("events").and_then(Value::as_array).ok_or_else(|| LinkError::BadParams("events must be an array".into()))?; + let mut names = Vec::with_capacity(list.len()); + for item in list { + let Some(name) = item.as_str() else { + return Err(LinkError::BadParams("event name must be a string".into())); + }; + names.push(name.to_string()); + } + events::subscribe(sub, names); + Ok(json!({"seq": 0})) +} + +fn activate(hub: &Hub, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let token = optional_string(obj, "token")?; + let from_pid = optional_u32(obj.get("from_pid"), "from_pid")?; + let open = match obj.get("open") { + None => None, + Some(Value::Object(open)) => { + if let Some(asset) = open.get("asset") { + let text = asset.as_str().ok_or_else(|| LinkError::BadParams("open.asset must be a moniker string".into()))?; + Moniker::parse(text).map_err(|err| LinkError::BadParams(format!("open.asset: {err}")))?; + } + Some(Value::Object(open.clone())) + } + Some(_) => return Err(LinkError::BadParams("open must be an object".into())), + }; + let provider = std::sync::Arc::clone(&hub.activate); + let activated = match provider::call(hub, "app.activate", move || provider.activate(token, from_pid, open))? { + Activation::Yes => Value::Bool(true), + Activation::No => Value::Bool(false), + Activation::Requested => Value::String("requested".into()), + }; + Ok(json!({"activated": activated})) +} + +/// Run an asset method on a helper thread, with the deadline and the stop check of [`provider::call`]. +fn with_assets(hub: &Hub, what: &str, params: &Value, run: fn(&dyn Assets, &Value) -> Result) -> Result { + let Some(assets) = &hub.assets else { + return Err(LinkError::NotReady("this app has no asset provider".into())); + }; + let assets = std::sync::Arc::clone(assets); + let params = params.clone(); + provider::call(hub, what, move || run(assets.as_ref(), ¶ms))? +} + +fn asset_param(params: &Value) -> Result { + let obj = wire::params_object(params)?; + let text = obj.get("asset").and_then(Value::as_str).ok_or_else(|| LinkError::BadParams("asset must be a moniker string".into()))?; + Moniker::parse(text).map_err(|err| LinkError::BadParams(format!("asset: {err}"))) +} + +fn resolve(assets: &dyn Assets, params: &Value) -> Result { + let asset = asset_param(params)?; + Ok(serde_json::to_value(assets.resolve(&asset)?)?) +} + +fn describe(assets: &dyn Assets, params: &Value) -> Result { + let asset = asset_param(params)?; + Ok(serde_json::to_value(assets.describe(&asset)?)?) +} + +fn refresh(assets: &dyn Assets, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let link_id = obj + .get("link_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .ok_or_else(|| LinkError::BadParams("link_id must be a non-empty string".into()))?; + Ok(serde_json::to_value(assets.refresh(link_id)?)?) +} + +/// `link.open` carries a `payload` whose shape depends on `format`. A bad one +/// never reaches the app. Params that are not an object stay `bad_params`. +fn validate_open(params: &Value) -> Result<(), LinkError> { + let obj = wire::params_object(params)?; + let payload = obj.get("payload").ok_or_else(|| LinkError::PayloadInvalid("payload is required".into()))?; + OpenPayload::from_value(payload)?; + match obj.get("result_path") { + None | Some(Value::Null) => Ok(()), + Some(Value::String(path)) => crate::payload::check_peer_path(path, "result_path"), + Some(_) => Err(LinkError::PayloadInvalid("result_path must be a string".into())), + } +} + +fn frames_open(hub: &Hub, conn_id: u64, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let opened = hub.frames.open(obj)?; + if opened.ticket.len() != 32 || !crate::auth::is_lower_hex(&opened.ticket) { + hub.frames.close_ticket(&opened.ticket); + return Err(LinkError::BadParams("ticket must be 32 lowercase hex characters".into())); + } + let mut tickets = lock(&hub.tickets); + if tickets.contains_key(&opened.ticket) { + return Err(LinkError::BadParams("ticket already registered".into())); + } + tickets.insert(opened.ticket.clone(), conn_id); + drop(tickets); + Ok(json!({ + "ticket": opened.ticket, + "w": opened.w, + "h": opened.h, + "stride": opened.stride, + "len": opened.len, + })) +} + +fn bulk_attach(hub: &Hub, first: bool, req: &Request) -> Outcome { + if !first { + let err = LinkError::BadParams("bulk.attach must be the first method".into()); + return Outcome::Close(Some(wire::reply_err(&req.id, &err))); + } + let ticket = match ticket_param(&req.params) { + Ok(ticket) => ticket, + Err(err) => return Outcome::Close(Some(wire::reply_err(&req.id, &err))), + }; + let known = lock(&hub.tickets).contains_key(&ticket); + if !known { + let err = LinkError::BadParams("unknown ticket".into()); + return Outcome::Close(Some(wire::reply_err(&req.id, &err))); + } + Outcome::Bulk { reply: wire::reply_ok(&req.id, json!({})), ticket } +} + +fn ticket_param(params: &Value) -> Result { + let obj = wire::params_object(params)?; + let Some(ticket) = obj.get("ticket").and_then(Value::as_str) else { + return Err(LinkError::BadParams("ticket required".into())); + }; + Ok(ticket.to_string()) +} + +fn forward(hub: &Hub, method: &str, id: &Value, params: Value) -> Outcome { + let Some(rx) = hub.sink.submit(method.to_string(), params) else { + return Outcome::Close(None); + }; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + loop { + if hub.stop.load(std::sync::atomic::Ordering::SeqCst) { + return Outcome::Close(None); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Outcome::Continue(timeout_reply(method, id)); + } + let slice = left.min(Duration::from_millis(200)); + match rx.recv_timeout(slice) { + Ok(result) => return Outcome::Continue(adopt_sink_reply(id, result)), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Outcome::Close(None), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +/// FilmCraft's UI sends a finished reply object (`ok` plus `result` or `error`). +/// A value with `ok` is that object. Insert `id` and do not wrap it again. +/// Any other value is a result payload and gets [`wire::reply_ok`]. +fn adopt_sink_reply(id: &Value, value: Value) -> Value { + let Some(obj) = value.as_object() else { + return wire::reply_ok(id, value); + }; + if !obj.contains_key("ok") { + return wire::reply_ok(id, value); + } + let mut reply = value; + if let Some(map) = reply.as_object_mut() + && !map.contains_key("id") + { + map.insert("id".to_string(), id.clone()); + } + reply +} + +fn timeout_reply(method: &str, id: &Value) -> Value { + if is_new_method(method) { wire::reply_err(id, &LinkError::Timeout) } else { wire::reply_err_text(id, "timeout") } +} + +/// New methods use an error object. Legacy names keep the string `"timeout"`. +fn is_new_method(method: &str) -> bool { + matches!( + method, + "link.open" + | "link.attach" + | "link.focus" + | "link.close" + | "link.hello" + | "events.subscribe" + | "events.unsubscribe" + | "link.bye" + | "app.activate" + | "frames.open" + | "bulk.attach" + | "link.resolve" + | "asset.describe" + ) +} + +fn optional_string(obj: &Map, key: &str) -> Result, LinkError> { + match obj.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) => Ok(Some(text.clone())), + Some(_) => Err(LinkError::BadParams(format!("{key} must be a string or null"))), + } +} + +fn optional_u32(value: Option<&Value>, key: &str) -> Result, LinkError> { + match value { + None => Ok(None), + Some(value) => required_u32(Some(value), key).map(Some), + } +} + +fn required_u32(value: Option<&Value>, key: &str) -> Result { + let Some(value) = value else { + return Err(LinkError::BadParams(format!("{key} is required"))); + }; + let Some(number) = value.as_u64() else { + return Err(LinkError::BadParams(format!("{key} must be a number"))); + }; + u32::try_from(number).map_err(|_| LinkError::BadParams(format!("{key} is out of range"))) +} diff --git a/crates/ipc/src/server/events.rs b/crates/ipc/src/server/events.rs new file mode 100644 index 000000000..6de840964 --- /dev/null +++ b/crates/ipc/src/server/events.rs @@ -0,0 +1,214 @@ +//! Per-connection event queue. `emit` only enqueues. +//! +//! The writer sleeps a short grace before it drains so a burst fills the +//! queue of [`crate::wire::EVENT_QUEUE`] instead of racing one write at a time +//! into the socket buffer. Overflow occupies one of those slots. + +use std::collections::{HashSet, VecDeque}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use super::{Hub, lock}; +use crate::wire::{self, EVENT_QUEUE}; + +/// How long a burst may land in the queue before the writer drains it. +const BATCH_GRACE: Duration = Duration::from_millis(30); +const WAIT_SLICE: Duration = Duration::from_millis(200); + +pub(crate) struct Subscription { + pub id: u64, + filter: Mutex, + outbox: Mutex, + cv: Condvar, + closed: AtomicBool, +} + +struct Filter { + names: Option>, +} + +struct Outbox { + items: VecDeque, + latched: bool, + next_seq: u64, +} + +struct Queued { + line: Value, + overflow: bool, +} + +impl Subscription { + pub(super) fn new(id: u64) -> Arc { + Arc::new(Self { + id, + filter: Mutex::new(Filter { names: None }), + outbox: Mutex::new(Outbox { items: VecDeque::new(), latched: false, next_seq: 1 }), + cv: Condvar::new(), + closed: AtomicBool::new(false), + }) + } +} + +pub(super) fn subscribe(sub: &Subscription, names: Vec) { + let mut filter = lock(&sub.filter); + filter.names = Some(names.into_iter().collect()); +} + +pub(super) fn is_subscribed(sub: &Subscription) -> bool { + lock(&sub.filter).names.is_some() +} + +pub(super) fn unsubscribe(sub: &Subscription) { + let mut filter = lock(&sub.filter); + filter.names = None; +} + +pub(super) fn enqueue(sub: &Subscription, event: &str, data: &Value) { + if sub.closed.load(Ordering::SeqCst) { + return; + } + let interested = { + let filter = lock(&sub.filter); + filter.names.as_ref().is_some_and(|names| names.contains(event)) + }; + if !interested { + return; + } + { + let mut outbox = lock(&sub.outbox); + if sub.closed.load(Ordering::SeqCst) || outbox.latched { + return; + } + let line = wire::push_event(event, outbox.next_seq, data.clone()); + let oversize = serde_json::to_vec(&line).map(|bytes| bytes.len() > wire::MAX_LINE).unwrap_or(true); + if oversize || outbox.items.len() >= EVENT_QUEUE { + // A full queue gives up its newest slot. An event that is too large for one line + // is dropped alone: everything queued before it is still delivered. + let since = if outbox.items.len() >= EVENT_QUEUE { + outbox.items.pop_back().map(|item| item_seq(&item)).unwrap_or(outbox.next_seq) + } else { + outbox.items.back().map(item_seq).unwrap_or(outbox.next_seq.saturating_sub(1)) + }; + let seq = outbox.next_seq; + outbox.next_seq = outbox.next_seq.saturating_add(1); + outbox.items.push_back(Queued { overflow: true, line: wire::push_event("events.overflow", seq, json!({"since": since})) }); + outbox.latched = true; + } else { + outbox.next_seq = outbox.next_seq.saturating_add(1); + outbox.items.push_back(Queued { overflow: false, line }); + } + } + sub.cv.notify_one(); +} + +fn item_seq(item: &Queued) -> u64 { + item.line.get("seq").and_then(Value::as_u64).unwrap_or(0) +} + +pub(super) fn close_sub(sub: &Subscription) { + sub.closed.store(true, Ordering::SeqCst); + // The outbox lock pairs with the check in `write_loop`, so the notify + // cannot land before the writer has entered `wait`. + drop(lock(&sub.outbox)); + sub.cv.notify_one(); +} + +pub(super) fn wake_all(subs: &Mutex>>) { + let guard = lock(subs); + for sub in guard.iter() { + drop(lock(&sub.outbox)); + sub.cv.notify_one(); + } +} + +pub(super) fn remove_sub(hub: &Hub, id: u64) { + lock(&hub.subs).retain(|sub| sub.id != id); +} + +pub(super) fn write_loop(sub: Arc, writer: Arc>>, stop: &AtomicBool) { + loop { + { + let guard = lock(&sub.outbox); + if sub.closed.load(Ordering::SeqCst) || stop.load(Ordering::SeqCst) { + return; + } + if guard.items.is_empty() { + let waited = sub.cv.wait_timeout(guard, WAIT_SLICE); + let (guard, _) = match waited { + Ok(pair) => pair, + Err(err) => err.into_inner(), + }; + drop(guard); + continue; + } + } + std::thread::sleep(BATCH_GRACE); + if sub.closed.load(Ordering::SeqCst) || stop.load(Ordering::SeqCst) { + return; + } + let batch = { + let mut guard = lock(&sub.outbox); + std::mem::take(&mut guard.items) + }; + for item in batch { + let wrote = { + let mut guard = lock(&writer); + wire::write_line(&mut *guard, &item.line).is_ok() + }; + if item.overflow { + lock(&sub.outbox).latched = false; + } + if !wrote { + sub.closed.store(true, Ordering::SeqCst); + return; + } + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn an_event_over_the_line_limit_becomes_an_overflow_marker() { + let sub = Subscription::new(8); + subscribe(&sub, vec!["doc.changed".into()]); + enqueue(&sub, "doc.changed", &json!({"n": 1})); + enqueue(&sub, "doc.changed", &json!({"blob": "x".repeat(wire::MAX_LINE)})); + let outbox = lock(&sub.outbox); + assert_eq!(outbox.items.len(), 2, "the earlier event stays queued"); + assert_eq!(outbox.items[0].line["event"], "doc.changed"); + assert_eq!(outbox.items[1].line["event"], "events.overflow"); + assert_eq!(outbox.items[1].line["data"]["since"], 1); + assert!(outbox.items.iter().all(|item| serde_json::to_vec(&item.line).unwrap().len() <= wire::MAX_LINE)); + } + + #[test] + fn overflow_occupies_one_queue_slot() { + let sub = Subscription::new(7); + subscribe(&sub, vec!["doc.changed".into()]); + enqueue(&sub, "other", &json!({"n": 0})); + assert!(lock(&sub.outbox).items.is_empty()); + for n in 0..EVENT_QUEUE { + enqueue(&sub, "doc.changed", &json!({"n": n})); + } + enqueue(&sub, "doc.changed", &json!({"n": EVENT_QUEUE})); + enqueue(&sub, "doc.changed", &json!({"n": EVENT_QUEUE + 1})); + let outbox = lock(&sub.outbox); + assert!(outbox.latched); + assert_eq!(outbox.items.len(), EVENT_QUEUE); + let overflow: Vec<_> = outbox.items.iter().filter(|item| item.overflow).collect(); + assert_eq!(overflow.len(), 1); + assert_eq!(outbox.items.iter().filter(|item| !item.overflow).count(), EVENT_QUEUE - 1); + let last = overflow[0]; + assert_eq!(last.line["event"], json!("events.overflow")); + assert!(last.line["data"]["since"].is_number()); + assert_eq!(outbox.items.front().expect("queued").line["seq"], json!(1)); + } +} diff --git a/crates/ipc/src/server/listener.rs b/crates/ipc/src/server/listener.rs new file mode 100644 index 000000000..eec2402e7 --- /dev/null +++ b/crates/ipc/src/server/listener.rs @@ -0,0 +1,633 @@ +//! Local socket accept loop, auth, and the per-connection threads. +//! +//! The 17th connection receives one refusal line and does not take a slot. +//! Bulk connections count toward the same cap. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use interprocess::TryClone; +#[cfg(unix)] +use interprocess::local_socket::GenericFilePath; +use interprocess::local_socket::prelude::*; +use interprocess::local_socket::{Listener, ListenerNonblockingMode, ListenerOptions, Stream}; +use serde_json::Value; + +use super::dispatch::{self, Outcome}; +use super::events::{self, Subscription}; +use super::tcp::BoundTcp; +use super::{Hub, lock}; +use crate::discovery::{Endpoint, RuntimeDir}; +use crate::error::LinkError; +use crate::wire::{self, LineReader, MAX_LINE}; + +const READ_POLL: Duration = Duration::from_millis(250); +const REFUSE_WRITE: Duration = Duration::from_secs(2); + +pub(super) fn bind(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + #[cfg(unix)] + { + bind_unix(runtime, app, pid) + } + #[cfg(windows)] + { + let _ = runtime; + bind_pipe(app, pid) + } +} + +pub(super) fn endpoint(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + #[cfg(unix)] + { + runtime.unix_endpoint(app, pid) + } + #[cfg(windows)] + { + let _ = runtime; + Ok(Endpoint::Pipe { name: pipe_name(app, pid) }) + } +} + +#[cfg(unix)] +fn bind_unix(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + let path = runtime.socket_file(app, pid); + let name = path.to_fs_name::()?; + let listener = ListenerOptions::new().name(name).nonblocking(ListenerNonblockingMode::Accept).reclaim_name(false).create_sync()?; + Ok(listener) +} + +/// Owner and SYSTEM only. A peer pid is not an allow; Windows `PeerCreds` has no user id. +#[cfg(windows)] +fn bind_pipe(app: &str, pid: u32) -> Result { + use interprocess::local_socket::GenericNamespaced; + use interprocess::os::windows::local_socket::ListenerOptionsExt; + use interprocess::os::windows::security_descriptor::SecurityDescriptor; + use widestring::U16CString; + + let wide = U16CString::from_str("D:P(A;;GA;;;OW)(A;;GA;;;SY)").map_err(LinkError::io)?; + let descriptor = SecurityDescriptor::deserialize(wide.as_ucstr())?; + let name = pipe_name(app, pid).to_ns_name::()?; + let listener = ListenerOptions::new() + .name(name) + .nonblocking(ListenerNonblockingMode::Accept) + .reclaim_name(false) + .security_descriptor(descriptor) + .create_sync()?; + Ok(listener) +} + +#[cfg(windows)] +fn pipe_name(app: &str, pid: u32) -> String { + match windows_user() { + Some(user) => format!("craft-{user}-{app}-{pid}"), + None => format!("craft-{app}-{pid}"), + } +} + +/// `USERNAME` with non-alphanumeric bytes replaced. Reading the env is enough; +/// there is no dependency-free Windows user API in this crate. +#[cfg(windows)] +fn windows_user() -> Option { + let raw = std::env::var("USERNAME").ok()?; + if raw.is_empty() { + return None; + } + let mut cleaned = String::with_capacity(raw.len()); + for ch in raw.chars() { + if ch.is_ascii_alphanumeric() { + cleaned.push(ch); + } else { + cleaned.push('_'); + } + } + let trimmed = cleaned.trim_matches('_'); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } +} + +pub(super) fn accept_loop(listener: Listener, tcp: Option, hub: Arc) { + while !hub.stop.load(Ordering::SeqCst) { + reap(&hub); + let mut busy = false; + match listener.accept() { + Ok(stream) => { + busy = true; + accept_local(stream, &hub); + } + Err(err) if idle_accept(&err) => {} + Err(_) => { + std::thread::sleep(Duration::from_millis(10)); + continue; + } + } + if let Some(bound) = tcp.as_ref() + && let Ok(Some(stream)) = super::tcp::poll_accept(&bound.listener) + { + busy = true; + accept_tcp(stream, &hub); + } + if !busy { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +fn idle_accept(err: &std::io::Error) -> bool { + matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted) +} + +fn reap(hub: &Hub) { + let mut threads = lock(&hub.threads); + let pending = std::mem::take(&mut *threads); + for handle in pending { + if handle.is_finished() { + let _ = handle.join(); + } else { + threads.push(handle); + } + } +} + +/// One connection slot. A connection that has to authenticate also holds a share of the small +/// unauthenticated pool until it does. The slots outside that pool are only for peers that proved +/// the token, so silent peers cannot fill the server. +struct Slot { + hub: Arc, + pending: bool, +} + +/// Why [`Slot::admit`] said no. +enum Refusal { + Full, + Unauthenticated, +} + +impl Refusal { + fn text(&self) -> &'static str { + match self { + Self::Full => "too many control connections", + Self::Unauthenticated => "too many unauthenticated connections", + } + } +} + +/// Add one to `counter` unless it is at `cap`. +fn take_one(counter: &AtomicUsize, cap: usize) -> bool { + loop { + let cur = counter.load(Ordering::SeqCst); + if cur >= cap { + return false; + } + if counter.compare_exchange(cur, cur + 1, Ordering::SeqCst, Ordering::SeqCst).is_ok() { + return true; + } + } +} + +impl Slot { + fn admit(hub: &Arc, needs_auth: bool) -> Result { + if !take_one(&hub.open, wire::MAX_CONNECTIONS) { + return Err(Refusal::Full); + } + if needs_auth && !take_one(&hub.pending, hub.limits.max_unauthenticated.min(wire::MAX_CONNECTIONS)) { + hub.open.fetch_sub(1, Ordering::SeqCst); + return Err(Refusal::Unauthenticated); + } + Ok(Slot { hub: Arc::clone(hub), pending: needs_auth }) + } + + /// The peer sent the right token. It leaves the unauthenticated pool and keeps its slot. + fn authenticated(&mut self) { + if self.pending { + self.pending = false; + self.hub.pending.fetch_sub(1, Ordering::SeqCst); + } + } +} + +impl Drop for Slot { + fn drop(&mut self) { + self.authenticated(); + self.hub.open.fetch_sub(1, Ordering::SeqCst); + } +} + +fn accept_local(mut stream: Stream, hub: &Arc) { + let _ = stream.set_send_timeout(Some(REFUSE_WRITE)); + let slot = match Slot::admit(hub, true) { + Ok(slot) => slot, + Err(refusal) => { + refuse(&mut stream, &refusal); + return; + } + }; + if crate::ensure_peer_user(&stream).is_err() { + let _ = wire::write_line(&mut stream, &wire::auth_failure()); + drop(slot); + return; + } + let _ = stream.set_recv_timeout(Some(READ_POLL)); + let Ok(writer_stream) = stream.try_clone() else { + return; + }; + #[cfg(windows)] + let (reader, writer): (Box, Box) = { + let (Ok(reader), Ok(writer)) = (crate::client::PipeTransport::new(stream), crate::client::PipeTransport::new(writer_stream)) else { + return; + }; + reader.set_timeouts(READ_POLL, REFUSE_WRITE); + writer.set_timeouts(READ_POLL, REFUSE_WRITE); + (Box::new(reader), Box::new(writer)) + }; + #[cfg(not(windows))] + let (reader, writer): (Box, Box) = (Box::new(stream), Box::new(writer_stream)); + spawn(slot, Arc::clone(hub), BufReader::new(reader), Arc::new(std::sync::Mutex::new(writer)), true); +} + +fn accept_tcp(mut stream: std::net::TcpStream, hub: &Arc) { + let _ = stream.set_write_timeout(Some(REFUSE_WRITE)); + let slot = match Slot::admit(hub, true) { + Ok(slot) => slot, + Err(refusal) => { + refuse(&mut stream, &refusal); + return; + } + }; + // The listener is bound to 127.0.0.1. Refuse anything else anyway. + if !stream.peer_addr().is_ok_and(|addr| addr.ip().is_loopback()) { + let _ = wire::write_line(&mut stream, &wire::auth_failure()); + return; + } + let _ = stream.set_read_timeout(Some(READ_POLL)); + let Ok(writer_stream) = stream.try_clone() else { + return; + }; + let reader: Box = Box::new(stream); + let writer: Box = Box::new(writer_stream); + spawn(slot, Arc::clone(hub), BufReader::new(reader), Arc::new(std::sync::Mutex::new(writer)), true); +} + +fn refuse(stream: &mut impl Write, refusal: &Refusal) { + let line = serde_json::json!({"ok": false, "error": refusal.text()}); + let _ = wire::write_line(stream, &line); +} + +fn spawn(slot: Slot, hub: Arc, reader: BufReader>, writer: Arc>>, auth: bool) { + let thread_hub = Arc::clone(&hub); + let started = std::thread::Builder::new().name("craft-ipc-conn".into()).spawn(move || { + let mut slot = slot; + connection(thread_hub, reader, writer, auth, &mut slot); + }); + if let Ok(handle) = started { + lock(&hub.threads).push(handle); + } +} + +struct Cleanup { + hub: Arc, + conn_id: u64, + sub: Arc, + writer: Option>, +} + +impl Drop for Cleanup { + fn drop(&mut self) { + events::close_sub(&self.sub); + if let Some(handle) = self.writer.take() { + let _ = handle.join(); + } + events::remove_sub(&self.hub, self.conn_id); + dispatch::release_tickets(&self.hub, self.conn_id); + } +} + +fn connection( + hub: Arc, + mut reader: BufReader>, + writer: Arc>>, + auth: bool, + slot: &mut Slot, +) { + let first_patience = if auth { hub.limits.partial_line.min(hub.limits.auth_deadline) } else { hub.limits.partial_line }; + let mut lines = LineReader::new(MAX_LINE).with_patience(first_patience); + if auth && authenticate(&hub, &mut reader, &mut lines, &writer).is_err() { + return; + } + slot.authenticated(); + lines.set_patience(hub.limits.partial_line); + let conn_id = hub.next_id.fetch_add(1, Ordering::SeqCst); + let sub = Subscription::new(conn_id); + lock(&hub.subs).push(Arc::clone(&sub)); + let write_sub = Arc::clone(&sub); + let write_io = Arc::clone(&writer); + let write_stop = Arc::clone(&hub); + let writer_thread = std::thread::Builder::new().name("craft-ipc-write".into()).spawn(move || { + events::write_loop(write_sub, write_io, &write_stop.stop); + }); + let Ok(writer_thread) = writer_thread else { + events::remove_sub(&hub, conn_id); + return; + }; + let mut cleanup = Cleanup { hub: Arc::clone(&hub), conn_id, sub: Arc::clone(&sub), writer: Some(writer_thread) }; + let mut first = true; + let mut last_input = Instant::now(); + loop { + if hub.stop.load(Ordering::SeqCst) { + break; + } + let line = match next_line(&hub, &mut reader, &mut lines) { + Incoming::Again => { + if lines.partial_age().is_some_and(|age| age >= hub.limits.partial_line) { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, "request line not finished in time; closing the connection")); + break; + } + if lines.partial_age().is_none() && last_input.elapsed() >= hub.limits.idle && !events::is_subscribed(&sub) { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, "idle connection closed")); + break; + } + continue; + } + Incoming::Stop => break, + Incoming::TooLarge => { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, &overlong())); + break; + } + Incoming::Line(bytes) => bytes, + }; + last_input = Instant::now(); + let req = match wire::parse_request(&line) { + Ok(None) => continue, + Ok(Some(req)) => req, + Err(text) => { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, &text)); + break; + } + }; + let outcome = dispatch::dispatch(&hub, conn_id, &sub, first, &req); + first = false; + match outcome { + Outcome::Continue(reply) => { + if write_value(&writer, &reply).is_err() { + break; + } + } + Outcome::Close(reply) => { + if let Some(reply) = reply { + let _ = write_value(&writer, &reply); + } + break; + } + Outcome::Bulk { reply, ticket } => { + events::close_sub(&sub); + if let Some(handle) = cleanup.writer.take() { + let _ = handle.join(); + } + if write_value(&writer, &reply).is_err() { + break; + } + bulk_loop(&mut reader, &writer, &hub, &ticket); + break; + } + } + } + drop(cleanup); +} + +fn authenticate( + hub: &Hub, + reader: &mut BufReader>, + lines: &mut LineReader, + writer: &Arc>>, +) -> Result<(), ()> { + let deadline = Instant::now() + hub.limits.auth_deadline; + loop { + if hub.stop.load(Ordering::SeqCst) { + return Err(()); + } + let line = match next_line(hub, reader, lines) { + Incoming::Again if Instant::now() >= deadline => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Incoming::Again => continue, + Incoming::Stop | Incoming::TooLarge => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + // A line that completes after the deadline is too late, even when the read poll has not run since. + Incoming::Line(_) if Instant::now() >= deadline => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Incoming::Line(bytes) => bytes, + }; + match wire::parse_request(&line) { + Ok(None) if Instant::now() < deadline => continue, + Ok(None) => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Ok(Some(req)) if req.method == "auth" && token_ok(&req.params, &hub.record.token) => { + return write_value(writer, &wire::auth_success(&req.id)).map_err(|_| ()); + } + Ok(Some(_)) | Err(_) => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + } + } +} + +fn token_ok(params: &Value, expected: &str) -> bool { + params.get("token").and_then(Value::as_str).is_some_and(|token| crate::tokens_equal(expected, token)) +} + +enum Incoming { + Line(Vec), + Again, + TooLarge, + Stop, +} + +fn next_line(hub: &Hub, reader: &mut BufReader>, lines: &mut LineReader) -> Incoming { + match lines.read(reader) { + Ok(Some(bytes)) => Incoming::Line(bytes), + Ok(None) => Incoming::Stop, + Err(LinkError::TooLarge) => Incoming::TooLarge, + Err(err) if is_timeout(&err) => { + if hub.stop.load(Ordering::SeqCst) { + Incoming::Stop + } else { + Incoming::Again + } + } + Err(_) => Incoming::Stop, + } +} + +fn is_timeout(err: &LinkError) -> bool { + if matches!(err, LinkError::Timeout) { + return true; + } + let LinkError::Io(msg) = err else { + return false; + }; + let lower = msg.to_ascii_lowercase(); + lower.contains("timed out") || lower.contains("timeout") +} + +fn overlong() -> String { + format!("request line longer than {MAX_LINE} bytes; closing the connection") +} + +fn write_value(writer: &Arc>>, value: &Value) -> Result<(), LinkError> { + let mut guard = lock(writer); + wire::write_reply_line(&mut *guard, value) +} + +/// Retries reads that time out until `deadline`, so a record split across the read poll is whole. +struct Patient<'a, R> { + inner: &'a mut R, + deadline: Instant, + stop: &'a std::sync::atomic::AtomicBool, +} + +impl Read for Patient<'_, R> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + loop { + match self.inner.read(out) { + Err(err) if idle_accept(&err) && Instant::now() < self.deadline && !self.stop.load(Ordering::SeqCst) => {} + other => return other, + } + } + } +} + +fn bulk_loop(reader: &mut BufReader>, writer: &Arc>>, hub: &Hub, ticket: &str) { + loop { + if hub.stop.load(Ordering::SeqCst) { + return; + } + // Wait for the first byte of a request. A timeout here loses nothing. + match reader.fill_buf() { + Ok([]) => return, + Ok(_) => {} + Err(err) if idle_accept(&err) => continue, + Err(_) => return, + } + // The record has started. A pause inside it is not a new request, so keep + // reading until it ends, the server stops, or the peer takes too long. + let mut patient = Patient { inner: &mut *reader, deadline: Instant::now() + hub.limits.partial_line, stop: &hub.stop }; + let header = match wire::read_cfrq(&mut patient) { + Ok(header) => header, + Err(_) => return, + }; + match hub.frames.render(ticket, &header) { + Ok((hdr, payload)) => { + if send_frame(writer, &header, &hdr, &payload).is_err() { + return; + } + } + Err(err) => { + let fail = error_header(&header, &err.to_string()); + if write_frame(writer, &fail, &[]).is_err() { + return; + } + } + } + } +} + +/// Send one frame record. A record that cannot be built (bad header, payload over the limit) has +/// written nothing, so an error record takes its place. A failure while writing may leave part of a +/// record on the stream, and a second record would read as its tail: that error is returned alone +/// and the caller closes the connection. +fn send_frame(writer: &Arc>>, request: &Value, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + if let Err(err) = wire::check_cfrm(header, payload) { + let fail = error_header(request, &err.to_string()); + write_frame(writer, &fail, &[])?; + return Err(err); + } + write_frame(writer, header, payload) +} + +fn error_header(request: &Value, message: &str) -> Value { + let req = request.get("req").cloned().unwrap_or(Value::Null); + serde_json::json!({"req": req, "error": message}) +} + +fn write_frame(writer: &Arc>>, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + let mut guard = lock(writer); + wire::write_cfrm(&mut *guard, header, payload) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::sync::Mutex; + + use serde_json::json; + + use super::*; + + /// Takes `room` bytes, fails one write, then takes everything: a socket whose send timeout + /// fired mid-record and whose peer then drained the buffer. + struct Choked { + seen: Arc>>, + room: usize, + failed: bool, + } + + impl Write for Choked { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.room == 0 && !self.failed { + self.failed = true; + self.room = usize::MAX; + return Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "choked")); + } + let take = buf.len().min(self.room); + self.seen.lock().unwrap().extend_from_slice(&buf[..take]); + self.room = self.room.saturating_sub(take); + Ok(take) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + type Writer = Arc>>; + + fn rig(room: usize) -> (Writer, Arc>>) { + let seen = Arc::new(Mutex::new(Vec::new())); + let writer: Box = Box::new(Choked { seen: Arc::clone(&seen), room, failed: false }); + (Arc::new(Mutex::new(writer)), seen) + } + + #[test] + fn a_failed_write_is_not_followed_by_an_error_record() { + let (writer, seen) = rig(10); + let request = json!({"req": 7}); + let header = json!({"req": 7, "w": 1}); + let payload = vec![1u8; 64]; + assert!(send_frame(&writer, &request, &header, &payload).is_err()); + let mut whole = Vec::new(); + wire::write_cfrm(&mut whole, &header, &payload).unwrap(); + assert_eq!(*seen.lock().unwrap(), whole[..10].to_vec(), "only the bytes of the first record, and no error record after them"); + } + + #[test] + fn a_record_that_cannot_be_built_gets_an_error_record() { + let (writer, seen) = rig(usize::MAX); + let request = json!({"req": 7}); + // A header that is not an object is refused before any byte is written. + assert!(matches!(send_frame(&writer, &request, &json!("no"), b"x"), Err(LinkError::BadParams(_)))); + let written = seen.lock().unwrap().clone(); + let (header, payload) = wire::read_cfrm(&mut std::io::Cursor::new(written)).unwrap(); + assert_eq!(header["req"], json!(7)); + assert!(header["error"].as_str().is_some_and(|text| text.contains("object")), "{header}"); + assert!(payload.is_empty()); + } +} diff --git a/crates/ipc/src/server/mod.rs b/crates/ipc/src/server/mod.rs new file mode 100644 index 000000000..7b28543a0 --- /dev/null +++ b/crates/ipc/src/server/mod.rs @@ -0,0 +1,376 @@ +//! Craft Link server. +//! +//! One accept thread, then a read thread and a write thread per connection. +//! There is no tokio runtime here. +//! +//! Shutdown. [`Server::shutdown`] and drop set the stop flag, then join the accept thread and every +//! connection thread. A connection thread leaves its loop within one read poll (250 ms) of the +//! flag. Calls into app code differ in what they can delay that: +//! +//! - [`ControlSink`], [`Assets`] and [`Activate`] calls never delay it for longer than a stop poll +//! (at most 200 ms). The server waits for a sink reply, or for an `Assets` or `Activate` answer, in +//! short slices and gives up when the flag is set. An `Assets` or `Activate` call runs on a helper +//! thread that the server detaches when it gives up, so a provider that never returns leaves one +//! idle thread behind and delays nothing. See [`Limits::provider_deadline`]. +//! - [`Frames`] calls run on the connection thread. The flag is checked before each request, but a +//! call that is already running is waited for. A `Frames` method that never returns blocks shutdown. + +mod dispatch; +mod events; +mod listener; +mod provider; +mod tcp; + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::Value; + +use crate::asset::{Described, Refreshed, Resolved}; +use crate::discovery::{Discovery, InstanceRecord, RuntimeDir}; +use crate::error::LinkError; +use crate::moniker::Moniker; + +/// App request channel. +/// +/// `submit` returns `None` when the app is gone. The connection then closes. +/// The `Receiver` yields either a result payload or a finished reply object. +/// A payload is wrapped with [`crate::wire::reply_ok`]. An object that already +/// contains `ok` is passed through, and `id` is inserted when it is missing. +/// This signature cannot carry a [`LinkError`]. +/// A sink that never replies hits [`crate::wire::REPLY_DEADLINE_SECS`]. +pub trait ControlSink: Send + Sync { + /// None means the app is gone. The connection must close. + fn submit(&self, method: String, params: Value) -> Option>; +} + +/// What [`Activate::activate`] decided. The server serializes this as JSON +/// `true`, `false`, or the string `"requested"`. It does not rewrite the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Activation { + /// The window has focus. JSON `true`. + Yes, + /// Focus was not taken. The app asked for user attention instead. JSON `false`. + No, + /// The app asked the OS to focus the window. The OS decides later and + /// nobody confirmed it. JSON `"requested"`. + Requested, +} + +/// Raises or focuses the app. There is no default success. +/// +/// A call runs on a helper thread with a time limit of [`Limits::provider_deadline`]. On timeout the +/// caller gets a timeout reply and the server detaches the thread. The thread keeps running until +/// `activate` returns, possibly after the server is gone, so an implementation must not rely on the +/// server being alive when it returns. +pub trait Activate: Send + Sync { + fn activate(&self, token: Option, from_pid: Option, open: Option) -> Activation; +} + +/// One bulk ticket from [`Frames::open`]. `ticket` is 32 lowercase hex characters. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameOpen { + pub ticket: String, + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +/// Frame tickets and rendered payloads. The server registers the ticket on the +/// control connection that called `frames.open`. +/// +/// Every method runs on a connection thread, one call per frame, with no helper thread and no +/// time limit. The server checks its stop flag before it reads each request, so shutdown takes +/// effect between frames. It cannot interrupt a call that is running: [`Server::shutdown`] and +/// drop wait for it, and `close_ticket` runs when the connection ends. Each method must return in +/// bounded time, for example by putting a timeout on any I/O it does. +pub trait Frames: Send + Sync { + fn open(&self, params: &serde_json::Map) -> Result; + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError>; + fn close_ticket(&self, ticket: &str); +} + +/// Answers asset questions without a UI frame (`link.resolve`, `asset.describe`, +/// `link.refresh`). Pass one in [`Extensions::assets`]. Without it these three +/// methods go to the [`ControlSink`] like any other link method. +/// +/// Methods do not run on the UI thread, so an implementation must be thread safe +/// and must not wait on the UI thread. +/// +/// A call runs on a helper thread with a time limit of [`Limits::provider_deadline`]. On timeout the +/// caller gets a timeout reply and the server detaches the thread. A call that never returns +/// leaves one thread behind and does not hold up [`Server::shutdown`]. The server keeps at most +/// 32 such threads, and a call past that gets a `busy` error. The thread keeps running until the +/// method returns, possibly after the server is gone, so an implementation must not rely on the +/// server being alive when it returns. +pub trait Assets: Send + Sync { + /// Where the last-known result of `asset` is. `Err(LinkError::Unknown)` when + /// the asset or its result file does not exist. + fn resolve(&self, asset: &Moniker) -> Result; + /// Name, kind and thumbnail of `asset`. + fn describe(&self, asset: &Moniker) -> Result; + /// Re-check the file behind `link_id` and report its state. + fn refresh(&self, link_id: &str) -> Result; +} + +/// Optional parts for [`Server::start_with`]. The default adds nothing. +#[derive(Default)] +pub struct Extensions { + /// Answers asset methods. Adds the `assets` cap. + pub assets: Option>, + /// Extra caps for the instance record and `link.hello`, such as `hub`. + pub caps: Vec, +} + +/// What [`Server::start`] needs. `tcp_port: None` does not bind a TCP socket. +pub struct ServerConfig { + pub app: String, + pub app_version: String, + pub instance: Option, + pub runtime: Option, + pub tcp_port: Option, + pub sink: Arc, + pub activate: Arc, + pub frames: Arc, +} + +/// Time and count limits on connections. `Limits::default()` is what the apps run. +/// Tests shorten the times through [`Server::start_with_limits`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// A new connection must send a valid `auth` line within this time, or it is closed. + pub auth_deadline: Duration, + /// A request line, once started, must end within this time, or the connection is closed. + pub partial_line: Duration, + /// An authenticated control connection that sends nothing for this long is closed. A + /// connection with an event subscription and a bulk connection are exempt. + pub idle: Duration, + /// Connections that have not authenticated yet. At most `MAX_CONNECTIONS` minus this many + /// slots can be held by them. + pub max_unauthenticated: usize, + /// How long an `Assets` or `Activate` call may run before the caller gets a timeout reply and + /// the server detaches the helper thread that runs it. + pub provider_deadline: Duration, +} + +impl Default for Limits { + fn default() -> Self { + Self { + auth_deadline: Duration::from_secs(2), + partial_line: Duration::from_secs(10), + idle: Duration::from_secs(60 * 60), + max_unauthenticated: crate::wire::MAX_UNAUTHENTICATED, + provider_deadline: Duration::from_secs(crate::wire::REPLY_DEADLINE_SECS), + } + } +} + +pub(super) struct Hub { + pub(super) limits: Limits, + pub(super) pending: AtomicUsize, + pub(super) record: InstanceRecord, + pub(super) sink: Arc, + pub(super) activate: Arc, + pub(super) frames: Arc, + pub(super) assets: Option>, + pub(super) stop: AtomicBool, + pub(super) open: AtomicUsize, + /// Live `Assets` and `Activate` helper threads, stuck ones included. See `provider`. + pub(super) providers: Arc, + pub(super) next_id: AtomicU64, + pub(super) tickets: Mutex>, + pub(in crate::server) subs: Mutex>>, + pub(super) threads: Mutex>>, +} + +/// A running link endpoint. [`Server::shutdown`] removes the instance record. +/// Drop only joins the accept thread and the connection threads. A helper thread that runs a +/// stuck `Assets` or `Activate` call is detached and not joined. +pub struct Server { + discovery: Option, + hub: Arc, + accept: Option>, + tcp_port: Option, + record: InstanceRecord, +} + +impl Server { + /// Bind, publish, then accept. Scan runs before the bind so a dead socket + /// from an earlier process is unlinked. A lock that is still held is left + /// alone. The instance record keeps the local socket as its endpoint. When `tcp_port` is set the + /// record also carries the bound port (`record.tcp_port()`), and the TCP listener requires the + /// same `auth` token line as the socket. + pub fn start(config: ServerConfig) -> Result { + Self::start_inner(config, Extensions::default(), Limits::default()) + } + + /// Like [`Server::start`], plus the optional parts in [`Extensions`]. + pub fn start_with(config: ServerConfig, extensions: Extensions) -> Result { + Self::start_inner(config, extensions, Limits::default()) + } + + /// Like [`Server::start_with`], with connection limits other than the defaults. + pub fn start_with_limits(config: ServerConfig, extensions: Extensions, limits: Limits) -> Result { + Self::start_inner(config, extensions, limits) + } + + fn start_inner(config: ServerConfig, extensions: Extensions, limits: Limits) -> Result { + let runtime = match config.runtime { + Some(dir) => dir, + None => RuntimeDir::resolve()?, + }; + let _existing = Discovery::scan(&runtime)?; + let pid = std::process::id(); + #[cfg(unix)] + let socket_path = runtime.socket_file(&config.app, pid); + let listener = listener::bind(&runtime, &config.app, pid)?; + #[cfg(unix)] + if let Err(err) = crate::discovery::enforce_mode(&socket_path, 0o600) { + drop(listener); + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + let endpoint = listener::endpoint(&runtime, &config.app, pid)?; + let mut record = match InstanceRecord::new(&config.app, &config.app_version, pid, endpoint) { + Ok(record) => record, + Err(err) => { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + }; + let Extensions { assets, caps } = extensions; + let mut wanted = caps; + if assets.is_some() { + wanted.push("assets".to_string()); + } + for cap in &wanted { + if let Err(err) = record.add_cap(cap) { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + } + if let Some(instance) = config.instance { + record.instance = instance; + if let Err(err) = record.validate() { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + } + // Bind TCP before publishing so the record carries the real port (port 0 picks one). + let tcp = match config.tcp_port { + Some(port) => match tcp::bind_loopback(port) { + Ok(bound) => Some(bound), + Err(err) => { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + }, + None => None, + }; + let tcp_port = tcp.as_ref().map(|bound| bound.port); + if let Some(port) = tcp_port { + record.set_tcp_port(port); + } + let discovery = match Discovery::publish(&runtime, record.clone()) { + Ok(discovery) => discovery, + Err(err) => { + drop(listener); + return Err(err); + } + }; + let hub = Arc::new(Hub { + limits, + pending: AtomicUsize::new(0), + record: record.clone(), + sink: config.sink, + activate: config.activate, + frames: config.frames, + assets, + stop: AtomicBool::new(false), + open: AtomicUsize::new(0), + providers: Arc::new(AtomicUsize::new(0)), + next_id: AtomicU64::new(1), + tickets: Mutex::new(HashMap::new()), + subs: Mutex::new(Vec::new()), + threads: Mutex::new(Vec::new()), + }); + let accept_hub = Arc::clone(&hub); + let accept = match std::thread::Builder::new().name("craft-ipc-accept".into()).spawn(move || { + listener::accept_loop(listener, tcp, accept_hub); + }) { + Ok(handle) => handle, + Err(err) => { + let _ = discovery.remove(); + return Err(err.into()); + } + }; + Ok(Self { discovery: Some(discovery), hub, accept: Some(accept), tcp_port, record }) + } + + pub fn record(&self) -> &InstanceRecord { + &self.record + } + + /// Queue `event` on subscribed connections. This returns after the enqueue, + /// not after the socket write. A full per-connection queue records one + /// `events.overflow` and then drops until that overflow line is written. + pub fn emit(&self, event: &str, data: Value) -> Result<(), LinkError> { + let subs = lock(&self.hub.subs); + for sub in subs.iter() { + events::enqueue(sub, event, &data); + } + Ok(()) + } + + /// Stop accept, join connection threads, and delete the instance record. A stuck `Assets` or + /// `Activate` call does not delay this. A running `Frames` call does. + pub fn shutdown(mut self) -> Result<(), LinkError> { + self.stop_threads(); + match self.discovery.take() { + Some(discovery) => discovery.remove(), + None => Ok(()), + } + } + + /// Bound loopback port, or `None` when TCP was not requested. + pub fn tcp_port(&self) -> Option { + self.tcp_port + } + + fn stop_threads(&mut self) { + self.hub.stop.store(true, Ordering::SeqCst); + events::wake_all(&self.hub.subs); + if let Some(accept) = self.accept.take() { + let _ = accept.join(); + } + let handles = std::mem::take(&mut *lock(&self.hub.threads)); + for handle in handles { + let _ = handle.join(); + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop_threads(); + } +} + +pub(super) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(err) => err.into_inner(), + } +} diff --git a/crates/ipc/src/server/provider.rs b/crates/ipc/src/server/provider.rs new file mode 100644 index 000000000..f74a62923 --- /dev/null +++ b/crates/ipc/src/server/provider.rs @@ -0,0 +1,71 @@ +//! Calls into app code that must not hold up shutdown. +//! +//! `Assets` and `Activate` are control plane calls, rare and slow enough to pay for a thread. Each +//! call runs on a helper thread while the connection thread waits for the answer, for at most +//! [`super::Limits::provider_deadline`] and not past the stop flag. A call that does not finish in +//! time gets a timeout reply, and the helper thread is detached: nobody joins it, so +//! [`super::Server::shutdown`] and drop do not wait for it. At most [`MAX_HELPERS`] helper +//! threads are alive at once. A new call past that gets a `busy` error until a stuck one returns. +//! +//! `Frames` is not run here. It is the hot path with one call per frame, so it stays on the +//! connection thread (see [`super::Frames`]). + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use super::Hub; +use crate::error::LinkError; +use crate::wire::MAX_CONNECTIONS; + +/// Most provider helper threads alive at once, running or stuck. Every connection can have one +/// call in flight, so this leaves room for the same number of stuck ones. +pub(super) const MAX_HELPERS: usize = 2 * MAX_CONNECTIONS; + +/// How often a waiting connection thread looks at the stop flag. +const STOP_POLL: Duration = Duration::from_millis(100); + +/// Counts one live helper thread. Dropping it, on any path, frees the slot. +struct Slot(Arc); + +impl Drop for Slot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Run `run` on a helper thread and wait for its result. +/// +/// `Err(Timeout)` when it takes longer than the deadline, `Err(Closed)` when the server stops +/// first, `Err(Busy)` when [`MAX_HELPERS`] helpers are already alive, and `Err(Io)` when the +/// helper could not start or panicked. +pub(super) fn call(hub: &Hub, what: &str, run: impl FnOnce() -> T + Send + 'static) -> Result { + if hub.providers.fetch_add(1, Ordering::SeqCst) >= MAX_HELPERS { + hub.providers.fetch_sub(1, Ordering::SeqCst); + return Err(LinkError::Busy(format!("{what}: too many provider calls are stuck"))); + } + let slot = Slot(Arc::clone(&hub.providers)); + let (tx, rx) = mpsc::channel(); + let started = std::thread::Builder::new().name("craft-ipc-provider".into()).spawn(move || { + let _slot = slot; + let _ = tx.send(run()); + }); + if let Err(err) = started { + return Err(LinkError::Io(format!("{what}: cannot start a helper thread: {err}"))); + } + let deadline = Instant::now() + hub.limits.provider_deadline; + loop { + if hub.stop.load(Ordering::SeqCst) { + return Err(LinkError::Closed); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(LinkError::Timeout); + } + match rx.recv_timeout(left.min(STOP_POLL)) { + Ok(value) => return Ok(value), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => return Err(LinkError::Io(format!("{what}: the call ended without an answer"))), + } + } +} diff --git a/crates/ipc/src/server/tcp.rs b/crates/ipc/src/server/tcp.rs new file mode 100644 index 000000000..6c1ae386b --- /dev/null +++ b/crates/ipc/src/server/tcp.rs @@ -0,0 +1,32 @@ +//! Optional loopback TCP. Off unless [`crate::server::ServerConfig::tcp_port`] is set. +//! It binds 127.0.0.1 only and runs the same token handshake, unauthenticated cap and +//! deadlines as the local socket. The 16-connection cap is shared +//! with the local listener. The instance record (0600) carries the port and the token. + +use std::net::{TcpListener, TcpStream}; + +use crate::error::LinkError; + +pub(super) struct BoundTcp { + pub listener: TcpListener, + pub port: u16, +} + +pub(super) fn bind_loopback(port: u16) -> Result { + let listener = TcpListener::bind(("127.0.0.1", port))?; + listener.set_nonblocking(true)?; + let port = listener.local_addr()?.port(); + Ok(BoundTcp { listener, port }) +} + +pub(super) fn poll_accept(listener: &TcpListener) -> std::io::Result> { + match listener.accept() { + Ok((stream, _addr)) => Ok(Some(stream)), + Err(err) if is_idle(&err) => Ok(None), + Err(err) => Err(err), + } +} + +fn is_idle(err: &std::io::Error) -> bool { + matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted) +} diff --git a/crates/ipc/src/trust.rs b/crates/ipc/src/trust.rs new file mode 100644 index 000000000..7f57f6d5b --- /dev/null +++ b/crates/ipc/src/trust.rs @@ -0,0 +1,215 @@ +//! Checks before this crate runs a program. +//! +//! The launcher finds an app binary through `CRAFT__BIN`, `installed.json`, `PATH` and a few +//! fixed directories. A same-user process can write most of those places, so a name that resolves +//! is not proof that the file is the app. These checks do not stop a process of the same user, +//! which can do anything the user can. They stop two mistakes: running a file that any local +//! user can replace, and running a file that a relative path or an empty `PATH` entry picked from +//! the current directory. +//! +//! Group write is not checked. A file or directory that is group-writable passes, so every other +//! member of its group can replace the program. The check accepts that because most Linux +//! distributions give each user a private group with no other members, and the check cannot see +//! who is in a group. + +use std::path::Path; + +use crate::error::LinkError; + +/// `Ok` when `path` is an absolute path to a regular, executable file, owned by this user or root +/// and not writable by every user, in a directory with the same owner and mode rules. +/// +/// Symlinks are followed, and the ancestors of both the supplied and resolved paths are checked. +/// A sticky, trusted-owner directory may appear above the immediate directory: it cannot let +/// another user replace the next trusted-owner path component (as with a private folder in /tmp). +/// A group-writable file or directory passes, and the other members of its group can replace it. +/// Most Linux distributions give each user a private group, and the check cannot see group +/// members. A file or directory that every user can write fails. +/// +/// Non-Unix targets only check that the path is an absolute regular file. The Windows ACL on the +/// install directory is the user's check there. +pub fn check_executable(path: &Path) -> Result<(), LinkError> { + if !path.is_absolute() { + return Err(refused(path, "path is not absolute")); + } + #[cfg(unix)] + { + unix::check(path) + } + #[cfg(not(unix))] + { + if std::fs::metadata(path).is_ok_and(|meta| meta.is_file()) { Ok(()) } else { Err(refused(path, "not a file")) } + } +} + +fn refused(path: &Path, reason: &str) -> LinkError { + LinkError::Refused { path: path.display().to_string(), reason: reason.to_string() } +} + +#[cfg(unix)] +mod unix { + use std::os::unix::fs::MetadataExt; + use std::path::Path; + + use super::refused; + use crate::error::LinkError; + + const WORLD_WRITE: u32 = 0o002; + const ANY_EXEC: u32 = 0o111; + const STICKY: u32 = 0o1000; + + pub(super) fn check(path: &Path) -> Result<(), LinkError> { + let euid = crate::discovery::current_euid()?; + let meta = std::fs::metadata(path).map_err(|err| refused(path, &format!("cannot stat: {err}")))?; + if !meta.is_file() { + return Err(refused(path, "not a regular file")); + } + if meta.mode() & ANY_EXEC == 0 { + return Err(refused(path, "not executable")); + } + owner_and_mode(path, &meta, euid, "file")?; + let real = std::fs::canonicalize(path).map_err(|err| refused(path, &format!("cannot resolve: {err}")))?; + for candidate in [path, real.as_path()] { + // Check each named component as well as its resolved directory. In particular a + // foreign-owned symlink in a sticky ancestor can be replaced by that symlink's owner. + for component in candidate.ancestors() { + let named = std::fs::symlink_metadata(component).map_err(|err| refused(component, &format!("cannot stat: {err}")))?; + if named.file_type().is_symlink() && named.uid() != euid && named.uid() != 0 { + return Err(refused(component, "symlink is owned by another user")); + } + } + for (depth, parent) in candidate.ancestors().skip(1).enumerate() { + let dir = std::fs::metadata(parent).map_err(|err| refused(parent, &format!("cannot stat: {err}")))?; + // Sticky ancestors protect the trusted-owned child checked on the preceding + // iteration. The immediate directory still must not be world-writable. + if depth > 0 && dir.mode() & STICKY != 0 { + if dir.uid() != euid && dir.uid() != 0 { + return Err(refused(parent, "sticky ancestor is owned by another user")); + } + } else { + owner_and_mode(parent, &dir, euid, "directory")?; + } + } + } + Ok(()) + } + + fn owner_and_mode(path: &Path, meta: &std::fs::Metadata, euid: u32, what: &str) -> Result<(), LinkError> { + if meta.uid() != euid && meta.uid() != 0 { + return Err(refused(path, &format!("{what} is owned by uid {}, not uid {euid} or root", meta.uid()))); + } + if meta.mode() & WORLD_WRITE != 0 { + return Err(refused(path, &format!("{what} is writable by every user"))); + } + Ok(()) + } +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::os::unix::fs::PermissionsExt; + use std::path::PathBuf; + + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-trust-{label}-{}", crate::auth::encode_hex(&bytes))); + std::fs::create_dir_all(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + path + } + + fn program(dir: &Path, mode: u32) -> PathBuf { + let file = dir.join("app"); + std::fs::write(&file, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(mode)).unwrap(); + file + } + + #[test] + fn a_private_executable_passes() { + let dir = scratch("ok"); + assert!(check_executable(&program(&dir, 0o755)).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn relative_and_empty_paths_fail() { + assert!(matches!(check_executable(Path::new("app")), Err(LinkError::Refused { .. }))); + assert!(matches!(check_executable(Path::new("")), Err(LinkError::Refused { .. }))); + assert!(matches!(check_executable(Path::new("./app")), Err(LinkError::Refused { .. }))); + } + + #[test] + fn loose_modes_and_non_programs_fail() { + let dir = scratch("modes"); + let plain = program(&dir, 0o644); + assert!(check_executable(&plain).is_err(), "not executable"); + let open = program(&dir, 0o777); + assert!(check_executable(&open).is_err(), "world-writable file"); + std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&open).is_ok()); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&open).is_err(), "world-writable directory"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o1777)).unwrap(); + assert!(check_executable(&open).is_err(), "sticky does not excuse a world-writable directory"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&dir).is_err(), "a directory is not a program"); + assert!(check_executable(&dir.join("missing")).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_symlink_into_a_loose_directory_fails() { + let good = scratch("linkgood"); + let loose = scratch("linkloose"); + let target = program(&loose, 0o755); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o777)).unwrap(); + let link = good.join("app"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + assert!(check_executable(&link).is_err()); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&link).is_ok()); + std::fs::remove_dir_all(&good).unwrap(); + std::fs::remove_dir_all(&loose).unwrap(); + } + + #[test] + fn system_programs_pass() { + assert!(check_executable(Path::new("/usr/bin/env")).is_ok() || check_executable(Path::new("/bin/sh")).is_ok()); + } + + #[test] + fn writable_ancestor_cannot_replace_a_private_program_directory() { + let ancestor = scratch("ancestor"); + let bin = ancestor.join("install/bin"); + std::fs::create_dir_all(&bin).unwrap(); + let app = program(&bin, 0o755); + assert!(check_executable(&app).is_ok()); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&app).is_err(), "a safe immediate parent does not protect its ancestors"); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o1777)).unwrap(); + assert!(check_executable(&app).is_ok(), "sticky ancestor protects the owned install directory"); + std::fs::remove_dir_all(&ancestor).unwrap(); + } + + #[test] + fn symlink_target_ancestors_are_checked_too() { + let good = scratch("ancestor-link"); + let ancestor = scratch("ancestor-target"); + let bin = ancestor.join("install/bin"); + std::fs::create_dir_all(&bin).unwrap(); + let target = program(&bin, 0o755); + let link = good.join("app"); + std::os::unix::fs::symlink(target, &link).unwrap(); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&link).is_err()); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&link).is_ok()); + std::fs::remove_dir_all(good).unwrap(); + std::fs::remove_dir_all(ancestor).unwrap(); + } +} diff --git a/crates/ipc/src/watch.rs b/crates/ipc/src/watch.rs new file mode 100644 index 000000000..cc765314c --- /dev/null +++ b/crates/ipc/src/watch.rs @@ -0,0 +1,397 @@ +//! Scheduled polling of linked files (`PROTOCOL.md` section 10, rule 4). +//! +//! [`crate::refresh::Tracker`] looks at the disk once per call. A [`Watcher`] +//! owns trackers and calls them on a timer: a shallow poll every +//! [`SHALLOW_INTERVAL`] (1 s) and a deep poll every [`DEEP_INTERVAL`] (30 s). +//! A deep poll replaces the shallow poll that falls on the same tick. Nothing +//! here needs the producer, so it works while the producer is offline. +//! +//! # Time +//! +//! The watcher never reads a clock. A [`Ticker`] blocks until the next tick and +//! returns the time elapsed since it started. Two tickers exist: +//! +//! - [`IntervalTicker`] waits on a condition variable with a timeout. It is the +//! native ticker. It uses no CPU between ticks and wakes at once on close. +//! - [`ManualTicker`] moves only when [`ManualTicker::advance`] is called. Tests +//! use it, so they never sleep. +//! +//! # Threads and shutdown +//! +//! [`Watcher::new`] starts one thread. Dropping the watcher closes the ticker +//! and joins the thread. A poll that is running finishes first. A panic in the +//! notice callback ends the thread, and the panic message goes to stderr. +//! +//! # Notices +//! +//! The callback runs on the watcher thread with no lock held, so it may call +//! back into the watcher. It receives [`Notice::Changed`] when a poll bumped the +//! epoch or moved the state (`current`, `modified`, `missing`), and +//! [`Notice::PollFailed`] when a poll hit a read error (rule 6). A repeated +//! identical error is reported once until a poll succeeds. An app turns +//! `Changed` into a `link.changed` event with the new epoch. +//! +//! `std` only. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use crate::error::LinkError; +use crate::refresh::{Change, FileState, Tracker}; + +/// How often a shallow poll runs (size and mtime, hash only on a difference). +pub const SHALLOW_INTERVAL: Duration = Duration::from_secs(1); +/// How often a deep poll runs (always hash). Covers coarse file system mtime. +pub const DEEP_INTERVAL: Duration = Duration::from_secs(30); + +/// Poll periods. [`Intervals::default`] is the contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Intervals { + pub shallow: Duration, + pub deep: Duration, +} + +impl Default for Intervals { + fn default() -> Self { + Self { shallow: SHALLOW_INTERVAL, deep: DEEP_INTERVAL } + } +} + +impl Intervals { + fn validate(self) -> Result<(), LinkError> { + if self.shallow.is_zero() { + return Err(LinkError::Invalid("the shallow interval must be above zero".into())); + } + if self.deep < self.shallow { + return Err(LinkError::Invalid("the deep interval must not be shorter than the shallow interval".into())); + } + Ok(()) + } +} + +/// A source of ticks for a [`Watcher`]. +pub trait Ticker: Send + Sync { + /// Block until the next tick, then return the time since the ticker started. + /// The value never decreases. Return `None` after [`Ticker::close`]. + fn wait(&self) -> Option; + /// Wake a blocked [`Ticker::wait`] and make every later call return `None`. + fn close(&self); +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// The native ticker: one tick per period on the wall clock. +/// +/// A tick that comes late skips the ticks it missed. It does not fire them in a burst. +#[derive(Debug)] +pub struct IntervalTicker { + start: Instant, + period: Duration, + state: Mutex, + wake: Condvar, +} + +#[derive(Debug)] +struct IntervalState { + /// The number of the next tick, counted from 1. + next: u32, + closed: bool, +} + +impl IntervalTicker { + /// `period` must be above zero. + pub fn new(period: Duration) -> Result { + if period.is_zero() { + return Err(LinkError::Invalid("the tick period must be above zero".into())); + } + Ok(Self { start: Instant::now(), period, state: Mutex::new(IntervalState { next: 1, closed: false }), wake: Condvar::new() }) + } +} + +impl Ticker for IntervalTicker { + fn wait(&self) -> Option { + let mut state = lock(&self.state); + loop { + if state.closed { + return None; + } + let deadline = self.start + self.period.saturating_mul(state.next); + let now = Instant::now(); + if now >= deadline { + let elapsed = now.duration_since(self.start); + let passed = elapsed.as_nanos() / self.period.as_nanos().max(1); + state.next = u32::try_from(passed.saturating_add(1)).unwrap_or(u32::MAX); + return Some(elapsed); + } + state = self.wake.wait_timeout(state, deadline - now).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } + + fn close(&self) { + lock(&self.state).closed = true; + self.wake.notify_all(); + } +} + +/// A ticker that moves only when told. For tests and for hosts with their own clock. +#[derive(Debug, Default)] +pub struct ManualTicker { + state: Mutex, + wake: Condvar, +} + +#[derive(Debug, Default)] +struct ManualState { + now: Duration, + delivered: Duration, + /// The watcher thread is blocked in `wait` and has handled every earlier tick. + idle: bool, + closed: bool, +} + +impl ManualTicker { + pub fn new() -> Self { + Self::default() + } + + /// Move time forward by `by` and wake the watcher. The watcher sees one tick + /// at the new time, however large `by` is. + pub fn advance(&self, by: Duration) { + let mut state = lock(&self.state); + let moved = state.now.saturating_add(by); + if moved > state.now { + state.now = moved; + state.idle = false; + } + drop(state); + self.wake.notify_all(); + } + + /// Block until the watcher has handled every tick so far and waits again. + /// Returns `false` when `limit` passes first or the ticker is closed. The + /// limit only guards a test from hanging. A healthy watcher settles at once. + pub fn settle(&self, limit: Duration) -> bool { + let deadline = Instant::now() + limit; + let mut state = lock(&self.state); + loop { + if state.closed { + return false; + } + if state.idle && state.delivered == state.now { + return true; + } + let now = Instant::now(); + if now >= deadline { + return false; + } + state = self.wake.wait_timeout(state, deadline - now).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } +} + +impl Ticker for ManualTicker { + fn wait(&self) -> Option { + let mut state = lock(&self.state); + state.idle = state.delivered == state.now; + self.wake.notify_all(); + loop { + if state.closed { + return None; + } + if state.now > state.delivered { + state.delivered = state.now; + state.idle = false; + return Some(state.now); + } + state = self.wake.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + + fn close(&self) { + lock(&self.state).closed = true; + self.wake.notify_all(); + } +} + +/// What a [`Watcher`] tells its callback. +#[derive(Debug, Clone, PartialEq)] +pub enum Notice { + /// A poll bumped the epoch or moved the state of `link_id`. + Changed { link_id: String, change: Change, deep: bool }, + /// A poll could not read the file. The state did not change. + PollFailed { link_id: String, error: LinkError }, +} + +/// How many polls the watcher thread has run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PollCounts { + pub shallow: u64, + pub deep: u64, +} + +type Callback = Box; + +struct Entry { + id: String, + state: Mutex, +} + +struct EntryState { + /// `None` after `unwatch`, for a poll that was already under way. + tracker: Option, + /// The state the callback last heard about. + reported: FileState, + last_error: Option, +} + +struct Shared { + entries: Mutex>>, + notify: Callback, + shallow_polls: AtomicU64, + deep_polls: AtomicU64, +} + +/// Polls a set of trackers on a schedule. See the module docs. +pub struct Watcher { + shared: Arc, + ticker: Arc, + thread: Option>, +} + +impl Watcher { + /// Watch on the wall clock with the contract intervals. + pub fn start(notify: impl Fn(Notice) + Send + Sync + 'static) -> Result { + let ticker = Arc::new(IntervalTicker::new(SHALLOW_INTERVAL)?); + Self::new(ticker, Intervals::default(), notify) + } + + /// Watch with a chosen ticker. The ticker should tick at least as often as + /// `intervals.shallow`, or polls run at the ticker rate. + pub fn new(ticker: Arc, intervals: Intervals, notify: impl Fn(Notice) + Send + Sync + 'static) -> Result { + intervals.validate()?; + let shared = Arc::new(Shared { + entries: Mutex::new(Vec::new()), + notify: Box::new(notify), + shallow_polls: AtomicU64::new(0), + deep_polls: AtomicU64::new(0), + }); + let thread = { + let shared = Arc::clone(&shared); + let ticker = Arc::clone(&ticker); + std::thread::Builder::new().name("craft-ipc-watch".into()).spawn(move || run(&shared, ticker.as_ref(), intervals))? + }; + Ok(Self { shared, ticker, thread: Some(thread) }) + } + + /// Start polling `tracker` under `link_id`. A second tracker for the same id is an error. + pub fn watch(&self, link_id: impl Into, tracker: Tracker) -> Result<(), LinkError> { + let id = link_id.into(); + let mut entries = lock(&self.shared.entries); + if entries.iter().any(|entry| entry.id == id) { + return Err(LinkError::Invalid(format!("{id} is already watched"))); + } + let reported = tracker.state(); + entries.push(Arc::new(Entry { id, state: Mutex::new(EntryState { tracker: Some(tracker), reported, last_error: None }) })); + Ok(()) + } + + /// Stop polling `link_id` and hand its tracker back. + pub fn unwatch(&self, link_id: &str) -> Option { + let entry = { + let mut entries = lock(&self.shared.entries); + let at = entries.iter().position(|entry| entry.id == link_id)?; + entries.remove(at) + }; + // The thread may hold a clone of the entry for one more poll, which skips it now. + lock(&entry.state).tracker.take() + } + + /// Run `act` on the tracker of `link_id`: `accept`, `relink`, `adopt_epoch` + /// or a read. `None` when the id is not watched. The callback does not hear + /// about the state that `act` leaves behind. + pub fn with_tracker(&self, link_id: &str, act: impl FnOnce(&mut Tracker) -> R) -> Option { + let entry = lock(&self.shared.entries).iter().find(|entry| entry.id == link_id).cloned()?; + let mut state = lock(&entry.state); + let tracker = state.tracker.as_mut()?; + let out = act(tracker); + let now = tracker.state(); + state.reported = now; + Some(out) + } + + pub fn poll_counts(&self) -> PollCounts { + PollCounts { shallow: self.shared.shallow_polls.load(Ordering::SeqCst), deep: self.shared.deep_polls.load(Ordering::SeqCst) } + } +} + +impl Drop for Watcher { + fn drop(&mut self) { + self.ticker.close(); + if let Some(thread) = self.thread.take() { + // An Err means the callback panicked. The panic message is already on stderr. + let _ = thread.join(); + } + } +} + +/// Next due time on the grid `first, first + step, ...` that is later than `now`. +fn next_after(due: u128, step: u128, now: u128) -> u128 { + if due > now { due } else { due + ((now - due) / step + 1) * step } +} + +fn run(shared: &Shared, ticker: &dyn Ticker, intervals: Intervals) { + let shallow = intervals.shallow.as_nanos(); + let deep = intervals.deep.as_nanos(); + let mut due_shallow = shallow; + let mut due_deep = deep; + while let Some(elapsed) = ticker.wait() { + let now = elapsed.as_nanos(); + if now >= due_deep { + due_deep = next_after(due_deep, deep, now); + due_shallow = next_after(due_shallow, shallow, now); + shared.deep_polls.fetch_add(1, Ordering::SeqCst); + poll_all(shared, true); + } else if now >= due_shallow { + due_shallow = next_after(due_shallow, shallow, now); + shared.shallow_polls.fetch_add(1, Ordering::SeqCst); + poll_all(shared, false); + } + } +} + +fn poll_all(shared: &Shared, deep: bool) { + let entries: Vec> = lock(&shared.entries).clone(); + for entry in entries { + let notice = poll_entry(&entry, deep); + if let Some(notice) = notice { + (shared.notify)(notice); + } + } +} + +fn poll_entry(entry: &Entry, deep: bool) -> Option { + let mut state = lock(&entry.state); + let polled = state.tracker.as_mut()?.poll(deep); + match polled { + Ok(change) => { + state.last_error = None; + if change.bumped || change.state != state.reported { + state.reported = change.state; + Some(Notice::Changed { link_id: entry.id.clone(), change, deep }) + } else { + None + } + } + Err(error) => { + if state.last_error.as_ref() == Some(&error) { + return None; + } + state.last_error = Some(error.clone()); + Some(Notice::PollFailed { link_id: entry.id.clone(), error }) + } + } +} diff --git a/crates/ipc/src/wire.rs b/crates/ipc/src/wire.rs new file mode 100644 index 000000000..f3dc76621 --- /dev/null +++ b/crates/ipc/src/wire.rs @@ -0,0 +1,561 @@ +//! JSON lines and bulk frame records. No sockets live here. + +use std::io::{BufRead, Read, Write}; +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use crate::error::LinkError; + +/// Longest accepted control line, without the newline. Matches the app control servers. +pub const MAX_LINE: usize = 4 * 1024 * 1024; +/// Open connections per server, authenticated or not. Bulk connections count. +pub const MAX_CONNECTIONS: usize = 16; +/// Connections that have not authenticated yet. The other `MAX_CONNECTIONS - this` slots are +/// reserved for authenticated peers. +pub const MAX_UNAUTHENTICATED: usize = 4; +pub const EVENT_QUEUE: usize = 256; +pub const REPLY_DEADLINE_SECS: u64 = 60; +/// `link.open` payload ceiling. The control line cap is the real limit: a longer request line +/// closes the connection. This number is the same, so `OpenPayload` gives `payload_too_large` +/// to a caller that does not come through the line reader. +pub const MAX_LINK_PAYLOAD: u64 = MAX_LINE as u64; +pub const BULK_HEADER_MAX: u32 = 1024 * 1024; +pub const FRAME_PAYLOAD_MAX: u64 = 256 * 1024 * 1024; +pub const PROTOCOL_MIN: u32 = 1; +pub const PROTOCOL_MAX: u32 = 1; +pub const AUTH_ERROR: &str = "authentication required"; + +const CFRQ: &[u8; 4] = b"CFRQ"; +const CFRM: &[u8; 4] = b"CFRM"; + +/// One control request. `id` is JSON null when the caller omitted it. +#[derive(Debug, Clone, PartialEq)] +pub struct Request { + pub id: Value, + pub method: String, + pub params: Value, +} + +/// Read one line of at most `max` bytes. `Ok(None)` is a clean end of stream. +/// +/// This keeps no state between calls. A read timeout drops the bytes read so far, so use it only +/// on a stream that cannot time out. A server loop that polls with a receive timeout uses +/// [`LineReader`]. +pub fn read_line(reader: &mut impl BufRead, max: usize) -> Result>, LinkError> { + LineReader::new(max).read(reader) +} + +/// Reads lines from a stream whose reads can time out. Bytes read before a timeout stay in the +/// reader, so a line that arrives in pieces more than one poll apart is not lost. +#[derive(Debug)] +pub struct LineReader { + buf: Vec, + max: usize, + started: Option, + patience: Option, +} + +impl LineReader { + pub fn new(max: usize) -> Self { + Self { buf: Vec::new(), max, started: None, patience: None } + } + + /// How long the current unfinished line has been open. `None` when no line is in progress. + pub fn partial_age(&self) -> Option { + self.started.map(|start| start.elapsed()) + } + + /// Return `Err(Timeout)` once a line has been open this long, even when bytes keep arriving. + /// Without it a peer that sends one byte per poll holds the connection for ever. + pub fn with_patience(mut self, patience: Duration) -> Self { + self.patience = Some(patience); + self + } + + /// Change the limit that [`LineReader::with_patience`] sets. + pub fn set_patience(&mut self, patience: Duration) { + self.patience = Some(patience); + } + + /// Read the next line. On `Err(Timeout)` the partial line stays here and the next call + /// continues it. `Ok(None)` is a clean end of stream with no partial line. + pub fn read(&mut self, reader: &mut impl BufRead) -> Result>, LinkError> { + loop { + let available = match reader.fill_buf() { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) => return Err(err.into()), + }; + if available.is_empty() { + // End of stream. A final line without a newline is still a line. + if self.buf.is_empty() { + return Ok(None); + } + let line = self.finish(); + if line.len() > self.max { + return Err(LinkError::TooLarge); + } + return Ok(Some(line)); + } + let newline = available.iter().position(|byte| *byte == b'\n'); + let take = newline.map_or(available.len(), |at| at + 1); + if self.buf.len().saturating_add(take) > self.max.saturating_add(1) { + return Err(LinkError::TooLarge); + } + self.buf.extend_from_slice(&available[..take]); + reader.consume(take); + if self.started.is_none() { + self.started = Some(Instant::now()); + } + if newline.is_some() { + let mut line = self.finish(); + line.pop(); + if line.last() == Some(&b'\r') { + line.pop(); + } + if line.len() > self.max { + return Err(LinkError::TooLarge); + } + return Ok(Some(line)); + } + if self.patience.zip(self.partial_age()).is_some_and(|(limit, age)| age >= limit) { + return Err(LinkError::Timeout); + } + } + } + + fn finish(&mut self) -> Vec { + self.started = None; + std::mem::take(&mut self.buf) + } +} + +/// Parse one control line. +/// +/// `Ok(None)` is a blank line (the caller skips it). `Err` is the existing +/// control-server reject text: one reply, then close. +pub fn parse_request(bytes: &[u8]) -> Result, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "not a control request (invalid UTF-8); closing the connection".to_string())?; + let text = text.trim(); + if text.is_empty() { + return Ok(None); + } + let msg: Value = + serde_json::from_str(text).map_err(|err| format!("bad JSON ({err}); expected one JSON request object per line; closing the connection"))?; + let Some(obj) = msg.as_object() else { + return Err("not a control request (expected a JSON object); closing the connection".into()); + }; + let Some(method) = obj.get("method").and_then(Value::as_str) else { + return Err("not a control request (missing string `method`); closing the connection".into()); + }; + if method.is_empty() { + return Err("not a control request (missing string `method`); closing the connection".into()); + } + let id = obj.get("id").cloned().unwrap_or(Value::Null); + if !(id.is_null() || id.is_number() || id.is_string()) { + return Err("not a control request (id must be a number, string, or null); closing the connection".into()); + } + let params = obj.get("params").cloned().unwrap_or_else(|| json!({})); + Ok(Some(Request { id, method: method.to_string(), params })) +} + +pub fn params_object(params: &Value) -> Result<&Map, LinkError> { + params.as_object().ok_or_else(|| LinkError::BadParams("params must be an object".into())) +} + +pub fn auth_failure() -> Value { + json!({"id": 0, "ok": false, "error": AUTH_ERROR}) +} + +pub fn auth_success(id: &Value) -> Value { + let id = if id.is_null() { json!(0) } else { id.clone() }; + json!({"id": id, "ok": true, "result": {"authenticated": true}}) +} + +pub fn reply_ok(id: &Value, result: Value) -> Value { + json!({"id": id, "ok": true, "result": result}) +} + +pub fn reply_err_text(id: &Value, error: &str) -> Value { + json!({"id": id, "ok": false, "error": error}) +} + +pub fn reply_err(id: &Value, err: &LinkError) -> Value { + json!({"id": id, "ok": false, "error": err.to_object()}) +} + +pub fn push_event(event: &str, seq: u64, data: Value) -> Value { + json!({"event": event, "seq": seq, "data": data}) +} + +/// Write a reply line that a peer will read with the [`MAX_LINE`] limit. A value longer than the +/// limit is replaced by a `payload_too_large` reply that keeps the request `id`. When even that +/// does not fit, or the value has no `id` to answer, nothing is written and the result is +/// `Err(TooLarge)`. The limit counts the content without the newline, as the readers do. +pub fn write_reply_line(writer: &mut impl Write, value: &Value) -> Result<(), LinkError> { + let mut bytes = serde_json::to_vec(value)?; + if bytes.len() > MAX_LINE { + let id = value.get("id").ok_or(LinkError::TooLarge)?; + bytes = serde_json::to_vec(&reply_err(id, &LinkError::TooLarge))?; + if bytes.len() > MAX_LINE { + return Err(LinkError::TooLarge); + } + } + bytes.push(b'\n'); + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +pub fn write_line(writer: &mut impl Write, value: &Value) -> Result<(), LinkError> { + let mut bytes = serde_json::to_vec(value)?; + bytes.push(b'\n'); + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +pub fn protocols_overlap(min: u32, max: u32) -> bool { + min <= max && min <= PROTOCOL_MAX && max >= PROTOCOL_MIN +} + +pub fn write_cfrq(writer: &mut impl Write, header: &Value) -> Result<(), LinkError> { + write_magic(writer, CFRQ, header)?; + Ok(()) +} + +pub fn read_cfrq(reader: &mut impl Read) -> Result { + read_magic(reader, CFRQ) +} + +/// The checks that [`write_cfrm`] makes before it writes a byte: the header is an object that fits +/// `BULK_HEADER_MAX`, and the payload fits `FRAME_PAYLOAD_MAX`. A record that fails them has put +/// nothing on the wire, so a caller may still send a different record. An error from `write_cfrm` +/// after this check passed came from the stream and may have left part of a record on it. +pub fn check_cfrm(header: &Value, payload: &[u8]) -> Result<(), LinkError> { + cfrm_parts(header, payload).map(|_| ()) +} + +fn cfrm_parts(header: &Value, payload: &[u8]) -> Result<(Vec, u64), LinkError> { + let header_bytes = header_bytes(header)?; + let payload_len = u64::try_from(payload.len()).map_err(|_| LinkError::TooLarge)?; + if payload_len > FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + Ok((header_bytes, payload_len)) +} + +pub fn write_cfrm(writer: &mut impl Write, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + let (header_bytes, payload_len) = cfrm_parts(header, payload)?; + writer.write_all(CFRM)?; + writer.write_all(&u32_len(&header_bytes)?.to_le_bytes())?; + writer.write_all(&payload_len.to_le_bytes())?; + writer.write_all(&header_bytes)?; + writer.write_all(payload)?; + writer.flush()?; + Ok(()) +} + +pub fn read_cfrm(reader: &mut impl Read) -> Result<(Value, Vec), LinkError> { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if &magic != CFRM { + return Err(LinkError::Invalid("bulk reply magic".into())); + } + let header_len = read_u32(reader)?; + if header_len > BULK_HEADER_MAX { + return Err(LinkError::TooLarge); + } + let payload_len = read_u64(reader)?; + if payload_len > FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + let header = read_header(reader, header_len)?; + let mut payload = vec![0u8; usize::try_from(payload_len).map_err(|_| LinkError::TooLarge)?]; + reader.read_exact(&mut payload)?; + Ok((header, payload)) +} + +fn write_magic(writer: &mut impl Write, magic: &[u8; 4], header: &Value) -> Result<(), LinkError> { + let bytes = header_bytes(header)?; + writer.write_all(magic)?; + writer.write_all(&u32_len(&bytes)?.to_le_bytes())?; + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +fn read_magic(reader: &mut impl Read, expect: &[u8; 4]) -> Result { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if &magic != expect { + return Err(LinkError::Invalid("bulk request magic".into())); + } + let len = read_u32(reader)?; + if len > BULK_HEADER_MAX { + return Err(LinkError::TooLarge); + } + read_header(reader, len) +} + +fn header_bytes(header: &Value) -> Result, LinkError> { + if !header.is_object() { + return Err(LinkError::BadParams("bulk header must be an object".into())); + } + let bytes = serde_json::to_vec(header)?; + if bytes.len() > usize::try_from(BULK_HEADER_MAX).unwrap_or(usize::MAX) { + return Err(LinkError::TooLarge); + } + Ok(bytes) +} + +fn u32_len(bytes: &[u8]) -> Result { + u32::try_from(bytes.len()).map_err(|_| LinkError::TooLarge) +} + +fn read_u32(reader: &mut impl Read) -> Result { + let mut buf = [0u8; 4]; + reader.read_exact(&mut buf)?; + Ok(u32::from_le_bytes(buf)) +} + +fn read_u64(reader: &mut impl Read) -> Result { + let mut buf = [0u8; 8]; + reader.read_exact(&mut buf)?; + Ok(u64::from_le_bytes(buf)) +} + +fn read_header(reader: &mut impl Read, len: u32) -> Result { + let mut buf = vec![0u8; len as usize]; + reader.read_exact(&mut buf)?; + let value: Value = serde_json::from_slice(&buf)?; + if !value.is_object() { + return Err(LinkError::BadParams("bulk header must be an object".into())); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::*; + + #[test] + fn blank_and_request_round_trip() { + assert!(parse_request(b" \n").unwrap().is_none()); + let req = parse_request(br#"{"id":1,"method":"engine.execute","params":{"command":"x"}}"#).unwrap().unwrap(); + assert_eq!(req.method, "engine.execute"); + assert_eq!(req.id, json!(1)); + assert!(parse_request(b"[]").is_err()); + assert!(parse_request(b"{").is_err()); + } + + #[test] + fn a_reply_over_the_line_limit_becomes_a_too_large_error_with_the_same_id() { + let mut out = Vec::new(); + let small = reply_ok(&json!(3), json!({"a": 1})); + write_reply_line(&mut out, &small).unwrap(); + assert_eq!(out, [serde_json::to_vec(&small).unwrap(), b"\n".to_vec()].concat(), "a reply inside the limit is written as it is"); + + // The content is exactly MAX_LINE bytes: still inside. + let overhead = serde_json::to_vec(&reply_ok(&json!(3), json!(""))).unwrap().len(); + let exact = reply_ok(&json!(3), json!("x".repeat(MAX_LINE - overhead))); + assert_eq!(serde_json::to_vec(&exact).unwrap().len(), MAX_LINE); + let mut out = Vec::new(); + write_reply_line(&mut out, &exact).unwrap(); + assert_eq!(out.len(), MAX_LINE + 1); + + let big = reply_ok(&json!(3), json!("x".repeat(MAX_LINE - overhead + 1))); + let mut out = Vec::new(); + write_reply_line(&mut out, &big).unwrap(); + let line = read_line(&mut Cursor::new(out), MAX_LINE).unwrap().unwrap(); + let reply: Value = serde_json::from_slice(&line).unwrap(); + assert_eq!(reply["id"], json!(3)); + assert_eq!(reply["ok"], json!(false)); + assert_eq!(reply["error"]["code"], json!("payload_too_large")); + + // No id to answer: nothing is written. + let mut out = Vec::new(); + let anonymous = json!({"ok": true, "result": "x".repeat(MAX_LINE)}); + assert!(matches!(write_reply_line(&mut out, &anonymous), Err(LinkError::TooLarge))); + assert!(out.is_empty()); + // An id so long that the error reply does not fit either. + let huge_id = json!({"id": "i".repeat(MAX_LINE + 1), "ok": true}); + assert!(matches!(write_reply_line(&mut out, &huge_id), Err(LinkError::TooLarge))); + assert!(out.is_empty()); + } + + #[test] + fn a_last_line_without_a_newline_obeys_the_limit() { + let at_limit = vec![b'a'; 8]; + let line = read_line(&mut Cursor::new(at_limit.clone()), 8).unwrap().unwrap(); + assert_eq!(line, at_limit); + let over = vec![b'a'; 9]; + assert!(matches!(read_line(&mut Cursor::new(over), 8), Err(LinkError::TooLarge))); + // The same bytes with a newline are refused too, so the two paths agree. + let mut with_newline = vec![b'a'; 9]; + with_newline.push(b'\n'); + assert!(matches!(read_line(&mut Cursor::new(with_newline), 8), Err(LinkError::TooLarge))); + } + + #[test] + fn line_limit_and_crlf() { + let mut cur = Cursor::new(b"abc\r\n".to_vec()); + let line = read_line(&mut cur, 8).unwrap().unwrap(); + assert_eq!(line, b"abc"); + let mut huge = vec![b'a'; 20]; + huge.push(b'\n'); + let mut cur = Cursor::new(huge); + assert!(matches!(read_line(&mut cur, 8), Err(LinkError::TooLarge))); + } + + /// A reader that hands out `chunks` and fails with a timeout after each one, like a socket + /// with a receive timeout and a slow peer. + struct Stuttering { + chunks: std::collections::VecDeque>, + pending_timeout: bool, + current: Vec, + pos: usize, + } + + impl Stuttering { + fn new(chunks: Vec>) -> Self { + Self { chunks: chunks.into(), pending_timeout: false, current: Vec::new(), pos: 0 } + } + } + + impl Read for Stuttering { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.pos >= self.current.len() { + if self.pending_timeout { + self.pending_timeout = false; + return Err(std::io::ErrorKind::WouldBlock.into()); + } + let Some(next) = self.chunks.pop_front() else { return Ok(0) }; + self.current = next; + self.pos = 0; + if self.current.is_empty() { + return Err(std::io::ErrorKind::WouldBlock.into()); + } + self.pending_timeout = true; + } + let n = out.len().min(self.current.len() - self.pos); + out[..n].copy_from_slice(&self.current[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } + + fn collect_lines(chunks: Vec>, max: usize) -> Vec> { + let mut reader = std::io::BufReader::with_capacity(4, Stuttering::new(chunks)); + let mut lines = LineReader::new(max); + let mut got = Vec::new(); + loop { + match lines.read(&mut reader) { + Ok(Some(line)) => got.push(line), + Ok(None) => return got, + Err(LinkError::Timeout) => {} + Err(other) => panic!("{other:?}"), + } + } + } + + #[test] + fn a_line_split_at_every_offset_survives_a_timeout() { + let text = b"{\"id\":1,\"method\":\"link.hello\"}\r\n{\"id\":2}\n"; + let want = vec![b"{\"id\":1,\"method\":\"link.hello\"}".to_vec(), b"{\"id\":2}".to_vec()]; + for cut in 0..=text.len() { + let chunks = vec![text[..cut].to_vec(), text[cut..].to_vec()]; + assert_eq!(collect_lines(chunks, 1024), want, "cut {cut}"); + } + // Two cuts, and one byte at a time. + for a in 0..text.len() { + for b in a..text.len() { + let chunks = vec![text[..a].to_vec(), text[a..b].to_vec(), text[b..].to_vec()]; + assert_eq!(collect_lines(chunks, 1024), want, "cuts {a} {b}"); + } + } + let bytes: Vec> = text.iter().map(|byte| vec![*byte]).collect(); + assert_eq!(collect_lines(bytes, 1024), want); + } + + #[test] + fn the_line_cap_holds_across_timeouts() { + let mut reader = std::io::BufReader::new(Stuttering::new(vec![vec![b'a'; 6], vec![b'b'; 6], b"\n".to_vec()])); + let mut lines = LineReader::new(8); + let mut result = None; + for _ in 0..8 { + match lines.read(&mut reader) { + Err(LinkError::Timeout) => {} + other => { + result = Some(other); + break; + } + } + } + assert!(matches!(result, Some(Err(LinkError::TooLarge))), "{result:?}"); + } + + /// One byte per read, a short pause each, and never a newline: a slow-loris peer whose reads + /// never time out. + struct Trickle; + + impl Read for Trickle { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + std::thread::sleep(Duration::from_millis(10)); + out[0] = b'a'; + Ok(1) + } + } + + #[test] + fn a_trickling_line_is_cut_by_patience() { + let mut reader = std::io::BufReader::with_capacity(1, Trickle); + let mut lines = LineReader::new(MAX_LINE).with_patience(Duration::from_millis(100)); + let start = Instant::now(); + assert!(matches!(lines.read(&mut reader), Err(LinkError::Timeout))); + assert!(start.elapsed() < Duration::from_secs(2), "{:?}", start.elapsed()); + assert!(lines.partial_age().is_some_and(|age| age >= Duration::from_millis(100))); + } + + #[test] + fn partial_age_tracks_an_open_line() { + let mut reader = std::io::BufReader::new(Stuttering::new(vec![b"abc".to_vec(), b"d\n".to_vec()])); + let mut lines = LineReader::new(64); + assert!(lines.partial_age().is_none()); + assert!(matches!(lines.read(&mut reader), Err(LinkError::Timeout))); + assert!(lines.partial_age().is_some()); + assert_eq!(lines.read(&mut reader).unwrap().unwrap(), b"abcd"); + assert!(lines.partial_age().is_none()); + } + + #[test] + fn bulk_round_trip() { + let header = json!({"req": 7, "time_s": 1.5, "epoch": 3, "region": null}); + let mut buf = Vec::new(); + write_cfrq(&mut buf, &header).unwrap(); + let got = read_cfrq(&mut Cursor::new(&buf)).unwrap(); + assert_eq!(got["req"], json!(7)); + + let mut buf = Vec::new(); + let reply = json!({"req": 7, "error": "no"}); + write_cfrm(&mut buf, &reply, b"").unwrap(); + let (h, payload) = read_cfrm(&mut Cursor::new(&buf)).unwrap(); + assert_eq!(h["error"], json!("no")); + assert!(payload.is_empty()); + + let mut buf = Vec::new(); + write_cfrm(&mut buf, &json!({"req": 1, "w": 2}), &[9, 8, 7]).unwrap(); + let (_h, payload) = read_cfrm(&mut Cursor::new(buf)).unwrap(); + assert_eq!(payload, vec![9, 8, 7]); + } + + #[test] + fn protocol_window() { + assert!(protocols_overlap(1, 1)); + assert!(!protocols_overlap(2, 2)); + assert!(!protocols_overlap(2, 1)); + } +} diff --git a/crates/ipc/tests/bulk.rs b/crates/ipc/tests/bulk.rs new file mode 100644 index 000000000..8cd8295a2 --- /dev/null +++ b/crates/ipc/tests/bulk.rs @@ -0,0 +1,334 @@ +//! Bulk frames over a real local socket with a fake `FrameSource`. +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use craft_ipc::bulk::{BulkClient, FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; +use craft_ipc::client::Client; +use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; +use craft_ipc::wire; +use craft_ipc::{Endpoint, InstanceRecord, LinkError, RuntimeDir}; + +const W: u32 = 4; +const H: u32 = 3; +const STRIDE: u32 = 16; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +/// Pixels encode the request so a wrong frame cannot match by accident. +struct FakeSource { + epoch: AtomicU64, + closed: Mutex>, +} + +/// One row holds `W` pixels: 4 bytes each in rgba8, 8 bytes each in rgba16f. +fn stride_for(format: FrameFormat) -> u32 { + match format { + FrameFormat::Rgba8Premul => STRIDE, + FrameFormat::Rgba16fPremul => W * 8, + } +} + +fn pixels_for(link_id: &str, time_s: f64, region: Option<[u32; 4]>) -> Vec { + let seed = + link_id.bytes().fold(0u8, |acc, b| acc.wrapping_add(b)).wrapping_add((time_s * 10.0) as u8).wrapping_add(region.map_or(0, |r| r[0] as u8)); + (0..(STRIDE * H) as usize).map(|i| seed.wrapping_add((i as u8).wrapping_mul(7))).collect() +} + +impl FrameSource for FakeSource { + fn open(&self, spec: &FrameSpec) -> Result { + if spec.link_id == "missing" { + return Err(LinkError::NotReady("link_unknown".into())); + } + let stride = stride_for(spec.format); + Ok(FrameLayout { w: W, h: H, stride, len: u64::from(stride * H) }) + } + + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { + let stride = stride_for(spec.format); + let mut pixels = pixels_for(&spec.link_id, request.time_s, request.region); + pixels.resize((stride * H) as usize, 0); + Ok(RenderedFrame { epoch: self.epoch.load(Ordering::SeqCst), w: W, h: H, stride, format: spec.format, render_ms: 0.25, pixels }) + } + + fn close(&self, spec: &FrameSpec) { + self.closed.lock().unwrap().push(spec.link_id.clone()); + } +} + +struct Rig { + _tmp: Tmp, + server: Server, + record: InstanceRecord, + source: Arc, + tickets: Arc, +} + +fn boot(epoch: u64) -> Rig { + // Parallel tests can observe the same clock tick on macOS. + static NEXT_RIG: AtomicU64 = AtomicU64::new(0); + let serial = NEXT_RIG.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-blk-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let source = Arc::new(FakeSource { epoch: AtomicU64::new(epoch), closed: Mutex::new(Vec::new()) }); + let tickets = Arc::new(FrameTickets::new(Arc::clone(&source) as Arc)); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(RuntimeDir::at(&path).unwrap()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: Arc::clone(&tickets) as Arc, + }) + .unwrap(); + let record = server.record().clone(); + Rig { _tmp: Tmp(path), server, record, source, tickets } +} + +fn wait_for(what: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while !done() { + assert!(Instant::now() < deadline, "timed out waiting for {what}"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn socket_of(record: &InstanceRecord) -> PathBuf { + match &record.endpoint { + Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + } +} + +#[test] +fn round_trip_returns_the_exact_bytes() { + let rig = boot(3); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + assert_eq!((ticket.w, ticket.h, ticket.stride, ticket.len), (W, H, STRIDE, u64::from(STRIDE * H))); + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + let whole = bulk.request(1.5, 3, None).unwrap(); + assert_eq!(whole.pixels, pixels_for("link-a", 1.5, None)); + assert_eq!((whole.req, whole.epoch, whole.w, whole.h, whole.stride), (1, 3, W, H, STRIDE)); + assert_eq!(whole.format, FrameFormat::Rgba8Premul); + assert!((whole.render_ms - 0.25).abs() < f64::EPSILON); + let part = bulk.request(2.0, 3, Some([5, 0, 2, 2])).unwrap(); + assert_eq!(part.pixels, pixels_for("link-a", 2.0, Some([5, 0, 2, 2]))); + assert_ne!(part.pixels, whole.pixels); + assert_eq!(part.req, 2); + drop(bulk); + drop(control); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_newer_epoch_is_returned_and_an_older_source_is_an_error() { + let rig = boot(7); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba16fPremul, 2.0).unwrap(); + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + let frame = bulk.request(0.0, 2, None).unwrap(); + assert_eq!(frame.epoch, 7, "asked for epoch 2, source is at 7"); + assert_eq!(frame.format, FrameFormat::Rgba16fPremul); + let err = bulk.request(0.0, 9, None).unwrap_err(); + assert!(err.to_string().contains("epoch 7"), "{err}"); + // The error did not poison the connection. + rig.source.epoch.store(9, Ordering::SeqCst); + assert_eq!(bulk.request(0.0, 9, None).unwrap().epoch, 9); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_ticket_is_refused_after_its_control_connection_closes() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut live = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + live.request(0.0, 1, None).unwrap(); + assert_eq!(rig.tickets.open_tickets(), 1); + drop(control); + wait_for("source.close", || !rig.source.closed.lock().unwrap().is_empty()); + assert_eq!(rig.tickets.open_tickets(), 0); + assert!(live.request(0.0, 1, None).is_err(), "an attached connection must lose its frames"); + let again = BulkClient::attach(&rig.record, &ticket.ticket); + assert!(again.is_err(), "reuse after control close must be refused"); + rig.server.shutdown().unwrap(); +} + +#[test] +fn bulk_attach_must_be_first_and_the_ticket_must_exist() { + let rig = boot(1); + let bogus = "0123456789abcdef0123456789abcdef"; + assert!(BulkClient::attach(&rig.record, bogus).is_err()); + let mut control = Client::connect(&rig.record).unwrap(); + assert!(control.frames_open("missing", FrameFormat::Rgba8Premul, 1.0).is_err()); + assert!(control.frames_open("a", FrameFormat::Rgba8Premul, 0.0).is_err()); + assert_eq!(rig.tickets.open_tickets(), 0); + rig.server.shutdown().unwrap(); +} + +fn raw_attach(rig: &Rig, ticket: &str) -> UnixStream { + let mut stream = UnixStream::connect(socket_of(&rig.record)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + wire::write_line(&mut stream, &json!({"id": 0, "method": "auth", "params": {"token": rig.record.token}})).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &json!({"id": 1, "method": "bulk.attach", "params": {"ticket": ticket}})).unwrap(); + let reply: Value = serde_json::from_slice(&read_line(&mut stream)).unwrap(); + assert_eq!(reply["ok"], json!(true), "{reply}"); + stream +} + +fn read_line(stream: &mut UnixStream) -> Vec { + let mut out = Vec::new(); + let mut byte = [0u8; 1]; + while stream.read_exact(&mut byte).is_ok() && byte[0] != b'\n' { + out.push(byte[0]); + } + out +} + +#[test] +fn an_oversized_request_header_closes_that_connection_only() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut raw = raw_attach(&rig, &ticket.ticket); + raw.write_all(b"CFRQ").unwrap(); + raw.write_all(&(wire::BULK_HEADER_MAX + 1).to_le_bytes()).unwrap(); + let mut rest = Vec::new(); + // The server closes without a reply, so the read ends at EOF instead of timing out. + let n = raw.read_to_end(&mut rest).unwrap(); + assert_eq!(n, 0, "no reply expected for an oversized header"); + // The control connection and a fresh bulk connection still work. + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + assert!(bulk.request(0.0, 1, None).is_ok()); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_bad_magic_and_a_bad_header_do_not_panic_the_server() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut raw = raw_attach(&rig, &ticket.ticket); + raw.write_all(b"XXXX\0\0\0\0").unwrap(); + let mut rest = Vec::new(); + assert_eq!(raw.read_to_end(&mut rest).unwrap(), 0); + // A well-formed record with a bad header gets an error record, not a close. + let mut raw = raw_attach(&rig, &ticket.ticket); + wire::write_cfrq(&mut raw, &json!({"req": 5, "epoch": 0})).unwrap(); + let (header, payload) = wire::read_cfrm(&mut raw).unwrap(); + assert_eq!(header["req"], json!(5)); + assert!(header["error"].as_str().unwrap().contains("time_s"), "{header}"); + assert!(payload.is_empty()); + rig.server.shutdown().unwrap(); +} + +/// A one-shot fake server. Answers auth and attach, then sends `bytes` and closes. +fn fake_server(bytes: Vec) -> (InstanceRecord, std::thread::JoinHandle<()>, Tmp) { + static NEXT_FAKE: AtomicU64 = AtomicU64::new(0); + let serial = NEXT_FAKE.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("cl-fk-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("s.sock"); + let listener = UnixListener::bind(&path).unwrap(); + let record = InstanceRecord::new("fake", "0", std::process::id(), Endpoint::Unix { path: path.to_string_lossy().into_owned() }).unwrap(); + let handle = std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { return }; + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &wire::auth_success(&json!(0))).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &wire::reply_ok(&json!(1), json!({}))).unwrap(); + let mut request = [0u8; 8]; + stream.read_exact(&mut request).unwrap(); + let mut header = vec![0u8; u32::from_le_bytes([request[4], request[5], request[6], request[7]]) as usize]; + stream.read_exact(&mut header).unwrap(); + stream.write_all(&bytes).unwrap(); + }); + (record, handle, Tmp(dir)) +} + +fn cfrm(header: &Value, declared_payload: u64, payload: &[u8]) -> Vec { + let header = serde_json::to_vec(header).unwrap(); + let mut out = b"CFRM".to_vec(); + out.extend((header.len() as u32).to_le_bytes()); + out.extend(declared_payload.to_le_bytes()); + out.extend(header); + out.extend(payload); + out +} + +#[test] +fn truncated_replies_are_errors_not_panics() { + let good_header = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + let full = cfrm(&good_header, 48, &[7u8; 48]); + for cut in [2, 6, 10, 20, full.len() - 1] { + let (record, handle, _tmp) = fake_server(full[..cut].to_vec()); + let mut bulk = BulkClient::attach(&record, &"a".repeat(32)).unwrap(); + let err = bulk.request(0.0, 0, None).unwrap_err(); + assert!(matches!(err, LinkError::Io(_)), "cut {cut}: {err:?}"); + handle.join().unwrap(); + } + // The uncut record is accepted by the same fake, so the cuts above are the cause. + let (record, handle, _tmp) = fake_server(full); + let mut bulk = BulkClient::attach(&record, &"a".repeat(32)).unwrap(); + assert_eq!(bulk.request(0.0, 0, None).unwrap().pixels, vec![7u8; 48]); + handle.join().unwrap(); +} + +#[test] +fn oversized_or_inconsistent_replies_are_rejected_by_the_client() { + let good = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + let mut huge_header = b"CFRM".to_vec(); + huge_header.extend((wire::BULK_HEADER_MAX + 1).to_le_bytes()); + huge_header.extend(0u64.to_le_bytes()); + let huge_payload = cfrm(&good, wire::FRAME_PAYLOAD_MAX + 1, &[]); + let short_payload = cfrm(&good, 47, &[1u8; 47]); + let narrow = cfrm(&json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 1, "format": "rgba8_premul", "render_ms": 1.0}), 3, &[1u8; 3]); + let wrong_req = cfrm(&json!({"req": 99, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}), 48, &[1u8; 48]); + for (name, bytes, want_too_large) in [ + ("header", huge_header, true), + ("payload", huge_payload, true), + ("short", short_payload, false), + ("narrow", narrow, false), + ("req", wrong_req, false), + ] { + let (record, handle, _tmp) = fake_server(bytes); + let mut bulk = BulkClient::attach(&record, &"b".repeat(32)).unwrap(); + let err = bulk.request(0.0, 0, None).unwrap_err(); + assert_eq!(matches!(err, LinkError::TooLarge), want_too_large, "{name}: {err:?}"); + handle.join().unwrap(); + } +} diff --git a/crates/ipc/tests/connection_limits.rs b/crates/ipc/tests/connection_limits.rs new file mode 100644 index 000000000..5c0888340 --- /dev/null +++ b/crates/ipc/tests/connection_limits.rs @@ -0,0 +1,280 @@ +//! Connection limits and partial-line handling. Each test asserts the secure behavior: silent +//! peers cannot fill the server, and a line that arrives in pieces is not lost. +#![cfg(unix)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, Extensions, FrameOpen, Frames, Limits, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; + +struct Sink; +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({})); + Some(rx) + } +} +struct NoActivate; +impl Activate for NoActivate { + fn activate(&self, _t: Option, _p: Option, _o: Option) -> Activation { + Activation::No + } +} +struct NoFrames; +impl Frames for NoFrames { + fn open(&self, _p: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _t: &str, _h: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _t: &str) {} +} + +fn boot_with(label: &str, limits: Limits) -> (Server, PathBuf, String, PathBuf) { + let dir = std::env::temp_dir().join(format!("cl-ipc-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let runtime = RuntimeDir::at(&dir).unwrap(); + let server = Server::start_with_limits( + ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(runtime), + tcp_port: None, + sink: Arc::new(Sink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }, + Extensions::default(), + limits, + ) + .unwrap(); + let socket = match &server.record().endpoint { + craft_ipc::Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("unix endpoint expected, got {other:?}"), + }; + let token = server.record().token.clone(); + (server, socket, token, dir) +} + +fn boot(label: &str) -> (Server, PathBuf, String, PathBuf) { + boot_with(label, Limits::default()) +} + +fn auth_line(token: &str) -> String { + format!("{{\"id\":0,\"method\":\"auth\",\"params\":{{\"token\":\"{token}\"}}}}\n") +} + +const HELLO: &str = "{\"id\":1,\"method\":\"link.hello\",\"params\":{\"protocol\":{\"min\":1,\"max\":1}}}\n"; + +fn arm_read_timeout(stream: &UnixStream, secs: u64) { + let result = stream.set_read_timeout(Some(Duration::from_secs(secs))); + // Darwin refuses socket options after shutdown. The queued denial/EOF can still be read. + #[cfg(target_os = "macos")] + if result.as_ref().is_err_and(|err| err.raw_os_error() == Some(libc::EINVAL)) { + return; + } + result.unwrap(); +} + +fn read_text(stream: &mut UnixStream, secs: u64) -> String { + arm_read_timeout(stream, secs); + let mut text = String::new(); + let _ = stream.read_to_string(&mut text); + text +} + +fn next_line(reader: &mut BufReader) -> String { + let mut line = String::new(); + arm_read_timeout(reader.get_ref(), 5); + reader.read_line(&mut line).unwrap(); + line +} + +/// 16 silent connections must not lock the legitimate client out. At most 4 unauthenticated +/// connections are admitted, the rest are refused at once, and the admitted ones are closed at the +/// auth deadline. A client that authenticated before the flood keeps working. +#[test] +fn silent_connections_cannot_lock_out_clients() { + let (server, socket, token, dir) = boot("slots"); + let early = UnixStream::connect(&socket).unwrap(); + let mut early_writer = early.try_clone().unwrap(); + let mut early_reader = BufReader::new(early); + early_writer.write_all(auth_line(&token).as_bytes()).unwrap(); + assert!(next_line(&mut early_reader).contains("authenticated")); + + let mut hogs = Vec::new(); + for _ in 0..16 { + hogs.push(UnixStream::connect(&socket).unwrap()); + } + // The authenticated client still gets its answer while the flood is open. + early_writer.write_all(HELLO.as_bytes()).unwrap(); + let line = next_line(&mut early_reader); + assert!(line.contains("\"ok\":true"), "authenticated client lost service: {line:?}"); + + // Past the auth deadline every silent connection is closed and a new client is served. + std::thread::sleep(Duration::from_millis(3500)); + for hog in &mut hogs { + let text = read_text(hog, 1); + assert!(!text.contains("\"authenticated\""), "{text:?}"); + } + let mut legit = UnixStream::connect(&socket).unwrap(); + legit.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(legit); + let line = next_line(&mut reader); + assert!(line.contains("authenticated"), "legitimate client was not served: {line:?}"); + drop(hogs); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// the unauthenticated pool is small, and the refusal for an extra peer is immediate. +#[test] +fn unauthenticated_peers_beyond_the_cap_are_refused_at_once() { + let limits = Limits { max_unauthenticated: 2, ..Limits::default() }; + let (server, socket, token, dir) = boot_with("unauth-cap", limits); + let a = UnixStream::connect(&socket).unwrap(); + let b = UnixStream::connect(&socket).unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let mut third = UnixStream::connect(&socket).unwrap(); + let text = read_text(&mut third, 1); + assert!(text.contains("too many unauthenticated connections"), "{text:?}"); + // Authenticating frees an unauthenticated slot. + let mut a = a; + a.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(a); + assert!(next_line(&mut reader).contains("authenticated")); + let mut fourth = UnixStream::connect(&socket).unwrap(); + fourth.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(fourth); + assert!(next_line(&mut reader).contains("authenticated")); + drop(b); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a connection that sends no auth line is closed after the deadline (default 2 s). +#[test] +fn silent_connection_is_closed_at_the_auth_deadline() { + let (server, socket, _token, dir) = boot("deadline"); + let mut s = UnixStream::connect(&socket).unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut s, 6); + let waited = start.elapsed(); + assert!(text.contains("authentication required"), "{text:?}"); + assert!(waited < Duration::from_secs(4), "closed after {waited:?}"); + assert!(waited >= Duration::from_millis(1500), "closed too early: {waited:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// an auth line that completes after the deadline does not authenticate, even when it lands +/// between two read polls (250 ms apart) and the poll has not yet seen the deadline pass. +#[test] +fn an_auth_line_that_completes_after_the_deadline_is_refused() { + let limits = Limits { auth_deadline: Duration::from_millis(300), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("lateauth", limits); + let mut s = UnixStream::connect(&socket).unwrap(); + std::thread::sleep(Duration::from_millis(400)); + if let Err(err) = s.write_all(auth_line(&token).as_bytes()) { + assert!( + matches!(err.kind(), std::io::ErrorKind::BrokenPipe | std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::NotConnected), + "{err}" + ); + } + let text = read_text(&mut s, 6); + assert!(text.contains("authentication required"), "{text:?}"); + assert!(!text.contains("\"authenticated\""), "{text:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a peer that starts a line and stalls is cut at the partial-line deadline. +#[test] +fn a_stalled_partial_line_is_closed() { + let limits = Limits { partial_line: Duration::from_millis(800), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("stall", limits); + let mut s = UnixStream::connect(&socket).unwrap(); + s.write_all(auth_line(&token).as_bytes()).unwrap(); + s.write_all(b"{\"id\":1,\"method\":\"link.he").unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut s, 6); + assert!(text.contains("authenticated"), "{text:?}"); + assert!(start.elapsed() < Duration::from_secs(4), "the stalled line stayed open for {:?}", start.elapsed()); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// an authenticated connection with no subscription and no traffic is closed at the idle limit. +/// A subscribed connection stays open. +#[test] +fn idle_connections_are_closed_but_subscribers_stay() { + let limits = Limits { idle: Duration::from_millis(900), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("idle", limits); + let mut idle = UnixStream::connect(&socket).unwrap(); + idle.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut sub = UnixStream::connect(&socket).unwrap(); + sub.write_all(auth_line(&token).as_bytes()).unwrap(); + sub.write_all(b"{\"id\":2,\"method\":\"events.subscribe\",\"params\":{\"events\":[\"doc.changed\"]}}\n").unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut idle, 6); + assert!(text.contains("authenticated"), "{text:?}"); + assert!(start.elapsed() < Duration::from_secs(4), "idle connection stayed open for {:?}", start.elapsed()); + // The subscriber is still served after the idle limit passed. + std::thread::sleep(Duration::from_millis(300)); + sub.write_all(HELLO.as_bytes()).unwrap(); + let mut reader = BufReader::new(sub); + let mut seen = String::new(); + for _ in 0..3 { + seen.push_str(&next_line(&mut reader)); + if seen.contains("\"id\":1") { + break; + } + } + assert!(seen.contains("\"id\":1"), "subscriber was cut: {seen:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a request split at several points, with a pause longer than the 250 ms read poll, is served. +#[test] +fn a_line_split_across_the_250ms_read_poll_is_kept() { + let (server, socket, token, dir) = boot("split"); + for cut in [1usize, 23, HELLO.len() - 2, HELLO.len() - 1] { + let mut s = UnixStream::connect(&socket).unwrap(); + s.write_all(auth_line(&token).as_bytes()).unwrap(); + s.write_all(&HELLO.as_bytes()[..cut]).unwrap(); + std::thread::sleep(Duration::from_millis(600)); + s.write_all(&HELLO.as_bytes()[cut..]).unwrap(); + let mut reader = BufReader::new(s); + assert!(next_line(&mut reader).contains("authenticated"), "cut {cut}"); + let line = next_line(&mut reader); + assert!(line.contains("\"ok\":true"), "cut {cut}: split request was not served: {line:?}"); + } + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// the auth line itself may arrive in pieces. +#[test] +fn an_auth_line_split_across_the_read_poll_is_kept() { + let (server, socket, token, dir) = boot("authsplit"); + let mut s = UnixStream::connect(&socket).unwrap(); + let line = auth_line(&token); + s.write_all(&line.as_bytes()[..20]).unwrap(); + std::thread::sleep(Duration::from_millis(600)); + s.write_all(&line.as_bytes()[20..]).unwrap(); + let mut reader = BufReader::new(s); + assert!(next_line(&mut reader).contains("authenticated")); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/ipc/tests/discovery.rs b/crates/ipc/tests/discovery.rs new file mode 100644 index 000000000..7d9f4a05f --- /dev/null +++ b/crates/ipc/tests/discovery.rs @@ -0,0 +1,97 @@ +//! A record held by a child dies with the child. +//! +//! `hold_until_killed` returns at once unless `CRAFT_HOLD_DIR` is set. +//! The parent test spawns that child, checks the live record, sends SIGKILL, +//! and checks that `Discovery::scan` removed it. Drop does not run on SIGKILL. + +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +use craft_ipc::{Discovery, InstanceRecord, RuntimeDir}; + +#[test] +fn hold_until_killed() { + let Ok(dir) = std::env::var("CRAFT_HOLD_DIR") else { + return; + }; + let runtime = RuntimeDir::at(&dir).unwrap(); + let app = "effectcraft"; + let pid = std::process::id(); + let endpoint = runtime.unix_endpoint(app, pid).unwrap(); + let record = InstanceRecord::new(app, "0.8.3", pid, endpoint).unwrap(); + let held = Discovery::publish(&runtime, record).unwrap(); + std::fs::write(runtime.path().join("ready"), b"ready").unwrap(); + let _held = held; + // park can wake without an unpark. Loop so Drop does not run until SIGKILL. + loop { + std::thread::park(); + } +} + +#[test] +fn sigkill_drops_the_record() { + let path = hold_dir(); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let exe = std::env::current_exe().unwrap(); + let child = Command::new(exe).args(["--exact", "hold_until_killed", "--test-threads=1"]).env("CRAFT_HOLD_DIR", &path).spawn().unwrap(); + let mut guard = HoldCleanup { dir: path.clone(), child: Some(child) }; + let ready = runtime.path().join("ready"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if ready.is_file() { + break; + } + let child = guard.child.as_mut().unwrap(); + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before ready: {status}"); + } + if Instant::now() >= deadline { + panic!("child did not write ready"); + } + std::thread::sleep(Duration::from_millis(20)); + } + let child_pid = guard.child.as_ref().unwrap().id(); + let live = Discovery::scan(&runtime).unwrap(); + assert_eq!(live.len(), 1, "expected one live record"); + assert_eq!(live[0].pid, child_pid); + assert_eq!(live[0].app, "effectcraft"); + let mut child = guard.child.take().unwrap(); + child.kill().unwrap(); + let status = child.wait().unwrap(); + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + assert_eq!(status.signal(), Some(9), "SIGKILL so the child's Drop does not unlock"); + } + #[cfg(not(unix))] + { + let _ = status; + } + let live = Discovery::scan(&runtime).unwrap(); + assert!(live.is_empty(), "scan should drop the record after SIGKILL, got {live:?}"); + assert!(!runtime.record_file("effectcraft", child_pid).exists(), "record json should be gone"); +} + +struct HoldCleanup { + dir: PathBuf, + child: Option, +} + +impl Drop for HoldCleanup { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn hold_dir() -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + std::env::temp_dir().join(format!("cl-hold-{hex}")) +} diff --git a/crates/ipc/tests/fetch.rs b/crates/ipc/tests/fetch.rs new file mode 100644 index 000000000..8e1dcde73 --- /dev/null +++ b/crates/ipc/tests/fetch.rs @@ -0,0 +1,366 @@ +//! `FrameFetcher` against a real server with a fake `FrameSource`: cache, deadline, dedupe, +//! parallel connections, and a peer that goes away. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use craft_ipc::bulk::{FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; +use craft_ipc::fetch::{FetchOptions, Fetched, FrameFetcher, FrameKey, QUEUE_MAX}; +use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; +use craft_ipc::{InstanceRecord, LinkError, RuntimeDir}; + +const W: u32 = 4; +const H: u32 = 3; +const STRIDE: u32 = 16; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +/// The first pixel byte is the frame number asked for, so a wrong frame cannot match by accident. +struct Fake { + epoch: AtomicU64, + delay_ms: AtomicU64, + renders: AtomicUsize, + times: Mutex>, + fail_time: Mutex>, +} + +impl FrameSource for Fake { + fn open(&self, _spec: &FrameSpec) -> Result { + Ok(FrameLayout { w: W, h: H, stride: STRIDE, len: u64::from(STRIDE * H) }) + } + + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { + self.renders.fetch_add(1, Ordering::SeqCst); + self.times.lock().unwrap().push(request.time_s); + let delay = self.delay_ms.load(Ordering::SeqCst); + if delay > 0 { + std::thread::sleep(Duration::from_millis(delay)); + } + if *self.fail_time.lock().unwrap() == Some(request.time_s) { + return Err(LinkError::NotReady("comp is busy".into())); + } + let mut pixels = vec![0u8; (STRIDE * H) as usize]; + pixels[0] = request.time_s as u8; + Ok(RenderedFrame { + epoch: self.epoch.load(Ordering::SeqCst).max(request.epoch), + w: W, + h: H, + stride: STRIDE, + format: spec.format, + render_ms: 1.0, + pixels, + }) + } + + fn close(&self, _spec: &FrameSpec) {} +} + +struct Rig { + _tmp: Tmp, + server: Option, + record: InstanceRecord, + fake: Arc, +} + +fn boot() -> Rig { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-fch-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let fake = Arc::new(Fake { + epoch: AtomicU64::new(1), + delay_ms: AtomicU64::new(0), + renders: AtomicUsize::new(0), + times: Mutex::new(Vec::new()), + fail_time: Mutex::new(None), + }); + let tickets = Arc::new(FrameTickets::new(Arc::clone(&fake) as Arc)); + let server = Server::start(ServerConfig { + app: format!("ec-{serial}"), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(RuntimeDir::at(&path).unwrap()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: tickets as Arc, + }) + .unwrap(); + let record = server.record().clone(); + Rig { _tmp: Tmp(path), server: Some(server), record, fake } +} + +fn connect(rig: &Rig, connections: usize) -> FrameFetcher { + let options = FetchOptions { connections, io_timeout: Duration::from_secs(2), ..FetchOptions::default() }; + FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap() +} + +const fn key(frame: i64) -> FrameKey { + FrameKey { frame, scale_milli: 1000 } +} + +const WAIT: Duration = Duration::from_secs(2); + +#[test] +fn fetch_returns_the_frame_and_the_second_ask_is_a_cache_hit() { + let rig = boot(); + let fetcher = connect(&rig, 1); + let layout = fetcher.layout(); + assert_eq!((layout.w, layout.h, layout.stride), (W, H, STRIDE)); + let Fetched::Frame(first) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; + assert_eq!(first.pixels[0], 7); + assert_eq!(first.epoch, 1); + let Fetched::Frame(second) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; + assert!(Arc::ptr_eq(&first, &second), "a hit returns the cached frame"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); + let stats = fetcher.stats(); + assert_eq!((stats.hits, stats.renders), (1, 1)); +} + +#[test] +fn a_late_frame_misses_the_deadline_then_lands_in_the_cache() { + let rig = boot(); + rig.fake.delay_ms.store(300, Ordering::SeqCst); + let fetcher = connect(&rig, 1); + let t0 = Instant::now(); + assert!(matches!(fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(40)), Fetched::TimedOut)); + assert!(t0.elapsed() < Duration::from_millis(200), "the call must not wait for the render: {:?}", t0.elapsed()); + assert!(fetcher.cached(key(3), 1).is_none(), "not there yet"); + let deadline = Instant::now() + Duration::from_secs(3); + while fetcher.cached(key(3), 1).is_none() { + assert!(Instant::now() < deadline, "the late frame never reached the cache"); + std::thread::sleep(Duration::from_millis(10)); + } + let Fetched::Frame(frame) = fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(10)) else { panic!("no frame") }; + assert_eq!(frame.pixels[0], 3); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1, "the timed-out ask and the retry share one render"); + assert_eq!(fetcher.stats().timeouts, 1); +} + +#[test] +fn a_newer_epoch_asked_for_renders_again_and_an_older_cached_one_does_not_count() { + let rig = boot(); + let fetcher = connect(&rig, 1); + let Fetched::Frame(old) = fetcher.fetch(key(1), 1.0, 1, WAIT) else { panic!("no frame") }; + assert_eq!(old.epoch, 1); + rig.fake.epoch.store(5, Ordering::SeqCst); + // Epoch 1 is still good for a caller that wants epoch 1. + assert!(fetcher.cached(key(1), 1).is_some()); + assert!(fetcher.cached(key(1), 2).is_none(), "an epoch 1 frame cannot answer a request for epoch 2"); + let Fetched::Frame(fresh) = fetcher.fetch(key(1), 1.0, 2, WAIT) else { panic!("no frame") }; + assert_eq!(fresh.epoch, 5, "the source may answer with a newer epoch"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2); + // The newer frame answers an older request too. + assert!(fetcher.cached(key(1), 1).is_some_and(|f| f.epoch == 5)); +} + +#[test] +fn two_asks_for_one_key_render_once() { + let rig = boot(); + rig.fake.delay_ms.store(120, Ordering::SeqCst); + let fetcher = Arc::new(connect(&rig, 2)); + let a = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) + }; + let b = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) + }; + assert!(a.join().unwrap() && b.join().unwrap()); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); +} + +#[test] +fn four_connections_render_four_frames_at_once() { + let rig = boot(); + rig.fake.delay_ms.store(200, Ordering::SeqCst); + let fetcher = Arc::new(connect(&rig, 4)); + let t0 = Instant::now(); + let handles: Vec<_> = (0..4) + .map(|n| { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))) + }) + .collect(); + for handle in handles { + assert!(handle.join().unwrap()); + } + let took = t0.elapsed(); + assert!(took < Duration::from_millis(600), "four 200 ms renders on four connections took {took:?}"); +} + +#[test] +fn a_frame_error_fails_that_frame_and_the_fetcher_keeps_working() { + let rig = boot(); + *rig.fake.fail_time.lock().unwrap() = Some(4.0); + let fetcher = connect(&rig, 1); + match fetcher.fetch(key(4), 4.0, 1, WAIT) { + Fetched::Failed(message) => assert!(message.contains("comp is busy"), "{message}"), + other => panic!("expected Failed, got {other:?}"), + } + assert!(!fetcher.is_down()); + assert!(matches!(fetcher.fetch(key(5), 5.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn a_peer_that_goes_away_makes_the_fetcher_down_and_asks_return_at_once() { + let mut rig = boot(); + let fetcher = connect(&rig, 2); + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); + rig.server.take().unwrap().shutdown().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !fetcher.is_down() { + // A miss makes the workers touch the dead socket. + let _ = fetcher.fetch(key(100), 100.0, 1, Duration::from_millis(50)); + assert!(Instant::now() < deadline, "the fetcher never noticed the peer is gone"); + } + let t0 = Instant::now(); + assert!(matches!(fetcher.fetch(key(200), 200.0, 1, WAIT), Fetched::Down)); + assert!(t0.elapsed() < Duration::from_millis(100)); + // Frames already in the cache still answer. + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn the_cache_stays_inside_its_byte_budget() { + let rig = boot(); + let options = FetchOptions { connections: 1, cache_bytes: usize::try_from(STRIDE * H).unwrap() * 3, ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + for n in 0..6 { + assert!(matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))); + } + let kept = (0..6).filter(|n| fetcher.cached(key(*n), 1).is_some()).count(); + assert_eq!(kept, 3, "a budget of three frames keeps three"); + assert!(fetcher.cached(key(5), 1).is_some(), "the newest frame stays"); +} + +#[test] +fn queued_asks_nobody_waits_for_are_dropped_when_old() { + let rig = boot(); + rig.fake.delay_ms.store(150, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + // Five asks with no patience: frame 0 starts at once, frames 1 to 4 wait behind it. + for n in 0..5 { + assert!(matches!(fetcher.fetch(key(n), n as f64, 1, Duration::from_millis(1)), Fetched::TimedOut)); + } + std::thread::sleep(Duration::from_millis(900)); + let renders = rig.fake.renders.load(Ordering::SeqCst); + assert!(renders < 5, "frames that waited longer than max_age must be skipped, got {renders} renders"); +} + +#[test] +fn a_time_that_is_not_finite_fails_the_ask_and_keeps_the_workers() { + let rig = boot(); + let fetcher = connect(&rig, 1); + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + match fetcher.fetch(key(1), bad, 1, WAIT) { + Fetched::Failed(message) => assert!(message.contains("finite"), "{message}"), + other => panic!("expected Failed for {bad}, got {other:?}"), + } + } + assert!(!fetcher.is_down(), "a bad time must not end the worker"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 0, "nothing was queued"); + assert!(matches!(fetcher.fetch(key(2), 2.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn an_ask_with_a_waiting_caller_is_not_dropped_for_age() { + let rig = boot(); + rig.fake.delay_ms.store(300, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; + let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); + // Frame 0 keeps the only worker busy for 300 ms. Frame 1 waits in the queue past max_age, + // and its caller is still there when the worker reaches it. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + let waiter = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || fetcher.fetch(key(1), 1.0, 1, WAIT)) + }; + // A second caller for the same key joins the queued job and shares its render. + std::thread::sleep(Duration::from_millis(20)); + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + assert!(matches!(waiter.join().unwrap(), Fetched::Frame(_))); + let stats = fetcher.stats(); + assert_eq!(stats.skipped, 0, "a job a caller waits for is not skipped"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2, "each frame renders once"); +} + +#[test] +fn the_queue_holds_at_most_queue_max_asks() { + let rig = boot(); + rig.fake.delay_ms.store(500, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + // Frame 0 holds the only worker. The asks after it only queue. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + std::thread::sleep(Duration::from_millis(100)); + let extra = 40; + for n in 1..=i64::try_from(QUEUE_MAX + extra).unwrap() { + assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); + } + assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap(), "the oldest asks past the cap are dropped"); +} + +#[test] +fn a_caller_waiting_on_a_dropped_ask_gets_failed_and_does_not_queue_it_again() { + let rig = boot(); + rig.fake.delay_ms.store(1500, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; + let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); + // Frame 0 holds the only worker. Frame 1 queues behind it with a caller that waits. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + std::thread::sleep(Duration::from_millis(100)); + let (done, finished) = std::sync::mpsc::channel(); + let waiter = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || { + let started = Instant::now(); + let _ = done.send((fetcher.fetch(key(1), 1.0, 1, Duration::from_secs(20)), started.elapsed())); + }) + }; + std::thread::sleep(Duration::from_millis(100)); + // Fill the queue past its cap: frame 1 is the oldest, so it goes first. + let extra = 5; + for n in 2..=i64::try_from(QUEUE_MAX + extra).unwrap() { + assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); + } + let (outcome, waited) = finished.recv_timeout(Duration::from_secs(1)).expect("the dropped caller must return at once"); + assert!(matches!(&outcome, Fetched::Failed(message) if message.contains("dropped")), "{outcome:?}"); + assert!(waited < Duration::from_secs(5), "{waited:?}"); + waiter.join().unwrap(); + std::thread::sleep(Duration::from_millis(100)); + // Frame 1 plus the others make QUEUE_MAX + extra asks for QUEUE_MAX places. A caller that queued + // frame 1 again would evict more. + assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap()); + // A new ask for the dropped frame is a new ask, not a repeat of the failure. + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); +} diff --git a/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json b/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json new file mode 100644 index 000000000..9b0330361 --- /dev/null +++ b/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json @@ -0,0 +1,13 @@ +{ + "v": 1, + "link_id": "550e8400-e29b-41d4-a716-446655440000", + "target_app": "effectcraft", + "target_project": "links/shot.ecproj", + "target_comp": "comp-1", + "nested_sequence": "item-9", + "result_path": "Project Links/550e8400-e29b-41d4-a716-446655440000.mov", + "result_epoch": 7, + "format": { "w": 1920, "h": 1080, "rate_num": 24, "rate_den": 1, "duration_ticks": 240, "alpha": true, "tint": "warm" }, + "created": "2026-10-11T15:04:05Z", + "badge": "kept" +} diff --git a/crates/ipc/tests/fixtures/link-record-still.json b/crates/ipc/tests/fixtures/link-record-still.json new file mode 100644 index 000000000..20cf44e85 --- /dev/null +++ b/crates/ipc/tests/fixtures/link-record-still.json @@ -0,0 +1,15 @@ +{ + "v": 1, + "link_id": "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", + "target_app": "photocraft", + "target_project": "art/poster.photocraft", + "result_path": "Project Links/0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d.png", + "result_epoch": 3, + "asset": { + "kind": "still", + "moniker": "craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/flattened" + }, + "producer": { "app": "photocraft", "app_version": "0.4.2", "doc_id": "9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f" }, + "format": { "w": 4096, "h": 2160, "alpha": false }, + "created": "2026-10-23T09:30:00.250+02:00" +} diff --git a/crates/ipc/tests/hub.rs b/crates/ipc/tests/hub.rs new file mode 100644 index 000000000..0a9482b41 --- /dev/null +++ b/crates/ipc/tests/hub.rs @@ -0,0 +1,334 @@ +//! A hub is a peer. Two fake apps share one runtime directory: +//! `photocraft`, which owns a still and answers the asset methods, and `hub`, +//! which owns nothing, writes `caps: ["hub"]` and receives `app.activate` +//! with `open:{asset}`. A third party finds both through discovery. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use craft_ipc::asset::{Described, Refreshed, Resolved}; +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::moniker::Moniker; +use craft_ipc::record::AssetKind; +use craft_ipc::refresh::{Tracker, write_atomic}; +use craft_ipc::server::{Activate, Activation, Assets, ControlSink, Extensions, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{Discovery, InstanceRecord, LinkError, RuntimeDir}; + +// Both tests bind the same app and process named pipe on Windows, regardless of runtime directory. +static HUB_TEST: Mutex<()> = Mutex::new(()); + +const DOC: &str = "9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f"; +const LINK: &str = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +struct Tmp(PathBuf); + +impl Tmp { + fn new() -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-hub-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[derive(Default)] +struct RecordingSink { + seen: Mutex>, +} + +impl ControlSink for RecordingSink { + fn submit(&self, method: String, _params: Value) -> Option> { + self.seen.lock().unwrap().push(method); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({"forwarded": true})); + Some(rx) + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[derive(Default)] +struct RecordingActivate { + opens: Mutex>>, +} + +impl Activate for RecordingActivate { + fn activate(&self, _token: Option, _from_pid: Option, open: Option) -> Activation { + self.opens.lock().unwrap().push(open); + Activation::Requested + } +} + +/// A producer with one still. The real file watcher answers `link.refresh`. +struct Stills { + asset: Moniker, + file: PathBuf, + thumb: PathBuf, + tracker: Mutex, + links: HashMap, +} + +impl Assets for Stills { + fn resolve(&self, asset: &Moniker) -> Result { + self.known(asset)?; + let fp = craft_ipc::refresh::fingerprint_if_present(&self.file)?.ok_or_else(|| LinkError::Unknown("no result file yet".into()))?; + let epoch = self.tracker.lock().unwrap().epoch(); + Ok(Resolved::from_fingerprint(asset.clone(), &self.file.display().to_string(), epoch, &fp)) + } + + fn describe(&self, asset: &Moniker) -> Result { + self.known(asset)?; + let fp = craft_ipc::refresh::fingerprint_if_present(&self.file)?; + Ok(Described { + asset: asset.clone(), + name: "Poster".into(), + kind: AssetKind::Still, + epoch: self.tracker.lock().unwrap().epoch(), + w: Some(640), + h: Some(480), + hash: fp.as_ref().map(|f| f.hash.clone()), + thumb_path: Some(self.thumb.display().to_string()), + modified_ms: fp.map(|f| f.mtime_ms), + }) + } + + fn refresh(&self, link_id: &str) -> Result { + if !self.links.contains_key(link_id) { + return Err(LinkError::Unknown(format!("link {link_id} is not known"))); + } + let mut tracker = self.tracker.lock().unwrap(); + let change = tracker.poll(true)?; + Ok(Refreshed::new(link_id, change.epoch, change.state, &self.file.display().to_string(), tracker.seen())) + } +} + +impl Stills { + fn known(&self, asset: &Moniker) -> Result<(), LinkError> { + if *asset == self.asset { Ok(()) } else { Err(LinkError::Unknown(format!("{asset} is not an asset of this document"))) } + } +} + +fn config(app: &str, runtime: &RuntimeDir, sink: Arc, activate: Arc) -> ServerConfig { + ServerConfig { + app: app.into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink, + activate, + frames: Arc::new(NoFrames), + } +} + +fn dial(runtime: &RuntimeDir, app: &str) -> (Client, InstanceRecord) { + let found = Discovery::scan(runtime).unwrap(); + let record = found.into_iter().find(|r| r.app == app).unwrap_or_else(|| panic!("no live {app} record")); + let mut client = Client::connect(&record).unwrap(); + let identity = HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440000".into(), + protocol_min: 1, + protocol_max: 1, + }; + let hello = client.hello(&identity).unwrap(); + assert_eq!(hello["app"], json!(app)); + (client, record) +} + +fn caps_of(hello_or_record: &Value) -> Vec { + hello_or_record["caps"].as_array().unwrap().iter().map(|c| c.as_str().unwrap().to_string()).collect() +} + +/// The two fake apps of the hub test and the files behind them. +struct World { + runtime: RuntimeDir, + asset: Moniker, + file: PathBuf, + producer: Server, + producer_sink: Arc, + hub: Server, + hub_activate: Arc, +} + +fn start_world(tmp: &Tmp) -> World { + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let asset = Moniker::parse(&format!("craft://photocraft/{DOC}/image/flattened")).unwrap(); + let file = tmp.0.join("flat.png"); + write_atomic(&file, b"png-v1").unwrap(); + let stills = Arc::new(Stills { + asset: asset.clone(), + file: file.clone(), + thumb: tmp.0.join("thumb.png"), + tracker: Mutex::new(Tracker::new(&file, 3).unwrap()), + links: HashMap::from([(LINK.to_string(), ())]), + }); + + let producer_sink = Arc::new(RecordingSink::default()); + let producer = Server::start_with( + config("photocraft", &runtime, Arc::clone(&producer_sink), Arc::new(RecordingActivate::default())), + Extensions { assets: Some(stills), caps: Vec::new() }, + ) + .unwrap(); + let hub_activate = Arc::new(RecordingActivate::default()); + let hub = Server::start_with( + config("hub", &runtime, Arc::new(RecordingSink::default()), Arc::clone(&hub_activate)), + Extensions { assets: None, caps: vec!["hub".into()] }, + ) + .unwrap(); + World { runtime, asset, file, producer, producer_sink, hub, hub_activate } +} + +/// Discovery: the hub is an ordinary record with app "hub" and caps ["hub"]. `link.hello` +/// advertises the base caps plus the record caps. +fn check_discovery_and_hello(world: &World, hub_client: &mut Client, hub_record: &InstanceRecord, producer_client: &mut Client) { + let live = Discovery::scan(&world.runtime).unwrap(); + let mut names: Vec<&str> = live.iter().map(|r| r.app.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["hub", "photocraft"]); + assert_eq!(world.hub.record().caps(), vec!["hub".to_string()]); + assert_eq!(world.producer.record().caps(), vec!["assets".to_string()]); + + let hello = hub_client.call("link.hello", json!({"protocol": {"min": 1, "max": 1}})).unwrap(); + assert_eq!(hello["app"], json!("hub")); + assert!(caps_of(&hello).contains(&"hub".to_string()), "{hello}"); + assert!(caps_of(&hello).contains(&"activate".to_string()), "{hello}"); + assert_eq!(hub_record.docs.len(), 0, "a hub owns no document"); + + let hello = producer_client.call("link.hello", json!({"protocol": {"min": 1, "max": 1}})).unwrap(); + assert!(caps_of(&hello).contains(&"assets".to_string()), "{hello}"); +} + +/// The hub browses the producer with no document open. Returns the hash of the first version. +fn check_browse(world: &World, producer_client: &mut Client) -> String { + let asset = &world.asset; + let described = producer_client.call("asset.describe", json!({"asset": asset.to_string()})).unwrap(); + assert_eq!(described["name"], json!("Poster")); + assert_eq!(described["kind"], json!("still")); + assert_eq!(described["epoch"], json!(3)); + assert_eq!(described["w"], json!(640)); + assert_eq!(described["asset"], json!(asset.to_string())); + let resolved = producer_client.call("link.resolve", json!({"asset": asset.to_string()})).unwrap(); + assert_eq!(resolved["path"], json!(world.file.display().to_string())); + assert_eq!(resolved["size"], json!(6)); + let v1_hash = resolved["hash"].as_str().unwrap().to_string(); + assert_eq!(described["hash"], json!(v1_hash)); + assert!(v1_hash.starts_with("sha256:") && v1_hash.len() == 71, "{v1_hash}"); + assert!(world.producer_sink.seen.lock().unwrap().is_empty(), "asset methods must not reach the UI sink"); + v1_hash +} + +/// Errors map to the protocol's error codes. +fn check_error_codes(producer_client: &mut Client) { + let other = format!("craft://photocraft/{DOC}/image/other"); + assert!(matches!(producer_client.call("asset.describe", json!({"asset": other})), Err(LinkError::Unknown(_)))); + assert!(matches!(producer_client.call("link.resolve", json!({"asset": "craft://nope"})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.resolve", json!({})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.refresh", json!({"link_id": ""})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.refresh", json!({"link_id": "7b1e0000-0000-4000-8000-000000000000"})), Err(LinkError::Unknown(_)))); +} + +/// `link.refresh` sees a new version on disk and bumps the epoch once. +fn check_refresh(world: &World, producer_client: &mut Client, v1_hash: &str) { + let file = &world.file; + let current = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(current["state"], json!("current")); + assert_eq!(current["epoch"], json!(3)); + write_atomic(file, b"png-version-two").unwrap(); + let modified = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(modified["state"], json!("modified")); + assert_eq!(modified["epoch"], json!(4)); + assert_ne!(modified["hash"], json!(v1_hash)); + let again = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(again["epoch"], json!(4), "no second bump without a new edit"); + std::fs::remove_file(file).unwrap(); + let missing = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(missing["state"], json!("missing")); + assert!(missing.get("hash").is_none() && missing.get("result_path").is_none(), "{missing}"); + assert!(matches!(producer_client.call("link.resolve", json!({"asset": world.asset.to_string()})), Err(LinkError::Unknown(_)))); +} + +/// `app.activate` carries open:{asset} to the hub's Activate. +fn check_activate(world: &World, hub_client: &mut Client) { + let asset = &world.asset; + let reply = hub_client.call("app.activate", json!({"token": null, "from_pid": 4242, "open": {"asset": asset.to_string()}})).unwrap(); + assert_eq!(reply["activated"], json!("requested")); + let reply = hub_client.call("app.activate", json!({"open": {"link_id": LINK, "asset": asset.to_string()}})).unwrap(); + assert_eq!(reply["activated"], json!("requested")); + // Legacy shapes still pass through unchanged. + hub_client.call("app.activate", json!({"open": {"link_id": LINK}})).unwrap(); + hub_client.call("app.activate", json!({})).unwrap(); + // A bad moniker never reaches the app. + assert!(matches!(hub_client.call("app.activate", json!({"open": {"asset": "craft://bad"}})), Err(LinkError::BadParams(_)))); + assert!(matches!(hub_client.call("app.activate", json!({"open": {"asset": 7}})), Err(LinkError::BadParams(_)))); + let opens = world.hub_activate.opens.lock().unwrap().clone(); + assert_eq!( + opens, + vec![ + Some(json!({"asset": asset.to_string()})), + Some(json!({"link_id": LINK, "asset": asset.to_string()})), + Some(json!({"link_id": LINK})), + None, + ] + ); +} + +#[test] +fn hub_is_a_peer_and_producer_answers_asset_methods() { + let _guard = HUB_TEST.lock().unwrap(); + let tmp = Tmp::new(); + let world = start_world(&tmp); + let (mut hub_client, hub_record) = dial(&world.runtime, "hub"); + let (mut producer_client, _) = dial(&world.runtime, "photocraft"); + + check_discovery_and_hello(&world, &mut hub_client, &hub_record, &mut producer_client); + let v1_hash = check_browse(&world, &mut producer_client); + check_error_codes(&mut producer_client); + check_refresh(&world, &mut producer_client, &v1_hash); + check_activate(&world, &mut hub_client); + + world.producer.shutdown().unwrap(); + world.hub.shutdown().unwrap(); +} + +#[test] +fn server_without_a_provider_forwards_asset_methods_to_the_app() { + let _guard = HUB_TEST.lock().unwrap(); + let tmp = Tmp::new(); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let sink = Arc::new(RecordingSink::default()); + let server = Server::start(config("photocraft", &runtime, Arc::clone(&sink), Arc::new(RecordingActivate::default()))).unwrap(); + assert!(server.record().caps().is_empty()); + let (mut client, _) = dial(&runtime, "photocraft"); + let asset = format!("craft://photocraft/{DOC}/image/x"); + for method in ["link.resolve", "asset.describe", "link.refresh"] { + let reply = client.call(method, json!({"asset": asset, "link_id": LINK})).unwrap(); + assert_eq!(reply, json!({"forwarded": true}), "{method}"); + } + assert_eq!(sink.seen.lock().unwrap().as_slice(), ["link.resolve", "asset.describe", "link.refresh"]); + server.shutdown().unwrap(); +} diff --git a/crates/ipc/tests/native_roundtrip.rs b/crates/ipc/tests/native_roundtrip.rs new file mode 100644 index 000000000..7539d91cf --- /dev/null +++ b/crates/ipc/tests/native_roundtrip.rs @@ -0,0 +1,93 @@ +//! Exercise the real native transport: Unix sockets or Windows named pipes. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::PathBuf; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; +use serde_json::{Value, json}; + +struct Cleanup(PathBuf); +impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct Echo; +impl ControlSink for Echo { + fn submit(&self, method: String, params: Value) -> Option> { + let (tx, rx) = mpsc::channel(); + tx.send(json!({"ok": true, "result": {"method": method, "params": params}})).unwrap(); + Some(rx) + } +} + +struct NoActivate; +impl Activate for NoActivate { + fn activate(&self, _token: Option, _pid: Option, _options: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _hints: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[test] +fn native_transport_authenticates_and_delivers_commands() { + let mut nonce = [0u8; 4]; + getrandom::getrandom(&mut nonce).unwrap(); + let path = std::env::temp_dir().join(format!("cl-nr-{}-{}", std::process::id(), u32::from_le_bytes(nonce))); + let cleanup = Cleanup(path.clone()); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "test".into(), + instance: None, + runtime: Some(RuntimeDir::at(path).unwrap()), + tcp_port: None, + sink: Arc::new(Echo), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let mut client = Client::connect(server.record()).unwrap(); + client + .hello(&HelloClient { + app: "ec".into(), + app_version: "test".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440000".into(), + protocol_min: craft_ipc::wire::PROTOCOL_MIN, + protocol_max: craft_ipc::wire::PROTOCOL_MAX, + }) + .unwrap(); + let reply = client.call("engine.execute", json!({"command": "ping"})).unwrap(); + assert_eq!(reply, json!({"method": "engine.execute", "params": {"command": "ping"}})); + + client.subscribe(&["doc.changed"]).unwrap(); + let start = Instant::now(); + assert!(client.next_event(Some(Duration::from_millis(50))).unwrap().is_none()); + assert!(start.elapsed() < Duration::from_secs(2), "quiet native transport must respect its deadline"); + server.emit("doc.changed", json!({"after_timeout": true})).unwrap(); + let event = client.next_event(Some(Duration::from_secs(2))).unwrap().expect("event after timeout"); + assert_eq!(event.data, json!({"after_timeout": true})); + + let mut wrong = server.record().clone(); + let opposite = if wrong.token.starts_with('0') { "1" } else { "0" }; + wrong.token.replace_range(..1, opposite); + assert!(matches!(Client::connect(&wrong), Err(LinkError::Auth))); + drop(client); + drop(server); + drop(cleanup); +} diff --git a/crates/ipc/tests/open_payload.rs b/crates/ipc/tests/open_payload.rs new file mode 100644 index 000000000..1a6f3453b --- /dev/null +++ b/crates/ipc/tests/open_payload.rs @@ -0,0 +1,169 @@ +//! `link.open` payload validation over a real server and a real client. +//! A bad payload gets `payload_invalid` and never reaches the app sink. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{Discovery, LinkError, RuntimeDir}; + +const ASSET: &str = "craft://photocraft/550e8400-e29b-41d4-a716-446655440000/image/flat"; +const LINK: &str = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +struct Tmp(PathBuf); + +impl Tmp { + fn new() -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-open-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Records every forwarded request and answers with a finished reply payload. +#[derive(Default)] +struct RecordingSink { + seen: Mutex>, +} + +impl ControlSink for RecordingSink { + fn submit(&self, method: String, params: Value) -> Option> { + self.seen.lock().unwrap().push((method, params)); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({"comp": "c1"})); + Some(rx) + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +fn native_path(name: &str) -> String { + std::env::temp_dir().join(name).to_str().unwrap().to_owned() +} + +fn file_url(path: &std::path::Path) -> String { + let text = path.to_str().unwrap().replace('\\', "/"); + if text.starts_with('/') { format!("file://{text}") } else { format!("file:///{text}") } +} + +fn open_params(payload: Value) -> Value { + json!({"link_id": LINK, "payload": payload, "result_path": native_path("x.png")}) +} + +#[test] +fn bad_payloads_get_payload_invalid_and_good_ones_reach_the_sink() { + let tmp = Tmp::new(); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let sink = Arc::new(RecordingSink::default()); + let server = Server::start(ServerConfig { + app: "effectcraft".into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink: Arc::clone(&sink) as Arc, + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let record = Discovery::scan(&runtime).unwrap().into_iter().next().unwrap(); + let mut client = Client::connect(&record).unwrap(); + client + .hello(&HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440001".into(), + protocol_min: 1, + protocol_max: 1, + }) + .unwrap(); + + let bad = [ + ("unknown format", json!({"format": "svg", "data": ""})), + ("no format", json!({"data": "{}"})), + ("payload is a string", json!("otio")), + ("otio without data", json!({"format": "otio-json"})), + ("otio empty data", json!({"format": "otio-json", "data": ""})), + ("image-doc relative path", json!({"format": "image-doc", "path": "rel.png", "asset": ASSET})), + ("image-doc no path", json!({"format": "image-doc", "asset": ASSET})), + ("image-doc bad moniker", json!({"format": "image-doc", "path": native_path("a.png"), "asset": "craft://bad"})), + ("image-doc no asset", json!({"format": "image-doc", "path": native_path("a.png")})), + ]; + for (name, payload) in bad { + let err = client.call("link.open", open_params(payload)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{name}: {err:?}"); + } + let err = client.call("link.open", json!({"link_id": LINK})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "missing payload: {err:?}"); + // A payload that names a file that is not there gets media_missing with the paths. + let missing_path = tmp.0.join("missing-clip.mov"); + let missing = json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": file_url(&missing_path)}).to_string(); + let err = client.call("link.open", open_params(json!({"format": "otio-json", "data": missing}))).unwrap_err(); + assert_eq!(err, LinkError::MediaMissing { paths: vec![missing_path.to_str().unwrap().replace('\\', "/")] }); + // A path that climbs, is relative or is not a string never reaches the app. + let good = json!({"format": "otio-json", "data": "{}"}); + for result_path in [json!("../x.mov"), json!("/tmp/../etc/x.mov"), json!("rel/x.mov"), json!(7), json!("/tmp/a\u{0}b")] { + let params = json!({"link_id": LINK, "payload": good, "result_path": result_path}); + let err = client.call("link.open", params).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{result_path}: {err:?}"); + } + let climbing = json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": file_url(&tmp.0.join("..").join("outside.mov"))}).to_string(); + let err = client.call("link.open", open_params(json!({"format": "otio-json", "data": climbing}))).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // Params that are not an object are still bad_params. + let err = client.call("link.open", json!("nope")).unwrap_err(); + assert!(matches!(err, LinkError::BadParams(_)), "{err:?}"); + assert!(sink.seen.lock().unwrap().is_empty(), "a bad payload must not reach the app: {:?}", sink.seen.lock().unwrap()); + + let otio = open_params(json!({"format": "otio-json", "data": "{\"OTIO_SCHEMA\":\"Timeline.1\"}"})); + let still = open_params(json!({"format": "image-doc", "path": native_path("a.photocraft"), "asset": ASSET})); + for params in [otio, still] { + let reply = client.call("link.open", params.clone()).unwrap(); + assert_eq!(reply, json!({"comp": "c1"})); + let seen = sink.seen.lock().unwrap(); + let (method, got) = seen.last().unwrap(); + assert_eq!(method, "link.open"); + assert_eq!(got, ¶ms, "the app gets the request unchanged"); + } + assert_eq!(sink.seen.lock().unwrap().len(), 2); + + // The wire carries the contract code, not bad_params. + let err = client.call("link.open", open_params(json!({"format": "svg"}))).unwrap_err(); + assert_eq!(err.code(), "payload_invalid"); + // The connection stays usable after a refusal. + client.call("link.open", open_params(json!({"format": "otio-json", "data": "{}"}))).unwrap(); + assert_eq!(sink.seen.lock().unwrap().len(), 3); + + server.shutdown().unwrap(); +} diff --git a/crates/ipc/tests/protocol.rs b/crates/ipc/tests/protocol.rs new file mode 100644 index 000000000..d77a46017 --- /dev/null +++ b/crates/ipc/tests/protocol.rs @@ -0,0 +1,361 @@ +//! Session behaviour. The tests own the sink, the activate trait, and frames. +//! They do not use the client module. +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Read}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::wire::{self, MAX_LINE}; +use craft_ipc::{Endpoint, LinkError, RuntimeDir}; + +const PAYLOAD: &[u8] = b"craft-frame"; +const TICKET: &str = "00112233445566778899aabbccddeeff"; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct ReplySink { + result: Mutex, + seen: Mutex>, +} + +impl ControlSink for ReplySink { + fn submit(&self, method: String, _params: Value) -> Option> { + self.seen.lock().unwrap().push(method); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(self.result.lock().unwrap().clone()); + Some(rx) + } +} + +struct SwitchActivate { + mode: Mutex, + seen_token: Mutex>>, + seen_pid: Mutex>>, +} + +impl Activate for SwitchActivate { + fn activate(&self, token: Option, from_pid: Option, _open: Option) -> Activation { + *self.seen_token.lock().unwrap() = Some(token); + *self.seen_pid.lock().unwrap() = Some(from_pid); + *self.mode.lock().unwrap() + } +} + +struct TestFrames { + open_count: Mutex, + closed: Mutex>, +} + +impl Frames for TestFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + *self.open_count.lock().unwrap() += 1; + Ok(FrameOpen { ticket: TICKET.into(), w: 4, h: 2, stride: 16, len: PAYLOAD.len() as u64 }) + } + + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError> { + if ticket != TICKET { + return Err(LinkError::BadParams("unknown ticket".into())); + } + let req = header.get("req").cloned().unwrap_or(Value::Null); + Ok((json!({"req": req, "epoch": 3, "w": 4, "h": 2, "stride": 16, "format": "rgba8_premul", "render_ms": 1.5}), PAYLOAD.to_vec())) + } + + fn close_ticket(&self, ticket: &str) { + self.closed.lock().unwrap().push(ticket.to_string()); + } +} + +struct Harness { + tmp: Tmp, + server: Server, + socket: PathBuf, + token: String, + sink: Arc, + activate: Arc, + frames: Arc, +} + +fn boot() -> Harness { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-proto-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let sink = Arc::new(ReplySink { result: Mutex::new(json!({"commands": ["cut"]})), seen: Mutex::new(Vec::new()) }); + let activate = Arc::new(SwitchActivate { mode: Mutex::new(Activation::Requested), seen_token: Mutex::new(None), seen_pid: Mutex::new(None) }); + let frames = Arc::new(TestFrames { open_count: Mutex::new(0), closed: Mutex::new(Vec::new()) }); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(runtime), + tcp_port: None, + sink: Arc::clone(&sink) as Arc, + activate: Arc::clone(&activate) as Arc, + frames: Arc::clone(&frames) as Arc, + }) + .unwrap(); + let socket = match &server.record().endpoint { + Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + }; + let token = server.record().token.clone(); + Harness { tmp: Tmp(path), server, socket, token, sink, activate, frames } +} + +struct Client { + reader: BufReader, +} + +impl Client { + fn connect(socket: &Path, token: &str) -> Self { + let stream = UnixStream::connect(socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut reader = BufReader::new(stream); + let req = json!({"id": 0, "method": "auth", "params": {"token": token}}); + wire::write_line(reader.get_mut(), &req).unwrap(); + let reply = read_value(&mut reader).unwrap_or_else(|| panic!("no auth reply")); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"]["authenticated"], json!(true), "{reply}"); + Self { reader } + } + + fn call(&mut self, id: i64, method: &str, params: Value) -> Value { + let req = json!({"id": id, "method": method, "params": params}); + wire::write_line(self.reader.get_mut(), &req).unwrap(); + read_value(&mut self.reader).unwrap_or_else(|| panic!("no reply for {method}")) + } + + fn set_timeout(&mut self, timeout: Duration) { + self.reader.get_mut().set_read_timeout(Some(timeout)).unwrap(); + } +} + +fn read_value(reader: &mut BufReader) -> Option { + match wire::read_line(reader, MAX_LINE) { + Ok(Some(bytes)) => Some(serde_json::from_slice(&bytes).unwrap()), + Ok(None) => None, + Err(_) => None, + } +} + +fn read_raw_line(stream: &mut impl Read) -> Vec { + let mut out = Vec::new(); + let mut byte = [0u8; 1]; + loop { + stream.read_exact(&mut byte).unwrap(); + if byte[0] == b'\n' { + break; + } + out.push(byte[0]); + } + if out.last() == Some(&b'\r') { + out.pop(); + } + out +} + +#[test] +fn legacy_command_keeps_the_request_id() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let reply = client.call(41, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(41), "{reply}"); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"], json!({"commands": ["cut"]}), "{reply}"); + assert_eq!(harness.sink.seen.lock().unwrap().as_slice(), ["engine.commands"]); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// A sink reply longer than `MAX_LINE` cannot be read by a client. The server answers that request +/// with `payload_too_large` and the same id, and the connection keeps working. +#[test] +fn a_reply_over_the_line_limit_is_answered_with_an_error_and_the_connection_survives() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + *harness.sink.result.lock().unwrap() = json!({"blob": "x".repeat(MAX_LINE + 16)}); + let reply = client.call(5, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(5), "{reply}"); + assert_eq!(reply["ok"], json!(false), "{reply}"); + assert_eq!(reply["error"]["code"], json!("payload_too_large"), "{reply}"); + *harness.sink.result.lock().unwrap() = json!({"commands": ["cut"]}); + let next = client.call(6, "engine.commands", json!({})); + assert_eq!(next["ok"], json!(true), "the connection must still work: {next}"); + assert_eq!(next["id"], json!(6), "{next}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// The server polls each connection with a 250 ms read timeout. On Linux that timeout surfaces as +/// `WouldBlock` ("Resource temporarily unavailable"), which must mean "idle, keep waiting". A +/// connection that stays quiet for longer than one poll must still answer the next request. +#[test] +fn a_connection_idle_for_longer_than_the_read_poll_still_answers() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let first = client.call(1, "engine.commands", json!({})); + assert_eq!(first["ok"], json!(true), "{first}"); + std::thread::sleep(Duration::from_millis(900)); + let second = client.call(2, "engine.commands", json!({})); + assert_eq!(second["id"], json!(2), "the idle connection must still be open: {second}"); + assert_eq!(second["ok"], json!(true), "{second}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// A sink value that already has `ok` is the finished reply. The server inserts `id` and does not nest it. +#[test] +fn sink_reply_with_ok_is_not_wrapped() { + let harness = boot(); + *harness.sink.result.lock().unwrap() = json!({"ok": true, "result": null}); + let mut client = Client::connect(&harness.socket, &harness.token); + let reply = client.call(3, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(3), "{reply}"); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"], Value::Null, "{reply}"); + assert!(reply.get("error").is_none(), "{reply}"); + + *harness.sink.result.lock().unwrap() = json!({"ok": false, "error": "missing command"}); + let bad = client.call(4, "engine.commands", json!({})); + assert_eq!(bad["id"], json!(4), "{bad}"); + assert_eq!(bad["ok"], json!(false), "{bad}"); + assert_eq!(bad["error"], json!("missing command"), "{bad}"); + assert!(bad.get("result").is_none(), "an error reply must stay flat: {bad}"); + harness.server.shutdown().unwrap(); +} + +#[test] +fn events_wait_for_subscribe_then_one_event_is_delivered() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + client.set_timeout(Duration::from_millis(200)); + harness.server.emit("doc.changed", json!({"doc_id": "a"})).unwrap(); + assert!(read_value(&mut client.reader).is_none(), "event before subscribe"); + client.set_timeout(Duration::from_secs(2)); + let sub = client.call(2, "events.subscribe", json!({"events": ["doc.changed"]})); + assert_eq!(sub["id"], json!(2), "{sub}"); + assert_eq!(sub["result"], json!({"seq": 0}), "{sub}"); + harness.server.emit("doc.changed", json!({"doc_id": "a"})).unwrap(); + let event = read_value(&mut client.reader).unwrap_or_else(|| panic!("missing event")); + assert_eq!(event["event"], json!("doc.changed"), "{event}"); + assert_eq!(event["seq"], json!(1), "{event}"); + assert_eq!(event["data"], json!({"doc_id": "a"}), "{event}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +#[test] +fn oversized_event_delivers_an_overflow_marker() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let sub = client.call(2, "events.subscribe", json!({"events": ["doc.changed"]})); + assert_eq!(sub["result"]["seq"], json!(0), "{sub}"); + harness.server.emit("doc.changed", json!({"blob": "x".repeat(MAX_LINE)})).unwrap(); + let event = read_value(&mut client.reader).expect("missing overflow event"); + assert_eq!(event["event"], json!("events.overflow")); + assert!(event["data"]["since"].is_number()); + harness.server.shutdown().unwrap(); +} + +#[test] +fn bulk_round_trip_returns_the_frame_bytes() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let hello = client.call( + 1, + "link.hello", + json!({"client": {"app": "fc", "app_version": "0.9.1", "pid": 7, "instance": "00000000-0000-4000-8000-000000000001"}, "protocol": {"min": 1, "max": 1}}), + ); + assert_eq!(hello["ok"], json!(true), "{hello}"); + assert_eq!(hello["result"]["app"], json!("ec"), "{hello}"); + assert_eq!(hello["result"]["app_version"], json!("0.8.3"), "{hello}"); + assert_eq!(hello["result"]["protocol"], json!(1), "{hello}"); + assert_eq!(hello["result"]["caps"], json!(["events", "activate", "link.receive", "frames"]), "{hello}"); + let opened = client.call(8, "frames.open", json!({"link_id": "L", "format": "rgba8_premul", "scale": 1.0})); + assert_eq!(opened["ok"], json!(true), "{opened}"); + assert_eq!(opened["result"]["ticket"], json!(TICKET), "{opened}"); + assert_eq!(opened["result"]["w"], json!(4), "{opened}"); + assert_eq!(opened["result"]["len"], json!(PAYLOAD.len()), "{opened}"); + + let mut bulk = UnixStream::connect(&harness.socket).unwrap(); + bulk.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + bulk.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let auth = json!({"id": 0, "method": "auth", "params": {"token": harness.token}}); + wire::write_line(&mut bulk, &auth).unwrap(); + let auth_reply: Value = serde_json::from_slice(&read_raw_line(&mut bulk)).unwrap(); + assert_eq!(auth_reply["ok"], json!(true), "{auth_reply}"); + let attach = json!({"id": 9, "method": "bulk.attach", "params": {"ticket": TICKET}}); + wire::write_line(&mut bulk, &attach).unwrap(); + let attach_reply: Value = serde_json::from_slice(&read_raw_line(&mut bulk)).unwrap(); + assert_eq!(attach_reply["ok"], json!(true), "{attach_reply}"); + assert_eq!(attach_reply["id"], json!(9), "{attach_reply}"); + + let header = json!({"req": 7, "time_s": 1.5, "epoch": 3, "region": null}); + wire::write_cfrq(&mut bulk, &header).unwrap(); + let (got, payload) = wire::read_cfrm(&mut bulk).unwrap(); + assert_eq!(got["req"], json!(7), "{got}"); + assert_eq!(payload, PAYLOAD); + + drop(bulk); + drop(client); + harness.server.shutdown().unwrap(); + assert_eq!(harness.frames.closed.lock().unwrap().as_slice(), [TICKET]); + drop(harness.tmp); +} + +#[test] +fn activate_returns_requested_from_the_trait() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let params = json!({"token": "wayland-token", "from_pid": 4242, "open": {"link_id": "abc"}}); + let reply = client.call(3, "app.activate", params); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"]["activated"], json!("requested"), "{reply}"); + assert_eq!(harness.activate.seen_token.lock().unwrap().clone().unwrap().as_deref(), Some("wayland-token")); + assert_eq!(harness.activate.seen_pid.lock().unwrap().unwrap(), Some(4242)); + + *harness.activate.mode.lock().unwrap() = Activation::No; + let reply = client.call(4, "app.activate", json!({"token": null})); + assert_eq!(reply["result"]["activated"], json!(false), "{reply}"); + + *harness.activate.mode.lock().unwrap() = Activation::Yes; + let reply = client.call(5, "app.activate", json!({})); + assert_eq!(reply["result"]["activated"], json!(true), "{reply}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +#[test] +fn the_17th_connection_is_refused_and_does_not_stay_open() { + let harness = boot(); + let mut held = Vec::new(); + // Authenticated connections fill the cap. Unauthenticated ones are limited to a smaller pool. + for _ in 0..wire::MAX_CONNECTIONS { + held.push(Client::connect(&harness.socket, &harness.token)); + } + let mut extra = UnixStream::connect(&harness.socket).unwrap(); + extra.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut line = String::new(); + BufReader::new(&mut extra).read_line(&mut line).unwrap(); + let value: Value = serde_json::from_str(line.trim()).unwrap(); + assert_eq!(value["ok"], json!(false), "{line}"); + assert_eq!(value["error"], json!("too many control connections"), "{line}"); + drop(held); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} diff --git a/crates/ipc/tests/provider_stall.rs b/crates/ipc/tests/provider_stall.rs new file mode 100644 index 000000000..3407df05d --- /dev/null +++ b/crates/ipc/tests/provider_stall.rs @@ -0,0 +1,248 @@ +//! A provider that never returns must not hold up shutdown. `Assets` and `Activate` calls run on +//! helper threads with a deadline: the caller gets a timeout reply, the helper is detached, and a +//! normal call still works afterwards. +#![cfg(unix)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use craft_ipc::asset::{Described, Refreshed, Resolved}; +use craft_ipc::moniker::Moniker; +use craft_ipc::record::AssetKind; +use craft_ipc::refresh::FileState; +use craft_ipc::server::{Activate, Activation, Assets, ControlSink, Extensions, FrameOpen, Frames, Limits, Server, ServerConfig}; +use craft_ipc::{Endpoint, LinkError, RuntimeDir}; + +const DEADLINE: Duration = Duration::from_millis(400); +/// `link.refresh` with this link id blocks until the test releases it. `app.activate` with this +/// token blocks the same way. +const STUCK: &str = "stuck"; + +/// Blocks while `release` is false, so a test can end its helper threads when it is done. +struct Gate { + release: AtomicBool, + entered: AtomicUsize, +} + +impl Gate { + fn wait(&self) { + self.entered.fetch_add(1, Ordering::SeqCst); + while !self.release.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +struct StuckAssets(Arc); + +impl Assets for StuckAssets { + fn resolve(&self, _asset: &Moniker) -> Result { + Err(LinkError::Unknown("no such asset".into())) + } + + fn describe(&self, asset: &Moniker) -> Result { + Ok(Described { + asset: asset.clone(), + name: "x".into(), + kind: AssetKind::Still, + epoch: 1, + w: None, + h: None, + hash: None, + thumb_path: None, + modified_ms: None, + }) + } + + fn refresh(&self, link_id: &str) -> Result { + if link_id == STUCK { + self.0.wait(); + } + Ok(Refreshed::new(link_id, 1, FileState::Current, "/tmp/x", None)) + } +} + +struct StuckActivate(Arc); + +impl Activate for StuckActivate { + fn activate(&self, token: Option, _from_pid: Option, _open: Option) -> Activation { + if token.as_deref() == Some(STUCK) { + self.0.wait(); + } + Activation::Yes + } +} + +struct Sink; + +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +struct Rig { + server: Server, + gate: Arc, + reader: BufReader, + dir: PathBuf, +} + +impl Rig { + fn boot(label: &str, deadline: Duration) -> Self { + let dir = std::env::temp_dir().join(format!("cl-stall-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let gate = Arc::new(Gate { release: AtomicBool::new(false), entered: AtomicUsize::new(0) }); + let server = Server::start_with_limits( + ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(RuntimeDir::at(&dir).unwrap()), + tcp_port: None, + sink: Arc::new(Sink), + activate: Arc::new(StuckActivate(Arc::clone(&gate))), + frames: Arc::new(NoFrames), + }, + Extensions { assets: Some(Arc::new(StuckAssets(Arc::clone(&gate)))), caps: Vec::new() }, + Limits { provider_deadline: deadline, ..Limits::default() }, + ) + .unwrap(); + let Endpoint::Unix { path } = &server.record().endpoint else { panic!("unix endpoint expected") }; + let stream = UnixStream::connect(path).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(stream); + let token = server.record().token.clone(); + send(&mut reader, &json!({"id": 0, "method": "auth", "params": {"token": token}})); + assert_eq!(read(&mut reader)["ok"], json!(true)); + Self { server, gate, reader, dir } + } + + fn call(&mut self, id: u64, method: &str, params: Value) -> Value { + send(&mut self.reader, &json!({"id": id, "method": method, "params": params})); + read(&mut self.reader) + } + + /// Shut down on a thread and return how long it took. A hang fails the test after `limit`. + fn shutdown_within(self, limit: Duration) -> Duration { + let Self { server, gate, reader, dir } = self; + let (done, finished) = mpsc::channel(); + let started = Instant::now(); + std::thread::spawn(move || { + let result = server.shutdown(); + let _ = done.send(result.is_ok()); + }); + let waited = finished.recv_timeout(limit); + // Let the detached helpers end, so the test process leaves no spinning threads. + gate.release.store(true, Ordering::SeqCst); + drop(reader); + let _ = std::fs::remove_dir_all(dir); + assert!(waited.expect("shutdown did not finish: a stuck provider held it up"), "shutdown failed"); + started.elapsed() + } +} + +fn send(reader: &mut BufReader, value: &Value) { + let mut bytes = serde_json::to_vec(value).unwrap(); + bytes.push(b'\n'); + reader.get_mut().write_all(&bytes).unwrap(); +} + +fn read(reader: &mut BufReader) -> Value { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + serde_json::from_str(&line).unwrap_or_else(|err| panic!("reply {line:?}: {err}")) +} + +fn timed_out(reply: &Value) -> bool { + reply["ok"] == json!(false) && reply["error"]["message"] == json!("timeout") +} + +#[test] +fn a_stuck_asset_call_times_out_and_the_connection_keeps_working() { + let mut rig = Rig::boot("assets", DEADLINE); + let started = Instant::now(); + let reply = rig.call(1, "link.refresh", json!({"link_id": STUCK})); + let waited = started.elapsed(); + assert!(timed_out(&reply), "{reply}"); + assert_eq!(reply["id"], json!(1)); + assert!(waited >= DEADLINE && waited < DEADLINE * 5, "{waited:?}"); + // Other calls on the same connection still work, asset calls included. + let fine = rig.call(2, "link.refresh", json!({"link_id": "ok"})); + assert_eq!(fine["ok"], json!(true), "{fine}"); + assert_eq!(fine["result"]["link_id"], json!("ok")); + let hello = rig.call(3, "link.hello", json!({"protocol": {"min": 1, "max": 1}})); + assert_eq!(hello["ok"], json!(true), "{hello}"); + assert_eq!(rig.gate.entered.load(Ordering::SeqCst), 1); + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn shutdown_does_not_wait_for_a_stuck_asset_call() { + // The deadline is long, so shutdown meets the call still in flight. + let mut rig = Rig::boot("assets-stop", Duration::from_secs(60)); + send(&mut rig.reader, &json!({"id": 1, "method": "link.refresh", "params": {"link_id": STUCK}})); + while rig.gate.entered.load(Ordering::SeqCst) == 0 { + std::thread::sleep(Duration::from_millis(10)); + } + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn a_stuck_activate_call_times_out_and_shutdown_does_not_wait_for_it() { + let mut rig = Rig::boot("activate", DEADLINE); + let reply = rig.call(1, "app.activate", json!({"token": STUCK})); + assert!(timed_out(&reply), "{reply}"); + let fine = rig.call(2, "app.activate", json!({"token": "ok"})); + assert_eq!(fine["result"]["activated"], json!(true), "{fine}"); + // A second stuck call, then shut down while it is in flight. + send(&mut rig.reader, &json!({"id": 3, "method": "app.activate", "params": {"token": STUCK}})); + while rig.gate.entered.load(Ordering::SeqCst) < 2 { + std::thread::sleep(Duration::from_millis(10)); + } + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn stuck_helpers_are_capped_and_the_server_says_busy() { + let mut rig = Rig::boot("cap", Duration::from_millis(60)); + let mut busy_at = None; + for id in 1..=40u64 { + let reply = rig.call(id, "link.refresh", json!({"link_id": STUCK})); + if reply["error"]["code"] == json!("busy") { + busy_at = Some(id); + break; + } + assert!(timed_out(&reply), "call {id}: {reply}"); + } + let busy_at = busy_at.expect("the server kept starting helper threads past its cap"); + assert_eq!(busy_at, 33, "32 stuck helpers are allowed, the next call is refused"); + assert_eq!(rig.gate.entered.load(Ordering::SeqCst), 32, "a refused call starts no thread"); + // A call that needs no helper still works while the cap is full. + let hello = rig.call(100, "link.hello", json!({"protocol": {"min": 1, "max": 1}})); + assert_eq!(hello["ok"], json!(true), "{hello}"); + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} diff --git a/crates/ipc/tests/schemas.rs b/crates/ipc/tests/schemas.rs new file mode 100644 index 000000000..4dbcd251b --- /dev/null +++ b/crates/ipc/tests/schemas.rs @@ -0,0 +1,318 @@ +//! The JSON schemas in `schemas/` and the Rust types +//! must accept the same documents. Each table row is checked against both. +//! The fixtures are stored records. The first one is the FilmCraft to +//! EffectCraft shape written before the asset fields existed. + +use std::path::PathBuf; + +use jsonschema::Validator; +use serde_json::{Value, json}; + +use craft_ipc::InstanceRecord; +use craft_ipc::record::{AssetKind, LinkRecord}; + +fn schema_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schemas") +} + +fn validator(name: &str) -> Validator { + let path = schema_dir().join(name); + let schema: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + jsonschema::options().should_validate_formats(true).build(&schema).unwrap() +} + +fn fixture(name: &str) -> Value { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures").join(name); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +} + +/// Schema verdict and Rust verdict for one document. Both must equal `want`. +fn check_link(schema: &Validator, doc: &Value, want: bool, why: &str) { + let by_schema = schema.is_valid(doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, want, "schema: {why}: {doc}"); + assert_eq!(by_rust, want, "rust: {why}: {doc}"); +} + +#[test] +fn stored_filmcraft_effectcraft_record_stays_valid() { + let schema = validator("link-record.schema.json"); + let doc = fixture("link-record-filmcraft-effectcraft.json"); + check_link(&schema, &doc, true, "legacy fixture"); + let record: LinkRecord = serde_json::from_value(doc.clone()).unwrap(); + assert_eq!(record.target_app, "effectcraft"); + assert_eq!(record.target_comp.as_deref(), Some("comp-1")); + assert_eq!(record.nested_sequence.as_deref(), Some("item-9")); + assert_eq!(record.format.rate_num, Some(24)); + assert_eq!(record.format.duration_ticks, Some(240)); + assert!(record.asset.is_none() && record.producer.is_none()); + // Unknown fields survive a load and save, byte-for-byte as JSON values. + assert_eq!(serde_json::to_value(&record).unwrap(), doc); +} + +#[test] +fn still_record_has_no_video_fields_and_round_trips() { + let schema = validator("link-record.schema.json"); + let doc = fixture("link-record-still.json"); + check_link(&schema, &doc, true, "still fixture"); + let record: LinkRecord = serde_json::from_value(doc.clone()).unwrap(); + assert_eq!(record.target_app, "photocraft"); + assert!(record.nested_sequence.is_none() && record.target_comp.is_none()); + assert_eq!(record.format.rate_num, None); + assert_eq!(record.format.rate_den, None); + assert_eq!(record.format.duration_ticks, None); + assert_eq!(record.asset.as_ref().map(|a| a.kind), Some(AssetKind::Still)); + let out = serde_json::to_value(&record).unwrap(); + assert_eq!(out, doc); + let format = out["format"].as_object().unwrap(); + for key in ["rate_num", "rate_den", "duration_ticks"] { + assert!(!format.contains_key(key), "{key} must not be written as null"); + } +} + +#[test] +fn schema_and_rust_agree_on_edge_cases() { + let schema = validator("link-record.schema.json"); + let legacy = fixture("link-record-filmcraft-effectcraft.json"); + let still = fixture("link-record-still.json"); + + let edit = |base: &Value, f: &dyn Fn(&mut Value)| { + let mut doc = base.clone(); + f(&mut doc); + doc + }; + let cases: Vec<(Value, bool, &str)> = vec![ + (edit(&legacy, &|d| d["target_app"] = json!("photocraft")), true, "any app id"), + (edit(&legacy, &|d| d["target_app"] = json!("Film")), false, "uppercase app id"), + (edit(&legacy, &|d| d["target_app"] = json!("")), false, "empty app id"), + (edit(&legacy, &|d| d["v"] = json!(2)), false, "v 2"), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("nested_sequence"); + }), + true, + "nested_sequence optional", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("duration_ticks"); + }), + true, + "duration optional", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("rate_num"); + }), + false, + "rate_den without rate_num", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("rate_den"); + }), + false, + "rate_num without rate_den", + ), + (edit(&legacy, &|d| d["format"]["rate_num"] = json!(0)), false, "rate_num 0"), + (edit(&legacy, &|d| d["format"]["w"] = json!(0)), false, "w 0"), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("alpha"); + }), + false, + "alpha stays required", + ), + (edit(&legacy, &|d| d["result_epoch"] = json!(0)), true, "epoch 0"), + (edit(&legacy, &|d| d["created"] = json!("yesterday")), false, "bad date"), + (edit(&legacy, &|d| d["created"] = json!("2026-02-31T09:30:00Z")), false, "31 February"), + (edit(&legacy, &|d| d["created"] = json!("2026-02-29T09:30:00Z")), false, "29 February in a common year"), + (edit(&legacy, &|d| d["created"] = json!("2026-04-31T09:30:00+02:00")), false, "31 April"), + (edit(&legacy, &|d| d["created"] = json!("1900-02-29T00:00:00Z")), false, "29 February in a century year"), + (edit(&legacy, &|d| d["created"] = json!("2024-02-29T09:30:00Z")), true, "29 February in a leap year"), + (edit(&legacy, &|d| d["created"] = json!("2000-02-29T09:30:00.250Z")), true, "29 February in a 400 year"), + (edit(&legacy, &|d| d["created"] = json!("2026-12-31T23:59:59-05:30")), true, "last day of the year"), + (edit(&legacy, &|d| d["link_id"] = json!("nope")), false, "bad link id"), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("target_comp"); + }), + false, + "no comp and no asset", + ), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("target_project"); + }), + false, + "target_project required", + ), + ( + edit(&still, &|d| { + d.as_object_mut().unwrap().remove("asset"); + }), + false, + "still without asset or comp", + ), + (edit(&still, &|d| d["asset"]["kind"] = json!("video")), false, "unknown asset kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("vector")), true, "vector kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("sequence")), true, "sequence kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("comp")), true, "comp kind"), + (edit(&still, &|d| d["asset"]["moniker"] = json!("craft://photocraft/not-a-uuid/image/x")), false, "bad moniker doc"), + (edit(&still, &|d| d["asset"]["moniker"] = json!("photocraft/x")), false, "moniker without scheme"), + ( + edit(&still, &|d| d["asset"]["moniker"] = json!("craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/a/b")), + false, + "moniker with extra segment", + ), + (edit(&still, &|d| d["producer"]["app"] = json!("Bad App")), false, "bad producer app"), + ( + edit(&still, &|d| { + d.as_object_mut().unwrap().remove("producer"); + }), + true, + "producer optional", + ), + (edit(&still, &|d| d["asset"]["extra"] = json!(1)), true, "unknown asset field kept"), + ]; + for (doc, want, why) in &cases { + check_link(&schema, doc, *want, why); + } +} + +const MONIKER_BASE: &str = "craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/"; + +fn with_moniker(still: &Value, id: &str) -> Value { + let mut doc = still.clone(); + doc["asset"]["moniker"] = json!(format!("{MONIKER_BASE}{id}")); + doc +} + +/// The moniker id rules of `Moniker::parse` as the schema states them: unreserved characters and +/// two-digit escapes only, 1 to 512 decoded bytes, and decoded bytes that are valid UTF-8. +#[test] +fn schema_and_rust_agree_on_moniker_ids() { + let schema = validator("link-record.schema.json"); + let still = fixture("link-record-still.json"); + let escaped = |count: usize| "%41".repeat(count); + let cases: Vec<(String, bool, &str)> = vec![ + ("layer-3".into(), true, "plain"), + ("~._-AZaz09".into(), true, "every unreserved class"), + ("a%20b".into(), true, "escaped space"), + ("a%2fb".into(), true, "lowercase escape"), + ("%C3%A9".into(), true, "two byte character"), + ("%c3%a9".into(), true, "two byte character in lowercase hex"), + ("%E2%82%AC".into(), true, "three byte character"), + ("%F0%9F%98%80".into(), true, "four byte character"), + ("%F4%8F%BF%BF".into(), true, "highest scalar"), + ("%ED%9F%BF".into(), true, "just below the surrogates"), + ("%00".into(), true, "escaped NUL is a byte like any other"), + ("%GG".into(), false, "bad escape"), + ("%4".into(), false, "truncated escape"), + ("%".into(), false, "lone percent"), + ("a b".into(), false, "raw space"), + ("caf\u{e9}".into(), false, "raw non-ASCII"), + ("a:b".into(), false, "raw colon"), + ("a%2".into(), false, "truncated escape after text"), + ("".into(), false, "empty id"), + ("%FF".into(), false, "invalid UTF-8 byte"), + ("%C3".into(), false, "truncated two byte character"), + ("%80".into(), false, "lone continuation byte"), + ("%C0%80".into(), false, "overlong two byte form"), + ("%E0%80%80".into(), false, "overlong three byte form"), + ("%ED%A0%80".into(), false, "surrogate"), + ("%F0%80%80%80".into(), false, "overlong four byte form"), + ("%F4%90%80%80".into(), false, "above U+10FFFF"), + ("%F5%80%80%80".into(), false, "lead byte F5"), + ("a/b".into(), false, "raw slash"), + ("a?b".into(), false, "raw query"), + ("a#b".into(), false, "raw fragment"), + ("a".repeat(512), true, "512 raw bytes"), + ("a".repeat(513), false, "513 raw bytes"), + (escaped(512), true, "512 escaped bytes"), + (escaped(513), false, "513 escaped bytes"), + (format!("{}%C3%A9", "a".repeat(510)), true, "512 decoded bytes ending in a two byte character"), + (format!("{}%C3%A9", "a".repeat(511)), false, "513 decoded bytes ending in a two byte character"), + ("%C3%A9".repeat(256), true, "256 two byte characters make 512 bytes"), + ("%C3%A9".repeat(257), false, "257 two byte characters make 514 bytes"), + ]; + for (id, want, why) in &cases { + check_link(&schema, &with_moniker(&still, id), *want, why); + } + + // Every lead byte against a spread of second and third bytes: the two verdicts must match. + let probes = [0x00u8, 0x41, 0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0]; + let mut seen_valid = 0; + for lead in 0u8..=0xFF { + for second in probes { + for tail in [&[][..], &[0x80][..], &[0x80, 0x80][..]] { + for hex in [|b: u8| format!("%{b:02X}"), |b: u8| format!("%{b:02x}")] { + let id: String = [lead, second].iter().chain(tail).map(|byte| hex(*byte)).collect(); + let doc = with_moniker(&still, &id); + let by_schema = schema.is_valid(&doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, by_rust, "bytes {lead:02X} {second:02X} {tail:02X?} as {id}"); + seen_valid += usize::from(by_schema); + } + } + } + } + assert!(seen_valid > 100, "the probe must reach valid sequences too, got {seen_valid}"); +} + +fn instance_doc() -> Value { + let endpoint = if cfg!(windows) { + json!({"kind": "pipe", "name": "craft-hub-4242"}) + } else { + json!({"kind": "unix", "path": "/run/user/1000/craft/hub-4242.sock"}) + }; + json!({ + "v": 1, + "app": "hub", + "app_version": "0.1.0", + "pid": 4242, + "instance": "550e8400-e29b-41d4-a716-446655440000", + "endpoint": endpoint, + "token": "ab".repeat(32), + "started": 1_760_000_000u64, + "protocol": {"link": 1}, + "docs": [] + }) +} + +fn check_instance(schema: &Validator, doc: &Value, want: bool, why: &str) { + let by_schema = schema.is_valid(doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, want, "schema: {why}: {doc}"); + assert_eq!(by_rust, want, "rust: {why}: {doc}"); +} + +#[test] +fn instance_record_caps_agree_between_schema_and_rust() { + let schema = validator("instance-record.schema.json"); + let base = instance_doc(); + check_instance(&schema, &base, true, "no caps (existing records)"); + for (caps, want, why) in [ + (json!(["hub"]), true, "hub"), + (json!(["hub", "assets"]), true, "two caps"), + (json!([]), true, "empty caps"), + (json!(["Hub"]), false, "uppercase"), + (json!([""]), false, "empty cap"), + (json!(["hub", "hub"]), false, "duplicate"), + (json!("hub"), false, "not an array"), + (json!([1]), false, "not strings"), + ] { + let mut doc = base.clone(); + doc["caps"] = caps; + check_instance(&schema, &doc, want, why); + } + let mut hub: InstanceRecord = serde_json::from_value(base).unwrap(); + assert!(hub.caps().is_empty()); + hub.add_cap("hub").unwrap(); + hub.add_cap("hub").unwrap(); + assert_eq!(hub.caps(), vec!["hub".to_string()]); + assert!(hub.add_cap("Not Valid").is_err()); + let value = serde_json::to_value(&hub).unwrap(); + assert_eq!(value["caps"], json!(["hub"])); + assert!(schema.is_valid(&value)); +} diff --git a/crates/ipc/tests/security.rs b/crates/ipc/tests/security.rs new file mode 100644 index 000000000..0c42c599a --- /dev/null +++ b/crates/ipc/tests/security.rs @@ -0,0 +1,290 @@ +//! Auth refusal on the local socket. These tests speak bytes themselves. +//! They do not use the client module. +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct Sink; + +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({})); + Some(rx) + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + + fn close_ticket(&self, _ticket: &str) {} +} + +struct Running { + _tmp: Tmp, + server: Server, + socket: PathBuf, + token: String, +} + +fn boot() -> Running { + boot_with(None) +} + +fn boot_with(tcp_port: Option) -> Running { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-sec-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(runtime), + tcp_port, + sink: Arc::new(Sink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let socket = match &server.record().endpoint { + craft_ipc::Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + }; + let token = server.record().token.clone(); + Running { _tmp: Tmp(path), server, socket, token } +} + +fn read_closed_line(socket: &Path, payload: &[u8]) -> Value { + let mut stream = std::os::unix::net::UnixStream::connect(socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.write_all(payload).unwrap(); + let mut text = String::new(); + stream.read_to_string(&mut text).unwrap_or_else(|err| panic!("socket did not close after auth failure: {err}")); + let lines: Vec<_> = text.lines().filter(|line| !line.is_empty()).collect(); + assert_eq!(lines.len(), 1, "{text}"); + assert!(!lines[0].contains("bad JSON"), "{text}"); + serde_json::from_str(lines[0]).unwrap() +} + +fn assert_auth_failure(value: &Value) { + assert_eq!(value["id"], json!(0), "{value}"); + assert_eq!(value["ok"], json!(false), "{value}"); + assert_eq!(value["error"], json!("authentication required"), "{value}"); +} + +#[test] +fn no_token_is_auth_failure_not_bad_json() { + let running = boot(); + let value = read_closed_line(&running.socket, b"{\"id\":1,\"method\":\"engine.commands\",\"params\":{}}\n"); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn wrong_token_is_auth_failure() { + let running = boot(); + let mut bad = "ab".repeat(32); + if bad == running.token { + bad = "cd".repeat(32); + } + let line = json!({"id": 4, "method": "auth", "params": {"token": bad}}); + let mut bytes = serde_json::to_vec(&line).unwrap(); + bytes.push(b'\n'); + let value = read_closed_line(&running.socket, &bytes); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn malformed_first_line_is_auth_failure() { + let running = boot(); + let value = read_closed_line(&running.socket, b"{\n"); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_is_off_when_the_port_is_none() { + let running = boot(); + assert_eq!(running.server.tcp_port(), None); + running.server.shutdown().unwrap(); +} + +fn tcp_exchange(port: u16, payload: &[u8]) -> String { + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.write_all(payload).unwrap(); + let mut text = String::new(); + let _ = stream.read_to_string(&mut text); + text +} + +#[test] +fn tcp_without_a_token_is_refused_and_runs_nothing() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + assert_ne!(port, 0); + let text = tcp_exchange(port, b"{\"id\":7,\"method\":\"engine.commands\",\"params\":{}}\n"); + let first = text.lines().find(|line| !line.is_empty()).unwrap_or_else(|| panic!("tcp closed with no refusal line")); + assert_auth_failure(&serde_json::from_str(first).unwrap()); + assert!(!text.contains("\"result\""), "{text}"); + let bad_json = tcp_exchange(port, b"{\n"); + assert!(!bad_json.contains("bad JSON"), "{bad_json}"); + assert!(bad_json.contains("authentication required"), "{bad_json}"); + let mut wrong = "ab".repeat(32); + if wrong == running.token { + wrong = "cd".repeat(32); + } + let line = json!({"id": 0, "method": "auth", "params": {"token": wrong}}); + let mut bytes = serde_json::to_vec(&line).unwrap(); + bytes.push(b'\n'); + assert_auth_failure(&serde_json::from_str(tcp_exchange(port, &bytes).lines().next().unwrap()).unwrap()); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_with_the_record_token_works_and_the_record_publishes_the_port() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + match &running.server.record().endpoint { + craft_ipc::Endpoint::Unix { .. } => {} + other => panic!("the instance record must keep the unix socket, got {other:?}"), + } + assert_eq!(running.server.record().tcp_port(), Some(port)); + let mut record = running.server.record().clone(); + record.endpoint = craft_ipc::Endpoint::Tcp { port }; + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + let auth = json!({"id": 0, "method": "auth", "params": {"token": running.token}}); + let mut bytes = serde_json::to_vec(&auth).unwrap(); + bytes.extend_from_slice(b"\n{\"id\":7,\"method\":\"engine.commands\",\"params\":{}}\n"); + stream.write_all(&bytes).unwrap(); + let mut buf = Vec::new(); + while buf.iter().filter(|b| **b == b'\n').count() < 2 { + let mut byte = [0u8; 1]; + assert_ne!(stream.read(&mut byte).unwrap(), 0, "tcp closed before a reply"); + buf.push(byte[0]); + } + let text = String::from_utf8(buf).unwrap(); + let mut lines = text.lines(); + let ok: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + assert_eq!(ok["ok"], json!(true), "{ok}"); + let value: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + assert_eq!(value["id"], json!(7), "{value}"); + assert_eq!(value["ok"], json!(true), "{value}"); + assert_eq!(value["result"], json!({}), "{value}"); + // The library client does the same from the published record. + let mut client = craft_ipc::client::Client::connect_tcp(running.server.record()).unwrap(); + assert_eq!(client.call("engine.commands", json!({})).unwrap(), json!({})); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_is_not_published_when_it_is_off() { + let running = boot(); + assert_eq!(running.server.record().tcp_port(), None); + assert!(craft_ipc::client::Client::connect_tcp(running.server.record()).is_err()); + running.server.shutdown().unwrap(); +} + +#[test] +fn silent_tcp_connections_hit_the_unauthenticated_cap() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + let silent: Vec<_> = (0..craft_ipc::wire::MAX_UNAUTHENTICATED).map(|_| std::net::TcpStream::connect(("127.0.0.1", port)).unwrap()).collect(); + std::thread::sleep(Duration::from_millis(400)); + let text = tcp_exchange(port, b""); + assert!(text.contains("too many unauthenticated connections"), "{text}"); + drop(silent); + running.server.shutdown().unwrap(); +} + +#[test] +fn other_uid_is_refused_when_the_env_is_set() { + let Ok(user) = std::env::var("CRAFT_TEST_OTHER_USER") else { + return; + }; + let running = boot(); + // 0600 blocks a non-owner before the peer check. Loosen it so this user + // can connect. The assertion is still the auth failure line. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&running.socket, std::fs::Permissions::from_mode(0o666)).unwrap(); + } + let script = r#" +import socket, sys +s = socket.socket(socket.AF_UNIX) +s.settimeout(5) +s.connect(sys.argv[1]) +token = sys.argv[2] +msg = '{"id":1,"method":"auth","params":{"token":"%s"}}\n' % token +s.sendall(msg.encode()) +data = b"" +while b"\n" not in data: + chunk = s.recv(4096) + if not chunk: + break + data += chunk +sys.stdout.buffer.write(data) +"#; + let output = run_as(&user, script, &running.socket, &running.token); + if !output.status.success() { + panic!("run as {user} failed: status {:?} stderr {}", output.status.code(), String::from_utf8_lossy(&output.stderr)); + } + let text = String::from_utf8_lossy(&output.stdout); + assert!(!text.contains("bad JSON"), "{text}"); + let value: Value = serde_json::from_str(text.lines().next().unwrap()).unwrap(); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +fn run_as(user: &str, script: &str, socket: &Path, token: &str) -> std::process::Output { + let args = ["-u", user, "--", "python3", "-c", script]; + match std::process::Command::new("runuser").args(args).arg(socket).arg(token).output() { + Ok(output) => output, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + std::process::Command::new("sudo").args(args).arg(socket).arg(token).output().unwrap_or_else(|err| { + panic!("sudo failed to start: {err}"); + }) + } + Err(err) => panic!("runuser failed to start: {err}"), + } +} diff --git a/crates/ipc/tests/watch.rs b/crates/ipc/tests/watch.rs new file mode 100644 index 000000000..185990e92 --- /dev/null +++ b/crates/ipc/tests/watch.rs @@ -0,0 +1,361 @@ +//! Scheduled polling. A `ManualTicker` stands in for the clock, so no test +//! sleeps to wait for an interval. Real-time waits only bound a failure. + +use std::fs::OpenOptions; +use std::path::PathBuf; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::json; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::refresh::{Change, FileState, Tracker, write_atomic}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::watch::{DEEP_INTERVAL, IntervalTicker, Intervals, ManualTicker, Notice, SHALLOW_INTERVAL, Ticker, Watcher}; +use craft_ipc::{Discovery, LinkError, RuntimeDir}; + +const LIMIT: Duration = Duration::from_secs(10); +const SECOND: Duration = Duration::from_secs(1); + +struct Tmp(PathBuf); + +impl Tmp { + fn new(label: &str) -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-watch-{label}-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn forward(tx: Sender) -> impl Fn(Notice) + Send + Sync + 'static { + let tx = Mutex::new(tx); + move |notice| { + let _ = tx.lock().unwrap().send(notice); + } +} + +struct Rig { + _tmp: Tmp, + file: PathBuf, + clock: Arc, + notices: Receiver, + watcher: Watcher, +} + +impl Rig { + /// A watcher on `a.png` with content "one" and epoch 4, on the contract intervals. + fn new(label: &str) -> Self { + let tmp = Tmp::new(label); + let file = tmp.0.join("a.png"); + std::fs::write(&file, b"one").unwrap(); + let clock = Arc::new(ManualTicker::new()); + let (tx, notices) = channel(); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), forward(tx)).unwrap(); + watcher.watch("link-1", Tracker::new(&file, 4).unwrap()).unwrap(); + Self { _tmp: tmp, file, clock, notices, watcher } + } + + /// Move time and wait until the watcher thread has handled it. + fn advance(&self, by: Duration) { + self.clock.advance(by); + assert!(self.clock.settle(LIMIT), "the watcher did not settle"); + } + + fn quiet(&self) { + let got = self.notices.try_recv(); + assert!(got.is_err(), "unexpected notice {got:?}"); + } + + fn next(&self) -> Notice { + self.notices.try_recv().expect("a notice was due") + } +} + +fn changed(link: &str, state: FileState, epoch: u64, bumped: bool, deep: bool) -> Notice { + Notice::Changed { link_id: link.into(), change: Change { state, epoch, bumped }, deep } +} + +#[test] +fn contract_intervals_are_one_and_thirty_seconds() { + assert_eq!(SHALLOW_INTERVAL, Duration::from_secs(1)); + assert_eq!(DEEP_INTERVAL, Duration::from_secs(30)); + assert_eq!(Intervals::default(), Intervals { shallow: SHALLOW_INTERVAL, deep: DEEP_INTERVAL }); +} + +#[test] +fn edit_on_disk_bumps_the_epoch_on_the_first_shallow_tick_and_not_before() { + let rig = Rig::new("shallow"); + write_atomic(&rig.file, b"two!").unwrap(); + rig.advance(Duration::from_millis(500)); + rig.advance(Duration::from_millis(499)); + rig.quiet(); + assert_eq!(rig.watcher.poll_counts().shallow, 0, "no poll before the 1 s interval"); + rig.advance(Duration::from_millis(1)); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, false)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (1, 0)); + // No further edit, no further notice, and the next poll comes one interval later. + rig.advance(Duration::from_millis(999)); + assert_eq!(rig.watcher.poll_counts().shallow, 1); + rig.advance(Duration::from_millis(1)); + assert_eq!(rig.watcher.poll_counts().shallow, 2); + rig.quiet(); + // A second edit bumps again. + write_atomic(&rig.file, b"three").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 6, true, false)); +} + +#[test] +fn deep_poll_runs_at_thirty_seconds_and_finds_an_edit_that_keeps_size_and_mtime() { + let rig = Rig::new("deep"); + let mtime = std::fs::metadata(&rig.file).unwrap().modified().unwrap(); + std::fs::write(&rig.file, b"two").unwrap(); + OpenOptions::new().write(true).open(&rig.file).unwrap().set_modified(mtime).unwrap(); + for _ in 1..30 { + rig.advance(SECOND); + } + rig.quiet(); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (29, 0), "shallow polls are blind to this edit"); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, true)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (29, 1), "the deep poll replaces the shallow poll on its tick"); + // The schedule goes on: shallow at 31 s, deep again at 60 s. + rig.advance(SECOND); + assert_eq!(rig.watcher.poll_counts().shallow, 30); + rig.advance(Duration::from_secs(29)); + assert_eq!(rig.watcher.poll_counts().deep, 2); +} + +#[test] +fn a_large_time_jump_is_one_deep_poll_and_the_grid_stays_aligned() { + let rig = Rig::new("jump"); + rig.advance(Duration::from_secs(100)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (0, 1)); + rig.advance(Duration::from_millis(500)); + assert_eq!(rig.watcher.poll_counts().shallow, 0, "100.5 s is before the next grid point at 101 s"); + rig.advance(Duration::from_millis(500)); + assert_eq!(rig.watcher.poll_counts().shallow, 1, "101 s is the next grid point"); +} + +#[test] +fn producer_offline_missing_and_return_are_reported_without_a_bump() { + let rig = Rig::new("offline"); + // A touch alone is silent. + let file = OpenOptions::new().write(true).open(&rig.file).unwrap(); + file.set_modified(std::time::SystemTime::now() + Duration::from_secs(60)).unwrap(); + drop(file); + rig.advance(SECOND); + rig.quiet(); + std::fs::remove_file(&rig.file).unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Missing, 4, false, false)); + rig.advance(SECOND); + rig.quiet(); + std::fs::write(&rig.file, b"one").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Current, 4, false, false)); + rig.quiet(); +} + +#[test] +fn read_errors_are_reported_once_and_recover() { + let rig = Rig::new("error"); + std::fs::remove_file(&rig.file).unwrap(); + std::fs::create_dir(&rig.file).unwrap(); + rig.advance(SECOND); + let Notice::PollFailed { link_id, error } = rig.next() else { + panic!("expected a failure notice"); + }; + assert_eq!(link_id, "link-1"); + assert!(matches!(error, LinkError::Invalid(_)), "{error:?}"); + rig.advance(SECOND); + rig.quiet(); + std::fs::remove_dir(&rig.file).unwrap(); + std::fs::write(&rig.file, b"different").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, false)); +} + +#[test] +fn with_tracker_accept_silences_the_next_notice_and_unwatch_stops_polling() { + let rig = Rig::new("accept"); + write_atomic(&rig.file, b"two!").unwrap(); + rig.advance(SECOND); + rig.next(); + let state = rig.watcher.with_tracker("link-1", |tracker| { + tracker.accept(); + tracker.state() + }); + assert_eq!(state, Some(FileState::Current)); + rig.advance(SECOND); + rig.quiet(); + assert!(rig.watcher.with_tracker("nope", |_| ()).is_none()); + assert!(rig.watcher.watch("link-1", Tracker::new(&rig.file, 0).unwrap()).is_err(), "duplicate id"); + let tracker = rig.watcher.unwatch("link-1").unwrap(); + assert_eq!(tracker.epoch(), 5); + write_atomic(&rig.file, b"after unwatch").unwrap(); + rig.advance(SECOND); + rig.quiet(); + assert!(rig.watcher.unwatch("link-1").is_none()); +} + +#[test] +fn several_links_are_polled_together() { + let rig = Rig::new("many"); + let other = rig._tmp.0.join("b.png"); + std::fs::write(&other, b"b").unwrap(); + rig.watcher.watch("link-2", Tracker::new(&other, 0).unwrap()).unwrap(); + write_atomic(&other, b"bb").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-2", FileState::Modified, 1, true, false)); + rig.quiet(); +} + +#[test] +fn bad_intervals_are_refused() { + let clock = Arc::new(ManualTicker::new()); + let zero = Intervals { shallow: Duration::ZERO, deep: SECOND }; + assert!(Watcher::new(Arc::clone(&clock) as Arc, zero, |_| ()).is_err()); + let inverted = Intervals { shallow: Duration::from_secs(5), deep: SECOND }; + assert!(Watcher::new(clock as Arc, inverted, |_| ()).is_err()); + assert!(IntervalTicker::new(Duration::ZERO).is_err()); +} + +#[test] +fn drop_stops_the_thread_at_once_even_with_a_slow_ticker() { + let ticker = Arc::new(IntervalTicker::new(Duration::from_secs(3600)).unwrap()); + let watcher = + Watcher::new(ticker as Arc, Intervals { shallow: Duration::from_secs(3600), deep: Duration::from_secs(7200) }, |_| ()).unwrap(); + let (done, finished) = channel(); + std::thread::spawn(move || { + drop(watcher); + let _ = done.send(()); + }); + finished.recv_timeout(LIMIT).expect("drop must not wait for the next tick"); + + let clock = Arc::new(ManualTicker::new()); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), |_| ()).unwrap(); + assert!(clock.settle(LIMIT)); + drop(watcher); + assert!(clock.wait().is_none(), "a closed ticker stays closed"); + assert!(!clock.settle(Duration::from_millis(1))); +} + +#[test] +fn the_native_ticker_drives_a_watcher_without_a_manual_clock() { + let tmp = Tmp::new("native"); + let file = tmp.0.join("a.png"); + std::fs::write(&file, b"one").unwrap(); + let (tx, notices) = channel(); + let ticker = Arc::new(IntervalTicker::new(Duration::from_millis(10)).unwrap()); + let intervals = Intervals { shallow: Duration::from_millis(10), deep: Duration::from_millis(50) }; + let watcher = Watcher::new(ticker as Arc, intervals, forward(tx)).unwrap(); + watcher.watch("link-1", Tracker::new(&file, 0).unwrap()).unwrap(); + write_atomic(&file, b"two!").unwrap(); + let got = notices.recv_timeout(LIMIT).expect("a notice within the limit"); + // The first poll that sees the edit is shallow or deep, depending on when the 50 ms deep deadline falls. + let deep = matches!(got, Notice::Changed { deep: true, .. }); + assert_eq!(got, changed("link-1", FileState::Modified, 1, true, deep)); + drop(watcher); +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: serde_json::Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &serde_json::Value) -> Result<(serde_json::Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[test] +fn a_disk_edit_reaches_a_subscribed_client_as_link_changed() { + let tmp = Tmp::new("wire"); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let server = Arc::new( + Server::start(ServerConfig { + app: "filmcraft".into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(), + ); + let record = Discovery::scan(&runtime).unwrap().into_iter().next().unwrap(); + let mut client = Client::connect(&record).unwrap(); + client + .hello(&HelloClient { + app: "hub".into(), + app_version: "0.1.0".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440002".into(), + protocol_min: 1, + protocol_max: 1, + }) + .unwrap(); + client.subscribe(&["link.changed"]).unwrap(); + + let file = tmp.0.join("still.png"); + std::fs::write(&file, b"v1").unwrap(); + let clock = Arc::new(ManualTicker::new()); + let emitter = Arc::clone(&server); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), move |notice| { + if let Notice::Changed { link_id, change, .. } = notice + && change.bumped + { + emitter.emit("link.changed", json!({"link_id": link_id, "epoch": change.epoch})).unwrap(); + } + }) + .unwrap(); + watcher.watch("link-9", Tracker::new(&file, 7).unwrap()).unwrap(); + + clock.advance(SECOND); + assert!(clock.settle(LIMIT)); + assert!(client.next_event(Some(Duration::from_millis(50))).unwrap().is_none(), "no edit, no event"); + write_atomic(&file, b"v2 is longer").unwrap(); + clock.advance(SECOND); + assert!(clock.settle(LIMIT)); + let event = client.next_event(Some(LIMIT)).unwrap().expect("a link.changed event"); + assert_eq!(event.event, "link.changed"); + assert_eq!(event.data, json!({"link_id": "link-9", "epoch": 8})); + + drop(watcher); + Arc::try_unwrap(server).ok().expect("the watcher released the server").shutdown().unwrap(); +} From 3919a2858374bada1e6542f9cae3db50f701f6fb Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:55:27 +0100 Subject: [PATCH 21/27] svg: a scene in local coordinates, with fill rules and element ids (#778, #789) (#1859) * svg: a scene in local coordinates, with fill rules and element ids (#778, #789) Document::from_tree_local keeps every node in its own user space, with its transform relative to its parent, so an app can keep geometry editable apart from where it is placed (EffectCraft's shape layers). Paths now carry their fill rule, id and transform, and groups their id. from_tree is unchanged: segments, strokes and group transforms stay absolute, path transforms are the identity. * craft-svg: place local text and glyph groups in their own frames (#789) * svg: pattern paints carry their tile and content (#778, #789) (#1863) Paint::Pattern now holds the resolved tile: id, rect, transform and the content as scene nodes in tile space. Each pattern is converted once and shared by every use; content past the depth limit, or a pattern that refers back to itself, has none. In absolute scenes the tile transform maps to the document. * craft-svg: report scene-omitted group effects and unsupported elements (#789) (#1870) --- crates/svg/src/tree.rs | 37 ++ crates/svg/src/tree/document.rs | 457 ++++++++++++++++++------- crates/svg/src/tree/document_tests.rs | 349 +++++++++++++++++++ crates/svg/src/tree/primitive_tests.rs | 3 + 4 files changed, 715 insertions(+), 131 deletions(-) create mode 100644 crates/svg/src/tree/document_tests.rs diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 99b6f73a4..ab04a3002 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -174,12 +174,49 @@ impl Preflight<'_> { self.depth } + /// The SVG elements in the document that the parsed tree leaves out, by name and count, + /// sorted by name: animation, scripts, embedded HTML and media, and text when this build + /// converts no text (neither `tree-text` nor `tree-render`). For an import's "not imported" notes. + pub fn unsupported_elements(&self) -> Vec<(&'static str, usize)> { + const SVG_NS: &str = "http://www.w3.org/2000/svg"; + let mut counts: std::collections::BTreeMap<&'static str, usize> = std::collections::BTreeMap::new(); + for node in self.xml.descendants().filter(roxmltree::Node::is_element) { + if !matches!(node.tag_name().namespace(), None | Some(SVG_NS)) { + continue; + } + if let Some(name) = UNSUPPORTED.iter().copied().find(|name| *name == node.tag_name().name()) { + *counts.entry(name).or_default() += 1; + } + } + counts.into_iter().collect() + } + /// Converts this already-checked XML into the configured usvg tree. pub fn parse(self, options: &usvg::Options<'_>) -> Result { usvg::Tree::from_xmltree(&self.xml, options).map_err(|e| Error::Parse(e.to_string())) } } +/// Elements the parsed tree never holds: usvg reads static content only, and text needs fonts. +const UNSUPPORTED: &[&str] = &[ + "animate", + "animateColor", + "animateMotion", + "animateTransform", + "audio", + "canvas", + "discard", + "foreignObject", + "iframe", + "script", + "set", + // usvg converts text when built with its `text` feature: `tree-text`, or `tree-render` + // through resvg. + #[cfg(not(any(feature = "tree-text", feature = "tree-render")))] + "text", + "video", +]; + /// Checks already-decoded SVG text once, retaining the parsed XML for both depth policy and tree conversion. pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { let preflight_limits = crate::preflight::Limits { diff --git a/crates/svg/src/tree/document.rs b/crates/svg/src/tree/document.rs index 838f62e98..19f594173 100644 --- a/crates/svg/src/tree/document.rs +++ b/crates/svg/src/tree/document.rs @@ -1,8 +1,12 @@ //! A format-neutral vector scene made from a bounded SVG tree. //! //! The scene preserves paths, groups, paints, transforms, opacity, strokes and image sources -//! without depending on an application's document model. Coordinates and stroke widths use the -//! SVG tree's CSS-pixel coordinate system. +//! without depending on an application's document model. +//! +//! [`Document::from_tree`] gives every coordinate and stroke width in the document's CSS pixels +//! ([`Coordinates::Absolute`]). [`Document::from_tree_local`] keeps each node in its own user +//! space with its transform relative to its parent ([`Coordinates::Local`]), for an app that +//! keeps a node's geometry editable apart from where it is placed. #[cfg(feature = "tree-render")] use super::draw::Refusal; @@ -18,9 +22,42 @@ pub struct Document { pub images: usize, /// True if nodes deeper than the requested depth were omitted. pub too_deep: bool, + /// The space segments, transforms, bounds and stroke widths are given in. + pub coordinates: Coordinates, + /// Group effects the scene flags ([`GroupFeatures`]) but doesn't carry, counted. + pub omitted: Omitted, source: Option>, } +/// How many groups have an effect the scene doesn't carry: an app that draws only the scene +/// draws them without it, and can say so. Elements usvg itself leaves out (animation, scripts…) +/// are listed by [`super::Preflight::unsupported_elements`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Omitted { + /// Groups with `filter`. + pub filters: usize, + /// Groups with `clip-path`. + pub clip_paths: usize, + /// Groups with `mask`. + pub masks: usize, + /// Groups with a `mix-blend-mode` other than normal. + pub blends: usize, +} + +/// Where a scene's geometry is given. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Coordinates { + /// Document CSS pixels: path segments, stroke widths and dashes are already transformed, a + /// path's `transform` is the identity, and a group's or image's `transform` is its full + /// transform to the document. + #[default] + Absolute, + /// Each node in its own user space: path segments, stroke widths, dashes, gradients and + /// bounds are as the SVG gives them, and every `transform` maps the node into its parent + /// (the document for top-level nodes). + Local, +} + impl PartialEq for Document { fn eq(&self, other: &Self) -> bool { self.width == other.width @@ -28,6 +65,8 @@ impl PartialEq for Document { && self.nodes == other.nodes && self.images == other.images && self.too_deep == other.too_deep + && self.coordinates == other.coordinates + && self.omitted == other.omitted } } @@ -44,7 +83,16 @@ impl NodePath { /// A group or path in document order. #[derive(Clone, Debug, PartialEq)] pub enum Node { - Group { children: Vec, opacity: f32, transform: [f64; 6], bounds: [f64; 4], features: GroupFeatures, source_path: NodePath }, + Group { + children: Vec, + opacity: f32, + transform: [f64; 6], + bounds: [f64; 4], + features: GroupFeatures, + source_path: NodePath, + /// The element's `id`, empty when it has none. + id: String, + }, Path(Box), Image(Image), } @@ -80,13 +128,27 @@ pub enum ImageKind { Svg, } -/// A transformed path with its resolved fill and stroke. +/// A path with its resolved fill and stroke. #[derive(Clone, Debug, PartialEq)] pub struct Path { pub segments: Vec, pub fill: Option, + /// How the fill decides what is inside (`fill-rule`); `NonZero` when there is no fill. + pub fill_rule: FillRule, pub stroke: Option, pub source_path: NodePath, + /// The element's `id`, empty when it has none. + pub id: String, + /// Maps the segments into the parent ([`Coordinates`]); the identity in absolute scenes. + pub transform: [f64; 6], +} + +/// The `fill-rule` property. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FillRule { + #[default] + NonZero, + EvenOdd, } impl Path { @@ -97,7 +159,7 @@ impl Path { } } -/// A path segment in CSS pixels after its absolute transform. +/// A path segment, in the space the scene's [`Coordinates`] name. #[derive(Clone, Copy, Debug, PartialEq)] pub enum Segment { MoveTo([f64; 2]), @@ -107,12 +169,42 @@ pub enum Segment { Close, } -/// A resolved color or an unsupported SVG paint server. +/// A resolved color, gradient or pattern. #[derive(Clone, Debug, PartialEq)] pub enum Paint { - Color { rgb: [u8; 3], opacity: f32 }, - Gradient { kind: GradientKind, stops: Vec, transform: [f64; 6], spread: Spread, opacity: f32 }, - Pattern, + Color { + rgb: [u8; 3], + opacity: f32, + }, + Gradient { + kind: GradientKind, + stops: Vec, + transform: [f64; 6], + spread: Spread, + opacity: f32, + }, + /// A ``; `None` when its content is past the depth limit or refers back to the + /// pattern itself. + Pattern { + pattern: Option, + opacity: f32, + }, +} + +/// A resolved `` tile. usvg has already applied `patternUnits`, +/// `patternContentUnits` and `viewBox`, so the tile and its content share one space. +#[derive(Clone, Debug, PartialEq)] +pub struct Pattern { + /// The element's `id`. + pub id: String, + /// The tile `[x, y, width, height]` in tile space. + pub rect: [f64; 4], + /// Maps tile space into the painted path's space: `patternTransform` in a local scene; in + /// an absolute scene, `patternTransform` followed by the path's transform to the document. + pub transform: [f64; 6], + /// What one tile draws, in tile space ([`Coordinates::Local`] whatever the scene's + /// coordinates). Shared by every use of the same pattern. + pub content: std::sync::Arc>, } #[derive(Clone, Debug, PartialEq)] @@ -162,20 +254,32 @@ pub enum LineJoin { } impl Document { - /// Convert a bounded usvg tree to a scene, omitting nodes beyond `max_depth`. + /// Convert a bounded usvg tree to a scene in document pixels ([`Coordinates::Absolute`]), + /// omitting nodes beyond `max_depth`. pub fn from_tree(tree: &usvg::Tree, max_depth: usize) -> Self { - let size = tree.size(); - let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), ..Self::default() }; - document.nodes = convert_group(tree.root(), 0, max_depth, &mut document, &[]); + Self::convert(tree, max_depth, Coordinates::Absolute) + } + + /// Convert a bounded usvg tree to a scene that keeps every node in its own user space with a + /// transform relative to its parent ([`Coordinates::Local`]), omitting nodes beyond + /// `max_depth`. + pub fn from_tree_local(tree: &usvg::Tree, max_depth: usize) -> Self { + Self::convert(tree, max_depth, Coordinates::Local) + } + + /// Build a scene in document pixels that retains its source tree for fallback rendering. + pub fn from_owned_tree(tree: usvg::Tree, max_depth: usize) -> Self { + let mut document = Self::convert(&tree, max_depth, Coordinates::Absolute); + document.source = Some(Box::new(tree)); document } - /// Build a scene that retains its source tree for fallback rendering. - pub fn from_owned_tree(tree: usvg::Tree, max_depth: usize) -> Self { + fn convert(tree: &usvg::Tree, max_depth: usize, coordinates: Coordinates) -> Self { let size = tree.size(); - let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), ..Self::default() }; - document.nodes = convert_group(tree.root(), 0, max_depth, &mut document, &[]); - document.source = Some(Box::new(tree)); + let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), coordinates, ..Self::default() }; + let mut cx = Convert { doc: &mut document, max_depth, local: coordinates == Coordinates::Local, patterns: std::collections::HashMap::new() }; + let nodes = cx.group(tree.root(), tree.root().abs_transform(), 0, &[]); + document.nodes = nodes; document } @@ -231,7 +335,7 @@ fn spread(s: usvg::SpreadMethod) -> Spread { } } -fn color(paint: &usvg::Paint, opacity: f32) -> Paint { +fn color(paint: &usvg::Paint, opacity: f32, pattern: impl FnOnce(&std::sync::Arc) -> Option) -> Paint { match paint { usvg::Paint::Color(c) => Paint::Color { rgb: [c.red, c.green, c.blue], opacity }, usvg::Paint::LinearGradient(g) => Paint::Gradient { @@ -269,7 +373,7 @@ fn color(paint: &usvg::Paint, opacity: f32) -> Paint { spread: spread(g.spread_method()), opacity, }, - usvg::Paint::Pattern(_) => Paint::Pattern, + usvg::Paint::Pattern(source) => Paint::Pattern { pattern: pattern(source), opacity }, } } @@ -279,122 +383,213 @@ fn point(transform: usvg::Transform, p: usvg::tiny_skia_path::Point) -> [f64; 2] [f64::from(p.x), f64::from(p.y)] } -fn convert_group(group: &usvg::Group, depth: usize, max_depth: usize, doc: &mut Document, parent_path: &[usize]) -> Vec { - let mut result = Vec::new(); - for (index, node) in group.children().iter().enumerate() { - let mut source_path = parent_path.to_vec(); - source_path.push(index); - match node { - usvg::Node::Path(path) => { - if !path.is_visible() || path.data().is_empty() { - continue; +const IDENTITY: [f64; 6] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + +/// A node's transform relative to `parent` (both absolute): `parent⁻¹ · node`. The identity +/// when `parent` cannot be inverted (a degenerate group draws nothing anyway). +fn relative(parent: usvg::Transform, node: usvg::Transform) -> usvg::Transform { + parent.invert().map_or(usvg::Transform::identity(), |inverse| inverse.pre_concat(node)) +} + +struct Convert<'d> { + doc: &'d mut Document, + max_depth: usize, + local: bool, + /// Pattern content already converted, by the address of usvg's shared pattern; `None` while + /// a pattern is being converted, so a pattern that refers back to itself gets no content. + patterns: std::collections::HashMap>>>, +} + +impl Convert<'_> { + /// The transform a node with absolute transform `node` gets under a group whose absolute + /// transform is `parent`. + fn placement(&self, parent: usvg::Transform, node: usvg::Transform) -> [f64; 6] { + if self.local { transform(relative(parent, node)) } else { transform(node) } + } + + /// `parent` is the absolute transform `group`'s leaves are placed against: its own, except + /// for flattened text, whose outline paths carry the text's while the group has none. + fn group(&mut self, group: &usvg::Group, parent: usvg::Transform, depth: usize, parent_path: &[usize]) -> Vec { + let mut result = Vec::new(); + for (index, node) in group.children().iter().enumerate() { + let mut source_path = parent_path.to_vec(); + source_path.push(index); + match node { + usvg::Node::Path(path) => { + if let Some(path) = self.path(path, parent, source_path, depth) { + result.push(Node::Path(Box::new(path))); + } } - let transform = path.abs_transform(); - let segments = path - .data() - .segments() - .map(|segment| { - use usvg::tiny_skia_path::PathSegment as S; - match segment { - S::MoveTo(a) => Segment::MoveTo(point(transform, a)), - S::LineTo(a) => Segment::LineTo(point(transform, a)), - S::QuadTo(a, b) => Segment::QuadTo(point(transform, a), point(transform, b)), - S::CubicTo(a, b, c) => Segment::CubicTo(point(transform, a), point(transform, b), point(transform, c)), - S::Close => Segment::Close, + usvg::Node::Group(child) => { + if depth >= self.max_depth { + self.doc.too_deep = true; + continue; + } + let children = self.group(child, child.abs_transform(), depth + 1, &source_path); + if !children.is_empty() { + let omitted = &mut self.doc.omitted; + omitted.filters += usize::from(!child.filters().is_empty()); + omitted.clip_paths += usize::from(child.clip_path().is_some()); + omitted.masks += usize::from(child.mask().is_some()); + omitted.blends += usize::from(child.blend_mode() != usvg::BlendMode::Normal); + let bounds = if self.local { child.layer_bounding_box().to_rect() } else { child.abs_layer_bounding_box().to_rect() }; + result.push(Node::Group { + children, + opacity: child.opacity().get(), + // Locally, the group's own transform: usvg leaves the absolute one at + // the identity for colour and bitmap glyph groups in text. + transform: if self.local { transform(child.transform()) } else { transform(child.abs_transform()) }, + bounds: rect(bounds), + features: GroupFeatures { + filters: !child.filters().is_empty(), + clip_path: child.clip_path().is_some(), + mask: child.mask().is_some(), + non_normal_blend: child.blend_mode() != usvg::BlendMode::Normal, + isolation: child.should_isolate(), + }, + source_path: NodePath(source_path), + id: child.id().to_string(), + }); + } + } + usvg::Node::Text(text) => { + if depth >= self.max_depth { + self.doc.too_deep = true; + continue; + } + let flat = text.flattened(); + // The flattened group has no transform of its own, but its outline paths + // have the text's: place its leaves, and the wrapper, in the text's frame. + let frame = if self.local { text.abs_transform() } else { flat.abs_transform() }; + let children = self.group(flat, frame, depth + 1, &source_path); + if !children.is_empty() { + let bounds = if self.local { text.bounding_box() } else { text.abs_bounding_box() }; + result.push(Node::Group { + children, + opacity: 1.0, + transform: self.placement(parent, frame), + bounds: rect(bounds), + features: GroupFeatures::default(), + source_path: NodePath(source_path), + id: text.id().to_string(), + }); + } + } + usvg::Node::Image(image) => { + self.doc.images += 1; + let kind = match image.kind() { + usvg::ImageKind::JPEG(_) => ImageKind::Jpeg, + usvg::ImageKind::PNG(_) => ImageKind::Png, + usvg::ImageKind::GIF(_) => ImageKind::Gif, + usvg::ImageKind::WEBP(_) => ImageKind::WebP, + usvg::ImageKind::SVG(_) => ImageKind::Svg, + }; + let (encoded, svg) = match image.kind() { + usvg::ImageKind::JPEG(data) | usvg::ImageKind::PNG(data) | usvg::ImageKind::GIF(data) | usvg::ImageKind::WEBP(data) => { + (Some(data.clone()), None) } - }) - .collect(); - let fill = path.fill().map(|f| color(f.paint(), f.opacity().get())); - let stroke = path.stroke().map(|s| { - let scale = f64::from((transform.sx * transform.sy - transform.kx * transform.ky).abs().sqrt()); - Stroke { - paint: color(s.paint(), s.opacity().get()), - width: f64::from(s.width().get()) * scale, - cap: match s.linecap() { - usvg::LineCap::Butt => LineCap::Butt, - usvg::LineCap::Round => LineCap::Round, - usvg::LineCap::Square => LineCap::Square, - }, - join: match s.linejoin() { - usvg::LineJoin::Round => LineJoin::Round, - usvg::LineJoin::Bevel => LineJoin::Bevel, - usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => LineJoin::Miter, - }, - miter_limit: f64::from(s.miterlimit().get()), - dash_array: s.dasharray().map(|d| d.iter().map(|v| f64::from(*v) * scale).collect()), - dash_offset: f64::from(s.dashoffset()) * scale, - } - }); - result.push(Node::Path(Box::new(Path { segments, fill, stroke, source_path: NodePath(source_path) }))); - } - usvg::Node::Group(child) => { - if depth >= max_depth { - doc.too_deep = true; - continue; - } - let children = convert_group(child, depth + 1, max_depth, doc, &source_path); - if !children.is_empty() { - result.push(Node::Group { - children, - opacity: child.opacity().get(), - transform: transform(child.abs_transform()), - bounds: rect(child.abs_layer_bounding_box().to_rect()), - features: GroupFeatures { - filters: !child.filters().is_empty(), - clip_path: child.clip_path().is_some(), - mask: child.mask().is_some(), - non_normal_blend: child.blend_mode() != usvg::BlendMode::Normal, - isolation: child.should_isolate(), - }, + usvg::ImageKind::SVG(tree) => { + let max_depth = self.max_depth.saturating_sub(depth + 1); + let nested = if self.local { + Document::from_tree_local(tree, max_depth) + } else { + Document::from_owned_tree(tree.clone(), max_depth) + }; + (None, Some(Box::new(nested))) + } + }; + let bounds = if self.local { image.bounding_box() } else { image.abs_bounding_box() }; + result.push(Node::Image(Image { + kind, + visible: image.is_visible(), + transform: self.placement(parent, image.abs_transform()), + bounds: rect(bounds), + encoded, + svg, source_path: NodePath(source_path), - }); + })); } } - usvg::Node::Text(text) => { - if depth >= max_depth { - doc.too_deep = true; - continue; - } - let children = convert_group(text.flattened(), depth + 1, max_depth, doc, &source_path); - if !children.is_empty() { - result.push(Node::Group { - children, - opacity: 1.0, - transform: transform(text.abs_transform()), - bounds: rect(text.abs_bounding_box()), - features: GroupFeatures::default(), - source_path: NodePath(source_path), - }); - } - } - usvg::Node::Image(image) => { - doc.images += 1; - let kind = match image.kind() { - usvg::ImageKind::JPEG(_) => ImageKind::Jpeg, - usvg::ImageKind::PNG(_) => ImageKind::Png, - usvg::ImageKind::GIF(_) => ImageKind::Gif, - usvg::ImageKind::WEBP(_) => ImageKind::WebP, - usvg::ImageKind::SVG(_) => ImageKind::Svg, - }; - let (encoded, svg) = match image.kind() { - usvg::ImageKind::JPEG(data) | usvg::ImageKind::PNG(data) | usvg::ImageKind::GIF(data) | usvg::ImageKind::WEBP(data) => { - (Some(data.clone()), None) - } - usvg::ImageKind::SVG(tree) => { - (None, Some(Box::new(Document::from_owned_tree(tree.clone(), max_depth.saturating_sub(depth + 1))))) - } - }; - result.push(Node::Image(Image { - kind, - visible: image.is_visible(), - transform: transform(image.abs_transform()), - bounds: rect(image.abs_bounding_box()), - encoded, - svg, - source_path: NodePath(source_path), - })); - } } + result + } + + /// A visible, non-empty path. Its own absolute transform is its group's in usvg, so in a + /// local scene its segments stay as they are and its transform is normally the identity. + /// The tile of `pattern` painted on a path at `depth` whose own transform to the document is + /// `abs`. + fn pattern(&mut self, pattern: &std::sync::Arc, abs: usvg::Transform, depth: usize) -> Option { + let key = std::sync::Arc::as_ptr(pattern) as usize; + let content = match self.patterns.get(&key) { + Some(done) => done.clone(), + None if depth >= self.max_depth => { + self.doc.too_deep = true; + None + } + None => { + self.patterns.insert(key, None); + let local = std::mem::replace(&mut self.local, true); + let nodes = self.group(pattern.root(), pattern.root().abs_transform(), depth + 1, &[]); + self.local = local; + let content = Some(std::sync::Arc::new(nodes)); + self.patterns.insert(key, content.clone()); + content + } + }?; + let placement = if self.local { pattern.transform() } else { abs.pre_concat(pattern.transform()) }; + Some(Pattern { id: pattern.id().to_string(), rect: rect(pattern.rect().to_rect()), transform: transform(placement), content }) + } + + fn path(&mut self, path: &usvg::Path, parent: usvg::Transform, source_path: Vec, depth: usize) -> Option { + if !path.is_visible() || path.data().is_empty() { + return None; + } + let abs = path.abs_transform(); + let (to_points, scale, placement) = if self.local { + (usvg::Transform::identity(), 1.0, transform(relative(parent, abs))) + } else { + (abs, f64::from((abs.sx * abs.sy - abs.kx * abs.ky).abs().sqrt()), IDENTITY) + }; + let segments = path + .data() + .segments() + .map(|segment| { + use usvg::tiny_skia_path::PathSegment as S; + match segment { + S::MoveTo(a) => Segment::MoveTo(point(to_points, a)), + S::LineTo(a) => Segment::LineTo(point(to_points, a)), + S::QuadTo(a, b) => Segment::QuadTo(point(to_points, a), point(to_points, b)), + S::CubicTo(a, b, c) => Segment::CubicTo(point(to_points, a), point(to_points, b), point(to_points, c)), + S::Close => Segment::Close, + } + }) + .collect(); + let fill = path.fill().map(|f| color(f.paint(), f.opacity().get(), |p| self.pattern(p, abs, depth))); + let fill_rule = match path.fill().map(usvg::Fill::rule) { + Some(usvg::FillRule::EvenOdd) => FillRule::EvenOdd, + _ => FillRule::NonZero, + }; + let stroke = path.stroke().map(|s| Stroke { + paint: color(s.paint(), s.opacity().get(), |p| self.pattern(p, abs, depth)), + width: f64::from(s.width().get()) * scale, + cap: match s.linecap() { + usvg::LineCap::Butt => LineCap::Butt, + usvg::LineCap::Round => LineCap::Round, + usvg::LineCap::Square => LineCap::Square, + }, + join: match s.linejoin() { + usvg::LineJoin::Round => LineJoin::Round, + usvg::LineJoin::Bevel => LineJoin::Bevel, + usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => LineJoin::Miter, + }, + miter_limit: f64::from(s.miterlimit().get()), + dash_array: s.dasharray().map(|d| d.iter().map(|v| f64::from(*v) * scale).collect()), + dash_offset: f64::from(s.dashoffset()) * scale, + }); + Some(Path { segments, fill, fill_rule, stroke, source_path: NodePath(source_path), id: path.id().to_string(), transform: placement }) } - result } + +#[cfg(test)] +#[path = "document_tests.rs"] +mod tests; diff --git a/crates/svg/src/tree/document_tests.rs b/crates/svg/src/tree/document_tests.rs new file mode 100644 index 000000000..a130a6f6f --- /dev/null +++ b/crates/svg/src/tree/document_tests.rs @@ -0,0 +1,349 @@ +#![allow(clippy::unwrap_used, clippy::panic, clippy::indexing_slicing, clippy::float_cmp)] + +use super::*; +use crate::tree::{Limits, parse}; + +fn tree(body: &str) -> usvg::Tree { + let svg = format!(r#"{body}"#); + parse(svg.as_bytes(), &Limits::default(), &usvg::Options::default()).unwrap() +} + +fn apply(m: [f64; 6], p: [f64; 2]) -> [f64; 2] { + [m[0] * p[0] + m[2] * p[1] + m[4], m[1] * p[0] + m[3] * p[1] + m[5]] +} + +fn then(outer: [f64; 6], inner: [f64; 6]) -> [f64; 6] { + let [a, b, c, d, e, f] = outer; + let [p, q, r, s, t, u] = inner; + [a * p + c * q, b * p + d * q, a * r + c * s, b * r + d * s, a * t + c * u + e, b * t + d * u + f] +} + +fn close(a: [f64; 2], b: [f64; 2]) -> bool { + (a[0] - b[0]).abs() < 1e-3 && (a[1] - b[1]).abs() < 1e-3 +} + +/// Every path's points, mapped to document pixels through the scene's transforms. +fn document_points(nodes: &[Node], outer: [f64; 6], local: bool, out: &mut Vec<[f64; 2]>) { + for node in nodes { + match node { + Node::Group { children, transform, .. } => { + let next = if local { then(outer, *transform) } else { outer }; + document_points(children, next, local, out); + } + Node::Path(path) => { + let m = then(outer, path.transform); + for segment in &path.segments { + match segment { + Segment::MoveTo(p) | Segment::LineTo(p) => out.push(apply(m, *p)), + Segment::QuadTo(a, b) => out.extend([apply(m, *a), apply(m, *b)]), + Segment::CubicTo(a, b, c) => out.extend([apply(m, *a), apply(m, *b), apply(m, *c)]), + Segment::Close => {} + } + } + } + Node::Image(_) => {} + } + } +} + +fn paths(nodes: &[Node]) -> Vec<&Path> { + let mut out = Vec::new(); + for node in nodes { + match node { + Node::Group { children, .. } => out.extend(paths(children)), + Node::Path(path) => out.push(path), + Node::Image(_) => {} + } + } + out +} + +fn group_ids(nodes: &[Node]) -> Vec<(String, [f64; 6])> { + let mut out = Vec::new(); + for node in nodes { + if let Node::Group { children, id, transform, .. } = node { + out.push((id.clone(), *transform)); + out.extend(group_ids(children)); + } + } + out +} + +const NESTED: &str = r##" + + + + +"##; + +#[test] +fn a_local_scene_keeps_user_space_geometry_and_relative_transforms() { + let tree = tree(NESTED); + let local = Document::from_tree_local(&tree, 64); + assert_eq!(local.coordinates, Coordinates::Local); + let groups = group_ids(&local.nodes); + let outer = groups.iter().find(|(id, _)| id == "outer").unwrap(); + assert!(close([outer.1[0], outer.1[3]], [2.0, 2.0]) && close([outer.1[4], outer.1[5]], [10.0, 20.0]), "{outer:?}"); + let inner = groups.iter().find(|(id, _)| id == "inner").unwrap(); + assert!(close([inner.1[0], inner.1[1]], [0.0, 1.0]) && close([inner.1[4], inner.1[5]], [0.0, 0.0]), "relative, not absolute: {inner:?}"); + + let all = paths(&local.nodes); + let rect = all.iter().find(|p| p.id == "box").unwrap(); + let Segment::MoveTo(start) = rect.segments[0] else { panic!("{:?}", rect.segments) }; + assert!(close(start, [1.0, 2.0]), "user space: {start:?}"); + let stroke = rect.stroke.as_ref().unwrap(); + assert_eq!(stroke.width, 2.0, "not scaled by the groups"); + assert_eq!(stroke.dash_array.as_deref(), Some([1.0, 3.0].as_slice())); +} + +#[test] +fn an_absolute_scene_is_unchanged_by_the_local_option() { + let tree = tree(NESTED); + let absolute = Document::from_tree(&tree, 64); + assert_eq!(absolute.coordinates, Coordinates::Absolute); + let rect = paths(&absolute.nodes).into_iter().find(|p| p.id == "box").unwrap(); + assert_eq!(rect.transform, IDENTITY); + // (1, 2) rotated 90° is (-2, 1), then scaled by 2 and moved by (10, 20). + let Segment::MoveTo(start) = rect.segments[0] else { panic!("{:?}", rect.segments) }; + assert!(close(start, [6.0, 22.0]), "{start:?}"); + assert_eq!(rect.stroke.as_ref().unwrap().width, 4.0); +} + +#[test] +fn local_and_absolute_scenes_put_every_point_in_the_same_place() { + let tree = tree(NESTED); + let (mut local, mut absolute) = (Vec::new(), Vec::new()); + document_points(&Document::from_tree_local(&tree, 64).nodes, IDENTITY, true, &mut local); + document_points(&Document::from_tree(&tree, 64).nodes, IDENTITY, false, &mut absolute); + assert_eq!(local.len(), absolute.len()); + assert!(!local.is_empty()); + for (l, a) in local.iter().zip(&absolute) { + assert!(close(*l, *a), "{l:?} vs {a:?}"); + } +} + +#[test] +fn fill_rule_and_ids_are_kept() { + let tree = + tree(r##""##); + for document in [Document::from_tree(&tree, 64), Document::from_tree_local(&tree, 64)] { + let all = paths(&document.nodes); + assert_eq!(all.len(), 2); + assert_eq!((all[0].id.as_str(), all[0].fill_rule), ("hole", FillRule::EvenOdd)); + assert_eq!((all[1].id.as_str(), all[1].fill_rule), ("", FillRule::NonZero), "no fill: the default"); + assert!(all[1].fill.is_none()); + } +} + +#[test] +fn a_local_scene_respects_the_depth_limit() { + let deep = format!("{}{}", "".repeat(6), "".repeat(6)); + let tree = tree(&deep); + let shallow = Document::from_tree_local(&tree, 2); + assert!(shallow.too_deep); + assert!(paths(&shallow.nodes).is_empty()); + assert!(!Document::from_tree_local(&tree, 64).too_deep); +} + +#[test] +fn an_embedded_svg_in_a_local_scene_is_local_too() { + use std::io::Write as _; + let inner = r##""##; + let mut b64 = Vec::new(); + { + // A tiny base64 encoder keeps the test free of extra dependencies. + const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + for chunk in inner.as_bytes().chunks(3) { + let n = chunk.iter().enumerate().fold(0u32, |n, (i, b)| n | (u32::from(*b) << (16 - 8 * i))); + for i in 0..4 { + if i <= chunk.len() { + b64.push(A[((n >> (18 - 6 * i)) & 63) as usize]); + } else { + b64.push(b'='); + } + } + } + } + let mut body = Vec::new(); + write!(body, r#""#, String::from_utf8(b64).unwrap()).unwrap(); + let svg = format!(r#"{}"#, String::from_utf8(body).unwrap()); + let options = usvg::Options { image_href_resolver: crate::tree::image_resolver(Limits::default()), ..usvg::Options::default() }; + let tree = parse(svg.as_bytes(), &Limits::default(), &options).unwrap(); + let local = Document::from_tree_local(&tree, 64); + let image = local + .nodes + .iter() + .find_map(|n| match n { + Node::Image(image) => Some(image), + Node::Group { children, .. } => children.iter().find_map(|c| if let Node::Image(image) = c { Some(image) } else { None }), + Node::Path(_) => None, + }) + .unwrap(); + let nested = image.svg.as_ref().unwrap(); + assert_eq!(nested.coordinates, Coordinates::Local); + let rect = paths(&nested.nodes)[0]; + let Segment::LineTo(p) = rect.segments[1] else { panic!("{:?}", rect.segments) }; + assert!(close(p, [1.0, 0.0]), "inside its own scaled group: {p:?}"); +} + +fn fill_pattern(path: &Path) -> Option<&Pattern> { + match path.fill.as_ref()? { + Paint::Pattern { pattern, .. } => pattern.as_ref(), + _ => None, + } +} + +const PATTERN: &str = r##" + + +"##; + +#[test] +fn a_pattern_paint_carries_its_tile_and_content() { + let tree = tree(PATTERN); + let local = Document::from_tree_local(&tree, 64); + let all = paths(&local.nodes); + let a = all.iter().find(|p| p.id == "a").unwrap(); + let pattern = fill_pattern(a).unwrap(); + assert_eq!(pattern.id, "dots"); + assert_eq!(pattern.rect, [0.0, 0.0, 4.0, 4.0]); + assert!(close([pattern.transform[4], pattern.transform[5]], [1.0, 2.0]) && close([pattern.transform[0], pattern.transform[3]], [1.0, 1.0])); + let tile = paths(&pattern.content); + assert_eq!(tile.len(), 1); + assert_eq!(tile[0].fill, Some(Paint::Color { rgb: [255, 0, 0], opacity: 1.0 })); + + // Every use shares one converted tile, and keeps its own opacity. + let b = all.iter().find(|p| p.id == "b").unwrap(); + assert!(std::sync::Arc::ptr_eq(&pattern.content, &fill_pattern(b).unwrap().content)); + assert!(matches!(b.fill, Some(Paint::Pattern { opacity, .. }) if (opacity - 0.5).abs() < 1e-6)); +} + +#[test] +fn an_absolute_scene_maps_the_tile_to_the_document() { + let tree = tree(PATTERN); + let absolute = Document::from_tree(&tree, 64); + let a = paths(&absolute.nodes).into_iter().find(|p| p.id == "a").unwrap(); + let pattern = fill_pattern(a).unwrap(); + // translate(1 2), then the group's scale(2). + assert!(close([pattern.transform[0], pattern.transform[3]], [2.0, 2.0]), "{:?}", pattern.transform); + assert!(close([pattern.transform[4], pattern.transform[5]], [2.0, 4.0]), "{:?}", pattern.transform); + // The tile's content stays in tile space. + let Segment::LineTo(p) = paths(&pattern.content)[0].segments[1] else { panic!() }; + assert!(close(p, [2.0, 0.0]), "{p:?}"); +} + +#[test] +fn bounding_box_pattern_units_are_resolved() { + let tree = tree( + r##""##, + ); + let document = Document::from_tree_local(&tree, 64); + let pattern = fill_pattern(paths(&document.nodes)[0]).unwrap(); + assert_eq!(pattern.rect, [0.0, 0.0, 5.0, 10.0]); +} + +#[test] +fn nested_patterns_convert_within_the_depth_limit() { + let svg = r##" + + + "##; + let tree = tree(svg); + let deep = Document::from_tree_local(&tree, 64); + let outer = fill_pattern(paths(&deep.nodes)[0]).unwrap(); + let inner = fill_pattern(paths(&outer.content)[0]).unwrap(); + assert_eq!(inner.id, "inner"); + assert!(!deep.too_deep); + + let shallow = Document::from_tree_local(&tree, 1); + assert!(shallow.too_deep); + let outer = fill_pattern(paths(&shallow.nodes)[0]).unwrap(); + assert!(fill_pattern(paths(&outer.content)[0]).is_none(), "past the limit: no content"); +} + +#[test] +fn a_pattern_that_refers_to_itself_ends() { + let tree = tree( + r##""##, + ); + let document = Document::from_tree_local(&tree, 64); + // usvg may drop the reference; if it keeps it, the inner use has no content. + for path in paths(&document.nodes) { + if let Some(pattern) = fill_pattern(path) { + for inner in paths(&pattern.content) { + assert!(fill_pattern(inner).is_none()); + } + } + } +} + +#[test] +fn group_effects_the_scene_leaves_out_are_counted() { + let tree = tree( + r##" + + + + + + "##, + ); + for document in [Document::from_tree(&tree, 64), Document::from_tree_local(&tree, 64)] { + assert_eq!(document.omitted, Omitted { filters: 1, clip_paths: 2, masks: 1, blends: 1 }); + } + assert_eq!(Document::from_tree(&self::tree(r#""#), 64).omitted, Omitted::default()); +} + +#[test] +fn elements_usvg_leaves_out_are_listed() { + let svg = r##" + + + + + "##; + let checked = crate::tree::preflight(svg, &Limits::default()).unwrap(); + assert_eq!( + checked.unsupported_elements(), + vec![("animate", 2), ("animateTransform", 1), ("foreignObject", 1), ("script", 1), ("set", 1)], + "the XHTML div is not an SVG element" + ); + // The same checked XML still parses. + assert!(checked.parse(&usvg::Options::default()).is_ok()); + let plain = crate::tree::preflight(r#""#, &Limits::default()).unwrap(); + assert!(plain.unsupported_elements().is_empty()); +} + +#[cfg(feature = "tree-text")] +#[test] +fn local_text_is_placed_and_bounded_in_its_parents_frame() { + let mut options = usvg::Options::default(); + let fonts = options.fontdb_mut(); + fonts.load_font_data(include_bytes!("../../../ui/tests/fonts/NotoSans-test.ttf").to_vec()); + let family = fonts.faces().next().unwrap().families[0].0.clone(); + let svg = format!( + r#"Hi"# + ); + let tree = parse(svg.as_bytes(), &Limits::default(), &options).unwrap(); + let local = Document::from_tree_local(&tree, 64); + let absolute = Document::from_tree(&tree, 64); + + let groups = group_ids(&local.nodes); + let text = groups.iter().find(|(id, _)| id == "t").unwrap(); + assert!(close([text.1[0], text.1[3]], [1.0, 1.0]) && close([text.1[4], text.1[5]], [0.0, 0.0]), "text sits in its group's frame: {text:?}"); + + let bounds = |nodes: &[Node]| -> [f64; 4] { + let Node::Group { children, .. } = &nodes[0] else { panic!() }; + let Node::Group { bounds, .. } = &children[0] else { panic!() }; + *bounds + }; + let (l, a) = (bounds(&local.nodes), bounds(&absolute.nodes)); + assert!(close([l[0] + 50.0, l[1]], [a[0], a[1]]) && close([l[2], l[3]], [a[2], a[3]]), "local {l:?} vs absolute {a:?}"); + assert!(l[0] > 0.0 && l[0] < 10.0, "near x=5 in the group's frame: {l:?}"); + + let (mut lp, mut ap) = (Vec::new(), Vec::new()); + document_points(&local.nodes, IDENTITY, true, &mut lp); + document_points(&absolute.nodes, IDENTITY, false, &mut ap); + assert!(!lp.is_empty() && lp.len() == ap.len()); + assert!(lp.iter().zip(&ap).all(|(l, a)| close(*l, *a))); +} diff --git a/crates/svg/src/tree/primitive_tests.rs b/crates/svg/src/tree/primitive_tests.rs index 81f4c0979..ac417eb6c 100644 --- a/crates/svg/src/tree/primitive_tests.rs +++ b/crates/svg/src/tree/primitive_tests.rs @@ -109,8 +109,11 @@ fn hand_written_segments_are_recognised_too() { Segment::Close, ], fill: None, + fill_rule: crate::tree::document::FillRule::NonZero, stroke: None, source_path: crate::tree::document::NodePath::default(), + id: String::new(), + transform: [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], }; assert_eq!(path.primitive(), Some(Primitive::Ellipse { cx: 5.0, cy: 5.0, rx: 5.0, ry: 5.0 })); } From 472956a8d7626643b0148de5573083474a595934 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:33 -0500 Subject: [PATCH 22/27] feat: add craft-control (control channel protocol) (#1879) * feat: add craft-control (control channel protocol) Port from solidcraft: the control channel any app may share, no UI toolkit. - Per-launch token files (32 random bytes, user-only permissions) - Loopback server with token-based authentication - Client that talks to the loopback server - Request/response types (ControlRequest, ControlResponse) - Document watch for subscribed connections waiting on changes - Effect type for requests still running when they finish Desktop-only (wasm32 excluded): uses getrandom for token generation. No external dependencies beyond serde and serde_json. * fix: update test directory names from solidcraft-control to craft-control * Record control crate dependencies in workspace lockfile * re-trigger CI * chore(lock): keep the wasm-bindgen family at the version main's runners install --- Cargo.lock | 10 + crates/control/Cargo.toml | 23 + crates/control/src/auth.rs | 673 +++++++++++++ crates/control/src/cancel.rs | 118 +++ crates/control/src/client.rs | 616 ++++++++++++ crates/control/src/client_tests.rs | 190 ++++ crates/control/src/effect.rs | 27 + crates/control/src/lib.rs | 26 + crates/control/src/request.rs | 125 +++ crates/control/src/server.rs | 1433 ++++++++++++++++++++++++++++ 10 files changed, 3241 insertions(+) create mode 100644 crates/control/Cargo.toml create mode 100644 crates/control/src/auth.rs create mode 100644 crates/control/src/cancel.rs create mode 100644 crates/control/src/client.rs create mode 100644 crates/control/src/client_tests.rs create mode 100644 crates/control/src/effect.rs create mode 100644 crates/control/src/lib.rs create mode 100644 crates/control/src/request.rs create mode 100644 crates/control/src/server.rs diff --git a/Cargo.lock b/Cargo.lock index 5f28d1878..50b715040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1498,6 +1498,16 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "craft-control" +version = "0.1.0" +dependencies = [ + "getrandom 0.3.4", + "proptest", + "serde", + "serde_json", +] + [[package]] name = "craft-control-protocol" version = "0.1.0" diff --git a/crates/control/Cargo.toml b/crates/control/Cargo.toml new file mode 100644 index 000000000..0a18ccc99 --- /dev/null +++ b/crates/control/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "craft-control" +description = "The control channel any app may share: token files, the loopback server, and its requests; no UI toolkit" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + +# The loopback server is for desktop apps; the web app is driven through the page instead. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# The per-launch control token: 32 random bytes from the operating system (MIT OR Apache-2.0). +getrandom = "0.3" + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/control/src/auth.rs b/crates/control/src/auth.rs new file mode 100644 index 000000000..bb2cef86b --- /dev/null +++ b/crates/control/src/auth.rs @@ -0,0 +1,673 @@ +//! The control channel's per-launch token (docs/control-protocol.md). An app listening for +//! control makes a random token, writes it to a file only the user can read +//! ([`App::token_path`]) and refuses every connection whose first request doesn't carry it, so +//! other users on the same machine (loopback is shared, on Windows across remote-desktop sessions +//! too) can't drive the app. Clients find it with [`App::client_token`]. Next to it, `.pid` +//! names the process that wrote it, so a client refused with a leftover token can say the file is +//! stale. + +use std::ffi::OsString; +use std::io::{Error, ErrorKind, Read, Result, Write}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Shortest token accepted from the app's `CONTROL_TOKEN` variable (hex characters). +pub const MIN_TOKEN_LEN: usize = 32; + +/// Ports tried from the app's default port upward when it is taken (a second window, another +/// program). +pub const PORT_TRIES: u16 = 16; + +/// Which app a control channel belongs to: where its token files live, its environment and its +/// name in messages. +#[derive(Clone, Copy)] +pub struct App { + /// The app's name in messages ("restart SolidCraft"). + pub name: &'static str, + /// The per-user data folder; the token files go in its `control` folder. + pub data_dir: fn() -> Option, + /// The app's environment variable `name` (`CONTROL_DIR`, `CONTROL_TOKEN`), with the app's prefix. + pub env: fn(&str) -> Option, + /// A time in seconds since the Unix epoch, as the app writes UTC times in messages. + pub utc: fn(u64) -> String, + /// The port the control channel tries first when the launch gives none. Below the ephemeral + /// ranges of Linux (32768+) and Windows/macOS (49152+), so the system never hands it to an + /// outgoing connection. + pub default_port: u16, +} + +/// Where the control channel listens, from the launch's `--control`/environment port and the app's +/// allow-agents setting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Listen { + /// Exactly this port (`--control PORT`, the app's `CONTROL_PORT`); no fallback. + Exact(u16), + /// The app's [`App::default_port`], or the next free one of [`PORT_TRIES`]. + Default, +} + +/// What a launch listens on. An explicit port wins over the app's allow-agents setting +/// (`enabled`), so a scripted launch works with the setting off; with the setting off and no port, +/// nothing listens. +pub fn plan(explicit: Option, enabled: bool) -> Option { + match explicit { + Some(port) => Some(Listen::Exact(port)), + None if enabled => Some(Listen::Default), + None => None, + } +} + +/// Bind a loopback-only listener on `first`, or on the next free port when it is taken, up to +/// `tries` ports in all. +pub fn bind_with_fallback(first: u16, tries: u16) -> Result { + let mut last = Error::new(ErrorKind::AddrInUse, "no port to try"); + for port in (first..=u16::MAX).take(usize::from(tries.max(1))) { + match std::net::TcpListener::bind(("127.0.0.1", port)) { + Ok(l) => return Ok(l), + // In use; Windows reports a port another program holds exclusively as denied. + Err(e) if matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) => last = e, + Err(e) => return Err(e), + } + } + Err(last) +} + +impl App { + /// The folder of the token files: `/control` (the app's `CONTROL_DIR` replaces it). + fn control_dir(&self) -> Option { + match self.custom_control_dir() { + Some(d) => Some(PathBuf::from(d)), + None => Some((self.data_dir)()?.join("control")), + } + } + + /// The folder the app's `CONTROL_DIR` names in place of the default one. + fn custom_control_dir(&self) -> Option { + (self.env)("CONTROL_DIR").filter(|d| !d.is_empty()) + } + + /// The token file of the app listening on `port`: `/control/.token` + /// (the app's `CONTROL_DIR` replaces the folder). + pub fn token_path(&self, port: u16) -> Option { + Some(self.control_dir()?.join(format!("{port}.token"))) + } + + /// The discovery file: the port of the most recently started app, so a client with no port + /// finds it. It holds only the port; the token stays in its own file. + pub fn current_port_path(&self) -> Option { + Some(self.control_dir()?.join("current.port")) + } + + /// Record `port` as the running app's, readable only by the user like the token files. + pub fn write_current_port(&self, port: u16) -> Result { + let path = self.current_port_path().ok_or_else(|| Error::new(ErrorKind::NotFound, "no data folder"))?; + self.write_token_file(&path, &port.to_string())?; + Ok(path) + } + + /// The port the discovery file names, if there is one. + pub fn read_current_port(&self) -> Option { + parse_port(&std::fs::read_to_string(self.current_port_path()?).ok().filter(|t| t.len() <= 64)?) + } + + /// Remove the discovery file (and its process file) at exit, unless a later app has taken it + /// over: it names another port, or another launch now holds this port's token file. Under + /// Flatpak every instance has its own loopback, so two windows can listen on the same port + /// number and write the same file names; the one closing first must leave the other's (#8794). + /// Run it after [`remove_token_file`], which takes this launch's own token file away first. + pub fn remove_current_port(&self, port: u16, token: &str) { + if let (Some(current), Some(file)) = (self.current_port_path(), self.token_path(port)) { + remove_discovery(¤t, &file, port, token); + } + } + + /// A token set on purpose with the app's `CONTROL_TOKEN` (test harnesses, sandboxed packages + /// whose data folder the client can't see), if it is long enough. + pub fn env_token(&self) -> Option { + (self.env)("CONTROL_TOKEN").and_then(|v| v.into_string().ok()).map(|t| t.trim().to_string()).filter(|t| t.len() >= MIN_TOKEN_LEN) + } + + /// The token a client sends to the app on `port`: `explicit` (`--token`), else the app's + /// `CONTROL_TOKEN`, else the app's token file. + pub fn client_token(&self, port: u16, explicit: Option<&str>) -> Option { + if let Some(t) = explicit.map(str::trim).filter(|t| !t.is_empty()) { + return Some(ClientToken { token: t.to_string(), file: None, app: *self }); + } + if let Some(t) = self.env_token() { + return Some(ClientToken { token: t, file: None, app: *self }); + } + // Read directly, not through any agent's file scope: the app's own file. + read_token_file(&self.token_path(port)?, *self) + } + + /// Write `token` to `path`, readable and writable only by the user: the folder is made `0700` + /// and the file `0600` on Unix; on Windows the file's inherited permissions are replaced by one + /// entry for the user (`icacls`) before the token goes in; when that fails, the still empty file + /// is removed and no token is written. A file left by an earlier launch is replaced. The process + /// file beside it ([`pid_path`]) records this process, best effort. + pub fn write_token_file(&self, path: &Path, token: &str) -> Result<()> { + #[cfg(windows)] + let restrict = |p: &Path| restrict_to_user(p, self.custom_control_dir().is_some()); + #[cfg(not(windows))] + let restrict = |_: &Path| Ok(()); + write_token_file_with(path, token, restrict) + } +} + +fn parse_port(text: &str) -> Option { + text.trim().parse::().ok().filter(|p| *p != 0) +} + +/// [`App::remove_current_port`] on given files: `current` is the discovery file, `file` the token +/// file of `port`. +fn remove_discovery(current: &Path, file: &Path, port: u16, token: &str) { + let named = std::fs::read_to_string(current).ok().filter(|t| t.len() <= 64).and_then(|t| parse_port(&t)); + if named != Some(port) || held_by_another(file, token) { + return; + } + let _ = std::fs::remove_file(current); + let _ = std::fs::remove_file(pid_path(current)); +} + +/// A client token and, when it came from a token file, where that file is. +pub struct ClientToken { + pub token: String, + pub file: Option, + /// The app it is for, named in an explanation. + pub app: App, +} + +/// The token file a client read: its path, when it was written and the process that wrote it. +pub struct TokenSource { + pub path: PathBuf, + pub modified: Option, + /// From `.pid`: the writer's process id and, on Linux, its start time. + pub writer: Option<(u32, Option)>, +} + +impl ClientToken { + /// The app's `error`, and for a refused token read from a file, which file it was, when it + /// was written and whether the instance that wrote it has exited. Never the token itself. + pub fn explain_error(&self, error: &str) -> String { + let Some(src) = &self.file else { return error.to_string() }; + if !error.starts_with("unauthorized") { + return error.to_string(); + } + let written = src + .modified + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map(|t| (self.app.utc)(t.as_secs())) + .unwrap_or_else(|| "at an unknown time".into()); + let age = src.modified.and_then(|t| t.elapsed().ok()).map(|d| format!(" ({} s ago)", d.as_secs())).unwrap_or_default(); + let path = src.path.display(); + let name = self.app.name; + match src.writer.filter(|(pid, start)| writer_exited(*pid, *start)) { + Some((pid, _)) => format!( + "{error}; stale token file from an instance that exited (process {pid}): {path}, written {written}{age}; delete that file or restart {name}" + ), + None => format!("{error}; token from {path}, written {written}{age}; if {name} was force-quit, restart it or delete that file"), + } + } +} + +/// Files bigger than this are not token files. +const MAX_TOKEN_FILE: u64 = 4096; + +fn read_token_file(path: &Path, app: App) -> Option { + let f = std::fs::File::open(path).ok()?; + let modified = f.metadata().ok().and_then(|m| m.modified().ok()); + let mut text = String::new(); + f.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?; + if text.len() as u64 > MAX_TOKEN_FILE { + return None; + } + let token = text.trim(); + if token.is_empty() { + return None; + } + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let writer = read_pid_file(&pid_path(&path)); + Some(ClientToken { token: token.to_string(), file: Some(TokenSource { path, modified, writer }), app }) +} + +/// The process file beside the token file `path`: `7878.token` → `7878.pid`. +pub fn pid_path(path: &Path) -> PathBuf { + path.with_extension("pid") +} + +/// ` []`; anything else (or no file) is no information. +fn read_pid_file(path: &Path) -> Option<(u32, Option)> { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(256).read_to_string(&mut text).ok()?; + let mut fields = text.split_whitespace(); + let pid = fields.next()?.parse::().ok().filter(|&pid| pid > 0)?; + Some((pid, fields.next().and_then(|s| s.parse().ok()))) +} + +/// Whether the process that wrote a token file is known to be gone: not running, or its id now +/// belongs to a process that started at another time. Unknown counts as running. +fn writer_exited(pid: u32, start: Option) -> bool { + process_running(pid) == Some(false) || start.zip(process_start(pid)).is_some_and(|(saved, now)| saved != now) +} + +/// Whether process `pid` runs; `None` when that can't be found out. +fn process_running(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + if !Path::new("/proc/self").exists() { + return None; + } + match std::fs::metadata(format!("/proc/{pid}")) { + Ok(_) => Some(true), + Err(e) if e.kind() == ErrorKind::NotFound => Some(false), + Err(_) => None, + } + } + #[cfg(all(unix, not(target_os = "linux")))] + { + let output = std::process::Command::new("ps").args(["-p", &pid.to_string(), "-o", "pid="]).output().ok()?; + if output.status.success() { + return Some(true); + } + (output.status.code() == Some(1) && output.stdout.iter().all(u8::is_ascii_whitespace)).then_some(false) + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + let output = std::process::Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .creation_flags(0x0800_0000) + .output() + .ok()?; + if !output.status.success() || output.stdout.is_empty() { + return None; + } + // `"image","pid",…` per process; with no match, a message line without a pid. + let text = String::from_utf8_lossy(&output.stdout); + Some(text.lines().any(|line| line.split("\",\"").nth(1).and_then(|s| s.trim_matches('"').parse::().ok()) == Some(pid))) + } + #[cfg(not(any(unix, windows)))] + { + let _ = pid; + None + } +} + +/// Linux: the process's start time (clock ticks after boot), which tells a reused id apart. +fn process_start(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let mut stat = String::new(); + std::fs::File::open(format!("/proc/{pid}/stat")).ok()?.take(4096).read_to_string(&mut stat).ok()?; + stat.rsplit_once(')')?.1.split_whitespace().nth(19)?.parse().ok() + } + #[cfg(not(target_os = "linux"))] + { + let _ = pid; + None + } +} + +/// `a == b`, taking the same time wherever they differ. +pub fn tokens_match(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// [`App::write_token_file`], with `restrict` limiting the new file to the user (Windows). +fn write_token_file_with(path: &Path, token: &str, restrict: impl FnOnce(&Path) -> Result<()>) -> Result<()> { + let dir = path.parent().ok_or_else(|| Error::new(ErrorKind::InvalidInput, "no folder"))?; + std::fs::create_dir_all(dir)?; + let _ = std::fs::remove_file(path); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + opts.mode(0o600); + } + let opened = opts.open(path)?; + // On a refusal the file goes with the closure, so it is closed before it is removed. + let mut f = match restrict(path).map(|()| opened) { + Ok(f) => f, + Err(e) => { + let _ = std::fs::remove_file(path); + return Err(e); + } + }; + f.write_all(token.as_bytes())?; + f.write_all(b"\n")?; + f.sync_all()?; + let pid = std::process::id(); + let line = process_start(pid).map_or_else(|| format!("{pid}\n"), |start| format!("{pid} {start}\n")); + let _ = std::fs::write(pid_path(path), line); + Ok(()) +} + +/// Remove a token file this launch wrote, and its process file, while it still holds `token`. +/// Another launch may have written the same file since (a second window on the same port, +/// #8794); its file stays. The process id can't tell the two apart, as every Flatpak instance +/// has the same one, but each launch's token is its own. +pub fn remove_token_file(path: &Path, token: &str) { + if read_token(path).is_some_and(|held| tokens_match(&held, token.trim())) { + let _ = std::fs::remove_file(path); + let _ = std::fs::remove_file(pid_path(path)); + } +} + +/// The token a token file holds, trimmed; `None` when there is no readable token file. +fn read_token(path: &Path) -> Option { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?; + let token = text.trim(); + (text.len() as u64 <= MAX_TOKEN_FILE && !token.is_empty()).then(|| token.to_string()) +} + +/// Whether the token file at `path` holds a token other than `token`: another launch's. +fn held_by_another(path: &Path, token: &str) -> bool { + read_token(path).is_some_and(|held| !tokens_match(&held, token.trim())) +} + +/// How long `icacls` may take (#6145). It finishes at once; on some computers the new process +/// is left suspended (security software holding it), and waiting on it without a limit froze +/// the app. +#[cfg(windows)] +const ICACLS_LIMIT: std::time::Duration = std::time::Duration::from_secs(5); + +/// How a run of `icacls` ended. +#[cfg(any(windows, test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum IcaclsEnd { + Succeeded, + Failed, + /// It didn't finish within [`ICACLS_LIMIT`] and was ended. + TimedOut, +} + +/// Whether a token may go into a file after `icacls` ended this way. `custom_folder`: the file is +/// in the folder the app's `CONTROL_DIR` names. A timeout is accepted only in the default folder, +/// the user's own local app data, which other users can't read by default; nothing is known about +/// the permissions of any other folder. +#[cfg(any(windows, test))] +fn restriction_result(end: IcaclsEnd, custom_folder: bool, path: &Path) -> Result<()> { + match end { + IcaclsEnd::Succeeded => Ok(()), + IcaclsEnd::TimedOut if !custom_folder => Ok(()), + IcaclsEnd::TimedOut => Err(Error::new(ErrorKind::TimedOut, format!("icacls did not finish in time to restrict {}", path.display()))), + IcaclsEnd::Failed => Err(Error::other(format!("icacls failed to restrict {}", path.display()))), + } +} + +/// Windows: drop inherited entries and grant only the current user full access. An `icacls` +/// that doesn't finish in time is ended, and [`restriction_result`] decides what that means. +#[cfg(windows)] +fn restrict_to_user(path: &Path, custom_folder: bool) -> Result<()> { + use std::os::windows::process::CommandExt; + let user = std::env::var("USERNAME").map_err(|_| Error::other("USERNAME is not set"))?; + let domain = std::env::var("USERDOMAIN").ok().filter(|d| !d.is_empty()); + let who = domain.map_or(user.clone(), |d| format!("{d}\\{user}")); + let child = std::process::Command::new("icacls") + .arg(path) + .args(["/inheritance:r", "/grant:r", &format!("{who}:F")]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .creation_flags(0x0800_0000) + .spawn()?; + let end = match wait_within(child, ICACLS_LIMIT)? { + Some(status) if status.success() => IcaclsEnd::Succeeded, + Some(_) => IcaclsEnd::Failed, + None => IcaclsEnd::TimedOut, + }; + restriction_result(end, custom_folder, path) +} + +/// Wait for `child` at most `limit`: its exit status, or `None` (and the child ended) when it +/// hasn't finished by then. +#[cfg(any(windows, test))] +fn wait_within(mut child: std::process::Child, limit: std::time::Duration) -> Result> { + let start = std::time::Instant::now(); + loop { + if let Some(status) = child.try_wait()? { + return Ok(Some(status)); + } + if start.elapsed() >= limit { + // Not waited for afterwards: a suspended process may not end at once. + let _ = child.kill(); + return Ok(None); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +/// Removes the token file (path, token) when the app ends, while it still holds that token. +pub struct TokenFile(pub PathBuf, pub String); + +impl Drop for TokenFile { + fn drop(&mut self) { + remove_token_file(&self.0, &self.1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An app with no data folder and no environment, whose times read `t`. + const TEST: App = App { name: "TestCraft", data_dir: || None, env: |_| None, utc: |secs| format!("t{secs}"), default_port: 27878 }; + + /// #6145: a child that doesn't finish in time is given up on, not waited for forever. + #[cfg(unix)] + #[test] + fn a_slow_child_is_waited_for_only_so_long() { + use std::time::{Duration, Instant}; + let start = Instant::now(); + let slow = std::process::Command::new("sleep").arg("30").spawn().unwrap(); + assert!(wait_within(slow, Duration::from_millis(200)).unwrap().is_none()); + assert!(start.elapsed() < Duration::from_secs(5), "{:?}", start.elapsed()); + let quick = std::process::Command::new("true").spawn().unwrap(); + assert!(wait_within(quick, Duration::from_secs(5)).unwrap().is_some_and(|s| s.success())); + let failing = std::process::Command::new("false").spawn().unwrap(); + assert!(wait_within(failing, Duration::from_secs(5)).unwrap().is_some_and(|s| !s.success())); + } + + /// #6145's freeze fix stays in the default folder, the user's own; a folder picked with + /// the app's `CONTROL_DIR` gets no token unless the file was really restricted. + #[test] + fn only_the_default_folder_accepts_an_icacls_that_timed_out() { + let path = Path::new("control/7878.token"); + for custom in [false, true] { + assert!(restriction_result(IcaclsEnd::Succeeded, custom, path).is_ok()); + assert!(restriction_result(IcaclsEnd::Failed, custom, path).is_err()); + } + assert!(restriction_result(IcaclsEnd::TimedOut, false, path).is_ok()); + let e = restriction_result(IcaclsEnd::TimedOut, true, path).unwrap_err(); + assert_eq!(e.kind(), ErrorKind::TimedOut); + assert!(e.to_string().contains("7878.token"), "{e}"); + } + + /// A file that couldn't be restricted is removed, empty and with no process file beside it, and + /// the error goes to the caller. A restricted one gets its token. + #[test] + fn a_token_file_that_could_not_be_restricted_is_removed() { + let dir = std::env::temp_dir().join(format!("craft-control-restrict-{}", std::process::id())); + let path = dir.join("7879.token"); + let e = write_token_file_with(&path, "secret-token", |p| { + assert_eq!(std::fs::read(p).unwrap(), b"", "restricted before the token goes in"); + restriction_result(IcaclsEnd::TimedOut, true, p) + }) + .unwrap_err(); + assert_eq!(e.kind(), ErrorKind::TimedOut); + assert!(!path.exists() && !pid_path(&path).exists()); + write_token_file_with(&path, "secret-token", |p| restriction_result(IcaclsEnd::TimedOut, false, p)).unwrap(); + assert_eq!(read_token(&path).as_deref(), Some("secret-token")); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn tokens_compare_whole() { + assert!(tokens_match("abc123", "abc123")); + assert!(!tokens_match("abc123", "abc124")); + assert!(!tokens_match("abc123", "abc12")); + assert!(!tokens_match("", "a")); + } + + /// The preference starts the listener by default, off it doesn't, and a port given on the + /// command line or in the environment wins over it either way. + #[test] + fn the_preference_decides_unless_a_port_is_given() { + assert_eq!(plan(None, true), Some(Listen::Default)); + assert_eq!(plan(None, false), None); + assert_eq!(plan(Some(7878), false), Some(Listen::Exact(7878))); + assert_eq!(plan(Some(7878), true), Some(Listen::Exact(7878))); + } + + /// A taken port falls through to the next free one, on loopback only; a run of taken ports + /// is an error, not a hang. + #[test] + fn a_taken_port_falls_back_to_the_next_free_one() { + let first = bind_with_fallback(TEST.default_port, PORT_TRIES).unwrap(); + let a = first.local_addr().unwrap(); + assert!(a.ip().is_loopback() && (TEST.default_port..TEST.default_port + PORT_TRIES).contains(&a.port()), "{a}"); + let second = bind_with_fallback(a.port(), PORT_TRIES).unwrap(); + assert!(second.local_addr().unwrap().port() > a.port()); + assert!(bind_with_fallback(a.port(), 1).is_err()); + } + + #[test] + fn the_discovery_file_holds_a_port() { + assert_eq!(parse_port("27879\n"), Some(27879)); + assert_eq!(parse_port("0"), None); + assert_eq!(parse_port("70000"), None); + assert_eq!(parse_port("nope"), None); + let dir = std::env::temp_dir().join(format!("solidcraft-current-port-{}", std::process::id())); + let path = dir.join("current.port"); + TEST.write_token_file(&path, "27881").unwrap(); + assert_eq!(parse_port(&std::fs::read_to_string(&path).unwrap()), Some(27881)); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A one-line token file (and one with no process file) still reads. + #[test] + fn a_token_file_reads_with_and_without_its_process_file() { + let dir = std::env::temp_dir().join(format!("craft-control-read-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("7879.token"); + std::fs::write(&path, "legacy-token\n").unwrap(); + let client = read_token_file(&path, TEST).unwrap(); + assert_eq!(client.token, "legacy-token"); + let src = client.file.as_ref().unwrap(); + assert_eq!(src.path, std::fs::canonicalize(&path).unwrap()); + assert!(src.modified.is_some() && src.writer.is_none()); + std::fs::write(pid_path(&path), "not a pid\n").unwrap(); + assert!(read_token_file(&path, TEST).unwrap().file.unwrap().writer.is_none()); + std::fs::write(&path, "x".repeat(4097)).unwrap(); + assert!(read_token_file(&path, TEST).is_none()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A refused token's explanation names the app and writes the time as the app does. + #[test] + fn an_explanation_names_the_app() { + let file = Some(TokenSource { path: PathBuf::from("7878.token"), modified: Some(UNIX_EPOCH), writer: None }); + let client = ClientToken { token: "secret-token".into(), file, app: TEST }; + let error = client.explain_error("unauthorized"); + assert!(error.contains("written t0 (") && error.contains("if TestCraft was force-quit"), "{error}"); + assert!(!error.contains("secret-token"), "{error}"); + let explicit = TEST.client_token(0, Some("explicit-token")).unwrap(); + assert!(explicit.token == "explicit-token" && explicit.file.is_none()); + assert_eq!(explicit.explain_error("unauthorized"), "unauthorized"); + } + + /// A process file naming a process that has exited marks the token file stale. + /// Where the process inspector cannot run (a sandboxed test runner), the exit + /// is unknown, and the explanation must not claim it. + #[cfg(any(unix, windows))] + #[test] + fn a_token_file_from_an_exited_process_is_stale() { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "auth::tests::tokens_compare_whole"]) + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + assert!(child.wait().unwrap().success()); + let source = |writer| Some(TokenSource { path: PathBuf::from("7878.token"), modified: None, writer }); + let stale = ClientToken { token: "secret-token".into(), file: source(Some((pid, None))), app: TEST }; + let error = stale.explain_error("unauthorized"); + match process_running(pid) { + Some(false) => { + assert!(error.contains(&format!("stale token file from an instance that exited (process {pid}): 7878.token")), "{error}"); + } + None => assert!(!error.contains("stale") && error.contains("restart it or delete that file"), "{error}"), + Some(true) => panic!("the reaped child {pid} is reported as running"), + } + assert!(!error.contains("secret-token")); + let live = ClientToken { token: "t".into(), file: source(Some((std::process::id(), process_start(std::process::id())))), app: TEST }; + assert!(!live.explain_error("unauthorized").contains("stale")); + #[cfg(target_os = "linux")] + { + let reused = process_start(std::process::id()).map(|s| s + 1); + let reused = ClientToken { token: "t".into(), file: source(Some((std::process::id(), reused))), app: TEST }; + assert!(reused.explain_error("unauthorized").contains("stale token file")); + } + } + + /// The token file is the user's alone, and a client reads it back. + #[cfg(unix)] + #[test] + fn the_token_file_is_private_to_the_user() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("craft-control-auth-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let path = dir.join("control/7878.token"); + std::fs::create_dir_all(dir.join("control")).unwrap(); + std::fs::write(&path, "stale").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + TEST.write_token_file(&path, "0123456789abcdef0123456789abcdef").unwrap(); + let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&path), 0o600); + assert_eq!(mode(path.parent().unwrap()), 0o700); + assert_eq!(std::fs::read_to_string(&path).unwrap().trim(), "0123456789abcdef0123456789abcdef"); + let writer = read_token_file(&path, TEST).unwrap().file.unwrap().writer; + assert_eq!(writer.map(|(pid, _)| pid), Some(std::process::id())); + drop(TokenFile(path.clone(), "0123456789abcdef0123456789abcdef".into())); + assert!(!path.exists() && !pid_path(&path).exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// #8794: under Flatpak two windows can listen on the same port number (each has its own + /// loopback) and write the same files. The one closing first leaves the other's token, process + /// and discovery files; the last one takes them all away, `current.pid` too. + #[test] + fn a_closing_launch_leaves_another_launchs_files() { + let dir = std::env::temp_dir().join(format!("craft-control-two-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let (file, current) = (dir.join("27878.token"), dir.join("current.port")); + let (first, second) = ("a".repeat(64), "b".repeat(64)); + // The first window, then the second on the same port, each writing both files. + for token in [&first, &second] { + TEST.write_token_file(&file, token).unwrap(); + TEST.write_token_file(¤t, "27878").unwrap(); + } + // The first closes: the second's files stay. + remove_token_file(&file, &first); + remove_discovery(¤t, &file, 27878, &first); + assert_eq!(read_token(&file).as_deref(), Some(second.as_str())); + assert!(pid_path(&file).exists() && current.exists() && pid_path(¤t).exists()); + // Another port named in the discovery file is another app's too. + remove_discovery(¤t, &file, 27879, &second); + assert!(current.exists()); + // The second closes: nothing of either is left. + remove_token_file(&file, &format!(" {second}\n")); + remove_discovery(¤t, &file, 27878, &second); + assert!(!file.exists() && !pid_path(&file).exists() && !current.exists() && !pid_path(¤t).exists()); + // With no token file left (removed by hand, or never written), the discovery file of the + // port still goes; a missing file is no error. + TEST.write_token_file(¤t, "27878").unwrap(); + remove_token_file(&file, &second); + remove_discovery(¤t, &file, 27878, &second); + assert!(!current.exists()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/control/src/cancel.rs b/crates/control/src/cancel.rs new file mode 100644 index 000000000..e03052307 --- /dev/null +++ b/crates/control/src/cancel.rs @@ -0,0 +1,118 @@ +//! Where a control request is, so the connection that sent it can cancel it (`control.cancel`, +//! #1989): shared between the window, which moves it along, and the connection. Whoever changes +//! it first wins. + +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::{Mutex, PoisonError}; + +/// A request's place in the app, and how to stop the work computing it. +#[derive(Default)] +pub struct Ticket { + state: AtomicU8, + /// It is an export computing on a copy, whose worker writes its file. + writes_file: AtomicBool, + /// Tells its worker to stop: set by the window when it hands the request to one. + stopper: Mutex>>, +} + +impl std::fmt::Debug for Ticket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Ticket").field("state", &self.state).field("writes_file", &self.writes_file).finish_non_exhaustive() + } +} + +/// Waiting its turn (sent, or held behind another request). +const WAITING: u8 = 0; +/// Running on the window thread (or handed to something that can't stop it, such as the file +/// loader). +const RUNNING: u8 = 1; +/// Computing off the window thread. +const COMPUTING: u8 = 2; +const CANCELLED: u8 = 3; + +/// What [`Ticket::cancel`] did. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Cancel { + /// It was waiting its turn: it will not run. + Removed, + /// It was computing off the window thread: it is answered as stopped and nothing of it is + /// applied. An export's worker may still write its file. + Stopped { writes_file: bool }, + /// It runs on the window thread: it will finish. + Running, + /// It was cancelled or stopped before. + Already, +} + +impl Ticket { + /// Cancel the request, unless it already runs on the window thread. + pub fn cancel(&self) -> Cancel { + let mut state = self.state.load(Ordering::Acquire); + loop { + let done = match state { + WAITING => Cancel::Removed, + COMPUTING => Cancel::Stopped { writes_file: self.writes_file.load(Ordering::Acquire) }, + CANCELLED => return Cancel::Already, + _ => return Cancel::Running, + }; + match self.state.compare_exchange(state, CANCELLED, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => { + self.stop_worker(); + return done; + } + Err(now) => state = now, + } + } + } + + pub fn is_cancelled(&self) -> bool { + self.state.load(Ordering::Acquire) == CANCELLED + } + + /// The window takes the request to run it: false when it was cancelled. + pub fn take(&self) -> bool { + self.moves(WAITING, RUNNING) + } + + /// The work computing it stops on `stop` (a worker's own stop flag). Cancelled already, it is + /// stopped at once. + pub fn on_stop(&self, stop: impl Fn() + Send + Sync + 'static) { + *self.stopper.lock().unwrap_or_else(PoisonError::into_inner) = Some(Box::new(stop)); + if self.is_cancelled() { + self.stop_worker(); + } + } + + /// The window hands it to a worker (`writes_file`: an export on a copy). + pub fn computing(&self, writes_file: bool) { + self.writes_file.store(writes_file, Ordering::Release); + let _ = self.moves(RUNNING, COMPUTING) || self.moves(WAITING, COMPUTING); + } + + /// Its worker is done and it runs on the window thread now (or it never left): false when it + /// was cancelled meanwhile. Only the window moves it on from running, so running stays running. + pub fn back(&self) -> bool { + self.moves(COMPUTING, RUNNING) || self.state.load(Ordering::Acquire) == RUNNING + } + + /// Computed, but it waits its turn again (a file opening meanwhile): false when cancelled. + pub fn wait_again(&self) -> bool { + self.moves(COMPUTING, WAITING) + } + + /// The person at the app stopped it (Stop): its worker stops too. + pub fn stop(&self) { + self.state.store(CANCELLED, Ordering::Release); + self.stop_worker(); + } + + fn stop_worker(&self) { + if let Some(stop) = self.stopper.lock().unwrap_or_else(PoisonError::into_inner).as_ref() { + stop(); + } + } + + fn moves(&self, from: u8, to: u8) -> bool { + self.state.compare_exchange(from, to, Ordering::AcqRel, Ordering::Acquire).is_ok() + } +} diff --git a/crates/control/src/client.rs b/crates/control/src/client.rs new file mode 100644 index 000000000..cbad78850 --- /dev/null +++ b/crates/control/src/client.rs @@ -0,0 +1,616 @@ +//! The control channel's client (docs/control-protocol.md): a connection to an app's loopback +//! control port, with the app's token ([`App::client_token`]) on every request, one bounded +//! round trip per request, a fresh connection after one is lost, and the wait for a request the +//! app says is still running (#1618), through `control.pending`. Its errors tell an app that is +//! not running from one that went away during a call ([`CallError`]). What a request means, and +//! anything an app adds to its parameters, is the caller's. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::Effect; +use crate::auth::App; + +/// Longest reply line accepted from the app. +pub const MAX_REPLY: u64 = 64 << 20; + +/// How long a reply may take: kernel operations (booleans, fillets) on big parts can take a while. +pub const READ_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long the app may take to answer a request sent [`Request::once`]: a little longer than its +/// control server waits for a long request (3600 s), so its "still running" reply arrives first. +pub const ONCE_TIMEOUT: Duration = Duration::from_secs(3720); + +/// How long connecting to the app may take. +const CONNECT_TIMEOUT: Duration = Duration::from_millis(800); + +/// How often a long wait is reported ([`Connection::with_progress`]). +const PROGRESS_EVERY: Duration = Duration::from_secs(30); + +/// How an error starts when a request is still running in the app after the wait: the request +/// was not cancelled and its reply waits on the connection; [`Connection::pending`] gets it. +pub use crate::STILL_RUNNING; + +/// How long to wait for a request the app says is still running. +#[derive(Clone, Copy, Debug)] +pub struct Waits { + /// How long a call goes on waiting, asking `control.pending`, after the app said its request + /// is still running (the app says so after 60 s), before it says so itself. + pub pending: Duration, + /// How long a later call, or [`Connection::pending`], waits for a request still running from + /// an earlier call. + pub settle: Duration, + /// How often `control.pending` is asked while waiting. + pub poll: Duration, +} + +impl Default for Waits { + fn default() -> Self { + Waits { pending: Duration::from_secs(240), settle: Duration::from_secs(60), poll: Duration::from_millis(500) } + } +} + +/// Why a call has no result. Each holds the message for the caller (an agent reads it). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CallError { + /// Nothing listens at the address: the app is not running, or it quit before the call was + /// sent. The message says how to start it. + NotRunning(String), + /// The app went away or did not answer during the call, after running it or not. + Lost(String), + /// The request is still running in the app after the wait (it starts with [`STILL_RUNNING`]), + /// or an earlier one is and this one was not sent. + StillRunning(String), + /// The app answered with an error (a refused token says which token file it came from). + Refused(String), + /// The reply was not a control reply. + BadReply(String), +} + +impl CallError { + /// The message. + pub fn message(&self) -> &str { + match self { + CallError::NotRunning(m) | CallError::Lost(m) | CallError::StillRunning(m) | CallError::Refused(m) | CallError::BadReply(m) => m, + } + } +} + +impl std::fmt::Display for CallError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message()) + } +} + +impl std::error::Error for CallError {} + +impl From for String { + fn from(e: CallError) -> String { + e.message().to_string() + } +} + +/// A request: its method and parameters, how messages name it and what it does if it is still +/// running when its call returns. +#[derive(Clone, Debug)] +pub struct Request { + pub method: String, + pub params: Value, + /// The request in messages ("`engine.execute` (solid.fillet)"). + pub what: String, + /// What it still does when it finishes after its wait. + pub effect: Effect, + /// Never sent twice: on a fresh connection, waiting up to [`ONCE_TIMEOUT`], and once it is + /// being written any error is final, as the app may have run it. Otherwise a request that + /// finds its connection lost is sent once more on a fresh one (the app may have restarted). + pub once: bool, +} + +impl Request { + /// `method` with `params`, named "`method`" in messages, taken to change the document, sent + /// again once on a fresh connection when its connection was lost. + pub fn new(method: &str, params: Value) -> Self { + Request { method: method.to_string(), params, what: format!("`{method}`"), effect: Effect::Changes, once: false } + } +} + +/// A control reply as a call's result: its `result`, or its `error`. +pub fn outcome(v: &Value) -> Result { + if v.get("ok").and_then(Value::as_bool) == Some(true) { + Ok(v.get("result").cloned().unwrap_or(Value::Null)) + } else { + Err(v.get("error").and_then(Value::as_str).unwrap_or("error").to_string()) + } +} + +/// A request the app said is still running. Its reply will come on this connection, through +/// `control.pending`, so the connection is kept for it. +struct Outstanding { + what: String, + /// What it does when it finishes; `None`: not known (a request this client did not send). + effect: Option, + since: Instant, +} + +impl Outstanding { + fn secs(&self) -> u64 { + self.since.elapsed().as_secs() + } +} + +/// What became of a request still running when its call returned, once a later call waited it +/// out: kept for [`Connection::pending`]. +enum Settled { + Finished { what: String, reply: Value }, + Lost { what: String }, +} + +/// The end of a wait for an outstanding request. +enum Waited { + /// It finished: its own reply. + Done(Value), + /// The app holds nothing (any more) for this connection. + Nothing, + /// Still running when the wait ran out. + Running, +} + +/// A connection to the app listening at an address. It connects when a call needs it, so a +/// client started before the app reaches it once it runs. +pub struct Connection { + app: App, + /// What the app calls its document in messages ("design"). + document: &'static str, + addr: String, + port: u16, + conn: Option<(BufReader, TcpStream)>, + /// Counts the connections made, so a caller can tell a fresh one from the one it asked. + serial: u64, + next_id: u64, + /// The token given, when one was. Otherwise each request looks the token up again, since + /// every launch of the app makes a new one. + explicit_token: Option, + /// A request still running in the app; its reply waits on `conn`. + outstanding: Option, + /// An outstanding request's end, once a later call waited for it (for `pending`). + settled: Option, + waits: Waits, + /// Where a long wait is reported. + progress: fn(&str), +} + +impl Connection { + /// A connection to `app` at `addr` (`host:port`), with `token` when given, else the app's + /// token found the usual way ([`App::client_token`]); not connected yet. + pub fn new(app: App, addr: &str, token: Option<&str>) -> Self { + let port = addr.rsplit(':').next().and_then(|p| p.parse::().ok()).unwrap_or(0); + Self { + app, + document: "design", + addr: addr.to_string(), + port, + conn: None, + serial: 0, + next_id: 1, + explicit_token: token.map(str::to_string), + outstanding: None, + settled: None, + waits: Waits::default(), + progress: |_| {}, + } + } + + /// What the app calls its document in messages ("design", the default). + #[must_use] + pub fn with_document(mut self, document: &'static str) -> Self { + self.document = document; + self + } + + /// Report a long wait for a request still running to `progress`, every 30 s. + #[must_use] + pub fn with_progress(mut self, progress: fn(&str)) -> Self { + self.progress = progress; + self + } + + /// How long to wait for a request still running. + pub fn set_waits(&mut self, waits: Waits) { + self.waits = waits; + } + + /// The address it connects to. + pub fn addr(&self) -> &str { + &self.addr + } + + /// The port of the address (0 when it names none). + pub fn port(&self) -> u16 { + self.port + } + + /// The token given, if one was. + pub fn explicit_token(&self) -> Option<&str> { + self.explicit_token.as_deref() + } + + /// Whether a connection is open now. + pub fn is_connected(&self) -> bool { + self.conn.is_some() + } + + /// The number of connections made so far: the same number while the same connection stays + /// open. + pub fn serial(&self) -> u64 { + self.serial + } + + /// Whether a request of this connection is still running in the app. + pub fn has_outstanding(&self) -> bool { + self.outstanding.is_some() + } + + /// Close the connection. A request still running loses its reply with it. + pub fn disconnect(&mut self) { + self.conn = None; + if let Some(o) = self.outstanding.take() { + self.settled = Some(Settled::Lost { what: o.what }); + } + } + + /// Connect afresh. Failing to reach the app at all is `NotConnected`, so [`Connection::explain`] + /// can tell it from a connection lost during a call. + /// + /// # Errors + /// `NotConnected` when nothing listens at the address (or it does not resolve). + pub fn connect(&mut self) -> std::io::Result<()> { + self.disconnect(); + let not_connected = |e: std::io::Error| std::io::Error::new(std::io::ErrorKind::NotConnected, e); + let mut last = std::io::Error::new(std::io::ErrorKind::NotFound, format!("cannot resolve {}", self.addr)); + for sa in self.addr.to_socket_addrs().map_err(not_connected)? { + match TcpStream::connect_timeout(&sa, CONNECT_TIMEOUT) { + Ok(s) => { + s.set_nodelay(true).ok(); + s.set_read_timeout(Some(READ_TIMEOUT)).ok(); + let read = s.try_clone()?; + self.conn = Some((BufReader::new(read), s)); + self.serial += 1; + return Ok(()); + } + Err(e) => last = e, + } + } + Err(not_connected(last)) + } + + /// See whether the app is there now; the error says why not. The connection doesn't stay + /// open: the app closes one that shows no token within a few seconds, and the first call may + /// be long away. That call connects again. + /// + /// # Errors + /// As [`Connection::connect`]. + pub fn try_connect(&mut self) -> std::io::Result<()> { + self.connect()?; + self.disconnect(); + Ok(()) + } + + /// The error for a failed exchange: whether the app is not running (and how to start it) or + /// went away during the call. + pub fn explain(&self, e: &std::io::Error) -> CallError { + let (addr, port, name, document) = (&self.addr, self.port, self.app.name, self.document); + let start = format!("{} --control {port}", name.to_lowercase()); + if e.kind() == std::io::ErrorKind::NotConnected { + CallError::NotRunning(format!( + "no {name} app is listening at {addr} ({e}): it is not running, or it quit. Start it with `{start}` (or ask the user to); the \ + next call connects to it" + )) + } else { + CallError::Lost(format!( + "the {name} app at {addr} did not answer ({e}): it may have quit or crashed during this call, after running it or not. Inspect \ + the {document} before sending a change again; if the app quit, start it with `{start}`" + )) + } + } + + /// The request line for `method`: a fresh id and the app's token (looked up again for every + /// request: an app restarted mid-session has a new one). + pub fn request_line(&mut self, method: &str, params: Value) -> String { + let id = self.next_id; + self.next_id += 1; + let mut msg = json!({"id": id, "method": method, "params": params}); + if let Some(t) = self.app.client_token(self.port, self.explicit_token.as_deref()) { + msg["token"] = json!(t.token); + } + msg.to_string() + } + + /// One request line out and one reply line back, at most [`MAX_REPLY`] long, connecting + /// first when not connected. Any failure closes the connection (a timed-out read leaves a + /// late reply on the stream). + fn roundtrip(&mut self, line: &str) -> std::io::Result { + if self.conn.is_none() { + self.connect()?; + } + let Some((reader, writer)) = self.conn.as_mut() else { return Err(std::io::Error::other("not connected")) }; + // One write: a request that loses its newline still reaches the app as a request. + let sent = writer.write_all(format!("{line}\n").as_bytes()).and_then(|()| writer.flush()); + if let Err(e) = sent { + self.disconnect(); + return Err(e); + } + let mut reply = String::new(); + let n = match reader.by_ref().take(MAX_REPLY).read_line(&mut reply) { + Ok(n) => n, + Err(e) => { + self.disconnect(); + return Err(e); + } + }; + if n == 0 || !reply.ends_with('\n') { + self.disconnect(); + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "connection closed")); + } + Ok(reply) + } + + fn set_read_timeout(&self, t: Duration) { + if let Some((_, writer)) = &self.conn { + writer.set_read_timeout(Some(t)).ok(); + } + } + + /// A reply line as JSON. + fn parse(reply: &str) -> Result { + serde_json::from_str(reply).map_err(|e| CallError::BadReply(format!("bad reply from the app: {e}"))) + } + + /// A control reply as a call's result; a refused token says which token file it came from + /// (and whether it is stale). + /// + /// # Errors + /// [`CallError::Refused`] with the app's error. + pub fn outcome(&self, v: &Value) -> Result { + outcome(v).map_err(|error| { + CallError::Refused( + self.app.client_token(self.port, self.explicit_token.as_deref()).map_or_else(|| error.clone(), |t| t.explain_error(&error)), + ) + }) + } + + /// `method` with `params`, at once and once: not behind a request still running on this + /// connection, no second try. The app's whole reply (`ok`, `result` or `error`, and anything + /// it adds), for a request the app answers at once on any connection + /// (`control.capabilities`, `control.running`). + /// + /// # Errors + /// As [`Connection::explain`]; [`CallError::BadReply`]. + pub fn ask(&mut self, method: &str, params: Value) -> Result { + let line = self.request_line(method, params); + let reply = self.roundtrip(&line).map_err(|e| self.explain(&e))?; + Self::parse(&reply) + } + + /// `method` with `params`, waiting at most `timeout` for the reply and never behind a request + /// still running: for a request the app answers within a frame, asked again later when the + /// app is busy. One try. + /// + /// # Errors + /// [`CallError::StillRunning`] when a request of this connection is still running; as + /// [`Connection::ask`] and [`Connection::outcome`]. + pub fn quick(&mut self, method: &str, params: Value, timeout: Duration) -> Result { + if self.outstanding.is_some() { + return Err(CallError::StillRunning("a request is still running in the app".into())); + } + if self.conn.is_none() { + self.connect().map_err(|e| self.explain(&e))?; + } + let line = self.request_line(method, params); + self.set_read_timeout(timeout); + let reply = self.roundtrip(&line); + self.set_read_timeout(READ_TIMEOUT); + let reply = reply.map_err(|e| self.explain(&e))?; + let v = Self::parse(&reply)?; + self.outcome(&v) + } + + /// Ask the app, on this connection only, about the request it still runs for it. + fn ask_pending(&mut self) -> Result { + if self.conn.is_none() { + self.disconnect(); + return Err(CallError::Lost("the connection to the app was lost".into())); + } + let v = self.ask("control.pending", json!({}))?; + outcome(&v).map_err(CallError::Refused).inspect_err(|_| self.disconnect()) + } + + /// `control.cancel` for the request still running on this connection. Sent at once, without + /// waiting for that request first, and never on a fresh connection, which has nothing to + /// cancel. Once the app no longer holds it (removed, stopped or finished), later calls don't + /// wait for it. + /// + /// # Errors + /// As [`Connection::ask`]; [`CallError::Refused`]. + pub fn cancel(&mut self, params: Value) -> Result { + if self.conn.is_none() { + return Ok(json!({"cancelled": null, "note": "nothing to cancel: no request of this client is running in the app"})); + } + let v = self.ask("control.cancel", params)?; + let r = outcome(&v).map_err(CallError::Refused); + if r.is_ok() { + self.outstanding = None; + } + r + } + + /// Wait up to `limit` for the outstanding request, asking `control.pending` now and then and + /// reporting how long it takes. An error: the connection, and so the reply, was lost. + fn wait_outstanding(&mut self, limit: Duration) -> Result { + let start = Instant::now(); + let mut told = start; + loop { + let Some(o) = &self.outstanding else { return Ok(Waited::Nothing) }; + let what = o.what.clone(); + let r = self.ask_pending()?; + if r.get("pending").is_none_or(Value::is_null) { + self.outstanding = None; + return Ok(Waited::Nothing); + } + if r.get("done").and_then(Value::as_bool) == Some(true) { + self.outstanding = None; + return Ok(Waited::Done(r.get("reply").cloned().unwrap_or(Value::Null))); + } + let waited = start.elapsed(); + if waited >= limit { + return Ok(Waited::Running); + } + if told.elapsed() >= PROGRESS_EVERY { + let secs = self.outstanding.as_ref().map_or(0, Outstanding::secs); + (self.progress)(&format!("{what} is still running in the app ({secs} s); waiting for it")); + told = Instant::now(); + } + std::thread::sleep(self.waits.poll.min(limit.saturating_sub(waited))); + } + } + + /// The error for the outstanding request still running after its wait. + fn still_running(&self, earlier: bool) -> CallError { + let Some(o) = &self.outstanding else { return CallError::StillRunning(format!("{STILL_RUNNING} a request is still running in the app")) }; + let document = self.document; + let effect = + o.effect.map_or_else(|| format!("it was not cancelled and may still change the {document} when it finishes"), |e| e.when_done(document)); + let settle = self.waits.settle.as_secs(); + CallError::StillRunning(if earlier { + format!( + "{STILL_RUNNING} an earlier request, {}, is still running in the app ({} s; {effect}), so this call was not sent. Call the \ + `pending` tool to wait for its result, then send this call again", + o.what, + o.secs() + ) + } else { + format!( + "{STILL_RUNNING} {} is still running in the app after {} s ({effect}); don't send it again. Call the `pending` tool to \ + wait for its result (up to {settle} s per call); other calls wait for it first", + o.what, + o.secs() + ) + }) + } + + /// Before a new request: wait for one still running from an earlier call, keeping its end for + /// `pending`. Still running after the wait: the new request is not sent. + fn settle_earlier(&mut self) -> Result<(), CallError> { + let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return Ok(()) }; + match self.wait_outstanding(self.waits.settle) { + Ok(Waited::Done(reply)) => self.settled = Some(Settled::Finished { what, reply }), + Ok(Waited::Nothing) => {} + Ok(Waited::Running) => return Err(self.still_running(true)), + // The connection was lost, and the earlier reply with it (`pending` says so): the new + // request goes on a fresh connection. + Err(_) => {} + } + Ok(()) + } + + /// `control.pending` for the client: the end of the request still running from an earlier + /// call, waiting for it up to the settle wait: `{pending: what | null, done?, reply?, lost?, + /// running_s?}`. Never sent on a fresh connection, which has nothing pending. + pub fn pending(&mut self) -> Value { + if let Some(s) = self.settled.take() { + return match s { + Settled::Finished { what, reply } => json!({"pending": what, "done": true, "reply": reply}), + Settled::Lost { what } => json!({"pending": what, "lost": true}), + }; + } + let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return json!({"pending": null}) }; + match self.wait_outstanding(self.waits.settle) { + Ok(Waited::Done(reply)) => json!({"pending": what, "done": true, "reply": reply}), + Ok(Waited::Nothing) => json!({"pending": null}), + Ok(Waited::Running) => json!({"pending": what, "done": false, "running_s": self.outstanding.as_ref().map_or(0, Outstanding::secs)}), + Err(_) => { + self.settled = None; + json!({"pending": what, "lost": true}) + } + } + } + + /// Send `request` and wait for its reply: the app's whole reply (`ok`, `result` or `error`, + /// and anything it adds). First a request still running from an earlier call is waited for + /// (except for `control.running`, which the app answers at once); a request the app says is + /// still running is waited for through `control.pending`; a request refused unrun as busy + /// behind one this client did not know of is sent once more after that one, when + /// `retry_busy`. + /// + /// # Errors + /// As [`Connection::explain`]: a request whose connection is lost is sent once more on a fresh + /// one (unless [`Request::once`]), and when nothing listens any more the app went away, maybe + /// while running it ([`CallError::Lost`]); [`CallError::StillRunning`]; [`CallError::BadReply`]. + pub fn exchange(&mut self, request: Request, retry_busy: bool) -> Result { + let Request { method, params, what, effect, once } = request; + if method != "control.running" { + self.settle_earlier()?; + } + let again = retry_busy.then(|| params.clone()); + let sent = Instant::now(); + let line = self.request_line(&method, params); + let reply = if once { + // A fresh connection (an idle one may have been closed by an app restart; connecting + // again sends nothing). + self.connect().or_else(|_| self.connect()).and_then(|()| { + self.set_read_timeout(ONCE_TIMEOUT); + let r = self.roundtrip(&line); + self.set_read_timeout(READ_TIMEOUT); + r + }) + } else { + self.roundtrip(&line).or_else(|lost| { + self.roundtrip(&line).map_err(|e| if e.kind() == std::io::ErrorKind::NotConnected && lost.kind() != e.kind() { lost } else { e }) + }) + } + .map_err(|e| self.explain(&e))?; + let v = Self::parse(&reply)?; + // Timed out in the app, still running: wait for its result on this connection. + if v.get("running").and_then(Value::as_bool) == Some(true) { + self.outstanding = Some(Outstanding { what: what.clone(), effect: Some(effect), since: sent }); + (self.progress)(&format!( + "{what} is still running in the app after {} s; waiting up to {} s more for it", + sent.elapsed().as_secs(), + self.waits.pending.as_secs() + )); + let document = self.document; + return match self.wait_outstanding(self.waits.pending)? { + Waited::Done(reply) => Ok(reply), + Waited::Nothing => Err(CallError::Lost(format!( + "{what} timed out in the app, which kept no result for it: inspect the {document} before sending it again" + ))), + Waited::Running => Err(self.still_running(false)), + }; + } + // Refused unrun behind a request this client did not know of: wait for that one, then + // send this one once more. + if v.get("busy").and_then(Value::as_bool) == Some(true) + && let Some(params) = again + { + self.outstanding = Some(Outstanding { what: "an earlier request".into(), effect: None, since: Instant::now() }); + self.settle_earlier()?; + return self.exchange(Request { method, params, what, effect, once }, false); + } + Ok(v) + } + + /// [`Connection::exchange`] (retrying a busy refusal once) and the reply's result. + /// + /// # Errors + /// As [`Connection::exchange`] and [`Connection::outcome`]. + pub fn call(&mut self, request: Request) -> Result { + let v = self.exchange(request, true)?; + self.outcome(&v) + } +} + +#[cfg(test)] +#[path = "client_tests.rs"] +mod tests; diff --git a/crates/control/src/client_tests.rs b/crates/control/src/client_tests.rs new file mode 100644 index 000000000..f5af19a00 --- /dev/null +++ b/crates/control/src/client_tests.rs @@ -0,0 +1,190 @@ +//! The client against the shared server ([`crate::server::Hub`]) on an ephemeral loopback port. + +use super::*; +use crate::ControlRequest; +use crate::server::{Hub, Protocol, Stop, Summary, Waits as ServerWaits}; +use std::ffi::OsString; +use std::net::TcpListener; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::mpsc::channel; + +const TOKEN: &str = "0123456789abcdef0123456789abcdef"; + +fn no_dir() -> Option { + None +} + +fn no_env(_: &str) -> Option { + None +} + +fn utc(secs: u64) -> String { + secs.to_string() +} + +/// An app with no token files and no environment: a client sends only the token it is given. +const APP: App = App { name: "Probe", data_dir: no_dir, env: no_env, utc, default_port: 0 }; + +/// An app whose requests all may change its document. +struct Probe; + +impl Protocol for Probe { + fn summary(&self, method: &str, _: &Value) -> Summary { + Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false } + } + fn document(&self) -> &'static str { + "design" + } + fn change_event(&self) -> &'static str { + "design.changed" + } + fn revision_method(&self) -> &'static str { + "document.revision" + } + fn docs(&self) -> &'static str { + "docs/control-protocol.md" + } + fn capabilities(&self) -> Value { + json!({}) + } +} + +/// The app on `port` (0: an ephemeral one) behind [`TOKEN`]: its window answers each request after +/// `delay` with the request's method and parameters, and the server says a request is still +/// running after `wait`. +fn serve(port: u16, delay: Duration, wait: Duration) -> Stop { + // A port just closed may take a moment to be free again on a loaded machine. + let started = Instant::now(); + let mut bound = TcpListener::bind(("127.0.0.1", port)); + while bound.is_err() && started.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(10)); + bound = TcpListener::bind(("127.0.0.1", port)); + } + let listener = bound.unwrap(); + let (tx, rx) = channel::(); + std::thread::spawn(move || { + for req in rx { + std::thread::sleep(delay); + let _ = req.reply.send(json!({"ok": true, "result": {"method": req.method, "params": req.params}})); + } + }); + let waits = ServerWaits { short: wait, long: wait, token: Duration::from_secs(5) }; + let hub = Arc::new(Hub::new(Probe, || {}, 1 << 20).with_waits(waits)); + hub.listen_stoppable(listener, TOKEN.into(), tx, false).unwrap() +} + +fn addr(port: u16) -> String { + format!("127.0.0.1:{port}") +} + +/// A port nothing listens on. +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() +} + +#[test] +fn a_call_is_one_round_trip_with_the_token() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + assert!(!c.is_connected()); + let r = c.call(Request::new("document.inspect", json!({"a": 1}))).unwrap(); + assert_eq!(r, json!({"method": "document.inspect", "params": {"a": 1}})); + assert!(c.is_connected()); + let serial = c.serial(); + // the next call goes on the same connection + c.call(Request::new("document.inspect", json!({}))).unwrap(); + assert_eq!(c.serial(), serial); + // `try_connect` sees the app there and leaves no connection open + let mut fresh = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + fresh.try_connect().unwrap(); + assert!(!fresh.is_connected()); +} + +#[test] +fn a_wrong_token_or_none_is_refused() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + for token in [Some("fedcba9876543210fedcba9876543210"), None] { + let mut c = Connection::new(APP, &addr(app.port()), token); + match c.call(Request::new("document.inspect", json!({}))) { + Err(CallError::Refused(e)) => assert!(e.starts_with("unauthorized"), "{token:?}: {e}"), + other => panic!("{token:?}: {other:?}"), + } + } +} + +#[test] +fn an_app_not_running_says_how_to_start_it() { + let port = free_port(); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + let CallError::NotRunning(m) = &e else { panic!("{e:?}") }; + assert!(m.starts_with(&format!("no Probe app is listening at 127.0.0.1:{port}")), "{m}"); + assert!(m.contains(&format!("`probe --control {port}`")), "{m}"); + assert_eq!(c.try_connect().unwrap_err().kind(), std::io::ErrorKind::NotConnected); +} + +/// An app that goes away during a call is lost, not "not running"; the next call says it is not +/// running, and once it runs again the next call reaches it. +#[test] +fn an_app_gone_during_a_call_is_lost_and_reached_again_once_back() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + let port = app.port(); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + c.call(Request::new("document.inspect", json!({}))).unwrap(); + app.stop(); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + let CallError::Lost(m) = &e else { panic!("{e:?}") }; + assert!(m.contains("may have quit") && m.contains("Inspect the design") && m.contains(&format!("`probe --control {port}`")), "{m}"); + assert!(matches!(c.call(Request::new("document.inspect", json!({}))), Err(CallError::NotRunning(_)))); + let _back = serve(port, Duration::ZERO, Duration::from_secs(10)); + assert_eq!(c.call(Request::new("ui.view", json!({}))).unwrap()["method"], json!("ui.view")); +} + +/// A server that reads a request and closes the connection without a reply: the call is lost +/// (after one more try on a fresh connection), and the reply is never taken for "not running". +#[test] +fn a_connection_closed_without_a_reply_is_lost() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let mut line = String::new(); + let _ = BufReader::new(stream).read_line(&mut line); + } + }); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + assert!(matches!(e, CallError::Lost(_)), "{e:?}"); + assert!(!c.is_connected()); +} + +/// A request the app says is still running is waited for through `control.pending`, on the same +/// connection, and its own reply is the call's result. +#[test] +fn a_request_still_running_is_waited_for_through_pending() { + let app = serve(0, Duration::from_millis(400), Duration::from_millis(50)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + c.set_waits(Waits { pending: Duration::from_secs(10), settle: Duration::from_secs(10), poll: Duration::from_millis(5) }); + let r = c.call(Request::new("solid.slow", json!({"n": 2}))).unwrap(); + assert_eq!(r, json!({"method": "solid.slow", "params": {"n": 2}})); + assert!(!c.has_outstanding()); + assert_eq!(c.pending(), json!({"pending": null})); +} + +/// Past the wait, the call says the request is still running, and `pending` later has its end. +#[test] +fn a_request_still_running_past_the_wait_is_reported_and_its_end_kept() { + let app = serve(0, Duration::from_millis(400), Duration::from_millis(20)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + c.set_waits(Waits { pending: Duration::from_millis(30), settle: Duration::from_secs(10), poll: Duration::from_millis(5) }); + let mut slow = Request::new("solid.slow", json!({})); + slow.what = "`solid.slow` (now)".into(); + let e = c.call(slow).unwrap_err(); + let CallError::StillRunning(m) = &e else { panic!("{e:?}") }; + assert!(m.starts_with(STILL_RUNNING) && m.contains("`solid.slow` (now)") && m.contains("will still apply"), "{m}"); + assert!(c.has_outstanding()); + let end = c.pending(); + assert_eq!(end["done"], json!(true), "{end}"); + assert_eq!(end["reply"]["result"]["method"], json!("solid.slow"), "{end}"); +} diff --git a/crates/control/src/effect.rs b/crates/control/src/effect.rs new file mode 100644 index 000000000..aed5cbe59 --- /dev/null +++ b/crates/control/src/effect.rs @@ -0,0 +1,27 @@ +//! What a client is told about a control request that outlasts its wait (docs/control-protocol.md, +//! #1618): the app's control server and a bridged client both say what the request still does when +//! it finishes. Which requests do what is the app's to say. + +/// What a request still running does when it finishes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Effect { + /// It only reads (the document, the window or the command list): nothing changes. + ReadsOnly, + /// It writes a file (an export, a save, a render to a path); the document stays as it is. + WritesFile, + /// It changes the document (or may: anything not known to read or only write a file). + Changes, +} + +impl Effect { + /// What happens when the request finishes, for a message: "it only reads: nothing changes", + /// "it was not cancelled and will still write the file when it finishes; the design does not + /// change", … `document` is what the app calls its document ("design"). + pub fn when_done(self, document: &str) -> String { + match self { + Effect::ReadsOnly => "it only reads: nothing changes".into(), + Effect::WritesFile => format!("it was not cancelled and will still write the file when it finishes; the {document} does not change"), + Effect::Changes => "it was not cancelled and will still apply when it finishes".into(), + } + } +} diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs new file mode 100644 index 000000000..26727a97c --- /dev/null +++ b/crates/control/src/lib.rs @@ -0,0 +1,26 @@ +//! The control channel any app may share (docs/control-protocol.md): the per-launch token and its +//! files ([`auth`]), the loopback server that admits a connection by the token and hands each +//! request to the window ([`server`]), the client that talks to it ([`client`]), the request the +//! window answers ([`ControlRequest`]), the document watch subscribed connections wait on +//! ([`DocumentWatch`]), and what a request still running does when it finishes ([`Effect`]). It +//! needs no UI toolkit; the window's own requests over egui (`ui.screenshot`, `ui.resize`) are +//! `craft-control-egui`. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +pub mod auth; +mod cancel; +#[cfg(not(target_arch = "wasm32"))] +pub mod client; +mod effect; +mod request; +#[cfg(not(target_arch = "wasm32"))] +pub mod server; + +pub use cancel::{Cancel, Ticket}; +pub use effect::Effect; +pub use request::{ControlRequest, ControlResponse, DOCUMENT_WATCH, DocumentWatch, Subscriber, cancelled_chat_response, chat_turn_cancelled}; + +/// How an error from the client (`client`, native only) starts when a request is still running +/// in the app after the wait. Outside the client so callers on every target can test for it. +pub const STILL_RUNNING: &str = "still running:"; diff --git a/crates/control/src/request.rs b/crates/control/src/request.rs new file mode 100644 index 000000000..b97c90b76 --- /dev/null +++ b/crates/control/src/request.rs @@ -0,0 +1,125 @@ +//! A control request as the window gets it, and the design watch subscribed connections wait on. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::Sender; +use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use serde_json::{Value, json}; + +pub type ControlResponse = Value; + +pub struct ControlRequest { + pub method: String, + pub params: Value, + pub reply: Sender, + /// Set once the window starts on it; until then it waits its turn (`control.running`, #3654). + pub started: std::sync::Arc, + /// Chat requests already admitted by the gate may outlive a tab switch. This turn's + /// validity follows them through queues, computation and cursor animation. + pub chat_turn: Option>, + /// Where the request is, shared with the connection that sent it so it can cancel it + /// (`control.cancel`, #1989). + pub ticket: std::sync::Arc, + /// Told apart from every other request of this run: names its row while it waits. + pub serial: u64, +} + +impl ControlRequest { + pub fn new(method: impl Into, params: Value) -> (Self, std::sync::mpsc::Receiver) { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let (tx, rx) = std::sync::mpsc::channel(); + let serial = NEXT.fetch_add(1, Ordering::Relaxed); + (Self { method: method.into(), params, reply: tx, started: Default::default(), chat_turn: None, ticket: Default::default(), serial }, rx) + } + + /// Answer a request whose agent chat turn has ended, and say so: such a request is not run. + pub fn refuse_cancelled_chat(&self) -> bool { + if chat_turn_cancelled(self.chat_turn.as_deref()) { + let _ = self.reply.send(cancelled_chat_response()); + true + } else { + false + } + } +} + +/// Whether the agent chat turn a request belongs to has ended (`None`: it belongs to none). +pub fn chat_turn_cancelled(turn: Option<&std::sync::atomic::AtomicBool>) -> bool { + turn.is_some_and(|active| !active.load(std::sync::atomic::Ordering::Relaxed)) +} + +/// The reply to a request whose agent chat turn has ended. +pub fn cancelled_chat_response() -> Value { + json!({"ok": false, "error": "not run: the agent chat turn ended or its document tab was left"}) +} + +/// The active design's id and revision for control connections that sent `control.subscribe` +/// (#6985): the window publishes them each frame while one is subscribed, and each such +/// connection waits for them to move, then sends the app's change event (`Protocol::change_event`). +#[derive(Default)] +pub struct DocumentWatch { + subscribers: AtomicUsize, + now: Mutex>, + moved: Condvar, +} + +/// The app's [`DocumentWatch`]. +pub static DOCUMENT_WATCH: DocumentWatch = DocumentWatch::new(); + +impl DocumentWatch { + pub const fn new() -> Self { + DocumentWatch { subscribers: AtomicUsize::new(0), now: Mutex::new(None), moved: Condvar::new() } + } + + fn now(&self) -> MutexGuard<'_, Option<(String, u64)>> { + self.now.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The design as last published (`None` before anything was). + pub fn current(&self) -> Option<(String, u64)> { + self.now().clone() + } + + /// The design as it is now; waiters wake only when it moved. Only the latest is kept, so + /// changes made faster than a connection sends them go out as one. + pub fn publish(&self, document_id: &str, revision: u64) { + let mut now = self.now(); + if now.as_ref().is_some_and(|(id, r)| id == document_id && *r == revision) { + return; + } + *now = Some((document_id.to_string(), revision)); + self.moved.notify_all(); + } + + /// Whether a connection is subscribed: the window publishes only then. + pub fn subscribed(&self) -> bool { + self.subscribers.load(Ordering::Relaxed) > 0 + } + + /// Count a subscribed connection until the returned guard is dropped. + pub fn join(&'static self) -> Subscriber { + self.subscribers.fetch_add(1, Ordering::Relaxed); + Subscriber(self) + } + + /// Wait up to `wait` for the design to differ from `seen`, or for `stop`; the design then + /// (`None` before anything was published). + pub fn wait(&self, seen: &(String, u64), stop: &AtomicBool, wait: Duration) -> Option<(String, u64)> { + let unmoved = |now: &mut Option<(String, u64)>| now.as_ref() == Some(seen) && !stop.load(Ordering::Relaxed); + let (now, _) = self.moved.wait_timeout_while(self.now(), wait, unmoved).unwrap_or_else(PoisonError::into_inner); + now.clone() + } +} + +/// A subscribed connection, counted by its [`DocumentWatch`] while this lives. +pub struct Subscriber(&'static DocumentWatch); + +impl Drop for Subscriber { + fn drop(&mut self) { + self.0.subscribers.fetch_sub(1, Ordering::Relaxed); + // Under the lock, so a connection about to wait sees its `stop` set before it does. + let _now = self.0.now(); + self.0.moved.notify_all(); + } +} diff --git a/crates/control/src/server.rs b/crates/control/src/server.rs new file mode 100644 index 000000000..85664a1aa --- /dev/null +++ b/crates/control/src/server.rs @@ -0,0 +1,1433 @@ +//! The loopback JSON-lines control server: one request per line, one reply per line, behind the +//! launch's token (docs/control-protocol.md). Agents and tests drive an app through it (adapted +//! from CADCraft, MIT OR Apache-2.0). What the app's requests mean is the app's [`Protocol`]; the +//! window answers them from the [`ControlRequest`]s it is sent. + +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::Effect; +use crate::auth::tokens_match; +use crate::cancel::{Cancel, Ticket}; +use crate::request::{ControlRequest, DOCUMENT_WATCH, DocumentWatch, Subscriber}; + +/// What the server needs to know of a request before it sends it to the window: how to name it +/// in messages, what it does if it outlasts its wait, and what `control.running` shows of it. +#[derive(Clone, Debug)] +pub struct Summary { + /// The request in messages ("`engine.execute` (solid.fillet)"). + pub what: String, + /// What it still does when it finishes after its wait. + pub effect: Effect, + /// The command it runs, for `control.running` (never its parameters). + pub command: Option<&'static str>, + /// The number of steps it runs, for `control.running`. + pub steps: Option, + /// It may take as long as a batch. + pub long: bool, +} + +/// What an app's requests mean to its control server. +pub trait Protocol: Send + Sync { + fn summary(&self, method: &str, params: &Value) -> Summary; + /// What the app calls its document in messages ("design"). + fn document(&self) -> &'static str; + /// The event a subscribed connection is sent when the document moves ("design.changed"). + fn change_event(&self) -> &'static str; + /// The method that answers the document's id and revision now ("document.revision"). + fn revision_method(&self) -> &'static str; + /// Where a client reads how to talk to the channel, named when its token is refused + /// ("docs/control-protocol.md"). + fn docs(&self) -> &'static str; + /// The `control.capabilities` result: what this app's channel does beyond the methods every + /// version has, for a client to find out before it relies on it (an older app answers + /// `control.capabilities` with "unknown method"). + fn capabilities(&self) -> Value; +} + +/// An app's control server: its [`Protocol`], how it wakes the window for a request, and the +/// requests every connection of its listeners has the window running (`control.running`). +pub struct Hub { + protocol: Box, + wake: Box, + /// Longest request line accepted. + max_line: usize, + waits: Waits, + watch: &'static DocumentWatch, + running: Arc, + open: AtomicUsize, +} + +impl Hub { + /// A server for `protocol`'s requests; `wake` makes the window look at its requests now. + pub fn new(protocol: impl Protocol + 'static, wake: impl Fn() + Send + Sync + 'static, max_line: usize) -> Self { + Hub { + protocol: Box::new(protocol), + wake: Box::new(wake), + max_line, + waits: Waits::default(), + watch: &DOCUMENT_WATCH, + running: Arc::new(Running::new()), + open: AtomicUsize::new(0), + } + } + + /// Other waits (tests). + pub fn with_waits(mut self, waits: Waits) -> Self { + self.waits = waits; + self + } + + /// Another document watch than the app's [`DOCUMENT_WATCH`] (tests). + pub fn with_watch(mut self, watch: &'static DocumentWatch) -> Self { + self.watch = watch; + self + } + + /// Serve `listener` with `token` on threads of their own, at most [`MAX_CONNECTIONS`] + /// connections at once. `patient`: every request may wait as long as a batch (a person decides + /// on it). A listener that is not on a loopback address is refused: the channel never goes on + /// the network. + pub fn listen(self: &Arc, listener: TcpListener, token: String, requests: Sender, patient: bool) -> std::io::Result<()> { + self.listen_stoppable(listener, token, requests, patient).map(drop) + } + + /// [`Hub::listen`], and a [`Stop`] that closes the listener and the connections it accepted + /// (a channel turned off in the app's settings). Dropping the [`Stop`] leaves it serving. + pub fn listen_stoppable( + self: &Arc, + listener: TcpListener, + token: String, + requests: Sender, + patient: bool, + ) -> std::io::Result { + let addr = listener.local_addr()?; + if !addr.ip().is_loopback() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("the control channel listens on loopback only, not {addr}"))); + } + let hub = Arc::clone(self); + let token = Arc::new(token); + let shared = Arc::new(Accepted::default()); + let (closed_tx, closed) = channel::<()>(); + let stop = Stop { addr, shared: Arc::clone(&shared), closed }; + std::thread::spawn(move || { + // Dropped when this thread ends, with the listener: `Stop::stop` waits for that. + let _closed = closed_tx; + for stream in listener.incoming().flatten() { + if shared.stopped.load(Ordering::SeqCst) { + break; + } + if hub.open.fetch_add(1, Ordering::Relaxed) >= MAX_CONNECTIONS { + hub.open.fetch_sub(1, Ordering::Relaxed); + let mut refused = stream; + let _ = send(&mut refused, &json!({"ok": false, "error": "too many connections"})); + continue; + } + let id = shared.next.fetch_add(1, Ordering::Relaxed); + if let Ok(copy) = stream.try_clone() { + shared.lock().push((id, copy)); + } + // Stopped while this one was being accepted: it is closed with the rest. + if shared.stopped.load(Ordering::SeqCst) { + shared.close_all(); + } + let (hub, tx, token, shared) = (Arc::clone(&hub), requests.clone(), Arc::clone(&token), Arc::clone(&shared)); + std::thread::spawn(move || { + hub.serve(stream, &token, tx, patient); + hub.open.fetch_sub(1, Ordering::Relaxed); + shared.lock().retain(|(i, _)| *i != id); + }); + } + // A captured value drops after the body's locals, so without this `_closed` would tell + // `Stop::stop` the port is free while the listener still holds it (#14752). + drop(listener); + }); + Ok(stop) + } +} + +/// The connections a stoppable listener accepted, so stopping it can close them too. +#[derive(Default)] +struct Accepted { + stopped: AtomicBool, + next: AtomicU64, + streams: Mutex>, +} + +impl Accepted { + fn lock(&self) -> MutexGuard<'_, Vec<(u64, TcpStream)>> { + self.streams.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn close_all(&self) { + for (_, stream) in self.lock().drain(..) { + let _ = stream.shutdown(std::net::Shutdown::Both); + } + } +} + +/// Stops a listener started with [`Hub::listen_stoppable`]. +pub struct Stop { + addr: std::net::SocketAddr, + shared: Arc, + closed: Receiver<()>, +} + +impl Stop { + /// The port the listener is on. + pub fn port(&self) -> u16 { + self.addr.port() + } + + /// Stop accepting, close the port and end every connection it accepted. Returns once the + /// port is closed (at most a couple of seconds), so the same port can be bound again at once. + pub fn stop(self) { + self.shared.stopped.store(true, Ordering::SeqCst); + // The listener waits in `accept`: a connection of our own wakes it to see the flag. + let _ = TcpStream::connect_timeout(&self.addr, Duration::from_secs(1)); + let _ = self.closed.recv_timeout(Duration::from_secs(2)); + self.shared.close_all(); + } +} + +/// Connections served at once, so a flood must not be able to spawn threads without end. +pub const MAX_CONNECTIONS: usize = 16; +/// Lines that are not JSON before the connection is closed, so garbage cannot loop forever. +const MAX_BAD_LINES: usize = 8; +/// How long a subscribed connection waits after a `design.changed` line before the next: what +/// changes meanwhile (a parameter dragged) goes out as one line, the latest (#6985). +const EVENT_GAP: Duration = Duration::from_millis(100); +/// How long a new connection has to show the token. A client connects when it has a request to +/// send, so this is plenty; one that doesn't send it loses the connection, and its seat among +/// [`MAX_CONNECTIONS`] goes to someone else. +const TOKEN_WAIT: Duration = Duration::from_secs(5); + +/// A new random token: 32 bytes from the operating system, as hex. +pub fn new_token() -> Option { + let mut bytes = [0u8; 32]; + getrandom::fill(&mut bytes).ok()?; + Some(bytes.iter().map(|b| format!("{b:02x}")).collect()) +} + +/// One line without its newline, or `None` at the end of the stream. A client that sends +/// gigabytes with no newline would otherwise make this thread allocate until the process is +/// killed, so the line is refused as soon as it passes `max`. +fn read_line(r: &mut impl BufRead, max: usize) -> std::io::Result>> { + let mut line = Vec::new(); + loop { + let (done, used) = { + let available = r.fill_buf()?; + if available.is_empty() { + return Ok((!line.is_empty()).then_some(line)); + } + match available.iter().position(|b| *b == b'\n') { + Some(i) => { + line.extend_from_slice(available.get(..i).unwrap_or(&[])); + (true, i + 1) + } + None => { + line.extend_from_slice(available); + (false, available.len()) + } + } + }; + r.consume(used); + if done { + return Ok(Some(line)); + } + if line.len() > max { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "request too large")); + } + } +} + +/// A reply line, written to the socket at once. `writeln!` with a JSON value hands the socket each +/// of its pieces apart, a system call for every string, number and comma: half a second for a +/// design's component tree when it has thousands of components. +fn send(out: &mut impl Write, reply: &Value) -> std::io::Result<()> { + let mut line = serde_json::to_vec(reply).map_err(std::io::Error::other)?; + line.push(b'\n'); + out.write_all(&line) +} + +/// The `"token"` of a request line that is a JSON object, read without building the rest of it. +/// Until a connection shows the token, a line costs only its own bytes: parsed whole, a 16 MiB +/// line of `[0,0,…]` is hundreds of MB, and 16 such connections could exhaust memory before any +/// token was checked. +fn token_of(line: &str) -> Option { + struct Token; + impl<'de> serde::de::Visitor<'de> for Token { + type Value = Option; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a request object") + } + fn visit_map>(self, mut map: A) -> Result { + let mut token = None; + while let Some(key) = map.next_key::()? { + if key == "token" { + token = Some(map.next_value::()?); + } else { + map.next_value::()?; + } + } + Ok(token) + } + } + let mut de = serde_json::Deserializer::from_str(line); + let token = serde::Deserializer::deserialize_map(&mut de, Token).ok()?; + de.end().ok()?; + token +} + +/// The reply to a connection that didn't show the token, naming the app's `docs`; the +/// connection is then closed. +fn unauthorized(docs: &str) -> Value { + json!({"ok": false, "error": format!("unauthorized: send the control token as \"token\" (see {docs})")}) +} + +/// Refuse the connection: the reply, then the end of what we send, then what the client had +/// already sent read and dropped (a little, briefly). Closing a socket with unread input resets +/// the connection, and a reset can reach the client before it has read the refusal. +fn refuse(out: &mut TcpStream, reader: &mut BufReader, docs: &str) { + use std::io::Read; + let _ = send(out, &unauthorized(docs)); + let _ = out.shutdown(std::net::Shutdown::Write); + reader.get_mut().deadline = Some(Instant::now() + Duration::from_millis(500)); + let _ = std::io::copy(&mut reader.by_ref().take(64 * 1024), &mut std::io::sink()); +} + +/// What a client sends. Until it shows the token, all of its reads share one deadline, so neither +/// blank lines nor a line sent a byte at a time buy it more time. After that it may stay idle as +/// long as it likes. +struct ClientInput { + socket: TcpStream, + deadline: Option, +} + +impl ClientInput { + /// No deadline any more: the client showed the token. + fn admit(&mut self) -> std::io::Result<()> { + self.deadline = None; + self.socket.set_read_timeout(None) + } +} + +impl std::io::Read for ClientInput { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(deadline) = self.deadline { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(std::io::ErrorKind::TimedOut.into()); + } + self.socket.set_read_timeout(Some(left))?; + } + self.socket.read(buf) + } +} + +/// How long a request waits for the app's reply: most requests, and a batch (or every request, +/// on a patient connection); and how long a new connection has to show the token. +#[derive(Clone, Copy, Debug)] +pub struct Waits { + pub short: Duration, + pub long: Duration, + pub token: Duration, +} + +impl Default for Waits { + fn default() -> Self { + Waits { short: Duration::from_secs(60), long: Duration::from_secs(3600), token: TOKEN_WAIT } + } +} + +/// A request that timed out but is still running in the app (#1618): its id and method, and where +/// its reply will land. +struct Pending { + id: Value, + method: String, + /// The request in messages ("`engine.execute` (solid.fillet)"). + what: String, + reply: Receiver, + /// Where it is in the app, to cancel it (`control.cancel`). + job: Arc, +} + +/// `control.cancel {id?}`: cancel the request this connection still waits for (the one +/// `control.pending` reports; `id`, when given, must name it). A connection can only cancel its +/// own requests. Waiting its turn, it is removed; computing off the window thread, it is stopped +/// as the app's Stop stops it. Either way nothing of it is applied and the connection is free. +/// One that runs on the window thread can't be cancelled; one that finished is answered. +fn cancel(pending: &mut Option, late: Option, id: Option<&Value>, document: &str) -> Value { + let id = id.filter(|v| !v.is_null()); + let mine = pending.as_ref().filter(|p| id.is_none_or(|id| *id == p.id)); + let Some(p) = mine else { + return match id { + None => json!({"ok": true, "result": {"cancelled": null, "note": "nothing to cancel: no request of this connection is running"}}), + Some(id) => json!({"ok": false, "error": format!( + "nothing to cancel: request {} is not running on this connection (a connection can only cancel its own requests)", + id.to_string().chars().take(64).collect::() + )}), + }; + }; + let (what, id) = (format!("request {} ({})", p.id, p.what), p.id.clone()); + let finished = |p: Pending, reply: Value| { + json!({"ok": true, "result": {"cancelled": null, "pending": p.id, "method": p.method, "done": true, "reply": reply, + "note": format!("nothing to cancel: {what} had already finished; its reply is here")}}) + }; + if let Some(reply) = late { + return pending.take().map_or(Value::Null, |p| finished(p, reply)); + } + let outcome = p.job.cancel(); + let note = match outcome { + Cancel::Removed => format!("cancelled: {what} was still waiting its turn and will not run; nothing was applied"), + Cancel::Stopped { writes_file: false } => format!("stopped: {what} was computing off the window thread; nothing was applied"), + Cancel::Stopped { writes_file: true } => { + format!("stopped: {what} was computing on a copy of the {document}, which is unchanged; its worker may still write the file") + } + Cancel::Already => format!("{what} was already cancelled or stopped; nothing of it was applied"), + Cancel::Running => { + // It may have just finished. + if let Some(reply) = pending.as_ref().and_then(|p| p.reply.try_recv().ok()) { + return pending.take().map_or(Value::Null, |p| finished(p, reply)); + } + return json!({"ok": false, "cancellable": false, "error": format!( + "cannot be cancelled: {what} already runs in the app (on its window thread, or as a file it opens) and will \ + still finish; ask `control.pending` for its result. Only a request still waiting its turn, or a change or \ + export computing off the window thread, can be cancelled" + )}); + } + }; + let method = pending.take().map(|p| p.method); + let how = match outcome { + Cancel::Removed => "removed", + Cancel::Stopped { .. } => "stopped", + _ => "already", + }; + json!({"ok": true, "result": {"cancelled": id, "method": method, "outcome": how, "note": note}}) +} + +/// Every request the app has been sent and not answered yet, from every connection, for +/// `control.running` (#3654). Each connection's own thread answers that, so it never waits behind +/// a running request. +struct Running { + list: Mutex>, + connections: AtomicU64, + keys: AtomicU64, +} + +/// What `control.running` says about a request: never its parameters beyond the command id. +struct InFlight { + key: u64, + connection: u64, + id: Value, + method: String, + command: Option<&'static str>, + steps: Option, + since: Instant, + started: Arc, +} + +impl Running { + fn new() -> Self { + Running { list: Mutex::new(Vec::new()), connections: AtomicU64::new(0), keys: AtomicU64::new(0) } + } + + fn lock(&self) -> MutexGuard<'_, Vec> { + self.list.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// List `req`, sent on `connection`, until the returned ticket is dropped; `command` and + /// `steps` as the app's [`Protocol`] summarised it. + fn add(self: &Arc, connection: u64, id: &Value, req: &ControlRequest, command: Option<&'static str>, steps: Option) -> Listing { + let key = self.keys.fetch_add(1, Ordering::Relaxed); + // Ids and methods come from the client: only short ones are repeated to the others. + let id = match id { + Value::Number(_) => id.clone(), + Value::String(s) if s.len() <= 64 => id.clone(), + _ => Value::Null, + }; + let method = req.method.chars().take(64).collect(); + let started = req.started.clone(); + self.lock().push(InFlight { key, connection, id, method, command, steps, since: Instant::now(), started }); + Listing { running: Arc::clone(self), key } + } + + /// The `control.running` result, as `asker`'s connection sees it. + fn result(&self, asker: u64) -> Value { + let rows: Vec = self + .lock() + .iter() + .map(|e| { + let mut r = json!({ + "connection": e.connection, + "id": e.id, + "method": e.method, + "running_s": (e.since.elapsed().as_secs_f64() * 10.0).round() / 10.0, + "queued": !e.started.load(Ordering::Relaxed), + }); + if let Some(c) = e.command { + r["command"] = json!(c); + } + if let Some(n) = e.steps { + r["steps"] = json!(n); + } + if e.connection == asker { + r["self"] = json!(true); + } + r + }) + .collect(); + json!({"running": rows}) + } +} + +/// A listed request; dropped when its reply arrives. +struct Listing { + running: Arc, + key: u64, +} + +impl Drop for Listing { + fn drop(&mut self) { + let key = self.key; + self.running.lock().retain(|e| e.key != key); + } +} + +/// The reply to a timed-out request, received on a thread of its own so the request stays listed +/// until the app answers, also once its connection has closed (#3654). +fn watch(reply: Receiver, ticket: Listing) -> Receiver { + let (late_tx, late_rx) = channel(); + // A small thread per timed-out request (each connection has at most one). If none can be + // made, the reply is lost and `control.pending` says the app ended the request without one. + // The reply is passed on before the request leaves the list: once `control.running` no longer + // shows it, its connection's `control.pending` has it. + let _ = std::thread::Builder::new().stack_size(64 * 1024).spawn(move || { + if let Ok(v) = reply.recv() { + let _ = late_tx.send(v); + } + drop(late_tx); + drop(ticket); + }); + late_rx +} + +/// The reply text for a request that outlasts its wait. A request that only reads changes +/// nothing when it finishes; an export or a save still writes its file, and any other request +/// still applies, as it was not cancelled. +pub fn timeout_message(what: &str, effect: Effect, document: &str) -> String { + match effect { + Effect::ReadsOnly => { + format!("timeout: {what} is still running in the app ({}); ask `control.pending` for its answer", effect.when_done(document)) + } + Effect::WritesFile => format!( + "timeout: {what} is still running in the app and will still write the file when it finishes (it was not cancelled: \ + don't send it again; the {document} does not change); ask `control.pending` for its result, or `control.cancel` to \ + cancel it" + ), + Effect::Changes => format!( + "timeout: {what} is still running in the app and will still apply when it finishes (it was not cancelled: don't send it \ + again); ask `control.pending` for its result, or `control.cancel` to cancel it" + ), + } +} + +/// The reply text for a request refused because an earlier one on the connection still runs. +fn busy_message(p: &Pending, method: &str) -> String { + let method: String = method.chars().take(128).collect(); + format!( + "busy: request {} ({}) is still running, so `{method}` was not run; ask `control.pending` for that request's result, then \ + send this one again", + p.id, p.what + ) +} + +/// A subscribed connection's socket, for its replies and its events: each line is written whole +/// and alone, so an event never lands inside a reply (#6985). +struct Lines { + socket: TcpStream, + turn: Mutex<()>, +} + +impl Lines { + fn send(&self, line: &Value) -> std::io::Result<()> { + let _turn = self.turn.lock().unwrap_or_else(PoisonError::into_inner); + send(&mut &self.socket, line) + } +} + +/// A subscribed connection's seat on the watch: it leaves exactly once, whichever thread sees +/// the connection's end first — the connection's own thread at the end of the stream, or the +/// events thread at the first write a closed client refuses. The seat used to wait for the +/// events thread alone, and that thread only leaves its wait when the design moves, so the +/// count could outlive the connection by a second or more (#8315). +struct Seat(Mutex>); + +impl Seat { + fn new(counted: Subscriber) -> Self { + Seat(Mutex::new(Some(counted))) + } + + /// Leave the watch now: the seat goes once, whoever gets here first. + fn release(&self) { + let counted = self.0.lock().unwrap_or_else(PoisonError::into_inner).take(); + drop(counted); + } +} + +impl Drop for Seat { + fn drop(&mut self) { + self.release(); + } +} + +/// A connection's `control.subscribe` (#6985): its `design.changed` lines, sent from a thread of +/// their own until the connection ends. +struct Subscription { + lines: Arc, + stop: Arc, + seat: Arc, +} + +impl Subscription { + /// Write `reply`, the window's answer to `control.subscribe` naming the design `seen`, on + /// `out`, then send `design`'s changes after it: the reply goes out before any event. `None`, + /// with nothing written, when no thread or socket was left for the events. + fn start( + watch: &'static DocumentWatch, + event: &'static str, + counted: Subscriber, + out: &TcpStream, + seen: (String, u64), + reply: &Value, + ) -> Option<(Self, std::io::Result<()>)> { + let lines = Arc::new(Lines { socket: out.try_clone().ok()?, turn: Mutex::new(()) }); + let stop = Arc::new(AtomicBool::new(false)); + let seat = Arc::new(Seat::new(counted)); + let first = lines.turn.lock().unwrap_or_else(PoisonError::into_inner); + let (l, s, watch_seat) = (lines.clone(), stop.clone(), Arc::clone(&seat)); + // The connection's seat on the watch rides the events thread: it leaves when the thread + // does, so the next failed write to a closed client retires it even while the connection's + // own thread is slow to see the end of the stream (#7226). The connection's own thread + // leaves it too, so a closed connection stops counting at once (#8315). + std::thread::Builder::new().stack_size(64 * 1024).spawn(move || send_events(watch, event, &l, &s, seen, watch_seat)).ok()?; + let sent = send(&mut &lines.socket, reply); + drop(first); + Some((Subscription { lines, stop, seat }, sent)) + } +} + +/// The design a `control.subscribe` reply names. +fn document_of(reply: &Value) -> Option<(String, u64)> { + let result = reply.get("result")?; + Some((result.get("document_id")?.as_str()?.to_string(), result.get("revision")?.as_u64()?)) +} + +impl Drop for Subscription { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + // The seat on the watch goes now: the connection is over, and nothing counts it any more + // (#8315). + self.seat.release(); + // The socket closes now, not when the thread next looks; a line blocked on a client that + // stopped reading ends too. + let _ = self.lines.socket.shutdown(std::net::Shutdown::Both); + } +} + +/// The `design.changed` lines of a subscribed connection, each time the design differs from the +/// last one it was told of (`seen`), until `stop`. Holds the connection's seat on the watch +/// (`seat`), which it leaves when the thread ends: on `stop`, or on the first write a closed +/// client refuses (#7226). +fn send_events(watch: &DocumentWatch, event: &str, lines: &Lines, stop: &AtomicBool, mut seen: (String, u64), seat: Arc) { + while !stop.load(Ordering::Relaxed) { + let Some(now) = watch.wait(&seen, stop, Duration::from_secs(1)).filter(|now| *now != seen) else { continue }; + if lines.send(&json!({"event": event, "params": {"document_id": now.0, "revision": now.1}})).is_err() { + break; + } + seen = now; + std::thread::sleep(EVENT_GAP); + } + // The client is gone: it is not counted on the watch any more, whatever the connection's own + // thread still has to notice (#8315). + seat.release(); +} + +/// Write `reply` on the connection: through its subscription once it has one. +fn answer(out: &mut TcpStream, events: Option<&Subscription>, reply: &Value) -> std::io::Result<()> { + match events { + Some(e) => e.lines.send(reply), + None => send(out, reply), + } +} + +impl Hub { + /// Serve one connection until it closes: the token first, then one request per line, each + /// sent to the window on `requests` and its reply written back. + pub fn serve(&self, stream: TcpStream, token: &str, requests: Sender, patient: bool) { + let (waits, running, documents, tx) = (self.waits, &self.running, self.watch, requests); + let connection = running.connections.fetch_add(1, Ordering::Relaxed) + 1; + let Ok(read) = stream.try_clone() else { return }; + let mut out = stream; + let mut reader = BufReader::new(ClientInput { socket: read, deadline: Some(Instant::now() + waits.token) }); + let mut bad = 0usize; + let mut authenticated = false; + let mut pending: Option = None; + let mut events: Option = None; + loop { + // A first `control.subscribe` on its way to the window (#6985). + let mut joining = None; + let line = match read_line(&mut reader, self.max_line) { + Ok(Some(l)) => String::from_utf8_lossy(&l).into_owned(), + Ok(None) => break, + // Closed without a word: a client that connected early and only now sends its first + // request would read a line here as the reply to it. It finds the connection closed + // and opens another. + Err(e) if !authenticated && matches!(e.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) => break, + Err(e) => { + let _ = answer(&mut out, events.as_ref(), &json!({"ok": false, "error": e.to_string()})); + break; + } + }; + if line.trim().is_empty() { + continue; + } + // The first request must carry the token; later ones on the same connection may. + if !authenticated { + match token_of(&line) { + Some(t) if tokens_match(&t, token) => { + if reader.get_mut().admit().is_err() { + break; + } + authenticated = true; + } + _ => { + refuse(&mut out, &mut reader, self.protocol.docs()); + break; + } + } + } + let reply = match serde_json::from_str::(&line) { + Ok(msg) => { + let id = msg.get("id").cloned().unwrap_or(Value::Null); + let method = msg.get("method").and_then(Value::as_str).unwrap_or("").to_string(); + let params = msg.get("params").cloned().unwrap_or(json!({})); + // A request that timed out may have finished since (#1618). `control.running` and + // `control.capabilities` only look, so they leave that reply where it is. + let late = pending.as_ref().filter(|_| !matches!(method.as_str(), "control.running" | "control.capabilities")).and_then(|p| { + match p.reply.try_recv() { + Ok(reply) => Some(reply), + Err(TryRecvError::Disconnected) => Some(json!({"ok": false, "error": "the app ended the request without a reply"})), + Err(TryRecvError::Empty) => None, + } + }); + let mut r = if method == "control.running" { + json!({"ok": true, "result": running.result(connection)}) + } else if method == "control.capabilities" { + json!({"ok": true, "result": self.protocol.capabilities()}) + } else if method == "control.cancel" { + let r = cancel(&mut pending, late, params.get("id"), self.protocol.document()); + (self.wake)(); + r + } else if method == "control.pending" { + match (pending.take(), late) { + (None, _) => json!({"ok": true, "result": {"pending": null}}), + (Some(p), Some(reply)) => { + json!({"ok": true, "result": {"pending": p.id, "method": p.method, "done": true, "reply": reply}}) + } + (Some(p), None) => { + let r = json!({"ok": true, "result": {"pending": p.id, "method": p.method, "done": false}}); + pending = Some(p); + r + } + } + } else if let Some(p) = pending.as_ref().filter(|_| late.is_none()) { + // Nothing more is queued behind it, to run unseen when it ends. + json!({"ok": false, "busy": true, "error": busy_message(p, &method)}) + } else if method == "control.subscribe" && events.is_some() { + if late.is_some() { + // As below: finished, and the client moved on without asking. + pending = None; + } + // Its events already say how the design moves; a second answer could only + // arrive behind a newer one. + { + let (event, revision, document) = + (self.protocol.change_event(), self.protocol.revision_method(), self.protocol.document()); + json!({"ok": false, "error": format!("already subscribed: this connection is sent {event} lines; ask {revision} for the {document} now")}) + } + } else { + if late.is_some() { + // Finished, and the client moved on without asking: it is dropped. + pending = None; + } + // A batch replaces many short requests: the whole of it may take long (#743). + let Summary { what, effect, command, steps, long } = self.protocol.summary(&method, ¶ms); + let wait = if patient || long { waits.long } else { waits.short }; + // Counted before the window answers, so it publishes every change from its + // answer on. + joining = (method == "control.subscribe").then(|| documents.join()); + let (req, rrx) = ControlRequest::new(method.clone(), params); + let job = Arc::clone(&req.ticket); + let ticket = running.add(connection, &id, &req, command, steps); + if tx.send(req).is_err() { + break; + } + (self.wake)(); + match rrx.recv_timeout(wait) { + Ok(r) => r, + Err(_) => { + let r = json!({"ok": false, "running": true, "error": timeout_message(&what, effect, self.protocol.document())}); + pending = Some(Pending { id: id.clone(), method, what, reply: watch(rrx, ticket), job }); + r + } + } + }; + if let Some(o) = r.as_object_mut() { + o.insert("id".into(), id); + } + r + } + Err(e) => { + bad += 1; + json!({"ok": false, "error": format!("bad JSON: {e}")}) + } + }; + // Subscribed once the window named the design (not when it refused or timed out). + let sent = match joining.zip(document_of(&reply)) { + Some((counted, seen)) => match Subscription::start(documents, self.protocol.change_event(), counted, &out, seen, &reply) { + Some((subscription, sent)) => { + events = Some(subscription); + sent + } + None => send( + &mut out, + &json!({"id": reply.get("id"), "ok": false, "error": "not subscribed: no thread or socket left for the events"}), + ), + }, + None => answer(&mut out, events.as_ref(), &reply), + }; + if sent.is_err() { + break; + } + if bad >= MAX_BAD_LINES { + break; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The longest request line, as an app might accept. + const MAX_LINE: usize = 16 << 20; + + /// An app whose requests all may change its document and take the short wait. + struct Plain; + + impl Protocol for Plain { + fn summary(&self, method: &str, _: &Value) -> Summary { + Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false } + } + fn document(&self) -> &'static str { + "design" + } + fn change_event(&self) -> &'static str { + "design.changed" + } + fn revision_method(&self) -> &'static str { + "document.revision" + } + fn docs(&self) -> &'static str { + "docs/control-protocol.md" + } + fn capabilities(&self) -> Value { + json!({}) + } + } + + /// A server for [`Plain`] with these waits and this document watch. + fn hub(waits: Waits, watch: &'static DocumentWatch) -> Arc { + Arc::new(Hub::new(Plain, || {}, MAX_LINE).with_waits(waits).with_watch(watch)) + } + + /// The channel never goes on the network: a listener on any address but loopback is refused, + /// and nothing is served on it. + #[test] + fn a_listener_off_loopback_is_refused() { + let (tx, _rx) = channel::(); + let open = TcpListener::bind("0.0.0.0:0").unwrap(); + let e = hub(Waits::default(), &DOCUMENT_WATCH).listen(open, "0123456789abcdef0123456789abcdef".into(), tx.clone(), false).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput, "{e}"); + assert!(e.to_string().contains("loopback only"), "{e}"); + let local = TcpListener::bind("127.0.0.1:0").unwrap(); + assert!(hub(Waits::default(), &DOCUMENT_WATCH).listen(local, "0123456789abcdef0123456789abcdef".into(), tx, false).is_ok()); + } + + /// A connection that doesn't show the token in time is closed without a word, however it fills + /// the time. One that does keeps working when it idles for longer than that. + #[test] + fn the_token_has_a_time_limit_and_an_authenticated_client_may_idle() { + use std::io::Read; + let token = "0123456789abcdef0123456789abcdef"; + let limit = Duration::from_millis(500); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + std::thread::spawn(move || { + for req in rx { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + }); + let waits = Waits { short: Duration::from_secs(10), long: Duration::from_secs(10), token: limit }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let open = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + s + }; + // Sends `drip` every 50 ms until the server closes; what it read, and when it closed. + let closed_after = |drip: &[u8]| { + let mut s = open(); + let start = Instant::now(); + s.set_read_timeout(Some(Duration::from_millis(50))).unwrap(); + let mut got = Vec::new(); + loop { + let mut buf = [0u8; 256]; + match s.read(&mut buf) { + Ok(0) => break, + Ok(n) => got.extend_from_slice(buf.get(..n).unwrap()), + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => { + assert!(start.elapsed() < Duration::from_secs(20), "still open"); + let _ = s.write_all(drip); + } + Err(_) => break, + } + } + (got, start.elapsed()) + }; + for drip in [&b""[..], b"\n", b"\r\n \n", b" ", b"{"] { + let (got, took) = closed_after(drip); + assert!(got.is_empty(), "{drip:?} was answered: {}", String::from_utf8_lossy(&got)); + assert!(took >= limit.saturating_sub(Duration::from_millis(50)), "{drip:?} closed after {took:?}"); + } + + let mut s = open(); + let mut lines = BufReader::new(s.try_clone().unwrap()); + let mut ask = |line: Value| -> Value { + writeln!(s, "{line}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + assert_eq!(ask(json!({"id": 1, "method": "ui.view", "token": token}))["ok"], true); + std::thread::sleep(limit * 3); + assert_eq!(ask(json!({"id": 2, "method": "ui.view"}))["ok"], true, "idle for longer than the limit"); + + // A wrong token is still answered, at once. + let mut wrong = open(); + writeln!(wrong, "{}", json!({"id": 1, "method": "ui.view", "token": "nope"})).unwrap(); + let mut l = String::new(); + BufReader::new(wrong).read_line(&mut l).unwrap(); + assert!(l.contains("unauthorized"), "{l}"); + } + + /// #1989: `control.cancel` cancels the connection's own outstanding request. One still + /// waiting its turn is removed and never runs, and the connection is free at once; one that + /// already runs in the app can't be, and says so; another connection's request is refused; + /// with nothing outstanding there is nothing to cancel. + #[test] + fn control_cancel_cancels_only_this_connections_request() { + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + let (gate_tx, gate_rx) = channel::<()>(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + // A stand-in for the window thread: `queued` waits behind the others (never taken until + // `slow` is done), `slow` runs until the gate opens. + std::thread::spawn(move || { + let mut held = Vec::new(); + for req in rx { + if req.method == "queued" { + held.push(req); + continue; + } + if !req.ticket.take() { + continue; + } + seen.lock().unwrap().push(req.method.clone()); + if req.method == "slow" { + let _ = gate_rx.recv(); + } + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + if req.method == "slow" { + for req in held.drain(..) { + if req.ticket.take() { + seen.lock().unwrap().push(req.method.clone()); + } + } + } + } + }); + let waits = Waits { short: Duration::from_millis(100), long: Duration::from_millis(100), token: TOKEN_WAIT }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let connect = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let out = s.try_clone().unwrap(); + (out, BufReader::new(s)) + }; + let ask = |(out, lines): &mut (TcpStream, BufReader), v: Value| -> Value { + writeln!(out, "{v}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let mut a = connect(); + let mut b = connect(); + let r = ask(&mut b, json!({"id": 1, "method": "control.cancel", "token": token})); + assert!(r["ok"] == true && r["result"]["cancelled"].is_null(), "nothing to cancel: {r}"); + + // Waiting its turn: removed, never run, and the connection is free. + let r = ask(&mut a, json!({"id": 1, "method": "queued", "token": token})); + assert_eq!(r["running"], true, "{r}"); + let r = ask(&mut a, json!({"id": 2, "method": "control.cancel", "params": {"id": 1}})); + assert_eq!((&r["ok"], &r["result"]["cancelled"], &r["result"]["outcome"]), (&json!(true), &json!(1), &json!("removed")), "{r}"); + assert!(r["result"]["note"].as_str().is_some_and(|n| n.contains("will not run")), "{r}"); + let r = ask(&mut a, json!({"id": 3, "method": "control.pending"})); + assert_eq!(r["result"]["pending"], Value::Null, "{r}"); + + // Already running in the app: it can't be, and it still answers. + let r = ask(&mut a, json!({"id": 4, "method": "slow"})); + assert_eq!(r["running"], true, "{r}"); + let r = ask(&mut b, json!({"id": 2, "method": "control.cancel", "params": {"id": 4}})); + assert!(r["ok"] == false && r["error"].as_str().is_some_and(|e| e.contains("its own requests")), "another connection's: {r}"); + let r = ask(&mut a, json!({"id": 5, "method": "control.cancel", "params": {"id": 99}})); + assert!(r["ok"] == false && r["error"].as_str().is_some_and(|e| e.starts_with("nothing to cancel")), "{r}"); + let r = ask(&mut a, json!({"id": 6, "method": "control.cancel"})); + assert!(r["ok"] == false && r["cancellable"] == false, "{r}"); + assert!(r["error"].as_str().is_some_and(|e| e.starts_with("cannot be cancelled") && e.contains("control.pending")), "{r}"); + gate_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let r = loop { + let r = ask(&mut a, json!({"id": 7, "method": "control.pending"})); + if r["result"]["done"] == true || Instant::now() > deadline { + break r; + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(r["result"]["reply"]["result"], "slow", "{r}"); + let r = ask(&mut a, json!({"id": 8, "method": "control.cancel"})); + assert!(r["ok"] == true && r["result"]["cancelled"].is_null(), "{r}"); + assert_eq!(*ran.lock().unwrap(), ["slow"], "the cancelled request never ran"); + } + + /// #1618: a request that outlasts its wait says it is still running; the connection keeps its + /// reply for `control.pending`, and refuses other requests as `busy` until it lands, so none + /// is queued to run unseen behind it. + #[test] + fn a_timed_out_request_can_be_asked_for_later() { + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The slow request is held until the test has seen it time out, so the test does not + // race the clock on a loaded machine. + let (gate_tx, gate_rx) = channel::<()>(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + std::thread::spawn(move || { + for req in rx { + seen.lock().unwrap().push(req.method.clone()); + if req.method == "slow" { + let _ = gate_rx.recv(); + } + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + }); + let waits = Waits { short: Duration::from_millis(100), long: Duration::from_millis(100), token: TOKEN_WAIT }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + if let Some(stream) = listener.incoming().flatten().next() { + hub.serve(stream, token, tx, false); + } + }); + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut out = s.try_clone().unwrap(); + let mut lines = BufReader::new(s); + let mut ask = |v: Value| -> Value { + writeln!(out, "{v}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let r = ask(json!({"id": 1, "method": "slow", "token": token})); + assert!(r["ok"] == false && r["running"] == true && r["error"].as_str().is_some_and(|e| e.starts_with("timeout")), "{r}"); + let r = ask(json!({"id": 2, "method": "ui.view"})); + assert!(r["busy"] == true && r["id"] == 2, "{r}"); + let r = ask(json!({"id": 3, "method": "control.pending"})); + assert_eq!((&r["result"]["pending"], &r["result"]["done"]), (&json!(1), &json!(false)), "{r}"); + gate_tx.send(()).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let r = loop { + let r = ask(json!({"id": 4, "method": "control.pending"})); + if r["result"]["done"] == true || std::time::Instant::now() > deadline { + break r; + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!( + (&r["result"]["pending"], &r["result"]["done"], &r["result"]["reply"]["result"]), + (&json!(1), &json!(true), &json!("slow")), + "{r}" + ); + let r = ask(json!({"id": 5, "method": "control.pending"})); + assert_eq!(r["result"]["pending"], Value::Null, "{r}"); + let r = ask(json!({"id": 6, "method": "ui.view"})); + assert_eq!((&r["ok"], &r["result"]), (&json!(true), &json!("ui.view")), "{r}"); + assert_eq!(*ran.lock().unwrap(), ["slow", "ui.view"], "the busy request never reached the app"); + } + + /// #6985: a connection that sends `control.subscribe` gets the design in the reply, then a + /// `design.changed` line each time the window publishes another: never before the reply, + /// none for the same design, the latest after a burst, each line whole between its replies. + /// A second subscribe is refused without reaching the window. A connection that did not + /// subscribe gets no events, and one that closed is no longer counted. + #[test] + fn subscribed_connections_are_told_of_design_changes() { + static WATCH: DocumentWatch = DocumentWatch::new(); + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The window: it answers `control.subscribe` as the app does, publishing the design first; + // then the design changes at once, before the reply may have been written. + std::thread::spawn(move || { + for req in rx { + if req.method == "control.subscribe" { + WATCH.publish("d-1", 3); + let _ = req.reply.send(json!({"ok": true, "result": {"document_id": "d-1", "revision": 3}})); + WATCH.publish("d-1", 4); + } else { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + } + }); + let hub = hub(Waits::default(), &WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let open = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + (s.try_clone().unwrap(), BufReader::new(s)) + }; + let next = |lines: &mut BufReader| -> Value { + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let changed = |id: &str, revision: u64| json!({"event": "design.changed", "params": {"document_id": id, "revision": revision}}); + + let (mut quiet_out, mut quiet) = open(); + writeln!(quiet_out, "{}", json!({"id": 1, "method": "ui.view", "token": token})).unwrap(); + assert_eq!(next(&mut quiet)["id"], 1); + + let (mut out, mut lines) = open(); + writeln!(out, "{}", json!({"id": 1, "method": "control.subscribe", "token": token})).unwrap(); + assert_eq!(next(&mut lines), json!({"id": 1, "ok": true, "result": {"document_id": "d-1", "revision": 3}})); + assert_eq!(next(&mut lines), changed("d-1", 4), "the change after the window's answer comes after the reply"); + assert!(WATCH.subscribed()); + // The design it was told of is no news; another revision is. + WATCH.publish("d-1", 4); + WATCH.publish("d-1", 5); + assert_eq!(next(&mut lines), changed("d-1", 5)); + // A burst ends with its latest. + for revision in 6..=10 { + WATCH.publish("d-1", revision); + } + let mut last = 5; + while last < 10 { + let e = next(&mut lines); + let revision = e["params"]["revision"].as_u64().unwrap(); + assert!(e["event"] == "design.changed" && revision > last, "{e} after {last}"); + last = revision; + } + // Another tab: another id, whatever its revision. + WATCH.publish("d-2", 2); + assert_eq!(next(&mut lines), changed("d-2", 2)); + // Subscribed already: refused here (the window would have published d-1 again). + writeln!(out, "{}", json!({"id": 2, "method": "control.subscribe"})).unwrap(); + let r = next(&mut lines); + assert!(r["id"] == 2 && r["ok"] == false && r["error"].as_str().is_some_and(|e| e.starts_with("already subscribed")), "{r}"); + // Replies still come, whole, and nothing else was sent meanwhile. + writeln!(out, "{}", json!({"id": 3, "method": "ui.view"})).unwrap(); + assert_eq!(next(&mut lines), json!({"id": 3, "ok": true, "result": "ui.view"})); + writeln!(quiet_out, "{}", json!({"id": 2, "method": "ui.view"})).unwrap(); + assert_eq!(next(&mut quiet), json!({"id": 2, "ok": true, "result": "ui.view"}), "no events without subscribing"); + // Closed, it is no longer counted. Each publish asks the events thread for one more line, + // and its write to the closed connection is what retires the seat; that keeps the wait off + // the connection's own thread, which anything else this process is doing may hold up past + // the deadline (#7226). + drop((out, lines)); + let mut revision = 11; + let deadline = Instant::now() + Duration::from_secs(10); + while WATCH.subscribed() && Instant::now() < deadline { + WATCH.publish("d-1", revision); + revision += 1; + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!WATCH.subscribed()); + } + + /// #8315: a closed connection leaves the watch at once. The seat of a subscribed connection + /// goes when the connection's own thread sees the end of the stream, not only when the + /// events thread notices: that thread waits for the design to move, so a connection that + /// closed while the design stood still kept the watch counting it for a second or more. + #[test] + fn a_closed_connection_leaves_the_watch_at_once() { + static WATCH: DocumentWatch = DocumentWatch::new(); + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The window names the design once and then leaves it alone: nothing moves it, so the + // events thread has nothing to write and only the end of the connection can retire it. + std::thread::spawn(move || { + for req in rx { + if req.method == "control.subscribe" { + WATCH.publish("d-1", 3); + let _ = req.reply.send(json!({"ok": true, "result": {"document_id": "d-1", "revision": 3}})); + } else { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + } + }); + let hub = hub(Waits::default(), &WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut out = s.try_clone().unwrap(); + let mut lines = BufReader::new(s); + writeln!(out, "{}", json!({"id": 1, "method": "control.subscribe", "token": token})).unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + let reply: Value = serde_json::from_str(&l).unwrap(); + assert!(reply["ok"] == true, "{reply}"); + assert!(WATCH.subscribed(), "the connection is subscribed"); + // Let the events thread settle into its wait: the design stands still, so it has nothing + // to write, and only the end of the connection can retire it. + std::thread::sleep(Duration::from_millis(200)); + // The client goes away without asking anything else. + drop((out, lines)); + let deadline = Instant::now() + Duration::from_millis(400); + while WATCH.subscribed() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!WATCH.subscribed(), "a closed connection is not counted on the watch"); + } + + /// The token is read from a request object without building the rest of it: other fields, + /// however large or nested, are only skipped. Anything but an object with a string token is + /// no token. + #[test] + fn token_is_read_without_parsing_the_request() { + assert_eq!(token_of(r#"{"id":1,"token":"abc","params":{"a":[1,[2,{"b":null}]]}}"#).as_deref(), Some("abc")); + let flood = format!(r#"{{"params":[{}0],"token":"t"}}"#, "0,".repeat(1 << 20)); + assert_eq!(token_of(&flood).as_deref(), Some("t")); + for no in [r#"["abc"]"#, r#"{"token":5}"#, r#"{"token":"a"} x"#, r#"{"id":1}"#, "GET / HTTP/1.1", r#""token""#] { + assert_eq!(token_of(no), None, "{no}"); + } + } + + /// #205: a connection runs nothing until a request carries the launch's token; a wrong or + /// missing token, or garbage, is refused and the connection closed. Once shown, the token + /// need not be repeated on that connection. + #[test] + fn requests_run_only_after_the_token() { + use std::io::Read; + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + std::thread::spawn(move || { + for req in rx { + seen.lock().unwrap().push(req.method.clone()); + let _ = req.reply.send(json!({"ok": true, "result": null})); + } + }); + let hub = hub(Waits::default(), &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let session = |lines: &[String]| { + let mut s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + for l in lines { + let _ = writeln!(s, "{l}"); + } + let _ = s.shutdown(std::net::Shutdown::Write); + let mut text = String::new(); + let _ = s.read_to_string(&mut text); + text.lines().map(|l| serde_json::from_str::(l).unwrap()).collect::>() + }; + for first in [ + json!({"id": 1, "method": "app.quit", "params": {"force": true}}).to_string(), + json!({"id": 1, "method": "app.quit", "token": "0123456789abcdef0123456789abcdee"}).to_string(), + json!({"id": 1, "method": "app.quit", "token": "0123"}).to_string(), + json!({"id": 1, "method": "app.quit", "token": 7}).to_string(), + "not json".to_string(), + ] { + let replies = session(&[first.clone(), json!({"id": 2, "method": "app.quit", "token": token}).to_string()]); + assert_eq!(replies.len(), 1, "{first}: closed after refusing: {replies:?}"); + assert!(replies[0]["error"].as_str().is_some_and(|e| e.starts_with("unauthorized")), "{first}: {replies:?}"); + } + assert!(ran.lock().unwrap().is_empty(), "nothing ran without the token"); + let replies = session(&[ + json!({"id": 1, "method": "ui.inspect", "token": token}).to_string(), + json!({"id": 2, "method": "engine.commands"}).to_string(), + ]); + assert_eq!(replies.len(), 2, "{replies:?}"); + assert!(replies.iter().all(|r| r["ok"] == true), "{replies:?}"); + assert_eq!(*ran.lock().unwrap(), ["ui.inspect", "engine.commands"]); + } + + /// A request longer than the cap is refused instead of allocated. + #[test] + fn lines_past_the_cap_are_refused() { + let flood = vec![b'A'; MAX_LINE + 2]; + let mut r = BufReader::new(flood.as_slice()); + let e = read_line(&mut r, MAX_LINE).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert!(e.to_string().contains("request too large"), "{e}"); + } + + /// Ordinary framing: lines split on newlines, blank lines kept, no trailing newline needed. + #[test] + fn lines_split_on_newlines() { + let text = b"{\"a\":1}\n\n{\"b\":2}\nlast".to_vec(); + let mut r = BufReader::new(text.as_slice()); + let mut got = Vec::new(); + while let Some(line) = read_line(&mut r, MAX_LINE).unwrap() { + got.push(String::from_utf8(line).unwrap()); + } + assert_eq!(got, vec!["{\"a\":1}", "", "{\"b\":2}", "last"]); + assert!(read_line(&mut r, MAX_LINE).unwrap().is_none()); + } + + /// A line that spans several reads of the underlying stream still arrives whole. + #[test] + fn lines_longer_than_one_buffer_arrive_whole() { + let long = "x".repeat(200_000); + let text = format!("{long}\n{long}\n").into_bytes(); + let mut r = BufReader::with_capacity(64, text.as_slice()); + assert_eq!(read_line(&mut r, MAX_LINE).unwrap().unwrap().len(), long.len()); + assert_eq!(read_line(&mut r, MAX_LINE).unwrap().unwrap().len(), long.len()); + assert!(read_line(&mut r, MAX_LINE).unwrap().is_none()); + // The same line against a smaller cap is refused. + let mut r = BufReader::with_capacity(64, text.as_slice()); + assert!(read_line(&mut r, 1000).is_err()); + } + + /// A large reply reaches the socket in one write, not one per JSON piece (a component tree of + /// thousands of components took half a second to send), and reads back as it was. + #[test] + fn a_large_reply_is_written_at_once() { + struct Counted { + writes: usize, + bytes: Vec, + } + impl Write for Counted { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.writes += 1; + self.bytes.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let components: Vec = (0..2000).map(|i| json!({"id": i, "name": format!("Part {i}"), "bodies": ["a", "b"], "parent": 0})).collect(); + let reply = json!({"ok": true, "id": 7, "result": {"components": components}}); + let mut out = Counted { writes: 0, bytes: Vec::new() }; + send(&mut out, &reply).unwrap(); + assert_eq!(out.writes, 1); + assert_eq!(out.bytes.last(), Some(&b'\n')); + assert_eq!(serde_json::from_slice::(&out.bytes).unwrap(), reply); + } + + /// A stopped listener frees its port at once and ends the connections it had, so a channel + /// turned off in the settings leaves nothing listening and can be turned on again on the + /// same port. + #[test] + fn a_stopped_listener_closes_its_port_and_its_connections() { + use std::io::Read; + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let (tx, _rx) = channel(); + let stop = hub(Waits::default(), &DOCUMENT_WATCH).listen_stoppable(listener, "0123456789abcdef0123456789abcdef".into(), tx, false).unwrap(); + assert_eq!(stop.port(), port); + let mut client = TcpStream::connect(("127.0.0.1", port)).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(4))).unwrap(); + // Wait until the server holds the connection, so stopping has one to close. + let held = Instant::now(); + while stop.shared.lock().is_empty() && held.elapsed() < Duration::from_secs(4) { + std::thread::sleep(Duration::from_millis(5)); + } + let started = Instant::now(); + stop.stop(); + let mut byte = [0u8; 1]; + match client.read(&mut byte) { + Ok(n) => assert_eq!(n, 0, "the connection ends"), + Err(e) => { + assert!(!matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut), "the connection ends, not waits: {e}") + } + } + assert!(started.elapsed() < Duration::from_secs(3), "before the token wait runs out"); + // The listener thread closes its socket when it notices the stop; on a loaded machine that can + // take a moment after `stop()` returns. Wait for the port to be free, not for one instant. + let freed = Instant::now(); + let mut bound = TcpListener::bind(("127.0.0.1", port)); + while bound.is_err() && freed.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(10)); + bound = TcpListener::bind(("127.0.0.1", port)); + } + assert!(bound.is_ok(), "the port is free again: {:?}", bound.err()); + } +} From bea91b1861eea3889bec9d26338076f2ff31c4ac Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:55:38 +0100 Subject: [PATCH 23/27] raw: charge each DNG chunk's own output allocation against the aggregate budget (craft-libs #778) (#1884) A lossy JPEG smaller than its tile is now budgeted by its frame, but decoding still fills the whole tile, so the chunk's w x h x spp output is charged as well (and refused over MAX_SAMPLES). Regression test: a 256x256 JPEG padded with comments in a 16384x16384 tile is TooLarge instead of allocating ~1.5 GiB. --- crates/raw/src/dng_lines.rs | 8 +++++++- crates/raw/tests/it/dng_lines.rs | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/raw/src/dng_lines.rs b/crates/raw/src/dng_lines.rs index 83877492e..8da1f45d8 100644 --- a/crates/raw/src/dng_lines.rs +++ b/crates/raw/src/dng_lines.rs @@ -342,7 +342,13 @@ pub fn read(t: &Tiff, ifd: &Ifd) -> Result { for c in &chunks { let n = c.w.checked_mul(c.h).and_then(|v| v.checked_mul(spp)).ok_or(Error::TooLarge)?; let budget = decode_budget(c, &l).filter(|&b| b <= MAX_SAMPLES).ok_or(Error::TooLarge)?; - decoded_samples = decoded_samples.checked_add(budget).ok_or(Error::TooLarge)?; + // the chunk's own `w` × `h` × `spp` output is allocated whatever its coded frame declares + // (a lossy JPEG smaller than its tile still fills a whole tile), so it is charged too + let alloc = budget.max(n); + if alloc > MAX_SAMPLES { + return Err(Error::TooLarge); + } + decoded_samples = decoded_samples.checked_add(alloc).ok_or(Error::TooLarge)?; if decoded_samples > MAX_DECODED_TILE_SAMPLES { return Err(Error::TooLarge); } diff --git a/crates/raw/tests/it/dng_lines.rs b/crates/raw/tests/it/dng_lines.rs index 1cf266d67..dd10c763c 100644 --- a/crates/raw/tests/it/dng_lines.rs +++ b/crates/raw/tests/it/dng_lines.rs @@ -277,3 +277,25 @@ fn short_lossy_jpeg_strips_are_charged_their_own_rows() { let band = w * rows * 3; assert_eq!(&img.data[..band], &img.data[img.data.len() - band..]); } + +#[test] +fn a_small_lossy_jpeg_in_a_huge_tile_is_charged_the_whole_tile() { + // a 256 x 256 RGB image in one 16384 x 16384 tile whose JPEG is only 256 x 256 (padded with + // comments to pass the bytes-per-sample guard): the JPEG's frame is small, but decoding fills + // the whole tile (about 1.5 GiB of samples), so it is refused before anything is allocated + let (w, tile) = (256usize, 16384usize); + let px: Vec = (0..w * w).flat_map(|i| [(i % 251) as u8, 90, 160]).collect(); + let mut jpeg = Vec::new(); + jpeg_encoder::Encoder::new(&mut jpeg, 90).encode(&px, w as u16, w as u16, jpeg_encoder::ColorType::Rgb).unwrap(); + let mut padded = jpeg[..2].to_vec(); + let need = tile * tile * 3 / 768 + 1; + while padded.len() < need { + padded.extend_from_slice(&[0xFF, 0xFE, 0xFF, 0xFF]); + padded.extend(std::iter::repeat_n(0u8, 0xFFFF - 2)); + } + padded.extend_from_slice(&jpeg[2..]); + let mut raw = base_ifd(w, w, 8, 3, false); + raw.set(t::COMPRESSION, Value::Short(vec![34892])); + raw.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles: vec![padded] }); + assert_eq!(lines(&dng(raw, ByteOrder::Little)).unwrap_err(), Error::TooLarge); +} From b774e4d1680a69ab0dc17bc437db93fe530cce36 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 16 Nov 2026 09:55:44 -0500 Subject: [PATCH 24/27] psd: add bounded PackBits recovery decoding for app cutovers (#1885) * psd: expose bounded PackBits recovery row decoding * Clarify recovery row budget and partial-error contract --- crates/psd/src/compression.rs | 32 +++++++++++++++++++++++++---- crates/psd/src/compression/tests.rs | 22 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/psd/src/compression.rs b/crates/psd/src/compression.rs index b41936a04..76891a421 100644 --- a/crates/psd/src/compression.rs +++ b/crates/psd/src/compression.rs @@ -206,10 +206,34 @@ pub mod packbits { /// `-128` no-op bytes (which are always permitted). Errors, rather than /// overflowing, when `out.len() + expected` does not fit in `usize`. pub(crate) fn decode_into(src: &[u8], expected: usize, out: &mut Vec) -> Result<()> { + decode_row(src, expected, out, false) + } + + /// Appends a recovery row: complete oversized packets are clipped and a short + /// row is zero-padded. Truncated packet payloads still fail; trailing packets + /// after the row is full are ignored. Strict decoding remains the default. + /// + /// Appended row bytes are bounded by `max_row_bytes` and the PSD decoded-size + /// limit; callers enforce their aggregate image budget. Allocation is fallible. + /// Existing output is retained; an error may leave a partially appended row. + pub fn decode_padded_into(src: &[u8], expected: usize, max_row_bytes: usize, out: &mut Vec) -> Result<()> { + if expected > max_row_bytes || expected as u64 > super::MAX_DECODED_BYTES { + return Err(PsdError::LimitExceeded("PackBits recovery row exceeds limit")); + } + out.len().checked_add(expected).ok_or(PsdError::LimitExceeded("PackBits output length overflows usize"))?; + out.try_reserve_exact(expected).map_err(|_| PsdError::OutOfMemory("PackBits recovery row allocation failed"))?; + decode_row(src, expected, out, true) + } + + fn decode_row(src: &[u8], expected: usize, out: &mut Vec, padded: bool) -> Result<()> { let target = out.len().checked_add(expected).ok_or(PsdError::LimitExceeded("PackBits output length overflows usize"))?; let mut i = 0; while out.len() < target { let Some(&h) = src.get(i) else { + if padded { + out.resize(target, 0); + break; + } return Err(PsdError::Decompress("PackBits row ended early".into())); }; i += 1; @@ -219,10 +243,10 @@ pub mod packbits { let Some(lit) = src.get(i..i + len) else { return Err(PsdError::Decompress("PackBits literal truncated".into())); }; - if len > target - out.len() { + if !padded && len > target - out.len() { return Err(PsdError::Decompress("PackBits literal overflows row".into())); } - out.extend_from_slice(lit); + out.extend_from_slice(&lit[..len.min(target - out.len())]); i += len; } else if h != -128 { let len = 1 - h as isize; @@ -231,10 +255,10 @@ pub mod packbits { return Err(PsdError::Decompress("PackBits run truncated".into())); }; i += 1; - if len > target - out.len() { + if !padded && len > target - out.len() { return Err(PsdError::Decompress("PackBits run overflows row".into())); } - out.extend(std::iter::repeat_n(b, len)); + out.extend(std::iter::repeat_n(b, len.min(target - out.len()))); } } Ok(()) diff --git a/crates/psd/src/compression/tests.rs b/crates/psd/src/compression/tests.rs index 6e0d86a7e..7cf876321 100644 --- a/crates/psd/src/compression/tests.rs +++ b/crates/psd/src/compression/tests.rs @@ -771,3 +771,25 @@ fn rle_chunks_use_the_callers_pool_and_report_panics() { assert!(map_chunks(2, vec![0, 1], |_| panic!("worker")).is_err()); }); } + +#[test] +fn padded_packbits_preserves_recovery_without_weakening_strict_decode() { + for (input, expected) in + [(&[0, 7][..], vec![7, 0, 0]), (&[3, 1, 2, 3, 4][..], vec![1, 2, 3]), (&[253, 9][..], vec![9, 9, 9]), (&[128][..], vec![0, 0, 0])] + { + let mut out = vec![42]; + super::packbits::decode_padded_into(input, 3, 3, &mut out).unwrap(); + assert_eq!(&out[1..], expected); + assert_eq!(out[0], 42); + assert!(super::packbits::decode(input, 3).is_err()); + } + for input in [&[3, 1][..], &[255][..]] { + assert!(super::packbits::decode_padded_into(input, 1, 1, &mut vec![]).is_err()); + } + let mut out = vec![42]; + assert!(super::packbits::decode_padded_into(&[], 4, 3, &mut out).is_err()); + assert_eq!(out, [42]); + super::packbits::decode_padded_into(&[255], 0, 0, &mut out).unwrap(); + super::packbits::decode_padded_into(&[0, 7, 255], 1, 1, &mut out).unwrap(); + assert_eq!(out, [42, 7]); +} From 9fc471fc5c18f467394a043adf17f021c2b27cae Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:09:54 +0100 Subject: [PATCH 25/27] psd: add Descriptor::parse_prefix for mid-stream reads (#1887) from_bytes still requires the slice to be fully consumed. parse_prefix returns the descriptor and the number of bytes used, matching VersionedDescriptor, so apps can read a descriptor that is followed by more data (layer effects, fill and adjustment blocks). Part of #778 and effectcraft#786. --- crates/psd/src/descriptor.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs index f3afae64a..96ad47a7b 100644 --- a/crates/psd/src/descriptor.rs +++ b/crates/psd/src/descriptor.rs @@ -300,6 +300,15 @@ impl Descriptor { Ok(d) } + /// Parses a descriptor at the start of `data`, returning it and the number of bytes consumed + /// (trailing data is allowed). Mid-stream reads of layer effects, fill and adjustment + /// descriptors use this; [`from_bytes`] still requires the slice to be fully consumed. + pub fn parse_prefix(data: &[u8]) -> Result<(Self, usize)> { + let mut r = Reader::new(data); + let d = Self::read(&mut r)?; + Ok((d, r.pos())) + } + /// Serializes the descriptor. pub fn to_bytes(&self) -> Vec { let mut v = Vec::new(); @@ -705,6 +714,18 @@ mod tests { assert_eq!(VersionedDescriptor::from_bytes(&b).unwrap(), v); } + #[test] + fn parse_prefix() { + let d = sample(); + let mut b = d.to_bytes(); + let n = b.len(); + b.extend_from_slice(&[1, 2, 3]); + let (p, used) = Descriptor::parse_prefix(&b).unwrap(); + assert_eq!(p, d); + assert_eq!(used, n); + assert!(Descriptor::from_bytes(&b).is_err(), "from_bytes still refuses trailing bytes"); + } + #[test] fn versioned_parse_prefix() { let v = VersionedDescriptor::new(sample()); From dd3015c678052b9a1f1732e70f83576c0a72e75f Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:23:22 +0100 Subject: [PATCH 26/27] wordmodel: keep the kashida length of a justified paragraph (#1875) * wordmodel: keep the kashida length of a justified paragraph ST_Jc lowKashida, mediumKashida and highKashida were all read as Align::Justify, so the length was lost. ParaProps and ResolvedPara now keep it; a direct both/justify cancels an inherited length. The DOCX reader and writer round-trip the three values. * docx: keep a style's kashida length when writing justify w:jc encodes alignment and kashida together. A paragraph or style with Align::Justify and kashida unset still inherits a length from the style chain; writing "both" would read back as Some(Kashida::None) and cancel it. Use the resolved length in that case. --- crates/docx/src/read/props.rs | 11 +- crates/docx/src/write.rs | 5 + crates/docx/src/write/props.rs | 11 +- crates/docx/src/write/story/para.rs | 6 + crates/docx/src/write/styles.rs | 8 +- crates/docx/tests/it/kashida.rs | 136 +++++++++++++++++++++++ crates/docx/tests/it/main.rs | 1 + crates/docx/tests/it/roundtrip.rs | 5 +- crates/wordmodel/src/props.rs | 2 +- crates/wordmodel/src/props/para_props.rs | 45 +++++++- crates/wordmodel/src/resolve.rs | 35 +++++- 11 files changed, 254 insertions(+), 11 deletions(-) create mode 100644 crates/docx/tests/it/kashida.rs diff --git a/crates/docx/src/read/props.rs b/crates/docx/src/read/props.rs index dc61b3a28..4d86ae164 100644 --- a/crates/docx/src/read/props.rs +++ b/crates/docx/src/read/props.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use craft_wordmodel::para::cap_alt; use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CellProps, CharProps, CombineBrackets, EastAsianLayout, HeightRule, Highlight, LineSpacing, NumRef, + Align, Border, BorderStyle, Borders, CellProps, CharProps, CombineBrackets, EastAsianLayout, HeightRule, Highlight, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, RowProps, TabAlign, TabLeader, TabStop, TableFloat, TableLook, TableProps, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, }; @@ -465,7 +465,14 @@ impl PropCtx { _ => {} } } - "w:jc" => p.align = k.attr("w:val").map(align), + "w:jc" => { + if let Some(v) = k.attr("w:val") { + p.align = Some(align(v)); + if let Some(kashida) = Kashida::from_jc(v) { + p.kashida = Some(kashida); + } + } + } "w:outlineLvl" => p.outline_level = k.attr("w:val").and_then(u32_of).map(|v| v.min(9) as u8), "w:rPr" => mark = self.rpr(k), _ => {} diff --git a/crates/docx/src/write.rs b/crates/docx/src/write.rs index 1d9eb0818..4ba486a6c 100644 --- a/crates/docx/src/write.rs +++ b/crates/docx/src/write.rs @@ -151,6 +151,11 @@ pub fn write_as(doc: &Document, flavor: Flavor) -> Result, DocxError> { } impl Writer<'_> { + /// The document's styles, for writers that need the resolved style chain. + pub(crate) fn styles(&self) -> &craft_wordmodel::styles::StyleSheet { + &self.doc.styles + } + fn assign_media(&mut self) { let mut used: std::collections::HashSet = std::collections::HashSet::new(); for (key, bytes) in &self.doc.media { diff --git a/crates/docx/src/write/props.rs b/crates/docx/src/write/props.rs index 0b92ce191..fb76da8d8 100644 --- a/crates/docx/src/write/props.rs +++ b/crates/docx/src/write/props.rs @@ -1,8 +1,8 @@ //! Property writers, in ECMA-376 schema element order. use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CellProps, CharProps, EastAsianLayout, HeightRule, Highlight, LineSpacing, ParaProps, RowProps, TabAlign, - TabLeader, TableFloat, TableLook, TableProps, TextColor, VAlign, VMerge, VertAlign, + Align, Border, BorderStyle, Borders, CellProps, CharProps, EastAsianLayout, HeightRule, Highlight, Kashida, LineSpacing, ParaProps, RowProps, + TabAlign, TabLeader, TableFloat, TableLook, TableProps, TextColor, VAlign, VMerge, VertAlign, }; use crate::units::{n, twips}; @@ -201,6 +201,11 @@ pub fn align_val(a: Align) -> &'static str { } } +/// `w:jc` for a paragraph, including the kashida length of a justified one. +pub fn jc_val(a: Align, kashida: Option) -> &'static str { + if a == Align::Justify { kashida.unwrap_or(Kashida::None).jc() } else { align_val(a) } +} + /// A tab stop's or positional tab's `w:leader` value. pub fn tab_leader(l: TabLeader) -> &'static str { match l { @@ -287,7 +292,7 @@ pub fn ppr_inner(w: &mut W, p: &ParaProps, framed: bool, num_change: Option<&Cha toggle(w, "w:contextualSpacing", p.contextual_spacing); toggle(w, "w:mirrorIndents", p.mirror_indents); if let Some(a) = p.align { - w.val("w:jc", align_val(a)); + w.val("w:jc", jc_val(a, p.kashida)); } if let Some(direction) = p.text_direction { w.val("w:textDirection", direction.ooxml()); diff --git a/crates/docx/src/write/story/para.rs b/crates/docx/src/write/story/para.rs index e0242d8eb..aa973184e 100644 --- a/crates/docx/src/write/story/para.rs +++ b/crates/docx/src/write/story/para.rs @@ -76,6 +76,12 @@ impl Writer<'_> { a.push(("w:original", c.original.clone())); a }); + // `w:jc` encodes alignment and kashida length together. A justified paragraph + // whose kashida is unset (`None`) still inherits a length from its style; writing + // `both` would read back as `Some(Kashida::None)` and cancel that length. + if pp.align == Some(craft_wordmodel::props::Align::Justify) && pp.kashida.is_none() { + pp.kashida = Some(self.styles().resolve_para(&pp).kashida); + } ppr_inner(w, &pp, framed, num_change.as_ref()); if has_mark { self.write_mark_rpr(w, &p.mark, &mark_revs); diff --git a/crates/docx/src/write/styles.rs b/crates/docx/src/write/styles.rs index 3bcf9ab97..0fddf33b4 100644 --- a/crates/docx/src/write/styles.rs +++ b/crates/docx/src/write/styles.rs @@ -184,7 +184,13 @@ fn style_property_xml(wr: &mut Writer<'_>, w: &mut W, st: &Style) { let para_change = st.para_change.as_deref().filter(|ch| ch.old.formatting() != st.para.formatting()); if (!st.para.is_empty() || para_change.is_some()) && st.kind != StyleKind::Character { w.open("w:pPr", &[]); - props::ppr_inner(w, &st.para, false, None); + let mut ppr = st.para.clone(); + if ppr.align == Some(craft_wordmodel::props::Align::Justify) && ppr.kashida.is_none() { + let mut probe = ppr.clone(); + probe.style = Some(st.id.clone()); + ppr.kashida = Some(wr.styles().resolve_para(&probe).kashida); + } + props::ppr_inner(w, &ppr, false, None); if let Some(ch) = para_change { let a = wr.change_attrs(ch.rev); open_change(w, "w:pPrChange", &a); diff --git a/crates/docx/tests/it/kashida.rs b/crates/docx/tests/it/kashida.rs new file mode 100644 index 000000000..02eaf2a49 --- /dev/null +++ b/crates/docx/tests/it/kashida.rs @@ -0,0 +1,136 @@ +//! Justified paragraphs keep their kashida length (`ST_Jc` `lowKashida`, `mediumKashida`, +//! `highKashida`, and ordinary `both` / `justify`) through read and write. + +use std::io::{Read, Write}; + +use std::sync::Arc; + +use craft_wordmodel::props::{Align, Kashida, ParaProps}; +use craft_wordmodel::styles::{Style, StyleKind, StyleSheet}; +use craft_wordmodel::{Document, Paragraph, para_block}; + +fn zip(entries: &[(&str, &[u8])]) -> Vec { + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, bytes) in entries { + zw.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +fn docx(body: &str) -> Vec { + let root = r#""#; + let doc = format!( + r#"{body}"# + ); + zip(&[("_rels/.rels", root.as_bytes()), ("word/document.xml", doc.as_bytes())]) +} + +fn paras(d: &Document) -> Vec<&Paragraph> { + d.body.iter().filter_map(|b| b.as_para()).collect() +} + +fn document_xml(bytes: &[u8]) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut s = String::new(); + z.by_name("word/document.xml").unwrap().read_to_string(&mut s).unwrap(); + s +} + +fn jc_para(body: &str, val: &str) -> String { + format!(r#"{body}"#) +} + +#[test] +fn kashida_jc_values_are_read() { + let body = [ + jc_para("both", "both"), + jc_para("justify", "justify"), + jc_para("low", "lowKashida"), + jc_para("medium", "mediumKashida"), + jc_para("high", "highKashida"), + jc_para("left", "left"), + ] + .join(""); + let d = craft_docx::read(&docx(&body)).unwrap(); + let ps = paras(&d); + let got: Vec<_> = ps.iter().map(|p| (p.props.align, p.props.kashida)).collect(); + assert_eq!( + got, + vec![ + (Some(Align::Justify), Some(Kashida::None)), + (Some(Align::Justify), Some(Kashida::None)), + (Some(Align::Justify), Some(Kashida::Short)), + (Some(Align::Justify), Some(Kashida::Medium)), + (Some(Align::Justify), Some(Kashida::Long)), + (Some(Align::Left), None), + ] + ); +} + +#[test] +fn kashida_length_round_trips() { + let cases = [ + (Some(Kashida::None), "w:val=\"both\""), + (Some(Kashida::Short), "w:val=\"lowKashida\""), + (Some(Kashida::Medium), "w:val=\"mediumKashida\""), + (Some(Kashida::Long), "w:val=\"highKashida\""), + ]; + for (kashida, token) in cases { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { align: Some(Align::Justify), kashida, ..Default::default() }; + let mut d = Document::new(); + d.body = vec![para_block(p.clone())]; + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains(token), "expected {token} in {xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.align, Some(Align::Justify)); + assert_eq!(q.props.kashida, kashida, "round-trip of {token}"); + } +} + +fn body_style(kashida: Kashida) -> StyleSheet { + let mut sheet = StyleSheet::empty(); + sheet.styles.push(Style { + id: "Body".into(), + name: "Body".into(), + kind: StyleKind::Paragraph, + para: ParaProps { align: Some(Align::Justify), kashida: Some(kashida), ..ParaProps::default() }, + ..Style::default() + }); + sheet +} + +#[test] +fn style_kashida_length_survives_a_write_when_the_paragraph_has_no_direct_jc() { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { style: Some("Body".into()), ..ParaProps::default() }; + let mut d = Document::new(); + d.styles = Arc::new(body_style(Kashida::Short)); + d.body = vec![para_block(p)]; + assert_eq!(d.styles.resolve_para(&d.body[0].as_para().unwrap().props).kashida, Kashida::Short); + let bytes = craft_docx::write(&d).unwrap(); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.kashida, None, "no direct w:jc on the paragraph"); + assert_eq!(r.styles.resolve_para(&q.props).kashida, Kashida::Short); +} + +#[test] +fn writing_justify_without_a_local_kashida_keeps_the_style_length() { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { style: Some("Body".into()), align: Some(Align::Justify), kashida: None, ..ParaProps::default() }; + let mut d = Document::new(); + d.styles = Arc::new(body_style(Kashida::Medium)); + d.body = vec![para_block(p)]; + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains("w:val=\"mediumKashida\""), "expected mediumKashida in {xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.align, Some(Align::Justify)); + assert_eq!(q.props.kashida, Some(Kashida::Medium)); + assert_eq!(r.styles.resolve_para(&q.props).kashida, Kashida::Medium); +} diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index 8696981c7..8d6be8a97 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -20,6 +20,7 @@ mod emphasis_marks; mod fixtures; mod glossary; mod heading_levels; +mod kashida; mod legacy_forms; mod ligatures; mod line_numbering; diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 65a01e6b4..2037f9849 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use craft_wordmodel::numbering::ListKind; use craft_wordmodel::para::{Anchor, Float, FloatAlign, NoteKind, ShapeKind, Wrap}; use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CharProps, HeightRule, Highlight, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, RowProps, - TabAlign, TabLeader, TabStop, TableLook, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, + Align, Border, BorderStyle, Borders, CharProps, HeightRule, Highlight, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, + RowProps, TabAlign, TabLeader, TabStop, TableLook, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, }; use craft_wordmodel::section::{Columns, DocGrid, DocGridType, LineNumberRestart, LineNumbering, NumFormat, PageVAlign, SectionProps, SectionStart}; use craft_wordmodel::styles::{Style, StyleKind}; @@ -163,6 +163,7 @@ fn every_para_prop_round_trips() { let pp = ParaProps { style: Some("Heading1".into()), align: Some(Align::Justify), + kashida: Some(Kashida::Medium), indent_left: Some(36.0), indent_right: Some(18.0), indent_first: Some(-18.0), diff --git a/crates/wordmodel/src/props.rs b/crates/wordmodel/src/props.rs index 65a59592c..318779acb 100644 --- a/crates/wordmodel/src/props.rs +++ b/crates/wordmodel/src/props.rs @@ -27,7 +27,7 @@ pub use char_props::{ }; pub use color::{Rgb, TextColor}; pub use para_props::{ - Align, Border, BorderStyle, Borders, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, + Align, Border, BorderStyle, Borders, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, TextAlignment, TextFrame, }; pub use table_props::{ diff --git a/crates/wordmodel/src/props/para_props.rs b/crates/wordmodel/src/props/para_props.rs index a48540248..b3dec00ad 100644 --- a/crates/wordmodel/src/props/para_props.rs +++ b/crates/wordmodel/src/props/para_props.rs @@ -26,6 +26,45 @@ impl Align { } } +/// The kashida (tatweel) length a justified paragraph uses (`ST_Jc` in ECMA-376 §17.18.44). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Kashida { + /// No kashidas: justified lines widen their word spaces (`both`, `justify`). + #[default] + None, + /// `lowKashida`. + Short, + /// `mediumKashida`. + Medium, + /// `highKashida`. + Long, +} + +impl Kashida { + /// The length named by a `w:jc` value, if that value is a kashida or ordinary justify form. + /// Other alignments return `None` so they leave an inherited length. + pub fn from_jc(value: &str) -> Option { + match value { + "both" | "justify" => Some(Kashida::None), + "lowKashida" => Some(Kashida::Short), + "mediumKashida" => Some(Kashida::Medium), + "highKashida" => Some(Kashida::Long), + _ => None, + } + } + + /// The `w:jc` value for a justified paragraph of this length. + pub fn jc(self) -> &'static str { + match self { + Kashida::None => "both", + Kashida::Short => "lowKashida", + Kashida::Medium => "mediumKashida", + Kashida::Long => "highKashida", + } + } +} + /// Paragraph-level text flow (`w:pPr/w:textDirection`, ECMA-376 §17.3.1.41). /// /// Kept separate from [`super::TextDirection`], whose three values model the table-cell direction @@ -383,6 +422,10 @@ pub struct ParaProps { pub style: Option, #[serde(skip_serializing_if = "Option::is_none")] pub align: Option, + /// Kashida length for a justified paragraph. `None` inherits; `Some(Kashida::None)` is + /// ordinary justification (`both` / `justify`) and cancels an inherited length. + #[serde(skip_serializing_if = "Option::is_none")] + pub kashida: Option, /// Left indent, points. #[serde(skip_serializing_if = "Option::is_none")] pub indent_left: Option, @@ -516,7 +559,7 @@ impl ParaProps { } fn overlay_layout(&mut self, patch: &ParaProps) { - overlay_fields!(self, patch; style, align, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, + overlay_fields!(self, patch; style, align, kashida, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, space_before, space_after, space_before_auto, space_after_auto, line_spacing, contextual_spacing, mirror_indents, adjust_right_indent, tabs, shading, borders); } diff --git a/crates/wordmodel/src/resolve.rs b/crates/wordmodel/src/resolve.rs index 7ef12ccdc..85a7a1550 100644 --- a/crates/wordmodel/src/resolve.rs +++ b/crates/wordmodel/src/resolve.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::props::{ - Align, Border, Borders, CharProps, Highlight, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, + Align, Border, Borders, CharProps, Highlight, Kashida, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, }; use crate::styles::{StyleKind, StyleSheet}; @@ -173,6 +173,9 @@ fn default_adjust_right_indent() -> bool { pub struct ResolvedPara { pub style: String, pub align: Align, + /// Kashida length when `align` is Justify; ignored for other alignments. + #[serde(default)] + pub kashida: Kashida, pub indent_left: f32, pub indent_right: f32, pub indent_first: f32, @@ -268,6 +271,7 @@ impl ResolvedPara { ResolvedPara { style: style.to_string(), align: p.align.unwrap_or_default(), + kashida: p.kashida.unwrap_or_default(), indent_left: clampi(p.indent_left), indent_right: clampi(p.indent_right), indent_first: clampi(p.indent_first), @@ -634,4 +638,33 @@ mod tests { assert_eq!(back.padding, [0.0; 4]); assert_eq!(back, ResolvedPara { padding: [0.0; 4], ..p }); } + + /// A style's kashida length is inherited, and a direct `both` / `justify` cancels it. + #[test] + fn kashida_length_follows_the_style_chain_and_direct_formatting() { + use crate::styles::{Style, StyleKind}; + let mut styles = StyleSheet::builtin(); + styles.upsert(Style { + id: "Body".into(), + name: "Body".into(), + kind: StyleKind::Paragraph, + based_on: Some("Normal".into()), + para: ParaProps { align: Some(Align::Justify), kashida: Some(Kashida::Short), ..Default::default() }, + ..Default::default() + }); + let inherited = styles.resolve_para(&ParaProps { style: Some("Body".into()), ..Default::default() }); + assert_eq!(inherited.align, Align::Justify); + assert_eq!(inherited.kashida, Kashida::Short); + + let cleared = styles.resolve_para(&ParaProps { + style: Some("Body".into()), + align: Some(Align::Justify), + kashida: Some(Kashida::None), + ..Default::default() + }); + assert_eq!(cleared.kashida, Kashida::None); + + let longer = styles.resolve_para(&ParaProps { style: Some("Body".into()), kashida: Some(Kashida::Long), ..Default::default() }); + assert_eq!(longer.kashida, Kashida::Long); + } } From a58989cae9173c7402a125bdd42e48fb5c878e09 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:26:36 +0100 Subject: [PATCH 27/27] image-codecs: skip a SubIFDs array whose offset is cut off (#1889) camera_raw_mode read a missing SubIFDs offset as 0, so it walked the TIFF header as the pointer array and could classify a malformed TIFF as RAW. Skip the entry instead, matching LightCraft's classifier. --- crates/image-codecs/src/codecs/tiff.rs | 6 ++++-- crates/image-codecs/tests/legacy_tiff_raw.rs | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/src/codecs/tiff.rs b/crates/image-codecs/src/codecs/tiff.rs index bf59157eb..0f53bb3da 100644 --- a/crates/image-codecs/src/codecs/tiff.rs +++ b/crates/image-codecs/src/codecs/tiff.rs @@ -207,8 +207,10 @@ fn camera_raw_mode(b: &[u8], legacy: bool) -> bool { } TAG_SUB_IFDS if count == 1 => queue.extend(value(e)), TAG_SUB_IFDS => { - // More than one offset: they are stored at the entry's offset. - let at = u32_at(e.saturating_add(8)).unwrap_or(0) as usize; + // More than one offset: they are stored at the entry's offset. A cut-off + // offset field means no array, not one at the start of the file. + let Some(at) = u32_at(e.saturating_add(8)) else { continue }; + let at = at as usize; for k in 0..count.min(8) as usize { queue.extend(u32_at(at.saturating_add(4 * k))); } diff --git a/crates/image-codecs/tests/legacy_tiff_raw.rs b/crates/image-codecs/tests/legacy_tiff_raw.rs index 770801734..27cf6b881 100644 --- a/crates/image-codecs/tests/legacy_tiff_raw.rs +++ b/crates/image-codecs/tests/legacy_tiff_raw.rs @@ -63,3 +63,23 @@ fn oversized_or_cut_directories_are_skipped() { cut.truncate(20); assert!(!legacy_tiff_raw_hint(&cut)); } + +#[test] +fn a_subifds_array_without_its_offset_is_skipped() { + // IFD0 at 64 holds one SubIFDs entry (LONG, count 3) whose offset field is cut off. A DNG + // directory sits at 32, and the bytes at 8 point to it. Reading the missing offset as 0 + // would treat the TIFF header as the pointer array and follow 8 -> 32. + let mut b = vec![0u8; 64]; + b[..8].copy_from_slice(b"II\x2a\0\x40\0\0\0"); + b[8..12].copy_from_slice(&32u32.to_le_bytes()); + b[32..34].copy_from_slice(&1u16.to_le_bytes()); + b[34..46].copy_from_slice(&[0x12, 0xc6, 1, 0, 4, 0, 0, 0, 1, 4, 0, 0]); + b.extend_from_slice(&1u16.to_le_bytes()); + b.extend_from_slice(&[0x4a, 0x01, 4, 0, 3, 0, 0, 0]); + assert_eq!(b.len(), 74); + assert!(!legacy_tiff_raw_hint(&b)); + // With the offset present (the array at 8), the same SubIFD is followed. + let mut whole = b.clone(); + whole.extend_from_slice(&8u32.to_le_bytes()); + assert!(legacy_tiff_raw_hint(&whole)); +}