diff --git a/Cargo.lock b/Cargo.lock index c462980f1..a398fa719 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,12 +116,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "serde", "version_check", "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned" version = "0.4.3" @@ -576,6 +586,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + [[package]] name = "bit-set" version = "0.10.0" @@ -594,6 +613,12 @@ dependencies = [ "bit-vec 0.10.1", ] +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -772,6 +797,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "borsh" version = "1.8.1" @@ -883,6 +914,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.2" @@ -921,6 +958,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bzip2-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beeb59e7e4c811ab37cc73680c798c7a5da77fc9989c62b09138e31ee740f735" +dependencies = [ + "crc32fast", + "tinyvec", +] + [[package]] name = "calloop" version = "0.13.0" @@ -1459,6 +1506,16 @@ dependencies = [ "serde", ] +[[package]] +name = "craft-control" +version = "0.1.0" +dependencies = [ + "getrandom 0.3.4", + "proptest", + "serde", + "serde_json", +] + [[package]] name = "craft-control-protocol" version = "0.1.0" @@ -2039,6 +2096,21 @@ dependencies = [ "indd", ] +[[package]] +name = "craft-ipc" +version = "0.1.0" +dependencies = [ + "chrono", + "getrandom 0.2.17", + "interprocess", + "jsonschema", + "libc", + "serde", + "serde_json", + "sha2 0.10.9", + "widestring", +] + [[package]] name = "craft-jmap" version = "0.1.0" @@ -2676,6 +2748,13 @@ dependencies = [ "zune-jpegxl", ] +[[package]] +name = "craft-render-parity" +version = "0.1.0" +dependencies = [ + "craft-cms", +] + [[package]] name = "craft-rtf" version = "0.1.0" @@ -2799,6 +2878,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-ssip" +version = "0.1.0" + [[package]] name = "craft-stencilcraft-format" version = "0.1.0" @@ -3155,12 +3238,33 @@ dependencies = [ name = "craft-zip" version = "0.1.0" dependencies = [ + "bzip2-rs", "craft-zip", "flate2", + "lzma-rs", + "miniz_oxide 0.8.9", + "ruzstd", + "serde", + "serde_json", "thiserror 2.0.21", "zip", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc24" version = "0.1.6" @@ -3679,6 +3783,12 @@ dependencies = [ "libloading", ] +[[package]] +name = "doctest-file" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359" + [[package]] name = "document-features" version = "0.2.12" @@ -4027,6 +4137,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "emath" version = "0.37.0" @@ -4239,6 +4358,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -4352,6 +4482,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fmt-derive" version = "0.1.2" @@ -4504,6 +4645,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs-set-times" version = "0.20.3" @@ -4935,6 +5086,8 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -5583,6 +5736,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "interprocess" +version = "2.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb02ffbf25e8ef0f4a95bff054eef75f6d1bf16919474cf2caa63b21997478a" +dependencies = [ + "doctest-file", + "libc", + "recvmsg", + "widestring", + "windows-sys 0.61.2", +] + [[package]] name = "io-extras" version = "0.19.0" @@ -5760,6 +5926,58 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonschema" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "jsonschema-regex", + "jsonschema-value", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "strum", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "jsonschema-value" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" +dependencies = [ + "ahash", + "bytecount", + "fraction", + "num-cmp", + "num-traits", + "serde_json", +] + [[package]] name = "jxl-bitstream" version = "1.1.0" @@ -6214,6 +6432,16 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "maybe-owned" version = "0.3.4" @@ -6274,6 +6502,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + [[package]] name = "mime" version = "0.3.17" @@ -6518,6 +6752,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -6545,6 +6793,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.6" @@ -7060,6 +7314,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -8185,6 +8445,12 @@ dependencies = [ "yoke", ] +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + [[package]] name = "redox_syscall" version = "0.4.1" @@ -8222,6 +8488,66 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "referencing" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown 0.17.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-lite" version = "0.1.9" @@ -10491,6 +10817,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "v_frame" version = "0.3.9" @@ -10813,6 +11149,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -11282,6 +11624,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi-util" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index beec53ea3..c42190431 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,10 @@ craft-shx = { path = "crates/shx" } craft-http-url = { path = "crates/http-url" } craft-usd = { path = "crates/usd" } craft-smime = { path = "crates/smime" } +craft-ssip = { path = "crates/ssip" } craft-ber = { path = "crates/ber" } craft-proto = { path = "crates/proto" } +craft-render-parity = { path = "crates/render-parity" } acadrust = "=0.6.6" log = "0.4" # internal: add each crate here as `craft- = { path = "crates/" }` @@ -96,6 +98,7 @@ craft-heif = { path = "crates/heif" } craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } +craft-ipc = { path = "crates/ipc" } craft-idml = { path = "crates/idml" } craft-indd = { path = "crates/indd" } craft-jmap = { path = "crates/jmap" } diff --git a/README.md b/README.md index 2709855d2..47b223d48 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through `craft-crw`) so far; demosaicing and everything after stay with the develop pipeline. +- [`craft-render-parity`](crates/render-parity/README.md): test support for the shared develop + pipeline (#587): how far two renders of the same photo are apart (per-channel error, CIEDE2000, + the worst pixel) and whether that is within a stated tolerance. - [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft (writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on @@ -236,6 +239,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit primitives (`.abr`, `.asl`, `.aco`, `.act`, `.ase`, `.grd`, `.pat`, `.atn`, `.csh`, `.shc`) and the Photoshop blocks of a layered TIFF. Bounded and panic-free on hostile input. Compositing and the mapping onto a document stay in each app. +- [`craft-ipc`](crates/ipc/README.md): Craft apps talking on one machine: per-user local sockets (Unix socket, Windows named pipe), token auth, instance discovery, launch on demand, focus hand-off, frame transfer and linked-file watching. Native only: it does not build for `wasm32`. Protocol in [PROTOCOL.md](crates/ipc/PROTOCOL.md). - [`craft-jmap`](crates/jmap/README.md): the JMAP protocol core (RFC 8620), from MailCraft, without sockets: session, requests of method calls with back-references, responses, ids, address templates, credentials and the judgement of HTTP answers, over any HTTP implementation. Bounded @@ -397,7 +401,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit - [`craft-kra`](crates/kra/README.md): Krita documents (`.kra`, `.krz`), read: the ZIP archive, the `maindoc.xml` node tree as written, and the tiled pixel files with their LZF compression, a tile at a time. From PhotoCraft. - [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. - [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. -- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. +- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. With the `lenient` feature, also a reader of the raw bytes without the `zip` crate, from Craft Anything, for inspectors that must describe damaged archives rather than refuse them: the central-directory listing with ZIP64, prepended-stub and multi-volume checks, CP437 and Info-ZIP Unicode Path names, and entry reads (stored, DEFLATE, and bzip2, LZMA, XZ and Zstandard behind `lenient-*` features) checked against the declared size and CRC-32. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/crates/atomic/src/lib.rs b/crates/atomic/src/lib.rs index 1590372ad..08b881e17 100644 --- a/crates/atomic/src/lib.rs +++ b/crates/atomic/src/lib.rs @@ -70,6 +70,9 @@ pub trait AtomicFs { fn sync_dir(&self, dir: &Path) -> io::Result<()>; /// Give the open temporary file its final permissions. fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()>; + /// Put the open temporary file in group `gid` (Unix only). + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()>; } /// The production file system implementation. @@ -126,6 +129,11 @@ impl AtomicFs for RealFs { fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> { file.set_permissions(perms) } + + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()> { + std::os::unix::fs::fchown(file, None, Some(gid)) + } } /// Flush `file` to stable storage (`sync_all`). @@ -342,10 +350,29 @@ fn split(target: &Path, path: &Path) -> io::Result<(PathBuf, String)> { Ok((dir, name.to_string_lossy().into_owned())) } -/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions of the file it -/// replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when the mode -/// already matches after `fill`. A leftover file with the same name is skipped. When any step -/// fails, the temporary file is removed. +/// Give `perms` the replaced file's group, or drop its group bits when the stage cannot join +/// that group. A stage already in the group needs no fchown, which some filesystems reject even +/// as a no-op, and dropping the bits there would narrow a mode the replaced file granted. +#[cfg(unix)] +fn adopt_group(fs: &dyn AtomicFs, file: &File, stage_gid: u32, replaced_gid: u32, perms: std::fs::Permissions) -> io::Result { + use std::os::unix::fs::PermissionsExt; + if perms.mode() & 0o070 == 0 || stage_gid == replaced_gid { + return Ok(perms); + } + if fs.set_group(file, replaced_gid).is_ok() { + return Ok(perms); + } + let mut narrowed = perms; + narrowed.set_mode(narrowed.mode() & !0o070); + Ok(narrowed) +} + +/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions and group of +/// the file it replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when +/// the mode already matches after `fill`; the group bits are dropped when the file cannot be put +/// in the replaced file's group, so they never open the file to a group that was not granted +/// them. A leftover file with the same name is skipped. When any step fails, the temporary file +/// is removed. fn stage( fs: &dyn AtomicFs, dir: &Path, @@ -383,6 +410,14 @@ fn stage( if (file.metadata()?.permissions().mode() & 0o7777) == (p.mode() & 0o7777) { return Ok(()); } + // The group bits belong to the replaced file's group, which a new + // inode need not be in (setgid directory, another primary group). + if let Some(replaced) = replaced { + let stage_gid = std::os::unix::fs::MetadataExt::gid(&file.metadata()?); + let replaced_gid = std::os::unix::fs::MetadataExt::gid(replaced); + let perms = adopt_group(fs, &file, stage_gid, replaced_gid, p)?; + return fs.set_permissions(&file, perms); + } } fs.set_permissions(&file, p) }) @@ -678,6 +713,9 @@ mod tests { fixed_mode: Option, fail_sync_dir: bool, fail_normal_create: bool, + /// Simulate a filesystem that refuses the stage's group change. + #[cfg(unix)] + fail_set_group: bool, normal_collisions: Cell, /// Another process creates the target just before the hard link. created_meanwhile: Option<&'static [u8]>, @@ -750,6 +788,15 @@ mod tests { } /// Record permission restoration and payload length, then inject a configured error /// or delegate the mode change to the real filesystem. + #[cfg(unix)] + fn set_group(&self, file: &File, gid: u32) -> io::Result<()> { + self.events.borrow_mut().push("set_group"); + if self.fail_set_group { + return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected: set_group failed")); + } + RealFs.set_group(file, gid) + } + fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> { self.events.borrow_mut().push("set_permissions"); #[cfg(unix)] @@ -1090,6 +1137,36 @@ mod tests { Ok(()) } + #[cfg(unix)] + #[test] + fn group_bits_are_dropped_when_the_stage_cannot_join_the_replaced_group() -> io::Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let d = TempDir::new("group-fallback"); + let file = File::create(d.0.join("stage"))?; + let gid = file.metadata()?.gid(); + let elsewhere = gid.wrapping_add(1); + + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o640))?.mode(); + assert_eq!(mode & 0o777, 0o600, "group bits are dropped when the group cannot be joined"); + assert!(fs.events.borrow().contains(&"set_group")); + + // No group bits: the group change is never attempted. + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o600))?.mode(); + assert_eq!(mode & 0o777, 0o600); + assert!(!fs.events.borrow().contains(&"set_group")); + + // A stage already in the replaced file's group keeps its bits untouched. + let fs = Faulty { fail_set_group: true, ..Default::default() }; + let mode = adopt_group(&fs, &file, gid, gid, std::fs::Permissions::from_mode(0o640))?.mode(); + assert_eq!(mode & 0o777, 0o640); + assert!(!fs.events.borrow().contains(&"set_group")); + + assert_no_temp(&d.0); + Ok(()) + } + #[cfg(unix)] #[test] fn normal_policy_preserves_existing_modes_with_private_staging() -> io::Result<()> { @@ -1102,6 +1179,11 @@ mod tests { let fs = Faulty { fail_normal_create: true, ..Default::default() }; replace_with_options(&fs, NO_RETRY, &p, NewFilePermissions::PlatformDefault, |f| f.write_all(b"replacement"))?; assert_eq!(std::fs::metadata(&p)?.permissions().mode() & 0o777, mode); + assert_eq!( + std::os::unix::fs::MetadataExt::gid(&std::fs::metadata(&p)?), + std::os::unix::fs::MetadataExt::gid(&d.0.metadata()?), + "kept in the directory's group" + ); assert!(!fs.events.borrow().contains(&"create_new_normal")); assert_eq!(*fs.created_modes.borrow(), [0o600]); } diff --git a/crates/control/Cargo.toml b/crates/control/Cargo.toml new file mode 100644 index 000000000..0a18ccc99 --- /dev/null +++ b/crates/control/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "craft-control" +description = "The control channel any app may share: token files, the loopback server, and its requests; no UI toolkit" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + +# The loopback server is for desktop apps; the web app is driven through the page instead. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# The per-launch control token: 32 random bytes from the operating system (MIT OR Apache-2.0). +getrandom = "0.3" + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/control/src/auth.rs b/crates/control/src/auth.rs new file mode 100644 index 000000000..bb2cef86b --- /dev/null +++ b/crates/control/src/auth.rs @@ -0,0 +1,673 @@ +//! The control channel's per-launch token (docs/control-protocol.md). An app listening for +//! control makes a random token, writes it to a file only the user can read +//! ([`App::token_path`]) and refuses every connection whose first request doesn't carry it, so +//! other users on the same machine (loopback is shared, on Windows across remote-desktop sessions +//! too) can't drive the app. Clients find it with [`App::client_token`]. Next to it, `.pid` +//! names the process that wrote it, so a client refused with a leftover token can say the file is +//! stale. + +use std::ffi::OsString; +use std::io::{Error, ErrorKind, Read, Result, Write}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Shortest token accepted from the app's `CONTROL_TOKEN` variable (hex characters). +pub const MIN_TOKEN_LEN: usize = 32; + +/// Ports tried from the app's default port upward when it is taken (a second window, another +/// program). +pub const PORT_TRIES: u16 = 16; + +/// Which app a control channel belongs to: where its token files live, its environment and its +/// name in messages. +#[derive(Clone, Copy)] +pub struct App { + /// The app's name in messages ("restart SolidCraft"). + pub name: &'static str, + /// The per-user data folder; the token files go in its `control` folder. + pub data_dir: fn() -> Option, + /// The app's environment variable `name` (`CONTROL_DIR`, `CONTROL_TOKEN`), with the app's prefix. + pub env: fn(&str) -> Option, + /// A time in seconds since the Unix epoch, as the app writes UTC times in messages. + pub utc: fn(u64) -> String, + /// The port the control channel tries first when the launch gives none. Below the ephemeral + /// ranges of Linux (32768+) and Windows/macOS (49152+), so the system never hands it to an + /// outgoing connection. + pub default_port: u16, +} + +/// Where the control channel listens, from the launch's `--control`/environment port and the app's +/// allow-agents setting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Listen { + /// Exactly this port (`--control PORT`, the app's `CONTROL_PORT`); no fallback. + Exact(u16), + /// The app's [`App::default_port`], or the next free one of [`PORT_TRIES`]. + Default, +} + +/// What a launch listens on. An explicit port wins over the app's allow-agents setting +/// (`enabled`), so a scripted launch works with the setting off; with the setting off and no port, +/// nothing listens. +pub fn plan(explicit: Option, enabled: bool) -> Option { + match explicit { + Some(port) => Some(Listen::Exact(port)), + None if enabled => Some(Listen::Default), + None => None, + } +} + +/// Bind a loopback-only listener on `first`, or on the next free port when it is taken, up to +/// `tries` ports in all. +pub fn bind_with_fallback(first: u16, tries: u16) -> Result { + let mut last = Error::new(ErrorKind::AddrInUse, "no port to try"); + for port in (first..=u16::MAX).take(usize::from(tries.max(1))) { + match std::net::TcpListener::bind(("127.0.0.1", port)) { + Ok(l) => return Ok(l), + // In use; Windows reports a port another program holds exclusively as denied. + Err(e) if matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) => last = e, + Err(e) => return Err(e), + } + } + Err(last) +} + +impl App { + /// The folder of the token files: `/control` (the app's `CONTROL_DIR` replaces it). + fn control_dir(&self) -> Option { + match self.custom_control_dir() { + Some(d) => Some(PathBuf::from(d)), + None => Some((self.data_dir)()?.join("control")), + } + } + + /// The folder the app's `CONTROL_DIR` names in place of the default one. + fn custom_control_dir(&self) -> Option { + (self.env)("CONTROL_DIR").filter(|d| !d.is_empty()) + } + + /// The token file of the app listening on `port`: `/control/.token` + /// (the app's `CONTROL_DIR` replaces the folder). + pub fn token_path(&self, port: u16) -> Option { + Some(self.control_dir()?.join(format!("{port}.token"))) + } + + /// The discovery file: the port of the most recently started app, so a client with no port + /// finds it. It holds only the port; the token stays in its own file. + pub fn current_port_path(&self) -> Option { + Some(self.control_dir()?.join("current.port")) + } + + /// Record `port` as the running app's, readable only by the user like the token files. + pub fn write_current_port(&self, port: u16) -> Result { + let path = self.current_port_path().ok_or_else(|| Error::new(ErrorKind::NotFound, "no data folder"))?; + self.write_token_file(&path, &port.to_string())?; + Ok(path) + } + + /// The port the discovery file names, if there is one. + pub fn read_current_port(&self) -> Option { + parse_port(&std::fs::read_to_string(self.current_port_path()?).ok().filter(|t| t.len() <= 64)?) + } + + /// Remove the discovery file (and its process file) at exit, unless a later app has taken it + /// over: it names another port, or another launch now holds this port's token file. Under + /// Flatpak every instance has its own loopback, so two windows can listen on the same port + /// number and write the same file names; the one closing first must leave the other's (#8794). + /// Run it after [`remove_token_file`], which takes this launch's own token file away first. + pub fn remove_current_port(&self, port: u16, token: &str) { + if let (Some(current), Some(file)) = (self.current_port_path(), self.token_path(port)) { + remove_discovery(¤t, &file, port, token); + } + } + + /// A token set on purpose with the app's `CONTROL_TOKEN` (test harnesses, sandboxed packages + /// whose data folder the client can't see), if it is long enough. + pub fn env_token(&self) -> Option { + (self.env)("CONTROL_TOKEN").and_then(|v| v.into_string().ok()).map(|t| t.trim().to_string()).filter(|t| t.len() >= MIN_TOKEN_LEN) + } + + /// The token a client sends to the app on `port`: `explicit` (`--token`), else the app's + /// `CONTROL_TOKEN`, else the app's token file. + pub fn client_token(&self, port: u16, explicit: Option<&str>) -> Option { + if let Some(t) = explicit.map(str::trim).filter(|t| !t.is_empty()) { + return Some(ClientToken { token: t.to_string(), file: None, app: *self }); + } + if let Some(t) = self.env_token() { + return Some(ClientToken { token: t, file: None, app: *self }); + } + // Read directly, not through any agent's file scope: the app's own file. + read_token_file(&self.token_path(port)?, *self) + } + + /// Write `token` to `path`, readable and writable only by the user: the folder is made `0700` + /// and the file `0600` on Unix; on Windows the file's inherited permissions are replaced by one + /// entry for the user (`icacls`) before the token goes in; when that fails, the still empty file + /// is removed and no token is written. A file left by an earlier launch is replaced. The process + /// file beside it ([`pid_path`]) records this process, best effort. + pub fn write_token_file(&self, path: &Path, token: &str) -> Result<()> { + #[cfg(windows)] + let restrict = |p: &Path| restrict_to_user(p, self.custom_control_dir().is_some()); + #[cfg(not(windows))] + let restrict = |_: &Path| Ok(()); + write_token_file_with(path, token, restrict) + } +} + +fn parse_port(text: &str) -> Option { + text.trim().parse::().ok().filter(|p| *p != 0) +} + +/// [`App::remove_current_port`] on given files: `current` is the discovery file, `file` the token +/// file of `port`. +fn remove_discovery(current: &Path, file: &Path, port: u16, token: &str) { + let named = std::fs::read_to_string(current).ok().filter(|t| t.len() <= 64).and_then(|t| parse_port(&t)); + if named != Some(port) || held_by_another(file, token) { + return; + } + let _ = std::fs::remove_file(current); + let _ = std::fs::remove_file(pid_path(current)); +} + +/// A client token and, when it came from a token file, where that file is. +pub struct ClientToken { + pub token: String, + pub file: Option, + /// The app it is for, named in an explanation. + pub app: App, +} + +/// The token file a client read: its path, when it was written and the process that wrote it. +pub struct TokenSource { + pub path: PathBuf, + pub modified: Option, + /// From `.pid`: the writer's process id and, on Linux, its start time. + pub writer: Option<(u32, Option)>, +} + +impl ClientToken { + /// The app's `error`, and for a refused token read from a file, which file it was, when it + /// was written and whether the instance that wrote it has exited. Never the token itself. + pub fn explain_error(&self, error: &str) -> String { + let Some(src) = &self.file else { return error.to_string() }; + if !error.starts_with("unauthorized") { + return error.to_string(); + } + let written = src + .modified + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map(|t| (self.app.utc)(t.as_secs())) + .unwrap_or_else(|| "at an unknown time".into()); + let age = src.modified.and_then(|t| t.elapsed().ok()).map(|d| format!(" ({} s ago)", d.as_secs())).unwrap_or_default(); + let path = src.path.display(); + let name = self.app.name; + match src.writer.filter(|(pid, start)| writer_exited(*pid, *start)) { + Some((pid, _)) => format!( + "{error}; stale token file from an instance that exited (process {pid}): {path}, written {written}{age}; delete that file or restart {name}" + ), + None => format!("{error}; token from {path}, written {written}{age}; if {name} was force-quit, restart it or delete that file"), + } + } +} + +/// Files bigger than this are not token files. +const MAX_TOKEN_FILE: u64 = 4096; + +fn read_token_file(path: &Path, app: App) -> Option { + let f = std::fs::File::open(path).ok()?; + let modified = f.metadata().ok().and_then(|m| m.modified().ok()); + let mut text = String::new(); + f.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?; + if text.len() as u64 > MAX_TOKEN_FILE { + return None; + } + let token = text.trim(); + if token.is_empty() { + return None; + } + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let writer = read_pid_file(&pid_path(&path)); + Some(ClientToken { token: token.to_string(), file: Some(TokenSource { path, modified, writer }), app }) +} + +/// The process file beside the token file `path`: `7878.token` → `7878.pid`. +pub fn pid_path(path: &Path) -> PathBuf { + path.with_extension("pid") +} + +/// ` []`; anything else (or no file) is no information. +fn read_pid_file(path: &Path) -> Option<(u32, Option)> { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(256).read_to_string(&mut text).ok()?; + let mut fields = text.split_whitespace(); + let pid = fields.next()?.parse::().ok().filter(|&pid| pid > 0)?; + Some((pid, fields.next().and_then(|s| s.parse().ok()))) +} + +/// Whether the process that wrote a token file is known to be gone: not running, or its id now +/// belongs to a process that started at another time. Unknown counts as running. +fn writer_exited(pid: u32, start: Option) -> bool { + process_running(pid) == Some(false) || start.zip(process_start(pid)).is_some_and(|(saved, now)| saved != now) +} + +/// Whether process `pid` runs; `None` when that can't be found out. +fn process_running(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + if !Path::new("/proc/self").exists() { + return None; + } + match std::fs::metadata(format!("/proc/{pid}")) { + Ok(_) => Some(true), + Err(e) if e.kind() == ErrorKind::NotFound => Some(false), + Err(_) => None, + } + } + #[cfg(all(unix, not(target_os = "linux")))] + { + let output = std::process::Command::new("ps").args(["-p", &pid.to_string(), "-o", "pid="]).output().ok()?; + if output.status.success() { + return Some(true); + } + (output.status.code() == Some(1) && output.stdout.iter().all(u8::is_ascii_whitespace)).then_some(false) + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + let output = std::process::Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .creation_flags(0x0800_0000) + .output() + .ok()?; + if !output.status.success() || output.stdout.is_empty() { + return None; + } + // `"image","pid",…` per process; with no match, a message line without a pid. + let text = String::from_utf8_lossy(&output.stdout); + Some(text.lines().any(|line| line.split("\",\"").nth(1).and_then(|s| s.trim_matches('"').parse::().ok()) == Some(pid))) + } + #[cfg(not(any(unix, windows)))] + { + let _ = pid; + None + } +} + +/// Linux: the process's start time (clock ticks after boot), which tells a reused id apart. +fn process_start(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let mut stat = String::new(); + std::fs::File::open(format!("/proc/{pid}/stat")).ok()?.take(4096).read_to_string(&mut stat).ok()?; + stat.rsplit_once(')')?.1.split_whitespace().nth(19)?.parse().ok() + } + #[cfg(not(target_os = "linux"))] + { + let _ = pid; + None + } +} + +/// `a == b`, taking the same time wherever they differ. +pub fn tokens_match(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// [`App::write_token_file`], with `restrict` limiting the new file to the user (Windows). +fn write_token_file_with(path: &Path, token: &str, restrict: impl FnOnce(&Path) -> Result<()>) -> Result<()> { + let dir = path.parent().ok_or_else(|| Error::new(ErrorKind::InvalidInput, "no folder"))?; + std::fs::create_dir_all(dir)?; + let _ = std::fs::remove_file(path); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + opts.mode(0o600); + } + let opened = opts.open(path)?; + // On a refusal the file goes with the closure, so it is closed before it is removed. + let mut f = match restrict(path).map(|()| opened) { + Ok(f) => f, + Err(e) => { + let _ = std::fs::remove_file(path); + return Err(e); + } + }; + f.write_all(token.as_bytes())?; + f.write_all(b"\n")?; + f.sync_all()?; + let pid = std::process::id(); + let line = process_start(pid).map_or_else(|| format!("{pid}\n"), |start| format!("{pid} {start}\n")); + let _ = std::fs::write(pid_path(path), line); + Ok(()) +} + +/// Remove a token file this launch wrote, and its process file, while it still holds `token`. +/// Another launch may have written the same file since (a second window on the same port, +/// #8794); its file stays. The process id can't tell the two apart, as every Flatpak instance +/// has the same one, but each launch's token is its own. +pub fn remove_token_file(path: &Path, token: &str) { + if read_token(path).is_some_and(|held| tokens_match(&held, token.trim())) { + let _ = std::fs::remove_file(path); + let _ = std::fs::remove_file(pid_path(path)); + } +} + +/// The token a token file holds, trimmed; `None` when there is no readable token file. +fn read_token(path: &Path) -> Option { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?; + let token = text.trim(); + (text.len() as u64 <= MAX_TOKEN_FILE && !token.is_empty()).then(|| token.to_string()) +} + +/// Whether the token file at `path` holds a token other than `token`: another launch's. +fn held_by_another(path: &Path, token: &str) -> bool { + read_token(path).is_some_and(|held| !tokens_match(&held, token.trim())) +} + +/// How long `icacls` may take (#6145). It finishes at once; on some computers the new process +/// is left suspended (security software holding it), and waiting on it without a limit froze +/// the app. +#[cfg(windows)] +const ICACLS_LIMIT: std::time::Duration = std::time::Duration::from_secs(5); + +/// How a run of `icacls` ended. +#[cfg(any(windows, test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum IcaclsEnd { + Succeeded, + Failed, + /// It didn't finish within [`ICACLS_LIMIT`] and was ended. + TimedOut, +} + +/// Whether a token may go into a file after `icacls` ended this way. `custom_folder`: the file is +/// in the folder the app's `CONTROL_DIR` names. A timeout is accepted only in the default folder, +/// the user's own local app data, which other users can't read by default; nothing is known about +/// the permissions of any other folder. +#[cfg(any(windows, test))] +fn restriction_result(end: IcaclsEnd, custom_folder: bool, path: &Path) -> Result<()> { + match end { + IcaclsEnd::Succeeded => Ok(()), + IcaclsEnd::TimedOut if !custom_folder => Ok(()), + IcaclsEnd::TimedOut => Err(Error::new(ErrorKind::TimedOut, format!("icacls did not finish in time to restrict {}", path.display()))), + IcaclsEnd::Failed => Err(Error::other(format!("icacls failed to restrict {}", path.display()))), + } +} + +/// Windows: drop inherited entries and grant only the current user full access. An `icacls` +/// that doesn't finish in time is ended, and [`restriction_result`] decides what that means. +#[cfg(windows)] +fn restrict_to_user(path: &Path, custom_folder: bool) -> Result<()> { + use std::os::windows::process::CommandExt; + let user = std::env::var("USERNAME").map_err(|_| Error::other("USERNAME is not set"))?; + let domain = std::env::var("USERDOMAIN").ok().filter(|d| !d.is_empty()); + let who = domain.map_or(user.clone(), |d| format!("{d}\\{user}")); + let child = std::process::Command::new("icacls") + .arg(path) + .args(["/inheritance:r", "/grant:r", &format!("{who}:F")]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .creation_flags(0x0800_0000) + .spawn()?; + let end = match wait_within(child, ICACLS_LIMIT)? { + Some(status) if status.success() => IcaclsEnd::Succeeded, + Some(_) => IcaclsEnd::Failed, + None => IcaclsEnd::TimedOut, + }; + restriction_result(end, custom_folder, path) +} + +/// Wait for `child` at most `limit`: its exit status, or `None` (and the child ended) when it +/// hasn't finished by then. +#[cfg(any(windows, test))] +fn wait_within(mut child: std::process::Child, limit: std::time::Duration) -> Result> { + let start = std::time::Instant::now(); + loop { + if let Some(status) = child.try_wait()? { + return Ok(Some(status)); + } + if start.elapsed() >= limit { + // Not waited for afterwards: a suspended process may not end at once. + let _ = child.kill(); + return Ok(None); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +/// Removes the token file (path, token) when the app ends, while it still holds that token. +pub struct TokenFile(pub PathBuf, pub String); + +impl Drop for TokenFile { + fn drop(&mut self) { + remove_token_file(&self.0, &self.1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An app with no data folder and no environment, whose times read `t`. + const TEST: App = App { name: "TestCraft", data_dir: || None, env: |_| None, utc: |secs| format!("t{secs}"), default_port: 27878 }; + + /// #6145: a child that doesn't finish in time is given up on, not waited for forever. + #[cfg(unix)] + #[test] + fn a_slow_child_is_waited_for_only_so_long() { + use std::time::{Duration, Instant}; + let start = Instant::now(); + let slow = std::process::Command::new("sleep").arg("30").spawn().unwrap(); + assert!(wait_within(slow, Duration::from_millis(200)).unwrap().is_none()); + assert!(start.elapsed() < Duration::from_secs(5), "{:?}", start.elapsed()); + let quick = std::process::Command::new("true").spawn().unwrap(); + assert!(wait_within(quick, Duration::from_secs(5)).unwrap().is_some_and(|s| s.success())); + let failing = std::process::Command::new("false").spawn().unwrap(); + assert!(wait_within(failing, Duration::from_secs(5)).unwrap().is_some_and(|s| !s.success())); + } + + /// #6145's freeze fix stays in the default folder, the user's own; a folder picked with + /// the app's `CONTROL_DIR` gets no token unless the file was really restricted. + #[test] + fn only_the_default_folder_accepts_an_icacls_that_timed_out() { + let path = Path::new("control/7878.token"); + for custom in [false, true] { + assert!(restriction_result(IcaclsEnd::Succeeded, custom, path).is_ok()); + assert!(restriction_result(IcaclsEnd::Failed, custom, path).is_err()); + } + assert!(restriction_result(IcaclsEnd::TimedOut, false, path).is_ok()); + let e = restriction_result(IcaclsEnd::TimedOut, true, path).unwrap_err(); + assert_eq!(e.kind(), ErrorKind::TimedOut); + assert!(e.to_string().contains("7878.token"), "{e}"); + } + + /// A file that couldn't be restricted is removed, empty and with no process file beside it, and + /// the error goes to the caller. A restricted one gets its token. + #[test] + fn a_token_file_that_could_not_be_restricted_is_removed() { + let dir = std::env::temp_dir().join(format!("craft-control-restrict-{}", std::process::id())); + let path = dir.join("7879.token"); + let e = write_token_file_with(&path, "secret-token", |p| { + assert_eq!(std::fs::read(p).unwrap(), b"", "restricted before the token goes in"); + restriction_result(IcaclsEnd::TimedOut, true, p) + }) + .unwrap_err(); + assert_eq!(e.kind(), ErrorKind::TimedOut); + assert!(!path.exists() && !pid_path(&path).exists()); + write_token_file_with(&path, "secret-token", |p| restriction_result(IcaclsEnd::TimedOut, false, p)).unwrap(); + assert_eq!(read_token(&path).as_deref(), Some("secret-token")); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn tokens_compare_whole() { + assert!(tokens_match("abc123", "abc123")); + assert!(!tokens_match("abc123", "abc124")); + assert!(!tokens_match("abc123", "abc12")); + assert!(!tokens_match("", "a")); + } + + /// The preference starts the listener by default, off it doesn't, and a port given on the + /// command line or in the environment wins over it either way. + #[test] + fn the_preference_decides_unless_a_port_is_given() { + assert_eq!(plan(None, true), Some(Listen::Default)); + assert_eq!(plan(None, false), None); + assert_eq!(plan(Some(7878), false), Some(Listen::Exact(7878))); + assert_eq!(plan(Some(7878), true), Some(Listen::Exact(7878))); + } + + /// A taken port falls through to the next free one, on loopback only; a run of taken ports + /// is an error, not a hang. + #[test] + fn a_taken_port_falls_back_to_the_next_free_one() { + let first = bind_with_fallback(TEST.default_port, PORT_TRIES).unwrap(); + let a = first.local_addr().unwrap(); + assert!(a.ip().is_loopback() && (TEST.default_port..TEST.default_port + PORT_TRIES).contains(&a.port()), "{a}"); + let second = bind_with_fallback(a.port(), PORT_TRIES).unwrap(); + assert!(second.local_addr().unwrap().port() > a.port()); + assert!(bind_with_fallback(a.port(), 1).is_err()); + } + + #[test] + fn the_discovery_file_holds_a_port() { + assert_eq!(parse_port("27879\n"), Some(27879)); + assert_eq!(parse_port("0"), None); + assert_eq!(parse_port("70000"), None); + assert_eq!(parse_port("nope"), None); + let dir = std::env::temp_dir().join(format!("solidcraft-current-port-{}", std::process::id())); + let path = dir.join("current.port"); + TEST.write_token_file(&path, "27881").unwrap(); + assert_eq!(parse_port(&std::fs::read_to_string(&path).unwrap()), Some(27881)); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A one-line token file (and one with no process file) still reads. + #[test] + fn a_token_file_reads_with_and_without_its_process_file() { + let dir = std::env::temp_dir().join(format!("craft-control-read-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("7879.token"); + std::fs::write(&path, "legacy-token\n").unwrap(); + let client = read_token_file(&path, TEST).unwrap(); + assert_eq!(client.token, "legacy-token"); + let src = client.file.as_ref().unwrap(); + assert_eq!(src.path, std::fs::canonicalize(&path).unwrap()); + assert!(src.modified.is_some() && src.writer.is_none()); + std::fs::write(pid_path(&path), "not a pid\n").unwrap(); + assert!(read_token_file(&path, TEST).unwrap().file.unwrap().writer.is_none()); + std::fs::write(&path, "x".repeat(4097)).unwrap(); + assert!(read_token_file(&path, TEST).is_none()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A refused token's explanation names the app and writes the time as the app does. + #[test] + fn an_explanation_names_the_app() { + let file = Some(TokenSource { path: PathBuf::from("7878.token"), modified: Some(UNIX_EPOCH), writer: None }); + let client = ClientToken { token: "secret-token".into(), file, app: TEST }; + let error = client.explain_error("unauthorized"); + assert!(error.contains("written t0 (") && error.contains("if TestCraft was force-quit"), "{error}"); + assert!(!error.contains("secret-token"), "{error}"); + let explicit = TEST.client_token(0, Some("explicit-token")).unwrap(); + assert!(explicit.token == "explicit-token" && explicit.file.is_none()); + assert_eq!(explicit.explain_error("unauthorized"), "unauthorized"); + } + + /// A process file naming a process that has exited marks the token file stale. + /// Where the process inspector cannot run (a sandboxed test runner), the exit + /// is unknown, and the explanation must not claim it. + #[cfg(any(unix, windows))] + #[test] + fn a_token_file_from_an_exited_process_is_stale() { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "auth::tests::tokens_compare_whole"]) + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + assert!(child.wait().unwrap().success()); + let source = |writer| Some(TokenSource { path: PathBuf::from("7878.token"), modified: None, writer }); + let stale = ClientToken { token: "secret-token".into(), file: source(Some((pid, None))), app: TEST }; + let error = stale.explain_error("unauthorized"); + match process_running(pid) { + Some(false) => { + assert!(error.contains(&format!("stale token file from an instance that exited (process {pid}): 7878.token")), "{error}"); + } + None => assert!(!error.contains("stale") && error.contains("restart it or delete that file"), "{error}"), + Some(true) => panic!("the reaped child {pid} is reported as running"), + } + assert!(!error.contains("secret-token")); + let live = ClientToken { token: "t".into(), file: source(Some((std::process::id(), process_start(std::process::id())))), app: TEST }; + assert!(!live.explain_error("unauthorized").contains("stale")); + #[cfg(target_os = "linux")] + { + let reused = process_start(std::process::id()).map(|s| s + 1); + let reused = ClientToken { token: "t".into(), file: source(Some((std::process::id(), reused))), app: TEST }; + assert!(reused.explain_error("unauthorized").contains("stale token file")); + } + } + + /// The token file is the user's alone, and a client reads it back. + #[cfg(unix)] + #[test] + fn the_token_file_is_private_to_the_user() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("craft-control-auth-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let path = dir.join("control/7878.token"); + std::fs::create_dir_all(dir.join("control")).unwrap(); + std::fs::write(&path, "stale").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + TEST.write_token_file(&path, "0123456789abcdef0123456789abcdef").unwrap(); + let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&path), 0o600); + assert_eq!(mode(path.parent().unwrap()), 0o700); + assert_eq!(std::fs::read_to_string(&path).unwrap().trim(), "0123456789abcdef0123456789abcdef"); + let writer = read_token_file(&path, TEST).unwrap().file.unwrap().writer; + assert_eq!(writer.map(|(pid, _)| pid), Some(std::process::id())); + drop(TokenFile(path.clone(), "0123456789abcdef0123456789abcdef".into())); + assert!(!path.exists() && !pid_path(&path).exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// #8794: under Flatpak two windows can listen on the same port number (each has its own + /// loopback) and write the same files. The one closing first leaves the other's token, process + /// and discovery files; the last one takes them all away, `current.pid` too. + #[test] + fn a_closing_launch_leaves_another_launchs_files() { + let dir = std::env::temp_dir().join(format!("craft-control-two-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let (file, current) = (dir.join("27878.token"), dir.join("current.port")); + let (first, second) = ("a".repeat(64), "b".repeat(64)); + // The first window, then the second on the same port, each writing both files. + for token in [&first, &second] { + TEST.write_token_file(&file, token).unwrap(); + TEST.write_token_file(¤t, "27878").unwrap(); + } + // The first closes: the second's files stay. + remove_token_file(&file, &first); + remove_discovery(¤t, &file, 27878, &first); + assert_eq!(read_token(&file).as_deref(), Some(second.as_str())); + assert!(pid_path(&file).exists() && current.exists() && pid_path(¤t).exists()); + // Another port named in the discovery file is another app's too. + remove_discovery(¤t, &file, 27879, &second); + assert!(current.exists()); + // The second closes: nothing of either is left. + remove_token_file(&file, &format!(" {second}\n")); + remove_discovery(¤t, &file, 27878, &second); + assert!(!file.exists() && !pid_path(&file).exists() && !current.exists() && !pid_path(¤t).exists()); + // With no token file left (removed by hand, or never written), the discovery file of the + // port still goes; a missing file is no error. + TEST.write_token_file(¤t, "27878").unwrap(); + remove_token_file(&file, &second); + remove_discovery(¤t, &file, 27878, &second); + assert!(!current.exists()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/control/src/cancel.rs b/crates/control/src/cancel.rs new file mode 100644 index 000000000..e03052307 --- /dev/null +++ b/crates/control/src/cancel.rs @@ -0,0 +1,118 @@ +//! Where a control request is, so the connection that sent it can cancel it (`control.cancel`, +//! #1989): shared between the window, which moves it along, and the connection. Whoever changes +//! it first wins. + +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::{Mutex, PoisonError}; + +/// A request's place in the app, and how to stop the work computing it. +#[derive(Default)] +pub struct Ticket { + state: AtomicU8, + /// It is an export computing on a copy, whose worker writes its file. + writes_file: AtomicBool, + /// Tells its worker to stop: set by the window when it hands the request to one. + stopper: Mutex>>, +} + +impl std::fmt::Debug for Ticket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Ticket").field("state", &self.state).field("writes_file", &self.writes_file).finish_non_exhaustive() + } +} + +/// Waiting its turn (sent, or held behind another request). +const WAITING: u8 = 0; +/// Running on the window thread (or handed to something that can't stop it, such as the file +/// loader). +const RUNNING: u8 = 1; +/// Computing off the window thread. +const COMPUTING: u8 = 2; +const CANCELLED: u8 = 3; + +/// What [`Ticket::cancel`] did. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Cancel { + /// It was waiting its turn: it will not run. + Removed, + /// It was computing off the window thread: it is answered as stopped and nothing of it is + /// applied. An export's worker may still write its file. + Stopped { writes_file: bool }, + /// It runs on the window thread: it will finish. + Running, + /// It was cancelled or stopped before. + Already, +} + +impl Ticket { + /// Cancel the request, unless it already runs on the window thread. + pub fn cancel(&self) -> Cancel { + let mut state = self.state.load(Ordering::Acquire); + loop { + let done = match state { + WAITING => Cancel::Removed, + COMPUTING => Cancel::Stopped { writes_file: self.writes_file.load(Ordering::Acquire) }, + CANCELLED => return Cancel::Already, + _ => return Cancel::Running, + }; + match self.state.compare_exchange(state, CANCELLED, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => { + self.stop_worker(); + return done; + } + Err(now) => state = now, + } + } + } + + pub fn is_cancelled(&self) -> bool { + self.state.load(Ordering::Acquire) == CANCELLED + } + + /// The window takes the request to run it: false when it was cancelled. + pub fn take(&self) -> bool { + self.moves(WAITING, RUNNING) + } + + /// The work computing it stops on `stop` (a worker's own stop flag). Cancelled already, it is + /// stopped at once. + pub fn on_stop(&self, stop: impl Fn() + Send + Sync + 'static) { + *self.stopper.lock().unwrap_or_else(PoisonError::into_inner) = Some(Box::new(stop)); + if self.is_cancelled() { + self.stop_worker(); + } + } + + /// The window hands it to a worker (`writes_file`: an export on a copy). + pub fn computing(&self, writes_file: bool) { + self.writes_file.store(writes_file, Ordering::Release); + let _ = self.moves(RUNNING, COMPUTING) || self.moves(WAITING, COMPUTING); + } + + /// Its worker is done and it runs on the window thread now (or it never left): false when it + /// was cancelled meanwhile. Only the window moves it on from running, so running stays running. + pub fn back(&self) -> bool { + self.moves(COMPUTING, RUNNING) || self.state.load(Ordering::Acquire) == RUNNING + } + + /// Computed, but it waits its turn again (a file opening meanwhile): false when cancelled. + pub fn wait_again(&self) -> bool { + self.moves(COMPUTING, WAITING) + } + + /// The person at the app stopped it (Stop): its worker stops too. + pub fn stop(&self) { + self.state.store(CANCELLED, Ordering::Release); + self.stop_worker(); + } + + fn stop_worker(&self) { + if let Some(stop) = self.stopper.lock().unwrap_or_else(PoisonError::into_inner).as_ref() { + stop(); + } + } + + fn moves(&self, from: u8, to: u8) -> bool { + self.state.compare_exchange(from, to, Ordering::AcqRel, Ordering::Acquire).is_ok() + } +} diff --git a/crates/control/src/client.rs b/crates/control/src/client.rs new file mode 100644 index 000000000..cbad78850 --- /dev/null +++ b/crates/control/src/client.rs @@ -0,0 +1,616 @@ +//! The control channel's client (docs/control-protocol.md): a connection to an app's loopback +//! control port, with the app's token ([`App::client_token`]) on every request, one bounded +//! round trip per request, a fresh connection after one is lost, and the wait for a request the +//! app says is still running (#1618), through `control.pending`. Its errors tell an app that is +//! not running from one that went away during a call ([`CallError`]). What a request means, and +//! anything an app adds to its parameters, is the caller's. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::Effect; +use crate::auth::App; + +/// Longest reply line accepted from the app. +pub const MAX_REPLY: u64 = 64 << 20; + +/// How long a reply may take: kernel operations (booleans, fillets) on big parts can take a while. +pub const READ_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long the app may take to answer a request sent [`Request::once`]: a little longer than its +/// control server waits for a long request (3600 s), so its "still running" reply arrives first. +pub const ONCE_TIMEOUT: Duration = Duration::from_secs(3720); + +/// How long connecting to the app may take. +const CONNECT_TIMEOUT: Duration = Duration::from_millis(800); + +/// How often a long wait is reported ([`Connection::with_progress`]). +const PROGRESS_EVERY: Duration = Duration::from_secs(30); + +/// How an error starts when a request is still running in the app after the wait: the request +/// was not cancelled and its reply waits on the connection; [`Connection::pending`] gets it. +pub use crate::STILL_RUNNING; + +/// How long to wait for a request the app says is still running. +#[derive(Clone, Copy, Debug)] +pub struct Waits { + /// How long a call goes on waiting, asking `control.pending`, after the app said its request + /// is still running (the app says so after 60 s), before it says so itself. + pub pending: Duration, + /// How long a later call, or [`Connection::pending`], waits for a request still running from + /// an earlier call. + pub settle: Duration, + /// How often `control.pending` is asked while waiting. + pub poll: Duration, +} + +impl Default for Waits { + fn default() -> Self { + Waits { pending: Duration::from_secs(240), settle: Duration::from_secs(60), poll: Duration::from_millis(500) } + } +} + +/// Why a call has no result. Each holds the message for the caller (an agent reads it). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CallError { + /// Nothing listens at the address: the app is not running, or it quit before the call was + /// sent. The message says how to start it. + NotRunning(String), + /// The app went away or did not answer during the call, after running it or not. + Lost(String), + /// The request is still running in the app after the wait (it starts with [`STILL_RUNNING`]), + /// or an earlier one is and this one was not sent. + StillRunning(String), + /// The app answered with an error (a refused token says which token file it came from). + Refused(String), + /// The reply was not a control reply. + BadReply(String), +} + +impl CallError { + /// The message. + pub fn message(&self) -> &str { + match self { + CallError::NotRunning(m) | CallError::Lost(m) | CallError::StillRunning(m) | CallError::Refused(m) | CallError::BadReply(m) => m, + } + } +} + +impl std::fmt::Display for CallError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message()) + } +} + +impl std::error::Error for CallError {} + +impl From for String { + fn from(e: CallError) -> String { + e.message().to_string() + } +} + +/// A request: its method and parameters, how messages name it and what it does if it is still +/// running when its call returns. +#[derive(Clone, Debug)] +pub struct Request { + pub method: String, + pub params: Value, + /// The request in messages ("`engine.execute` (solid.fillet)"). + pub what: String, + /// What it still does when it finishes after its wait. + pub effect: Effect, + /// Never sent twice: on a fresh connection, waiting up to [`ONCE_TIMEOUT`], and once it is + /// being written any error is final, as the app may have run it. Otherwise a request that + /// finds its connection lost is sent once more on a fresh one (the app may have restarted). + pub once: bool, +} + +impl Request { + /// `method` with `params`, named "`method`" in messages, taken to change the document, sent + /// again once on a fresh connection when its connection was lost. + pub fn new(method: &str, params: Value) -> Self { + Request { method: method.to_string(), params, what: format!("`{method}`"), effect: Effect::Changes, once: false } + } +} + +/// A control reply as a call's result: its `result`, or its `error`. +pub fn outcome(v: &Value) -> Result { + if v.get("ok").and_then(Value::as_bool) == Some(true) { + Ok(v.get("result").cloned().unwrap_or(Value::Null)) + } else { + Err(v.get("error").and_then(Value::as_str).unwrap_or("error").to_string()) + } +} + +/// A request the app said is still running. Its reply will come on this connection, through +/// `control.pending`, so the connection is kept for it. +struct Outstanding { + what: String, + /// What it does when it finishes; `None`: not known (a request this client did not send). + effect: Option, + since: Instant, +} + +impl Outstanding { + fn secs(&self) -> u64 { + self.since.elapsed().as_secs() + } +} + +/// What became of a request still running when its call returned, once a later call waited it +/// out: kept for [`Connection::pending`]. +enum Settled { + Finished { what: String, reply: Value }, + Lost { what: String }, +} + +/// The end of a wait for an outstanding request. +enum Waited { + /// It finished: its own reply. + Done(Value), + /// The app holds nothing (any more) for this connection. + Nothing, + /// Still running when the wait ran out. + Running, +} + +/// A connection to the app listening at an address. It connects when a call needs it, so a +/// client started before the app reaches it once it runs. +pub struct Connection { + app: App, + /// What the app calls its document in messages ("design"). + document: &'static str, + addr: String, + port: u16, + conn: Option<(BufReader, TcpStream)>, + /// Counts the connections made, so a caller can tell a fresh one from the one it asked. + serial: u64, + next_id: u64, + /// The token given, when one was. Otherwise each request looks the token up again, since + /// every launch of the app makes a new one. + explicit_token: Option, + /// A request still running in the app; its reply waits on `conn`. + outstanding: Option, + /// An outstanding request's end, once a later call waited for it (for `pending`). + settled: Option, + waits: Waits, + /// Where a long wait is reported. + progress: fn(&str), +} + +impl Connection { + /// A connection to `app` at `addr` (`host:port`), with `token` when given, else the app's + /// token found the usual way ([`App::client_token`]); not connected yet. + pub fn new(app: App, addr: &str, token: Option<&str>) -> Self { + let port = addr.rsplit(':').next().and_then(|p| p.parse::().ok()).unwrap_or(0); + Self { + app, + document: "design", + addr: addr.to_string(), + port, + conn: None, + serial: 0, + next_id: 1, + explicit_token: token.map(str::to_string), + outstanding: None, + settled: None, + waits: Waits::default(), + progress: |_| {}, + } + } + + /// What the app calls its document in messages ("design", the default). + #[must_use] + pub fn with_document(mut self, document: &'static str) -> Self { + self.document = document; + self + } + + /// Report a long wait for a request still running to `progress`, every 30 s. + #[must_use] + pub fn with_progress(mut self, progress: fn(&str)) -> Self { + self.progress = progress; + self + } + + /// How long to wait for a request still running. + pub fn set_waits(&mut self, waits: Waits) { + self.waits = waits; + } + + /// The address it connects to. + pub fn addr(&self) -> &str { + &self.addr + } + + /// The port of the address (0 when it names none). + pub fn port(&self) -> u16 { + self.port + } + + /// The token given, if one was. + pub fn explicit_token(&self) -> Option<&str> { + self.explicit_token.as_deref() + } + + /// Whether a connection is open now. + pub fn is_connected(&self) -> bool { + self.conn.is_some() + } + + /// The number of connections made so far: the same number while the same connection stays + /// open. + pub fn serial(&self) -> u64 { + self.serial + } + + /// Whether a request of this connection is still running in the app. + pub fn has_outstanding(&self) -> bool { + self.outstanding.is_some() + } + + /// Close the connection. A request still running loses its reply with it. + pub fn disconnect(&mut self) { + self.conn = None; + if let Some(o) = self.outstanding.take() { + self.settled = Some(Settled::Lost { what: o.what }); + } + } + + /// Connect afresh. Failing to reach the app at all is `NotConnected`, so [`Connection::explain`] + /// can tell it from a connection lost during a call. + /// + /// # Errors + /// `NotConnected` when nothing listens at the address (or it does not resolve). + pub fn connect(&mut self) -> std::io::Result<()> { + self.disconnect(); + let not_connected = |e: std::io::Error| std::io::Error::new(std::io::ErrorKind::NotConnected, e); + let mut last = std::io::Error::new(std::io::ErrorKind::NotFound, format!("cannot resolve {}", self.addr)); + for sa in self.addr.to_socket_addrs().map_err(not_connected)? { + match TcpStream::connect_timeout(&sa, CONNECT_TIMEOUT) { + Ok(s) => { + s.set_nodelay(true).ok(); + s.set_read_timeout(Some(READ_TIMEOUT)).ok(); + let read = s.try_clone()?; + self.conn = Some((BufReader::new(read), s)); + self.serial += 1; + return Ok(()); + } + Err(e) => last = e, + } + } + Err(not_connected(last)) + } + + /// See whether the app is there now; the error says why not. The connection doesn't stay + /// open: the app closes one that shows no token within a few seconds, and the first call may + /// be long away. That call connects again. + /// + /// # Errors + /// As [`Connection::connect`]. + pub fn try_connect(&mut self) -> std::io::Result<()> { + self.connect()?; + self.disconnect(); + Ok(()) + } + + /// The error for a failed exchange: whether the app is not running (and how to start it) or + /// went away during the call. + pub fn explain(&self, e: &std::io::Error) -> CallError { + let (addr, port, name, document) = (&self.addr, self.port, self.app.name, self.document); + let start = format!("{} --control {port}", name.to_lowercase()); + if e.kind() == std::io::ErrorKind::NotConnected { + CallError::NotRunning(format!( + "no {name} app is listening at {addr} ({e}): it is not running, or it quit. Start it with `{start}` (or ask the user to); the \ + next call connects to it" + )) + } else { + CallError::Lost(format!( + "the {name} app at {addr} did not answer ({e}): it may have quit or crashed during this call, after running it or not. Inspect \ + the {document} before sending a change again; if the app quit, start it with `{start}`" + )) + } + } + + /// The request line for `method`: a fresh id and the app's token (looked up again for every + /// request: an app restarted mid-session has a new one). + pub fn request_line(&mut self, method: &str, params: Value) -> String { + let id = self.next_id; + self.next_id += 1; + let mut msg = json!({"id": id, "method": method, "params": params}); + if let Some(t) = self.app.client_token(self.port, self.explicit_token.as_deref()) { + msg["token"] = json!(t.token); + } + msg.to_string() + } + + /// One request line out and one reply line back, at most [`MAX_REPLY`] long, connecting + /// first when not connected. Any failure closes the connection (a timed-out read leaves a + /// late reply on the stream). + fn roundtrip(&mut self, line: &str) -> std::io::Result { + if self.conn.is_none() { + self.connect()?; + } + let Some((reader, writer)) = self.conn.as_mut() else { return Err(std::io::Error::other("not connected")) }; + // One write: a request that loses its newline still reaches the app as a request. + let sent = writer.write_all(format!("{line}\n").as_bytes()).and_then(|()| writer.flush()); + if let Err(e) = sent { + self.disconnect(); + return Err(e); + } + let mut reply = String::new(); + let n = match reader.by_ref().take(MAX_REPLY).read_line(&mut reply) { + Ok(n) => n, + Err(e) => { + self.disconnect(); + return Err(e); + } + }; + if n == 0 || !reply.ends_with('\n') { + self.disconnect(); + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "connection closed")); + } + Ok(reply) + } + + fn set_read_timeout(&self, t: Duration) { + if let Some((_, writer)) = &self.conn { + writer.set_read_timeout(Some(t)).ok(); + } + } + + /// A reply line as JSON. + fn parse(reply: &str) -> Result { + serde_json::from_str(reply).map_err(|e| CallError::BadReply(format!("bad reply from the app: {e}"))) + } + + /// A control reply as a call's result; a refused token says which token file it came from + /// (and whether it is stale). + /// + /// # Errors + /// [`CallError::Refused`] with the app's error. + pub fn outcome(&self, v: &Value) -> Result { + outcome(v).map_err(|error| { + CallError::Refused( + self.app.client_token(self.port, self.explicit_token.as_deref()).map_or_else(|| error.clone(), |t| t.explain_error(&error)), + ) + }) + } + + /// `method` with `params`, at once and once: not behind a request still running on this + /// connection, no second try. The app's whole reply (`ok`, `result` or `error`, and anything + /// it adds), for a request the app answers at once on any connection + /// (`control.capabilities`, `control.running`). + /// + /// # Errors + /// As [`Connection::explain`]; [`CallError::BadReply`]. + pub fn ask(&mut self, method: &str, params: Value) -> Result { + let line = self.request_line(method, params); + let reply = self.roundtrip(&line).map_err(|e| self.explain(&e))?; + Self::parse(&reply) + } + + /// `method` with `params`, waiting at most `timeout` for the reply and never behind a request + /// still running: for a request the app answers within a frame, asked again later when the + /// app is busy. One try. + /// + /// # Errors + /// [`CallError::StillRunning`] when a request of this connection is still running; as + /// [`Connection::ask`] and [`Connection::outcome`]. + pub fn quick(&mut self, method: &str, params: Value, timeout: Duration) -> Result { + if self.outstanding.is_some() { + return Err(CallError::StillRunning("a request is still running in the app".into())); + } + if self.conn.is_none() { + self.connect().map_err(|e| self.explain(&e))?; + } + let line = self.request_line(method, params); + self.set_read_timeout(timeout); + let reply = self.roundtrip(&line); + self.set_read_timeout(READ_TIMEOUT); + let reply = reply.map_err(|e| self.explain(&e))?; + let v = Self::parse(&reply)?; + self.outcome(&v) + } + + /// Ask the app, on this connection only, about the request it still runs for it. + fn ask_pending(&mut self) -> Result { + if self.conn.is_none() { + self.disconnect(); + return Err(CallError::Lost("the connection to the app was lost".into())); + } + let v = self.ask("control.pending", json!({}))?; + outcome(&v).map_err(CallError::Refused).inspect_err(|_| self.disconnect()) + } + + /// `control.cancel` for the request still running on this connection. Sent at once, without + /// waiting for that request first, and never on a fresh connection, which has nothing to + /// cancel. Once the app no longer holds it (removed, stopped or finished), later calls don't + /// wait for it. + /// + /// # Errors + /// As [`Connection::ask`]; [`CallError::Refused`]. + pub fn cancel(&mut self, params: Value) -> Result { + if self.conn.is_none() { + return Ok(json!({"cancelled": null, "note": "nothing to cancel: no request of this client is running in the app"})); + } + let v = self.ask("control.cancel", params)?; + let r = outcome(&v).map_err(CallError::Refused); + if r.is_ok() { + self.outstanding = None; + } + r + } + + /// Wait up to `limit` for the outstanding request, asking `control.pending` now and then and + /// reporting how long it takes. An error: the connection, and so the reply, was lost. + fn wait_outstanding(&mut self, limit: Duration) -> Result { + let start = Instant::now(); + let mut told = start; + loop { + let Some(o) = &self.outstanding else { return Ok(Waited::Nothing) }; + let what = o.what.clone(); + let r = self.ask_pending()?; + if r.get("pending").is_none_or(Value::is_null) { + self.outstanding = None; + return Ok(Waited::Nothing); + } + if r.get("done").and_then(Value::as_bool) == Some(true) { + self.outstanding = None; + return Ok(Waited::Done(r.get("reply").cloned().unwrap_or(Value::Null))); + } + let waited = start.elapsed(); + if waited >= limit { + return Ok(Waited::Running); + } + if told.elapsed() >= PROGRESS_EVERY { + let secs = self.outstanding.as_ref().map_or(0, Outstanding::secs); + (self.progress)(&format!("{what} is still running in the app ({secs} s); waiting for it")); + told = Instant::now(); + } + std::thread::sleep(self.waits.poll.min(limit.saturating_sub(waited))); + } + } + + /// The error for the outstanding request still running after its wait. + fn still_running(&self, earlier: bool) -> CallError { + let Some(o) = &self.outstanding else { return CallError::StillRunning(format!("{STILL_RUNNING} a request is still running in the app")) }; + let document = self.document; + let effect = + o.effect.map_or_else(|| format!("it was not cancelled and may still change the {document} when it finishes"), |e| e.when_done(document)); + let settle = self.waits.settle.as_secs(); + CallError::StillRunning(if earlier { + format!( + "{STILL_RUNNING} an earlier request, {}, is still running in the app ({} s; {effect}), so this call was not sent. Call the \ + `pending` tool to wait for its result, then send this call again", + o.what, + o.secs() + ) + } else { + format!( + "{STILL_RUNNING} {} is still running in the app after {} s ({effect}); don't send it again. Call the `pending` tool to \ + wait for its result (up to {settle} s per call); other calls wait for it first", + o.what, + o.secs() + ) + }) + } + + /// Before a new request: wait for one still running from an earlier call, keeping its end for + /// `pending`. Still running after the wait: the new request is not sent. + fn settle_earlier(&mut self) -> Result<(), CallError> { + let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return Ok(()) }; + match self.wait_outstanding(self.waits.settle) { + Ok(Waited::Done(reply)) => self.settled = Some(Settled::Finished { what, reply }), + Ok(Waited::Nothing) => {} + Ok(Waited::Running) => return Err(self.still_running(true)), + // The connection was lost, and the earlier reply with it (`pending` says so): the new + // request goes on a fresh connection. + Err(_) => {} + } + Ok(()) + } + + /// `control.pending` for the client: the end of the request still running from an earlier + /// call, waiting for it up to the settle wait: `{pending: what | null, done?, reply?, lost?, + /// running_s?}`. Never sent on a fresh connection, which has nothing pending. + pub fn pending(&mut self) -> Value { + if let Some(s) = self.settled.take() { + return match s { + Settled::Finished { what, reply } => json!({"pending": what, "done": true, "reply": reply}), + Settled::Lost { what } => json!({"pending": what, "lost": true}), + }; + } + let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return json!({"pending": null}) }; + match self.wait_outstanding(self.waits.settle) { + Ok(Waited::Done(reply)) => json!({"pending": what, "done": true, "reply": reply}), + Ok(Waited::Nothing) => json!({"pending": null}), + Ok(Waited::Running) => json!({"pending": what, "done": false, "running_s": self.outstanding.as_ref().map_or(0, Outstanding::secs)}), + Err(_) => { + self.settled = None; + json!({"pending": what, "lost": true}) + } + } + } + + /// Send `request` and wait for its reply: the app's whole reply (`ok`, `result` or `error`, + /// and anything it adds). First a request still running from an earlier call is waited for + /// (except for `control.running`, which the app answers at once); a request the app says is + /// still running is waited for through `control.pending`; a request refused unrun as busy + /// behind one this client did not know of is sent once more after that one, when + /// `retry_busy`. + /// + /// # Errors + /// As [`Connection::explain`]: a request whose connection is lost is sent once more on a fresh + /// one (unless [`Request::once`]), and when nothing listens any more the app went away, maybe + /// while running it ([`CallError::Lost`]); [`CallError::StillRunning`]; [`CallError::BadReply`]. + pub fn exchange(&mut self, request: Request, retry_busy: bool) -> Result { + let Request { method, params, what, effect, once } = request; + if method != "control.running" { + self.settle_earlier()?; + } + let again = retry_busy.then(|| params.clone()); + let sent = Instant::now(); + let line = self.request_line(&method, params); + let reply = if once { + // A fresh connection (an idle one may have been closed by an app restart; connecting + // again sends nothing). + self.connect().or_else(|_| self.connect()).and_then(|()| { + self.set_read_timeout(ONCE_TIMEOUT); + let r = self.roundtrip(&line); + self.set_read_timeout(READ_TIMEOUT); + r + }) + } else { + self.roundtrip(&line).or_else(|lost| { + self.roundtrip(&line).map_err(|e| if e.kind() == std::io::ErrorKind::NotConnected && lost.kind() != e.kind() { lost } else { e }) + }) + } + .map_err(|e| self.explain(&e))?; + let v = Self::parse(&reply)?; + // Timed out in the app, still running: wait for its result on this connection. + if v.get("running").and_then(Value::as_bool) == Some(true) { + self.outstanding = Some(Outstanding { what: what.clone(), effect: Some(effect), since: sent }); + (self.progress)(&format!( + "{what} is still running in the app after {} s; waiting up to {} s more for it", + sent.elapsed().as_secs(), + self.waits.pending.as_secs() + )); + let document = self.document; + return match self.wait_outstanding(self.waits.pending)? { + Waited::Done(reply) => Ok(reply), + Waited::Nothing => Err(CallError::Lost(format!( + "{what} timed out in the app, which kept no result for it: inspect the {document} before sending it again" + ))), + Waited::Running => Err(self.still_running(false)), + }; + } + // Refused unrun behind a request this client did not know of: wait for that one, then + // send this one once more. + if v.get("busy").and_then(Value::as_bool) == Some(true) + && let Some(params) = again + { + self.outstanding = Some(Outstanding { what: "an earlier request".into(), effect: None, since: Instant::now() }); + self.settle_earlier()?; + return self.exchange(Request { method, params, what, effect, once }, false); + } + Ok(v) + } + + /// [`Connection::exchange`] (retrying a busy refusal once) and the reply's result. + /// + /// # Errors + /// As [`Connection::exchange`] and [`Connection::outcome`]. + pub fn call(&mut self, request: Request) -> Result { + let v = self.exchange(request, true)?; + self.outcome(&v) + } +} + +#[cfg(test)] +#[path = "client_tests.rs"] +mod tests; diff --git a/crates/control/src/client_tests.rs b/crates/control/src/client_tests.rs new file mode 100644 index 000000000..f5af19a00 --- /dev/null +++ b/crates/control/src/client_tests.rs @@ -0,0 +1,190 @@ +//! The client against the shared server ([`crate::server::Hub`]) on an ephemeral loopback port. + +use super::*; +use crate::ControlRequest; +use crate::server::{Hub, Protocol, Stop, Summary, Waits as ServerWaits}; +use std::ffi::OsString; +use std::net::TcpListener; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::mpsc::channel; + +const TOKEN: &str = "0123456789abcdef0123456789abcdef"; + +fn no_dir() -> Option { + None +} + +fn no_env(_: &str) -> Option { + None +} + +fn utc(secs: u64) -> String { + secs.to_string() +} + +/// An app with no token files and no environment: a client sends only the token it is given. +const APP: App = App { name: "Probe", data_dir: no_dir, env: no_env, utc, default_port: 0 }; + +/// An app whose requests all may change its document. +struct Probe; + +impl Protocol for Probe { + fn summary(&self, method: &str, _: &Value) -> Summary { + Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false } + } + fn document(&self) -> &'static str { + "design" + } + fn change_event(&self) -> &'static str { + "design.changed" + } + fn revision_method(&self) -> &'static str { + "document.revision" + } + fn docs(&self) -> &'static str { + "docs/control-protocol.md" + } + fn capabilities(&self) -> Value { + json!({}) + } +} + +/// The app on `port` (0: an ephemeral one) behind [`TOKEN`]: its window answers each request after +/// `delay` with the request's method and parameters, and the server says a request is still +/// running after `wait`. +fn serve(port: u16, delay: Duration, wait: Duration) -> Stop { + // A port just closed may take a moment to be free again on a loaded machine. + let started = Instant::now(); + let mut bound = TcpListener::bind(("127.0.0.1", port)); + while bound.is_err() && started.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(10)); + bound = TcpListener::bind(("127.0.0.1", port)); + } + let listener = bound.unwrap(); + let (tx, rx) = channel::(); + std::thread::spawn(move || { + for req in rx { + std::thread::sleep(delay); + let _ = req.reply.send(json!({"ok": true, "result": {"method": req.method, "params": req.params}})); + } + }); + let waits = ServerWaits { short: wait, long: wait, token: Duration::from_secs(5) }; + let hub = Arc::new(Hub::new(Probe, || {}, 1 << 20).with_waits(waits)); + hub.listen_stoppable(listener, TOKEN.into(), tx, false).unwrap() +} + +fn addr(port: u16) -> String { + format!("127.0.0.1:{port}") +} + +/// A port nothing listens on. +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() +} + +#[test] +fn a_call_is_one_round_trip_with_the_token() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + assert!(!c.is_connected()); + let r = c.call(Request::new("document.inspect", json!({"a": 1}))).unwrap(); + assert_eq!(r, json!({"method": "document.inspect", "params": {"a": 1}})); + assert!(c.is_connected()); + let serial = c.serial(); + // the next call goes on the same connection + c.call(Request::new("document.inspect", json!({}))).unwrap(); + assert_eq!(c.serial(), serial); + // `try_connect` sees the app there and leaves no connection open + let mut fresh = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + fresh.try_connect().unwrap(); + assert!(!fresh.is_connected()); +} + +#[test] +fn a_wrong_token_or_none_is_refused() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + for token in [Some("fedcba9876543210fedcba9876543210"), None] { + let mut c = Connection::new(APP, &addr(app.port()), token); + match c.call(Request::new("document.inspect", json!({}))) { + Err(CallError::Refused(e)) => assert!(e.starts_with("unauthorized"), "{token:?}: {e}"), + other => panic!("{token:?}: {other:?}"), + } + } +} + +#[test] +fn an_app_not_running_says_how_to_start_it() { + let port = free_port(); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + let CallError::NotRunning(m) = &e else { panic!("{e:?}") }; + assert!(m.starts_with(&format!("no Probe app is listening at 127.0.0.1:{port}")), "{m}"); + assert!(m.contains(&format!("`probe --control {port}`")), "{m}"); + assert_eq!(c.try_connect().unwrap_err().kind(), std::io::ErrorKind::NotConnected); +} + +/// An app that goes away during a call is lost, not "not running"; the next call says it is not +/// running, and once it runs again the next call reaches it. +#[test] +fn an_app_gone_during_a_call_is_lost_and_reached_again_once_back() { + let app = serve(0, Duration::ZERO, Duration::from_secs(10)); + let port = app.port(); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + c.call(Request::new("document.inspect", json!({}))).unwrap(); + app.stop(); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + let CallError::Lost(m) = &e else { panic!("{e:?}") }; + assert!(m.contains("may have quit") && m.contains("Inspect the design") && m.contains(&format!("`probe --control {port}`")), "{m}"); + assert!(matches!(c.call(Request::new("document.inspect", json!({}))), Err(CallError::NotRunning(_)))); + let _back = serve(port, Duration::ZERO, Duration::from_secs(10)); + assert_eq!(c.call(Request::new("ui.view", json!({}))).unwrap()["method"], json!("ui.view")); +} + +/// A server that reads a request and closes the connection without a reply: the call is lost +/// (after one more try on a fresh connection), and the reply is never taken for "not running". +#[test] +fn a_connection_closed_without_a_reply_is_lost() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let mut line = String::new(); + let _ = BufReader::new(stream).read_line(&mut line); + } + }); + let mut c = Connection::new(APP, &addr(port), Some(TOKEN)); + let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err(); + assert!(matches!(e, CallError::Lost(_)), "{e:?}"); + assert!(!c.is_connected()); +} + +/// A request the app says is still running is waited for through `control.pending`, on the same +/// connection, and its own reply is the call's result. +#[test] +fn a_request_still_running_is_waited_for_through_pending() { + let app = serve(0, Duration::from_millis(400), Duration::from_millis(50)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + c.set_waits(Waits { pending: Duration::from_secs(10), settle: Duration::from_secs(10), poll: Duration::from_millis(5) }); + let r = c.call(Request::new("solid.slow", json!({"n": 2}))).unwrap(); + assert_eq!(r, json!({"method": "solid.slow", "params": {"n": 2}})); + assert!(!c.has_outstanding()); + assert_eq!(c.pending(), json!({"pending": null})); +} + +/// Past the wait, the call says the request is still running, and `pending` later has its end. +#[test] +fn a_request_still_running_past_the_wait_is_reported_and_its_end_kept() { + let app = serve(0, Duration::from_millis(400), Duration::from_millis(20)); + let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN)); + c.set_waits(Waits { pending: Duration::from_millis(30), settle: Duration::from_secs(10), poll: Duration::from_millis(5) }); + let mut slow = Request::new("solid.slow", json!({})); + slow.what = "`solid.slow` (now)".into(); + let e = c.call(slow).unwrap_err(); + let CallError::StillRunning(m) = &e else { panic!("{e:?}") }; + assert!(m.starts_with(STILL_RUNNING) && m.contains("`solid.slow` (now)") && m.contains("will still apply"), "{m}"); + assert!(c.has_outstanding()); + let end = c.pending(); + assert_eq!(end["done"], json!(true), "{end}"); + assert_eq!(end["reply"]["result"]["method"], json!("solid.slow"), "{end}"); +} diff --git a/crates/control/src/effect.rs b/crates/control/src/effect.rs new file mode 100644 index 000000000..aed5cbe59 --- /dev/null +++ b/crates/control/src/effect.rs @@ -0,0 +1,27 @@ +//! What a client is told about a control request that outlasts its wait (docs/control-protocol.md, +//! #1618): the app's control server and a bridged client both say what the request still does when +//! it finishes. Which requests do what is the app's to say. + +/// What a request still running does when it finishes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Effect { + /// It only reads (the document, the window or the command list): nothing changes. + ReadsOnly, + /// It writes a file (an export, a save, a render to a path); the document stays as it is. + WritesFile, + /// It changes the document (or may: anything not known to read or only write a file). + Changes, +} + +impl Effect { + /// What happens when the request finishes, for a message: "it only reads: nothing changes", + /// "it was not cancelled and will still write the file when it finishes; the design does not + /// change", … `document` is what the app calls its document ("design"). + pub fn when_done(self, document: &str) -> String { + match self { + Effect::ReadsOnly => "it only reads: nothing changes".into(), + Effect::WritesFile => format!("it was not cancelled and will still write the file when it finishes; the {document} does not change"), + Effect::Changes => "it was not cancelled and will still apply when it finishes".into(), + } + } +} diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs new file mode 100644 index 000000000..26727a97c --- /dev/null +++ b/crates/control/src/lib.rs @@ -0,0 +1,26 @@ +//! The control channel any app may share (docs/control-protocol.md): the per-launch token and its +//! files ([`auth`]), the loopback server that admits a connection by the token and hands each +//! request to the window ([`server`]), the client that talks to it ([`client`]), the request the +//! window answers ([`ControlRequest`]), the document watch subscribed connections wait on +//! ([`DocumentWatch`]), and what a request still running does when it finishes ([`Effect`]). It +//! needs no UI toolkit; the window's own requests over egui (`ui.screenshot`, `ui.resize`) are +//! `craft-control-egui`. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +pub mod auth; +mod cancel; +#[cfg(not(target_arch = "wasm32"))] +pub mod client; +mod effect; +mod request; +#[cfg(not(target_arch = "wasm32"))] +pub mod server; + +pub use cancel::{Cancel, Ticket}; +pub use effect::Effect; +pub use request::{ControlRequest, ControlResponse, DOCUMENT_WATCH, DocumentWatch, Subscriber, cancelled_chat_response, chat_turn_cancelled}; + +/// How an error from the client (`client`, native only) starts when a request is still running +/// in the app after the wait. Outside the client so callers on every target can test for it. +pub const STILL_RUNNING: &str = "still running:"; diff --git a/crates/control/src/request.rs b/crates/control/src/request.rs new file mode 100644 index 000000000..b97c90b76 --- /dev/null +++ b/crates/control/src/request.rs @@ -0,0 +1,125 @@ +//! A control request as the window gets it, and the design watch subscribed connections wait on. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::Sender; +use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use serde_json::{Value, json}; + +pub type ControlResponse = Value; + +pub struct ControlRequest { + pub method: String, + pub params: Value, + pub reply: Sender, + /// Set once the window starts on it; until then it waits its turn (`control.running`, #3654). + pub started: std::sync::Arc, + /// Chat requests already admitted by the gate may outlive a tab switch. This turn's + /// validity follows them through queues, computation and cursor animation. + pub chat_turn: Option>, + /// Where the request is, shared with the connection that sent it so it can cancel it + /// (`control.cancel`, #1989). + pub ticket: std::sync::Arc, + /// Told apart from every other request of this run: names its row while it waits. + pub serial: u64, +} + +impl ControlRequest { + pub fn new(method: impl Into, params: Value) -> (Self, std::sync::mpsc::Receiver) { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let (tx, rx) = std::sync::mpsc::channel(); + let serial = NEXT.fetch_add(1, Ordering::Relaxed); + (Self { method: method.into(), params, reply: tx, started: Default::default(), chat_turn: None, ticket: Default::default(), serial }, rx) + } + + /// Answer a request whose agent chat turn has ended, and say so: such a request is not run. + pub fn refuse_cancelled_chat(&self) -> bool { + if chat_turn_cancelled(self.chat_turn.as_deref()) { + let _ = self.reply.send(cancelled_chat_response()); + true + } else { + false + } + } +} + +/// Whether the agent chat turn a request belongs to has ended (`None`: it belongs to none). +pub fn chat_turn_cancelled(turn: Option<&std::sync::atomic::AtomicBool>) -> bool { + turn.is_some_and(|active| !active.load(std::sync::atomic::Ordering::Relaxed)) +} + +/// The reply to a request whose agent chat turn has ended. +pub fn cancelled_chat_response() -> Value { + json!({"ok": false, "error": "not run: the agent chat turn ended or its document tab was left"}) +} + +/// The active design's id and revision for control connections that sent `control.subscribe` +/// (#6985): the window publishes them each frame while one is subscribed, and each such +/// connection waits for them to move, then sends the app's change event (`Protocol::change_event`). +#[derive(Default)] +pub struct DocumentWatch { + subscribers: AtomicUsize, + now: Mutex>, + moved: Condvar, +} + +/// The app's [`DocumentWatch`]. +pub static DOCUMENT_WATCH: DocumentWatch = DocumentWatch::new(); + +impl DocumentWatch { + pub const fn new() -> Self { + DocumentWatch { subscribers: AtomicUsize::new(0), now: Mutex::new(None), moved: Condvar::new() } + } + + fn now(&self) -> MutexGuard<'_, Option<(String, u64)>> { + self.now.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The design as last published (`None` before anything was). + pub fn current(&self) -> Option<(String, u64)> { + self.now().clone() + } + + /// The design as it is now; waiters wake only when it moved. Only the latest is kept, so + /// changes made faster than a connection sends them go out as one. + pub fn publish(&self, document_id: &str, revision: u64) { + let mut now = self.now(); + if now.as_ref().is_some_and(|(id, r)| id == document_id && *r == revision) { + return; + } + *now = Some((document_id.to_string(), revision)); + self.moved.notify_all(); + } + + /// Whether a connection is subscribed: the window publishes only then. + pub fn subscribed(&self) -> bool { + self.subscribers.load(Ordering::Relaxed) > 0 + } + + /// Count a subscribed connection until the returned guard is dropped. + pub fn join(&'static self) -> Subscriber { + self.subscribers.fetch_add(1, Ordering::Relaxed); + Subscriber(self) + } + + /// Wait up to `wait` for the design to differ from `seen`, or for `stop`; the design then + /// (`None` before anything was published). + pub fn wait(&self, seen: &(String, u64), stop: &AtomicBool, wait: Duration) -> Option<(String, u64)> { + let unmoved = |now: &mut Option<(String, u64)>| now.as_ref() == Some(seen) && !stop.load(Ordering::Relaxed); + let (now, _) = self.moved.wait_timeout_while(self.now(), wait, unmoved).unwrap_or_else(PoisonError::into_inner); + now.clone() + } +} + +/// A subscribed connection, counted by its [`DocumentWatch`] while this lives. +pub struct Subscriber(&'static DocumentWatch); + +impl Drop for Subscriber { + fn drop(&mut self) { + self.0.subscribers.fetch_sub(1, Ordering::Relaxed); + // Under the lock, so a connection about to wait sees its `stop` set before it does. + let _now = self.0.now(); + self.0.moved.notify_all(); + } +} diff --git a/crates/control/src/server.rs b/crates/control/src/server.rs new file mode 100644 index 000000000..85664a1aa --- /dev/null +++ b/crates/control/src/server.rs @@ -0,0 +1,1433 @@ +//! The loopback JSON-lines control server: one request per line, one reply per line, behind the +//! launch's token (docs/control-protocol.md). Agents and tests drive an app through it (adapted +//! from CADCraft, MIT OR Apache-2.0). What the app's requests mean is the app's [`Protocol`]; the +//! window answers them from the [`ControlRequest`]s it is sent. + +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::Effect; +use crate::auth::tokens_match; +use crate::cancel::{Cancel, Ticket}; +use crate::request::{ControlRequest, DOCUMENT_WATCH, DocumentWatch, Subscriber}; + +/// What the server needs to know of a request before it sends it to the window: how to name it +/// in messages, what it does if it outlasts its wait, and what `control.running` shows of it. +#[derive(Clone, Debug)] +pub struct Summary { + /// The request in messages ("`engine.execute` (solid.fillet)"). + pub what: String, + /// What it still does when it finishes after its wait. + pub effect: Effect, + /// The command it runs, for `control.running` (never its parameters). + pub command: Option<&'static str>, + /// The number of steps it runs, for `control.running`. + pub steps: Option, + /// It may take as long as a batch. + pub long: bool, +} + +/// What an app's requests mean to its control server. +pub trait Protocol: Send + Sync { + fn summary(&self, method: &str, params: &Value) -> Summary; + /// What the app calls its document in messages ("design"). + fn document(&self) -> &'static str; + /// The event a subscribed connection is sent when the document moves ("design.changed"). + fn change_event(&self) -> &'static str; + /// The method that answers the document's id and revision now ("document.revision"). + fn revision_method(&self) -> &'static str; + /// Where a client reads how to talk to the channel, named when its token is refused + /// ("docs/control-protocol.md"). + fn docs(&self) -> &'static str; + /// The `control.capabilities` result: what this app's channel does beyond the methods every + /// version has, for a client to find out before it relies on it (an older app answers + /// `control.capabilities` with "unknown method"). + fn capabilities(&self) -> Value; +} + +/// An app's control server: its [`Protocol`], how it wakes the window for a request, and the +/// requests every connection of its listeners has the window running (`control.running`). +pub struct Hub { + protocol: Box, + wake: Box, + /// Longest request line accepted. + max_line: usize, + waits: Waits, + watch: &'static DocumentWatch, + running: Arc, + open: AtomicUsize, +} + +impl Hub { + /// A server for `protocol`'s requests; `wake` makes the window look at its requests now. + pub fn new(protocol: impl Protocol + 'static, wake: impl Fn() + Send + Sync + 'static, max_line: usize) -> Self { + Hub { + protocol: Box::new(protocol), + wake: Box::new(wake), + max_line, + waits: Waits::default(), + watch: &DOCUMENT_WATCH, + running: Arc::new(Running::new()), + open: AtomicUsize::new(0), + } + } + + /// Other waits (tests). + pub fn with_waits(mut self, waits: Waits) -> Self { + self.waits = waits; + self + } + + /// Another document watch than the app's [`DOCUMENT_WATCH`] (tests). + pub fn with_watch(mut self, watch: &'static DocumentWatch) -> Self { + self.watch = watch; + self + } + + /// Serve `listener` with `token` on threads of their own, at most [`MAX_CONNECTIONS`] + /// connections at once. `patient`: every request may wait as long as a batch (a person decides + /// on it). A listener that is not on a loopback address is refused: the channel never goes on + /// the network. + pub fn listen(self: &Arc, listener: TcpListener, token: String, requests: Sender, patient: bool) -> std::io::Result<()> { + self.listen_stoppable(listener, token, requests, patient).map(drop) + } + + /// [`Hub::listen`], and a [`Stop`] that closes the listener and the connections it accepted + /// (a channel turned off in the app's settings). Dropping the [`Stop`] leaves it serving. + pub fn listen_stoppable( + self: &Arc, + listener: TcpListener, + token: String, + requests: Sender, + patient: bool, + ) -> std::io::Result { + let addr = listener.local_addr()?; + if !addr.ip().is_loopback() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("the control channel listens on loopback only, not {addr}"))); + } + let hub = Arc::clone(self); + let token = Arc::new(token); + let shared = Arc::new(Accepted::default()); + let (closed_tx, closed) = channel::<()>(); + let stop = Stop { addr, shared: Arc::clone(&shared), closed }; + std::thread::spawn(move || { + // Dropped when this thread ends, with the listener: `Stop::stop` waits for that. + let _closed = closed_tx; + for stream in listener.incoming().flatten() { + if shared.stopped.load(Ordering::SeqCst) { + break; + } + if hub.open.fetch_add(1, Ordering::Relaxed) >= MAX_CONNECTIONS { + hub.open.fetch_sub(1, Ordering::Relaxed); + let mut refused = stream; + let _ = send(&mut refused, &json!({"ok": false, "error": "too many connections"})); + continue; + } + let id = shared.next.fetch_add(1, Ordering::Relaxed); + if let Ok(copy) = stream.try_clone() { + shared.lock().push((id, copy)); + } + // Stopped while this one was being accepted: it is closed with the rest. + if shared.stopped.load(Ordering::SeqCst) { + shared.close_all(); + } + let (hub, tx, token, shared) = (Arc::clone(&hub), requests.clone(), Arc::clone(&token), Arc::clone(&shared)); + std::thread::spawn(move || { + hub.serve(stream, &token, tx, patient); + hub.open.fetch_sub(1, Ordering::Relaxed); + shared.lock().retain(|(i, _)| *i != id); + }); + } + // A captured value drops after the body's locals, so without this `_closed` would tell + // `Stop::stop` the port is free while the listener still holds it (#14752). + drop(listener); + }); + Ok(stop) + } +} + +/// The connections a stoppable listener accepted, so stopping it can close them too. +#[derive(Default)] +struct Accepted { + stopped: AtomicBool, + next: AtomicU64, + streams: Mutex>, +} + +impl Accepted { + fn lock(&self) -> MutexGuard<'_, Vec<(u64, TcpStream)>> { + self.streams.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn close_all(&self) { + for (_, stream) in self.lock().drain(..) { + let _ = stream.shutdown(std::net::Shutdown::Both); + } + } +} + +/// Stops a listener started with [`Hub::listen_stoppable`]. +pub struct Stop { + addr: std::net::SocketAddr, + shared: Arc, + closed: Receiver<()>, +} + +impl Stop { + /// The port the listener is on. + pub fn port(&self) -> u16 { + self.addr.port() + } + + /// Stop accepting, close the port and end every connection it accepted. Returns once the + /// port is closed (at most a couple of seconds), so the same port can be bound again at once. + pub fn stop(self) { + self.shared.stopped.store(true, Ordering::SeqCst); + // The listener waits in `accept`: a connection of our own wakes it to see the flag. + let _ = TcpStream::connect_timeout(&self.addr, Duration::from_secs(1)); + let _ = self.closed.recv_timeout(Duration::from_secs(2)); + self.shared.close_all(); + } +} + +/// Connections served at once, so a flood must not be able to spawn threads without end. +pub const MAX_CONNECTIONS: usize = 16; +/// Lines that are not JSON before the connection is closed, so garbage cannot loop forever. +const MAX_BAD_LINES: usize = 8; +/// How long a subscribed connection waits after a `design.changed` line before the next: what +/// changes meanwhile (a parameter dragged) goes out as one line, the latest (#6985). +const EVENT_GAP: Duration = Duration::from_millis(100); +/// How long a new connection has to show the token. A client connects when it has a request to +/// send, so this is plenty; one that doesn't send it loses the connection, and its seat among +/// [`MAX_CONNECTIONS`] goes to someone else. +const TOKEN_WAIT: Duration = Duration::from_secs(5); + +/// A new random token: 32 bytes from the operating system, as hex. +pub fn new_token() -> Option { + let mut bytes = [0u8; 32]; + getrandom::fill(&mut bytes).ok()?; + Some(bytes.iter().map(|b| format!("{b:02x}")).collect()) +} + +/// One line without its newline, or `None` at the end of the stream. A client that sends +/// gigabytes with no newline would otherwise make this thread allocate until the process is +/// killed, so the line is refused as soon as it passes `max`. +fn read_line(r: &mut impl BufRead, max: usize) -> std::io::Result>> { + let mut line = Vec::new(); + loop { + let (done, used) = { + let available = r.fill_buf()?; + if available.is_empty() { + return Ok((!line.is_empty()).then_some(line)); + } + match available.iter().position(|b| *b == b'\n') { + Some(i) => { + line.extend_from_slice(available.get(..i).unwrap_or(&[])); + (true, i + 1) + } + None => { + line.extend_from_slice(available); + (false, available.len()) + } + } + }; + r.consume(used); + if done { + return Ok(Some(line)); + } + if line.len() > max { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "request too large")); + } + } +} + +/// A reply line, written to the socket at once. `writeln!` with a JSON value hands the socket each +/// of its pieces apart, a system call for every string, number and comma: half a second for a +/// design's component tree when it has thousands of components. +fn send(out: &mut impl Write, reply: &Value) -> std::io::Result<()> { + let mut line = serde_json::to_vec(reply).map_err(std::io::Error::other)?; + line.push(b'\n'); + out.write_all(&line) +} + +/// The `"token"` of a request line that is a JSON object, read without building the rest of it. +/// Until a connection shows the token, a line costs only its own bytes: parsed whole, a 16 MiB +/// line of `[0,0,…]` is hundreds of MB, and 16 such connections could exhaust memory before any +/// token was checked. +fn token_of(line: &str) -> Option { + struct Token; + impl<'de> serde::de::Visitor<'de> for Token { + type Value = Option; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a request object") + } + fn visit_map>(self, mut map: A) -> Result { + let mut token = None; + while let Some(key) = map.next_key::()? { + if key == "token" { + token = Some(map.next_value::()?); + } else { + map.next_value::()?; + } + } + Ok(token) + } + } + let mut de = serde_json::Deserializer::from_str(line); + let token = serde::Deserializer::deserialize_map(&mut de, Token).ok()?; + de.end().ok()?; + token +} + +/// The reply to a connection that didn't show the token, naming the app's `docs`; the +/// connection is then closed. +fn unauthorized(docs: &str) -> Value { + json!({"ok": false, "error": format!("unauthorized: send the control token as \"token\" (see {docs})")}) +} + +/// Refuse the connection: the reply, then the end of what we send, then what the client had +/// already sent read and dropped (a little, briefly). Closing a socket with unread input resets +/// the connection, and a reset can reach the client before it has read the refusal. +fn refuse(out: &mut TcpStream, reader: &mut BufReader, docs: &str) { + use std::io::Read; + let _ = send(out, &unauthorized(docs)); + let _ = out.shutdown(std::net::Shutdown::Write); + reader.get_mut().deadline = Some(Instant::now() + Duration::from_millis(500)); + let _ = std::io::copy(&mut reader.by_ref().take(64 * 1024), &mut std::io::sink()); +} + +/// What a client sends. Until it shows the token, all of its reads share one deadline, so neither +/// blank lines nor a line sent a byte at a time buy it more time. After that it may stay idle as +/// long as it likes. +struct ClientInput { + socket: TcpStream, + deadline: Option, +} + +impl ClientInput { + /// No deadline any more: the client showed the token. + fn admit(&mut self) -> std::io::Result<()> { + self.deadline = None; + self.socket.set_read_timeout(None) + } +} + +impl std::io::Read for ClientInput { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(deadline) = self.deadline { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(std::io::ErrorKind::TimedOut.into()); + } + self.socket.set_read_timeout(Some(left))?; + } + self.socket.read(buf) + } +} + +/// How long a request waits for the app's reply: most requests, and a batch (or every request, +/// on a patient connection); and how long a new connection has to show the token. +#[derive(Clone, Copy, Debug)] +pub struct Waits { + pub short: Duration, + pub long: Duration, + pub token: Duration, +} + +impl Default for Waits { + fn default() -> Self { + Waits { short: Duration::from_secs(60), long: Duration::from_secs(3600), token: TOKEN_WAIT } + } +} + +/// A request that timed out but is still running in the app (#1618): its id and method, and where +/// its reply will land. +struct Pending { + id: Value, + method: String, + /// The request in messages ("`engine.execute` (solid.fillet)"). + what: String, + reply: Receiver, + /// Where it is in the app, to cancel it (`control.cancel`). + job: Arc, +} + +/// `control.cancel {id?}`: cancel the request this connection still waits for (the one +/// `control.pending` reports; `id`, when given, must name it). A connection can only cancel its +/// own requests. Waiting its turn, it is removed; computing off the window thread, it is stopped +/// as the app's Stop stops it. Either way nothing of it is applied and the connection is free. +/// One that runs on the window thread can't be cancelled; one that finished is answered. +fn cancel(pending: &mut Option, late: Option, id: Option<&Value>, document: &str) -> Value { + let id = id.filter(|v| !v.is_null()); + let mine = pending.as_ref().filter(|p| id.is_none_or(|id| *id == p.id)); + let Some(p) = mine else { + return match id { + None => json!({"ok": true, "result": {"cancelled": null, "note": "nothing to cancel: no request of this connection is running"}}), + Some(id) => json!({"ok": false, "error": format!( + "nothing to cancel: request {} is not running on this connection (a connection can only cancel its own requests)", + id.to_string().chars().take(64).collect::() + )}), + }; + }; + let (what, id) = (format!("request {} ({})", p.id, p.what), p.id.clone()); + let finished = |p: Pending, reply: Value| { + json!({"ok": true, "result": {"cancelled": null, "pending": p.id, "method": p.method, "done": true, "reply": reply, + "note": format!("nothing to cancel: {what} had already finished; its reply is here")}}) + }; + if let Some(reply) = late { + return pending.take().map_or(Value::Null, |p| finished(p, reply)); + } + let outcome = p.job.cancel(); + let note = match outcome { + Cancel::Removed => format!("cancelled: {what} was still waiting its turn and will not run; nothing was applied"), + Cancel::Stopped { writes_file: false } => format!("stopped: {what} was computing off the window thread; nothing was applied"), + Cancel::Stopped { writes_file: true } => { + format!("stopped: {what} was computing on a copy of the {document}, which is unchanged; its worker may still write the file") + } + Cancel::Already => format!("{what} was already cancelled or stopped; nothing of it was applied"), + Cancel::Running => { + // It may have just finished. + if let Some(reply) = pending.as_ref().and_then(|p| p.reply.try_recv().ok()) { + return pending.take().map_or(Value::Null, |p| finished(p, reply)); + } + return json!({"ok": false, "cancellable": false, "error": format!( + "cannot be cancelled: {what} already runs in the app (on its window thread, or as a file it opens) and will \ + still finish; ask `control.pending` for its result. Only a request still waiting its turn, or a change or \ + export computing off the window thread, can be cancelled" + )}); + } + }; + let method = pending.take().map(|p| p.method); + let how = match outcome { + Cancel::Removed => "removed", + Cancel::Stopped { .. } => "stopped", + _ => "already", + }; + json!({"ok": true, "result": {"cancelled": id, "method": method, "outcome": how, "note": note}}) +} + +/// Every request the app has been sent and not answered yet, from every connection, for +/// `control.running` (#3654). Each connection's own thread answers that, so it never waits behind +/// a running request. +struct Running { + list: Mutex>, + connections: AtomicU64, + keys: AtomicU64, +} + +/// What `control.running` says about a request: never its parameters beyond the command id. +struct InFlight { + key: u64, + connection: u64, + id: Value, + method: String, + command: Option<&'static str>, + steps: Option, + since: Instant, + started: Arc, +} + +impl Running { + fn new() -> Self { + Running { list: Mutex::new(Vec::new()), connections: AtomicU64::new(0), keys: AtomicU64::new(0) } + } + + fn lock(&self) -> MutexGuard<'_, Vec> { + self.list.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// List `req`, sent on `connection`, until the returned ticket is dropped; `command` and + /// `steps` as the app's [`Protocol`] summarised it. + fn add(self: &Arc, connection: u64, id: &Value, req: &ControlRequest, command: Option<&'static str>, steps: Option) -> Listing { + let key = self.keys.fetch_add(1, Ordering::Relaxed); + // Ids and methods come from the client: only short ones are repeated to the others. + let id = match id { + Value::Number(_) => id.clone(), + Value::String(s) if s.len() <= 64 => id.clone(), + _ => Value::Null, + }; + let method = req.method.chars().take(64).collect(); + let started = req.started.clone(); + self.lock().push(InFlight { key, connection, id, method, command, steps, since: Instant::now(), started }); + Listing { running: Arc::clone(self), key } + } + + /// The `control.running` result, as `asker`'s connection sees it. + fn result(&self, asker: u64) -> Value { + let rows: Vec = self + .lock() + .iter() + .map(|e| { + let mut r = json!({ + "connection": e.connection, + "id": e.id, + "method": e.method, + "running_s": (e.since.elapsed().as_secs_f64() * 10.0).round() / 10.0, + "queued": !e.started.load(Ordering::Relaxed), + }); + if let Some(c) = e.command { + r["command"] = json!(c); + } + if let Some(n) = e.steps { + r["steps"] = json!(n); + } + if e.connection == asker { + r["self"] = json!(true); + } + r + }) + .collect(); + json!({"running": rows}) + } +} + +/// A listed request; dropped when its reply arrives. +struct Listing { + running: Arc, + key: u64, +} + +impl Drop for Listing { + fn drop(&mut self) { + let key = self.key; + self.running.lock().retain(|e| e.key != key); + } +} + +/// The reply to a timed-out request, received on a thread of its own so the request stays listed +/// until the app answers, also once its connection has closed (#3654). +fn watch(reply: Receiver, ticket: Listing) -> Receiver { + let (late_tx, late_rx) = channel(); + // A small thread per timed-out request (each connection has at most one). If none can be + // made, the reply is lost and `control.pending` says the app ended the request without one. + // The reply is passed on before the request leaves the list: once `control.running` no longer + // shows it, its connection's `control.pending` has it. + let _ = std::thread::Builder::new().stack_size(64 * 1024).spawn(move || { + if let Ok(v) = reply.recv() { + let _ = late_tx.send(v); + } + drop(late_tx); + drop(ticket); + }); + late_rx +} + +/// The reply text for a request that outlasts its wait. A request that only reads changes +/// nothing when it finishes; an export or a save still writes its file, and any other request +/// still applies, as it was not cancelled. +pub fn timeout_message(what: &str, effect: Effect, document: &str) -> String { + match effect { + Effect::ReadsOnly => { + format!("timeout: {what} is still running in the app ({}); ask `control.pending` for its answer", effect.when_done(document)) + } + Effect::WritesFile => format!( + "timeout: {what} is still running in the app and will still write the file when it finishes (it was not cancelled: \ + don't send it again; the {document} does not change); ask `control.pending` for its result, or `control.cancel` to \ + cancel it" + ), + Effect::Changes => format!( + "timeout: {what} is still running in the app and will still apply when it finishes (it was not cancelled: don't send it \ + again); ask `control.pending` for its result, or `control.cancel` to cancel it" + ), + } +} + +/// The reply text for a request refused because an earlier one on the connection still runs. +fn busy_message(p: &Pending, method: &str) -> String { + let method: String = method.chars().take(128).collect(); + format!( + "busy: request {} ({}) is still running, so `{method}` was not run; ask `control.pending` for that request's result, then \ + send this one again", + p.id, p.what + ) +} + +/// A subscribed connection's socket, for its replies and its events: each line is written whole +/// and alone, so an event never lands inside a reply (#6985). +struct Lines { + socket: TcpStream, + turn: Mutex<()>, +} + +impl Lines { + fn send(&self, line: &Value) -> std::io::Result<()> { + let _turn = self.turn.lock().unwrap_or_else(PoisonError::into_inner); + send(&mut &self.socket, line) + } +} + +/// A subscribed connection's seat on the watch: it leaves exactly once, whichever thread sees +/// the connection's end first — the connection's own thread at the end of the stream, or the +/// events thread at the first write a closed client refuses. The seat used to wait for the +/// events thread alone, and that thread only leaves its wait when the design moves, so the +/// count could outlive the connection by a second or more (#8315). +struct Seat(Mutex>); + +impl Seat { + fn new(counted: Subscriber) -> Self { + Seat(Mutex::new(Some(counted))) + } + + /// Leave the watch now: the seat goes once, whoever gets here first. + fn release(&self) { + let counted = self.0.lock().unwrap_or_else(PoisonError::into_inner).take(); + drop(counted); + } +} + +impl Drop for Seat { + fn drop(&mut self) { + self.release(); + } +} + +/// A connection's `control.subscribe` (#6985): its `design.changed` lines, sent from a thread of +/// their own until the connection ends. +struct Subscription { + lines: Arc, + stop: Arc, + seat: Arc, +} + +impl Subscription { + /// Write `reply`, the window's answer to `control.subscribe` naming the design `seen`, on + /// `out`, then send `design`'s changes after it: the reply goes out before any event. `None`, + /// with nothing written, when no thread or socket was left for the events. + fn start( + watch: &'static DocumentWatch, + event: &'static str, + counted: Subscriber, + out: &TcpStream, + seen: (String, u64), + reply: &Value, + ) -> Option<(Self, std::io::Result<()>)> { + let lines = Arc::new(Lines { socket: out.try_clone().ok()?, turn: Mutex::new(()) }); + let stop = Arc::new(AtomicBool::new(false)); + let seat = Arc::new(Seat::new(counted)); + let first = lines.turn.lock().unwrap_or_else(PoisonError::into_inner); + let (l, s, watch_seat) = (lines.clone(), stop.clone(), Arc::clone(&seat)); + // The connection's seat on the watch rides the events thread: it leaves when the thread + // does, so the next failed write to a closed client retires it even while the connection's + // own thread is slow to see the end of the stream (#7226). The connection's own thread + // leaves it too, so a closed connection stops counting at once (#8315). + std::thread::Builder::new().stack_size(64 * 1024).spawn(move || send_events(watch, event, &l, &s, seen, watch_seat)).ok()?; + let sent = send(&mut &lines.socket, reply); + drop(first); + Some((Subscription { lines, stop, seat }, sent)) + } +} + +/// The design a `control.subscribe` reply names. +fn document_of(reply: &Value) -> Option<(String, u64)> { + let result = reply.get("result")?; + Some((result.get("document_id")?.as_str()?.to_string(), result.get("revision")?.as_u64()?)) +} + +impl Drop for Subscription { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + // The seat on the watch goes now: the connection is over, and nothing counts it any more + // (#8315). + self.seat.release(); + // The socket closes now, not when the thread next looks; a line blocked on a client that + // stopped reading ends too. + let _ = self.lines.socket.shutdown(std::net::Shutdown::Both); + } +} + +/// The `design.changed` lines of a subscribed connection, each time the design differs from the +/// last one it was told of (`seen`), until `stop`. Holds the connection's seat on the watch +/// (`seat`), which it leaves when the thread ends: on `stop`, or on the first write a closed +/// client refuses (#7226). +fn send_events(watch: &DocumentWatch, event: &str, lines: &Lines, stop: &AtomicBool, mut seen: (String, u64), seat: Arc) { + while !stop.load(Ordering::Relaxed) { + let Some(now) = watch.wait(&seen, stop, Duration::from_secs(1)).filter(|now| *now != seen) else { continue }; + if lines.send(&json!({"event": event, "params": {"document_id": now.0, "revision": now.1}})).is_err() { + break; + } + seen = now; + std::thread::sleep(EVENT_GAP); + } + // The client is gone: it is not counted on the watch any more, whatever the connection's own + // thread still has to notice (#8315). + seat.release(); +} + +/// Write `reply` on the connection: through its subscription once it has one. +fn answer(out: &mut TcpStream, events: Option<&Subscription>, reply: &Value) -> std::io::Result<()> { + match events { + Some(e) => e.lines.send(reply), + None => send(out, reply), + } +} + +impl Hub { + /// Serve one connection until it closes: the token first, then one request per line, each + /// sent to the window on `requests` and its reply written back. + pub fn serve(&self, stream: TcpStream, token: &str, requests: Sender, patient: bool) { + let (waits, running, documents, tx) = (self.waits, &self.running, self.watch, requests); + let connection = running.connections.fetch_add(1, Ordering::Relaxed) + 1; + let Ok(read) = stream.try_clone() else { return }; + let mut out = stream; + let mut reader = BufReader::new(ClientInput { socket: read, deadline: Some(Instant::now() + waits.token) }); + let mut bad = 0usize; + let mut authenticated = false; + let mut pending: Option = None; + let mut events: Option = None; + loop { + // A first `control.subscribe` on its way to the window (#6985). + let mut joining = None; + let line = match read_line(&mut reader, self.max_line) { + Ok(Some(l)) => String::from_utf8_lossy(&l).into_owned(), + Ok(None) => break, + // Closed without a word: a client that connected early and only now sends its first + // request would read a line here as the reply to it. It finds the connection closed + // and opens another. + Err(e) if !authenticated && matches!(e.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) => break, + Err(e) => { + let _ = answer(&mut out, events.as_ref(), &json!({"ok": false, "error": e.to_string()})); + break; + } + }; + if line.trim().is_empty() { + continue; + } + // The first request must carry the token; later ones on the same connection may. + if !authenticated { + match token_of(&line) { + Some(t) if tokens_match(&t, token) => { + if reader.get_mut().admit().is_err() { + break; + } + authenticated = true; + } + _ => { + refuse(&mut out, &mut reader, self.protocol.docs()); + break; + } + } + } + let reply = match serde_json::from_str::(&line) { + Ok(msg) => { + let id = msg.get("id").cloned().unwrap_or(Value::Null); + let method = msg.get("method").and_then(Value::as_str).unwrap_or("").to_string(); + let params = msg.get("params").cloned().unwrap_or(json!({})); + // A request that timed out may have finished since (#1618). `control.running` and + // `control.capabilities` only look, so they leave that reply where it is. + let late = pending.as_ref().filter(|_| !matches!(method.as_str(), "control.running" | "control.capabilities")).and_then(|p| { + match p.reply.try_recv() { + Ok(reply) => Some(reply), + Err(TryRecvError::Disconnected) => Some(json!({"ok": false, "error": "the app ended the request without a reply"})), + Err(TryRecvError::Empty) => None, + } + }); + let mut r = if method == "control.running" { + json!({"ok": true, "result": running.result(connection)}) + } else if method == "control.capabilities" { + json!({"ok": true, "result": self.protocol.capabilities()}) + } else if method == "control.cancel" { + let r = cancel(&mut pending, late, params.get("id"), self.protocol.document()); + (self.wake)(); + r + } else if method == "control.pending" { + match (pending.take(), late) { + (None, _) => json!({"ok": true, "result": {"pending": null}}), + (Some(p), Some(reply)) => { + json!({"ok": true, "result": {"pending": p.id, "method": p.method, "done": true, "reply": reply}}) + } + (Some(p), None) => { + let r = json!({"ok": true, "result": {"pending": p.id, "method": p.method, "done": false}}); + pending = Some(p); + r + } + } + } else if let Some(p) = pending.as_ref().filter(|_| late.is_none()) { + // Nothing more is queued behind it, to run unseen when it ends. + json!({"ok": false, "busy": true, "error": busy_message(p, &method)}) + } else if method == "control.subscribe" && events.is_some() { + if late.is_some() { + // As below: finished, and the client moved on without asking. + pending = None; + } + // Its events already say how the design moves; a second answer could only + // arrive behind a newer one. + { + let (event, revision, document) = + (self.protocol.change_event(), self.protocol.revision_method(), self.protocol.document()); + json!({"ok": false, "error": format!("already subscribed: this connection is sent {event} lines; ask {revision} for the {document} now")}) + } + } else { + if late.is_some() { + // Finished, and the client moved on without asking: it is dropped. + pending = None; + } + // A batch replaces many short requests: the whole of it may take long (#743). + let Summary { what, effect, command, steps, long } = self.protocol.summary(&method, ¶ms); + let wait = if patient || long { waits.long } else { waits.short }; + // Counted before the window answers, so it publishes every change from its + // answer on. + joining = (method == "control.subscribe").then(|| documents.join()); + let (req, rrx) = ControlRequest::new(method.clone(), params); + let job = Arc::clone(&req.ticket); + let ticket = running.add(connection, &id, &req, command, steps); + if tx.send(req).is_err() { + break; + } + (self.wake)(); + match rrx.recv_timeout(wait) { + Ok(r) => r, + Err(_) => { + let r = json!({"ok": false, "running": true, "error": timeout_message(&what, effect, self.protocol.document())}); + pending = Some(Pending { id: id.clone(), method, what, reply: watch(rrx, ticket), job }); + r + } + } + }; + if let Some(o) = r.as_object_mut() { + o.insert("id".into(), id); + } + r + } + Err(e) => { + bad += 1; + json!({"ok": false, "error": format!("bad JSON: {e}")}) + } + }; + // Subscribed once the window named the design (not when it refused or timed out). + let sent = match joining.zip(document_of(&reply)) { + Some((counted, seen)) => match Subscription::start(documents, self.protocol.change_event(), counted, &out, seen, &reply) { + Some((subscription, sent)) => { + events = Some(subscription); + sent + } + None => send( + &mut out, + &json!({"id": reply.get("id"), "ok": false, "error": "not subscribed: no thread or socket left for the events"}), + ), + }, + None => answer(&mut out, events.as_ref(), &reply), + }; + if sent.is_err() { + break; + } + if bad >= MAX_BAD_LINES { + break; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The longest request line, as an app might accept. + const MAX_LINE: usize = 16 << 20; + + /// An app whose requests all may change its document and take the short wait. + struct Plain; + + impl Protocol for Plain { + fn summary(&self, method: &str, _: &Value) -> Summary { + Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false } + } + fn document(&self) -> &'static str { + "design" + } + fn change_event(&self) -> &'static str { + "design.changed" + } + fn revision_method(&self) -> &'static str { + "document.revision" + } + fn docs(&self) -> &'static str { + "docs/control-protocol.md" + } + fn capabilities(&self) -> Value { + json!({}) + } + } + + /// A server for [`Plain`] with these waits and this document watch. + fn hub(waits: Waits, watch: &'static DocumentWatch) -> Arc { + Arc::new(Hub::new(Plain, || {}, MAX_LINE).with_waits(waits).with_watch(watch)) + } + + /// The channel never goes on the network: a listener on any address but loopback is refused, + /// and nothing is served on it. + #[test] + fn a_listener_off_loopback_is_refused() { + let (tx, _rx) = channel::(); + let open = TcpListener::bind("0.0.0.0:0").unwrap(); + let e = hub(Waits::default(), &DOCUMENT_WATCH).listen(open, "0123456789abcdef0123456789abcdef".into(), tx.clone(), false).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput, "{e}"); + assert!(e.to_string().contains("loopback only"), "{e}"); + let local = TcpListener::bind("127.0.0.1:0").unwrap(); + assert!(hub(Waits::default(), &DOCUMENT_WATCH).listen(local, "0123456789abcdef0123456789abcdef".into(), tx, false).is_ok()); + } + + /// A connection that doesn't show the token in time is closed without a word, however it fills + /// the time. One that does keeps working when it idles for longer than that. + #[test] + fn the_token_has_a_time_limit_and_an_authenticated_client_may_idle() { + use std::io::Read; + let token = "0123456789abcdef0123456789abcdef"; + let limit = Duration::from_millis(500); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + std::thread::spawn(move || { + for req in rx { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + }); + let waits = Waits { short: Duration::from_secs(10), long: Duration::from_secs(10), token: limit }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let open = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + s + }; + // Sends `drip` every 50 ms until the server closes; what it read, and when it closed. + let closed_after = |drip: &[u8]| { + let mut s = open(); + let start = Instant::now(); + s.set_read_timeout(Some(Duration::from_millis(50))).unwrap(); + let mut got = Vec::new(); + loop { + let mut buf = [0u8; 256]; + match s.read(&mut buf) { + Ok(0) => break, + Ok(n) => got.extend_from_slice(buf.get(..n).unwrap()), + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => { + assert!(start.elapsed() < Duration::from_secs(20), "still open"); + let _ = s.write_all(drip); + } + Err(_) => break, + } + } + (got, start.elapsed()) + }; + for drip in [&b""[..], b"\n", b"\r\n \n", b" ", b"{"] { + let (got, took) = closed_after(drip); + assert!(got.is_empty(), "{drip:?} was answered: {}", String::from_utf8_lossy(&got)); + assert!(took >= limit.saturating_sub(Duration::from_millis(50)), "{drip:?} closed after {took:?}"); + } + + let mut s = open(); + let mut lines = BufReader::new(s.try_clone().unwrap()); + let mut ask = |line: Value| -> Value { + writeln!(s, "{line}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + assert_eq!(ask(json!({"id": 1, "method": "ui.view", "token": token}))["ok"], true); + std::thread::sleep(limit * 3); + assert_eq!(ask(json!({"id": 2, "method": "ui.view"}))["ok"], true, "idle for longer than the limit"); + + // A wrong token is still answered, at once. + let mut wrong = open(); + writeln!(wrong, "{}", json!({"id": 1, "method": "ui.view", "token": "nope"})).unwrap(); + let mut l = String::new(); + BufReader::new(wrong).read_line(&mut l).unwrap(); + assert!(l.contains("unauthorized"), "{l}"); + } + + /// #1989: `control.cancel` cancels the connection's own outstanding request. One still + /// waiting its turn is removed and never runs, and the connection is free at once; one that + /// already runs in the app can't be, and says so; another connection's request is refused; + /// with nothing outstanding there is nothing to cancel. + #[test] + fn control_cancel_cancels_only_this_connections_request() { + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + let (gate_tx, gate_rx) = channel::<()>(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + // A stand-in for the window thread: `queued` waits behind the others (never taken until + // `slow` is done), `slow` runs until the gate opens. + std::thread::spawn(move || { + let mut held = Vec::new(); + for req in rx { + if req.method == "queued" { + held.push(req); + continue; + } + if !req.ticket.take() { + continue; + } + seen.lock().unwrap().push(req.method.clone()); + if req.method == "slow" { + let _ = gate_rx.recv(); + } + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + if req.method == "slow" { + for req in held.drain(..) { + if req.ticket.take() { + seen.lock().unwrap().push(req.method.clone()); + } + } + } + } + }); + let waits = Waits { short: Duration::from_millis(100), long: Duration::from_millis(100), token: TOKEN_WAIT }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let connect = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let out = s.try_clone().unwrap(); + (out, BufReader::new(s)) + }; + let ask = |(out, lines): &mut (TcpStream, BufReader), v: Value| -> Value { + writeln!(out, "{v}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let mut a = connect(); + let mut b = connect(); + let r = ask(&mut b, json!({"id": 1, "method": "control.cancel", "token": token})); + assert!(r["ok"] == true && r["result"]["cancelled"].is_null(), "nothing to cancel: {r}"); + + // Waiting its turn: removed, never run, and the connection is free. + let r = ask(&mut a, json!({"id": 1, "method": "queued", "token": token})); + assert_eq!(r["running"], true, "{r}"); + let r = ask(&mut a, json!({"id": 2, "method": "control.cancel", "params": {"id": 1}})); + assert_eq!((&r["ok"], &r["result"]["cancelled"], &r["result"]["outcome"]), (&json!(true), &json!(1), &json!("removed")), "{r}"); + assert!(r["result"]["note"].as_str().is_some_and(|n| n.contains("will not run")), "{r}"); + let r = ask(&mut a, json!({"id": 3, "method": "control.pending"})); + assert_eq!(r["result"]["pending"], Value::Null, "{r}"); + + // Already running in the app: it can't be, and it still answers. + let r = ask(&mut a, json!({"id": 4, "method": "slow"})); + assert_eq!(r["running"], true, "{r}"); + let r = ask(&mut b, json!({"id": 2, "method": "control.cancel", "params": {"id": 4}})); + assert!(r["ok"] == false && r["error"].as_str().is_some_and(|e| e.contains("its own requests")), "another connection's: {r}"); + let r = ask(&mut a, json!({"id": 5, "method": "control.cancel", "params": {"id": 99}})); + assert!(r["ok"] == false && r["error"].as_str().is_some_and(|e| e.starts_with("nothing to cancel")), "{r}"); + let r = ask(&mut a, json!({"id": 6, "method": "control.cancel"})); + assert!(r["ok"] == false && r["cancellable"] == false, "{r}"); + assert!(r["error"].as_str().is_some_and(|e| e.starts_with("cannot be cancelled") && e.contains("control.pending")), "{r}"); + gate_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let r = loop { + let r = ask(&mut a, json!({"id": 7, "method": "control.pending"})); + if r["result"]["done"] == true || Instant::now() > deadline { + break r; + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(r["result"]["reply"]["result"], "slow", "{r}"); + let r = ask(&mut a, json!({"id": 8, "method": "control.cancel"})); + assert!(r["ok"] == true && r["result"]["cancelled"].is_null(), "{r}"); + assert_eq!(*ran.lock().unwrap(), ["slow"], "the cancelled request never ran"); + } + + /// #1618: a request that outlasts its wait says it is still running; the connection keeps its + /// reply for `control.pending`, and refuses other requests as `busy` until it lands, so none + /// is queued to run unseen behind it. + #[test] + fn a_timed_out_request_can_be_asked_for_later() { + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The slow request is held until the test has seen it time out, so the test does not + // race the clock on a loaded machine. + let (gate_tx, gate_rx) = channel::<()>(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + std::thread::spawn(move || { + for req in rx { + seen.lock().unwrap().push(req.method.clone()); + if req.method == "slow" { + let _ = gate_rx.recv(); + } + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + }); + let waits = Waits { short: Duration::from_millis(100), long: Duration::from_millis(100), token: TOKEN_WAIT }; + let hub = hub(waits, &DOCUMENT_WATCH); + std::thread::spawn(move || { + if let Some(stream) = listener.incoming().flatten().next() { + hub.serve(stream, token, tx, false); + } + }); + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut out = s.try_clone().unwrap(); + let mut lines = BufReader::new(s); + let mut ask = |v: Value| -> Value { + writeln!(out, "{v}").unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let r = ask(json!({"id": 1, "method": "slow", "token": token})); + assert!(r["ok"] == false && r["running"] == true && r["error"].as_str().is_some_and(|e| e.starts_with("timeout")), "{r}"); + let r = ask(json!({"id": 2, "method": "ui.view"})); + assert!(r["busy"] == true && r["id"] == 2, "{r}"); + let r = ask(json!({"id": 3, "method": "control.pending"})); + assert_eq!((&r["result"]["pending"], &r["result"]["done"]), (&json!(1), &json!(false)), "{r}"); + gate_tx.send(()).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let r = loop { + let r = ask(json!({"id": 4, "method": "control.pending"})); + if r["result"]["done"] == true || std::time::Instant::now() > deadline { + break r; + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!( + (&r["result"]["pending"], &r["result"]["done"], &r["result"]["reply"]["result"]), + (&json!(1), &json!(true), &json!("slow")), + "{r}" + ); + let r = ask(json!({"id": 5, "method": "control.pending"})); + assert_eq!(r["result"]["pending"], Value::Null, "{r}"); + let r = ask(json!({"id": 6, "method": "ui.view"})); + assert_eq!((&r["ok"], &r["result"]), (&json!(true), &json!("ui.view")), "{r}"); + assert_eq!(*ran.lock().unwrap(), ["slow", "ui.view"], "the busy request never reached the app"); + } + + /// #6985: a connection that sends `control.subscribe` gets the design in the reply, then a + /// `design.changed` line each time the window publishes another: never before the reply, + /// none for the same design, the latest after a burst, each line whole between its replies. + /// A second subscribe is refused without reaching the window. A connection that did not + /// subscribe gets no events, and one that closed is no longer counted. + #[test] + fn subscribed_connections_are_told_of_design_changes() { + static WATCH: DocumentWatch = DocumentWatch::new(); + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The window: it answers `control.subscribe` as the app does, publishing the design first; + // then the design changes at once, before the reply may have been written. + std::thread::spawn(move || { + for req in rx { + if req.method == "control.subscribe" { + WATCH.publish("d-1", 3); + let _ = req.reply.send(json!({"ok": true, "result": {"document_id": "d-1", "revision": 3}})); + WATCH.publish("d-1", 4); + } else { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + } + }); + let hub = hub(Waits::default(), &WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let open = || { + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + (s.try_clone().unwrap(), BufReader::new(s)) + }; + let next = |lines: &mut BufReader| -> Value { + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + serde_json::from_str(&l).unwrap() + }; + let changed = |id: &str, revision: u64| json!({"event": "design.changed", "params": {"document_id": id, "revision": revision}}); + + let (mut quiet_out, mut quiet) = open(); + writeln!(quiet_out, "{}", json!({"id": 1, "method": "ui.view", "token": token})).unwrap(); + assert_eq!(next(&mut quiet)["id"], 1); + + let (mut out, mut lines) = open(); + writeln!(out, "{}", json!({"id": 1, "method": "control.subscribe", "token": token})).unwrap(); + assert_eq!(next(&mut lines), json!({"id": 1, "ok": true, "result": {"document_id": "d-1", "revision": 3}})); + assert_eq!(next(&mut lines), changed("d-1", 4), "the change after the window's answer comes after the reply"); + assert!(WATCH.subscribed()); + // The design it was told of is no news; another revision is. + WATCH.publish("d-1", 4); + WATCH.publish("d-1", 5); + assert_eq!(next(&mut lines), changed("d-1", 5)); + // A burst ends with its latest. + for revision in 6..=10 { + WATCH.publish("d-1", revision); + } + let mut last = 5; + while last < 10 { + let e = next(&mut lines); + let revision = e["params"]["revision"].as_u64().unwrap(); + assert!(e["event"] == "design.changed" && revision > last, "{e} after {last}"); + last = revision; + } + // Another tab: another id, whatever its revision. + WATCH.publish("d-2", 2); + assert_eq!(next(&mut lines), changed("d-2", 2)); + // Subscribed already: refused here (the window would have published d-1 again). + writeln!(out, "{}", json!({"id": 2, "method": "control.subscribe"})).unwrap(); + let r = next(&mut lines); + assert!(r["id"] == 2 && r["ok"] == false && r["error"].as_str().is_some_and(|e| e.starts_with("already subscribed")), "{r}"); + // Replies still come, whole, and nothing else was sent meanwhile. + writeln!(out, "{}", json!({"id": 3, "method": "ui.view"})).unwrap(); + assert_eq!(next(&mut lines), json!({"id": 3, "ok": true, "result": "ui.view"})); + writeln!(quiet_out, "{}", json!({"id": 2, "method": "ui.view"})).unwrap(); + assert_eq!(next(&mut quiet), json!({"id": 2, "ok": true, "result": "ui.view"}), "no events without subscribing"); + // Closed, it is no longer counted. Each publish asks the events thread for one more line, + // and its write to the closed connection is what retires the seat; that keeps the wait off + // the connection's own thread, which anything else this process is doing may hold up past + // the deadline (#7226). + drop((out, lines)); + let mut revision = 11; + let deadline = Instant::now() + Duration::from_secs(10); + while WATCH.subscribed() && Instant::now() < deadline { + WATCH.publish("d-1", revision); + revision += 1; + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!WATCH.subscribed()); + } + + /// #8315: a closed connection leaves the watch at once. The seat of a subscribed connection + /// goes when the connection's own thread sees the end of the stream, not only when the + /// events thread notices: that thread waits for the design to move, so a connection that + /// closed while the design stood still kept the watch counting it for a second or more. + #[test] + fn a_closed_connection_leaves_the_watch_at_once() { + static WATCH: DocumentWatch = DocumentWatch::new(); + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + // The window names the design once and then leaves it alone: nothing moves it, so the + // events thread has nothing to write and only the end of the connection can retire it. + std::thread::spawn(move || { + for req in rx { + if req.method == "control.subscribe" { + WATCH.publish("d-1", 3); + let _ = req.reply.send(json!({"ok": true, "result": {"document_id": "d-1", "revision": 3}})); + } else { + let _ = req.reply.send(json!({"ok": true, "result": req.method})); + } + } + }); + let hub = hub(Waits::default(), &WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut out = s.try_clone().unwrap(); + let mut lines = BufReader::new(s); + writeln!(out, "{}", json!({"id": 1, "method": "control.subscribe", "token": token})).unwrap(); + let mut l = String::new(); + lines.read_line(&mut l).unwrap(); + let reply: Value = serde_json::from_str(&l).unwrap(); + assert!(reply["ok"] == true, "{reply}"); + assert!(WATCH.subscribed(), "the connection is subscribed"); + // Let the events thread settle into its wait: the design stands still, so it has nothing + // to write, and only the end of the connection can retire it. + std::thread::sleep(Duration::from_millis(200)); + // The client goes away without asking anything else. + drop((out, lines)); + let deadline = Instant::now() + Duration::from_millis(400); + while WATCH.subscribed() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!WATCH.subscribed(), "a closed connection is not counted on the watch"); + } + + /// The token is read from a request object without building the rest of it: other fields, + /// however large or nested, are only skipped. Anything but an object with a string token is + /// no token. + #[test] + fn token_is_read_without_parsing_the_request() { + assert_eq!(token_of(r#"{"id":1,"token":"abc","params":{"a":[1,[2,{"b":null}]]}}"#).as_deref(), Some("abc")); + let flood = format!(r#"{{"params":[{}0],"token":"t"}}"#, "0,".repeat(1 << 20)); + assert_eq!(token_of(&flood).as_deref(), Some("t")); + for no in [r#"["abc"]"#, r#"{"token":5}"#, r#"{"token":"a"} x"#, r#"{"id":1}"#, "GET / HTTP/1.1", r#""token""#] { + assert_eq!(token_of(no), None, "{no}"); + } + } + + /// #205: a connection runs nothing until a request carries the launch's token; a wrong or + /// missing token, or garbage, is refused and the connection closed. Once shown, the token + /// need not be repeated on that connection. + #[test] + fn requests_run_only_after_the_token() { + use std::io::Read; + let token = "0123456789abcdef0123456789abcdef"; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = channel::(); + let ran = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = ran.clone(); + std::thread::spawn(move || { + for req in rx { + seen.lock().unwrap().push(req.method.clone()); + let _ = req.reply.send(json!({"ok": true, "result": null})); + } + }); + let hub = hub(Waits::default(), &DOCUMENT_WATCH); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (tx, hub) = (tx.clone(), Arc::clone(&hub)); + std::thread::spawn(move || hub.serve(stream, token, tx, false)); + } + }); + let session = |lines: &[String]| { + let mut s = TcpStream::connect(addr).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + for l in lines { + let _ = writeln!(s, "{l}"); + } + let _ = s.shutdown(std::net::Shutdown::Write); + let mut text = String::new(); + let _ = s.read_to_string(&mut text); + text.lines().map(|l| serde_json::from_str::(l).unwrap()).collect::>() + }; + for first in [ + json!({"id": 1, "method": "app.quit", "params": {"force": true}}).to_string(), + json!({"id": 1, "method": "app.quit", "token": "0123456789abcdef0123456789abcdee"}).to_string(), + json!({"id": 1, "method": "app.quit", "token": "0123"}).to_string(), + json!({"id": 1, "method": "app.quit", "token": 7}).to_string(), + "not json".to_string(), + ] { + let replies = session(&[first.clone(), json!({"id": 2, "method": "app.quit", "token": token}).to_string()]); + assert_eq!(replies.len(), 1, "{first}: closed after refusing: {replies:?}"); + assert!(replies[0]["error"].as_str().is_some_and(|e| e.starts_with("unauthorized")), "{first}: {replies:?}"); + } + assert!(ran.lock().unwrap().is_empty(), "nothing ran without the token"); + let replies = session(&[ + json!({"id": 1, "method": "ui.inspect", "token": token}).to_string(), + json!({"id": 2, "method": "engine.commands"}).to_string(), + ]); + assert_eq!(replies.len(), 2, "{replies:?}"); + assert!(replies.iter().all(|r| r["ok"] == true), "{replies:?}"); + assert_eq!(*ran.lock().unwrap(), ["ui.inspect", "engine.commands"]); + } + + /// A request longer than the cap is refused instead of allocated. + #[test] + fn lines_past_the_cap_are_refused() { + let flood = vec![b'A'; MAX_LINE + 2]; + let mut r = BufReader::new(flood.as_slice()); + let e = read_line(&mut r, MAX_LINE).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert!(e.to_string().contains("request too large"), "{e}"); + } + + /// Ordinary framing: lines split on newlines, blank lines kept, no trailing newline needed. + #[test] + fn lines_split_on_newlines() { + let text = b"{\"a\":1}\n\n{\"b\":2}\nlast".to_vec(); + let mut r = BufReader::new(text.as_slice()); + let mut got = Vec::new(); + while let Some(line) = read_line(&mut r, MAX_LINE).unwrap() { + got.push(String::from_utf8(line).unwrap()); + } + assert_eq!(got, vec!["{\"a\":1}", "", "{\"b\":2}", "last"]); + assert!(read_line(&mut r, MAX_LINE).unwrap().is_none()); + } + + /// A line that spans several reads of the underlying stream still arrives whole. + #[test] + fn lines_longer_than_one_buffer_arrive_whole() { + let long = "x".repeat(200_000); + let text = format!("{long}\n{long}\n").into_bytes(); + let mut r = BufReader::with_capacity(64, text.as_slice()); + assert_eq!(read_line(&mut r, MAX_LINE).unwrap().unwrap().len(), long.len()); + assert_eq!(read_line(&mut r, MAX_LINE).unwrap().unwrap().len(), long.len()); + assert!(read_line(&mut r, MAX_LINE).unwrap().is_none()); + // The same line against a smaller cap is refused. + let mut r = BufReader::with_capacity(64, text.as_slice()); + assert!(read_line(&mut r, 1000).is_err()); + } + + /// A large reply reaches the socket in one write, not one per JSON piece (a component tree of + /// thousands of components took half a second to send), and reads back as it was. + #[test] + fn a_large_reply_is_written_at_once() { + struct Counted { + writes: usize, + bytes: Vec, + } + impl Write for Counted { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.writes += 1; + self.bytes.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let components: Vec = (0..2000).map(|i| json!({"id": i, "name": format!("Part {i}"), "bodies": ["a", "b"], "parent": 0})).collect(); + let reply = json!({"ok": true, "id": 7, "result": {"components": components}}); + let mut out = Counted { writes: 0, bytes: Vec::new() }; + send(&mut out, &reply).unwrap(); + assert_eq!(out.writes, 1); + assert_eq!(out.bytes.last(), Some(&b'\n')); + assert_eq!(serde_json::from_slice::(&out.bytes).unwrap(), reply); + } + + /// A stopped listener frees its port at once and ends the connections it had, so a channel + /// turned off in the settings leaves nothing listening and can be turned on again on the + /// same port. + #[test] + fn a_stopped_listener_closes_its_port_and_its_connections() { + use std::io::Read; + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let (tx, _rx) = channel(); + let stop = hub(Waits::default(), &DOCUMENT_WATCH).listen_stoppable(listener, "0123456789abcdef0123456789abcdef".into(), tx, false).unwrap(); + assert_eq!(stop.port(), port); + let mut client = TcpStream::connect(("127.0.0.1", port)).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(4))).unwrap(); + // Wait until the server holds the connection, so stopping has one to close. + let held = Instant::now(); + while stop.shared.lock().is_empty() && held.elapsed() < Duration::from_secs(4) { + std::thread::sleep(Duration::from_millis(5)); + } + let started = Instant::now(); + stop.stop(); + let mut byte = [0u8; 1]; + match client.read(&mut byte) { + Ok(n) => assert_eq!(n, 0, "the connection ends"), + Err(e) => { + assert!(!matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut), "the connection ends, not waits: {e}") + } + } + assert!(started.elapsed() < Duration::from_secs(3), "before the token wait runs out"); + // The listener thread closes its socket when it notices the stop; on a loaded machine that can + // take a moment after `stop()` returns. Wait for the port to be free, not for one instant. + let freed = Instant::now(); + let mut bound = TcpListener::bind(("127.0.0.1", port)); + while bound.is_err() && freed.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(10)); + bound = TcpListener::bind(("127.0.0.1", port)); + } + assert!(bound.is_ok(), "the port is free again: {:?}", bound.err()); + } +} diff --git a/crates/docx/src/read/props.rs b/crates/docx/src/read/props.rs index dc61b3a28..4d86ae164 100644 --- a/crates/docx/src/read/props.rs +++ b/crates/docx/src/read/props.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use craft_wordmodel::para::cap_alt; use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CellProps, CharProps, CombineBrackets, EastAsianLayout, HeightRule, Highlight, LineSpacing, NumRef, + Align, Border, BorderStyle, Borders, CellProps, CharProps, CombineBrackets, EastAsianLayout, HeightRule, Highlight, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, RowProps, TabAlign, TabLeader, TabStop, TableFloat, TableLook, TableProps, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, }; @@ -465,7 +465,14 @@ impl PropCtx { _ => {} } } - "w:jc" => p.align = k.attr("w:val").map(align), + "w:jc" => { + if let Some(v) = k.attr("w:val") { + p.align = Some(align(v)); + if let Some(kashida) = Kashida::from_jc(v) { + p.kashida = Some(kashida); + } + } + } "w:outlineLvl" => p.outline_level = k.attr("w:val").and_then(u32_of).map(|v| v.min(9) as u8), "w:rPr" => mark = self.rpr(k), _ => {} diff --git a/crates/docx/src/read/shape_style.rs b/crates/docx/src/read/shape_style.rs index 9f28ef2ff..983218f9e 100644 --- a/crates/docx/src/read/shape_style.rs +++ b/crates/docx/src/read/shape_style.rs @@ -47,6 +47,7 @@ pub(crate) fn preset_kind(prst: &str) -> ShapeKind { "downArrow" => ShapeKind::ArrowDown, "star4" | "star5" | "star6" | "star7" | "star8" | "star10" | "star12" | "star16" | "star24" | "star32" => ShapeKind::Star, "heart" => ShapeKind::Heart, + "frame" => ShapeKind::Frame, _ => ShapeKind::Rectangle, } } diff --git a/crates/docx/src/read/story.rs b/crates/docx/src/read/story.rs index 770375365..86ceb1651 100644 --- a/crates/docx/src/read/story.rs +++ b/crates/docx/src/read/story.rs @@ -22,6 +22,7 @@ use craft_wordmodel::wordart::TextWarp; mod blocks; mod drawing; mod fields; +mod locked_canvas; mod runs; mod vml; diff --git a/crates/docx/src/read/story/drawing.rs b/crates/docx/src/read/story/drawing.rs index cb1152da8..e791f1e12 100644 --- a/crates/docx/src/read/story/drawing.rs +++ b/crates/docx/src/read/story/drawing.rs @@ -82,6 +82,10 @@ impl Reader<'_> { self.read_canvas(sc, c, rels, w, h, float) } else if let Some(g) = gd.child("wpg:wgp") { self.read_group(sc, g, rels, w, h, float)? + } else if let Some(lc) = gd.child("lc:lockedCanvas") { + // Word 2007's shapes, pictures and text in a drawing: read as the group they are. + let g = super::locked_canvas::as_group(lc, &self.doc.settings.theme_colors); + self.read_group(sc, &g, rels, w, h, float)? } else if direct_shape.is_none() && gd.find("a:blip").is_some() { self.read_pic(gd, rels, w, h, alt.clone(), float)? } else { @@ -526,12 +530,17 @@ impl Reader<'_> { outline_fill: paint.outline_fill, ..ShapeExtra::default() }; - // A rounded rectangle's own corner (`adj`, ECMA-376 §20.1.10.56); other presets drawn as - // rounded rectangles keep the default. - if prst == "roundRect" { + // A rounded rectangle's own corner (`adj`, ECMA-376 §20.1.10.56), a frame's own border + // width (`adj1`); other presets drawn as rounded rectangles keep the default. + let adj = match prst { + "roundRect" => Some("adj"), + "frame" => Some("adj1"), + _ => None, + }; + if let Some(adj) = adj { extra.corner = sppr .and_then(|s| s.child("a:prstGeom")) - .and_then(|g| preset_adj(g, "adj")) + .and_then(|g| preset_adj(g, adj)) .and_then(|v| u32::try_from(v.clamp(0, i64::from(craft_wordmodel::connector::MAX_CORNER))).ok()); } // Arrowheads: any end type but `none` (ECMA-376 §20.1.10.33) is drawn as a triangle of its diff --git a/crates/docx/src/read/story/locked_canvas.rs b/crates/docx/src/read/story/locked_canvas.rs new file mode 100644 index 000000000..59a52cea2 --- /dev/null +++ b/crates/docx/src/read/story/locked_canvas.rs @@ -0,0 +1,265 @@ +//! A DrawingML locked canvas (`lc:lockedCanvas`, ECMA-376 Part 1 §20.3.2.1): the shapes, +//! pictures and text Word 2007 writes in a drawing in compatibility mode (wordcraft#5060). It is a group +//! shape in the DrawingML namespace (`a:sp`, `a:pic`, `a:grpSp`, `a:cxnSp`, text in `a:txSp`), so +//! it is read as the `wpg:wgp` group the reader already knows: each member renamed into its +//! WordprocessingML drawing form, and its DrawingML text into `w:txbxContent` paragraphs. + +use super::{El, Node, Rgb}; + +/// Groups nested deeper than this are left out, as `group_members` does. +const MAX_NEST: usize = 8; +/// DrawingML's text size when neither the run nor the list style gives one (18 pt, in +/// hundredths of a point). +const DEFAULT_SZ: i64 = 1800; + +/// The locked canvas `lc` as a `wpg:wgp` group. `theme` resolves its text colours. +pub(super) fn as_group(lc: &El, theme: &[Rgb]) -> El { + el("wpg:wgp", Vec::new(), members(lc, theme, 0)) +} + +/// The members of DrawingML group `g`, in their WordprocessingML drawing form. +fn members(g: &El, theme: &[Rgb], depth: usize) -> Vec { + g.els() + .filter_map(|e| match e.name.as_str() { + "a:grpSpPr" => Some(renamed(e, "wpg:grpSpPr")), + "a:grpSp" if depth < MAX_NEST => Some(el("wpg:grpSp", Vec::new(), members(e, theme, depth + 1))), + "a:sp" => Some(shape(e, theme)), + "a:cxnSp" => Some(connector(e)), + "a:pic" => Some(picture(e)), + "a:txSp" => Some(text_shape(e, theme)), + _ => None, + }) + .map(Node::El) + .collect() +} + +/// `a:sp` as `wps:wsp`: its name, properties and style, and its text as a text box's. +fn shape(sp: &El, theme: &[Rgb]) -> El { + let nv = sp.child("a:nvSpPr"); + let mut kids: Vec = [ + nv.and_then(|n| n.child("a:cNvPr")).map(|e| renamed(e, "wps:cNvPr")), + nv.and_then(|n| n.child("a:cNvSpPr")).map(|e| renamed(e, "wps:cNvSpPr")), + sp.child("a:spPr").map(|e| renamed(e, "wps:spPr")), + sp.child("a:style").map(|e| renamed(e, "wps:style")), + ] + .into_iter() + .flatten() + .map(Node::El) + .collect(); + // The text sits in the shape's own rectangle (`a:useSpRect`); a text rectangle of its own + // (`a:txSp/a:xfrm`) is not kept apart. + if let Some(body) = sp.child("a:txSp").and_then(|t| t.child("a:txBody")) { + kids.extend(text_box(body, theme)); + } + el("wps:wsp", Vec::new(), kids) +} + +/// A group's text shape (`a:txSp` as a member, text with no shape of its own) as a `wps:wsp` text +/// box without fill or outline at its own rectangle (`a:xfrm`). +fn text_shape(tx: &El, theme: &[Rgb]) -> El { + let none = || Node::El(el("a:noFill", Vec::new(), Vec::new())); + let geometry = el("a:prstGeom", attrs(&[("prst", "rect")]), vec![Node::El(el("a:avLst", Vec::new(), Vec::new()))]); + let mut sp_pr: Vec = tx.child("a:xfrm").map(|x| Node::El(x.clone())).into_iter().collect(); + sp_pr.extend([Node::El(geometry), none(), Node::El(el("a:ln", Vec::new(), vec![none()]))]); + let mut kids = vec![Node::El(el("wps:cNvSpPr", attrs(&[("txBox", "1")]), Vec::new())), Node::El(el("wps:spPr", Vec::new(), sp_pr))]; + if let Some(body) = tx.child("a:txBody") { + kids.extend(text_box(body, theme)); + } + el("wps:wsp", Vec::new(), kids) +} + +/// DrawingML text body `body` as a text box's content and body properties. +fn text_box(body: &El, theme: &[Rgb]) -> Vec { + let content = el("w:txbxContent", Vec::new(), paragraphs(body, theme)); + let mut kids = vec![Node::El(el("wps:txbx", Vec::new(), vec![Node::El(content)]))]; + if let Some(bp) = body.child("a:bodyPr") { + kids.push(Node::El(renamed(bp, "wps:bodyPr"))); + } + kids +} + +/// `a:cxnSp` as a `wps:wsp` line. Its glued ends name shapes by the canvas's own ids, which +/// aren't the document's, so they are left out. +fn connector(cx: &El) -> El { + let kids = [ + cx.child("a:nvCxnSpPr").and_then(|n| n.child("a:cNvPr")).map(|e| renamed(e, "wps:cNvPr")), + cx.child("a:spPr").map(|e| renamed(e, "wps:spPr")), + cx.child("a:style").map(|e| renamed(e, "wps:style")), + ]; + el("wps:wsp", Vec::new(), kids.into_iter().flatten().map(Node::El).collect()) +} + +/// `a:pic` as `pic:pic`. +fn picture(pic: &El) -> El { + let nv = pic.child("a:nvPicPr").and_then(|n| n.child("a:cNvPr")).map(|e| el("pic:nvPicPr", Vec::new(), vec![Node::El(renamed(e, "pic:cNvPr"))])); + let kids = [nv, pic.child("a:blipFill").map(|e| renamed(e, "pic:blipFill")), pic.child("a:spPr").map(|e| renamed(e, "pic:spPr"))]; + el("pic:pic", Vec::new(), kids.into_iter().flatten().map(Node::El).collect()) +} + +/// The paragraphs of DrawingML text body `body` as `w:p`s. DrawingML text has no space before +/// or after a paragraph and single lines, so each says so rather than taking the document's +/// defaults. +fn paragraphs(body: &El, theme: &[Rgb]) -> Vec { + body.children("a:p") + .map(|p| { + // A paragraph's properties default to the list style's for its level (`lvl`, 0–8), + // then to the list style's defaults (`a:defPPr`), property by property; a run's to its + // paragraph's own (`a:pPr/a:defRPr`), then to those two's. + let ppr_el = p.child("a:pPr"); + let level = ppr_el.and_then(|pp| pp.attr("lvl")).and_then(|v| v.trim().parse::().ok()).filter(|l| *l <= 8).unwrap_or(0); + let list_style = body.child("a:lstStyle"); + let paragraph_styles: Vec<&El> = + [ppr_el, list_style.and_then(|l| l.child(&format!("a:lvl{}pPr", level + 1))), list_style.and_then(|l| l.child("a:defPPr"))] + .into_iter() + .flatten() + .collect(); + let defaults: Vec<&El> = paragraph_styles.iter().filter_map(|pp| pp.child("a:defRPr")).collect(); + let defaults = defaults.as_slice(); + let mut ppr = + vec![Node::El(el("w:spacing", attrs(&[("w:before", "0"), ("w:after", "0"), ("w:line", "240"), ("w:lineRule", "auto")]), Vec::new()))]; + let jc = match paragraph_styles.iter().find_map(|pp| pp.attr("algn")) { + Some("ctr") => Some("center"), + Some("r") => Some("right"), + Some("just") => Some("both"), + Some("dist") => Some("distribute"), + Some("l") => Some("left"), + _ => None, + }; + if let Some(jc) = jc { + ppr.push(Node::El(el("w:jc", attrs(&[("w:val", jc)]), Vec::new()))); + } + // The paragraph mark's own size and font (`a:endParaRPr`), which an empty one's + // height follows. + ppr.push(Node::El(run_props(p.child("a:endParaRPr"), defaults, theme))); + let mut kids = vec![Node::El(el("w:pPr", Vec::new(), ppr))]; + for r in p.els() { + let text = match r.name.as_str() { + "a:r" | "a:fld" => Node::El(el( + "w:t", + attrs(&[("xml:space", "preserve")]), + vec![Node::Text(r.child("a:t").map(|t| t.text()).unwrap_or_default())], + )), + "a:br" => Node::El(el("w:br", Vec::new(), Vec::new())), + _ => continue, + }; + let rpr = run_props(r.child("a:rPr"), defaults, theme); + kids.push(Node::El(el("w:r", Vec::new(), vec![Node::El(rpr), text]))); + } + Node::El(el("w:p", Vec::new(), kids)) + }) + .collect() +} + +/// A DrawingML run's properties (`a:rPr`, each falling back on the first of `defaults`, the +/// `a:defRPr`s in order, that gives it) as `w:rPr`: font, size, bold, italic, underline, +/// strikethrough, caps, baseline and colour. +fn run_props(rpr: Option<&El>, defaults: &[&El], theme: &[Rgb]) -> El { + let get = |n: &str| rpr.and_then(|r| r.attr(n)).or_else(|| defaults.iter().find_map(|d| d.attr(n))); + let child = |n: &str| rpr.and_then(|r| r.child(n)).or_else(|| defaults.iter().find_map(|d| d.child(n))); + // On, off (kept, so the document's style doesn't turn it back on), or not given. + let toggle = |n: &str| match get(n) { + Some("1" | "true") => Some(true), + Some("0" | "false") => Some(false), + _ => None, + }; + let mut kids = Vec::new(); + let mut push = |name: &str, a: &[(&str, &str)]| kids.push(Node::El(el(name, attrs(a), Vec::new()))); + // Theme fonts (`+mn-lt`) are left to the document's. + let face = |n: &str| child(n).and_then(|f| f.attr("typeface")).filter(|t| !t.is_empty() && !t.starts_with('+')); + let mut fonts = Vec::new(); + if let Some(f) = face("a:latin") { + fonts.extend([("w:ascii", f), ("w:hAnsi", f)]); + } + if let Some(f) = face("a:ea") { + fonts.push(("w:eastAsia", f)); + } + if let Some(f) = face("a:cs") { + fonts.push(("w:cs", f)); + } + if !fonts.is_empty() { + push("w:rFonts", &fonts); + } + for (drawingml, word) in [("b", "w:b"), ("i", "w:i")] { + match toggle(drawingml) { + Some(true) => push(word, &[]), + Some(false) => push(word, &[("w:val", "0")]), + None => {} + } + } + match get("cap") { + Some("all") => push("w:caps", &[]), + Some("small") => push("w:smallCaps", &[]), + _ => {} + } + match get("strike") { + Some("sngStrike") => push("w:strike", &[]), + Some("dblStrike") => push("w:dstrike", &[]), + _ => {} + } + match get("u") { + None | Some("none") => {} + Some("dbl") => push("w:u", &[("w:val", "double")]), + Some(_) => push("w:u", &[("w:val", "single")]), + } + match get("baseline").and_then(|v| v.trim().parse::().ok()) { + Some(b) if b > 0 => push("w:vertAlign", &[("w:val", "superscript")]), + Some(b) if b < 0 => push("w:vertAlign", &[("w:val", "subscript")]), + _ => {} + } + if let Some(rgb) = child("a:solidFill").and_then(|f| crate::read::drawing_color::color(f, theme)) { + push("w:color", &[("w:val", &rgb.hex())]); + } + // Hundredths of a point into half-points, within Word's 1–1638 pt. + let sz = get("sz").and_then(|v| v.trim().parse::().ok()).unwrap_or(DEFAULT_SZ).clamp(100, 163_800) / 50; + let sz = sz.to_string(); + push("w:sz", &[("w:val", &sz)]); + push("w:szCs", &[("w:val", &sz)]); + // WordArt's outline, shadow, glow and reflection, as Word 2010 writes them in `w:rPr`. + kids.extend(text_effects(child("a:ln"), child("a:effectLst"), theme).into_iter().map(Node::El)); + el("w:rPr", Vec::new(), kids) +} + +/// A DrawingML run's text outline (`a:ln`) and effects (`a:effectLst`: outer shadow, glow, +/// reflection) as the `w14:` run properties ([MS-DOCX] §2.6.1) the run reader takes for WordArt. +/// Colours are resolved here, theme and modifiers included, into a `w14:srgbClr` with its +/// transparency. Lengths (EMUs), angles and percentages carry over as they are. +fn text_effects(ln: Option<&El>, fx: Option<&El>, theme: &[Rgb]) -> Vec { + let color = |e: &El| -> Option { + let (rgb, transparency) = crate::read::drawing_color::color_alpha(e, theme, None)?; + let alpha = ((transparency.clamp(0.0, 100.0) * 1000.0).round() as i64).to_string(); + let kids = if transparency > 0.0 { vec![Node::El(el("w14:alpha", attrs(&[("w14:val", &alpha)]), Vec::new()))] } else { Vec::new() }; + Some(Node::El(el("w14:srgbClr", attrs(&[("w14:val", &rgb.hex())]), kids))) + }; + // The attributes `names` of `e`, under the `w14:` prefix. + let copied = |e: &El, names: &[&str]| -> Vec<(String, String)> { + names.iter().filter_map(|n| e.attr(n).map(|v| (format!("w14:{n}"), v.to_string()))).collect() + }; + let mut out = Vec::new(); + if let Some(ln) = ln + && let Some(c) = ln.child("a:solidFill").and_then(color) + { + out.push(el("w14:textOutline", copied(ln, &["w"]), vec![Node::El(el("w14:solidFill", Vec::new(), vec![c]))])); + } + for e in fx.into_iter().flat_map(|f| f.els()) { + match e.name.as_str() { + "a:outerShdw" => out.push(el("w14:shadow", copied(e, &["blurRad", "dist", "dir"]), color(e).into_iter().collect())), + "a:glow" => out.push(el("w14:glow", copied(e, &["rad"]), color(e).into_iter().collect())), + "a:reflection" => out.push(el("w14:reflection", copied(e, &["blurRad", "stA", "endPos", "dist"]), Vec::new())), + _ => {} + } + } + out +} + +fn el(name: &str, attrs: Vec<(String, String)>, kids: Vec) -> El { + El { name: name.to_string(), attrs, kids } +} + +fn attrs(a: &[(&str, &str)]) -> Vec<(String, String)> { + a.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() +} + +/// `e` under another name, with its attributes and children. +fn renamed(e: &El, name: &str) -> El { + El { name: name.to_string(), ..e.clone() } +} diff --git a/crates/docx/src/write.rs b/crates/docx/src/write.rs index 1d9eb0818..4ba486a6c 100644 --- a/crates/docx/src/write.rs +++ b/crates/docx/src/write.rs @@ -151,6 +151,11 @@ pub fn write_as(doc: &Document, flavor: Flavor) -> Result, DocxError> { } impl Writer<'_> { + /// The document's styles, for writers that need the resolved style chain. + pub(crate) fn styles(&self) -> &craft_wordmodel::styles::StyleSheet { + &self.doc.styles + } + fn assign_media(&mut self) { let mut used: std::collections::HashSet = std::collections::HashSet::new(); for (key, bytes) in &self.doc.media { diff --git a/crates/docx/src/write/props.rs b/crates/docx/src/write/props.rs index 0b92ce191..fb76da8d8 100644 --- a/crates/docx/src/write/props.rs +++ b/crates/docx/src/write/props.rs @@ -1,8 +1,8 @@ //! Property writers, in ECMA-376 schema element order. use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CellProps, CharProps, EastAsianLayout, HeightRule, Highlight, LineSpacing, ParaProps, RowProps, TabAlign, - TabLeader, TableFloat, TableLook, TableProps, TextColor, VAlign, VMerge, VertAlign, + Align, Border, BorderStyle, Borders, CellProps, CharProps, EastAsianLayout, HeightRule, Highlight, Kashida, LineSpacing, ParaProps, RowProps, + TabAlign, TabLeader, TableFloat, TableLook, TableProps, TextColor, VAlign, VMerge, VertAlign, }; use crate::units::{n, twips}; @@ -201,6 +201,11 @@ pub fn align_val(a: Align) -> &'static str { } } +/// `w:jc` for a paragraph, including the kashida length of a justified one. +pub fn jc_val(a: Align, kashida: Option) -> &'static str { + if a == Align::Justify { kashida.unwrap_or(Kashida::None).jc() } else { align_val(a) } +} + /// A tab stop's or positional tab's `w:leader` value. pub fn tab_leader(l: TabLeader) -> &'static str { match l { @@ -287,7 +292,7 @@ pub fn ppr_inner(w: &mut W, p: &ParaProps, framed: bool, num_change: Option<&Cha toggle(w, "w:contextualSpacing", p.contextual_spacing); toggle(w, "w:mirrorIndents", p.mirror_indents); if let Some(a) = p.align { - w.val("w:jc", align_val(a)); + w.val("w:jc", jc_val(a, p.kashida)); } if let Some(direction) = p.text_direction { w.val("w:textDirection", direction.ooxml()); diff --git a/crates/docx/src/write/story/drawing.rs b/crates/docx/src/write/story/drawing.rs index 94154f649..3986d7964 100644 --- a/crates/docx/src/write/story/drawing.rs +++ b/crates/docx/src/write/story/drawing.rs @@ -126,16 +126,22 @@ impl Writer<'_> { ShapeKind::ArrowDown => "downArrow", ShapeKind::Star => "star5", ShapeKind::Heart => "heart", + ShapeKind::Frame => "frame", }; match geom { Some(f) => cust_geom(w, f, *sw, *sh), None => { w.open("a:prstGeom", &[("prst", prst)]); - match extra.corner.filter(|_| *kind == ShapeKind::RoundedRectangle) { - Some(c) => { + let adj = match kind { + ShapeKind::RoundedRectangle => Some("adj"), + ShapeKind::Frame => Some("adj1"), + _ => None, + }; + match adj.zip(extra.corner) { + Some((name, c)) => { w.open("a:avLst", &[]); let fmla = format!("val {}", c.min(craft_wordmodel::connector::MAX_CORNER)); - w.empty("a:gd", &[("name", "adj"), ("fmla", &fmla)]); + w.empty("a:gd", &[("name", name), ("fmla", &fmla)]); w.close("a:avLst"); } None => w.empty("a:avLst", &[]), diff --git a/crates/docx/src/write/story/para.rs b/crates/docx/src/write/story/para.rs index e0242d8eb..aa973184e 100644 --- a/crates/docx/src/write/story/para.rs +++ b/crates/docx/src/write/story/para.rs @@ -76,6 +76,12 @@ impl Writer<'_> { a.push(("w:original", c.original.clone())); a }); + // `w:jc` encodes alignment and kashida length together. A justified paragraph + // whose kashida is unset (`None`) still inherits a length from its style; writing + // `both` would read back as `Some(Kashida::None)` and cancel that length. + if pp.align == Some(craft_wordmodel::props::Align::Justify) && pp.kashida.is_none() { + pp.kashida = Some(self.styles().resolve_para(&pp).kashida); + } ppr_inner(w, &pp, framed, num_change.as_ref()); if has_mark { self.write_mark_rpr(w, &p.mark, &mark_revs); diff --git a/crates/docx/src/write/styles.rs b/crates/docx/src/write/styles.rs index 3bcf9ab97..0fddf33b4 100644 --- a/crates/docx/src/write/styles.rs +++ b/crates/docx/src/write/styles.rs @@ -184,7 +184,13 @@ fn style_property_xml(wr: &mut Writer<'_>, w: &mut W, st: &Style) { let para_change = st.para_change.as_deref().filter(|ch| ch.old.formatting() != st.para.formatting()); if (!st.para.is_empty() || para_change.is_some()) && st.kind != StyleKind::Character { w.open("w:pPr", &[]); - props::ppr_inner(w, &st.para, false, None); + let mut ppr = st.para.clone(); + if ppr.align == Some(craft_wordmodel::props::Align::Justify) && ppr.kashida.is_none() { + let mut probe = ppr.clone(); + probe.style = Some(st.id.clone()); + ppr.kashida = Some(wr.styles().resolve_para(&probe).kashida); + } + props::ppr_inner(w, &ppr, false, None); if let Some(ch) = para_change { let a = wr.change_attrs(ch.rev); open_change(w, "w:pPrChange", &a); diff --git a/crates/docx/src/xml.rs b/crates/docx/src/xml.rs index bb86755d9..fa50d217a 100644 --- a/crates/docx/src/xml.rs +++ b/crates/docx/src/xml.rs @@ -52,8 +52,10 @@ pub const NAMESPACES: &[(&str, &str)] = &[ pub const DECORATIVE_EXT: &str = "{C183D7F6-B498-43B3-948B-1728B52AA6E4}"; pub const DECORATIVE_NS: &str = "http://schemas.microsoft.com/office/drawing/2017/decorative"; -/// Namespaces that only appear in non-body parts. +/// Namespaces that only appear in non-body parts, or that the reader reads but never writes. const OTHER_NAMESPACES: &[(&str, &str)] = &[ + // A locked canvas, read into a drawing group (wordcraft#5060). + ("lc", "http://schemas.openxmlformats.org/drawingml/2006/lockedCanvas"), ("b", "http://schemas.openxmlformats.org/officeDocument/2006/bibliography"), ("rel", "http://schemas.openxmlformats.org/package/2006/relationships"), ("ct", "http://schemas.openxmlformats.org/package/2006/content-types"), diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 348d6239e..47663008d 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -2729,6 +2729,39 @@ fn rounded_rectangle_corners_are_read_and_saved() { assert_eq!(corners(&craft_docx::read(&bytes).unwrap()), expected); } +/// A `frame` preset (a letterhead's page border) is a frame, not a filled rectangle: its border +/// width is its `adj1`, read, capped and written back as the same preset. +#[test] +fn frame_presets_keep_their_hole_and_border_width() { + let geom = + |av: &str| format!(r#"{av}"#); + let gd = |fmla: &str| format!(r#""#); + let body = [geom(&gd("val 2604")), geom(""), geom(&gd("val 90000")), geom(r#""#)] + .iter() + .map(|sppr| anchored_wsp("Frame", 2540000, 1270000, sppr, "", "")) + .collect::(); + let frames = |d: &Document| -> Vec<(ShapeKind, Option)> { + paras(d)[0] + .objects + .iter() + .filter_map(|o| match o { + InlineObject::Shape { kind, extra, .. } => Some((*kind, extra.corner)), + _ => None, + }) + .collect() + }; + let f = ShapeKind::Frame; + let expected = vec![(f, Some(2604)), (f, None), (f, Some(50_000)), (f, None)]; + let d = read_body(&format!("{body}")); + assert_eq!(frames(&d), expected); + let bytes = craft_docx::write(&d).unwrap(); + let mut xml = String::new(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes.as_slice())).unwrap(); + std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap(); + assert!(xml.contains(r#""#), "{xml}"); + assert_eq!(frames(&craft_docx::read(&bytes).unwrap()), expected); +} + /// #611: legacy VML lines and rectangles (documents without DrawingML). #[test] fn vml_lines_and_rectangles() { diff --git a/crates/docx/tests/it/kashida.rs b/crates/docx/tests/it/kashida.rs new file mode 100644 index 000000000..02eaf2a49 --- /dev/null +++ b/crates/docx/tests/it/kashida.rs @@ -0,0 +1,136 @@ +//! Justified paragraphs keep their kashida length (`ST_Jc` `lowKashida`, `mediumKashida`, +//! `highKashida`, and ordinary `both` / `justify`) through read and write. + +use std::io::{Read, Write}; + +use std::sync::Arc; + +use craft_wordmodel::props::{Align, Kashida, ParaProps}; +use craft_wordmodel::styles::{Style, StyleKind, StyleSheet}; +use craft_wordmodel::{Document, Paragraph, para_block}; + +fn zip(entries: &[(&str, &[u8])]) -> Vec { + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, bytes) in entries { + zw.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +fn docx(body: &str) -> Vec { + let root = r#""#; + let doc = format!( + r#"{body}"# + ); + zip(&[("_rels/.rels", root.as_bytes()), ("word/document.xml", doc.as_bytes())]) +} + +fn paras(d: &Document) -> Vec<&Paragraph> { + d.body.iter().filter_map(|b| b.as_para()).collect() +} + +fn document_xml(bytes: &[u8]) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut s = String::new(); + z.by_name("word/document.xml").unwrap().read_to_string(&mut s).unwrap(); + s +} + +fn jc_para(body: &str, val: &str) -> String { + format!(r#"{body}"#) +} + +#[test] +fn kashida_jc_values_are_read() { + let body = [ + jc_para("both", "both"), + jc_para("justify", "justify"), + jc_para("low", "lowKashida"), + jc_para("medium", "mediumKashida"), + jc_para("high", "highKashida"), + jc_para("left", "left"), + ] + .join(""); + let d = craft_docx::read(&docx(&body)).unwrap(); + let ps = paras(&d); + let got: Vec<_> = ps.iter().map(|p| (p.props.align, p.props.kashida)).collect(); + assert_eq!( + got, + vec![ + (Some(Align::Justify), Some(Kashida::None)), + (Some(Align::Justify), Some(Kashida::None)), + (Some(Align::Justify), Some(Kashida::Short)), + (Some(Align::Justify), Some(Kashida::Medium)), + (Some(Align::Justify), Some(Kashida::Long)), + (Some(Align::Left), None), + ] + ); +} + +#[test] +fn kashida_length_round_trips() { + let cases = [ + (Some(Kashida::None), "w:val=\"both\""), + (Some(Kashida::Short), "w:val=\"lowKashida\""), + (Some(Kashida::Medium), "w:val=\"mediumKashida\""), + (Some(Kashida::Long), "w:val=\"highKashida\""), + ]; + for (kashida, token) in cases { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { align: Some(Align::Justify), kashida, ..Default::default() }; + let mut d = Document::new(); + d.body = vec![para_block(p.clone())]; + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains(token), "expected {token} in {xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.align, Some(Align::Justify)); + assert_eq!(q.props.kashida, kashida, "round-trip of {token}"); + } +} + +fn body_style(kashida: Kashida) -> StyleSheet { + let mut sheet = StyleSheet::empty(); + sheet.styles.push(Style { + id: "Body".into(), + name: "Body".into(), + kind: StyleKind::Paragraph, + para: ParaProps { align: Some(Align::Justify), kashida: Some(kashida), ..ParaProps::default() }, + ..Style::default() + }); + sheet +} + +#[test] +fn style_kashida_length_survives_a_write_when_the_paragraph_has_no_direct_jc() { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { style: Some("Body".into()), ..ParaProps::default() }; + let mut d = Document::new(); + d.styles = Arc::new(body_style(Kashida::Short)); + d.body = vec![para_block(p)]; + assert_eq!(d.styles.resolve_para(&d.body[0].as_para().unwrap().props).kashida, Kashida::Short); + let bytes = craft_docx::write(&d).unwrap(); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.kashida, None, "no direct w:jc on the paragraph"); + assert_eq!(r.styles.resolve_para(&q.props).kashida, Kashida::Short); +} + +#[test] +fn writing_justify_without_a_local_kashida_keeps_the_style_length() { + let mut p = Paragraph::with_text("مثال", Default::default()); + p.props = ParaProps { style: Some("Body".into()), align: Some(Align::Justify), kashida: None, ..ParaProps::default() }; + let mut d = Document::new(); + d.styles = Arc::new(body_style(Kashida::Medium)); + d.body = vec![para_block(p)]; + let bytes = craft_docx::write(&d).unwrap(); + let xml = document_xml(&bytes); + assert!(xml.contains("w:val=\"mediumKashida\""), "expected mediumKashida in {xml}"); + let r = craft_docx::read(&bytes).unwrap(); + let q = paras(&r)[0]; + assert_eq!(q.props.align, Some(Align::Justify)); + assert_eq!(q.props.kashida, Some(Kashida::Medium)); + assert_eq!(r.styles.resolve_para(&q.props).kashida, Kashida::Medium); +} diff --git a/crates/docx/tests/it/locked_canvas.rs b/crates/docx/tests/it/locked_canvas.rs new file mode 100644 index 000000000..fb01406a4 --- /dev/null +++ b/crates/docx/tests/it/locked_canvas.rs @@ -0,0 +1,163 @@ +//! A DrawingML locked canvas (`lc:lockedCanvas`, ECMA-376 Part 1 §20.3.2.1), as Word 2007 writes +//! shapes, pictures and WordArt in a drawing in compatibility mode (wordcraft#5060). Fixtures are +//! hand-written from ECMA-376 Part 1 §20.1 (DrawingML main) and §20.3 (locked canvas). + +use std::io::Write; + +use craft_wordmodel::para::ShapeKind; +use craft_wordmodel::props::{Rgb, TextColor}; +use craft_wordmodel::{Document, InlineObject, StoryRef}; + +const NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main""#; + +fn docx(body: &str) -> Vec { + let doc = format!(r#"{body}"#); + let types = + r#""#; + let root = r#""#; + let rels = r#""#; + let entries: [(&str, &[u8]); 5] = [ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", root.as_bytes()), + ("word/document.xml", doc.as_bytes()), + ("word/_rels/document.xml.rels", rels.as_bytes()), + ("word/media/image1.png", "\u{89}PNG stand-in".as_bytes()), + ]; + let mut zw = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, bytes) in entries { + zw.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + zw.write_all(bytes).unwrap(); + } + zw.finish().unwrap().into_inner() +} + +/// A floating drawing, 200 × 100 pt, holding `canvas` (the locked canvas's members) in a canvas +/// whose members' space starts at 10 pt, 10 pt. +fn drawing(canvas: &str) -> String { + format!( + r#"00{canvas}"# + ) +} + +const OVAL: &str = r#""#; + +/// WordArt in a nested group: bold, all-caps red text with a green outline and a reflection, +/// 32 pt, centred, bent by a chevron warp; a second paragraph takes the list style's size, and an +/// empty third one its mark's. +const WORD_ART: &str = r#"Youngkeepers"#; + +const PICTURE: &str = r#""#; + +/// A line glued to the oval by the canvas's own id, which isn't a document shape's. +const LINE: &str = r#""#; + +fn group(d: &Document) -> (f32, f32, Vec) { + let p = d.body.iter().find_map(|b| b.as_para()).unwrap(); + match p.objects.first() { + Some(InlineObject::Group { w, h, children, canvas: None, .. }) => (*w, *h, children.clone()), + o => panic!("{o:?}"), + } +} + +/// wordcraft#5060: the locked canvas is a group of its shapes, text, picture and line, each where the +/// canvas (and the group inside it) puts it, and nothing is lost on save. +#[test] +fn a_locked_canvas_reads_as_a_group_of_its_members() { + let d = craft_docx::read(&docx(&drawing(&[OVAL, WORD_ART, PICTURE, LINE].concat()))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (w, h, children) = group(d); + assert_eq!((w, h), (200.0, 100.0)); + let at: Vec<(f32, f32)> = children.iter().map(|c| (c.x, c.y)).collect(); + assert_eq!(at, [(0.0, 0.0), (110.0, 60.0), (0.0, 50.0), (0.0, 80.0)]); + let [oval, art, picture, line] = &children[..] else { panic!("{children:?}") }; + let InlineObject::Shape { kind: ShapeKind::Ellipse, w: 100.0, h: 50.0, fill: Some(fill), stroke: Some(stroke), .. } = &oval.obj else { + panic!("{:?}", oval.obj) + }; + assert_eq!((fill.main_color(), *stroke), (Rgb(0xFF, 0xFF, 0x66), Rgb(0, 0, 0))); + let InlineObject::Shape { kind: ShapeKind::TextBox, w: 20.0, h: 10.0, story: Some(id), extra, .. } = &art.obj else { + panic!("{:?}", art.obj) + }; + assert_eq!(extra.warp.as_ref().map(|w| w.preset.as_str()), Some("textChevron")); + assert_eq!(d.plain_text(StoryRef::Part(*id)).trim_end(), "Young\nkeepers"); + let blocks = &d.parts.get(id).unwrap().blocks; + let first = blocks[0].as_para().unwrap(); + assert_eq!(first.props.align, Some(craft_wordmodel::props::Align::Center)); + assert_eq!((first.props.space_before, first.props.space_after), (Some(0.0), Some(0.0)), "DrawingML text has no paragraph spacing"); + let run = &first.runs[0].props; + assert_eq!((run.size, run.bold, run.caps, run.color), (Some(32.0), Some(true), Some(true), Some(TextColor::Rgb(Rgb(0xFF, 0, 0))))); + assert_eq!(run.font.as_deref(), Some("Corsiva Stand-in")); + let fx = run.text_effects.as_deref().unwrap(); + let outline = fx.outline.unwrap(); + assert_eq!((outline.color, outline.width), (Some(Rgb(0, 0xFF, 0)), 0.75)); + assert!(fx.reflection.is_some(), "{fx:?}"); + assert_eq!(blocks[1].as_para().unwrap().runs[0].props.size, Some(14.0), "the list style's size"); + assert_eq!(blocks[2].as_para().unwrap().mark.size, Some(10.0), "an empty paragraph's mark"); + let InlineObject::Image { w: 50.0, h: 50.0, alt, .. } = &picture.obj else { panic!("{:?}", picture.obj) }; + assert_eq!(alt, "A logo"); + let InlineObject::Shape { kind: ShapeKind::Line, stroke: Some(blue), extra, .. } = &line.obj else { panic!("{:?}", line.obj) }; + assert_eq!((*blue, extra.start), (Rgb(0, 0, 0xFF), None), "no glue to a canvas id"); + } +} + +/// A text shape as a member of the canvas (`a:txSp`, text with no shape of its own) is a text box +/// at its own rectangle. Runs take their paragraph's own defaults (`a:pPr/a:defRPr`) before the +/// list style's for the paragraph's level. +#[test] +fn a_locked_canvas_text_shape_and_paragraph_defaults() { + let text = r#"OwnLevelFirst"#; + let d = craft_docx::read(&docx(&drawing(text))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (_, _, children) = group(d); + let [shape] = &children[..] else { panic!("{children:?}") }; + assert_eq!((shape.x, shape.y), (40.0, 20.0)); + let InlineObject::Shape { kind: ShapeKind::TextBox, w: 100.0, h: 40.0, fill: None, stroke: None, story: Some(id), .. } = &shape.obj else { + panic!("{:?}", shape.obj) + }; + assert_eq!(d.plain_text(StoryRef::Part(*id)).trim_end(), "Own\nLevel\nFirst"); + let blocks = &d.parts.get(id).unwrap().blocks; + let run = |i: usize| blocks[i].as_para().unwrap().runs[0].props.clone(); + assert_eq!((run(0).size, run(0).italic), (Some(32.0), Some(true)), "the paragraph's own size, the level's italic"); + assert_eq!((run(1).size, run(1).italic), (Some(20.0), None), "level 2's"); + assert_eq!((run(2).size, run(2).italic), (Some(14.0), Some(true)), "level 1's"); + } +} + +/// A paragraph's alignment and its runs' properties fall back on its level's style, then on the +/// list style's defaults (`a:defPPr`); a run's bold set off stays off. +#[test] +fn a_locked_canvas_paragraph_takes_its_level_and_the_list_defaults() { + use craft_wordmodel::props::Align; + let text = r#"PlainCentredOwn"#; + let d = craft_docx::read(&docx(&drawing(text))).unwrap(); + for d in [&d, &craft_docx::read(&craft_docx::write(&d).unwrap()).unwrap()] { + let (_, _, children) = group(d); + let [shape] = &children[..] else { panic!("{children:?}") }; + let InlineObject::Shape { story: Some(id), .. } = &shape.obj else { panic!("{:?}", shape.obj) }; + let blocks = &d.parts.get(id).unwrap().blocks; + let para = |i: usize| blocks[i].as_para().unwrap(); + let aligns: Vec> = (0..3).map(|i| para(i).props.align).collect(); + assert_eq!(aligns, [Some(Align::Right), Some(Align::Center), Some(Align::Left)], "the list defaults', the level's, its own"); + let runs: Vec<(Option, Option)> = (0..3).map(|i| (para(i).runs[0].props.size, para(i).runs[0].props.bold)).collect(); + assert_eq!(runs, [(Some(16.0), Some(false)), (Some(16.0), Some(true)), (Some(16.0), Some(true))]); + } +} + +/// Before wordcraft#5060 only the canvas's first picture showed: the shapes and their text were dropped. +#[test] +fn a_locked_canvas_is_not_read_as_its_picture_alone() { + let d = craft_docx::read(&docx(&drawing(&[OVAL, PICTURE].concat()))).unwrap(); + let (_, _, children) = group(&d); + assert_eq!(children.len(), 2); +} + +/// Hostile canvases never panic: no members, groups nested past the limit, text with no runs, +/// huge and broken sizes. +#[test] +fn hostile_locked_canvases_never_panic() { + let deep = format!("{}{}{}", r#""#.repeat(40), OVAL, "".repeat(40)); + let odd = r#"f"#; + for canvas in ["", deep.as_str(), odd] { + let d = craft_docx::read(&docx(&drawing(canvas))).unwrap(); + let _ = craft_docx::write(&d).unwrap(); + } +} diff --git a/crates/docx/tests/it/main.rs b/crates/docx/tests/it/main.rs index d1def3f8c..8d6be8a97 100644 --- a/crates/docx/tests/it/main.rs +++ b/crates/docx/tests/it/main.rs @@ -20,9 +20,11 @@ mod emphasis_marks; mod fixtures; mod glossary; mod heading_levels; +mod kashida; mod legacy_forms; mod ligatures; mod line_numbering; +mod locked_canvas; mod malformed; mod missing_pictures; mod nested_block_controls; diff --git a/crates/docx/tests/it/roundtrip.rs b/crates/docx/tests/it/roundtrip.rs index 65a01e6b4..2037f9849 100644 --- a/crates/docx/tests/it/roundtrip.rs +++ b/crates/docx/tests/it/roundtrip.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use craft_wordmodel::numbering::ListKind; use craft_wordmodel::para::{Anchor, Float, FloatAlign, NoteKind, ShapeKind, Wrap}; use craft_wordmodel::props::{ - Align, Border, BorderStyle, Borders, CharProps, HeightRule, Highlight, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, RowProps, - TabAlign, TabLeader, TabStop, TableLook, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, + Align, Border, BorderStyle, Borders, CharProps, HeightRule, Highlight, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, Rgb, + RowProps, TabAlign, TabLeader, TabStop, TableLook, TextAlignment, TextColor, TextDirection, Underline, VAlign, VMerge, VertAlign, }; use craft_wordmodel::section::{Columns, DocGrid, DocGridType, LineNumberRestart, LineNumbering, NumFormat, PageVAlign, SectionProps, SectionStart}; use craft_wordmodel::styles::{Style, StyleKind}; @@ -163,6 +163,7 @@ fn every_para_prop_round_trips() { let pp = ParaProps { style: Some("Heading1".into()), align: Some(Align::Justify), + kashida: Some(Kashida::Medium), indent_left: Some(36.0), indent_right: Some(18.0), indent_first: Some(-18.0), diff --git a/crates/flow/src/from_doc.rs b/crates/flow/src/from_doc.rs index 810e67d55..b7204f1bf 100644 --- a/crates/flow/src/from_doc.rs +++ b/crates/flow/src/from_doc.rs @@ -342,6 +342,20 @@ fn rich_inlines(doc: &Document, o: &InlineObject, f: &Fmt, opts: FlowOpts) -> Op }) } +/// The freeform a `w` × `h` frame shape with border `corner` (see +/// [`craft_wordmodel::connector::frame_width`]) draws: its outline clockwise and its hole the +/// other way, so the hole stays empty under either fill rule. +fn frame_freeform(corner: Option, w: f32, h: f32) -> craft_wordmodel::freeform::Freeform { + use craft_wordmodel::freeform::{FreePath, Freeform}; + let (w, h) = (craft_wordmodel::geom::finite(w).max(0.0), craft_wordmodel::geom::finite(h).max(0.0)); + let t = craft_wordmodel::connector::frame_width(corner, w, h); + let mut paths = vec![FreePath { pts: vec![[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]], closed: true }]; + if w - 2.0 * t > 0.0 && h - 2.0 * t > 0.0 { + paths.push(FreePath { pts: vec![[t, t], [t, h - t], [w - t, h - t], [w - t, t]], closed: true }); + } + Freeform { w, h, paths, ..Default::default() } +} + /// Comment `c` (a reply to `parent`) of `doc` as a flow comment: its text holds no comments or /// notes of its own. /// The flow inline for picture, shape or group `o` of `doc` (a member of groups `depth` deep), @@ -358,8 +372,12 @@ fn drawing_inline(doc: &Document, o: &InlineObject, opts: FlowOpts, depth: usize { // A connector stays one in formats that glue them (ODT, with boxes); elsewhere it is a // straight line. + // A frame (a border with a hole), which the other formats have no shape for, is the + // freeform it draws: its outline and, wound the other way, its hole. + let frame = (*kind == ShapeKind::Frame).then(|| std::sync::Arc::new(frame_freeform(extra.corner, *w, *h))); let kind = match kind { ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector if !opts.boxes => ShapeKind::Line, + ShapeKind::Frame => ShapeKind::Freeform, k => *k, }; let inner = FlowOpts { notes: false, comments: false, boxes: false, ..opts }; @@ -383,7 +401,7 @@ fn drawing_inline(doc: &Document, o: &InlineObject, opts: FlowOpts, depth: usize Box::new(crate::Chain { name: format!("Frame{id}"), next }) }), }; - let path = freeform.clone().filter(|_| kind == ShapeKind::Freeform); + let path = frame.or_else(|| freeform.clone()).filter(|_| kind == ShapeKind::Freeform); Some(Inline::Shape { kind, w: w * sx, h: h * sy, float: kept_float(float), blocks, look, path }) } InlineObject::Image { media, w, h, alt, float, ole, crop, .. } => { diff --git a/crates/flow/src/to_doc.rs b/crates/flow/src/to_doc.rs index ce29a4448..0375c448c 100644 --- a/crates/flow/src/to_doc.rs +++ b/crates/flow/src/to_doc.rs @@ -841,7 +841,8 @@ impl<'a> Builder<'a> { tb.props.style = None; } if t.widths.len() == cols && t.widths.iter().all(|w| w.is_finite() && *w > 1.0) { - tb.grid = t.widths.iter().map(|w| w.clamp(6.0, 1584.0)).collect(); + // Narrow columns stay as they are: raising them would widen the table past its source. + tb.grid = t.widths.iter().map(|w| w.min(1584.0)).collect(); } tb.rows.clear(); if let Some(x) = &t.extra { diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs index 2a7c82d1a..09f95e0b8 100644 --- a/crates/frame/src/lib.rs +++ b/crates/frame/src/lib.rs @@ -221,6 +221,10 @@ impl VideoFrame { /// without alpha, and a copy for RGBA8). pub fn to_rgba8(&self) -> Vec { let (w, h) = (self.width as usize, self.height as usize); + // Empty pictures have no rows; Rayon rejects a zero-sized row chunk. + if w == 0 || h == 0 { + return Vec::new(); + } match &self.data { PixelData::Rgba8(d) => d.as_ref().clone(), PixelData::Yuv8 { planes, chroma, alpha: None } if matches!(self.color.transfer, Transfer::Bt709 | Transfer::Srgb) => { @@ -303,6 +307,16 @@ pub fn default_matrix(width: u32, height: u32) -> Matrix { mod tests { use super::*; + #[test] + fn empty_frames_convert_without_zero_sized_chunks() { + for (w, h) in [(0, 0), (0, 1), (1, 0)] { + let frame = VideoFrame::transparent_f32(w, h); + assert!(frame.to_rgba8().is_empty()); + assert!(frame.to_linear_f32().is_empty()); + assert!(frame.luma8().is_empty()); + } + } + #[test] fn rgba8_roundtrips_through_to_rgba8() { let px = vec![10u8, 20, 30, 255, 0, 0, 0, 0]; diff --git a/crates/image-codecs/src/codecs/tiff.rs b/crates/image-codecs/src/codecs/tiff.rs index bf59157eb..0f53bb3da 100644 --- a/crates/image-codecs/src/codecs/tiff.rs +++ b/crates/image-codecs/src/codecs/tiff.rs @@ -207,8 +207,10 @@ fn camera_raw_mode(b: &[u8], legacy: bool) -> bool { } TAG_SUB_IFDS if count == 1 => queue.extend(value(e)), TAG_SUB_IFDS => { - // More than one offset: they are stored at the entry's offset. - let at = u32_at(e.saturating_add(8)).unwrap_or(0) as usize; + // More than one offset: they are stored at the entry's offset. A cut-off + // offset field means no array, not one at the start of the file. + let Some(at) = u32_at(e.saturating_add(8)) else { continue }; + let at = at as usize; for k in 0..count.min(8) as usize { queue.extend(u32_at(at.saturating_add(4 * k))); } diff --git a/crates/image-codecs/tests/legacy_tiff_raw.rs b/crates/image-codecs/tests/legacy_tiff_raw.rs index 770801734..27cf6b881 100644 --- a/crates/image-codecs/tests/legacy_tiff_raw.rs +++ b/crates/image-codecs/tests/legacy_tiff_raw.rs @@ -63,3 +63,23 @@ fn oversized_or_cut_directories_are_skipped() { cut.truncate(20); assert!(!legacy_tiff_raw_hint(&cut)); } + +#[test] +fn a_subifds_array_without_its_offset_is_skipped() { + // IFD0 at 64 holds one SubIFDs entry (LONG, count 3) whose offset field is cut off. A DNG + // directory sits at 32, and the bytes at 8 point to it. Reading the missing offset as 0 + // would treat the TIFF header as the pointer array and follow 8 -> 32. + let mut b = vec![0u8; 64]; + b[..8].copy_from_slice(b"II\x2a\0\x40\0\0\0"); + b[8..12].copy_from_slice(&32u32.to_le_bytes()); + b[32..34].copy_from_slice(&1u16.to_le_bytes()); + b[34..46].copy_from_slice(&[0x12, 0xc6, 1, 0, 4, 0, 0, 0, 1, 4, 0, 0]); + b.extend_from_slice(&1u16.to_le_bytes()); + b.extend_from_slice(&[0x4a, 0x01, 4, 0, 3, 0, 0, 0]); + assert_eq!(b.len(), 74); + assert!(!legacy_tiff_raw_hint(&b)); + // With the offset present (the array at 8), the same SubIFD is followed. + let mut whole = b.clone(); + whole.extend_from_slice(&8u32.to_le_bytes()); + assert!(legacy_tiff_raw_hint(&whole)); +} diff --git a/crates/ipc/Cargo.toml b/crates/ipc/Cargo.toml new file mode 100644 index 000000000..5d4b0ff06 --- /dev/null +++ b/crates/ipc/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "craft-ipc" +description = "Local-socket control server, client and discovery for Craft apps on one machine: token auth, launch on demand, focus hand-off. Native only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +# Native only: `interprocess` has no wasm32 build, so this crate does not build for +# wasm32-unknown-unknown. Apps gate it by target. interprocess is pinned: the transport behavior is +# what the tests were run against. +[dependencies] +# Calendar checks only (AGENTS.md: all date logic goes through chrono). +chrono.workspace = true +getrandom = "0.2.15" +interprocess = "=2.4.4" +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10.9" + +[target.'cfg(unix)'.dependencies] +# O_NONBLOCK and O_NOFOLLOW for opens that must not wait on a FIFO or follow a symlink. Constants only. +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +widestring = "1" + +[dev-dependencies] +# Default features off: the tests read local schemas and need no network resolver. +jsonschema = { version = "0.49.9", default-features = false } + +[lints] +workspace = true diff --git a/crates/ipc/PROTOCOL.md b/crates/ipc/PROTOCOL.md new file mode 100644 index 000000000..27945f021 --- /dev/null +++ b/crates/ipc/PROTOCOL.md @@ -0,0 +1,297 @@ +# Craft Link protocol, version 1 + +Extends the existing Craft control channel (the control protocol document of each app). Every +existing method keeps its exact behaviour. New methods are additive. `craft-ipc` implements this +protocol. + +## 1. Transport and framing + +- **Endpoint**: from the instance record (`schemas/instance-record.schema.json`). Unix domain socket file + (mode 0600, directory 0700) on Linux and macOS; named pipe with an owner-only SDDL descriptor on + Windows; loopback TCP only when the user opts in (`--control `, legacy). +- **Control framing**: one UTF-8 JSON object per line, max 4 MiB per line. A request has `method` + (string), optional `id` (number or string), optional `params` (object). A reply has the request's + `id`, `ok` (bool), and `result` or `error`. Malformed input gets one error line and the connection is + closed (existing behaviour). +- **Push events** (new): lines with `event` and `seq` and no `id`: + `{"event": "", "seq": , "data": {...}}`. Sent only on connections that called + `events.subscribe`. Per-subscriber queue of 256; on overflow the server sends + `{"event":"events.overflow","seq":n,"data":{"since":}}` and drops events until the client + catches up. +- **Bulk connection** (new): a second connection to the same endpoint, used only for frames (section 6). +- **Connection limit**: 16 concurrent connections per app (existing). Bulk connections count. At most + 4 of the 16 may be unauthenticated at one time; the other 12 slots go only to peers that sent the + right token. A peer past either limit gets one line `{"ok":false,"error":"too many control + connections"}` or `"too many unauthenticated connections"` and a close. +- **Time limits** (server defaults): a connection must send a valid `auth` line within 2 s of the + accept; a request line that has started must end within 10 s; an authenticated control + connection with no input for 1 h, no event subscription and no bulk role is closed with + `{"id":null,"ok":false,"error":"idle connection closed"}`. A client that keeps a quiet connection + longer must call `events.subscribe` or reconnect. A request may arrive in any number of pieces + with pauses between them; the server keeps the bytes of an unfinished line. +- **Launch on demand** (library rule, not wire): `craft_ipc::launch` runs an app binary only if the + path is absolute and the file is regular and executable, and the file and its directory belong to + the user or root and are not writable by every user. Empty and relative `PATH` entries are + skipped. A fork that starts an app by another route must call `craft_ipc::trust::check_executable`. + Caches use `craft_ipc::cache::private_cache_dir` (`0700`, owner checked, no symlink, no + fallback to a shared temp directory); `CRAFT_LINK_CACHE_DIR` must be absolute. + +## 2. Authentication (required on every transport: sockets, pipes, loopback TCP) + +The first line on every connection must be: + +```json +{"id": 0, "method": "auth", "params": {"token": "<64 hex from the instance record>"}} +``` + +Reply `{"id":0,"ok":true,"result":{"authenticated":true}}`, or +`{"id":0,"ok":false,"error":"authentication required"}` followed by close. The server also checks +the peer's user id (Linux/macOS euid; Windows token user) and refuses other users. Any other first +line gets the same refusal. (Shape copied from PhotoCraft's existing handshake.) + +Loopback TCP (`--control `, legacy) runs the same handshake, the same unauthenticated cap and +the same 2 s auth deadline. It binds 127.0.0.1 only. There is no uid check on TCP, so the token is +the only proof. The instance record (0600) publishes the port as `tcp_port` next to the `token`; +a TCP client reads both from the record (`Client::connect_tcp`). Never print the token or put it +in a config file or on a command line. A TCP client without a token gets the auth refusal. + +## 3. Session methods (answered on the connection thread, no UI frame needed) + +### `link.hello` +```json +{"id":1,"method":"link.hello","params":{"client":{"app":"filmcraft","app_version":"0.9.1","pid":4242,"instance":""},"protocol":{"min":1,"max":1}}} +-> {"id":1,"ok":true,"result":{"app":"effectcraft","app_version":"0.8.3","pid":5151,"instance":"","protocol":1,"caps":["events","frames","activate","link.receive"]}} +``` +Errors: `protocol_unsupported{server_min, server_max}`. The client must check `pid` and `instance` +equal the instance record. Legacy clients that never call `link.hello` keep today's behaviour. + +`caps` is the fixed base set (`events`, `activate`, `link.receive`, `frames`) plus the `caps` array of +the instance record (`schemas/instance-record.schema.json`), without duplicates. Roles and optional verbs use +this array. Defined names: + +| Cap | Meaning | +|---|---| +| `hub` | The peer is a browser or launcher hub. It owns no document, it only reads and opens assets (section 9). | +| `assets` | The peer answers `link.resolve` and `asset.describe` itself, on the connection thread (section 4). | + +A client that does not know a cap ignores it. + +### `events.subscribe` / `events.unsubscribe` +```json +{"id":2,"method":"events.subscribe","params":{"events":["doc.changed","link.changed","app.closing"]}} +-> {"id":2,"ok":true,"result":{"seq":0}} +``` + +### `app.activate` +```json +{"id":3,"method":"app.activate","params":{"token":"","from_pid":4242,"open":{"link_id":""}}} +-> {"id":3,"ok":true,"result":{"activated":"requested"}} +``` +`activated` has three values: + +| Value | Meaning | +|---|---| +| `true` | The window has focus now. | +| `"requested"` | The app asked the OS to focus the window. The OS decides later. Nobody confirmed it, and the reply does not say it will happen. | +| `false` | Focus was not taken. The app asked for user attention instead (taskbar flash, dock bounce). | + +On Linux, `"requested"` needs a token that the app could apply to its existing window. An app whose +toolkit can mint a token but cannot apply one (stock winit 0.30 on Wayland) answers `false`. The +sender can still pass `token`. Old senders and receivers keep working: the three values did not change. +`activated` is distinct from the existing `ui.focus` method, which never steals focus by design. + +`open` is an object that names what to show. It carries `link_id` (open a link), `asset` (a moniker, +section 9: open that asset in its document) or both. A hub or launcher uses `asset` to send the user +to the app and document that owns the asset, with no link in between. The server rejects an `asset` +that is not a valid moniker with `bad_params`. An `open` with neither key is passed on unchanged +(existing behaviour). The receiving app decides what "show" means for the asset. + +### `link.bye` +Clean close. The server drops subscriptions and bulk tickets of this connection. + +## 4. Link methods (run on the UI thread through the app's existing request channel) + +### `link.open` (FilmCraft -> EffectCraft) +```json +{"id":10,"method":"link.open","params":{ + "link_id":"", + "payload":{"format":"otio-json","data":""}, + "place":{"seconds":12.48,"selection":[{"start_s":10.0,"end_s":14.0,"track_index":0}],"view":{"zoom":1.0}}, + "result_path":"/abs/path/Project Links/.mov", + "source":{"app":"filmcraft","doc_id":"","item":""}}} +-> {"id":10,"ok":true,"result":{"comp":"","project":"/abs/path/Project Links/.ecproj", + "format":{"w":1920,"h":1080,"rate_num":24,"rate_den":1,"duration_ticks":...,"alpha":true}, + "unsupported":[{"what":"effect","name":"Lumetri Color","kept_as":"none"}]}} +``` +Errors: `payload_invalid`, `media_missing`, `payload_too_large` (OTIO string > 4 MiB), `io_error`. +A request line longer than 4 MiB (`MAX_LINE`) is not an error reply: the server answers once in text +and closes the connection. `unsupported` lists parts that +did not survive the hand-off. + +**Payload formats.** `payload.format` selects the shape of the rest of `payload`: + +| `format` | Other fields | Use | +|---|---|---| +| `otio-json` | `data`: OTIO JSON string | Video: a nested sequence (above). | +| `image-doc` | `asset`: moniker (section 9), `path`: absolute path of the producer document or image file | Stills, vectors and other single images. `result_path` is the last-known flattened image. `place`, `source` and `format` in the reply stay as above. `rate_num`, `rate_den` and `duration_ticks` are absent from `format` in the reply. | + +An unknown `format` gets `payload_invalid`. A new `format` value is additive and needs no protocol +bump. The server checks the shape of `payload` with `craft_ipc::payload::OpenPayload` before it +forwards `link.open` to the app. A missing `payload`, a `payload` that is not an object, a missing or +unknown `format`, a missing field, a relative `path` or a bad `asset` moniker gets `payload_invalid` +and the app never sees the request. `params` that is not an object stays `bad_params`. The server +checks shape, and for `otio-json` it also opens every `ExternalReference` whose `target_url` is a +`file://` URL or an absolute path. A file that does not open gets `media_missing` with +`data.missing` (the paths) and the app never sees the request. A `file://` URL whose path has a bad +percent escape, or does not decode to UTF-8, gets `payload_invalid`. Relative targets are left to the app. + +Path rules. `result_path`, `payload.path` and every absolute media `target_url` must +be absolute, hold no NUL byte and no `..` component, or the request gets `payload_invalid`. The +server does not canonicalise and does not confine paths to a root: footage lives anywhere and +may be a symlink. The media check is the only file probe. It looks only at paths the payload +names, at most 4096 distinct ones, and reports a path as missing when it is not a regular file +(after symlinks) or does not open for reading. A missing file, a directory, a FIFO, a device and a +file without read access all look the same, and the probe never opens a FIFO. An OTIO document +with more than 4096 distinct files, more than 2^20 JSON values or nesting over 128 gets +`payload_invalid`. The app must apply its own checks to `result_path` before it writes there; +the server check is a first filter. +An app still answers `payload_invalid` when the content is bad (for example OTIO that does not +parse). + +### `link.attach` (FilmCraft -> EffectCraft, reopen an existing link) +`{"link_id","project","comp"}` -> `{"format", "epoch"}`. Errors: `comp_missing` (link becomes `Broken`). +The receiver makes the comp of the link the active one. It reads `project` (the sidecar written by +`link.open`) when it holds no such link and has no unsaved edits, otherwise `busy`. +FilmCraft sends this when `link.editIn` runs on a clip that plays a linked item already. It nests +nothing and creates no item. On `comp_missing` or `io_error` (no sidecar, damaged sidecar) it sends +`link.open` again with the same `link_id` and `result_path`, built from the nest in `nested_sequence`. + +### `link.focus` (either direction, used by Back) +`{"link_id","place"}` -> `{}`. The receiver moves to `place` and is then activated with `app.activate`. + +### `link.close` +`{"link_id"}` -> `{}`. EffectCraft drops the binding; the sidecar project stays on disk. + +### `link.refresh` (consumer side, any app that holds a link) +`{"link_id"}` -> `{"link_id","epoch","state":"current"|"modified"|"missing","result_path","hash"}`. +The receiver compares the last-known result file with the disk (rules in section 10), bumps the epoch +when the content changed, and answers the state. `hash` is `sha256:<64 hex>` and is absent when the +state is `missing`. A link that exists in the receiver but whose file is gone is not an error. The +answer is `state:"missing"`. Errors: `link_unknown`, `bad_params`, `io_error`. + +### `link.resolve` (producer or hub, any app that owns assets) +`{"asset":"craft://..."}` -> `{"asset","path","epoch","size","hash","mtime_ms"}`. `path` is an +absolute path of the last-known flattened result that a consumer can read now. `hash` is +`sha256:<64 hex>` of that file. `epoch` is the producer epoch. Errors: `link_unknown` (no such asset, +or the asset has no result file yet), `bad_params` (not a moniker), `io_error`. + +### `asset.describe` (producer or hub) +`{"asset":"craft://..."}` -> `{"asset","name","kind","epoch","w","h","hash","thumb_path","modified_ms"}`. +`kind` is `still`, `vector`, `sequence` or `comp`. `w`, `h`, `hash` and `thumb_path` are `null` when +unknown. `thumb_path` is an absolute path of a PNG, at most 512 px on the long side, that the +producer keeps in a per-document sidecar. A hub browses assets with this call and never opens the +document. Errors as `link.resolve`. + +`link.resolve` and `asset.describe` have no side effects and never need a UI frame. A server that was +started with an asset provider (`Server::start_with` and `Extensions`) answers both on the connection thread +and lists `assets` in `caps`. `link.refresh` is answered by the provider the same way. A server +started without a provider forwards all three to the app like any other link method (existing +behaviour). + +### Events +- `link.changed`: `{"link_id","epoch"}`. Sent by EffectCraft when the bound comp changes (any + `bump()` whose change touches the comp). +- `link.result`: `{"link_id","epoch","path"}`. A fresh last-known result was written. +- `doc.changed`: `{"doc_id","revision","cause"}`. Generic change feed (from `craft-command`). +- `app.closing`: `{}`. Sent before a clean exit. + +## 5. Command passthrough (existing, unchanged) + +`engine.execute {command, params}`, `engine.commands`, `ui.*`, `perf.stats` keep their current +contracts. `engine.commands` now also lists plugin commands and carries a `schema` per command +(`x-craft-source`: `doc` | `derived` | `opaque`). + +## 6. Bulk frames + +1. Control connection: `frames.open {"link_id","format":"rgba8_premul"|"rgba16f_premul","scale":1.0}` + -> `{"ticket":"<32 hex>","w","h","stride","len"}`. +2. New connection: `auth`, then `{"method":"bulk.attach","params":{"ticket":"..."}}` -> `{"ok":true}`. + The connection is now binary in both directions. +3. Client request record (binary): `'CFRQ' | u32 header_len | header JSON` with header + `{"req":,"time_s":,"epoch":,"region":[x,y,w,h]|null}`. +4. Server reply record: `'CFRM' | u32 header_len | u64 payload_len | header JSON | payload`, header + `{"req","epoch","w","h","stride","format","render_ms"}`. `epoch` is the epoch the frame was + rendered at; a client that asked for an older epoch may get a newer one. `payload_len` is + `stride * h`, and `stride` is at least one row: `w` times 4 bytes for `rgba8_premul`, `w` times 8 + bytes for `rgba16f_premul`. A layout that breaks this is refused at `frames.open`, and a reply + that breaks it is refused by the server and by the client. +5. Errors: header `{"req","error":"..."}` with `payload_len` 0. +All integers little endian. Tickets expire when the control connection closes. + +## 7. Error codes (in `error` objects for new methods) + +`{"code": "", "message": "", "data": {...}}`. Codes: `auth_required`, +`protocol_unsupported`, `unknown_method`, `bad_params`, `not_ready`, `payload_invalid`, +`media_missing`, `payload_too_large`, `comp_missing`, `link_unknown`, `io_error`, `busy`, `internal`. +`link_unknown` also covers an unknown asset moniker. +Existing methods keep their current string `error` field. + +## 8. Versioning + +- `protocol` is an integer. A server serves any client version in `[min, max]`. New methods and new + optional fields do not bump the version. Removing or changing the meaning of a field does. +- Method names are never reused with a different meaning. + +## 9. Asset identity: the moniker + +An asset is addressed by a moniker, a string that stays valid across restarts: + +``` +craft:///// +``` + +| Part | Rule | +|---|---| +| `app` | Same pattern as the instance record `app`: `^[a-z][a-z0-9-]{1,31}$`. The app that owns the document. | +| `doc` | The document id, a UUID in 8-4-4-4-12 hex form. Parsers accept either hex case and keep the text as written. | +| `kind` | `^[a-z][a-z0-9-]{0,31}$`. Known values: `comp`, `image`, `vector`, `sequence`, `layer`. An unknown value is valid and is kept. | +| `id` | Non-empty, at most 512 bytes after decoding, valid UTF-8. Characters other than `A-Z a-z 0-9 - . _ ~` are percent-encoded as `%XX` (uppercase hex) by the formatter. The parser accepts either hex case. A raw `/`, `?`, `#` or `%` not followed by two hex digits is invalid. | + +The `asset.moniker` pattern in `schemas/link-record.schema.json` states the same `id` rules, so a +record that the schema accepts also parses in Rust. + +A moniker has no query and no fragment. It names an asset, not a state: the state is the epoch. +A moniker can outlive its asset. Resolving a dead moniker gives `link_unknown`. Parsing then +formatting a moniker gives the canonical text (uppercase `%XX`). Formatting then parsing gives back +an equal moniker. `craft_ipc::moniker::Moniker` implements both. The module uses only `serde` and +`std`, so an app can copy it to a wasm build. The `craft-ipc` crate as a whole still does not build +for wasm32 (local sockets). + +## 10. Refresh and watch rules for linked files + +These rules apply to any linked asset with a `result_path` (a still, a rendered movie). + +1. **Producer writes are atomic.** The producer writes the file to a temporary name in the same + directory (`....tmp`), flushes it, renames it over `result_path`, and then flushes + the directory. A reader sees the old file or the new file, never a half file. A reader ignores + names that start with `.` and end with `.tmp`. +2. **Epoch.** `result_epoch` is a counter of result versions. The producer adds 1 on every saved + change and emits `link.changed` and `link.result`. A consumer that detects a change by polling + (rule 4) adds 1 to its own stored epoch. When the producer is reachable again, the consumer keeps + the larger of the two values. An epoch never decreases. +3. **Fingerprint.** `(size, mtime_ms, sha256)` of the file. The hash is of the full content. +4. **Polling when the producer is offline.** The consumer calls `poll` about once per second + (`craft_ipc::watch::Watcher` runs this schedule with a thread and an injectable ticker). A poll + reads size and mtime. It hashes the file only when size or mtime differ from the last fingerprint. + Some file systems have coarse mtime, so the consumer also calls a deep poll (always hash) about + once every 30 seconds. A touch that leaves the hash equal does not bump the epoch. +5. **States.** `current` (disk equals what the consumer loaded), `modified` (disk differs from what + the consumer loaded; the epoch was bumped), `missing` (the file does not exist). The consumer + moves `modified` to `current` when it has loaded the new content. A file that returns with the + hash it had when loaded goes back to `current` with no epoch bump. A file that returns with a + different hash is `modified` and bumps the epoch. +6. **Other read errors** (permission denied, I/O error) are errors, not `missing`. The state does not + change and the next poll tries again. +7. **Relink.** A consumer can point a link at a new path. The new path is polled at once. The state + is `current` when its hash equals what the consumer loaded and `modified` (epoch bump) otherwise. + A missing new path is rejected and the old path stays. diff --git a/crates/ipc/README.md b/crates/ipc/README.md new file mode 100644 index 000000000..8bcdbea66 --- /dev/null +++ b/crates/ipc/README.md @@ -0,0 +1,38 @@ +# craft-ipc + +`craft-ipc` is how a Craft app talks to another Craft app on the same machine. It has a server, a client, instance discovery, launch on demand and focus hand-off. It has no async runtime. The wire protocol is in [PROTOCOL.md](PROTOCOL.md). The crate depends on no app and on no other Craft crate. + +**Native only.** `craft-ipc` does not build for `wasm32-unknown-unknown`, because `interprocess` (local sockets and named pipes) has no wasm32 build. An app gates the dependency by target. The pure data modules (`moniker`, `record`, `payload`, `asset`, `ids`) use only `std` and `serde`. + +## Boundaries + +- Never crashes: no panics, no `unsafe`. +- Every connection must present a 64 hex token within 2 seconds. The server also checks the peer user id (Unix) or the token user (Windows). At most 16 connections are open, and at most 4 of them are unauthenticated. +- One UTF-8 JSON object per line, at most 4 MiB per line. A request can arrive in pieces with pauses between them. `wire::LineReader` keeps the partial line. Do not read control lines with a stateless `read_line` on a stream that has a receive timeout. +- On Unix, the socket file is mode `0600` inside a `0700` directory. The Windows code paths (named pipe, token user check) have not been run on Windows. + +## Modules + +| Module | Role | +|---|---| +| `server`, `client`, `wire`, `bulk`, `auth`, `launch`, `activate`, `discovery` | transport, session, frames, focus hand-off, instance records | +| `moniker`, `record`, `payload`, `asset`, `ids` | pure data types | +| `refresh` | `Tracker` (poll, epoch bump, relink) and `write_atomic` for linked files | +| `fetch` | `FrameFetcher`: ticket, a few `BulkClient` workers, a RAM cache keyed `FrameKey` with a byte budget and LRU eviction. `fetch` never waits past its `wait` argument. A late frame still lands in the cache | +| `watch` | `Watcher` thread that polls `Tracker`s: shallow every second, deep every 30 seconds. `IntervalTicker` runs on real time and `ManualTicker` is for tests | +| `trust`, `cache` | `check_executable` guards every launch (absolute path, regular file, executable, owner and mode checks). `private_cache_dir`, `write_private` and `read_private` make a `0700` cache with no `/tmp` fallback | + +- `server::Limits` holds the connection limits: auth deadline 2 s, started line 10 s, idle 1 h for unsubscribed control connections, 4 unauthenticated of 16. +- `payload::check_peer_path` is the one path filter: absolute, no NUL, no `..`. The media probe only touches regular files and never opens a FIFO. +- `Server::start_with(config, Extensions { assets, caps })` adds an asset provider and extra caps. `InstanceRecord` keeps `caps` in `extra`. Use `caps()` and `add_cap()`. +- Discovery: each app writes one JSON record, mode `0600`, into a per-user runtime directory. A lock file held for the life of the app shows whether the owner is alive. +- Launch: `launch::resolve` finds an app binary (environment, a read-only parse of the launcher's install record, `PATH`, usual install locations). `launch` starts it only when `trust::check_executable` accepts the file. +- Loopback TCP is available only when the app opts in. It binds 127.0.0.1 and runs the same handshake. A TCP client reads the port and token from the instance record (`Client::connect_tcp`). + +## Schemas and fixtures + +`schemas/instance-record.schema.json` and `schemas/link-record.schema.json` describe the stored records. `tests/schemas.rs` checks that each schema and its Rust type agree, and that the stored records in `tests/fixtures/` stay valid. + +## Tests + +A forked child holds a copy of every open file descriptor until exec, so a `flock` released by `drop` can look held. Unit tests that start a process take `crate::SPAWN_GUARD.write()`. Unit tests that publish or scan a discovery lock take `.read()`. diff --git a/crates/ipc/schemas/instance-record.schema.json b/crates/ipc/schemas/instance-record.schema.json new file mode 100644 index 000000000..6cc9b4f15 --- /dev/null +++ b/crates/ipc/schemas/instance-record.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "craft://schemas/link/instance-record/1", + "title": "Craft Link instance record", + "description": "Written by each running Craft app to /craft/-.json (mode 0600). See PROTOCOL.md.", + "type": "object", + "required": ["v", "app", "app_version", "pid", "instance", "endpoint", "token", "started", "protocol", "docs"], + "additionalProperties": true, + "properties": { + "v": { "const": 1 }, + "app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$" }, + "app_version": { "type": "string" }, + "pid": { "type": "integer", "minimum": 1 }, + "instance": { "type": "string", "format": "uuid" }, + "endpoint": { + "oneOf": [ + { + "type": "object", + "required": ["kind", "path"], + "properties": { "kind": { "const": "unix" }, "path": { "type": "string", "maxLength": 100 } } + }, + { + "type": "object", + "required": ["kind", "name"], + "properties": { "kind": { "const": "pipe" }, "name": { "type": "string", "pattern": "^craft-" } } + }, + { + "type": "object", + "required": ["kind", "port"], + "properties": { "kind": { "const": "tcp" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 } } + } + ] + }, + "token": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "started": { "type": "integer", "description": "unix seconds" }, + "protocol": { + "type": "object", + "required": ["link"], + "properties": { + "link": { "type": "integer", "minimum": 1 }, + "commands_schema": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } + } + }, + "caps": { + "type": "array", + "description": "Roles and optional verbs this app offers. Merged into the link.hello caps (link-protocol.md section 3). 'hub' marks a hub peer, 'assets' marks an app that answers link.resolve and asset.describe on the connection thread. Absent means none.", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z][a-z0-9._-]{0,63}$" } + }, + "docs": { + "type": "array", + "items": { + "type": "object", + "required": ["doc_id", "title"], + "properties": { + "doc_id": { "type": "string" }, + "title": { "type": "string" }, + "path": { "type": "string" } + } + } + } + } +} diff --git a/crates/ipc/schemas/link-record.schema.json b/crates/ipc/schemas/link-record.schema.json new file mode 100644 index 000000000..0a1183085 --- /dev/null +++ b/crates/ipc/schemas/link-record.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "craft://schemas/link/link-record/1", + "title": "craft.link project item metadata", + "description": "Stored in the consumer's project item metadata[\"craft.link\"] on the linked media item. Unknown fields must be preserved on save. Records written before the asset fields existed (FilmCraft to EffectCraft) stay valid.", + "type": "object", + "required": ["v", "link_id", "target_app", "target_project", "result_path", "result_epoch", "format", "created"], + "anyOf": [ + { "required": ["target_comp"] }, + { "required": ["asset"] } + ], + "additionalProperties": true, + "properties": { + "v": { "const": 1 }, + "link_id": { "type": "string", "format": "uuid" }, + "target_app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$", "description": "app id of the producer, same pattern as the instance record app" }, + "target_project": { "type": "string", "description": "path of the producer document, relative to the consumer project file when possible" }, + "target_comp": { "type": "string", "description": "comp id inside the producer document. Required unless asset is present." }, + "nested_sequence": { "type": "string", "description": "FilmCraft item id of the nest created by link.editIn. Absent for stills." }, + "asset": { + "type": "object", + "required": ["kind", "moniker"], + "additionalProperties": true, + "properties": { + "kind": { "enum": ["still", "vector", "sequence", "comp"] }, + "moniker": { + "type": "string", + "description": "craft://app/doc/kind/id. The id is 1 to 512 bytes. Each byte is an unreserved character (A-Z a-z 0-9 . _ ~ -) or a percent escape of two hex digits, and the bytes the escapes decode to must be valid UTF-8. The first pattern checks the shape and the byte count (one character or one escape is one byte). The second checks the UTF-8.", + "allOf": [ + { "pattern": "^craft://[a-z][a-z0-9-]{1,31}/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/[a-z][a-z0-9-]{0,31}/(?:[A-Za-z0-9._~-]|%[0-9A-Fa-f]{2}){1,512}$" }, + { "pattern": "^craft://[^/]+/[^/]+/[^/]+/(?:[A-Za-z0-9._~-]|%[0-7][0-9A-Fa-f]|%(?:[Cc][2-9A-Fa-f]|[Dd][0-9A-Fa-f])%[89ABab][0-9A-Fa-f]|%[Ee]0%[ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ee](?:[1-9A-Ca-c]|[EeFf])%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ee][Dd]%[89][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff]0%(?:9|[ABab])[0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff][1-3]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f]|%[Ff]4%8[0-9A-Fa-f]%[89ABab][0-9A-Fa-f]%[89ABab][0-9A-Fa-f])+$" } + ] + } + } + }, + "producer": { + "type": "object", + "required": ["app"], + "additionalProperties": true, + "properties": { + "app": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,31}$" }, + "app_version": { "type": "string" }, + "doc_id": { "type": "string" } + } + }, + "result_path": { "type": "string" }, + "result_epoch": { "type": "integer", "minimum": 0 }, + "format": { + "type": "object", + "required": ["w", "h", "alpha"], + "dependentRequired": { "rate_num": ["rate_den"], "rate_den": ["rate_num"] }, + "properties": { + "w": { "type": "integer", "minimum": 1 }, + "h": { "type": "integer", "minimum": 1 }, + "rate_num": { "type": "integer", "minimum": 1 }, + "rate_den": { "type": "integer", "minimum": 1 }, + "duration_ticks": { "type": "integer", "minimum": 0 }, + "alpha": { "type": "boolean" } + } + }, + "created": { "type": "string", "format": "date-time" } + } +} diff --git a/crates/ipc/src/activate.rs b/crates/ipc/src/activate.rs new file mode 100644 index 000000000..476634cdc --- /dev/null +++ b/crates/ipc/src/activate.rs @@ -0,0 +1,362 @@ +//! Focus handoff between Craft apps. +//! +//! The platform calls live behind [`ActivationBackend`]. An app adapter +//! implements it with the real winit and egui calls, so this crate depends on +//! neither. The decision logic here is plain Rust and runs against a fake +//! backend in tests. +//! +//! | Platform | Sender | Receiver | +//! |---|---|---| +//! | Linux | mint a token from the focused window | apply the token if the backend can, then `Focus`; else ask for attention | +//! | Windows | none | `Focus` (winit's foreground path) | +//! | macOS | yield activation to the target bundle | `activate`, then `Focus` | +//! +//! The three answers mean: +//! +//! - [`Activated::Yes`]: the window has focus. +//! - [`Activated::Requested`]: the app asked the OS to focus the window. The OS +//! decides later. Nobody confirmed it. +//! - [`Activated::No`]: focus was not taken. The app asked for user attention +//! instead (taskbar flash, dock bounce). +//! +//! On Linux, a token only counts when the backend can apply it to an existing +//! window ([`ActivationBackend::can_apply_token`]). Stock winit 0.30 cannot on +//! Wayland: the app can mint a token but has no call to use one. In that case +//! the answer is [`Activated::No`], never `Requested`. + +use crate::server::Activation; + +/// The answer to `app.activate`. Same three values as the wire reply. +pub type Activated = Activation; + +/// What a focus request did, as far as the adapter can tell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FocusResult { + /// The window has focus now. + Focused, + /// The request was sent. The compositor or window manager decides later. + Pending, + /// The platform refused or the window is gone. + Refused, +} + +/// Platform calls an app adapter provides. +/// +/// Every method is called on the thread that owns the UI. The adapter maps +/// them to winit and egui: +/// +/// - `mint_token`: egui's `Frame::request_activation_token` on an earlier +/// frame, then `Frame::take_activation_token`. Return the token or `None`. +/// - `can_apply_token`: true only if the winit in use can activate an existing window with a token. +/// - `set_pending_token`: the winit call that stores the token for that window. +/// - `focus`: `ViewportCommand::Focus`. +/// - `request_user_attention`: `ViewportCommand::RequestUserAttention`. +/// - `yield_activation_to` and `activate_application`: the `objc2-app-kit` +/// `NSApplication` and `NSRunningApplication` calls, macOS only. +pub trait ActivationBackend { + /// A fresh activation token, or `None` when the platform cannot mint one. + fn mint_token(&self) -> Option; + /// True when `set_pending_token` followed by `focus` really hands the token + /// to the compositor for an existing window. The default is false: no + /// default success. An adapter returns true only when its winit has a call + /// that applies a token to a live window. + fn can_apply_token(&self) -> bool { + false + } + /// Store `token` so the next `focus` call applies it. Called only when + /// `can_apply_token` is true. + fn set_pending_token(&self, token: &str); + /// Ask the window to take focus. + fn focus(&self) -> FocusResult; + /// Flash the taskbar or dock entry. Used when focus is refused. + fn request_user_attention(&self); + /// macOS sender. Hand activation to the app with `bundle_id`. Returns false on failure. + fn yield_activation_to(&self, bundle_id: &str) -> bool; + /// macOS receiver. Activate this app. Returns false on failure. + fn activate_application(&self) -> bool; +} + +/// The platform whose rules apply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Platform { + Linux, + Windows, + MacOs, + /// Anything else. No handoff is possible. + Other, +} + +impl Platform { + pub const fn current() -> Self { + if cfg!(target_os = "linux") { + Self::Linux + } else if cfg!(target_os = "windows") { + Self::Windows + } else if cfg!(target_os = "macos") { + Self::MacOs + } else { + Self::Other + } + } +} + +/// Sender side. A token to put in `app.activate`, or `None`. +/// +/// Only Linux has tokens. Windows and macOS return `None`. +pub fn mint_token(backend: &dyn ActivationBackend) -> Option { + mint_token_on(Platform::current(), backend) +} + +pub fn mint_token_on(platform: Platform, backend: &dyn ActivationBackend) -> Option { + match platform { + Platform::Linux => backend.mint_token().filter(|token| !token.is_empty()), + Platform::Windows | Platform::MacOs | Platform::Other => None, + } +} + +/// Sender side, before `app.activate`. macOS yields activation to `bundle_id`. +/// Other platforms do nothing and return true. A false return means the yield failed. +pub fn yield_to(backend: &dyn ActivationBackend, bundle_id: &str) -> bool { + yield_to_on(Platform::current(), backend, bundle_id) +} + +pub fn yield_to_on(platform: Platform, backend: &dyn ActivationBackend, bundle_id: &str) -> bool { + match platform { + Platform::MacOs => backend.yield_activation_to(bundle_id), + Platform::Linux | Platform::Windows | Platform::Other => true, + } +} + +/// Receiver side. Apply `app.activate`. `token` is the sender's token or `None`. +pub fn apply_activation(backend: &dyn ActivationBackend, token: Option<&str>) -> Activated { + apply_activation_on(Platform::current(), backend, token) +} + +pub fn apply_activation_on(platform: Platform, backend: &dyn ActivationBackend, token: Option<&str>) -> Activated { + match platform { + Platform::Linux => { + let token = token.filter(|text| !text.is_empty()); + let applied = match token { + Some(token) if backend.can_apply_token() => { + backend.set_pending_token(token); + true + } + _ => false, + }; + let result = backend.focus(); + // Pending means "the compositor may act". Without an applied token it will not. + let result = match (result, applied) { + (FocusResult::Pending, false) => FocusResult::Refused, + (other, _) => other, + }; + settle(backend, result) + } + Platform::Windows => settle(backend, backend.focus()), + Platform::MacOs => { + if !backend.activate_application() { + backend.request_user_attention(); + return Activated::No; + } + settle(backend, backend.focus()) + } + Platform::Other => { + backend.request_user_attention(); + Activated::No + } + } +} + +fn settle(backend: &dyn ActivationBackend, result: FocusResult) -> Activated { + match result { + FocusResult::Focused => Activated::Yes, + FocusResult::Pending => Activated::Requested, + FocusResult::Refused => { + backend.request_user_attention(); + Activated::No + } + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + + use super::*; + + #[derive(Default)] + struct Fake { + minted: Option, + focus: Option, + yield_ok: bool, + activate_ok: bool, + can_apply: bool, + calls: RefCell>, + } + + impl Fake { + fn calls(&self) -> Vec { + self.calls.borrow().clone() + } + fn log(&self, text: impl Into) { + self.calls.borrow_mut().push(text.into()); + } + } + + impl ActivationBackend for Fake { + fn mint_token(&self) -> Option { + self.log("mint"); + self.minted.clone() + } + fn can_apply_token(&self) -> bool { + self.can_apply + } + fn set_pending_token(&self, token: &str) { + self.log(format!("pending:{token}")); + } + fn focus(&self) -> FocusResult { + self.log("focus"); + self.focus.unwrap_or(FocusResult::Refused) + } + fn request_user_attention(&self) { + self.log("attention"); + } + fn yield_activation_to(&self, bundle_id: &str) -> bool { + self.log(format!("yield:{bundle_id}")); + self.yield_ok + } + fn activate_application(&self) -> bool { + self.log("activate"); + self.activate_ok + } + } + + #[test] + fn linux_sets_the_token_before_focus_and_reports_requested() { + let fake = Fake { focus: Some(FocusResult::Pending), can_apply: true, ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("tok")); + assert_eq!(got, Activated::Requested); + assert_eq!(fake.calls(), ["pending:tok", "focus"]); + } + + #[test] + fn linux_token_the_backend_cannot_apply_is_not_a_request() { + // Stock winit on Wayland: a token can be minted but not applied to an existing window. + let fake = Fake { focus: Some(FocusResult::Pending), can_apply: false, ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("tok")); + assert_eq!(got, Activated::No); + assert_eq!(fake.calls(), ["focus", "attention"]); + } + + #[test] + fn linux_token_the_backend_cannot_apply_still_reports_real_focus() { + let fake = Fake { focus: Some(FocusResult::Focused), can_apply: false, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Linux, &fake, Some("tok")), Activated::Yes); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn can_apply_token_defaults_to_false() { + struct Bare; + impl ActivationBackend for Bare { + fn mint_token(&self) -> Option { + None + } + fn set_pending_token(&self, _: &str) {} + fn focus(&self) -> FocusResult { + FocusResult::Pending + } + fn request_user_attention(&self) {} + fn yield_activation_to(&self, _: &str) -> bool { + true + } + fn activate_application(&self) -> bool { + true + } + } + assert!(!Bare.can_apply_token()); + } + + #[test] + fn linux_without_a_token_cannot_claim_requested() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, None); + assert_eq!(got, Activated::No); + assert_eq!(fake.calls(), ["focus", "attention"]); + } + + #[test] + fn linux_empty_token_counts_as_none() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + let got = apply_activation_on(Platform::Linux, &fake, Some("")); + assert_eq!(got, Activated::No); + assert!(!fake.calls().iter().any(|call| call.starts_with("pending"))); + } + + #[test] + fn linux_focused_window_is_yes_without_attention() { + let fake = Fake { focus: Some(FocusResult::Focused), ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Linux, &fake, None), Activated::Yes); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn refusal_requests_attention_on_every_platform() { + for platform in [Platform::Linux, Platform::Windows, Platform::MacOs] { + let fake = Fake { focus: Some(FocusResult::Refused), activate_ok: true, ..Fake::default() }; + assert_eq!(apply_activation_on(platform, &fake, Some("t")), Activated::No, "{platform:?}"); + assert_eq!(fake.calls().last().map(String::as_str), Some("attention"), "{platform:?}"); + } + } + + #[test] + fn windows_ignores_the_token() { + let fake = Fake { focus: Some(FocusResult::Pending), ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::Windows, &fake, Some("t")), Activated::Requested); + assert_eq!(fake.calls(), ["focus"]); + } + + #[test] + fn macos_activates_then_focuses() { + let fake = Fake { focus: Some(FocusResult::Focused), activate_ok: true, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::MacOs, &fake, None), Activated::Yes); + assert_eq!(fake.calls(), ["activate", "focus"]); + } + + #[test] + fn macos_activation_failure_is_no_with_attention() { + let fake = Fake { focus: Some(FocusResult::Focused), activate_ok: false, ..Fake::default() }; + assert_eq!(apply_activation_on(Platform::MacOs, &fake, None), Activated::No); + assert_eq!(fake.calls(), ["activate", "attention"]); + } + + #[test] + fn other_platforms_only_ask_for_attention() { + let fake = Fake::default(); + assert_eq!(apply_activation_on(Platform::Other, &fake, Some("t")), Activated::No); + assert_eq!(fake.calls(), ["attention"]); + } + + #[test] + fn only_linux_mints_tokens() { + let fake = Fake { minted: Some("abc".into()), ..Fake::default() }; + assert_eq!(mint_token_on(Platform::Linux, &fake).as_deref(), Some("abc")); + assert_eq!(mint_token_on(Platform::Windows, &fake), None); + assert_eq!(mint_token_on(Platform::MacOs, &fake), None); + let empty = Fake { minted: Some(String::new()), ..Fake::default() }; + assert_eq!(mint_token_on(Platform::Linux, &empty), None); + } + + #[test] + fn only_macos_yields() { + let fake = Fake { yield_ok: false, ..Fake::default() }; + assert!(yield_to_on(Platform::Linux, &fake, "com.x")); + assert!(fake.calls().is_empty()); + assert!(!yield_to_on(Platform::MacOs, &fake, "com.x")); + assert_eq!(fake.calls(), ["yield:com.x"]); + } + + #[test] + fn current_platform_matches_the_build() { + assert_eq!(Platform::current() == Platform::Linux, cfg!(target_os = "linux")); + } +} diff --git a/crates/ipc/src/asset.rs b/crates/ipc/src/asset.rs new file mode 100644 index 000000000..9ffd6f429 --- /dev/null +++ b/crates/ipc/src/asset.rs @@ -0,0 +1,70 @@ +//! Reply bodies for `link.resolve`, `asset.describe` and `link.refresh` +//! (`PROTOCOL.md` section 4). +//! +//! `std`, `serde` and the sibling pure modules only. + +use serde::Serialize; + +use crate::moniker::Moniker; +use crate::record::AssetKind; +use crate::refresh::{FileState, Fingerprint}; + +/// Answer to `link.resolve`: where the last-known result is and which version it is. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Resolved { + pub asset: Moniker, + pub path: String, + pub epoch: u64, + pub size: u64, + pub hash: String, + pub mtime_ms: u64, +} + +impl Resolved { + /// Build the answer from a file fingerprint. + pub fn from_fingerprint(asset: Moniker, path: &str, epoch: u64, fp: &Fingerprint) -> Self { + Self { asset, path: path.to_string(), epoch, size: fp.size, hash: fp.hash.clone(), mtime_ms: fp.mtime_ms } + } +} + +/// Answer to `asset.describe`: enough for a hub to list an asset without opening its document. +/// Unknown values serialize as `null`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Described { + pub asset: Moniker, + pub name: String, + pub kind: AssetKind, + pub epoch: u64, + pub w: Option, + pub h: Option, + pub hash: Option, + pub thumb_path: Option, + pub modified_ms: Option, +} + +/// Answer to `link.refresh`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Refreshed { + pub link_id: String, + pub epoch: u64, + pub state: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub result_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hash: Option, +} + +impl Refreshed { + /// Build the answer from a tracker state. `hash` and `result_path` are + /// dropped when the file is missing. + pub fn new(link_id: &str, epoch: u64, state: FileState, result_path: &str, seen: Option<&Fingerprint>) -> Self { + let present = state != FileState::Missing; + Self { + link_id: link_id.to_string(), + epoch, + state: state.as_str(), + result_path: present.then(|| result_path.to_string()), + hash: seen.filter(|_| present).map(|fp| fp.hash.clone()), + } + } +} diff --git a/crates/ipc/src/auth.rs b/crates/ipc/src/auth.rs new file mode 100644 index 000000000..851ec7d8d --- /dev/null +++ b/crates/ipc/src/auth.rs @@ -0,0 +1,107 @@ +//! Session tokens and the peer user check. +//! +//! On Unix the peer check is the effective uid from `interprocess`. A missing +//! credential is a refusal. On Windows, `interprocess` 2.4.4 `PeerCreds` only +//! exposes a pid. The pipe SDDL is the user check there. A pid is not a user. + +use interprocess::local_socket::Stream; +#[cfg(unix)] +use interprocess::local_socket::traits::StreamCommon; + +#[cfg(unix)] +use crate::discovery::current_euid; +use crate::error::LinkError; + +const HEX: &[u8; 16] = b"0123456789abcdef"; + +pub fn encode_hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} + +pub fn is_lower_hex(text: &str) -> bool { + text.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// 32 random bytes as 64 lowercase hex characters. +pub fn generate_token() -> Result { + let mut bytes = [0u8; 32]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + Ok(encode_hex(&bytes)) +} + +/// UUID version 4. The instance record stores this string. +pub fn random_uuid() -> Result { + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + let hex = encode_hex(&bytes); + Ok(format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32])) +} + +/// Compare two tokens without stopping at the first different byte. +/// Length is public (always 64). A different length is not equal. +pub fn tokens_equal(expected: &str, presented: &str) -> bool { + if expected.len() != 64 || presented.len() != 64 { + return false; + } + let mut diff = 0u8; + for (left, right) in expected.bytes().zip(presented.bytes()) { + diff |= left ^ right; + } + diff == 0 +} + +/// `Ok` when this Unix peer is the current effective user. +/// Missing credentials are `Err(Auth)`, not a pass. +/// +/// Windows returns `Ok`. The named-pipe SDDL admits only the owner and SYSTEM. +/// This function cannot see a Windows user id through interprocess 2.4.4. +pub fn ensure_peer_user(stream: &Stream) -> Result<(), LinkError> { + #[cfg(unix)] + { + let creds = stream.peer_creds().map_err(|_| LinkError::Auth)?; + let Some(peer) = creds.euid() else { + return Err(LinkError::Auth); + }; + let me = current_euid().map_err(|_| LinkError::Auth)?; + if peer == me { Ok(()) } else { Err(LinkError::Auth) } + } + #[cfg(not(unix))] + { + let _ = stream; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_is_64_lower_hex_and_compare_is_exact() { + let token = generate_token().unwrap(); + assert_eq!(token.len(), 64); + assert!(is_lower_hex(&token)); + assert!(tokens_equal(&token, &token)); + let mut other = token.clone(); + other.replace_range(0..1, if token.starts_with('a') { "b" } else { "a" }); + assert!(!tokens_equal(&token, &other)); + assert!(!tokens_equal(&token, "abcd")); + assert!(!tokens_equal(&token, &token.to_uppercase())); + } + + #[test] + fn uuid_v4_shape() { + let id = random_uuid().unwrap(); + assert_eq!(id.len(), 36); + assert_eq!(id.as_bytes()[14], b'4'); + let variant = id.as_bytes()[19]; + assert!(matches!(variant, b'8' | b'9' | b'a' | b'b')); + } +} diff --git a/crates/ipc/src/bulk.rs b/crates/ipc/src/bulk.rs new file mode 100644 index 000000000..488288778 --- /dev/null +++ b/crates/ipc/src/bulk.rs @@ -0,0 +1,487 @@ +//! Bulk frames (contract section 6). +//! +//! Server side. An app implements [`FrameSource`]. [`FrameTickets`] adapts it +//! to the server's [`Frames`] trait: it mints the 32 hex tickets, checks every +//! `CFRQ` header, and checks every reply before it reaches the wire. The server +//! drops a ticket when its control connection closes, which calls +//! [`Frames::close_ticket`] and ends the source's session for that ticket. +//! +//! Client side. [`Client::frames_open`] asks for a ticket. [`BulkClient`] opens +//! the second connection, attaches the ticket, and requests frames. + +use std::collections::HashMap; +use std::io::BufReader; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::{Map, Value, json}; + +use crate::InstanceRecord as Record; +use crate::auth::{encode_hex, is_lower_hex}; +use crate::client::{Client, Transport, map_reply, open_reader, read_json, write_json}; +use crate::error::LinkError; +use crate::server::{FrameOpen, Frames}; +use crate::wire; + +/// Largest `scale` a client may ask for in `frames.open`. +pub const SCALE_MAX: f64 = 16.0; + +/// Pixel format of a frame. Both are premultiplied alpha. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameFormat { + Rgba8Premul, + Rgba16fPremul, +} + +impl FrameFormat { + pub const fn as_str(self) -> &'static str { + match self { + Self::Rgba8Premul => "rgba8_premul", + Self::Rgba16fPremul => "rgba16f_premul", + } + } + + /// Bytes one pixel takes in a row. + pub(crate) const fn bytes_per_pixel(self) -> u32 { + match self { + Self::Rgba8Premul => 4, + Self::Rgba16fPremul => 8, + } + } + + pub fn parse(text: &str) -> Result { + match text { + "rgba8_premul" => Ok(Self::Rgba8Premul), + "rgba16f_premul" => Ok(Self::Rgba16fPremul), + other => Err(LinkError::BadParams(format!("unknown frame format {other}"))), + } + } +} + +/// What a client asked for in `frames.open`. +#[derive(Debug, Clone, PartialEq)] +pub struct FrameSpec { + pub link_id: String, + pub format: FrameFormat, + pub scale: f64, +} + +impl FrameSpec { + pub fn from_params(params: &Map) -> Result { + let link_id = params + .get("link_id") + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) + .ok_or_else(|| LinkError::BadParams("link_id must be a non-empty string".into()))? + .to_string(); + let format = + FrameFormat::parse(params.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::BadParams("format is required".into()))?)?; + let scale = match params.get("scale") { + None => 1.0, + Some(value) => value.as_f64().ok_or_else(|| LinkError::BadParams("scale must be a number".into()))?, + }; + if !scale.is_finite() || scale <= 0.0 || scale > SCALE_MAX { + return Err(LinkError::BadParams(format!("scale must be above 0 and at most {SCALE_MAX}"))); + } + Ok(Self { link_id, format, scale }) + } +} + +/// Size of the frames a ticket yields. `len` is `stride * h`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameLayout { + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +/// A row of `w` pixels in `format` must fit in `stride` bytes, or a reader that splits the payload +/// into rows of `stride` bytes cannot decode the frame. +fn check_row(w: u32, stride: u32, format: FrameFormat) -> Result<(), LinkError> { + if u64::from(stride) < u64::from(w) * u64::from(format.bytes_per_pixel()) { + return Err(LinkError::Invalid("frame layout: stride is smaller than one row of w pixels".into())); + } + Ok(()) +} + +impl FrameLayout { + fn check(&self) -> Result<(), LinkError> { + if u64::from(self.stride) * u64::from(self.h) != self.len { + return Err(LinkError::Invalid("frame layout: len is not stride * h".into())); + } + if self.len > wire::FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + Ok(()) + } +} + +/// One `CFRQ` header. +#[derive(Debug, Clone, PartialEq)] +pub struct FrameRequest { + pub req: u64, + pub time_s: f64, + /// The epoch the client last saw. The source may answer with a newer one. + pub epoch: u64, + /// `[x, y, w, h]` in pixels, or `None` for the whole frame. + pub region: Option<[u32; 4]>, +} + +impl FrameRequest { + pub fn from_header(header: &Value) -> Result { + let req = header.get("req").and_then(Value::as_u64).ok_or_else(|| LinkError::BadParams("req must be an unsigned integer".into()))?; + let time_s = header + .get("time_s") + .and_then(Value::as_f64) + .filter(|t| t.is_finite()) + .ok_or_else(|| LinkError::BadParams("time_s must be a finite number".into()))?; + let epoch = header.get("epoch").and_then(Value::as_u64).ok_or_else(|| LinkError::BadParams("epoch must be an unsigned integer".into()))?; + let region = match header.get("region") { + None | Some(Value::Null) => None, + Some(Value::Array(items)) if items.len() == 4 => { + let mut out = [0u32; 4]; + for (slot, item) in out.iter_mut().zip(items) { + let number = item.as_u64().ok_or_else(|| LinkError::BadParams("region entries must be unsigned integers".into()))?; + *slot = u32::try_from(number).map_err(|_| LinkError::BadParams("region entry is out of range".into()))?; + } + Some(out) + } + Some(_) => return Err(LinkError::BadParams("region must be null or [x, y, w, h]".into())), + }; + Ok(Self { req, time_s, epoch, region }) + } + + pub fn to_header(&self) -> Value { + json!({"req": self.req, "time_s": self.time_s, "epoch": self.epoch, "region": self.region}) + } +} + +/// A rendered frame from the source. +#[derive(Debug, Clone, PartialEq)] +pub struct RenderedFrame { + /// The epoch this frame was rendered at. + pub epoch: u64, + pub w: u32, + pub h: u32, + pub stride: u32, + pub format: FrameFormat, + pub render_ms: f64, + pub pixels: Vec, +} + +/// The app side of bulk frames. Calls may come from several connection threads. +pub trait FrameSource: Send + Sync { + /// Validate `spec` and report the frame layout. Errors become `frames.open` errors. + fn open(&self, spec: &FrameSpec) -> Result; + /// Render one frame. The returned epoch must be at least `request.epoch`. + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result; + /// The ticket for `spec` is gone. Release anything held for it. + fn close(&self, spec: &FrameSpec); +} + +/// [`Frames`] over a [`FrameSource`]. Pass it as `ServerConfig::frames`. +pub struct FrameTickets { + source: Arc, + open: Mutex>, +} + +impl FrameTickets { + pub fn new(source: Arc) -> Self { + Self { source, open: Mutex::new(HashMap::new()) } + } + + /// Count of tickets that are still valid. + pub fn open_tickets(&self) -> usize { + crate::server::lock(&self.open).len() + } + + fn spec(&self, ticket: &str) -> Result { + crate::server::lock(&self.open).get(ticket).cloned().ok_or_else(|| LinkError::BadParams("unknown or closed ticket".into())) + } +} + +fn new_ticket() -> Result { + let mut bytes = [0u8; 16]; + getrandom::getrandom(&mut bytes).map_err(LinkError::io)?; + Ok(encode_hex(&bytes)) +} + +impl Frames for FrameTickets { + fn open(&self, params: &Map) -> Result { + let spec = FrameSpec::from_params(params)?; + let layout = self.source.open(&spec)?; + layout.check()?; + check_row(layout.w, layout.stride, spec.format)?; + let ticket = new_ticket()?; + crate::server::lock(&self.open).insert(ticket.clone(), spec); + Ok(FrameOpen { ticket, w: layout.w, h: layout.h, stride: layout.stride, len: layout.len }) + } + + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError> { + let spec = self.spec(ticket)?; + let request = FrameRequest::from_header(header)?; + let frame = self.source.render(&spec, &request)?; + if frame.epoch < request.epoch { + return Err(LinkError::NotReady(format!("source is at epoch {}, request wants {}", frame.epoch, request.epoch))); + } + check_row(frame.w, frame.stride, frame.format)?; + let expect = u64::from(frame.stride) * u64::from(frame.h); + if u64::try_from(frame.pixels.len()).ok() != Some(expect) { + return Err(LinkError::Invalid("rendered payload is not stride * h bytes".into())); + } + let reply = json!({ + "req": request.req, + "epoch": frame.epoch, + "w": frame.w, + "h": frame.h, + "stride": frame.stride, + "format": frame.format.as_str(), + "render_ms": frame.render_ms, + }); + Ok((reply, frame.pixels)) + } + + fn close_ticket(&self, ticket: &str) { + let removed = crate::server::lock(&self.open).remove(ticket); + if let Some(spec) = removed { + self.source.close(&spec); + } + } +} + +/// The `frames.open` reply. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameTicket { + pub ticket: String, + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +impl Client { + /// `frames.open`. The ticket lives until this connection closes. + pub fn frames_open(&mut self, link_id: &str, format: FrameFormat, scale: f64) -> Result { + let result = self.call("frames.open", json!({"link_id": link_id, "format": format.as_str(), "scale": scale}))?; + let ticket = result + .get("ticket") + .and_then(Value::as_str) + .filter(|t| t.len() == 32 && is_lower_hex(t)) + .ok_or_else(|| LinkError::Invalid("frames.open ticket is not 32 hex characters".into()))?; + Ok(FrameTicket { + ticket: ticket.to_string(), + w: result_u32(&result, "w")?, + h: result_u32(&result, "h")?, + stride: result_u32(&result, "stride")?, + len: result.get("len").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("frames.open result has no len".into()))?, + }) + } +} + +fn result_u32(result: &Value, key: &str) -> Result { + result + .get(key) + .and_then(Value::as_u64) + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| LinkError::Invalid(format!("frames.open result has no {key}"))) +} + +/// One `CFRM` reply that carried pixels. +#[derive(Debug, Clone, PartialEq)] +pub struct Frame { + pub req: u64, + /// The epoch the frame was rendered at. It can be newer than the epoch asked for. + pub epoch: u64, + pub w: u32, + pub h: u32, + pub stride: u32, + pub format: FrameFormat, + pub render_ms: f64, + pub pixels: Vec, +} + +/// The second connection of contract section 6. It is binary after the attach. +pub struct BulkClient { + reader: BufReader, + next_req: u64, + io_timeout: Duration, +} + +impl BulkClient { + /// Connect, authenticate, and `bulk.attach` with `ticket`. + pub fn attach(record: &Record, ticket: &str) -> Result { + let mut reader = open_reader(record)?; + write_json(&mut reader, &json!({"id": 1, "method": "bulk.attach", "params": {"ticket": ticket}}))?; + let deadline = std::time::Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + let reply = read_json(&mut reader, &mut Vec::new(), deadline)?.ok_or(LinkError::Closed)?; + map_reply(&reply)?; + Ok(Self { reader, next_req: 1, io_timeout: Duration::from_secs(wire::REPLY_DEADLINE_SECS) }) + } + + /// How long one request may wait for the socket to take the request or give the reply. + /// The default is [`wire::REPLY_DEADLINE_SECS`]. A request that runs out of time is + /// `Err(Timeout)`, and the connection must then be dropped, because a reply that arrives + /// late cannot be told apart from the next one. + pub fn set_io_timeout(&mut self, timeout: Duration) { + self.io_timeout = timeout; + } + + /// Ask for one frame and wait for its reply. + /// + /// A reply with an `error` header is `Err(Invalid(message))`. The + /// connection stays usable after it. A short or malformed record is an + /// error and the connection must be dropped. [`BulkClient::request_reply`] + /// tells the two apart. + pub fn request(&mut self, time_s: f64, epoch: u64, region: Option<[u32; 4]>) -> Result { + self.request_reply(time_s, epoch, region)?.map_err(LinkError::Invalid) + } + + /// Like [`BulkClient::request`], with the two failures apart. The outer `Err` is the + /// connection: drop it. The inner `Err` is the text of an `error` header from the server: + /// the connection is still good. + pub fn request_reply(&mut self, time_s: f64, epoch: u64, region: Option<[u32; 4]>) -> Result, LinkError> { + if !time_s.is_finite() { + return Err(LinkError::BadParams("time_s must be finite".into())); + } + let req = self.next_req; + self.next_req = req.checked_add(1).ok_or_else(|| LinkError::Invalid("request id overflow".into()))?; + let header = FrameRequest { req, time_s, epoch, region }.to_header(); + let transport = self.reader.get_ref(); + transport.arm_send_timeout(Some(self.io_timeout))?; + transport.arm_recv_timeout(Some(self.io_timeout))?; + wire::write_cfrq(self.reader.get_mut(), &header)?; + let (reply, pixels) = wire::read_cfrm(&mut self.reader)?; + if reply.get("req").and_then(Value::as_u64) != Some(req) { + return Err(LinkError::Invalid("reply req does not match the request".into())); + } + if let Some(message) = reply.get("error") { + return Ok(Err(message.as_str().unwrap_or("frame error").to_string())); + } + parse_reply(req, &reply, pixels).map(Ok) + } +} + +fn parse_reply(req: u64, reply: &Value, pixels: Vec) -> Result { + let number = |key: &str| reply.get(key).and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid(format!("frame header has no {key}"))); + let small = |key: &str| number(key).and_then(|n| u32::try_from(n).map_err(|_| LinkError::Invalid(format!("frame header {key} is out of range")))); + let (w, h, stride) = (small("w")?, small("h")?, small("stride")?); + let format = + FrameFormat::parse(reply.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::Invalid("frame header has no format".into()))?)?; + check_row(w, stride, format)?; + if u64::try_from(pixels.len()).ok() != Some(u64::from(stride) * u64::from(h)) { + return Err(LinkError::Invalid("payload is not stride * h bytes".into())); + } + let render_ms = reply.get("render_ms").and_then(Value::as_f64).ok_or_else(|| LinkError::Invalid("frame header has no render_ms".into()))?; + Ok(Frame { req, epoch: number("epoch")?, w, h, stride, format, render_ms, pixels }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_header_round_trips() { + let request = FrameRequest { req: 9, time_s: 2.5, epoch: 4, region: Some([1, 2, 3, 4]) }; + assert_eq!(FrameRequest::from_header(&request.to_header()).unwrap(), request); + let whole = FrameRequest { req: 1, time_s: 0.0, epoch: 0, region: None }; + assert_eq!(FrameRequest::from_header(&whole.to_header()).unwrap(), whole); + } + + #[test] + fn bad_request_headers_are_rejected() { + for header in [ + json!({"time_s": 1.0, "epoch": 0}), + json!({"req": 1, "epoch": 0}), + json!({"req": 1, "time_s": 1.0}), + json!({"req": 1, "time_s": 1.0, "epoch": 0, "region": [1, 2, 3]}), + json!({"req": 1, "time_s": 1.0, "epoch": 0, "region": [1, 2, 3, -4]}), + json!({"req": -1, "time_s": 1.0, "epoch": 0}), + ] { + assert!(FrameRequest::from_header(&header).is_err(), "{header}"); + } + } + + #[test] + fn spec_rejects_bad_scale_and_format() { + let ok = json!({"link_id": "a", "format": "rgba8_premul", "scale": 2.0}); + assert!(FrameSpec::from_params(ok.as_object().unwrap()).is_ok()); + for bad in [ + json!({"link_id": "a", "format": "rgba8_premul", "scale": 0.0}), + json!({"link_id": "a", "format": "rgba8_premul", "scale": 17.0}), + json!({"link_id": "a", "format": "bgra"}), + json!({"format": "rgba8_premul"}), + ] { + assert!(FrameSpec::from_params(bad.as_object().unwrap()).is_err(), "{bad}"); + } + } + + struct OneLayout { + w: u32, + stride: u32, + h: u32, + } + + impl FrameSource for OneLayout { + fn open(&self, _spec: &FrameSpec) -> Result { + Ok(FrameLayout { w: self.w, h: self.h, stride: self.stride, len: u64::from(self.stride) * u64::from(self.h) }) + } + + fn render(&self, spec: &FrameSpec, _request: &FrameRequest) -> Result { + let len = usize::try_from(u64::from(self.stride) * u64::from(self.h)).unwrap(); + Ok(RenderedFrame { epoch: 1, w: self.w, h: self.h, stride: self.stride, format: spec.format, render_ms: 0.0, pixels: vec![0; len] }) + } + + fn close(&self, _spec: &FrameSpec) {} + } + + fn open_params(format: &str) -> Map { + json!({"link_id": "a", "format": format}).as_object().unwrap().clone() + } + + #[test] + fn a_stride_that_cannot_hold_one_row_is_refused_before_a_ticket_exists() { + // (w, stride, format, accepted): one row is w * 4 bytes for rgba8 and w * 8 for rgba16f. + for (w, stride, format, accepted) in [ + (4, 16, "rgba8_premul", true), + (4, 17, "rgba8_premul", true), + (4, 15, "rgba8_premul", false), + (4, 1, "rgba8_premul", false), + (4, 32, "rgba16f_premul", true), + (4, 16, "rgba16f_premul", false), + (0, 0, "rgba8_premul", true), + (u32::MAX, u32::MAX, "rgba16f_premul", false), + ] { + let tickets = FrameTickets::new(Arc::new(OneLayout { w, stride, h: 3 })); + let got = tickets.open(&open_params(format)); + assert_eq!(got.is_ok(), accepted, "w {w} stride {stride} {format}: {got:?}"); + assert_eq!(tickets.open_tickets(), usize::from(accepted), "a refused layout leaves no ticket"); + } + } + + #[test] + fn a_rendered_frame_with_a_short_stride_is_refused_and_the_client_refuses_it_too() { + let render = json!({"req": 1, "time_s": 0.0, "epoch": 0}); + let tickets = FrameTickets::new(Arc::new(OneLayout { w: 4, stride: 16, h: 3 })); + let ticket = tickets.open(&open_params("rgba8_premul")).unwrap().ticket; + assert!(tickets.render(&ticket, &render).is_ok()); + // A source that reports a narrower stride than it promised at open. + let lying = FrameTickets::new(Arc::new(OneLayout { w: 4, stride: 2, h: 3 })); + lying.open.lock().unwrap().insert(ticket.clone(), FrameSpec { link_id: "a".into(), format: FrameFormat::Rgba8Premul, scale: 1.0 }); + assert!(matches!(lying.render(&ticket, &render), Err(LinkError::Invalid(_)))); + + let reply = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 1, "format": "rgba8_premul", "render_ms": 1.0}); + assert!(matches!(parse_reply(1, &reply, vec![0; 3]), Err(LinkError::Invalid(_)))); + let fine = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + assert!(parse_reply(1, &fine, vec![0; 48]).is_ok()); + } + + #[test] + fn tickets_are_32_lower_hex_and_distinct() { + let a = new_ticket().unwrap(); + let b = new_ticket().unwrap(); + assert_eq!(a.len(), 32); + assert!(is_lower_hex(&a)); + assert_ne!(a, b); + } +} diff --git a/crates/ipc/src/cache.rs b/crates/ipc/src/cache.rs new file mode 100644 index 000000000..6cc77545d --- /dev/null +++ b/crates/ipc/src/cache.rs @@ -0,0 +1,278 @@ +//! A private per-user cache directory for Craft Link. +//! +//! Prepared media and other link caches must not sit in a directory that another user can create +//! first. This module picks the directory, creates it `0700`, refuses a symlink, a wrong owner or a +//! loose mode, and writes files `0600` through a temp name and a rename. It never falls back to +//! `$TMPDIR` or `/tmp`: with no home and no override it returns an error. +//! +//! Order: `CRAFT_LINK_CACHE_DIR` (absolute), `$XDG_CACHE_HOME/artcraft/link`, then +//! `$HOME/.cache/artcraft/link`. A leaf name `sub` is added under the chosen directory. + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::error::LinkError; + +/// Environment variable that overrides the cache root. +pub const CACHE_DIR_ENV: &str = "CRAFT_LINK_CACHE_DIR"; + +/// [`cache_dir_with`] for the process environment. +pub fn private_cache_dir(sub: &str) -> Result { + let env = std::env::vars().collect::>(); + cache_dir_with(&env, sub) +} + +/// Pick, create and check the cache directory `sub` for `env`. +pub fn cache_dir_with(env: &BTreeMap, sub: &str) -> Result { + if sub.is_empty() || sub.starts_with('.') || !sub.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { + return Err(refused(Path::new(sub), "cache name must be letters, digits, dot, dash or underscore")); + } + let (base, rest) = base_and_rest(env)?; + let mut path = base.clone(); + create_base(&path)?; + for part in rest.iter().map(String::as_str).chain([sub]) { + path.push(part); + crate::discovery::ensure_private_dir(&path)?; + } + Ok(path) +} + +fn base_and_rest(env: &BTreeMap) -> Result<(PathBuf, Vec), LinkError> { + let absolute = |key: &str| env.get(key).filter(|value| !value.is_empty()).map(PathBuf::from); + if let Some(root) = absolute(CACHE_DIR_ENV) { + if !root.is_absolute() { + return Err(refused(&root, "CRAFT_LINK_CACHE_DIR is not absolute")); + } + let (Some(parent), Some(leaf)) = (root.parent(), root.file_name().and_then(|name| name.to_str())) else { + return Err(refused(&root, "CRAFT_LINK_CACHE_DIR has no parent directory")); + }; + return Ok((parent.to_path_buf(), vec![leaf.to_string()])); + } + if let Some(xdg) = absolute("XDG_CACHE_HOME").filter(|path| path.is_absolute()) { + return Ok((xdg, vec!["artcraft".into(), "link".into()])); + } + if let Some(home) = absolute("HOME").filter(|path| path.is_absolute()) { + return Ok((home.join(".cache"), vec!["artcraft".into(), "link".into()])); + } + Err(refused(Path::new(""), "no cache directory: set CRAFT_LINK_CACHE_DIR, XDG_CACHE_HOME or HOME (a shared temp directory is not used)")) +} + +/// Create `path` and missing parents `0700`. An existing directory is left as it is: it is the +/// user's own `~/.cache` or the parent of the override. +fn create_base(path: &Path) -> Result<(), LinkError> { + let mut builder = std::fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path)?; + Ok(()) +} + +/// Write `bytes` to `/` with mode `0600`. The file is made under a temp name with +/// `create_new` (which does not follow a symlink or reuse a file) and renamed into place, so a +/// reader never sees half a file and a planted symlink at `name` is replaced, not followed. +pub fn write_private(dir: &Path, name: &str, bytes: &[u8]) -> Result<(), LinkError> { + if name.is_empty() || name.contains(['/', '\\', '\0']) || name.starts_with('.') { + return Err(refused(&dir.join(name), "file name must be one plain component")); + } + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let tmp = dir.join(format!(".{name}.{}.tmp", crate::auth::encode_hex(&rand))); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&tmp)?; + let written = file.write_all(bytes).and_then(|()| file.sync_all()); + drop(file); + if let Err(err) = written.and_then(|()| std::fs::rename(&tmp, dir.join(name))) { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(()) +} + +/// Read `/` when it is a regular file (not a symlink) that this user owns and no one +/// else can access. Anything else is `Refused`. A missing file is an I/O error. +/// +/// On Unix the open refuses a symlink and does not wait on a FIFO, and the checks and the read use +/// that one handle, so the entry cannot change between the check and the read. +pub fn read_private(dir: &Path, name: &str) -> Result, LinkError> { + let path = dir.join(name); + #[cfg(unix)] + { + use std::io::Read; + use std::os::unix::fs::MetadataExt; + let mut file = match crate::fsopen::open_read(&path, false) { + Ok(file) => file, + // The errno for a symlink in the last component differs between Unix systems. + Err(_) if std::fs::symlink_metadata(&path).is_ok_and(|meta| meta.file_type().is_symlink()) => { + return Err(refused(&path, "not a regular file (a symlink is not followed)")); + } + Err(err) => return Err(err.into()), + }; + let meta = file.metadata()?; + if !meta.file_type().is_file() { + return Err(refused(&path, "not a regular file")); + } + let euid = crate::discovery::current_euid()?; + if meta.uid() != euid { + return Err(refused(&path, &format!("owner is {}, euid is {euid}", meta.uid()))); + } + if meta.mode() & 0o077 != 0 { + return Err(refused(&path, &format!("mode is {:o}, group and others must have no access", meta.mode() & 0o777))); + } + let mut bytes = Vec::with_capacity(usize::try_from(meta.len()).unwrap_or(0)); + file.read_to_end(&mut bytes)?; + Ok(bytes) + } + #[cfg(not(unix))] + { + let meta = std::fs::symlink_metadata(&path)?; + if !meta.file_type().is_file() { + return Err(refused(&path, "not a regular file (a symlink is not followed)")); + } + Ok(std::fs::read(&path)?) + } +} + +fn refused(path: &Path, reason: &str) -> LinkError { + LinkError::Refused { path: path.display().to_string(), reason: reason.to_string() } +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; + + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-cache-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + path + } + + fn env(pairs: &[(&str, &Path)]) -> BTreeMap { + pairs.iter().map(|(key, value)| ((*key).to_string(), value.display().to_string())).collect() + } + + fn mode(path: &Path) -> u32 { + std::fs::symlink_metadata(path).unwrap().mode() & 0o777 + } + + #[test] + fn home_and_xdg_give_a_0700_chain() { + let home = scratch("home"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + assert_eq!(dir, home.join(".cache/artcraft/link/prepared")); + for part in [".cache/artcraft", ".cache/artcraft/link", ".cache/artcraft/link/prepared"] { + assert_eq!(mode(&home.join(part)), 0o700, "{part}"); + } + let xdg = scratch("xdg"); + let dir = cache_dir_with(&env(&[("HOME", &home), ("XDG_CACHE_HOME", &xdg)]), "prepared").unwrap(); + assert_eq!(dir, xdg.join("artcraft/link/prepared")); + assert_eq!(mode(&dir), 0o700); + // Same call twice is fine. + assert_eq!(cache_dir_with(&env(&[("XDG_CACHE_HOME", &xdg)]), "prepared").unwrap(), dir); + std::fs::remove_dir_all(&home).unwrap(); + std::fs::remove_dir_all(&xdg).unwrap(); + } + + #[test] + fn no_home_means_an_error_not_tmp() { + let err = cache_dir_with(&BTreeMap::new(), "prepared").unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err:?}"); + let relative: BTreeMap<_, _> = + [("HOME".to_string(), "relative".to_string()), ("XDG_CACHE_HOME".to_string(), "also/relative".to_string())].into(); + assert!(cache_dir_with(&relative, "prepared").is_err()); + let rel_override: BTreeMap<_, _> = [(CACHE_DIR_ENV.to_string(), "cache".to_string())].into(); + assert!(cache_dir_with(&rel_override, "prepared").is_err()); + } + + #[test] + fn the_override_is_used_and_checked() { + let root = scratch("override"); + let chosen = root.join("mine"); + let dir = cache_dir_with(&env(&[(CACHE_DIR_ENV, &chosen), ("HOME", Path::new("/nonexistent"))]), "prepared").unwrap(); + assert_eq!(dir, chosen.join("prepared")); + assert_eq!(mode(&chosen), 0o700); + // A loose existing directory is refused, not tightened behind the user's back. + std::fs::set_permissions(&chosen, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(cache_dir_with(&env(&[(CACHE_DIR_ENV, &chosen)]), "prepared").is_err()); + // A symlink is refused. + let real = root.join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = root.join("link"); + symlink(&real, &link).unwrap(); + assert!(cache_dir_with(&env(&[(CACHE_DIR_ENV, &link)]), "prepared").is_err()); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn a_fifo_or_a_missing_file_is_not_read_and_does_not_block() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let home = scratch("fifo"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + let made = std::process::Command::new("mkfifo").arg("-m").arg("600").arg(dir.join("pipe.bin")).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + let (tx, rx) = std::sync::mpsc::channel(); + let reading = dir.clone(); + std::thread::spawn(move || { + let _ = tx.send((read_private(&reading, "pipe.bin"), read_private(&reading, "absent.bin"))); + }); + let (fifo, absent) = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("a FIFO blocked the read"); + assert!(matches!(fifo, Err(LinkError::Refused { .. })), "{fifo:?}"); + assert!(matches!(absent, Err(LinkError::Io(_))), "{absent:?}"); + std::fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn bad_names_are_refused() { + let home = scratch("names"); + std::fs::create_dir_all(&home).unwrap(); + for bad in ["", ".hidden", "a/b", "..", "a b", "x\0y"] { + assert!(cache_dir_with(&env(&[("HOME", &home)]), bad).is_err(), "{bad:?}"); + } + std::fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn files_are_0600_and_symlinks_are_replaced_not_followed() { + let home = scratch("files"); + std::fs::create_dir_all(&home).unwrap(); + let dir = cache_dir_with(&env(&[("HOME", &home)]), "prepared").unwrap(); + write_private(&dir, "a.bin", b"one").unwrap(); + assert_eq!(mode(&dir.join("a.bin")), 0o600); + assert_eq!(read_private(&dir, "a.bin").unwrap(), b"one"); + // A symlink planted at the name is replaced. The target keeps its content. + let victim = home.join("victim"); + std::fs::write(&victim, b"keep").unwrap(); + let planted = dir.join("b.bin"); + symlink(&victim, &planted).unwrap(); + assert!(matches!(read_private(&dir, "b.bin"), Err(LinkError::Refused { .. })), "a symlink is refused, not read"); + write_private(&dir, "b.bin", b"two").unwrap(); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep"); + assert_eq!(read_private(&dir, "b.bin").unwrap(), b"two"); + // A loose file is refused on read. + std::fs::set_permissions(dir.join("a.bin"), std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(read_private(&dir, "a.bin").is_err()); + for bad in ["", "../x", "a/b", ".x"] { + assert!(write_private(&dir, bad, b"x").is_err(), "{bad:?}"); + } + std::fs::remove_dir_all(&home).unwrap(); + } +} diff --git a/crates/ipc/src/client.rs b/crates/ipc/src/client.rs new file mode 100644 index 000000000..8090be1d3 --- /dev/null +++ b/crates/ipc/src/client.rs @@ -0,0 +1,853 @@ +//! Craft Link client. +//! +//! Connects to one instance record, authenticates on local sockets and pipes, +//! and exchanges JSON lines. Push events that arrive before a reply are kept +//! until [`Client::next_event`]. + +use std::collections::VecDeque; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +use interprocess::local_socket::prelude::*; +use interprocess::local_socket::{ConnectOptions, GenericFilePath, Stream}; +use serde_json::{Map, Value, json}; + +use crate::error::LinkError; +use crate::wire; +use crate::{Endpoint, InstanceRecord}; + +/// Identity sent in `link.hello`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HelloClient { + pub app: String, + pub app_version: String, + pub pid: u32, + pub instance: String, + pub protocol_min: u32, + pub protocol_max: u32, +} + +/// One push line: `event`, `seq`, and `data`. +#[derive(Debug, Clone, PartialEq)] +pub struct Event { + pub event: String, + pub seq: u64, + pub data: Value, +} + +/// A connected Craft Link session. +pub struct Client { + record: InstanceRecord, + reader: BufReader, + next_id: u64, + events: VecDeque, + hello: Option, + /// Bytes of a reply line read before a timeout, kept so the next read resumes the line. + partial: Vec, +} + +impl Client { + /// Connect to `record.endpoint`. Every transport sends auth id 0 with the record token first. + pub fn connect(record: &InstanceRecord) -> Result { + let reader = open_reader(record)?; + Ok(Self { record: record.clone(), reader, next_id: 1, events: VecDeque::new(), hello: None, partial: Vec::new() }) + } + + /// Connect to the loopback TCP control port published in `record` (`record.tcp_port()`), + /// with the same token handshake. `Err(Invalid)` when the app opened no TCP port. + pub fn connect_tcp(record: &InstanceRecord) -> Result { + let Some(port) = record.tcp_port() else { + return Err(LinkError::Invalid("the instance record has no tcp_port; start the app with --control".into())); + }; + let mut tcp = record.clone(); + tcp.endpoint = Endpoint::Tcp { port }; + Self::connect(&tcp) + } + + /// Send one request. `id` starts at 1 and increases by one each call. + /// Lines with `event` and a different id are buffered. + pub fn call(&mut self, method: &str, params: Value) -> Result { + let id = self.next_id; + self.next_id = self.next_id.checked_add(1).ok_or_else(|| LinkError::Invalid("request id overflow".into()))?; + let msg = json!({"id": id, "method": method, "params": params}); + write_json(&mut self.reader, &msg)?; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + loop { + let value = match read_json(&mut self.reader, &mut self.partial, deadline)? { + Some(value) => value, + None => return Err(LinkError::Closed), + }; + if !value.is_object() { + return Err(LinkError::Invalid("reply is not an object".into())); + } + if id_is(&value, id) { + return map_reply(&value); + } + if value.get("event").is_some() { + self.events.push_back(parse_event(&value)?); + continue; + } + return Err(LinkError::Invalid("reply id does not match the request".into())); + } + } + + /// `link.hello`. Checks `pid` and `instance` against the connected record. + /// A successful hello is stored and sent again by [`Client::reconnect`]. + pub fn hello(&mut self, identity: &HelloClient) -> Result { + let result = self.call("link.hello", hello_params(identity))?; + check_identity(&result, &self.record)?; + self.hello = Some(identity.clone()); + Ok(result) + } + + /// `events.subscribe`. Returns `result.seq`. + pub fn subscribe(&mut self, events: &[&str]) -> Result { + let result = self.call("events.subscribe", json!({"events": events}))?; + result.get("seq").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("events.subscribe result has no seq".into())) + } + + /// `events.unsubscribe`. + pub fn unsubscribe(&mut self) -> Result<(), LinkError> { + let _result = self.call("events.unsubscribe", json!({}))?; + Ok(()) + } + + /// The next buffered event, or the next event read before `timeout`. + /// `None` waits up to [`wire::REPLY_DEADLINE_SECS`]. A zero duration does not read. + pub fn next_event(&mut self, timeout: Option) -> Result, LinkError> { + if let Some(event) = self.events.pop_front() { + return Ok(Some(event)); + } + if timeout.is_some_and(|wait| wait.is_zero()) { + return Ok(None); + } + let wait = timeout.unwrap_or(Duration::from_secs(wire::REPLY_DEADLINE_SECS)); + let deadline = Instant::now() + wait; + if Instant::now() >= deadline { + return Ok(None); + } + match read_json(&mut self.reader, &mut self.partial, deadline) { + Ok(None) => Err(LinkError::Closed), + Ok(Some(value)) => { + if value.get("event").is_some() { + Ok(Some(parse_event(&value)?)) + } else { + Err(LinkError::Invalid("expected an event line".into())) + } + } + Err(LinkError::Timeout) => Ok(None), + Err(err) => Err(err), + } + } + + /// Connect to the stored record again. Does not scan discovery. + /// Sends auth again. Sends the stored hello when one succeeded. + pub fn reconnect(&mut self) -> Result<(), LinkError> { + let reader = open_reader(&self.record)?; + self.reader = reader; + self.partial.clear(); + self.events.clear(); + self.next_id = 1; + if let Some(identity) = self.hello.clone() { + self.hello(&identity)?; + } + Ok(()) + } +} + +pub(crate) enum Transport { + Local(Stream), + Tcp(TcpStream), + #[cfg(windows)] + Pipe(PipeTransport), +} + +/// Windows named pipes have no socket-style timeouts. Poll their nonblocking I/O so +/// authentication and frame requests retain the same bounded waits as other transports. +#[cfg(windows)] +pub(crate) struct PipeTransport { + stream: Stream, + recv_timeout: std::cell::Cell>, + send_timeout: std::cell::Cell>, +} + +#[cfg(windows)] +impl PipeTransport { + pub(crate) fn new(stream: Stream) -> std::io::Result { + stream.set_nonblocking(true)?; + Ok(Self { stream, recv_timeout: std::cell::Cell::new(None), send_timeout: std::cell::Cell::new(None) }) + } + + pub(crate) fn set_timeouts(&self, recv: Duration, send: Duration) { + self.recv_timeout.set(Some(recv)); + self.send_timeout.set(Some(send)); + } + + fn poll(timeout: Option, mut operation: impl FnMut() -> std::io::Result) -> std::io::Result { + let start = Instant::now(); + loop { + match operation() { + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + if timeout.is_some_and(|limit| start.elapsed() >= limit) { + return Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "named pipe I/O timed out")); + } + std::thread::sleep(Duration::from_millis(1)); + } + result => return result, + } + } + } +} + +#[cfg(windows)] +impl Read for PipeTransport { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + Self::poll(self.recv_timeout.get(), || match self.stream.read(buf) { + // interprocess converts Windows ERROR_NO_DATA on an empty nonblocking pipe + // to EOF. A disconnected pipe has no peer; otherwise retry until the deadline. + Ok(0) if !buf.is_empty() && self.stream.peer_creds().is_ok() => Err(std::io::ErrorKind::WouldBlock.into()), + result => result, + }) + } +} + +#[cfg(windows)] +impl Write for PipeTransport { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Self::poll(self.send_timeout.get(), || match self.stream.write(buf) { + Ok(0) if !buf.is_empty() => Err(std::io::ErrorKind::WouldBlock.into()), + result => result, + }) + } + fn flush(&mut self) -> std::io::Result<()> { + self.stream.flush() + } +} + +impl Transport { + pub(crate) fn arm_recv_timeout(&self, timeout: Option) -> std::io::Result<()> { + match self { + Self::Local(stream) => { + let result = stream.set_recv_timeout(timeout); + // Darwin rejects socket options after full shutdown (sosetoptlock), even when + // unread reply bytes remain. Reading those bytes or EOF cannot block, so let + // the read report the actual protocol result instead of this timeout error. + #[cfg(target_os = "macos")] + if result.as_ref().is_err_and(|err| err.raw_os_error() == Some(libc::EINVAL)) { + return Ok(()); + } + result + } + Self::Tcp(stream) => stream.set_read_timeout(timeout), + #[cfg(windows)] + Self::Pipe(stream) => { + stream.recv_timeout.set(timeout); + Ok(()) + } + } + } + + pub(crate) fn arm_send_timeout(&self, timeout: Option) -> std::io::Result<()> { + match self { + Self::Local(stream) => stream.set_send_timeout(timeout), + Self::Tcp(stream) => stream.set_write_timeout(timeout), + #[cfg(windows)] + Self::Pipe(stream) => { + stream.send_timeout.set(timeout); + Ok(()) + } + } + } +} + +impl Read for Transport { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + Self::Local(stream) => stream.read(buf), + Self::Tcp(stream) => stream.read(buf), + #[cfg(windows)] + Self::Pipe(stream) => stream.read(buf), + } + } +} + +impl Write for Transport { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + Self::Local(stream) => stream.write(buf), + Self::Tcp(stream) => stream.write(buf), + #[cfg(windows)] + Self::Pipe(stream) => stream.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Local(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + #[cfg(windows)] + Self::Pipe(stream) => stream.flush(), + } + } +} + +pub(crate) fn open_reader(record: &InstanceRecord) -> Result, LinkError> { + let (transport, needs_auth) = open_transport(&record.endpoint)?; + let mut reader = BufReader::new(transport); + if needs_auth { + authenticate(&mut reader, &record.token)?; + } + Ok(reader) +} + +fn open_transport(endpoint: &Endpoint) -> Result<(Transport, bool), LinkError> { + match endpoint { + Endpoint::Unix { path } => Ok((Transport::Local(connect_unix(path)?), true)), + Endpoint::Pipe { name } => { + #[cfg(windows)] + { + Ok((Transport::Pipe(PipeTransport::new(connect_pipe(name)?)?), true)) + } + #[cfg(not(windows))] + { + Ok((Transport::Local(connect_pipe(name)?), true)) + } + } + Endpoint::Tcp { port } => { + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, *port)); + let wait = Duration::from_secs(wire::REPLY_DEADLINE_SECS); + match TcpStream::connect_timeout(&addr, wait) { + Ok(stream) => Ok((Transport::Tcp(stream), true)), + Err(err) if is_timeout(&err) => Err(LinkError::Timeout), + Err(err) => Err(err.into()), + } + } + } +} + +fn connect_unix(path: &str) -> Result { + let name = path.to_fs_name::().map_err(LinkError::io)?; + ConnectOptions::new().name(name).connect_sync().map_err(LinkError::io) +} + +/// Named pipes are the Windows endpoint. Other platforms cannot open one. +/// This returns [`LinkError::Invalid`] instead of pretending the pipe connected. +#[cfg(not(windows))] +fn connect_pipe(name: &str) -> Result { + let _ = name; + Err(LinkError::Invalid("named pipes are a Windows endpoint and are not connected on this platform".into())) +} + +#[cfg(windows)] +fn connect_pipe(name: &str) -> Result { + use interprocess::local_socket::GenericNamespaced; + let socket_name = name.to_ns_name::().map_err(LinkError::io)?; + ConnectOptions::new().name(socket_name).connect_sync().map_err(LinkError::io) +} + +fn authenticate(reader: &mut BufReader, token: &str) -> Result<(), LinkError> { + let request = json!({"id": 0, "method": "auth", "params": {"token": token}}); + write_json(reader, &request)?; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + let expected = wire::auth_success(&json!(0)); + let mut partial = Vec::new(); + loop { + let line = match pull_line(reader, &mut partial, deadline) { + Ok(Some(bytes)) => bytes, + Ok(None) => return Err(LinkError::Auth), + Err(LinkError::Timeout) => return Err(LinkError::Timeout), + Err(LinkError::TooLarge) => return Err(LinkError::TooLarge), + Err(LinkError::Io(text)) => return Err(LinkError::Io(text)), + Err(_) => return Err(LinkError::Auth), + }; + let Ok(text) = std::str::from_utf8(&line) else { + return Err(LinkError::Auth); + }; + let text = text.trim(); + if text.is_empty() { + continue; + } + let Ok(value) = serde_json::from_str::(text) else { + return Err(LinkError::Auth); + }; + if value == expected { + return Ok(()); + } + return Err(LinkError::Auth); + } +} + +pub(crate) fn write_json(reader: &mut BufReader, value: &Value) -> Result<(), LinkError> { + reader.get_ref().arm_send_timeout(Some(Duration::from_secs(wire::REPLY_DEADLINE_SECS)))?; + wire::write_line(reader.get_mut(), value) +} + +/// Reads one JSON line. `partial` holds the bytes of a line that a timeout cut short, so a caller +/// that keeps the connection after `Err(Timeout)` passes the same buffer again and gets the whole line. +pub(crate) fn read_json(reader: &mut BufReader, partial: &mut Vec, deadline: Instant) -> Result, LinkError> { + loop { + let Some(bytes) = pull_line(reader, partial, deadline)? else { + return Ok(None); + }; + let text = std::str::from_utf8(&bytes).map_err(|err| LinkError::Invalid(err.to_string()))?; + let text = text.trim(); + if text.is_empty() { + continue; + } + return Ok(Some(serde_json::from_str(text)?)); + } +} + +/// Same line limit as [`wire::read_line`]. The io error kind is kept so a +/// socket timeout becomes [`LinkError::Timeout`] instead of a generic io string. +/// +/// Bytes read before a timeout stay in `partial` and the next call continues that line. +fn pull_line(reader: &mut BufReader, partial: &mut Vec, deadline: Instant) -> Result>, LinkError> { + let max = wire::MAX_LINE; + let limit = u64::try_from(max).map_err(|_| LinkError::TooLarge)?.saturating_add(1); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(LinkError::Timeout); + } + reader.get_ref().arm_recv_timeout(Some(remaining))?; + let left = limit.saturating_sub(u64::try_from(partial.len()).unwrap_or(u64::MAX)); + match reader.take(left).read_until(b'\n', partial) { + Ok(0) if partial.is_empty() => return Ok(None), + Ok(_) => return finish_line(std::mem::take(partial), max), + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) if is_timeout(&err) => return Err(LinkError::Timeout), + Err(err) => return Err(err.into()), + } + } +} + +fn finish_line(mut buf: Vec, max: usize) -> Result>, LinkError> { + if buf.last() == Some(&b'\n') { + buf.pop(); + if buf.last() == Some(&b'\r') { + buf.pop(); + } + } else if buf.len() > max { + return Err(LinkError::TooLarge); + } + if buf.len() > max { + return Err(LinkError::TooLarge); + } + Ok(Some(buf)) +} + +fn is_timeout(err: &std::io::Error) -> bool { + if matches!(err.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock) { + return true; + } + // Linux EAGAIN 11, ETIMEDOUT 110. macOS EAGAIN 35, ETIMEDOUT 60. + // Windows ERROR_SEM_TIMEOUT 121, WAIT_TIMEOUT 258. + matches!(err.raw_os_error(), Some(11 | 35 | 60 | 110 | 121 | 258)) +} + +fn hello_params(identity: &HelloClient) -> Value { + json!({ + "client": { + "app": identity.app, + "app_version": identity.app_version, + "pid": identity.pid, + "instance": identity.instance, + }, + "protocol": {"min": identity.protocol_min, "max": identity.protocol_max}, + }) +} + +fn check_identity(result: &Value, record: &InstanceRecord) -> Result<(), LinkError> { + let got_pid = result.get("pid"); + if got_pid.and_then(Value::as_u64) != Some(u64::from(record.pid)) { + return Err(LinkError::Mismatch { + field: "pid".into(), + expected: record.pid.to_string(), + got: got_pid.map(json_text).unwrap_or_else(|| "missing".into()), + }); + } + let got_instance = result.get("instance"); + if got_instance.and_then(Value::as_str) != Some(record.instance.as_str()) { + return Err(LinkError::Mismatch { + field: "instance".into(), + expected: record.instance.clone(), + got: got_instance.map(json_text).unwrap_or_else(|| "missing".into()), + }); + } + Ok(()) +} + +fn json_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Number(number) => number.to_string(), + Value::Bool(flag) => flag.to_string(), + Value::Null => "null".into(), + other => other.to_string(), + } +} + +fn id_is(value: &Value, id: u64) -> bool { + value.get("id").and_then(Value::as_u64) == Some(id) +} + +fn parse_event(value: &Value) -> Result { + let event = value.get("event").and_then(Value::as_str).ok_or_else(|| LinkError::Invalid("event line has no event name".into()))?; + let seq = value.get("seq").and_then(Value::as_u64).ok_or_else(|| LinkError::Invalid("event line has no seq".into()))?; + let data = value.get("data").cloned().unwrap_or(Value::Null); + Ok(Event { event: event.to_string(), seq, data }) +} + +pub(crate) fn map_reply(value: &Value) -> Result { + let Some(ok) = value.get("ok").and_then(Value::as_bool) else { + return Err(LinkError::Invalid("reply has no ok flag".into())); + }; + if ok { + return value.get("result").cloned().ok_or_else(|| LinkError::Invalid("reply has no result".into())); + } + match value.get("error") { + Some(Value::String(text)) => Err(LinkError::Invalid(text.clone())), + Some(Value::Object(map)) => map_error_object(map), + Some(_) => Err(LinkError::Invalid("error is not a string or an object".into())), + None => Err(LinkError::Invalid("reply has no error".into())), + } +} + +fn map_error_object(map: &Map) -> Result { + let code = map.get("code").and_then(Value::as_str).unwrap_or(""); + let message = map.get("message").and_then(Value::as_str).filter(|text| !text.is_empty()).unwrap_or(code).to_string(); + match code { + "protocol_unsupported" => { + let data = map.get("data").and_then(Value::as_object); + let server_min = protocol_bound(data, "server_min")?; + let server_max = protocol_bound(data, "server_max")?; + Err(LinkError::Protocol { server_min, server_max }) + } + "auth_required" => Err(LinkError::Auth), + "bad_params" => Err(LinkError::BadParams(message)), + "payload_too_large" => Err(LinkError::TooLarge), + "payload_invalid" => Err(LinkError::PayloadInvalid(message)), + "media_missing" => { + let paths = map + .get("data") + .and_then(Value::as_object) + .and_then(|data| data.get("missing")) + .and_then(Value::as_array) + .map(|list| list.iter().filter_map(Value::as_str).map(str::to_string).collect()) + .unwrap_or_default(); + Err(LinkError::MediaMissing { paths }) + } + "not_ready" => Err(LinkError::NotReady(message)), + "link_unknown" => Err(LinkError::Unknown(message)), + "busy" => Err(LinkError::Busy(message)), + _ => Err(LinkError::Invalid(if message.is_empty() { "unknown error".into() } else { message })), + } +} + +fn protocol_bound(data: Option<&Map>, key: &str) -> Result { + let Some(value) = data.and_then(|map| map.get(key)).and_then(Value::as_u64) else { + return Err(LinkError::Invalid(format!("protocol_unsupported is missing {key}"))); + }; + u32::try_from(value).map_err(|_| LinkError::Invalid(format!("{key} does not fit in u32"))) +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::io::{BufReader, Write}; + use std::os::unix::net::{UnixListener, UnixStream}; + use std::path::PathBuf; + use std::thread::JoinHandle; + use std::time::{Duration, Instant}; + + use serde_json::{Value, json}; + + use super::*; + + struct Peer { + path: PathBuf, + record: InstanceRecord, + join: Option>, + } + + impl Drop for Peer { + fn drop(&mut self) { + if let Some(join) = self.join.take() { + let _ = join.join(); + } + let _ = std::fs::remove_file(&self.path); + } + } + + fn sock_path(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))) + } + + fn peer(label: &str, script: F) -> Peer + where + F: FnOnce(UnixStream, InstanceRecord) + Send + 'static, + { + let path = sock_path(label); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let endpoint = Endpoint::Unix { path: path.to_str().unwrap().to_string() }; + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let moved = record.clone(); + let join = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(10))).unwrap(); + script(stream, moved); + }); + Peer { path, record, join: Some(join) } + } + + fn read_req(reader: &mut BufReader) -> Value { + let bytes = crate::wire::read_line(reader, crate::wire::MAX_LINE).unwrap().expect("peer eof"); + serde_json::from_slice(&bytes).unwrap() + } + + fn write_val(reader: &mut BufReader, value: &Value) { + crate::wire::write_line(reader.get_mut(), value).unwrap(); + } + + fn accept_auth(reader: &mut BufReader, token: &str) { + let req = read_req(reader); + assert_eq!(req["method"], "auth"); + assert_eq!(req["id"], json!(0)); + assert_eq!(req["params"]["token"], token); + // Key order differs from serde_json's BTreeMap on purpose. + reader.get_mut().write_all(b"{\"result\":{\"authenticated\":true},\"ok\":true,\"id\":0}\n").unwrap(); + reader.get_mut().flush().unwrap(); + } + + #[test] + fn auth_then_call_parses_the_result() { + let server = peer("call", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + assert_eq!(req["method"], "engine.execute"); + assert_eq!(req["id"], json!(1)); + assert_eq!(req["params"]["command"], "ping"); + let id = req["id"].clone(); + write_val(&mut reader, &crate::wire::reply_ok(&id, json!({"pong": true}))); + }); + let mut client = Client::connect(&server.record).unwrap(); + let result = client.call("engine.execute", json!({"command": "ping"})).unwrap(); + assert_eq!(result, json!({"pong": true})); + } + + #[test] + fn event_line_before_the_reply_is_returned_by_next_event() { + let server = peer("ev", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + let id = req["id"].clone(); + write_val(&mut reader, &json!({"event": "doc.changed", "seq": 4, "data": {"doc_id": "abc"}})); + write_val(&mut reader, &crate::wire::reply_ok(&id, json!({"n": 1}))); + }); + let mut client = Client::connect(&server.record).unwrap(); + assert!(client.next_event(Some(Duration::ZERO)).unwrap().is_none()); + let result = client.call("link.focus", json!({})).unwrap(); + assert_eq!(result, json!({"n": 1})); + let event = client.next_event(Some(Duration::ZERO)).unwrap().expect("buffered event"); + assert_eq!(event.event, "doc.changed"); + assert_eq!(event.seq, 4); + assert_eq!(event.data, json!({"doc_id": "abc"})); + assert!(client.next_event(Some(Duration::ZERO)).unwrap().is_none()); + } + + #[test] + fn queued_events_are_read_after_the_peer_has_closed() { + let mut server = peer("closed", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + for seq in [1, 2] { + write_val(&mut reader, &json!({"event": "doc.changed", "seq": seq, "data": {}})); + } + }); + let mut client = Client::connect(&server.record).unwrap(); + server.join.take().unwrap().join().unwrap(); + for seq in [1, 2] { + let event = client.next_event(Some(Duration::from_secs(1))).unwrap().expect("queued event after peer close"); + assert_eq!(event.seq, seq); + } + assert!(matches!(client.next_event(Some(Duration::from_secs(1))), Err(LinkError::Closed))); + } + + #[test] + fn reconnect_sends_auth_again() { + let path = sock_path("re"); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let endpoint = Endpoint::Unix { path: path.to_str().unwrap().to_string() }; + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let token = record.token.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let join = std::thread::spawn(move || { + for _ in 0..2 { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(stream); + let req = read_req(&mut reader); + tx.send(req).unwrap(); + write_val(&mut reader, &crate::wire::auth_success(&json!(0))); + } + }); + let mut client = Client::connect(&record).unwrap(); + client.reconnect().unwrap(); + drop(client); + let first = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let second = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(first["method"], "auth"); + assert_eq!(second["method"], "auth"); + assert_eq!(first["params"]["token"], token); + assert_eq!(second["params"]["token"], token); + join.join().unwrap(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn failed_auth_reply_is_auth() { + let server = peer("bad", |stream, _record| { + let mut reader = BufReader::new(stream); + let _req = read_req(&mut reader); + write_val(&mut reader, &crate::wire::auth_failure()); + }); + let err = match Client::connect(&server.record) { + Err(err) => err, + Ok(_client) => panic!("connect should fail auth"), + }; + assert!(matches!(err, LinkError::Auth), "{err}"); + } + + #[test] + fn closed_before_auth_reply_is_auth() { + let server = peer("eof", |stream, _record| { + let mut reader = BufReader::new(stream); + let _req = read_req(&mut reader); + drop(reader); + }); + let err = match Client::connect(&server.record) { + Err(err) => err, + Ok(_client) => panic!("connect should fail when the peer closes"), + }; + assert!(matches!(err, LinkError::Auth), "{err}"); + } + + #[test] + fn hello_pid_mismatch_is_mismatch() { + let server = peer("hi", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let req = read_req(&mut reader); + assert_eq!(req["method"], "link.hello"); + let id = req["id"].clone(); + let result = json!({ + "app": "effectcraft", + "app_version": "0.8.3", + "pid": 4243, + "instance": record.instance, + "protocol": 1, + }); + write_val(&mut reader, &crate::wire::reply_ok(&id, result)); + }); + let mut client = Client::connect(&server.record).unwrap(); + let identity = HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "11111111-1111-4111-8111-111111111111".into(), + protocol_min: wire::PROTOCOL_MIN, + protocol_max: wire::PROTOCOL_MAX, + }; + let err = client.hello(&identity).unwrap_err(); + match err { + LinkError::Mismatch { field, expected, got } => { + assert_eq!(field, "pid"); + assert_eq!(expected, "4242"); + assert_eq!(got, "4243"); + } + other => panic!("expected mismatch, got {other}"), + } + } + + #[test] + fn error_object_and_string_map_onto_link_error() { + let server = peer("err", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + for _ in 0..3 { + let req = read_req(&mut reader); + let id = req["id"].clone(); + let method = req["method"].as_str().unwrap().to_string(); + let error = match method.as_str() { + "proto" => json!({"code": "protocol_unsupported", "message": "no", "data": {"server_min": 2, "server_max": 5}}), + "text" => json!("no such method"), + "params" => json!({"code": "bad_params", "message": "missing command", "data": {}}), + other => panic!("unexpected method {other}"), + }; + write_val(&mut reader, &json!({"id": id, "ok": false, "error": error})); + } + }); + let mut client = Client::connect(&server.record).unwrap(); + match client.call("proto", json!({})).unwrap_err() { + LinkError::Protocol { server_min, server_max } => { + assert_eq!(server_min, 2); + assert_eq!(server_max, 5); + } + other => panic!("{other}"), + } + match client.call("text", json!({})).unwrap_err() { + LinkError::Invalid(text) => assert_eq!(text, "no such method"), + other => panic!("{other}"), + } + match client.call("params", json!({})).unwrap_err() { + LinkError::BadParams(text) => assert_eq!(text, "missing command"), + other => panic!("{other}"), + } + } + + #[test] + fn next_event_resumes_a_line_cut_short_by_a_timeout() { + let server = peer("split", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + let out = reader.get_mut(); + out.write_all(b"{\"event\":\"doc.changed\",\"seq\":9,").unwrap(); + out.flush().unwrap(); + std::thread::sleep(Duration::from_millis(600)); + out.write_all(b"\"data\":{\"doc_id\":\"abc\"}}\n").unwrap(); + out.flush().unwrap(); + std::thread::sleep(Duration::from_millis(300)); + }); + let mut client = Client::connect(&server.record).unwrap(); + assert!(client.next_event(Some(Duration::from_millis(200))).unwrap().is_none()); + let event = client.next_event(Some(Duration::from_secs(5))).unwrap().expect("the whole event"); + assert_eq!(event.event, "doc.changed"); + assert_eq!(event.seq, 9); + assert_eq!(event.data, json!({"doc_id": "abc"})); + } + + #[test] + fn next_event_times_out_when_the_peer_is_quiet() { + let server = peer("wait", |stream, record| { + let mut reader = BufReader::new(stream); + accept_auth(&mut reader, &record.token); + std::thread::sleep(Duration::from_millis(800)); + }); + let mut client = Client::connect(&server.record).unwrap(); + let started = Instant::now(); + let got = client.next_event(Some(Duration::from_millis(300))).unwrap(); + let elapsed = started.elapsed(); + assert!(got.is_none()); + assert!(elapsed >= Duration::from_millis(200), "returned too fast: {elapsed:?}"); + assert!(elapsed < Duration::from_secs(5), "waited too long: {elapsed:?}"); + } +} diff --git a/crates/ipc/src/discovery/dir.rs b/crates/ipc/src/discovery/dir.rs new file mode 100644 index 000000000..976c316bb --- /dev/null +++ b/crates/ipc/src/discovery/dir.rs @@ -0,0 +1,354 @@ +//! Per-user runtime directory. Hand-rolled environment lookups. No `dirs` crate. + +use std::path::{Path, PathBuf}; + +use crate::error::LinkError; + +/// The directory that holds instance records, locks, and socket files. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeDir { + path: PathBuf, +} + +impl RuntimeDir { + /// Resolve the directory for this process and create it when it is missing. + pub fn resolve() -> Result { + Self::from_resolved_path(runtime_path()?) + } + + fn from_resolved_path(path: PathBuf) -> Result { + // LOCALAPPDATA exists on a fresh Windows install, but its Craft namespace does not. + // Create and validate that single namespace first; do not recursively follow arbitrary + // missing ancestors. RuntimeDir::at keeps its existing caller-provided-path contract. + #[cfg(windows)] + if let Some(parent) = path.parent() { + ensure_private_dir(parent)?; + } + Self::at(path) + } + + /// Use `path` as the runtime directory. Tests use this so they do not touch + /// the real `$XDG_RUNTIME_DIR/craft`. + pub fn at(path: impl Into) -> Result { + let path = path.into(); + ensure_private_dir(&path)?; + Ok(Self { path }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn record_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.json")) + } + + pub fn lock_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.lock")) + } + + pub fn socket_file(&self, app: &str, pid: u32) -> PathBuf { + self.path.join(format!("{app}-{pid}.sock")) + } + + /// Unix socket endpoint. The path must fit the instance-record limit of 100 bytes. + pub fn unix_endpoint(&self, app: &str, pid: u32) -> Result { + let path = self.socket_file(app, pid); + let text = path.to_str().ok_or_else(|| LinkError::Invalid("socket path is not utf-8".into()))?.to_string(); + if text.len() > 100 { + return Err(LinkError::Invalid(format!("socket path is {} bytes, the record allows 100", text.len()))); + } + Ok(super::Endpoint::Unix { path: text }) + } +} + +/// Set `path` to `mode` (for example `0o600`) and refuse a different result. +/// Unix socket mode cannot be set through `ListenerOptions::set_mode` (it is +/// `pub(crate)` in interprocess 2.4.4). Call this after the socket file exists. +#[cfg(unix)] +pub fn enforce_mode(path: &Path, mode: u32) -> Result<(), LinkError> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode))?; + let got = std::fs::symlink_metadata(path)?.mode() & 0o777; + if got != mode { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("mode is {got:o}, want {mode:o}") }); + } + Ok(()) +} + +#[cfg(unix)] +pub(crate) fn current_euid() -> Result { + #[cfg(target_os = "linux")] + { + let text = std::fs::read_to_string("/proc/self/status")?; + for line in text.lines() { + let Some(rest) = line.strip_prefix("Uid:") else { continue }; + let mut fields = rest.split_whitespace(); + let _real = fields.next(); + let Some(eff) = fields.next() else { + return Err(LinkError::Io("Uid line is short".into())); + }; + return eff.parse().map_err(LinkError::io); + } + Err(LinkError::Io("no Uid line in /proc/self/status".into())) + } + #[cfg(all(unix, not(target_os = "linux")))] + { + probe_euid(&std::env::temp_dir()) + } +} + +/// The effective uid, read from the owner of a file this process creates in `dir`. The std library +/// has no safe call for the euid and this crate forbids `unsafe`. The file has a random name and is +/// made with `create_new`, which fails on an existing path (a planted file or symlink) instead of +/// opening it, so the probe only ever touches a file it made. The file is removed before the +/// function returns. +#[cfg(any(all(unix, not(target_os = "linux")), all(test, unix)))] +fn probe_euid(dir: &Path) -> Result { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let path = dir.join(format!(".craft-uid-{}-{}", std::process::id(), crate::auth::encode_hex(&rand))); + let file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&path)?; + let uid = file.metadata().map(|meta| meta.uid()); + drop(file); + let _ = std::fs::remove_file(&path); + Ok(uid?) +} + +fn runtime_path() -> Result { + #[cfg(target_os = "linux")] + { + let xdg = std::env::var_os("XDG_RUNTIME_DIR"); + Ok(linux_runtime_path(xdg.as_deref(), current_euid()?)) + } + #[cfg(target_os = "macos")] + { + let tmpdir = std::env::var_os("TMPDIR"); + Ok(macos_runtime_path(tmpdir.as_deref(), current_euid()?)) + } + #[cfg(windows)] + { + let value = std::env::var_os("LOCALAPPDATA").filter(|v| !v.is_empty()); + let Some(value) = value else { + return Err(LinkError::Refused { path: "%LOCALAPPDATA%".into(), reason: "LOCALAPPDATA is unset".into() }); + }; + let base = PathBuf::from(value); + if !base.is_absolute() { + return Err(LinkError::Refused { path: base.display().to_string(), reason: "LOCALAPPDATA is not absolute".into() }); + } + Ok(base.join("Craft").join("run")) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + Err(LinkError::Refused { path: String::new(), reason: "no runtime directory for this platform".into() }) + } +} + +#[cfg(target_os = "linux")] +fn linux_runtime_path(xdg: Option<&std::ffi::OsStr>, euid: u32) -> PathBuf { + if let Some(xdg) = xdg { + let path = PathBuf::from(xdg); + if !xdg.is_empty() && path.is_absolute() { + return path.join("craft"); + } + } + PathBuf::from(format!("/tmp/craft-{euid}")) +} + +/// `$TMPDIR/craft` when `TMPDIR` is absolute. Otherwise `/tmp/craft-`, because a shared +/// `/tmp/craft` belongs to the first user who creates it and refuses everyone else. +#[cfg(any(target_os = "macos", all(test, unix)))] +fn macos_runtime_path(tmpdir: Option<&std::ffi::OsStr>, euid: u32) -> PathBuf { + if let Some(tmpdir) = tmpdir { + let path = PathBuf::from(tmpdir); + if !tmpdir.is_empty() && path.is_absolute() { + return path.join("craft"); + } + } + PathBuf::from(format!("/tmp/craft-{euid}")) +} + +pub(crate) fn ensure_private_dir(path: &Path) -> Result<(), LinkError> { + if is_symlink(path) { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "runtime directory is a symlink".into() }); + } + let created = match create_dir(path) { + Ok(()) => true, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => false, + Err(err) => return Err(LinkError::io(err)), + }; + if is_symlink(path) { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "runtime directory is a symlink".into() }); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if created { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))?; + } + verify_unix_dir(path)?; + } + #[cfg(not(unix))] + { + let _ = created; + let meta = std::fs::metadata(path)?; + if !meta.is_dir() { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "not a directory".into() }); + } + } + Ok(()) +} + +fn create_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path) + } + #[cfg(not(unix))] + { + std::fs::create_dir(path) + } +} + +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) +} + +#[cfg(unix)] +fn verify_unix_dir(path: &Path) -> Result<(), LinkError> { + use std::os::unix::fs::MetadataExt; + let meta = std::fs::symlink_metadata(path)?; + if !meta.is_dir() { + return Err(LinkError::Refused { path: path.display().to_string(), reason: "not a directory".into() }); + } + let uid = current_euid()?; + if meta.uid() != uid { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("owner is {}, euid is {uid}", meta.uid()) }); + } + let mode = meta.mode() & 0o777; + if mode != 0o700 { + return Err(LinkError::Refused { path: path.display().to_string(), reason: format!("mode is {mode:o}, want 700") }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + path + } + + #[test] + fn creates_0700_and_refuses_a_loose_dir() { + let path = scratch("dir"); + let dir = RuntimeDir::at(&path).unwrap(); + assert_eq!(dir.path(), path.as_path()); + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + assert_eq!(std::fs::symlink_metadata(&path).unwrap().mode() & 0o777, 0o700); + let loose = scratch("loose"); + std::fs::create_dir(&loose).unwrap(); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o755)).unwrap(); + let err = RuntimeDir::at(&loose).unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err}"); + let _ = std::fs::remove_dir_all(&loose); + } + let _ = std::fs::remove_dir_all(&path); + } + + #[cfg(unix)] + #[test] + fn refuses_a_symlink() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let real = scratch("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = scratch("link"); + symlink(&real, &link).unwrap(); + assert!(RuntimeDir::at(&link).is_err()); + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_dir_all(&real); + } + + #[cfg(target_os = "linux")] + #[test] + fn xdg_fallback_is_pure() { + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("/run/user/1000")), 5), PathBuf::from("/run/user/1000/craft")); + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(linux_runtime_path(Some(std::ffi::OsStr::new("relative")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(linux_runtime_path(None, 9), PathBuf::from("/tmp/craft-9")); + } + + #[cfg(unix)] + #[test] + fn macos_fallback_is_per_user() { + use std::ffi::OsStr; + assert_eq!(macos_runtime_path(Some(OsStr::new("/var/folders/ab/T")), 5), PathBuf::from("/var/folders/ab/T/craft")); + assert_eq!(macos_runtime_path(Some(OsStr::new("")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(macos_runtime_path(Some(OsStr::new("relative")), 5), PathBuf::from("/tmp/craft-5")); + assert_eq!(macos_runtime_path(None, 9), PathBuf::from("/tmp/craft-9")); + assert_ne!(macos_runtime_path(None, 5), macos_runtime_path(None, 6)); + } + + #[cfg(unix)] + #[test] + fn the_uid_probe_leaves_a_planted_file_alone_and_cleans_up() { + use std::os::unix::fs::symlink; + let dir = scratch("probe"); + std::fs::create_dir_all(&dir).unwrap(); + // The names the probe used to pick: a planted file and a planted symlink must not be opened. + let planted = dir.join(format!(".craft-uid-{}", std::process::id())); + std::fs::write(&planted, b"keep me").unwrap(); + let victim = dir.join("victim"); + std::fs::write(&victim, b"keep me too").unwrap(); + let link = dir.join(format!(".craft-uid-{}-0000000000000000", std::process::id())); + symlink(&victim, &link).unwrap(); + let uid = probe_euid(&dir).unwrap(); + assert_eq!(uid, current_euid().unwrap()); + assert_eq!(std::fs::read(&planted).unwrap(), b"keep me"); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep me too"); + let mut names: Vec = std::fs::read_dir(&dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + names.sort(); + assert_eq!(names.len(), 3, "the probe file is gone: {names:?}"); + // Two probes do not collide. + assert_eq!(probe_euid(&dir).unwrap(), uid); + // A directory that cannot be written is an error, not a guess. + assert!(probe_euid(&dir.join("missing")).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(windows)] + #[test] + fn a_fresh_windows_runtime_creates_its_namespace() { + let base = scratch("fresh-runtime"); + std::fs::create_dir_all(&base).unwrap(); + let path = base.join("Craft").join("run"); + let dir = RuntimeDir::from_resolved_path(path.clone()).unwrap(); + assert_eq!(dir.path(), path); + assert!(base.join("Craft").is_dir()); + std::fs::remove_dir_all(base).unwrap(); + } + + #[test] + fn resolve_points_at_a_private_craft_dir() { + let dir = RuntimeDir::resolve().unwrap(); + #[cfg(windows)] + assert!(dir.path().ends_with(Path::new("Craft").join("run")), "{}", dir.path().display()); + #[cfg(unix)] + assert!(dir.path().ends_with("craft") || dir.path().ends_with(format!("craft-{}", current_euid().unwrap())), "{}", dir.path().display()); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!(std::fs::symlink_metadata(dir.path()).unwrap().mode() & 0o777, 0o700); + } + } +} diff --git a/crates/ipc/src/discovery/mod.rs b/crates/ipc/src/discovery/mod.rs new file mode 100644 index 000000000..f81731970 --- /dev/null +++ b/crates/ipc/src/discovery/mod.rs @@ -0,0 +1,11 @@ +//! Discovery records for Craft apps on this machine. + +mod dir; +mod record; + +pub use dir::RuntimeDir; +pub use record::{Discovery, DocInfo, Endpoint, InstanceRecord, Protocol, valid_app_name}; + +pub(crate) use dir::ensure_private_dir; +#[cfg(unix)] +pub(crate) use dir::{current_euid, enforce_mode}; diff --git a/crates/ipc/src/discovery/record.rs b/crates/ipc/src/discovery/record.rs new file mode 100644 index 000000000..290ac0bb5 --- /dev/null +++ b/crates/ipc/src/discovery/record.rs @@ -0,0 +1,575 @@ +//! Instance records. A live process holds `-.lock`. `scan` deletes +//! the record, the socket, and the lock when `try_lock` acquires that file. + +use std::collections::BTreeSet; +use std::fs::{File, OpenOptions, TryLockError}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Deserialize; +use serde::Serialize; +use serde_json::{Map, Value}; + +use crate::auth::{self, is_lower_hex}; +use crate::error::LinkError; +use crate::wire::PROTOCOL_MAX; + +use super::RuntimeDir; + +/// Where clients connect. Serialized with an internal `kind` tag. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Endpoint { + Unix { path: String }, + Pipe { name: String }, + Tcp { port: u16 }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Protocol { + pub link: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commands_schema: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DocInfo { + pub doc_id: String, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// One running app. Unknown JSON fields are kept in `extra`. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct InstanceRecord { + pub v: u32, + pub app: String, + pub app_version: String, + pub pid: u32, + pub instance: String, + pub endpoint: Endpoint, + pub token: String, + pub started: u64, + pub protocol: Protocol, + pub docs: Vec, + #[serde(default, flatten)] + pub extra: Map, +} + +impl std::fmt::Debug for InstanceRecord { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InstanceRecord") + .field("v", &self.v) + .field("app", &self.app) + .field("app_version", &self.app_version) + .field("pid", &self.pid) + .field("instance", &self.instance) + .field("endpoint", &self.endpoint) + .field("token", &"redacted") + .field("started", &self.started) + .field("protocol", &self.protocol) + .field("docs", &self.docs) + .finish_non_exhaustive() + } +} + +impl InstanceRecord { + pub fn new(app: &str, app_version: &str, pid: u32, endpoint: Endpoint) -> Result { + let record = Self { + v: 1, + app: app.to_string(), + app_version: app_version.to_string(), + pid, + instance: auth::random_uuid()?, + endpoint, + token: auth::generate_token()?, + started: unix_now()?, + protocol: Protocol { link: PROTOCOL_MAX, commands_schema: None, extra: Map::new() }, + docs: Vec::new(), + extra: Map::new(), + }; + record.validate()?; + Ok(record) + } + + pub fn validate(&self) -> Result<(), LinkError> { + if self.v != 1 { + return Err(LinkError::Invalid(format!("record v is {}, want 1", self.v))); + } + if !valid_app_name(&self.app) { + return Err(LinkError::Invalid(format!("app {:?} does not match the record pattern", self.app))); + } + if self.app_version.is_empty() { + return Err(LinkError::Invalid("app_version is empty".into())); + } + if self.pid < 1 { + return Err(LinkError::Invalid("pid must be >= 1".into())); + } + if !valid_uuid(&self.instance) { + return Err(LinkError::Invalid("instance is not a uuid".into())); + } + if self.token.len() != 64 || !is_lower_hex(&self.token) { + return Err(LinkError::Invalid("token must be 64 lowercase hex characters".into())); + } + if self.protocol.link < 1 { + return Err(LinkError::Invalid("protocol.link must be >= 1".into())); + } + if let Some(schema) = &self.protocol.commands_schema { + let Some(hex) = schema.strip_prefix("sha256:") else { + return Err(LinkError::Invalid("commands_schema must start with sha256:".into())); + }; + if hex.len() != 64 || !is_lower_hex(hex) { + return Err(LinkError::Invalid("commands_schema digest is not 64 lowercase hex characters".into())); + } + } + match &self.endpoint { + Endpoint::Unix { path } => { + if path.len() > 100 || path.is_empty() || path.contains('\0') || !Path::new(path).is_absolute() { + return Err(LinkError::Invalid("unix endpoint path must be absolute and at most 100 bytes".into())); + } + } + Endpoint::Pipe { name } => { + if !name.starts_with("craft-") || name.contains('/') || name.contains('\\') { + return Err(LinkError::Invalid("pipe name must start with craft-".into())); + } + } + Endpoint::Tcp { port } => { + if *port < 1 { + return Err(LinkError::Invalid("tcp port must be >= 1".into())); + } + } + } + for doc in &self.docs { + if doc.doc_id.is_empty() || doc.title.is_empty() { + return Err(LinkError::Invalid("doc_id and title are required".into())); + } + } + self.check_caps()?; + self.check_tcp_port()?; + Ok(()) + } + + /// Roles and optional verbs from the optional `caps` field (stored in + /// `extra`, so the struct shape and old struct literals stay valid). + /// Empty when the field is absent or malformed. [`Self::validate`] rejects + /// a malformed field. + pub fn caps(&self) -> Vec { + self.extra + .get("caps") + .and_then(Value::as_array) + .map(|list| list.iter().filter_map(|c| c.as_str().map(str::to_string)).collect()) + .unwrap_or_default() + } + + /// Add `cap` to `caps` when it is not there yet. + pub fn add_cap(&mut self, cap: &str) -> Result<(), LinkError> { + if !valid_cap(cap) { + return Err(LinkError::Invalid(format!("cap {cap:?} must match [a-z][a-z0-9._-]{{0,63}}"))); + } + let mut caps = self.caps(); + if !caps.iter().any(|c| c == cap) { + caps.push(cap.to_string()); + } + self.extra.insert("caps".to_string(), Value::from(caps)); + Ok(()) + } + + /// Loopback TCP control port, when the app opened one (`--control `). Stored in + /// `extra`. The record file is 0600, so only the owner can read this port and the token + /// that the TCP listener requires. + pub fn tcp_port(&self) -> Option { + self.extra.get("tcp_port").and_then(Value::as_u64).and_then(|port| u16::try_from(port).ok()).filter(|port| *port >= 1) + } + + /// Publish the TCP control port in the record. + pub fn set_tcp_port(&mut self, port: u16) { + self.extra.insert("tcp_port".to_string(), Value::from(port)); + } + + fn check_tcp_port(&self) -> Result<(), LinkError> { + match self.extra.get("tcp_port") { + None => Ok(()), + Some(value) if value.as_u64().is_some_and(|port| (1..=65535).contains(&port)) => Ok(()), + Some(_) => Err(LinkError::Invalid("tcp_port must be an integer from 1 to 65535".into())), + } + } + + fn check_caps(&self) -> Result<(), LinkError> { + let Some(value) = self.extra.get("caps") else { + return Ok(()); + }; + let Some(list) = value.as_array() else { + return Err(LinkError::Invalid("caps must be an array of strings".into())); + }; + let mut seen = BTreeSet::new(); + for item in list { + let Some(cap) = item.as_str().filter(|c| valid_cap(c)) else { + return Err(LinkError::Invalid("each cap must match [a-z][a-z0-9._-]{0,63}".into())); + }; + if !seen.insert(cap) { + return Err(LinkError::Invalid(format!("cap {cap:?} is listed twice"))); + } + } + Ok(()) + } +} + +fn valid_cap(cap: &str) -> bool { + let bytes = cap.as_bytes(); + (1..=64).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_lowercase) + && bytes.iter().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-')) +} + +/// Holds the lock file for this process. Drop releases the lock and leaves the +/// files in place. [`Discovery::remove`] deletes them. [`Discovery::scan`] +/// deletes them when the holder is gone. +pub struct Discovery { + dir: PathBuf, + stem: String, + record: InstanceRecord, + lock: File, +} + +impl std::fmt::Debug for Discovery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Discovery").field("stem", &self.stem).field("record", &self.record).finish_non_exhaustive() + } +} + +impl Discovery { + /// Write the record and hold `-.lock` until drop. + /// + /// `try_lock` is the acquire. A successful `try_lock` is the same exclusive + /// lock as `File::lock`, and it returns [`LinkError::Busy`] when a live + /// process already holds the file instead of waiting. + pub fn publish(dir: &RuntimeDir, record: InstanceRecord) -> Result { + record.validate()?; + let stem = format!("{}-{}", record.app, record.pid); + let lock_path = dir.path().join(format!("{stem}.lock")); + let mut opts = OpenOptions::new(); + opts.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let lock = opts.open(&lock_path)?; + #[cfg(unix)] + crate::discovery::enforce_mode(&lock_path, 0o600)?; + match lock.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(LinkError::Busy(format!("{stem} is already live"))); + } + Err(TryLockError::Error(err)) => return Err(err.into()), + } + let keep = unix_path(&record); + clear_payload(dir.path(), &stem, keep.as_deref())?; + let bytes = serde_json::to_vec(&record)?; + write_atomic(dir.path(), &stem, &bytes)?; + Ok(Self { dir: dir.path().to_path_buf(), stem, record, lock }) + } + + pub fn record(&self) -> &InstanceRecord { + &self.record + } + + /// Delete this process's record, socket, and lock. + pub fn remove(self) -> Result<(), LinkError> { + reap_dead(&self.dir, &self.stem, &self.lock) + } + + /// Live records in `dir`. A lock that `try_lock` acquires belongs to a dead + /// process: this deletes its record, socket, and lock. + pub fn scan(dir: &RuntimeDir) -> Result, LinkError> { + let mut stems = BTreeSet::new(); + let entries = std::fs::read_dir(dir.path())?; + for entry in entries { + let entry = entry?; + let Some(name) = entry.file_name().into_string().ok() else { continue }; + if let Some(stem) = stem_from_entry(&name) { + stems.insert(stem); + } + } + let mut live = Vec::new(); + for stem in stems { + if let Some(record) = settle(dir.path(), &stem)? { + live.push(record); + } + } + Ok(live) + } +} + +pub fn valid_app_name(app: &str) -> bool { + crate::ids::valid_app_id(app) +} + +fn valid_uuid(text: &str) -> bool { + crate::ids::valid_uuid(text) +} + +fn unix_now() -> Result { + Ok(SystemTime::now().duration_since(UNIX_EPOCH).map_err(LinkError::io)?.as_secs()) +} + +fn stem_from_entry(name: &str) -> Option { + if let Some(stem) = name.strip_suffix(".json").or_else(|| name.strip_suffix(".lock")).or_else(|| name.strip_suffix(".sock")) { + return accept_stem(stem); + } + let rest = name.strip_prefix('.')?.strip_suffix(".tmp")?; + let (stem, hex) = rest.rsplit_once('.')?; + if hex.len() < 2 || !is_lower_hex(hex) { + return None; + } + accept_stem(stem) +} + +fn accept_stem(stem: &str) -> Option { + let (app, pid) = stem.rsplit_once('-')?; + let pid: u32 = pid.parse().ok()?; + if pid < 1 || !valid_app_name(app) { + return None; + } + Some(stem.to_string()) +} + +fn settle(dir: &Path, stem: &str) -> Result, LinkError> { + let lock_path = dir.join(format!("{stem}.lock")); + let json_path = dir.join(format!("{stem}.json")); + if lock_path.is_file() { + let lock = OpenOptions::new().read(true).write(true).open(&lock_path)?; + match lock.try_lock() { + Err(TryLockError::WouldBlock) => return read_live(&json_path), + Err(TryLockError::Error(err)) => return Err(err.into()), + Ok(()) => { + reap_dead(dir, stem, &lock)?; + return Ok(None); + } + } + } + if json_path.exists() || dir.join(format!("{stem}.sock")).exists() { + clear_payload(dir, stem, None)?; + } else { + remove_tmps(dir, stem)?; + } + Ok(None) +} + +fn read_live(json_path: &Path) -> Result, LinkError> { + if !json_path.is_file() { + return Ok(None); + } + let bytes = std::fs::read(json_path)?; + let record: InstanceRecord = match serde_json::from_slice(&bytes) { + Ok(record) => record, + Err(_) => return Ok(None), + }; + if record.validate().is_err() { + return Ok(None); + } + Ok(Some(record)) +} + +fn reap_dead(dir: &Path, stem: &str, lock: &File) -> Result<(), LinkError> { + clear_payload(dir, stem, None)?; + let lock_path = dir.join(format!("{stem}.lock")); + match std::fs::remove_file(&lock_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + lock.unlock()?; + Ok(()) +} + +fn unix_path(record: &InstanceRecord) -> Option { + match &record.endpoint { + Endpoint::Unix { path } => Some(PathBuf::from(path)), + Endpoint::Pipe { .. } | Endpoint::Tcp { .. } => None, + } +} + +/// Remove the json file, temp files, and socket files. `keep` is the socket the +/// caller just bound and is about to publish. Deleting that name would unlist +/// a live listener. +fn clear_payload(dir: &Path, stem: &str, keep: Option<&Path>) -> Result<(), LinkError> { + let json_path = dir.join(format!("{stem}.json")); + if let Ok(bytes) = std::fs::read(&json_path) + && let Ok(record) = serde_json::from_slice::(&bytes) + && let Some(old) = unix_path(&record) + && keep.is_none_or(|path| path != old.as_path()) + { + remove_socket_inside(dir, &old); + } + let conventional = dir.join(format!("{stem}.sock")); + if keep.is_none_or(|path| path != conventional.as_path()) { + remove_socket_inside(dir, &conventional); + } + match std::fs::remove_file(&json_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + remove_tmps(dir, stem) +} + +fn remove_tmps(dir: &Path, stem: &str) -> Result<(), LinkError> { + let prefix = format!(".{stem}."); + let entries = std::fs::read_dir(dir)?; + for entry in entries { + let entry = entry?; + let Some(name) = entry.file_name().into_string().ok() else { continue }; + if name.starts_with(&prefix) && name.ends_with(".tmp") { + std::fs::remove_file(entry.path())?; + } + } + Ok(()) +} + +#[cfg(unix)] +fn remove_socket_inside(runtime: &Path, candidate: &Path) { + use std::os::unix::fs::FileTypeExt; + let Ok(meta) = std::fs::symlink_metadata(candidate) else { return }; + if !meta.file_type().is_socket() { + return; + } + let Some(parent) = candidate.parent() else { return }; + let Ok(parent_real) = std::fs::canonicalize(parent) else { return }; + let Ok(runtime_real) = std::fs::canonicalize(runtime) else { return }; + if parent_real == runtime_real { + let _ = std::fs::remove_file(candidate); + } +} + +/// Only Unix endpoints leave socket files behind. A named pipe has no file to remove. +#[cfg(not(unix))] +fn remove_socket_inside(runtime: &Path, candidate: &Path) { + let _ = (runtime, candidate); +} + +/// `.-..tmp`, then rename onto `-.json`. +fn write_atomic(dir: &Path, stem: &str, bytes: &[u8]) -> Result<(), LinkError> { + let mut rand = [0u8; 8]; + getrandom::getrandom(&mut rand).map_err(LinkError::io)?; + let tmp = dir.join(format!(".{stem}.{}.tmp", auth::encode_hex(&rand))); + let final_path = dir.join(format!("{stem}.json")); + let mut opts = OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut file = opts.open(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + if let Err(err) = std::fs::rename(&tmp, &final_path) { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(label: &str) -> (PathBuf, RuntimeDir) { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + let dir = RuntimeDir::at(&path).unwrap(); + (path, dir) + } + + #[test] + fn schema_accepts_a_record_and_rejects_a_bad_token() { + let (_path, dir) = scratch("schema"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let mut value = serde_json::to_value(&record).unwrap(); + value["note"] = Value::String("kept".into()); + let restored: InstanceRecord = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(restored.extra.get("note").and_then(Value::as_str), Some("kept")); + let schema_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schemas/instance-record.schema.json"); + let schema: Value = serde_json::from_slice(&std::fs::read(&schema_path).unwrap()).unwrap(); + let validator = jsonschema::options().should_validate_formats(true).build(&schema).unwrap(); + let errors: Vec<_> = validator.iter_errors(&value).map(|err| err.to_string()).collect(); + assert!(errors.is_empty(), "{errors:?}"); + value["token"] = Value::String("nope".into()); + assert!(!validator.is_valid(&value)); + let _ = std::fs::remove_dir_all(dir.path()); + } + + #[test] + fn scan_keeps_a_held_record_and_drops_it_after_unlock() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let (path, dir) = scratch("scan"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let record = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let decoy = dir.socket_file("effectcraft", 4242); + std::fs::write(&decoy, b"not a socket").unwrap(); + let held = Discovery::publish(&dir, record.clone()).unwrap(); + let live = Discovery::scan(&dir).unwrap(); + assert_eq!(live.len(), 1); + assert_eq!(live[0].instance, record.instance); + assert_eq!(live[0].token, record.token); + drop(held); + assert!(Discovery::scan(&dir).unwrap().is_empty()); + assert!(!dir.record_file("effectcraft", 4242).exists()); + assert!(!dir.lock_file("effectcraft", 4242).exists()); + assert!(decoy.exists(), "scan must not delete a regular file"); + let _ = std::fs::remove_dir_all(&path); + } + + #[cfg(unix)] + #[test] + fn scan_unlinks_a_dead_socket() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + use interprocess::local_socket::prelude::*; + use interprocess::local_socket::{GenericFilePath, ListenerOptions}; + + let (path, dir) = scratch("sock"); + let endpoint = dir.unix_endpoint("filmcraft", 5151).unwrap(); + let sock = match &endpoint { + Endpoint::Unix { path } => path.clone(), + Endpoint::Pipe { .. } | Endpoint::Tcp { .. } => panic!("unix endpoint"), + }; + let name = Path::new(&sock).to_fs_name::().unwrap(); + let listener = ListenerOptions::new().name(name).reclaim_name(false).create_sync().unwrap(); + crate::discovery::enforce_mode(Path::new(&sock), 0o600).unwrap(); + let record = InstanceRecord::new("filmcraft", "0.9.1", 5151, endpoint).unwrap(); + let held = Discovery::publish(&dir, record).unwrap(); + assert!(Path::new(&sock).exists(), "publish must leave the bound socket in place"); + drop(held); + assert!(Path::new(&sock).exists(), "dropping the lock must not unlink the socket"); + assert!(Discovery::scan(&dir).unwrap().is_empty()); + assert!(!Path::new(&sock).exists(), "scan removes the socket once the lock is free"); + drop(listener); + let _ = std::fs::remove_dir_all(&path); + } + + #[test] + fn second_publish_of_a_live_stem_is_busy() { + let _no_fork = crate::SPAWN_GUARD.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let (path, dir) = scratch("busy"); + let endpoint = dir.unix_endpoint("effectcraft", 4242).unwrap(); + let first = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint.clone()).unwrap(); + let held = Discovery::publish(&dir, first).unwrap(); + let second = InstanceRecord::new("effectcraft", "0.8.3", 4242, endpoint).unwrap(); + let err = Discovery::publish(&dir, second).unwrap_err(); + assert!(matches!(err, LinkError::Busy(_)), "{err}"); + drop(held); + let _ = std::fs::remove_dir_all(&path); + } +} diff --git a/crates/ipc/src/error.rs b/crates/ipc/src/error.rs new file mode 100644 index 000000000..65259fd58 --- /dev/null +++ b/crates/ipc/src/error.rs @@ -0,0 +1,134 @@ +//! Failures a Craft Link caller can show or turn into a protocol error object. + +use serde::Serialize; +use serde_json::{Map, Value}; + +/// Wire error object for new methods. Existing methods keep a string `error`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ErrorObject { + pub code: String, + pub message: String, + pub data: Map, +} + +/// A link failure. The first-line auth refusal is still the string +/// `authentication required`. [`LinkError::to_object`] is for new methods. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LinkError { + Io(String), + /// The runtime directory or socket mode is not acceptable. + Refused { + path: String, + reason: String, + }, + Invalid(String), + Busy(String), + Auth, + Protocol { + server_min: u32, + server_max: u32, + }, + BadParams(String), + NotInstalled { + app: String, + }, + Mismatch { + field: String, + expected: String, + got: String, + }, + Closed, + Timeout, + TooLarge, + NotReady(String), + /// An asset moniker, link id or file that the peer does not know. + Unknown(String), + /// A `link.open` payload with a bad shape or an unknown `format`. + PayloadInvalid(String), + /// A `link.open` payload names media files that do not exist or cannot be opened. The + /// sender renders or relinks them and sends the call again. + MediaMissing { + paths: Vec, + }, +} + +impl LinkError { + pub fn io(err: impl std::fmt::Display) -> Self { + Self::Io(err.to_string()) + } + + /// Contract section 7 code. + pub fn code(&self) -> &'static str { + match self { + Self::Auth => "auth_required", + Self::Protocol { .. } => "protocol_unsupported", + Self::BadParams(_) | Self::Invalid(_) => "bad_params", + Self::NotReady(_) => "not_ready", + Self::Unknown(_) => "link_unknown", + Self::PayloadInvalid(_) => "payload_invalid", + Self::MediaMissing { .. } => "media_missing", + Self::TooLarge => "payload_too_large", + Self::Io(_) => "io_error", + Self::Busy(_) => "busy", + Self::NotInstalled { .. } => "not_ready", + Self::Refused { .. } | Self::Mismatch { .. } | Self::Closed | Self::Timeout => "internal", + } + } + + pub fn to_object(&self) -> ErrorObject { + let mut data = Map::new(); + if let Self::Protocol { server_min, server_max } = self { + data.insert("server_min".into(), Value::from(*server_min)); + data.insert("server_max".into(), Value::from(*server_max)); + } + if let Self::MediaMissing { paths } = self { + data.insert("missing".into(), Value::from(paths.clone())); + } + ErrorObject { code: self.code().to_string(), message: self.to_string(), data } + } +} + +impl std::fmt::Display for LinkError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(msg) => write!(f, "{msg}"), + Self::Refused { path, reason } => write!(f, "{path}: {reason}"), + Self::Invalid(msg) => write!(f, "{msg}"), + Self::Busy(msg) => write!(f, "{msg}"), + Self::Auth => write!(f, "authentication required"), + Self::Protocol { server_min, server_max } => { + write!(f, "protocol not supported (server {server_min} to {server_max})") + } + Self::BadParams(msg) => write!(f, "{msg}"), + Self::NotInstalled { app } => write!(f, "{app} is not installed"), + Self::Mismatch { field, expected, got } => { + write!(f, "{field} is {got}, record says {expected}") + } + Self::Closed => write!(f, "connection closed"), + Self::Timeout => write!(f, "timeout"), + Self::TooLarge => write!(f, "payload too large"), + Self::NotReady(msg) => write!(f, "{msg}"), + Self::Unknown(msg) | Self::PayloadInvalid(msg) => write!(f, "{msg}"), + Self::MediaMissing { paths } => write!(f, "the payload names media that cannot be opened: {}", paths.join(", ")), + } + } +} + +impl std::error::Error for LinkError {} + +impl From for LinkError { + /// A socket read timeout is `WouldBlock` on Linux ("Resource temporarily unavailable") and + /// `TimedOut` elsewhere. Both mean the peer was quiet, not that the connection broke. + fn from(err: std::io::Error) -> Self { + match err.kind() { + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut => Self::Timeout, + _ => Self::Io(err.to_string()), + } + } +} + +impl From for LinkError { + fn from(err: serde_json::Error) -> Self { + Self::Invalid(err.to_string()) + } +} diff --git a/crates/ipc/src/fetch.rs b/crates/ipc/src/fetch.rs new file mode 100644 index 000000000..ca3f4d194 --- /dev/null +++ b/crates/ipc/src/fetch.rs @@ -0,0 +1,370 @@ +//! Pull frames from a peer without blocking the caller past a deadline. +//! +//! [`FrameFetcher`] owns the control connection that holds the `frames.open` ticket and a few +//! bulk connections ([`crate::bulk::BulkClient`]), each on its own worker thread. A caller asks +//! with [`FrameFetcher::fetch`] and a time limit. +//! +//! - A frame in the RAM cache answers at once. The cache keeps the newest epoch of each +//! [`FrameKey`] inside a byte budget and drops the least recently used frame first. +//! - A miss goes to a worker. Asks for one key share one render. +//! - A caller that runs out of time gets [`Fetched::TimedOut`]. The render goes on, and its frame +//! lands in the cache for the next ask. The caller never waits for the socket. +//! - A queued ask older than [`FetchOptions::max_age`] is dropped when a worker reaches it, unless +//! a caller is still waiting for it. The queue holds at most [`QUEUE_MAX`] asks: a new miss past +//! that drops the oldest. +//! - A connection error ends that worker. When the last worker ends the fetcher is down and every +//! ask that misses the cache returns [`Fetched::Down`] at once. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use crate::InstanceRecord; +use crate::bulk::{BulkClient, Frame, FrameFormat, FrameTicket}; +use crate::client::Client; +use crate::error::LinkError; + +/// Most bulk connections one fetcher opens. +pub const CONNECTIONS_MAX: usize = 8; + +/// Most asks that wait for a worker at once. A new miss past this drops the oldest queued ask. +pub const QUEUE_MAX: usize = 256; + +/// What identifies a frame for the cache: the caller's frame number and scale (in thousandths). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrameKey { + pub frame: i64, + pub scale_milli: u32, +} + +/// Tuning of a [`FrameFetcher`]. +#[derive(Debug, Clone)] +pub struct FetchOptions { + /// Bulk connections, and so renders in flight. Clamped to `1..=`[`CONNECTIONS_MAX`]. + pub connections: usize, + /// How long one socket read or write may take before the connection is dropped. + pub io_timeout: Duration, + /// Cache size in bytes of pixel data. + pub cache_bytes: usize, + /// A queued ask older than this is skipped, unless a caller is still waiting for it. + pub max_age: Duration, +} + +impl Default for FetchOptions { + fn default() -> Self { + Self { connections: 3, io_timeout: Duration::from_secs(5), cache_bytes: 256 * 1024 * 1024, max_age: Duration::from_secs(1) } + } +} + +/// The answer to one [`FrameFetcher::fetch`]. +#[derive(Debug, Clone)] +pub enum Fetched { + /// A frame whose epoch is at least the one asked for. + Frame(Arc), + /// The time limit passed. The render may still finish and fill the cache. + TimedOut, + /// The peer answered this frame with an error. The fetcher still works. + /// + /// Also the answer to a caller whose ask was dropped from a full queue (see [`QUEUE_MAX`]). That + /// caller is not re-queued: it asks again if it still wants the frame. + Failed(String), + /// The connections are gone and the cache has no answer. + Down, +} + +/// Counters since [`FrameFetcher::connect`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FetchStats { + /// Calls to [`FrameFetcher::fetch`]. + pub requests: u64, + /// Calls answered by the cache on the first look. + pub hits: u64, + /// Frames a worker received. + pub renders: u64, + pub timeouts: u64, + pub failures: u64, + /// Queued asks dropped for age or for a full queue. + pub skipped: u64, +} + +struct Job { + key: FrameKey, + time_s: f64, + epoch: u64, + queued: Instant, + /// The latest deadline of a caller that is waiting for this job. + wanted_until: Instant, +} + +struct Cached { + frame: Arc, + used: u64, +} + +#[derive(Default)] +struct Cache { + frames: HashMap, + bytes: usize, + tick: u64, +} + +impl Cache { + fn get(&mut self, key: FrameKey, min_epoch: u64) -> Option> { + self.tick += 1; + let tick = self.tick; + let entry = self.frames.get_mut(&key).filter(|entry| entry.frame.epoch >= min_epoch)?; + entry.used = tick; + Some(Arc::clone(&entry.frame)) + } + + fn insert(&mut self, key: FrameKey, frame: Arc, budget: usize) { + if self.frames.get(&key).is_some_and(|old| old.frame.epoch > frame.epoch) { + return; + } + self.tick += 1; + self.bytes += frame.pixels.len(); + if let Some(old) = self.frames.insert(key, Cached { frame, used: self.tick }) { + self.bytes -= old.frame.pixels.len(); + } + while self.bytes > budget && self.frames.len() > 1 { + let Some(oldest) = self.frames.iter().min_by_key(|(_, entry)| entry.used).map(|(key, _)| *key) else { break }; + if let Some(gone) = self.frames.remove(&oldest) { + self.bytes -= gone.frame.pixels.len(); + } + } + } +} + +#[derive(Default)] +struct State { + queue: VecDeque, + inflight: HashSet, + cache: Cache, + failed: HashMap, + fail_seq: u64, + stats: FetchStats, + stop: bool, +} + +/// Most failure records kept. Callers only look for a record newer than their own ask, so old +/// ones go first. +const FAILED_MAX: usize = 4 * QUEUE_MAX; + +impl State { + /// Record a failure of `key` that callers waiting for it will see. + fn note_failed(&mut self, key: FrameKey, message: String) { + self.fail_seq += 1; + self.failed.insert(key, (self.fail_seq, message)); + if self.failed.len() > FAILED_MAX { + let keep_from = self.fail_seq.saturating_sub(u64::try_from(FAILED_MAX).unwrap_or(u64::MAX)); + self.failed.retain(|_, (seq, _)| *seq > keep_from); + } + } +} + +struct Shared { + state: Mutex, + /// Workers wait here for a job. + work: Condvar, + /// Callers wait here for a frame, a failure or the end. + done: Condvar, + alive: AtomicUsize, + options: FetchOptions, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +/// Frames pulled from one link of a peer. See the module docs. +pub struct FrameFetcher { + shared: Arc, + ticket: FrameTicket, + /// Holds the ticket: the peer drops it when this connection closes. + control: Mutex, +} + +impl FrameFetcher { + /// Open the ticket and the bulk connections. Any failure is returned here, so a caller knows + /// the peer can serve frames before it relies on them. + pub fn connect(record: &InstanceRecord, link_id: &str, format: FrameFormat, scale: f64, options: FetchOptions) -> Result { + let mut control = Client::connect(record)?; + let ticket = control.frames_open(link_id, format, scale)?; + let connections = options.connections.clamp(1, CONNECTIONS_MAX); + let mut clients = Vec::with_capacity(connections); + for _ in 0..connections { + let mut client = BulkClient::attach(record, &ticket.ticket)?; + client.set_io_timeout(options.io_timeout); + clients.push(client); + } + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + work: Condvar::new(), + done: Condvar::new(), + alive: AtomicUsize::new(clients.len()), + options, + }); + for client in clients { + let worker = Arc::clone(&shared); + // Workers are not joined: one may sit in a socket read for up to `io_timeout`, and a + // drop on a UI thread must not wait for that. A worker sees `stop` and leaves. + if let Err(err) = std::thread::Builder::new().name("craft-ipc-fetch".into()).spawn(move || run_worker(&worker, client)) { + shared_stop(&shared); + return Err(err.into()); + } + } + Ok(Self { shared, ticket, control: Mutex::new(control) }) + } + + /// Size of the frames this fetcher yields. + pub fn layout(&self) -> FrameTicket { + self.ticket.clone() + } + + /// The cached frame for `key` if its epoch is at least `min_epoch`. Never asks the peer. + pub fn cached(&self, key: FrameKey, min_epoch: u64) -> Option> { + self.shared.lock().cache.get(key, min_epoch) + } + + /// The frame for `key` at an epoch of at least `min_epoch`, waiting at most `wait`. + /// `time_s` is the comp time the peer renders. + /// + /// A `time_s` that is not finite is answered with [`Fetched::Failed`] and is never queued: the + /// request would fail before it reaches the socket, and a worker reads that as a dead connection. + pub fn fetch(&self, key: FrameKey, time_s: f64, min_epoch: u64, wait: Duration) -> Fetched { + if !time_s.is_finite() { + return Fetched::Failed("time_s must be finite".into()); + } + let deadline = Instant::now() + wait; + let shared = &self.shared; + let mut state = shared.lock(); + state.stats.requests += 1; + let started_at = state.fail_seq; + let mut first = true; + loop { + if let Some(frame) = state.cache.get(key, min_epoch) { + if first { + state.stats.hits += 1; + } + return Fetched::Frame(frame); + } + first = false; + if let Some((seq, message)) = state.failed.get(&key) + && *seq > started_at + { + return Fetched::Failed(message.clone()); + } + if shared.alive.load(Ordering::SeqCst) == 0 || state.stop { + return Fetched::Down; + } + if state.inflight.contains(&key) { + // Still waiting for a worker: this caller keeps the job alive until its own deadline. + if let Some(job) = state.queue.iter_mut().find(|job| job.key == key) { + job.wanted_until = job.wanted_until.max(deadline); + } + } else { + if state.queue.len() >= QUEUE_MAX + && let Some(oldest) = state.queue.pop_front() + { + state.inflight.remove(&oldest.key); + state.stats.skipped += 1; + // Callers waiting for the dropped key see this before they could queue it again. + state.note_failed(oldest.key, "dropped: the fetch queue is full".into()); + shared.done.notify_all(); + } + state.inflight.insert(key); + state.queue.push_back(Job { key, time_s, epoch: min_epoch, queued: Instant::now(), wanted_until: deadline }); + shared.work.notify_one(); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + state.stats.timeouts += 1; + return Fetched::TimedOut; + } + state = shared.done.wait_timeout(state, left).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } + + /// True once every connection has ended. + pub fn is_down(&self) -> bool { + self.shared.alive.load(Ordering::SeqCst) == 0 + } + + pub fn stats(&self) -> FetchStats { + self.shared.lock().stats + } + + /// Ask the peer to close the ticket now instead of at drop. + pub fn close(&self) { + shared_stop(&self.shared); + let mut control = self.control.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _ = control.call("link.bye", serde_json::json!({})); + } +} + +impl Drop for FrameFetcher { + fn drop(&mut self) { + shared_stop(&self.shared); + } +} + +fn shared_stop(shared: &Shared) { + shared.lock().stop = true; + shared.work.notify_all(); + shared.done.notify_all(); +} + +fn run_worker(shared: &Shared, mut client: BulkClient) { + while let Some(job) = next_job(shared) { + let reply = client.request_reply(job.time_s, job.epoch, None); + let mut state = shared.lock(); + state.inflight.remove(&job.key); + match reply { + Ok(Ok(frame)) => { + state.stats.renders += 1; + state.failed.remove(&job.key); + let budget = shared.options.cache_bytes; + state.cache.insert(job.key, Arc::new(frame), budget); + } + Ok(Err(message)) => { + state.stats.failures += 1; + state.note_failed(job.key, message); + } + Err(_) => { + drop(state); + break; + } + } + drop(state); + shared.done.notify_all(); + } + if shared.alive.fetch_sub(1, Ordering::SeqCst) == 1 { + let mut state = shared.lock(); + state.queue.clear(); + state.inflight.clear(); + } + shared.done.notify_all(); +} + +/// The oldest queued job that is still worth rendering. `None` when the fetcher stops. +fn next_job(shared: &Shared) -> Option { + let mut state = shared.lock(); + loop { + if state.stop { + return None; + } + while let Some(job) = state.queue.pop_front() { + if job.queued.elapsed() > shared.options.max_age && job.wanted_until <= Instant::now() { + state.inflight.remove(&job.key); + state.stats.skipped += 1; + shared.done.notify_all(); + continue; + } + return Some(job); + } + state = shared.work.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } +} diff --git a/crates/ipc/src/fsopen.rs b/crates/ipc/src/fsopen.rs new file mode 100644 index 000000000..ec1316f68 --- /dev/null +++ b/crates/ipc/src/fsopen.rs @@ -0,0 +1,25 @@ +//! Read-only opens that no file type at the path can stall. +//! +//! `File::open` on a FIFO with no writer blocks until a writer appears. A path that was checked +//! and then replaced gives the same wait, so a type check before the open does not help. These +//! opens ask the kernel not to wait, and the caller checks the type on the open handle. + +use std::fs::File; +use std::path::Path; + +/// Open `path` for reading without waiting on a FIFO. On Unix `follow` false also refuses a +/// symlink in the last component (the error is not specific, so a caller that must tell a symlink +/// apart looks at `symlink_metadata` after a failed open). Other platforms open as `File::open`. +pub(crate) fn open_read(path: &Path, follow: bool) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + let flags = if follow { libc::O_NONBLOCK } else { libc::O_NONBLOCK | libc::O_NOFOLLOW }; + std::fs::OpenOptions::new().read(true).custom_flags(flags).open(path) + } + #[cfg(not(unix))] + { + let _ = follow; + File::open(path) + } +} diff --git a/crates/ipc/src/ids.rs b/crates/ipc/src/ids.rs new file mode 100644 index 000000000..c2ed1abd4 --- /dev/null +++ b/crates/ipc/src/ids.rs @@ -0,0 +1,41 @@ +//! Id checks shared by the instance record, the moniker and the link record. +//! `std` only, so the pure modules can use it. + +/// App id: `^[a-z][a-z0-9-]{1,31}$`. +pub fn valid_app_id(app: &str) -> bool { + let bytes = app.as_bytes(); + (2..=32).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_lowercase) + && bytes.iter().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') +} + +/// UUID text in 8-4-4-4-12 form. Either hex case. +pub fn valid_uuid(text: &str) -> bool { + let bytes = text.as_bytes(); + bytes.len() == 36 + && bytes.iter().enumerate().all(|(index, byte)| if matches!(index, 8 | 13 | 18 | 23) { *byte == b'-' } else { byte.is_ascii_hexdigit() }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn app_ids() { + for ok in ["ec", "filmcraft", "a-b-1", &"a".repeat(32)] { + assert!(valid_app_id(ok), "{ok}"); + } + for bad in ["", "a", "A1", "1a", "a_b", "Film", &"a".repeat(33)] { + assert!(!valid_app_id(bad), "{bad}"); + } + } + + #[test] + fn uuids() { + assert!(valid_uuid("550e8400-e29b-41d4-a716-446655440000")); + assert!(valid_uuid("550E8400-E29B-41D4-A716-446655440000")); + assert!(!valid_uuid("550e8400e29b41d4a716446655440000")); + assert!(!valid_uuid("550e8400-e29b-41d4-a716-44665544000g")); + assert!(!valid_uuid("")); + } +} diff --git a/crates/ipc/src/launch.rs b/crates/ipc/src/launch.rs new file mode 100644 index 000000000..73365d30f --- /dev/null +++ b/crates/ipc/src/launch.rs @@ -0,0 +1,777 @@ +//! Find and start a Craft app. +//! +//! `resolve` reads the process environment. The decision itself is +//! `resolve_with`, which takes an owned env map, a home directory, and the +//! candidate directories. Tests call that function and do not change the +//! process environment. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use crate::client::{Client, HelloClient}; +use crate::error::LinkError; +use crate::trust; +use crate::{Discovery, InstanceRecord, RuntimeDir}; + +const RECORD_WAIT: Duration = Duration::from_secs(15); +const RECORD_POLL: Duration = Duration::from_millis(50); + +/// Progress reported by [`launch_at`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + Spawned, + RecordSeen, + HelloOk, +} + +/// Find the binary for `app` from the process environment. +pub fn resolve(app: &str) -> Result { + let env = std::env::vars().collect::>(); + let home = std::env::var_os("HOME").map(PathBuf::from); + let exe_dir = std::env::current_exe().ok().and_then(|path| path.parent().map(Path::to_path_buf)); + let search = default_search_dirs(home.as_deref()); + resolve_with(app, &env, home.as_deref(), exe_dir.as_deref(), &search) +} + +/// [`resolve`] then [`launch_at`] with no args and the real runtime directory. +pub fn launch(app: &str, extra_env: &[(&str, &str)]) -> Result { + let bin = resolve(app)?; + launch_at(app, &bin, &[], extra_env, None, &mut |_| {}) +} + +/// Spawn `bin` and wait until its discovery record answers `link.hello`. +/// +/// `extra_env` is applied with [`Command::env`]. This function does not mint +/// an activation token. `XDG_ACTIVATION_TOKEN` and `DESKTOP_STARTUP_ID` are +/// passed through when the caller puts them in `extra_env`. +/// +/// `runtime` `None` scans [`RuntimeDir::resolve`]. The returned record is one +/// this function observed for the spawned pid. +pub fn launch_at( + app: &str, + bin: &Path, + args: &[OsString], + extra_env: &[(&str, &str)], + runtime: Option<&RuntimeDir>, + on_phase: &mut dyn FnMut(Phase), +) -> Result { + // `resolve` already checked `bin`. The caller may pass any path, and time has passed. + trust::check_executable(bin)?; + let mut cmd = Command::new(bin); + cmd.args(args); + for (key, value) in extra_env { + cmd.env(key, value); + } + // A new process group. The child is not in this process's group. + // The waiter thread reaps it. kill_on_drop stays off. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let mut child = cmd.spawn().map_err(LinkError::io)?; + let pid = child.id(); + std::thread::Builder::new() + .name("craft-ipc-wait".into()) + .spawn(move || { + let _ = child.wait(); + }) + .map_err(LinkError::io)?; + on_phase(Phase::Spawned); + if let Some(dir) = runtime { + finish_launch(app, pid, dir, on_phase) + } else { + let owned = RuntimeDir::resolve()?; + finish_launch(app, pid, &owned, on_phase) + } +} + +fn finish_launch(app: &str, pid: u32, dir: &RuntimeDir, on_phase: &mut dyn FnMut(Phase)) -> Result { + let record = wait_for_record(dir, app, pid)?; + on_phase(Phase::RecordSeen); + greet(&record, app)?; + on_phase(Phase::HelloOk); + Ok(record) +} + +fn wait_for_record(dir: &RuntimeDir, app: &str, pid: u32) -> Result { + let deadline = Instant::now() + RECORD_WAIT; + loop { + let records = Discovery::scan(dir)?; + if let Some(record) = records.into_iter().find(|record| record.app == app && record.pid == pid) { + return Ok(record); + } + let now = Instant::now(); + if now >= deadline { + return Err(LinkError::Timeout); + } + std::thread::sleep(RECORD_POLL.min(deadline.saturating_duration_since(now))); + } +} + +fn greet(record: &InstanceRecord, app: &str) -> Result<(), LinkError> { + let mut client = Client::connect(record).map_err(as_not_ready)?; + let identity = HelloClient { + app: app.to_string(), + app_version: "launch".into(), + pid: std::process::id(), + instance: crate::random_uuid().map_err(as_not_ready)?, + protocol_min: crate::wire::PROTOCOL_MIN, + protocol_max: crate::wire::PROTOCOL_MAX, + }; + client.hello(&identity).map_err(as_not_ready)?; + Ok(()) +} + +fn as_not_ready(err: LinkError) -> LinkError { + match err { + LinkError::Timeout => LinkError::Timeout, + LinkError::NotReady(_) => err, + other => LinkError::NotReady(other.to_string()), + } +} + +fn resolve_with( + app: &str, + env: &BTreeMap, + home: Option<&Path>, + exe_dir: Option<&Path>, + search_dirs: &[PathBuf], +) -> Result { + if let Some(found) = env_override(app, env) { + return found; + } + if let Some(manifest) = manifest_path(env, home, exe_dir) + && let Some(bin) = from_manifest(app, &manifest) + { + return Ok(bin); + } + if let Some(bin) = from_path_env(app, env) { + return Ok(bin); + } + for dir in search_dirs { + if let Some(bin) = search_dir(app, dir) { + return Ok(bin); + } + } + if let Some(bin) = platform_bundle(app, home) { + return Ok(bin); + } + Err(LinkError::NotInstalled { app: app.to_string() }) +} + +/// `CRAFT__BIN`. Set means authoritative, including a missing file. +fn env_override(app: &str, env: &BTreeMap) -> Option> { + let value = env.get(&bin_env_key(app))?; + let path = PathBuf::from(value); + if !path.is_file() { + return Some(Err(LinkError::NotInstalled { app: app.to_string() })); + } + Some(trust::check_executable(&path).map(|()| path)) +} + +fn bin_env_key(app: &str) -> String { + let mut key = String::from("CRAFT_"); + for byte in app.bytes() { + if byte.is_ascii_alphanumeric() { + key.push(byte.to_ascii_uppercase() as char); + } else { + key.push('_'); + } + } + key.push_str("_BIN"); + key +} + +fn manifest_path(env: &BTreeMap, home: Option<&Path>, exe_dir: Option<&Path>) -> Option { + if let Some(dir) = exe_dir + && dir.join("portable.txt").is_file() + { + return Some(dir.join("ArtCraftLauncherData").join("data").join("installed.json")); + } + Some(data_dir(env, home)?.join(launcher_dir_name()).join("installed.json")) +} + +fn launcher_dir_name() -> &'static str { + if cfg!(any(target_os = "macos", windows)) { "ArtCraft Launcher" } else { "artcraft-launcher" } +} + +fn data_dir(env: &BTreeMap, home: Option<&Path>) -> Option { + #[cfg(target_os = "linux")] + { + linux_data_dir(env, home) + } + #[cfg(target_os = "macos")] + { + let _ = env; + home.map(|dir| dir.join("Library").join("Application Support")) + } + #[cfg(windows)] + { + let _ = home; + env.get("LOCALAPPDATA").map(PathBuf::from).filter(|path| path.is_absolute()) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + let _ = (env, home); + None + } +} + +#[cfg(target_os = "linux")] +fn linux_data_dir(env: &BTreeMap, home: Option<&Path>) -> Option { + if let Some(xdg) = env.get("XDG_DATA_HOME") { + let path = PathBuf::from(xdg); + if path.is_absolute() { + return Some(path); + } + } + home.map(|dir| dir.join(".local").join("share")) +} + +/// Only `path` is required. An unknown `kind` is ignored. A path that is not a file is skipped. +fn from_manifest(app: &str, path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + let text = value.get("apps")?.get(app)?.get("path")?.as_str()?; + let bin = PathBuf::from(text); + trust::check_executable(&bin).is_ok().then_some(bin) +} + +/// A `PATH` entry that is empty or relative names the current directory, so it is skipped. +fn from_path_env(app: &str, env: &BTreeMap) -> Option { + let path = env.get("PATH")?; + for dir in std::env::split_paths(path) { + if !dir.is_absolute() { + continue; + } + let candidate = dir.join(app); + if trust::check_executable(&candidate).is_ok() { + return Some(candidate); + } + } + None +} + +fn default_search_dirs(home: Option<&Path>) -> Vec { + #[cfg(target_os = "linux")] + { + let mut dirs = Vec::new(); + if let Some(home) = home { + dirs.push(home.join("Applications")); + dirs.push(home.join("AppImages")); + dirs.push(home.join(".local").join("bin")); + } + dirs.push(PathBuf::from("/usr/bin")); + dirs.push(PathBuf::from("/usr/local/bin")); + dirs + } + #[cfg(not(target_os = "linux"))] + { + let _ = home; + Vec::new() + } +} + +fn search_dir(app: &str, dir: &Path) -> Option { + let exact = dir.join(app); + if trust::check_executable(&exact).is_ok() { + return Some(exact); + } + let entries = std::fs::read_dir(dir).ok()?; + let mut names = Vec::new(); + for entry in entries { + let Ok(entry) = entry else { continue }; + let Ok(name) = entry.file_name().into_string() else { continue }; + if is_appimage(app, &name) { + names.push(name); + } + } + names.sort(); + for name in names { + let path = dir.join(name); + if trust::check_executable(&path).is_ok() { + return Some(path); + } + } + None +} + +/// `{app}.appimage` in any ASCII case, or a name that starts with `{app}-` and ends with `.AppImage`. +fn is_appimage(app: &str, name: &str) -> bool { + const EXT: &str = ".appimage"; + let Some(split) = name.len().checked_sub(EXT.len()) else { + return false; + }; + let Some(ext) = name.get(split..) else { + return false; + }; + let Some(base) = name.get(..split) else { + return false; + }; + if !ext.eq_ignore_ascii_case(EXT) { + return false; + } + if base.eq_ignore_ascii_case(app) { + return true; + } + let mut prefix = String::with_capacity(app.len() + 1); + prefix.push_str(app); + prefix.push('-'); + name.starts_with(&prefix) +} + +/// An id that ends with `craft` becomes `PrefixCraft` (`filmcraft` to `FilmCraft`). +/// Used by the macOS bundle search. Linux resolves AppImages without this name. +#[cfg_attr(not(any(test, target_os = "macos")), allow(dead_code))] +fn display_name(app: &str) -> String { + if let Some(prefix) = app.strip_suffix("craft") { + let mut name = String::new(); + let mut chars = prefix.chars(); + if let Some(first) = chars.next() { + name.push(first.to_ascii_uppercase()); + name.push_str(chars.as_str()); + } + name.push_str("Craft"); + return name; + } + let mut chars = app.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + let mut name = String::new(); + name.push(first.to_ascii_uppercase()); + name.push_str(chars.as_str()); + name +} + +/// macOS bundles. Windows uninstall-registry lookup is not implemented (no winreg). +/// PATH, `CRAFT__BIN`, and `installed.json` still apply on Windows. +#[cfg(target_os = "macos")] +fn platform_bundle(app: &str, home: Option<&Path>) -> Option { + let bundle = format!("{}.app", display_name(app)); + let mut candidates = vec![PathBuf::from("/Applications").join(&bundle)]; + if let Some(home) = home { + candidates.push(home.join("Applications").join(&bundle)); + } + for bundle_path in candidates { + if !bundle_path.is_dir() { + continue; + } + let exe = bundle_path.join("Contents").join("MacOS").join(app); + if trust::check_executable(&exe).is_ok() { + return Some(exe); + } + } + None +} + +#[cfg(not(target_os = "macos"))] +fn platform_bundle(_app: &str, _home: Option<&Path>) -> Option { + None +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::collections::BTreeMap; + use std::path::{Path, PathBuf}; + + use super::*; + + struct Tmp { + path: PathBuf, + } + + impl Tmp { + fn new(label: &str) -> Self { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-{label}-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } + } + + fn touch(path: &Path) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, b"bin").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + + /// Mode of the directory that holds `path`, for the trust tests. + #[cfg(unix)] + fn chmod(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + #[test] + fn relative_and_empty_path_entries_are_skipped() { + let root = Tmp::new("relpath"); + let cwd = std::env::current_dir().unwrap(); + // A program in the current directory must not be found through "" or ".". + let here = cwd.join(format!("effectcraft-relpath-{}", std::process::id())); + touch(&here); + let name = here.file_name().unwrap().to_str().unwrap().to_string(); + let good_dir = root.path().join("bin"); + for path in ["", ".", "./", "bin:", ":bin"] { + let env = env_map(&[("PATH", path)]); + let err = super::resolve_with(&name, &env, None, None, &[]); + assert!(err.is_err(), "PATH={path:?} found {err:?}"); + } + // An absolute entry later in the list still works. + touch(&good_dir.join(&name)); + let path = format!(":.:{}", good_dir.display()); + let env = env_map(&[("PATH", path.as_str())]); + assert_eq!(super::resolve_with(&name, &env, None, None, &[]).unwrap(), good_dir.join(&name)); + std::fs::remove_file(&here).unwrap(); + } + + #[cfg(unix)] + #[test] + fn loose_programs_and_directories_are_not_run() { + let root = Tmp::new("loose"); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let env = env_map(&[("PATH", bin_dir.to_str().unwrap())]); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_ok()); + chmod(&bin_dir, 0o777); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "world-writable directory"); + // The same holds for the override, the manifest and the fixed search directories. + let by_env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", bin.to_str().unwrap())]); + assert!(matches!(super::resolve_with("effectcraft", &by_env, None, None, &[]), Err(LinkError::Refused { .. }))); + assert!(super::resolve_with("effectcraft", &env_map(&[]), None, None, std::slice::from_ref(&bin_dir)).is_err()); + let data = root.path().join("xdg"); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&data)), "effectcraft", &bin); + let by_manifest = data_env(&data); + chmod(&bin_dir, 0o755); + assert_eq!(super::resolve_with("effectcraft", &by_manifest, Some(&home), None, &[]).unwrap(), bin); + chmod(&bin_dir, 0o777); + assert!(super::resolve_with("effectcraft", &by_manifest, Some(&home), None, &[]).is_err()); + chmod(&bin_dir, 0o755); + chmod(&bin, 0o666); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "not executable and world-writable"); + chmod(&bin, 0o644); + assert!(super::resolve_with("effectcraft", &env, None, None, &[]).is_err(), "not executable"); + // A relative override is refused, not run from the current directory. + let relative = env_map(&[("CRAFT_EFFECTCRAFT_BIN", "Cargo.toml")]); + assert!(super::resolve_with("effectcraft", &relative, None, None, &[]).is_err()); + // launch_at checks the path too. + let err = launch_at("effectcraft", &bin, &[], &[], None, &mut |_| {}).unwrap_err(); + assert!(matches!(err, LinkError::Refused { .. }), "{err:?}"); + } + + fn env_map(pairs: &[(&str, &str)]) -> BTreeMap { + pairs.iter().map(|(key, value)| ((*key).to_string(), (*value).to_string())).collect() + } + + // Independent fixture paths for the launcher's documented native install record. + fn installed_manifest(home: &Path, xdg: Option<&Path>) -> PathBuf { + if cfg!(target_os = "macos") { + home.join("Library/Application Support/ArtCraft Launcher/installed.json") + } else if cfg!(windows) { + xdg.unwrap_or(home).join("ArtCraft Launcher/installed.json") + } else { + xdg.map_or_else(|| home.join(".local/share"), Path::to_path_buf).join("artcraft-launcher/installed.json") + } + } + + fn data_env(data: &Path) -> BTreeMap { + let key = if cfg!(windows) { "LOCALAPPDATA" } else { "XDG_DATA_HOME" }; + env_map(&[(key, data.to_str().unwrap())]) + } + + fn write_manifest(path: &Path, app: &str, bin: &Path) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + let body = serde_json::json!({ + "apps": { + app: {"kind": "mystery", "path": bin} + } + }); + std::fs::write(path, serde_json::to_vec(&body).unwrap()).unwrap(); + } + + #[test] + fn bin_env_key_uppercases_and_replaces_other_bytes() { + assert_eq!(super::bin_env_key("effectcraft"), "CRAFT_EFFECTCRAFT_BIN"); + assert_eq!(super::bin_env_key("film-craft"), "CRAFT_FILM_CRAFT_BIN"); + } + + #[test] + fn display_name_for_craft_apps() { + assert_eq!(super::display_name("filmcraft"), "FilmCraft"); + assert_eq!(super::display_name("effectcraft"), "EffectCraft"); + assert_eq!(super::display_name("pdfcraft"), "PdfCraft"); + } + + #[test] + fn appimage_names() { + assert!(super::is_appimage("effectcraft", "effectcraft.AppImage")); + assert!(super::is_appimage("effectcraft", "EffectCraft.appimage")); + assert!(super::is_appimage("effectcraft", "effectcraft-0.8.3-linux-x86_64.AppImage")); + assert!(!super::is_appimage("effectcraft", "effectcraft-0.8.3.deb")); + assert!(!super::is_appimage("effectcraft", "not-effectcraft.AppImage")); + assert!(!super::is_appimage("effectcraft", "effectcraft.AppImage.zsync")); + } + + #[test] + fn env_override_missing_file_does_not_fall_through() { + let root = Tmp::new("miss"); + let missing = root.path().join("nope"); + let real_dir = root.path().join("bin"); + let real = real_dir.join("effectcraft"); + touch(&real); + let env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", missing.to_str().unwrap()), ("PATH", real_dir.to_str().unwrap())]); + let err = super::resolve_with("effectcraft", &env, None, None, &[real_dir]).unwrap_err(); + match err { + LinkError::NotInstalled { app } => assert_eq!(app, "effectcraft"), + other => panic!("expected NotInstalled, got {other}"), + } + } + + #[test] + fn env_override_existing_file_is_that_path() { + let root = Tmp::new("envok"); + let bin = root.path().join("custom"); + touch(&bin); + let env = env_map(&[("CRAFT_EFFECTCRAFT_BIN", bin.to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn installed_json_supplies_the_path() { + let root = Tmp::new("man"); + let data = root.path().join("xdg"); + let bin = root.path().join("from-manifest"); + touch(&bin); + let home = root.path().join("home"); + let manifest = installed_manifest(&home, Some(&data)); + write_manifest(&manifest, "effectcraft", &bin); + let env = data_env(&data); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[cfg(target_os = "linux")] + #[test] + fn relative_xdg_data_home_uses_home() { + let root = Tmp::new("rel"); + let home = root.path().join("home"); + let bin = root.path().join("home-bin"); + touch(&bin); + let manifest = installed_manifest(&home, None); + write_manifest(&manifest, "effectcraft", &bin); + let env = env_map(&[("XDG_DATA_HOME", "relative/data")]); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[cfg(windows)] + #[test] + fn relative_localappdata_is_not_used() { + let env = env_map(&[("LOCALAPPDATA", "relative/data")]); + assert!(matches!(super::resolve_with("effectcraft", &env, None, None, &[]), Err(LinkError::NotInstalled { .. }))); + } + + #[test] + fn portable_manifest_wins_over_native_data_directory() { + let root = Tmp::new("port"); + let exe = root.path().join("exe-dir"); + std::fs::create_dir_all(&exe).unwrap(); + std::fs::write(exe.join("portable.txt"), b"1").unwrap(); + let bin = root.path().join("portable-bin"); + touch(&bin); + write_manifest(&exe.join("ArtCraftLauncherData").join("data").join("installed.json"), "effectcraft", &bin); + let xdg = root.path().join("xdg"); + let decoy = root.path().join("decoy-bin"); + touch(&decoy); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&xdg)), "effectcraft", &decoy); + let env = data_env(&xdg); + let got = super::resolve_with("effectcraft", &env, Some(&home), Some(&exe), &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn manifest_directory_is_skipped_for_path() { + let root = Tmp::new("skip"); + let data = root.path().join("xdg"); + let folder = root.path().join("not-a-file"); + std::fs::create_dir_all(&folder).unwrap(); + let home = root.path().join("home"); + write_manifest(&installed_manifest(&home, Some(&data)), "effectcraft", &folder); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let mut env = data_env(&data); + env.insert("PATH".into(), bin_dir.to_str().unwrap().into()); + let got = super::resolve_with("effectcraft", &env, Some(&home), None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn path_supplies_the_path() { + let root = Tmp::new("path"); + let bin_dir = root.path().join("bin"); + let bin = bin_dir.join("effectcraft"); + touch(&bin); + let env = env_map(&[("PATH", bin_dir.to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[]).unwrap(); + assert_eq!(got, bin); + } + + #[test] + fn linux_location_supplies_the_path() { + let root = Tmp::new("fs"); + let apps = root.path().join("Applications"); + let image = apps.join("effectcraft-1.0.0.AppImage"); + touch(&image); + std::fs::write(apps.join("not-effectcraft.AppImage"), b"no").unwrap(); + let later = root.path().join("usr-bin"); + let exact = later.join("effectcraft"); + touch(&exact); + let env = env_map(&[("PATH", root.path().join("empty").to_str().unwrap())]); + let got = super::resolve_with("effectcraft", &env, None, None, &[apps.clone(), later]).unwrap(); + assert_eq!(got, image); + } + + #[cfg(unix)] + #[test] + fn launch_at_reaches_hello() { + // The child branch returns at once when CRAFT_LAUNCH_DIR is unset. + // A normal cargo test takes the parent path and does not park. + if let Ok(dir) = std::env::var("CRAFT_LAUNCH_DIR") { + child_launch(&dir); + } + parent_launch(); + } + + #[cfg(unix)] + fn child_launch(dir: &str) -> ! { + use std::io::BufReader; + use std::os::unix::net::UnixListener; + + let runtime = RuntimeDir::at(dir).unwrap(); + let app = "effectcraft"; + let pid = std::process::id(); + std::fs::write(runtime.path().join("child.pid"), pid.to_string()).unwrap(); + let sock = runtime.socket_file(app, pid); + let _ = std::fs::remove_file(&sock); + let listener = UnixListener::bind(&sock).unwrap(); + crate::discovery::enforce_mode(&sock, 0o600).unwrap(); + let endpoint = runtime.unix_endpoint(app, pid).unwrap(); + let record = InstanceRecord::new(app, "0.8.3", pid, endpoint).unwrap(); + let published = Discovery::publish(&runtime, record.clone()).unwrap(); + std::fs::write(runtime.path().join("ready"), pid.to_string()).unwrap(); + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(20))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(20))).unwrap(); + let mut reader = BufReader::new(stream); + let auth = crate::wire::read_line(&mut reader, crate::wire::MAX_LINE).unwrap().unwrap(); + let auth_value: Value = serde_json::from_slice(&auth).unwrap(); + assert_eq!(auth_value["method"], "auth"); + assert_eq!(auth_value["params"]["token"], record.token); + crate::wire::write_line(reader.get_mut(), &crate::wire::auth_success(&serde_json::json!(0))).unwrap(); + let hello = crate::wire::read_line(&mut reader, crate::wire::MAX_LINE).unwrap().unwrap(); + let hello_value: Value = serde_json::from_slice(&hello).unwrap(); + assert_eq!(hello_value["method"], "link.hello"); + let id = hello_value.get("id").cloned().unwrap_or(serde_json::json!(1)); + let result = serde_json::json!({ + "app": app, + "app_version": record.app_version, + "pid": record.pid, + "instance": record.instance, + "protocol": 1, + }); + crate::wire::write_line(reader.get_mut(), &crate::wire::reply_ok(&id, result)).unwrap(); + let _keep = (published, listener, reader); + // park can wake without an unpark. Loop so Drop does not run until SIGKILL. + loop { + std::thread::park(); + } + } + + #[cfg(unix)] + fn parent_launch() { + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-launch-{}", crate::auth::encode_hex(&bytes))); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let _cleanup = LaunchCleanup { dir: path.clone() }; + let exe = std::env::current_exe().unwrap(); + let args = vec![OsString::from("--exact"), OsString::from("launch::tests::launch_at_reaches_hello"), OsString::from("--test-threads=1")]; + let dir_text = path.to_str().unwrap(); + let mut phases = Vec::new(); + // The test binary can sit on a file system that marks every file 0777, and `launch_at` + // refuses such a program. A private launcher script that execs it keeps the pid. + let script = path.join("launcher"); + std::fs::write(&script, format!("#!/bin/sh\nexec '{}' \"$@\"\n", exe.display())).unwrap(); + chmod(&script, 0o755); + let result = launch_at("effectcraft", &script, &args, &[("CRAFT_LAUNCH_DIR", dir_text)], Some(&runtime), &mut |phase| phases.push(phase)); + let record = result.unwrap_or_else(|err| panic!("launch_at failed: {err}; phases={phases:?}")); + assert_eq!(phases, vec![Phase::Spawned, Phase::RecordSeen, Phase::HelloOk]); + let ready = std::fs::read_to_string(path.join("ready")).unwrap(); + let child_pid: u32 = ready.trim().parse().unwrap(); + assert_eq!(record.pid, child_pid); + assert_eq!(record.app, "effectcraft"); + } + + #[cfg(unix)] + struct LaunchCleanup { + dir: PathBuf, + } + + #[cfg(unix)] + impl Drop for LaunchCleanup { + fn drop(&mut self) { + kill_recorded(&self.dir); + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + #[cfg(unix)] + fn kill_recorded(dir: &Path) { + for name in ["child.pid", "ready"] { + let Ok(text) = std::fs::read_to_string(dir.join(name)) else { continue }; + let Ok(pid) = text.trim().parse::() else { continue }; + let _ = Command::new("kill").args(["-KILL", &pid.to_string()]).status(); + let proc = PathBuf::from(format!("/proc/{pid}")); + let deadline = Instant::now() + Duration::from_secs(2); + while proc.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + return; + } + } +} diff --git a/crates/ipc/src/lib.rs b/crates/ipc/src/lib.rs new file mode 100644 index 000000000..53b071fa6 --- /dev/null +++ b/crates/ipc/src/lib.rs @@ -0,0 +1,39 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +//! Craft Link transport. +//! +//! Local sockets, discovery files, and the JSON-lines session. The server and +//! the client are separate modules. There is no tokio runtime in this crate. + +pub mod activate; +pub mod asset; +pub mod auth; +pub mod bulk; +pub mod cache; +pub mod client; +pub mod discovery; +pub mod error; +pub mod fetch; +mod fsopen; +pub mod ids; +pub mod launch; +pub mod moniker; +pub mod payload; +pub mod record; +pub mod refresh; +pub mod server; +pub mod trust; +pub mod watch; +pub mod wire; + +/// Unit tests that hold a discovery lock take `read`. Unit tests that start a +/// child process take `write`. A forked child owns a copy of every open file +/// until it calls exec, and a `flock` lock lives until the last copy closes. +/// Without this guard a lock released by `drop` can look held for a few +/// milliseconds, and `scan` or `publish` in another test then fails at random. +#[cfg(test)] +pub(crate) static SPAWN_GUARD: std::sync::RwLock<()> = std::sync::RwLock::new(()); + +pub use auth::{ensure_peer_user, generate_token, random_uuid, tokens_equal}; +pub use discovery::{Discovery, DocInfo, Endpoint, InstanceRecord, Protocol, RuntimeDir}; +pub use error::{ErrorObject, LinkError}; diff --git a/crates/ipc/src/moniker.rs b/crates/ipc/src/moniker.rs new file mode 100644 index 000000000..5bee3e77c --- /dev/null +++ b/crates/ipc/src/moniker.rs @@ -0,0 +1,313 @@ +//! Asset monikers: `craft://///`. +//! +//! The grammar is in `PROTOCOL.md` section 9. This module uses +//! only `std` and `serde`, so an app can copy it to a wasm build. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::error::LinkError; +use crate::ids::{valid_app_id, valid_uuid}; + +const SCHEME: &str = "craft://"; +/// Longest decoded `id`, in bytes. +pub const MAX_ID_BYTES: usize = 512; + +/// What an asset is. Unknown words are valid and kept in [`MonikerKind::Other`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum MonikerKind { + Comp, + Image, + Vector, + Sequence, + Layer, + Other(String), +} + +impl MonikerKind { + pub fn as_str(&self) -> &str { + match self { + Self::Comp => "comp", + Self::Image => "image", + Self::Vector => "vector", + Self::Sequence => "sequence", + Self::Layer => "layer", + Self::Other(word) => word, + } + } + + fn parse(word: &str) -> Result { + if !valid_word(word, 32) { + return Err(bad(format!("kind {word:?} must match [a-z][a-z0-9-]{{0,31}}"))); + } + Ok(match word { + "comp" => Self::Comp, + "image" => Self::Image, + "vector" => Self::Vector, + "sequence" => Self::Sequence, + "layer" => Self::Layer, + other => Self::Other(other.to_string()), + }) + } +} + +/// A parsed moniker. `doc` keeps the hex case it was written with. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Moniker { + app: String, + doc: String, + kind: MonikerKind, + id: String, +} + +impl Moniker { + /// Build a moniker from parts. Every part is checked, so a value of this + /// type always formats to text that parses back to an equal value. + pub fn new(app: &str, doc: &str, kind: MonikerKind, id: &str) -> Result { + if !valid_app_id(app) { + return Err(bad(format!("app {app:?} must match [a-z][a-z0-9-]{{1,31}}"))); + } + if !valid_uuid(doc) { + return Err(bad(format!("doc {doc:?} is not a uuid"))); + } + if let MonikerKind::Other(word) = &kind + && (!valid_word(word, 32) || is_known_word(word)) + { + return Err(bad(format!("kind {word:?} is not a valid unknown kind"))); + } + if id.is_empty() || id.len() > MAX_ID_BYTES { + return Err(bad(format!("id must be 1 to {MAX_ID_BYTES} bytes"))); + } + Ok(Self { app: app.to_string(), doc: doc.to_string(), kind, id: id.to_string() }) + } + + pub fn parse(text: &str) -> Result { + let Some(rest) = text.strip_prefix(SCHEME) else { + return Err(bad(format!("moniker must start with {SCHEME}"))); + }; + let mut parts = rest.split('/'); + let (Some(app), Some(doc), Some(kind), Some(id), None) = (parts.next(), parts.next(), parts.next(), parts.next(), parts.next()) else { + return Err(bad("moniker must have exactly app, doc, kind and id".to_string())); + }; + if id.contains(['?', '#']) { + return Err(bad("moniker has no query or fragment".to_string())); + } + let kind = MonikerKind::parse(kind)?; + let id = decode_id(id)?; + Self::new(app, doc, kind, &id) + } + + pub fn app(&self) -> &str { + &self.app + } + + pub fn doc(&self) -> &str { + &self.doc + } + + pub fn kind(&self) -> &MonikerKind { + &self.kind + } + + /// The decoded id. + pub fn id(&self) -> &str { + &self.id + } +} + +impl fmt::Display for Moniker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{SCHEME}{}/{}/{}/", self.app, self.doc, self.kind.as_str())?; + for byte in self.id.bytes() { + if is_unreserved(byte) { + write!(f, "{}", char::from(byte))?; + } else { + write!(f, "%{byte:02X}")?; + } + } + Ok(()) + } +} + +impl FromStr for Moniker { + type Err = LinkError; + + fn from_str(text: &str) -> Result { + Self::parse(text) + } +} + +impl Serialize for Moniker { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Moniker { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + Self::parse(&text).map_err(serde::de::Error::custom) + } +} + +fn bad(message: String) -> LinkError { + LinkError::BadParams(message) +} + +fn is_known_word(word: &str) -> bool { + matches!(word, "comp" | "image" | "vector" | "sequence" | "layer") +} + +fn is_unreserved(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') +} + +fn valid_word(word: &str, max: usize) -> bool { + let mut bytes = word.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + word.len() <= max && first.is_ascii_lowercase() && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +fn decode_id(text: &str) -> Result { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + let pair = bytes.get(index + 1..index + 3).ok_or_else(|| bad("id has a truncated %XX escape".to_string()))?; + let hi = hex_value(pair.first().copied()); + let lo = hex_value(pair.get(1).copied()); + let (Some(hi), Some(lo)) = (hi, lo) else { + return Err(bad("id has a bad %XX escape".to_string())); + }; + out.push(hi * 16 + lo); + index += 3; + } else if is_unreserved(byte) { + out.push(byte); + index += 1; + } else { + return Err(bad(format!("id has the raw byte 0x{byte:02X}, which must be percent-encoded"))); + } + } + String::from_utf8(out).map_err(|_| bad("id is not valid UTF-8".to_string())) +} + +fn hex_value(byte: Option) -> Option { + match byte? { + b @ b'0'..=b'9' => Some(b - b'0'), + b @ b'a'..=b'f' => Some(b - b'a' + 10), + b @ b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + const DOC: &str = "550e8400-e29b-41d4-a716-446655440000"; + + fn sample(id: &str) -> Moniker { + Moniker::new("photocraft", DOC, MonikerKind::Image, id).unwrap() + } + + #[test] + fn plain_id_formats_and_parses() { + let text = format!("craft://photocraft/{DOC}/image/layer-3"); + assert_eq!(sample("layer-3").to_string(), text); + assert_eq!(Moniker::parse(&text).unwrap(), sample("layer-3")); + } + + #[test] + fn round_trip_keeps_awkward_ids() { + for id in ["a b", "a/b", "100%", "x?y#z", "caf\u{e9}", "\u{1F600}", "~._-", "%2F", "a%20b", &"x".repeat(MAX_ID_BYTES)] { + let m = sample(id); + let text = m.to_string(); + let back = Moniker::parse(&text).unwrap(); + assert_eq!(back, m, "{id:?} via {text}"); + assert_eq!(back.id(), id); + assert_eq!(back.to_string(), text); + } + } + + #[test] + fn encoded_slash_stays_inside_the_id() { + let m = sample("a/b"); + assert_eq!(m.to_string(), format!("craft://photocraft/{DOC}/image/a%2Fb")); + assert_eq!(Moniker::parse(&m.to_string()).unwrap().id(), "a/b"); + } + + #[test] + fn lowercase_escapes_parse_and_format_uppercase() { + let m = Moniker::parse(&format!("craft://photocraft/{DOC}/image/a%2fb")).unwrap(); + assert_eq!(m.id(), "a/b"); + assert_eq!(m.to_string(), format!("craft://photocraft/{DOC}/image/a%2Fb")); + } + + #[test] + fn doc_case_is_kept() { + let upper = DOC.to_uppercase(); + let m = Moniker::parse(&format!("craft://photocraft/{upper}/comp/c1")).unwrap(); + assert_eq!(m.doc(), upper); + assert_eq!(m.to_string(), format!("craft://photocraft/{upper}/comp/c1")); + } + + #[test] + fn kinds_known_and_unknown() { + for (word, kind) in [ + ("comp", MonikerKind::Comp), + ("image", MonikerKind::Image), + ("vector", MonikerKind::Vector), + ("sequence", MonikerKind::Sequence), + ("layer", MonikerKind::Layer), + ("palette", MonikerKind::Other("palette".into())), + ] { + let m = Moniker::parse(&format!("craft://designcraft/{DOC}/{word}/x")).unwrap(); + assert_eq!(m.kind(), &kind); + assert_eq!(m.to_string(), format!("craft://designcraft/{DOC}/{word}/x")); + } + assert!(Moniker::new("designcraft", DOC, MonikerKind::Other("comp".into()), "x").is_err(), "known word must use its variant"); + } + + #[test] + fn rejects_bad_text() { + let bad_texts = [ + "".to_string(), + "craft:/photocraft".to_string(), + "http://photocraft/x/image/a".to_string(), + format!("craft://photocraft/{DOC}/image"), + format!("craft://photocraft/{DOC}/image/a/b"), + format!("craft://photocraft/{DOC}/image/"), + format!("craft://photocraft/{DOC}/image/a?x=1"), + format!("craft://photocraft/{DOC}/image/a#frag"), + format!("craft://photocraft/{DOC}/image/a b"), + format!("craft://photocraft/{DOC}/image/%"), + format!("craft://photocraft/{DOC}/image/%4"), + format!("craft://photocraft/{DOC}/image/%GG"), + format!("craft://photocraft/{DOC}/image/%FF"), + format!("craft://Photo/{DOC}/image/a"), + format!("craft://p/{DOC}/image/a"), + "craft://photocraft/not-a-uuid/image/a".to_string(), + format!("craft://photocraft/{DOC}/Image/a"), + format!("craft://photocraft/{DOC}/1image/a"), + format!("craft://photocraft/{DOC}/image/{}", "x".repeat(MAX_ID_BYTES + 1)), + ]; + for text in bad_texts { + assert!(Moniker::parse(&text).is_err(), "{text:?} should not parse"); + } + } + + #[test] + fn serde_uses_the_string_form() { + let m = sample("a b"); + let value = serde_json::to_value(&m).unwrap(); + assert_eq!(value, serde_json::json!(format!("craft://photocraft/{DOC}/image/a%20b"))); + assert_eq!(serde_json::from_value::(value).unwrap(), m); + assert!(serde_json::from_value::(serde_json::json!("craft://nope")).is_err()); + } +} diff --git a/crates/ipc/src/payload.rs b/crates/ipc/src/payload.rs new file mode 100644 index 000000000..ddbd7d2dd --- /dev/null +++ b/crates/ipc/src/payload.rs @@ -0,0 +1,396 @@ +//! The `payload` object of `link.open`. +//! +//! Formats are listed in `PROTOCOL.md` section 4. The server +//! calls [`OpenPayload::from_value`] on every `link.open` before it forwards the +//! request, and answers `payload_invalid` on an error. An app may call it again. +//! +//! This module uses only `std`, `serde_json`, [`crate::moniker`] and the size constants in +//! [`crate::wire`]. + +use serde_json::Value; + +use crate::error::LinkError; +use crate::moniker::Moniker; +use crate::wire::MAX_LINK_PAYLOAD; + +/// A parsed `link.open` payload. +#[derive(Debug, Clone, PartialEq)] +pub enum OpenPayload { + /// Video: an OTIO JSON string. + OtioJson { data: String }, + /// A still or vector: the producer document path and the asset moniker. + ImageDoc { path: String, asset: Moniker }, +} + +impl OpenPayload { + pub fn from_value(payload: &Value) -> Result { + let obj = payload.as_object().ok_or_else(|| LinkError::PayloadInvalid("payload must be an object".into()))?; + let format = obj.get("format").and_then(Value::as_str).ok_or_else(|| LinkError::PayloadInvalid("payload.format must be a string".into()))?; + match format { + "otio-json" => { + let data = text_field(obj, "data")?; + if data.len() as u64 > MAX_LINK_PAYLOAD { + return Err(LinkError::TooLarge); + } + let missing = missing_media(&data)?; + if !missing.is_empty() { + return Err(LinkError::MediaMissing { paths: missing }); + } + Ok(Self::OtioJson { data }) + } + "image-doc" => { + let path = text_field(obj, "path")?; + check_peer_path(&path, "payload.path")?; + let asset = Moniker::parse(&text_field(obj, "asset")?).map_err(|err| LinkError::PayloadInvalid(format!("payload.asset: {err}")))?; + Ok(Self::ImageDoc { path, asset }) + } + other => Err(LinkError::PayloadInvalid(format!("payload.format {other:?} is not supported"))), + } + } +} + +/// The most distinct files one OTIO document may name. A timeline with more is not an editor's +/// timeline, and each one costs a file system call. +pub const MAX_MEDIA_REFS: usize = 4096; +/// The most JSON values one OTIO document may hold. The line cap (4 MiB) allows about two million. +pub const MAX_NODES: usize = 1 << 20; +/// Deepest nesting walked. `serde_json` stops parsing at 128, so a deeper document never gets here. +pub const MAX_DEPTH: usize = 128; + +/// Check a path that a peer names (`payload.path`, a media `target_url`). It must be absolute, hold +/// no NUL byte and no `..` component. The server does not canonicalise it: the app opens it, and a +/// symlink to a file elsewhere is a normal way to keep footage. This check only refuses a path +/// that is written to climb out of the directory it starts in. +pub fn check_peer_path(path: &str, what: &str) -> Result<(), LinkError> { + let parsed = std::path::Path::new(path); + if !parsed.is_absolute() { + return Err(LinkError::PayloadInvalid(format!("{what} must be absolute"))); + } + if path.contains('\0') { + return Err(LinkError::PayloadInvalid(format!("{what} must not hold a NUL byte"))); + } + if parsed.components().any(|part| matches!(part, std::path::Component::ParentDir)) { + return Err(LinkError::PayloadInvalid(format!("{what} must not hold a `..` component"))); + } + Ok(()) +} + +/// The files that `ExternalReference`s of an OTIO document name and that cannot be used. Each path +/// appears once, in document order. +/// +/// A file is usable when it is a regular file (after symlinks) and opens for reading. A missing +/// file, a directory, a FIFO, a device and a file without read access all give the same answer, so +/// the reply says nothing beyond "this path from your payload does not work". The probe never opens +/// anything that is not a regular file, so a FIFO cannot hold the connection thread. Only paths the +/// payload itself names are probed, and at most [`MAX_MEDIA_REFS`] of them. +/// +/// Empty when every file works, when `data` is not JSON (the app reports that), and for references +/// that are not an absolute path or a `file://` URL: they have no base here to resolve against. +/// `Err` when the document is too deep or too big, or names an absolute path with `..` or NUL. +pub fn missing_media(data: &str) -> Result, LinkError> { + let Ok(document) = serde_json::from_str::(data) else { return Ok(Vec::new()) }; + let mut walk = Walk { seen: Vec::new(), nodes: 0 }; + walk.collect(&document, 0)?; + walk.seen.retain(|path| !media_usable(path)); + Ok(walk.seen) +} + +fn media_usable(path: &str) -> bool { + let path = std::path::Path::new(path); + std::fs::metadata(path).is_ok_and(|meta| meta.is_file()) && std::fs::File::open(path).is_ok() +} + +struct Walk { + seen: Vec, + nodes: usize, +} + +impl Walk { + fn collect(&mut self, value: &Value, depth: usize) -> Result<(), LinkError> { + if depth > MAX_DEPTH { + return Err(LinkError::PayloadInvalid(format!("the OTIO document nests deeper than {MAX_DEPTH}"))); + } + self.nodes += 1; + if self.nodes > MAX_NODES { + return Err(LinkError::PayloadInvalid(format!("the OTIO document holds more than {MAX_NODES} values"))); + } + match value { + Value::Object(map) => { + let external = map.get("OTIO_SCHEMA").and_then(Value::as_str).is_some_and(|schema| schema.starts_with("ExternalReference")); + let local = match map.get("target_url").and_then(Value::as_str) { + Some(url) if external => local_path(url)?, + _ => None, + }; + if let Some(path) = local { + check_peer_path(&path, "target_url")?; + if !self.seen.contains(&path) { + if self.seen.len() >= MAX_MEDIA_REFS { + return Err(LinkError::PayloadInvalid(format!("the OTIO document names more than {MAX_MEDIA_REFS} media files"))); + } + self.seen.push(path); + } + } + for child in map.values() { + self.collect(child, depth + 1)?; + } + } + Value::Array(items) => { + for child in items { + self.collect(child, depth + 1)?; + } + } + _ => {} + } + Ok(()) + } +} + +/// The local path a `target_url` names: a `file://` URL (percent-decoded, host empty or +/// `localhost`) or an absolute path. `Ok(None)` for anything else. `Err` for a `file://` URL +/// whose path does not percent-decode to UTF-8: it names a local file, so it cannot be skipped. +fn local_path(url: &str) -> Result, LinkError> { + let path = match url.strip_prefix("file://") { + Some(rest) => { + let rest = rest.strip_prefix("localhost").unwrap_or(rest); + if !rest.starts_with('/') { + return Ok(None); + } + percent_decode(rest).ok_or_else(|| LinkError::PayloadInvalid("target_url is a file:// URL with a bad percent escape".into()))? + } + None => url.to_string(), + }; + // `/C:/dir/a.mov` is how a Windows path sits in a URL. + let path = match path.as_bytes() { + [b'/', drive, b':', ..] if drive.is_ascii_alphabetic() && cfg!(windows) => path[1..].to_string(), + _ => path, + }; + Ok(std::path::Path::new(&path).is_absolute().then_some(path)) +} + +fn percent_decode(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while let Some(&byte) = bytes.get(index) { + if byte == b'%' { + let pair = bytes.get(index + 1..index + 3)?; + // `from_str_radix` also takes a leading `+`, so the two digits are checked first. + if !pair.iter().all(u8::is_ascii_hexdigit) { + return None; + } + out.push(u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok()?); + index += 3; + } else { + out.push(byte); + index += 1; + } + } + String::from_utf8(out).ok() +} + +fn text_field(obj: &serde_json::Map, key: &str) -> Result { + match obj.get(key) { + Some(Value::String(text)) if !text.is_empty() => Ok(text.clone()), + _ => Err(LinkError::PayloadInvalid(format!("payload.{key} must be a non-empty string"))), + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use serde_json::json; + + use super::*; + + const ASSET: &str = "craft://photocraft/550e8400-e29b-41d4-a716-446655440000/image/flat"; + + fn absolute(path: &str) -> String { + if cfg!(windows) { format!("C:{path}") } else { path.to_owned() } + } + + fn file_url(path: &str) -> String { + let path = path.replace('\\', "/").replace(' ', "%20"); + format!("file://{}{path}", if path.starts_with('/') { "" } else { "/" }) + } + + #[test] + fn parses_otio() { + let got = OpenPayload::from_value(&json!({"format": "otio-json", "data": "{}"})).unwrap(); + assert_eq!(got, OpenPayload::OtioJson { data: "{}".into() }); + } + + fn otio_with(url: &str) -> String { + json!({ + "OTIO_SCHEMA": "Timeline.1", + "tracks": {"OTIO_SCHEMA": "Stack.1", "children": [{"OTIO_SCHEMA": "Track.1", "children": [{ + "OTIO_SCHEMA": "Clip.2", + "media_references": {"DEFAULT_MEDIA": {"OTIO_SCHEMA": "ExternalReference.1", "target_url": url}}, + "active_media_reference_key": "DEFAULT_MEDIA" + }]}]} + }) + .to_string() + } + + #[test] + fn rejects_media_that_does_not_exist_with_the_paths() { + let missing = absolute("/nowhere/My Clip.mov"); + let payload = json!({"format": "otio-json", "data": otio_with(&file_url(&missing))}); + let err = OpenPayload::from_value(&payload).unwrap_err(); + assert_eq!(err, LinkError::MediaMissing { paths: vec![missing.clone()] }); + assert_eq!(err.code(), "media_missing"); + assert_eq!(err.to_object().data.get("missing"), Some(&json!([missing]))); + assert!(err.to_string().contains(&missing), "{err}"); + // A bare absolute path counts the same. + let bare = json!({"format": "otio-json", "data": otio_with(&absolute("/nowhere/clip.mov"))}); + assert!(matches!(OpenPayload::from_value(&bare), Err(LinkError::MediaMissing { .. }))); + } + + #[test] + fn accepts_media_that_opens_and_references_it_cannot_resolve() { + let dir = std::env::temp_dir().join(format!("craft-payload-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("My Clip.mov"); + std::fs::write(&file, b"x").unwrap(); + let url = file_url(file.to_str().unwrap()); + assert_eq!(std::path::PathBuf::from(local_path(&url).unwrap().unwrap()), file); + let good = json!({"format": "otio-json", "data": otio_with(&url)}); + assert!(OpenPayload::from_value(&good).is_ok()); + // A relative target has no base here. The app resolves it. + let relative = json!({"format": "otio-json", "data": otio_with("clips/a.mov")}); + assert!(OpenPayload::from_value(&relative).is_ok()); + // A generator or a missing reference names no file. + let none = json!({"format": "otio-json", "data": "{\"OTIO_SCHEMA\":\"GeneratorReference.1\"}"}); + assert!(OpenPayload::from_value(&none).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn parses_image_doc() { + let expected = absolute("/tmp/a.photocraft"); + let got = OpenPayload::from_value(&json!({"format": "image-doc", "path": expected, "asset": ASSET})).unwrap(); + let OpenPayload::ImageDoc { path, asset } = got else { + panic!("wrong variant"); + }; + assert_eq!(path, expected); + assert_eq!(asset.to_string(), ASSET); + } + + fn payload_with(url: &str) -> Value { + json!({"format": "otio-json", "data": otio_with(url)}) + } + + #[test] + fn traversal_in_a_named_path_is_payload_invalid() { + for bad in [ + absolute("/tmp/../etc/passwd"), + file_url(&absolute("/tmp/a/../../etc/passwd")), + file_url(&absolute("/tmp/%2e%2e/etc/passwd")), + absolute("/tmp/a\u{0}b"), + file_url(&absolute("/tmp/a%00b")), + ] { + let err = OpenPayload::from_value(&payload_with(&bad)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad:?}: {err:?}"); + } + let err = OpenPayload::from_value(&json!({"format": "image-doc", "path": absolute("/tmp/../etc/x.png"), "asset": ASSET})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // A relative target is the app's business, `..` or not. + assert!(OpenPayload::from_value(&payload_with("clips/../a.mov")).is_ok()); + } + + #[test] + fn a_file_url_that_does_not_decode_is_payload_invalid() { + for bad in [ + "file:///tmp/clip%GG.mov", + "file:///tmp/clip%4", + "file:///tmp/clip%", + "file://localhost/tmp/clip%zz.mov", + "file:///tmp/clip%FF.mov", + // A sign or a space is not a hex digit, although `from_str_radix` takes `+`. + "file:///tmp/a%+fb", + "file:///tmp/a%-fb", + "file:///tmp/a% fb", + "file:///tmp/a%f", + ] { + let err = OpenPayload::from_value(&payload_with(bad)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad:?}: {err:?}"); + assert!(matches!(missing_media(&otio_with(bad)), Err(LinkError::PayloadInvalid(_))), "{bad:?}"); + } + // A reference that names no local file stays the app's business. + for skipped in ["file://remote-host/tmp/clip%GG.mov", "http://example.com/clip%GG.mov", "clips/clip%GG.mov"] { + assert!(OpenPayload::from_value(&payload_with(skipped)).is_ok(), "{skipped:?}"); + } + } + + #[cfg(unix)] + #[test] + fn only_regular_files_count_as_present_and_a_fifo_does_not_block() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("craft-payload-fifo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("pipe.mov"); + let made = std::process::Command::new("mkfifo").arg(&fifo).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + let real = dir.join("real.mov"); + std::fs::write(&real, b"x").unwrap(); + let link = dir.join("link.mov"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let start = std::time::Instant::now(); + let data = json!({"OTIO_SCHEMA": "Timeline.1", "refs": [ + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": fifo.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": dir.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": link.to_str().unwrap()}, + {"OTIO_SCHEMA": "ExternalReference.1", "target_url": real.to_str().unwrap()}, + ]}) + .to_string(); + let missing = missing_media(&data).unwrap(); + assert!(start.elapsed() < std::time::Duration::from_secs(2), "the probe waited on a FIFO"); + assert_eq!(missing, vec![fifo.to_str().unwrap().to_string(), dir.to_str().unwrap().to_string()]); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn size_depth_and_count_are_bounded() { + // Too many distinct media files. + let refs: Vec = (0..=MAX_MEDIA_REFS) + .map(|n| json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": absolute(&format!("/nowhere/{n}.mov"))})) + .collect(); + let many = json!({"OTIO_SCHEMA": "Timeline.1", "refs": refs}).to_string(); + let err = OpenPayload::from_value(&json!({"format": "otio-json", "data": many})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // The same file named many times counts once. + let same: Vec = + (0..MAX_MEDIA_REFS * 2).map(|_| json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": absolute("/nowhere/a.mov")})).collect(); + let doc = json!({"OTIO_SCHEMA": "Timeline.1", "refs": same}).to_string(); + assert_eq!(missing_media(&doc).unwrap(), vec![absolute("/nowhere/a.mov")]); + // Too deep. serde_json refuses at 128, so the document reads as "not JSON" and the app reports it. + let deep = format!("{}{}", "[".repeat(300), "]".repeat(300)); + assert_eq!(missing_media(&deep).unwrap(), Vec::::new()); + // Just inside serde_json's limit the walker still finishes. + let nested = format!("{}{}", "[".repeat(MAX_DEPTH - 1), "]".repeat(MAX_DEPTH - 1)); + assert_eq!(missing_media(&nested).unwrap(), Vec::::new()); + // Too big. + let big = format!("{{\"pad\":\"{}\"}}", "a".repeat(usize::try_from(MAX_LINK_PAYLOAD).unwrap())); + let err = OpenPayload::from_value(&json!({"format": "otio-json", "data": big})).unwrap_err(); + assert_eq!(err, LinkError::TooLarge); + assert_eq!(err.code(), "payload_too_large"); + } + + #[test] + fn rejects_bad_payloads() { + for bad in [ + json!("x"), + json!({}), + json!({"format": "otio-json"}), + json!({"format": "otio-json", "data": ""}), + json!({"format": "image-doc", "path": "rel.png", "asset": ASSET}), + json!({"format": "image-doc", "path": "/a.png", "asset": "craft://bad"}), + json!({"format": "image-doc", "asset": ASSET}), + json!({"format": "svg", "data": "x"}), + ] { + let err = OpenPayload::from_value(&bad).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{bad}: {err:?}"); + assert_eq!(err.code(), "payload_invalid"); + } + } +} diff --git a/crates/ipc/src/record.rs b/crates/ipc/src/record.rs new file mode 100644 index 000000000..6b8b9f09f --- /dev/null +++ b/crates/ipc/src/record.rs @@ -0,0 +1,237 @@ +//! The `craft.link` project item record. +//! +//! Schema: `schemas/link-record.schema.json`. A consumer stores one of these +//! on the linked media item. Records written before the asset fields existed +//! (FilmCraft to EffectCraft) parse and validate unchanged. Unknown JSON fields +//! stay in the `extra` maps and come back out on save. +//! +//! This module uses only `std` and `serde`, so an app can copy it to a wasm build. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::error::LinkError; +use crate::ids::{valid_app_id, valid_uuid}; +use crate::moniker::Moniker; + +/// What the linked asset is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AssetKind { + Still, + Vector, + Sequence, + Comp, +} + +impl AssetKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Still => "still", + Self::Vector => "vector", + Self::Sequence => "sequence", + Self::Comp => "comp", + } + } +} + +/// The linked asset: its kind and its durable id. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AssetRef { + pub kind: AssetKind, + /// A moniker string. [`LinkRecord::validate`] parses it. + pub moniker: String, + #[serde(default, flatten)] + pub extra: Map, +} + +/// Who wrote the result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Producer { + pub app: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub doc_id: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// Size of the linked result. The video fields are absent for stills. +/// +/// `rate_num` and `rate_den` come as a pair. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Format { + pub w: u64, + pub h: u64, + pub alpha: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_num: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_den: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ticks: Option, + #[serde(default, flatten)] + pub extra: Map, +} + +/// One link from a consumer media item to a producer asset. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LinkRecord { + pub v: u32, + pub link_id: String, + pub target_app: String, + pub target_project: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_comp: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nested_sequence: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub producer: Option, + pub result_path: String, + pub result_epoch: u64, + pub format: Format, + pub created: String, + #[serde(default, flatten)] + pub extra: Map, +} + +impl LinkRecord { + /// Check the record rules. Deserialization can accept a record this rejects. + pub fn validate(&self) -> Result<(), LinkError> { + if self.v != 1 { + return Err(invalid(format!("v is {}, want 1", self.v))); + } + if !valid_uuid(&self.link_id) { + return Err(invalid("link_id is not a uuid".into())); + } + validate_app("target_app", &self.target_app)?; + require_nonempty("target_project", &self.target_project)?; + require_nonempty("result_path", &self.result_path)?; + if let Some(comp) = &self.target_comp { + require_nonempty("target_comp", comp)?; + } + if let Some(nest) = &self.nested_sequence { + require_nonempty("nested_sequence", nest)?; + } + if self.target_comp.is_none() && self.asset.is_none() { + return Err(invalid("a record needs target_comp or asset".into())); + } + if let Some(asset) = &self.asset { + Moniker::parse(&asset.moniker).map_err(|err| invalid(format!("asset.moniker: {err}")))?; + } + if let Some(producer) = &self.producer { + validate_app("producer.app", &producer.app)?; + } + self.format.validate()?; + if !valid_rfc3339(&self.created) { + return Err(invalid("created is not an RFC 3339 date-time".into())); + } + Ok(()) + } +} + +impl Format { + fn validate(&self) -> Result<(), LinkError> { + require_at_least_one("format.w", self.w)?; + require_at_least_one("format.h", self.h)?; + match (self.rate_num, self.rate_den) { + (Some(num), Some(den)) => { + require_at_least_one("format.rate_num", num)?; + require_at_least_one("format.rate_den", den) + } + (None, None) => Ok(()), + _ => Err(invalid("format.rate_num and format.rate_den come as a pair".into())), + } + } +} + +fn invalid(message: String) -> LinkError { + LinkError::Invalid(message) +} + +fn validate_app(name: &str, app: &str) -> Result<(), LinkError> { + if valid_app_id(app) { Ok(()) } else { Err(invalid(format!("{name} {app:?} must match [a-z][a-z0-9-]{{1,31}}"))) } +} + +fn require_nonempty(name: &str, value: &str) -> Result<(), LinkError> { + if value.is_empty() { Err(invalid(format!("{name} is empty"))) } else { Ok(()) } +} + +fn require_at_least_one(name: &str, value: u64) -> Result<(), LinkError> { + if value >= 1 { Ok(()) } else { Err(invalid(format!("{name} must be >= 1"))) } +} + +/// `YYYY-MM-DDTHH:MM:SS` with an optional fraction and a `Z` or numeric offset. +fn valid_rfc3339(text: &str) -> bool { + let bytes = text.as_bytes(); + if bytes.len() < 20 { + return false; + } + let (Some(month), Some(day), Some(hour), Some(minute), Some(second)) = + (two_digits(bytes, 5), two_digits(bytes, 8), two_digits(bytes, 11), two_digits(bytes, 14), two_digits(bytes, 17)) + else { + return false; + }; + if !four_digits(bytes, 0) + || byte_at(bytes, 4) != Some(b'-') + || byte_at(bytes, 7) != Some(b'-') + || !matches!(byte_at(bytes, 10), Some(b'T' | b't')) + || byte_at(bytes, 13) != Some(b':') + || byte_at(bytes, 16) != Some(b':') + { + return false; + } + if !(1..=12).contains(&month) || !(1..=31).contains(&day) || hour > 23 || minute > 59 || second > 60 { + return false; + } + let (Some(century), Some(year_in_century)) = (two_digits(bytes, 0), two_digits(bytes, 2)) else { + return false; + }; + if chrono::NaiveDate::from_ymd_opt(i32::from(century * 100 + year_in_century), u32::from(month), u32::from(day)).is_none() { + return false; + } + let mut index = 19; + if byte_at(bytes, index) == Some(b'.') { + index += 1; + let start = index; + while byte_at(bytes, index).is_some_and(|byte| byte.is_ascii_digit()) { + index += 1; + } + if index == start { + return false; + } + } + match byte_at(bytes, index) { + Some(b'Z' | b'z') => index + 1 == bytes.len(), + Some(b'+' | b'-') => numeric_offset(bytes, index + 1), + _ => false, + } +} + +fn numeric_offset(bytes: &[u8], index: usize) -> bool { + if bytes.len() != index + 5 || byte_at(bytes, index + 2) != Some(b':') { + return false; + } + matches!((two_digits(bytes, index), two_digits(bytes, index + 3)), (Some(hour), Some(minute)) if hour <= 23 && minute <= 59) +} + +fn four_digits(bytes: &[u8], index: usize) -> bool { + bytes.get(index..index + 4).is_some_and(|slice| slice.iter().all(u8::is_ascii_digit)) +} + +fn two_digits(bytes: &[u8], index: usize) -> Option { + let slice = bytes.get(index..index + 2)?; + let hi = slice.first().copied()?; + let lo = slice.get(1).copied()?; + if !hi.is_ascii_digit() || !lo.is_ascii_digit() { + return None; + } + Some(u16::from(hi - b'0') * 10 + u16::from(lo - b'0')) +} + +fn byte_at(bytes: &[u8], index: usize) -> Option { + bytes.get(index).copied() +} diff --git a/crates/ipc/src/refresh.rs b/crates/ipc/src/refresh.rs new file mode 100644 index 000000000..f4b48a72e --- /dev/null +++ b/crates/ipc/src/refresh.rs @@ -0,0 +1,572 @@ +//! Watch and refresh rules for linked files (`PROTOCOL.md` section 10). +//! +//! A producer writes `result_path` with [`write_atomic`]. A consumer keeps one +//! [`Tracker`] per linked file and calls [`Tracker::poll`] on a timer. The +//! tracker needs no connection to the producer, so it also works while the +//! producer is offline. `std` only. +//! +//! | State | Meaning | +//! |---|---| +//! | `Current` | the disk has the content the consumer loaded | +//! | `Modified` | the disk differs from what the consumer loaded; the epoch went up | +//! | `Missing` | the file does not exist | + +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::UNIX_EPOCH; + +use sha2::{Digest, Sha256}; + +use crate::error::LinkError; + +/// Size, modification time and content hash of a file at one moment. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fingerprint { + pub size: u64, + pub mtime_ms: u64, + /// `sha256:` followed by 64 lowercase hex characters. + pub hash: String, +} + +/// What a consumer shows in its Links panel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileState { + Current, + Modified, + Missing, +} + +impl FileState { + /// The wire word used by `link.refresh`. + pub fn as_str(self) -> &'static str { + match self { + Self::Current => "current", + Self::Modified => "modified", + Self::Missing => "missing", + } + } +} + +/// The result of one [`Tracker::poll`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Change { + pub state: FileState, + pub epoch: u64, + /// True when this poll added 1 to the epoch. + pub bumped: bool, +} + +/// Watches one file for a consumer. +#[derive(Debug)] +pub struct Tracker { + path: PathBuf, + epoch: u64, + /// What the consumer has loaded. + loaded: Option, + /// What the last poll saw on disk. + seen: Option, + state: FileState, +} + +impl Tracker { + /// Start watching `path` at `epoch`. A file that exists now counts as loaded. + pub fn new(path: impl Into, epoch: u64) -> Result { + let path = path.into(); + let seen = fingerprint_if_present(&path)?; + let state = if seen.is_some() { FileState::Current } else { FileState::Missing }; + Ok(Self { path, epoch, loaded: seen.clone(), seen, state }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn epoch(&self) -> u64 { + self.epoch + } + + pub fn state(&self) -> FileState { + self.state + } + + /// The fingerprint from the last poll, or `None` while the file is missing. + pub fn seen(&self) -> Option<&Fingerprint> { + self.seen.as_ref() + } + + /// Raise the epoch to `epoch` when it is larger. Used when the producer is + /// reachable again and reports its own epoch. An epoch never decreases. + pub fn adopt_epoch(&mut self, epoch: u64) { + self.epoch = self.epoch.max(epoch); + } + + /// Look at the disk once. + /// + /// A shallow poll (`deep` false) reads size and mtime and hashes the file + /// only when either differs from the last poll. A deep poll always hashes. + /// An edit that keeps size and mtime is invisible to a shallow poll. + /// + /// A missing file gives `Missing` and no epoch bump. Any other read error + /// is returned and leaves the state as it was. So does a change that needs + /// a bump at epoch `u64::MAX`: that is `Err(Invalid)`, because an epoch that + /// cannot go up would let a cached frame pass for the new content. + pub fn poll(&mut self, deep: bool) -> Result { + let Some(mut file) = open_if_present(&self.path)? else { + self.seen = None; + self.state = FileState::Missing; + return Ok(self.change(false)); + }; + let (size, mtime_ms) = stat(&file, &self.path)?; + if !deep + && let Some(prev) = &self.seen + && prev.size == size + && prev.mtime_ms == mtime_ms + { + return Ok(self.change(false)); + } + let now = Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? }; + let changed = self.seen.as_ref().is_none_or(|prev| prev.hash != now.hash); + let bumped = changed && self.differs_from_loaded_or_was_seen(&now); + self.apply(now, bumped)?; + Ok(self.change(bumped)) + } + + /// The consumer loaded the content it last saw. `Modified` becomes `Current`. + pub fn accept(&mut self) { + self.loaded = self.seen.clone(); + self.state = if self.seen.is_some() { FileState::Current } else { FileState::Missing }; + } + + /// Point the link at `new_path`. The new file is read at once. Its hash + /// equal to what the consumer loaded gives `Current` with no bump. A + /// different hash gives `Modified` and one bump. A missing `new_path` is an + /// error and the old path stays. So it does when the bump would pass `u64::MAX`. + pub fn relink(&mut self, new_path: impl Into) -> Result { + let new_path = new_path.into(); + let Some(now) = fingerprint_if_present(&new_path)? else { + return Err(LinkError::Unknown(format!("{} does not exist", new_path.display()))); + }; + let same = self.loaded.as_ref().is_some_and(|loaded| loaded.hash == now.hash); + self.apply(now, !same)?; + self.path = new_path; + Ok(self.change(!same)) + } + + /// Whether a changed hash counts as a new version. The first sighting after + /// `Missing` that equals the loaded content is not a new version. + fn differs_from_loaded_or_was_seen(&self, now: &Fingerprint) -> bool { + match (&self.seen, &self.loaded) { + (Some(_), _) => true, + (None, Some(loaded)) => loaded.hash != now.hash, + (None, None) => true, + } + } + + /// Record `now`. A bump adds 1 to the epoch, or fails with the tracker untouched. + fn apply(&mut self, now: Fingerprint, bumped: bool) -> Result<(), LinkError> { + if bumped { + self.epoch = self.epoch.checked_add(1).ok_or_else(|| LinkError::Invalid("the epoch is at its maximum and cannot go up".into()))?; + } + let current = self.loaded.as_ref().is_some_and(|loaded| loaded.hash == now.hash); + self.seen = Some(now); + self.state = if current { FileState::Current } else { FileState::Modified }; + Ok(()) + } + + fn change(&self, bumped: bool) -> Change { + Change { state: self.state, epoch: self.epoch, bumped } + } +} + +/// Fingerprint `path`. A missing file is an error. +pub fn fingerprint(path: &Path) -> Result { + let mut file = crate::fsopen::open_read(path, true)?; + let (size, mtime_ms) = stat(&file, path)?; + Ok(Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? }) +} + +/// `None` when the file does not exist. +pub fn fingerprint_if_present(path: &Path) -> Result, LinkError> { + match open_if_present(path)? { + None => Ok(None), + Some(mut file) => { + let (size, mtime_ms) = stat(&file, path)?; + Ok(Some(Fingerprint { size, mtime_ms, hash: hash_reader(&mut file)? })) + } + } +} + +fn open_if_present(path: &Path) -> Result, LinkError> { + // `stat` checks the type on this handle. An open that waits for a FIFO writer would stall the caller. + match crate::fsopen::open_read(path, true) { + Ok(file) => Ok(Some(file)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + // Windows cannot open a directory for reading while Unix can, so `stat` would never + // run there and callers would see an OS-specific Io instead. Classify a path that + // exists but is not a regular file here, so both platforms report Invalid. + Err(err) => match std::fs::metadata(path) { + Ok(meta) if !meta.is_file() => Err(LinkError::Invalid(format!("{} is not a regular file", path.display()))), + _ => Err(LinkError::Io(format!("{}: {err}", path.display()))), + }, + } +} + +fn stat(file: &File, path: &Path) -> Result<(u64, u64), LinkError> { + let meta = file.metadata().map_err(|err| LinkError::Io(format!("{}: {err}", path.display())))?; + if !meta.is_file() { + return Err(LinkError::Invalid(format!("{} is not a regular file", path.display()))); + } + let modified = meta.modified().map_err(|err| LinkError::Io(format!("{}: {err}", path.display())))?; + let since = modified.duration_since(UNIX_EPOCH).map_err(|_| LinkError::Io(format!("{}: mtime is before 1970", path.display())))?; + let ms = u64::try_from(since.as_millis()).map_err(|_| LinkError::Io(format!("{}: mtime is out of range", path.display())))?; + Ok((meta.len(), ms)) +} + +fn hash_reader(reader: &mut impl Read) -> Result { + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(buf.get(..n).unwrap_or_default()); + } + Ok(format!("sha256:{}", crate::auth::encode_hex(&hasher.finalize()))) +} + +/// True for the temporary names [`write_atomic`] uses. Readers skip them. +pub fn is_temp_name(name: &str) -> bool { + name.starts_with('.') && name.ends_with(".tmp") +} + +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Write `bytes` to `path` so a reader sees the old file or the new file. +/// +/// The bytes go to `....tmp` in the same directory, are flushed, +/// and are renamed over `path`. The directory is flushed on Unix. +/// +/// An error before the rename removes the temporary file and leaves `path` as it was. The +/// directory flush and the final fingerprint run after the rename, so an error from either can +/// come back while `path` already holds the new bytes. Such an error means the write is not +/// confirmed durable or not fingerprinted, not that `path` is unchanged. +pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result { + let name = path.file_name().and_then(|n| n.to_str()).ok_or_else(|| LinkError::Invalid(format!("{} has no file name", path.display())))?; + let dir = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let tmp = dir.join(format!(".{name}.{}.{}.tmp", std::process::id(), TEMP_COUNTER.fetch_add(1, Ordering::Relaxed))); + let written = write_and_rename(&tmp, path, bytes); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written?; + #[cfg(unix)] + File::open(dir)?.sync_all()?; + fingerprint(path) +} + +fn write_and_rename(tmp: &Path, path: &Path, bytes: &[u8]) -> Result<(), LinkError> { + let mut file = OpenOptions::new().write(true).create_new(true).open(tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(tmp, path)?; + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::time::Duration; + + use super::*; + + struct Tmp(PathBuf); + + impl Tmp { + fn new(label: &str) -> Self { + let nanos = std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let dir = std::env::temp_dir().join(format!("cl-refresh-{label}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + Self(dir) + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn sha_of(bytes: &[u8]) -> String { + format!("sha256:{}", crate::auth::encode_hex(&Sha256::digest(bytes))) + } + + #[test] + fn hash_is_sha256_of_the_content() { + let tmp = Tmp::new("hash"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"abc").unwrap(); + let fp = fingerprint(&path).unwrap(); + assert_eq!(fp.hash, "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert_eq!(fp.size, 3); + assert!(fp.mtime_ms > 0); + } + + #[test] + fn unchanged_file_stays_current_with_no_bump() { + let tmp = Tmp::new("same"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 4).unwrap(); + assert_eq!(tracker.state(), FileState::Current); + for deep in [false, true] { + let change = tracker.poll(deep).unwrap(); + assert_eq!(change, Change { state: FileState::Current, epoch: 4, bumped: false }); + } + } + + #[test] + fn changed_content_is_modified_and_bumps_once() { + let tmp = Tmp::new("mod"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 4).unwrap(); + write_atomic(&path, b"two!").unwrap(); + let change = tracker.poll(false).unwrap(); + assert_eq!(change, Change { state: FileState::Modified, epoch: 5, bumped: true }); + // Polling again with no new edit does not bump again. + let again = tracker.poll(true).unwrap(); + assert_eq!(again, Change { state: FileState::Modified, epoch: 5, bumped: false }); + // A second edit bumps again. + write_atomic(&path, b"three").unwrap(); + assert_eq!(tracker.poll(false).unwrap().epoch, 6); + // Loading the new content clears the state. + tracker.accept(); + assert_eq!(tracker.state(), FileState::Current); + assert_eq!(tracker.poll(true).unwrap(), Change { state: FileState::Current, epoch: 6, bumped: false }); + } + + #[test] + fn touch_without_content_change_does_not_bump() { + let tmp = Tmp::new("touch"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let file = OpenOptions::new().write(true).open(&path).unwrap(); + file.set_modified(std::time::SystemTime::now() + Duration::from_secs(60)).unwrap(); + drop(file); + let change = tracker.poll(false).unwrap(); + assert_eq!(change, Change { state: FileState::Current, epoch: 0, bumped: false }); + assert!(tracker.seen().unwrap().mtime_ms > 0); + } + + #[test] + fn shallow_poll_catches_a_same_size_edit_when_mtime_moves() { + let tmp = Tmp::new("mtime"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"aaaa").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let before = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"bbbb").unwrap(); + OpenOptions::new().write(true).open(&path).unwrap().set_modified(before + Duration::from_secs(10)).unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn shallow_poll_misses_an_edit_that_keeps_size_and_mtime_and_deep_poll_finds_it() { + let tmp = Tmp::new("deep"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"aaaa").unwrap(); + let mut tracker = Tracker::new(&path, 0).unwrap(); + let mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::fs::write(&path, b"bbbb").unwrap(); + OpenOptions::new().write(true).open(&path).unwrap().set_modified(mtime).unwrap(); + assert_eq!(tracker.poll(false).unwrap().state, FileState::Current, "documented limit of a shallow poll"); + let change = tracker.poll(true).unwrap(); + assert_eq!(change, Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn missing_then_same_content_returns_to_current_without_bump() { + let tmp = Tmp::new("missing"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 2).unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Missing, epoch: 2, bumped: false }); + assert!(tracker.seen().is_none()); + std::fs::write(&path, b"one").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Current, epoch: 2, bumped: false }); + } + + #[test] + fn missing_then_new_content_is_modified_with_a_bump() { + let tmp = Tmp::new("missing2"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 2).unwrap(); + std::fs::remove_file(&path).unwrap(); + tracker.poll(false).unwrap(); + std::fs::write(&path, b"different").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 3, bumped: true }); + } + + #[test] + fn tracker_for_a_file_that_does_not_exist_yet() { + let tmp = Tmp::new("pending"); + let path = tmp.0.join("later.png"); + let mut tracker = Tracker::new(&path, 0).unwrap(); + assert_eq!(tracker.state(), FileState::Missing); + write_atomic(&path, b"first").unwrap(); + assert_eq!(tracker.poll(false).unwrap(), Change { state: FileState::Modified, epoch: 1, bumped: true }); + } + + #[test] + fn unreadable_path_is_an_error_not_missing() { + let tmp = Tmp::new("dir"); + let path = tmp.0.join("a.png"); + std::fs::create_dir(&path).unwrap(); + assert!(Tracker::new(&path, 0).is_err(), "a directory is not a result file"); + } + + #[cfg(unix)] + #[test] + fn a_fifo_at_the_path_is_an_error_and_never_blocks() { + // Forking `mkfifo` while another test holds a discovery lock would hand the child a copy of the lock fd. + let _spawning = crate::SPAWN_GUARD.write().unwrap_or_else(std::sync::PoisonError::into_inner); + let tmp = Tmp::new("fifo"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let mut tracker = Tracker::new(&path, 3).unwrap(); + std::fs::remove_file(&path).unwrap(); + let made = std::process::Command::new("mkfifo").arg(&path).status().map(|s| s.success()).unwrap_or(false); + assert!(made, "mkfifo failed"); + // The open of a FIFO with no writer waits for one. Run each call where a hang fails the test instead of the run. + let (tx, rx) = std::sync::mpsc::channel(); + let watched = path.clone(); + std::thread::spawn(move || { + let polled = tracker.poll(true); + let kept = (tracker.state(), tracker.epoch()); + let _ = tx.send((polled, kept, Tracker::new(&watched, 0).map(|_| ()), fingerprint(&watched).map(|_| ()))); + }); + let (polled, kept, created, printed) = rx.recv_timeout(Duration::from_secs(5)).expect("a FIFO blocked the open"); + assert!(matches!(polled, Err(LinkError::Invalid(_))), "{polled:?}"); + assert_eq!(kept, (FileState::Current, 3), "a failed poll leaves the state as it was"); + assert!(matches!(created, Err(LinkError::Invalid(_))), "{created:?}"); + assert!(matches!(printed, Err(LinkError::Invalid(_))), "{printed:?}"); + } + + #[test] + fn relink_compares_hashes_and_rejects_a_missing_target() { + let tmp = Tmp::new("relink"); + let old = tmp.0.join("old.png"); + let same = tmp.0.join("same.png"); + let other = tmp.0.join("other.png"); + std::fs::write(&old, b"one").unwrap(); + std::fs::write(&same, b"one").unwrap(); + std::fs::write(&other, b"two").unwrap(); + let mut tracker = Tracker::new(&old, 7).unwrap(); + + let err = tracker.relink(tmp.0.join("nope.png")).unwrap_err(); + assert!(matches!(err, LinkError::Unknown(_)), "{err:?}"); + assert_eq!(tracker.path(), old.as_path(), "old path stays"); + + assert_eq!(tracker.relink(&same).unwrap(), Change { state: FileState::Current, epoch: 7, bumped: false }); + assert_eq!(tracker.path(), same.as_path()); + assert_eq!(tracker.relink(&other).unwrap(), Change { state: FileState::Modified, epoch: 8, bumped: true }); + } + + #[test] + fn a_bump_at_the_largest_epoch_is_an_error_and_changes_nothing() { + let tmp = Tmp::new("maxepoch"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"one").unwrap(); + let other = tmp.0.join("other.png"); + std::fs::write(&other, b"two").unwrap(); + let mut tracker = Tracker::new(&path, u64::MAX).unwrap(); + // No bump needed, no error. + assert_eq!(tracker.poll(true).unwrap(), Change { state: FileState::Current, epoch: u64::MAX, bumped: false }); + write_atomic(&path, b"changed").unwrap(); + for _ in 0..2 { + assert!(matches!(tracker.poll(true), Err(LinkError::Invalid(_)))); + assert_eq!((tracker.state(), tracker.epoch()), (FileState::Current, u64::MAX)); + } + assert!(matches!(tracker.relink(&other), Err(LinkError::Invalid(_)))); + assert_eq!(tracker.path(), path.as_path(), "the old path stays"); + assert_eq!((tracker.state(), tracker.epoch()), (FileState::Current, u64::MAX)); + } + + #[test] + fn adopt_epoch_never_decreases() { + let tmp = Tmp::new("adopt"); + let path = tmp.0.join("a.png"); + std::fs::write(&path, b"x").unwrap(); + let mut tracker = Tracker::new(&path, 5).unwrap(); + tracker.adopt_epoch(3); + assert_eq!(tracker.epoch(), 5); + tracker.adopt_epoch(9); + assert_eq!(tracker.epoch(), 9); + } + + #[test] + fn write_atomic_replaces_whole_and_leaves_no_temp() { + let tmp = Tmp::new("atomic"); + let path = tmp.0.join("out.png"); + let first = write_atomic(&path, b"first").unwrap(); + assert_eq!(first.hash, sha_of(b"first")); + let second = write_atomic(&path, b"second version").unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"second version"); + assert_eq!(second.hash, sha_of(b"second version")); + let names: Vec = std::fs::read_dir(&tmp.0).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert_eq!(names, vec!["out.png".to_string()], "no temp file left"); + } + + #[test] + fn write_atomic_failure_keeps_the_old_file_and_removes_the_temp() { + let tmp = Tmp::new("atomic-fail"); + let path = tmp.0.join("out.png"); + write_atomic(&path, b"keep").unwrap(); + // A directory at the destination makes rename fail after the temp file is written. + let blocked = tmp.0.join("blocked"); + std::fs::create_dir(&blocked).unwrap(); + std::fs::write(blocked.join("child"), b"x").unwrap(); + assert!(write_atomic(&blocked, b"nope").is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"keep"); + let temps = std::fs::read_dir(&tmp.0).unwrap().filter(|e| is_temp_name(e.as_ref().unwrap().file_name().to_str().unwrap())).count(); + assert_eq!(temps, 0); + } + + #[test] + fn reader_never_sees_a_half_file_while_a_writer_replaces_it() { + let tmp = Tmp::new("race"); + let path = tmp.0.join("out.bin"); + let a = vec![b'a'; 512 * 1024]; + let b = vec![b'b'; 768 * 1024]; + write_atomic(&path, &a).unwrap(); + let writer_path = path.clone(); + let (wa, wb) = (a.clone(), b.clone()); + let writer = std::thread::spawn(move || { + for i in 0..60 { + write_atomic(&writer_path, if i % 2 == 0 { &wb } else { &wa }).unwrap(); + } + }); + while !writer.is_finished() { + let got = std::fs::read(&path).unwrap(); + assert!(got == a || got == b, "saw a torn file of {} bytes", got.len()); + } + writer.join().unwrap(); + } + + #[test] + fn temp_names() { + assert!(is_temp_name(".out.png.12.0.tmp")); + assert!(!is_temp_name("out.png")); + assert!(!is_temp_name("notes.tmp")); + } +} diff --git a/crates/ipc/src/server/dispatch.rs b/crates/ipc/src/server/dispatch.rs new file mode 100644 index 000000000..5c7e24d33 --- /dev/null +++ b/crates/ipc/src/server/dispatch.rs @@ -0,0 +1,316 @@ +//! Methods answered on the connection thread, plus the sink forward. + +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use super::events::{self, Subscription}; +use super::provider; +use super::{Activation, Assets, Hub, lock}; +use crate::error::LinkError; +use crate::moniker::Moniker; +use crate::payload::OpenPayload; +use crate::wire::{self, Request}; + +pub(super) enum Outcome { + Continue(Value), + Close(Option), + Bulk { reply: Value, ticket: String }, +} + +pub(super) fn dispatch(hub: &Hub, conn_id: u64, sub: &Subscription, first: bool, req: &Request) -> Outcome { + match req.method.as_str() { + "link.hello" => continue_result(&req.id, hello(hub, &req.params)), + "events.subscribe" => continue_result(&req.id, subscribe(sub, &req.params)), + "events.unsubscribe" => { + events::unsubscribe(sub); + Outcome::Continue(wire::reply_ok(&req.id, json!({}))) + } + "link.bye" => { + events::unsubscribe(sub); + release_tickets(hub, conn_id); + Outcome::Close(Some(wire::reply_ok(&req.id, json!({})))) + } + "app.activate" => continue_result(&req.id, activate(hub, &req.params)), + "link.resolve" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "link.resolve", &req.params, resolve)), + "asset.describe" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "asset.describe", &req.params, describe)), + "link.refresh" if hub.assets.is_some() => continue_result(&req.id, with_assets(hub, "link.refresh", &req.params, refresh)), + "link.open" => match validate_open(&req.params) { + Ok(()) => forward(hub, &req.method, &req.id, req.params.clone()), + Err(err) => Outcome::Continue(wire::reply_err(&req.id, &err)), + }, + "frames.open" => continue_result(&req.id, frames_open(hub, conn_id, &req.params)), + "bulk.attach" => bulk_attach(hub, first, req), + method => forward(hub, method, &req.id, req.params.clone()), + } +} + +pub(super) fn release_tickets(hub: &Hub, conn_id: u64) { + let owned = { + let mut tickets = lock(&hub.tickets); + let owned: Vec = tickets.iter().filter(|(_, id)| **id == conn_id).map(|(ticket, _)| ticket.clone()).collect(); + for ticket in &owned { + tickets.remove(ticket); + } + owned + }; + for ticket in owned { + hub.frames.close_ticket(&ticket); + } +} + +fn continue_result(id: &Value, result: Result) -> Outcome { + match result { + Ok(result) => Outcome::Continue(wire::reply_ok(id, result)), + Err(err) => Outcome::Continue(wire::reply_err(id, &err)), + } +} + +fn hello(hub: &Hub, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let protocol = obj.get("protocol").and_then(Value::as_object).ok_or_else(|| LinkError::BadParams("protocol required".into()))?; + let min = required_u32(protocol.get("min"), "protocol.min")?; + let max = required_u32(protocol.get("max"), "protocol.max")?; + if !wire::protocols_overlap(min, max) { + return Err(LinkError::Protocol { server_min: wire::PROTOCOL_MIN, server_max: wire::PROTOCOL_MAX }); + } + let record = &hub.record; + let mut caps: Vec = ["events", "activate", "link.receive", "frames"].iter().map(|c| (*c).to_string()).collect(); + for cap in record.caps() { + if !caps.contains(&cap) { + caps.push(cap); + } + } + Ok(json!({ + "app": record.app, + "app_version": record.app_version, + "pid": record.pid, + "instance": record.instance, + "protocol": wire::PROTOCOL_MAX, + "caps": caps, + })) +} + +fn subscribe(sub: &Subscription, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let list = obj.get("events").and_then(Value::as_array).ok_or_else(|| LinkError::BadParams("events must be an array".into()))?; + let mut names = Vec::with_capacity(list.len()); + for item in list { + let Some(name) = item.as_str() else { + return Err(LinkError::BadParams("event name must be a string".into())); + }; + names.push(name.to_string()); + } + events::subscribe(sub, names); + Ok(json!({"seq": 0})) +} + +fn activate(hub: &Hub, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let token = optional_string(obj, "token")?; + let from_pid = optional_u32(obj.get("from_pid"), "from_pid")?; + let open = match obj.get("open") { + None => None, + Some(Value::Object(open)) => { + if let Some(asset) = open.get("asset") { + let text = asset.as_str().ok_or_else(|| LinkError::BadParams("open.asset must be a moniker string".into()))?; + Moniker::parse(text).map_err(|err| LinkError::BadParams(format!("open.asset: {err}")))?; + } + Some(Value::Object(open.clone())) + } + Some(_) => return Err(LinkError::BadParams("open must be an object".into())), + }; + let provider = std::sync::Arc::clone(&hub.activate); + let activated = match provider::call(hub, "app.activate", move || provider.activate(token, from_pid, open))? { + Activation::Yes => Value::Bool(true), + Activation::No => Value::Bool(false), + Activation::Requested => Value::String("requested".into()), + }; + Ok(json!({"activated": activated})) +} + +/// Run an asset method on a helper thread, with the deadline and the stop check of [`provider::call`]. +fn with_assets(hub: &Hub, what: &str, params: &Value, run: fn(&dyn Assets, &Value) -> Result) -> Result { + let Some(assets) = &hub.assets else { + return Err(LinkError::NotReady("this app has no asset provider".into())); + }; + let assets = std::sync::Arc::clone(assets); + let params = params.clone(); + provider::call(hub, what, move || run(assets.as_ref(), ¶ms))? +} + +fn asset_param(params: &Value) -> Result { + let obj = wire::params_object(params)?; + let text = obj.get("asset").and_then(Value::as_str).ok_or_else(|| LinkError::BadParams("asset must be a moniker string".into()))?; + Moniker::parse(text).map_err(|err| LinkError::BadParams(format!("asset: {err}"))) +} + +fn resolve(assets: &dyn Assets, params: &Value) -> Result { + let asset = asset_param(params)?; + Ok(serde_json::to_value(assets.resolve(&asset)?)?) +} + +fn describe(assets: &dyn Assets, params: &Value) -> Result { + let asset = asset_param(params)?; + Ok(serde_json::to_value(assets.describe(&asset)?)?) +} + +fn refresh(assets: &dyn Assets, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let link_id = obj + .get("link_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .ok_or_else(|| LinkError::BadParams("link_id must be a non-empty string".into()))?; + Ok(serde_json::to_value(assets.refresh(link_id)?)?) +} + +/// `link.open` carries a `payload` whose shape depends on `format`. A bad one +/// never reaches the app. Params that are not an object stay `bad_params`. +fn validate_open(params: &Value) -> Result<(), LinkError> { + let obj = wire::params_object(params)?; + let payload = obj.get("payload").ok_or_else(|| LinkError::PayloadInvalid("payload is required".into()))?; + OpenPayload::from_value(payload)?; + match obj.get("result_path") { + None | Some(Value::Null) => Ok(()), + Some(Value::String(path)) => crate::payload::check_peer_path(path, "result_path"), + Some(_) => Err(LinkError::PayloadInvalid("result_path must be a string".into())), + } +} + +fn frames_open(hub: &Hub, conn_id: u64, params: &Value) -> Result { + let obj = wire::params_object(params)?; + let opened = hub.frames.open(obj)?; + if opened.ticket.len() != 32 || !crate::auth::is_lower_hex(&opened.ticket) { + hub.frames.close_ticket(&opened.ticket); + return Err(LinkError::BadParams("ticket must be 32 lowercase hex characters".into())); + } + let mut tickets = lock(&hub.tickets); + if tickets.contains_key(&opened.ticket) { + return Err(LinkError::BadParams("ticket already registered".into())); + } + tickets.insert(opened.ticket.clone(), conn_id); + drop(tickets); + Ok(json!({ + "ticket": opened.ticket, + "w": opened.w, + "h": opened.h, + "stride": opened.stride, + "len": opened.len, + })) +} + +fn bulk_attach(hub: &Hub, first: bool, req: &Request) -> Outcome { + if !first { + let err = LinkError::BadParams("bulk.attach must be the first method".into()); + return Outcome::Close(Some(wire::reply_err(&req.id, &err))); + } + let ticket = match ticket_param(&req.params) { + Ok(ticket) => ticket, + Err(err) => return Outcome::Close(Some(wire::reply_err(&req.id, &err))), + }; + let known = lock(&hub.tickets).contains_key(&ticket); + if !known { + let err = LinkError::BadParams("unknown ticket".into()); + return Outcome::Close(Some(wire::reply_err(&req.id, &err))); + } + Outcome::Bulk { reply: wire::reply_ok(&req.id, json!({})), ticket } +} + +fn ticket_param(params: &Value) -> Result { + let obj = wire::params_object(params)?; + let Some(ticket) = obj.get("ticket").and_then(Value::as_str) else { + return Err(LinkError::BadParams("ticket required".into())); + }; + Ok(ticket.to_string()) +} + +fn forward(hub: &Hub, method: &str, id: &Value, params: Value) -> Outcome { + let Some(rx) = hub.sink.submit(method.to_string(), params) else { + return Outcome::Close(None); + }; + let deadline = Instant::now() + Duration::from_secs(wire::REPLY_DEADLINE_SECS); + loop { + if hub.stop.load(std::sync::atomic::Ordering::SeqCst) { + return Outcome::Close(None); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Outcome::Continue(timeout_reply(method, id)); + } + let slice = left.min(Duration::from_millis(200)); + match rx.recv_timeout(slice) { + Ok(result) => return Outcome::Continue(adopt_sink_reply(id, result)), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Outcome::Close(None), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +/// FilmCraft's UI sends a finished reply object (`ok` plus `result` or `error`). +/// A value with `ok` is that object. Insert `id` and do not wrap it again. +/// Any other value is a result payload and gets [`wire::reply_ok`]. +fn adopt_sink_reply(id: &Value, value: Value) -> Value { + let Some(obj) = value.as_object() else { + return wire::reply_ok(id, value); + }; + if !obj.contains_key("ok") { + return wire::reply_ok(id, value); + } + let mut reply = value; + if let Some(map) = reply.as_object_mut() + && !map.contains_key("id") + { + map.insert("id".to_string(), id.clone()); + } + reply +} + +fn timeout_reply(method: &str, id: &Value) -> Value { + if is_new_method(method) { wire::reply_err(id, &LinkError::Timeout) } else { wire::reply_err_text(id, "timeout") } +} + +/// New methods use an error object. Legacy names keep the string `"timeout"`. +fn is_new_method(method: &str) -> bool { + matches!( + method, + "link.open" + | "link.attach" + | "link.focus" + | "link.close" + | "link.hello" + | "events.subscribe" + | "events.unsubscribe" + | "link.bye" + | "app.activate" + | "frames.open" + | "bulk.attach" + | "link.resolve" + | "asset.describe" + ) +} + +fn optional_string(obj: &Map, key: &str) -> Result, LinkError> { + match obj.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) => Ok(Some(text.clone())), + Some(_) => Err(LinkError::BadParams(format!("{key} must be a string or null"))), + } +} + +fn optional_u32(value: Option<&Value>, key: &str) -> Result, LinkError> { + match value { + None => Ok(None), + Some(value) => required_u32(Some(value), key).map(Some), + } +} + +fn required_u32(value: Option<&Value>, key: &str) -> Result { + let Some(value) = value else { + return Err(LinkError::BadParams(format!("{key} is required"))); + }; + let Some(number) = value.as_u64() else { + return Err(LinkError::BadParams(format!("{key} must be a number"))); + }; + u32::try_from(number).map_err(|_| LinkError::BadParams(format!("{key} is out of range"))) +} diff --git a/crates/ipc/src/server/events.rs b/crates/ipc/src/server/events.rs new file mode 100644 index 000000000..6de840964 --- /dev/null +++ b/crates/ipc/src/server/events.rs @@ -0,0 +1,214 @@ +//! Per-connection event queue. `emit` only enqueues. +//! +//! The writer sleeps a short grace before it drains so a burst fills the +//! queue of [`crate::wire::EVENT_QUEUE`] instead of racing one write at a time +//! into the socket buffer. Overflow occupies one of those slots. + +use std::collections::{HashSet, VecDeque}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use super::{Hub, lock}; +use crate::wire::{self, EVENT_QUEUE}; + +/// How long a burst may land in the queue before the writer drains it. +const BATCH_GRACE: Duration = Duration::from_millis(30); +const WAIT_SLICE: Duration = Duration::from_millis(200); + +pub(crate) struct Subscription { + pub id: u64, + filter: Mutex, + outbox: Mutex, + cv: Condvar, + closed: AtomicBool, +} + +struct Filter { + names: Option>, +} + +struct Outbox { + items: VecDeque, + latched: bool, + next_seq: u64, +} + +struct Queued { + line: Value, + overflow: bool, +} + +impl Subscription { + pub(super) fn new(id: u64) -> Arc { + Arc::new(Self { + id, + filter: Mutex::new(Filter { names: None }), + outbox: Mutex::new(Outbox { items: VecDeque::new(), latched: false, next_seq: 1 }), + cv: Condvar::new(), + closed: AtomicBool::new(false), + }) + } +} + +pub(super) fn subscribe(sub: &Subscription, names: Vec) { + let mut filter = lock(&sub.filter); + filter.names = Some(names.into_iter().collect()); +} + +pub(super) fn is_subscribed(sub: &Subscription) -> bool { + lock(&sub.filter).names.is_some() +} + +pub(super) fn unsubscribe(sub: &Subscription) { + let mut filter = lock(&sub.filter); + filter.names = None; +} + +pub(super) fn enqueue(sub: &Subscription, event: &str, data: &Value) { + if sub.closed.load(Ordering::SeqCst) { + return; + } + let interested = { + let filter = lock(&sub.filter); + filter.names.as_ref().is_some_and(|names| names.contains(event)) + }; + if !interested { + return; + } + { + let mut outbox = lock(&sub.outbox); + if sub.closed.load(Ordering::SeqCst) || outbox.latched { + return; + } + let line = wire::push_event(event, outbox.next_seq, data.clone()); + let oversize = serde_json::to_vec(&line).map(|bytes| bytes.len() > wire::MAX_LINE).unwrap_or(true); + if oversize || outbox.items.len() >= EVENT_QUEUE { + // A full queue gives up its newest slot. An event that is too large for one line + // is dropped alone: everything queued before it is still delivered. + let since = if outbox.items.len() >= EVENT_QUEUE { + outbox.items.pop_back().map(|item| item_seq(&item)).unwrap_or(outbox.next_seq) + } else { + outbox.items.back().map(item_seq).unwrap_or(outbox.next_seq.saturating_sub(1)) + }; + let seq = outbox.next_seq; + outbox.next_seq = outbox.next_seq.saturating_add(1); + outbox.items.push_back(Queued { overflow: true, line: wire::push_event("events.overflow", seq, json!({"since": since})) }); + outbox.latched = true; + } else { + outbox.next_seq = outbox.next_seq.saturating_add(1); + outbox.items.push_back(Queued { overflow: false, line }); + } + } + sub.cv.notify_one(); +} + +fn item_seq(item: &Queued) -> u64 { + item.line.get("seq").and_then(Value::as_u64).unwrap_or(0) +} + +pub(super) fn close_sub(sub: &Subscription) { + sub.closed.store(true, Ordering::SeqCst); + // The outbox lock pairs with the check in `write_loop`, so the notify + // cannot land before the writer has entered `wait`. + drop(lock(&sub.outbox)); + sub.cv.notify_one(); +} + +pub(super) fn wake_all(subs: &Mutex>>) { + let guard = lock(subs); + for sub in guard.iter() { + drop(lock(&sub.outbox)); + sub.cv.notify_one(); + } +} + +pub(super) fn remove_sub(hub: &Hub, id: u64) { + lock(&hub.subs).retain(|sub| sub.id != id); +} + +pub(super) fn write_loop(sub: Arc, writer: Arc>>, stop: &AtomicBool) { + loop { + { + let guard = lock(&sub.outbox); + if sub.closed.load(Ordering::SeqCst) || stop.load(Ordering::SeqCst) { + return; + } + if guard.items.is_empty() { + let waited = sub.cv.wait_timeout(guard, WAIT_SLICE); + let (guard, _) = match waited { + Ok(pair) => pair, + Err(err) => err.into_inner(), + }; + drop(guard); + continue; + } + } + std::thread::sleep(BATCH_GRACE); + if sub.closed.load(Ordering::SeqCst) || stop.load(Ordering::SeqCst) { + return; + } + let batch = { + let mut guard = lock(&sub.outbox); + std::mem::take(&mut guard.items) + }; + for item in batch { + let wrote = { + let mut guard = lock(&writer); + wire::write_line(&mut *guard, &item.line).is_ok() + }; + if item.overflow { + lock(&sub.outbox).latched = false; + } + if !wrote { + sub.closed.store(true, Ordering::SeqCst); + return; + } + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn an_event_over_the_line_limit_becomes_an_overflow_marker() { + let sub = Subscription::new(8); + subscribe(&sub, vec!["doc.changed".into()]); + enqueue(&sub, "doc.changed", &json!({"n": 1})); + enqueue(&sub, "doc.changed", &json!({"blob": "x".repeat(wire::MAX_LINE)})); + let outbox = lock(&sub.outbox); + assert_eq!(outbox.items.len(), 2, "the earlier event stays queued"); + assert_eq!(outbox.items[0].line["event"], "doc.changed"); + assert_eq!(outbox.items[1].line["event"], "events.overflow"); + assert_eq!(outbox.items[1].line["data"]["since"], 1); + assert!(outbox.items.iter().all(|item| serde_json::to_vec(&item.line).unwrap().len() <= wire::MAX_LINE)); + } + + #[test] + fn overflow_occupies_one_queue_slot() { + let sub = Subscription::new(7); + subscribe(&sub, vec!["doc.changed".into()]); + enqueue(&sub, "other", &json!({"n": 0})); + assert!(lock(&sub.outbox).items.is_empty()); + for n in 0..EVENT_QUEUE { + enqueue(&sub, "doc.changed", &json!({"n": n})); + } + enqueue(&sub, "doc.changed", &json!({"n": EVENT_QUEUE})); + enqueue(&sub, "doc.changed", &json!({"n": EVENT_QUEUE + 1})); + let outbox = lock(&sub.outbox); + assert!(outbox.latched); + assert_eq!(outbox.items.len(), EVENT_QUEUE); + let overflow: Vec<_> = outbox.items.iter().filter(|item| item.overflow).collect(); + assert_eq!(overflow.len(), 1); + assert_eq!(outbox.items.iter().filter(|item| !item.overflow).count(), EVENT_QUEUE - 1); + let last = overflow[0]; + assert_eq!(last.line["event"], json!("events.overflow")); + assert!(last.line["data"]["since"].is_number()); + assert_eq!(outbox.items.front().expect("queued").line["seq"], json!(1)); + } +} diff --git a/crates/ipc/src/server/listener.rs b/crates/ipc/src/server/listener.rs new file mode 100644 index 000000000..eec2402e7 --- /dev/null +++ b/crates/ipc/src/server/listener.rs @@ -0,0 +1,633 @@ +//! Local socket accept loop, auth, and the per-connection threads. +//! +//! The 17th connection receives one refusal line and does not take a slot. +//! Bulk connections count toward the same cap. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use interprocess::TryClone; +#[cfg(unix)] +use interprocess::local_socket::GenericFilePath; +use interprocess::local_socket::prelude::*; +use interprocess::local_socket::{Listener, ListenerNonblockingMode, ListenerOptions, Stream}; +use serde_json::Value; + +use super::dispatch::{self, Outcome}; +use super::events::{self, Subscription}; +use super::tcp::BoundTcp; +use super::{Hub, lock}; +use crate::discovery::{Endpoint, RuntimeDir}; +use crate::error::LinkError; +use crate::wire::{self, LineReader, MAX_LINE}; + +const READ_POLL: Duration = Duration::from_millis(250); +const REFUSE_WRITE: Duration = Duration::from_secs(2); + +pub(super) fn bind(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + #[cfg(unix)] + { + bind_unix(runtime, app, pid) + } + #[cfg(windows)] + { + let _ = runtime; + bind_pipe(app, pid) + } +} + +pub(super) fn endpoint(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + #[cfg(unix)] + { + runtime.unix_endpoint(app, pid) + } + #[cfg(windows)] + { + let _ = runtime; + Ok(Endpoint::Pipe { name: pipe_name(app, pid) }) + } +} + +#[cfg(unix)] +fn bind_unix(runtime: &RuntimeDir, app: &str, pid: u32) -> Result { + let path = runtime.socket_file(app, pid); + let name = path.to_fs_name::()?; + let listener = ListenerOptions::new().name(name).nonblocking(ListenerNonblockingMode::Accept).reclaim_name(false).create_sync()?; + Ok(listener) +} + +/// Owner and SYSTEM only. A peer pid is not an allow; Windows `PeerCreds` has no user id. +#[cfg(windows)] +fn bind_pipe(app: &str, pid: u32) -> Result { + use interprocess::local_socket::GenericNamespaced; + use interprocess::os::windows::local_socket::ListenerOptionsExt; + use interprocess::os::windows::security_descriptor::SecurityDescriptor; + use widestring::U16CString; + + let wide = U16CString::from_str("D:P(A;;GA;;;OW)(A;;GA;;;SY)").map_err(LinkError::io)?; + let descriptor = SecurityDescriptor::deserialize(wide.as_ucstr())?; + let name = pipe_name(app, pid).to_ns_name::()?; + let listener = ListenerOptions::new() + .name(name) + .nonblocking(ListenerNonblockingMode::Accept) + .reclaim_name(false) + .security_descriptor(descriptor) + .create_sync()?; + Ok(listener) +} + +#[cfg(windows)] +fn pipe_name(app: &str, pid: u32) -> String { + match windows_user() { + Some(user) => format!("craft-{user}-{app}-{pid}"), + None => format!("craft-{app}-{pid}"), + } +} + +/// `USERNAME` with non-alphanumeric bytes replaced. Reading the env is enough; +/// there is no dependency-free Windows user API in this crate. +#[cfg(windows)] +fn windows_user() -> Option { + let raw = std::env::var("USERNAME").ok()?; + if raw.is_empty() { + return None; + } + let mut cleaned = String::with_capacity(raw.len()); + for ch in raw.chars() { + if ch.is_ascii_alphanumeric() { + cleaned.push(ch); + } else { + cleaned.push('_'); + } + } + let trimmed = cleaned.trim_matches('_'); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } +} + +pub(super) fn accept_loop(listener: Listener, tcp: Option, hub: Arc) { + while !hub.stop.load(Ordering::SeqCst) { + reap(&hub); + let mut busy = false; + match listener.accept() { + Ok(stream) => { + busy = true; + accept_local(stream, &hub); + } + Err(err) if idle_accept(&err) => {} + Err(_) => { + std::thread::sleep(Duration::from_millis(10)); + continue; + } + } + if let Some(bound) = tcp.as_ref() + && let Ok(Some(stream)) = super::tcp::poll_accept(&bound.listener) + { + busy = true; + accept_tcp(stream, &hub); + } + if !busy { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +fn idle_accept(err: &std::io::Error) -> bool { + matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted) +} + +fn reap(hub: &Hub) { + let mut threads = lock(&hub.threads); + let pending = std::mem::take(&mut *threads); + for handle in pending { + if handle.is_finished() { + let _ = handle.join(); + } else { + threads.push(handle); + } + } +} + +/// One connection slot. A connection that has to authenticate also holds a share of the small +/// unauthenticated pool until it does. The slots outside that pool are only for peers that proved +/// the token, so silent peers cannot fill the server. +struct Slot { + hub: Arc, + pending: bool, +} + +/// Why [`Slot::admit`] said no. +enum Refusal { + Full, + Unauthenticated, +} + +impl Refusal { + fn text(&self) -> &'static str { + match self { + Self::Full => "too many control connections", + Self::Unauthenticated => "too many unauthenticated connections", + } + } +} + +/// Add one to `counter` unless it is at `cap`. +fn take_one(counter: &AtomicUsize, cap: usize) -> bool { + loop { + let cur = counter.load(Ordering::SeqCst); + if cur >= cap { + return false; + } + if counter.compare_exchange(cur, cur + 1, Ordering::SeqCst, Ordering::SeqCst).is_ok() { + return true; + } + } +} + +impl Slot { + fn admit(hub: &Arc, needs_auth: bool) -> Result { + if !take_one(&hub.open, wire::MAX_CONNECTIONS) { + return Err(Refusal::Full); + } + if needs_auth && !take_one(&hub.pending, hub.limits.max_unauthenticated.min(wire::MAX_CONNECTIONS)) { + hub.open.fetch_sub(1, Ordering::SeqCst); + return Err(Refusal::Unauthenticated); + } + Ok(Slot { hub: Arc::clone(hub), pending: needs_auth }) + } + + /// The peer sent the right token. It leaves the unauthenticated pool and keeps its slot. + fn authenticated(&mut self) { + if self.pending { + self.pending = false; + self.hub.pending.fetch_sub(1, Ordering::SeqCst); + } + } +} + +impl Drop for Slot { + fn drop(&mut self) { + self.authenticated(); + self.hub.open.fetch_sub(1, Ordering::SeqCst); + } +} + +fn accept_local(mut stream: Stream, hub: &Arc) { + let _ = stream.set_send_timeout(Some(REFUSE_WRITE)); + let slot = match Slot::admit(hub, true) { + Ok(slot) => slot, + Err(refusal) => { + refuse(&mut stream, &refusal); + return; + } + }; + if crate::ensure_peer_user(&stream).is_err() { + let _ = wire::write_line(&mut stream, &wire::auth_failure()); + drop(slot); + return; + } + let _ = stream.set_recv_timeout(Some(READ_POLL)); + let Ok(writer_stream) = stream.try_clone() else { + return; + }; + #[cfg(windows)] + let (reader, writer): (Box, Box) = { + let (Ok(reader), Ok(writer)) = (crate::client::PipeTransport::new(stream), crate::client::PipeTransport::new(writer_stream)) else { + return; + }; + reader.set_timeouts(READ_POLL, REFUSE_WRITE); + writer.set_timeouts(READ_POLL, REFUSE_WRITE); + (Box::new(reader), Box::new(writer)) + }; + #[cfg(not(windows))] + let (reader, writer): (Box, Box) = (Box::new(stream), Box::new(writer_stream)); + spawn(slot, Arc::clone(hub), BufReader::new(reader), Arc::new(std::sync::Mutex::new(writer)), true); +} + +fn accept_tcp(mut stream: std::net::TcpStream, hub: &Arc) { + let _ = stream.set_write_timeout(Some(REFUSE_WRITE)); + let slot = match Slot::admit(hub, true) { + Ok(slot) => slot, + Err(refusal) => { + refuse(&mut stream, &refusal); + return; + } + }; + // The listener is bound to 127.0.0.1. Refuse anything else anyway. + if !stream.peer_addr().is_ok_and(|addr| addr.ip().is_loopback()) { + let _ = wire::write_line(&mut stream, &wire::auth_failure()); + return; + } + let _ = stream.set_read_timeout(Some(READ_POLL)); + let Ok(writer_stream) = stream.try_clone() else { + return; + }; + let reader: Box = Box::new(stream); + let writer: Box = Box::new(writer_stream); + spawn(slot, Arc::clone(hub), BufReader::new(reader), Arc::new(std::sync::Mutex::new(writer)), true); +} + +fn refuse(stream: &mut impl Write, refusal: &Refusal) { + let line = serde_json::json!({"ok": false, "error": refusal.text()}); + let _ = wire::write_line(stream, &line); +} + +fn spawn(slot: Slot, hub: Arc, reader: BufReader>, writer: Arc>>, auth: bool) { + let thread_hub = Arc::clone(&hub); + let started = std::thread::Builder::new().name("craft-ipc-conn".into()).spawn(move || { + let mut slot = slot; + connection(thread_hub, reader, writer, auth, &mut slot); + }); + if let Ok(handle) = started { + lock(&hub.threads).push(handle); + } +} + +struct Cleanup { + hub: Arc, + conn_id: u64, + sub: Arc, + writer: Option>, +} + +impl Drop for Cleanup { + fn drop(&mut self) { + events::close_sub(&self.sub); + if let Some(handle) = self.writer.take() { + let _ = handle.join(); + } + events::remove_sub(&self.hub, self.conn_id); + dispatch::release_tickets(&self.hub, self.conn_id); + } +} + +fn connection( + hub: Arc, + mut reader: BufReader>, + writer: Arc>>, + auth: bool, + slot: &mut Slot, +) { + let first_patience = if auth { hub.limits.partial_line.min(hub.limits.auth_deadline) } else { hub.limits.partial_line }; + let mut lines = LineReader::new(MAX_LINE).with_patience(first_patience); + if auth && authenticate(&hub, &mut reader, &mut lines, &writer).is_err() { + return; + } + slot.authenticated(); + lines.set_patience(hub.limits.partial_line); + let conn_id = hub.next_id.fetch_add(1, Ordering::SeqCst); + let sub = Subscription::new(conn_id); + lock(&hub.subs).push(Arc::clone(&sub)); + let write_sub = Arc::clone(&sub); + let write_io = Arc::clone(&writer); + let write_stop = Arc::clone(&hub); + let writer_thread = std::thread::Builder::new().name("craft-ipc-write".into()).spawn(move || { + events::write_loop(write_sub, write_io, &write_stop.stop); + }); + let Ok(writer_thread) = writer_thread else { + events::remove_sub(&hub, conn_id); + return; + }; + let mut cleanup = Cleanup { hub: Arc::clone(&hub), conn_id, sub: Arc::clone(&sub), writer: Some(writer_thread) }; + let mut first = true; + let mut last_input = Instant::now(); + loop { + if hub.stop.load(Ordering::SeqCst) { + break; + } + let line = match next_line(&hub, &mut reader, &mut lines) { + Incoming::Again => { + if lines.partial_age().is_some_and(|age| age >= hub.limits.partial_line) { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, "request line not finished in time; closing the connection")); + break; + } + if lines.partial_age().is_none() && last_input.elapsed() >= hub.limits.idle && !events::is_subscribed(&sub) { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, "idle connection closed")); + break; + } + continue; + } + Incoming::Stop => break, + Incoming::TooLarge => { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, &overlong())); + break; + } + Incoming::Line(bytes) => bytes, + }; + last_input = Instant::now(); + let req = match wire::parse_request(&line) { + Ok(None) => continue, + Ok(Some(req)) => req, + Err(text) => { + let _ = write_value(&writer, &wire::reply_err_text(&Value::Null, &text)); + break; + } + }; + let outcome = dispatch::dispatch(&hub, conn_id, &sub, first, &req); + first = false; + match outcome { + Outcome::Continue(reply) => { + if write_value(&writer, &reply).is_err() { + break; + } + } + Outcome::Close(reply) => { + if let Some(reply) = reply { + let _ = write_value(&writer, &reply); + } + break; + } + Outcome::Bulk { reply, ticket } => { + events::close_sub(&sub); + if let Some(handle) = cleanup.writer.take() { + let _ = handle.join(); + } + if write_value(&writer, &reply).is_err() { + break; + } + bulk_loop(&mut reader, &writer, &hub, &ticket); + break; + } + } + } + drop(cleanup); +} + +fn authenticate( + hub: &Hub, + reader: &mut BufReader>, + lines: &mut LineReader, + writer: &Arc>>, +) -> Result<(), ()> { + let deadline = Instant::now() + hub.limits.auth_deadline; + loop { + if hub.stop.load(Ordering::SeqCst) { + return Err(()); + } + let line = match next_line(hub, reader, lines) { + Incoming::Again if Instant::now() >= deadline => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Incoming::Again => continue, + Incoming::Stop | Incoming::TooLarge => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + // A line that completes after the deadline is too late, even when the read poll has not run since. + Incoming::Line(_) if Instant::now() >= deadline => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Incoming::Line(bytes) => bytes, + }; + match wire::parse_request(&line) { + Ok(None) if Instant::now() < deadline => continue, + Ok(None) => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + Ok(Some(req)) if req.method == "auth" && token_ok(&req.params, &hub.record.token) => { + return write_value(writer, &wire::auth_success(&req.id)).map_err(|_| ()); + } + Ok(Some(_)) | Err(_) => { + let _ = write_value(writer, &wire::auth_failure()); + return Err(()); + } + } + } +} + +fn token_ok(params: &Value, expected: &str) -> bool { + params.get("token").and_then(Value::as_str).is_some_and(|token| crate::tokens_equal(expected, token)) +} + +enum Incoming { + Line(Vec), + Again, + TooLarge, + Stop, +} + +fn next_line(hub: &Hub, reader: &mut BufReader>, lines: &mut LineReader) -> Incoming { + match lines.read(reader) { + Ok(Some(bytes)) => Incoming::Line(bytes), + Ok(None) => Incoming::Stop, + Err(LinkError::TooLarge) => Incoming::TooLarge, + Err(err) if is_timeout(&err) => { + if hub.stop.load(Ordering::SeqCst) { + Incoming::Stop + } else { + Incoming::Again + } + } + Err(_) => Incoming::Stop, + } +} + +fn is_timeout(err: &LinkError) -> bool { + if matches!(err, LinkError::Timeout) { + return true; + } + let LinkError::Io(msg) = err else { + return false; + }; + let lower = msg.to_ascii_lowercase(); + lower.contains("timed out") || lower.contains("timeout") +} + +fn overlong() -> String { + format!("request line longer than {MAX_LINE} bytes; closing the connection") +} + +fn write_value(writer: &Arc>>, value: &Value) -> Result<(), LinkError> { + let mut guard = lock(writer); + wire::write_reply_line(&mut *guard, value) +} + +/// Retries reads that time out until `deadline`, so a record split across the read poll is whole. +struct Patient<'a, R> { + inner: &'a mut R, + deadline: Instant, + stop: &'a std::sync::atomic::AtomicBool, +} + +impl Read for Patient<'_, R> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + loop { + match self.inner.read(out) { + Err(err) if idle_accept(&err) && Instant::now() < self.deadline && !self.stop.load(Ordering::SeqCst) => {} + other => return other, + } + } + } +} + +fn bulk_loop(reader: &mut BufReader>, writer: &Arc>>, hub: &Hub, ticket: &str) { + loop { + if hub.stop.load(Ordering::SeqCst) { + return; + } + // Wait for the first byte of a request. A timeout here loses nothing. + match reader.fill_buf() { + Ok([]) => return, + Ok(_) => {} + Err(err) if idle_accept(&err) => continue, + Err(_) => return, + } + // The record has started. A pause inside it is not a new request, so keep + // reading until it ends, the server stops, or the peer takes too long. + let mut patient = Patient { inner: &mut *reader, deadline: Instant::now() + hub.limits.partial_line, stop: &hub.stop }; + let header = match wire::read_cfrq(&mut patient) { + Ok(header) => header, + Err(_) => return, + }; + match hub.frames.render(ticket, &header) { + Ok((hdr, payload)) => { + if send_frame(writer, &header, &hdr, &payload).is_err() { + return; + } + } + Err(err) => { + let fail = error_header(&header, &err.to_string()); + if write_frame(writer, &fail, &[]).is_err() { + return; + } + } + } + } +} + +/// Send one frame record. A record that cannot be built (bad header, payload over the limit) has +/// written nothing, so an error record takes its place. A failure while writing may leave part of a +/// record on the stream, and a second record would read as its tail: that error is returned alone +/// and the caller closes the connection. +fn send_frame(writer: &Arc>>, request: &Value, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + if let Err(err) = wire::check_cfrm(header, payload) { + let fail = error_header(request, &err.to_string()); + write_frame(writer, &fail, &[])?; + return Err(err); + } + write_frame(writer, header, payload) +} + +fn error_header(request: &Value, message: &str) -> Value { + let req = request.get("req").cloned().unwrap_or(Value::Null); + serde_json::json!({"req": req, "error": message}) +} + +fn write_frame(writer: &Arc>>, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + let mut guard = lock(writer); + wire::write_cfrm(&mut *guard, header, payload) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::sync::Mutex; + + use serde_json::json; + + use super::*; + + /// Takes `room` bytes, fails one write, then takes everything: a socket whose send timeout + /// fired mid-record and whose peer then drained the buffer. + struct Choked { + seen: Arc>>, + room: usize, + failed: bool, + } + + impl Write for Choked { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.room == 0 && !self.failed { + self.failed = true; + self.room = usize::MAX; + return Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "choked")); + } + let take = buf.len().min(self.room); + self.seen.lock().unwrap().extend_from_slice(&buf[..take]); + self.room = self.room.saturating_sub(take); + Ok(take) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + type Writer = Arc>>; + + fn rig(room: usize) -> (Writer, Arc>>) { + let seen = Arc::new(Mutex::new(Vec::new())); + let writer: Box = Box::new(Choked { seen: Arc::clone(&seen), room, failed: false }); + (Arc::new(Mutex::new(writer)), seen) + } + + #[test] + fn a_failed_write_is_not_followed_by_an_error_record() { + let (writer, seen) = rig(10); + let request = json!({"req": 7}); + let header = json!({"req": 7, "w": 1}); + let payload = vec![1u8; 64]; + assert!(send_frame(&writer, &request, &header, &payload).is_err()); + let mut whole = Vec::new(); + wire::write_cfrm(&mut whole, &header, &payload).unwrap(); + assert_eq!(*seen.lock().unwrap(), whole[..10].to_vec(), "only the bytes of the first record, and no error record after them"); + } + + #[test] + fn a_record_that_cannot_be_built_gets_an_error_record() { + let (writer, seen) = rig(usize::MAX); + let request = json!({"req": 7}); + // A header that is not an object is refused before any byte is written. + assert!(matches!(send_frame(&writer, &request, &json!("no"), b"x"), Err(LinkError::BadParams(_)))); + let written = seen.lock().unwrap().clone(); + let (header, payload) = wire::read_cfrm(&mut std::io::Cursor::new(written)).unwrap(); + assert_eq!(header["req"], json!(7)); + assert!(header["error"].as_str().is_some_and(|text| text.contains("object")), "{header}"); + assert!(payload.is_empty()); + } +} diff --git a/crates/ipc/src/server/mod.rs b/crates/ipc/src/server/mod.rs new file mode 100644 index 000000000..7b28543a0 --- /dev/null +++ b/crates/ipc/src/server/mod.rs @@ -0,0 +1,376 @@ +//! Craft Link server. +//! +//! One accept thread, then a read thread and a write thread per connection. +//! There is no tokio runtime here. +//! +//! Shutdown. [`Server::shutdown`] and drop set the stop flag, then join the accept thread and every +//! connection thread. A connection thread leaves its loop within one read poll (250 ms) of the +//! flag. Calls into app code differ in what they can delay that: +//! +//! - [`ControlSink`], [`Assets`] and [`Activate`] calls never delay it for longer than a stop poll +//! (at most 200 ms). The server waits for a sink reply, or for an `Assets` or `Activate` answer, in +//! short slices and gives up when the flag is set. An `Assets` or `Activate` call runs on a helper +//! thread that the server detaches when it gives up, so a provider that never returns leaves one +//! idle thread behind and delays nothing. See [`Limits::provider_deadline`]. +//! - [`Frames`] calls run on the connection thread. The flag is checked before each request, but a +//! call that is already running is waited for. A `Frames` method that never returns blocks shutdown. + +mod dispatch; +mod events; +mod listener; +mod provider; +mod tcp; + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::Value; + +use crate::asset::{Described, Refreshed, Resolved}; +use crate::discovery::{Discovery, InstanceRecord, RuntimeDir}; +use crate::error::LinkError; +use crate::moniker::Moniker; + +/// App request channel. +/// +/// `submit` returns `None` when the app is gone. The connection then closes. +/// The `Receiver` yields either a result payload or a finished reply object. +/// A payload is wrapped with [`crate::wire::reply_ok`]. An object that already +/// contains `ok` is passed through, and `id` is inserted when it is missing. +/// This signature cannot carry a [`LinkError`]. +/// A sink that never replies hits [`crate::wire::REPLY_DEADLINE_SECS`]. +pub trait ControlSink: Send + Sync { + /// None means the app is gone. The connection must close. + fn submit(&self, method: String, params: Value) -> Option>; +} + +/// What [`Activate::activate`] decided. The server serializes this as JSON +/// `true`, `false`, or the string `"requested"`. It does not rewrite the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Activation { + /// The window has focus. JSON `true`. + Yes, + /// Focus was not taken. The app asked for user attention instead. JSON `false`. + No, + /// The app asked the OS to focus the window. The OS decides later and + /// nobody confirmed it. JSON `"requested"`. + Requested, +} + +/// Raises or focuses the app. There is no default success. +/// +/// A call runs on a helper thread with a time limit of [`Limits::provider_deadline`]. On timeout the +/// caller gets a timeout reply and the server detaches the thread. The thread keeps running until +/// `activate` returns, possibly after the server is gone, so an implementation must not rely on the +/// server being alive when it returns. +pub trait Activate: Send + Sync { + fn activate(&self, token: Option, from_pid: Option, open: Option) -> Activation; +} + +/// One bulk ticket from [`Frames::open`]. `ticket` is 32 lowercase hex characters. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameOpen { + pub ticket: String, + pub w: u32, + pub h: u32, + pub stride: u32, + pub len: u64, +} + +/// Frame tickets and rendered payloads. The server registers the ticket on the +/// control connection that called `frames.open`. +/// +/// Every method runs on a connection thread, one call per frame, with no helper thread and no +/// time limit. The server checks its stop flag before it reads each request, so shutdown takes +/// effect between frames. It cannot interrupt a call that is running: [`Server::shutdown`] and +/// drop wait for it, and `close_ticket` runs when the connection ends. Each method must return in +/// bounded time, for example by putting a timeout on any I/O it does. +pub trait Frames: Send + Sync { + fn open(&self, params: &serde_json::Map) -> Result; + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError>; + fn close_ticket(&self, ticket: &str); +} + +/// Answers asset questions without a UI frame (`link.resolve`, `asset.describe`, +/// `link.refresh`). Pass one in [`Extensions::assets`]. Without it these three +/// methods go to the [`ControlSink`] like any other link method. +/// +/// Methods do not run on the UI thread, so an implementation must be thread safe +/// and must not wait on the UI thread. +/// +/// A call runs on a helper thread with a time limit of [`Limits::provider_deadline`]. On timeout the +/// caller gets a timeout reply and the server detaches the thread. A call that never returns +/// leaves one thread behind and does not hold up [`Server::shutdown`]. The server keeps at most +/// 32 such threads, and a call past that gets a `busy` error. The thread keeps running until the +/// method returns, possibly after the server is gone, so an implementation must not rely on the +/// server being alive when it returns. +pub trait Assets: Send + Sync { + /// Where the last-known result of `asset` is. `Err(LinkError::Unknown)` when + /// the asset or its result file does not exist. + fn resolve(&self, asset: &Moniker) -> Result; + /// Name, kind and thumbnail of `asset`. + fn describe(&self, asset: &Moniker) -> Result; + /// Re-check the file behind `link_id` and report its state. + fn refresh(&self, link_id: &str) -> Result; +} + +/// Optional parts for [`Server::start_with`]. The default adds nothing. +#[derive(Default)] +pub struct Extensions { + /// Answers asset methods. Adds the `assets` cap. + pub assets: Option>, + /// Extra caps for the instance record and `link.hello`, such as `hub`. + pub caps: Vec, +} + +/// What [`Server::start`] needs. `tcp_port: None` does not bind a TCP socket. +pub struct ServerConfig { + pub app: String, + pub app_version: String, + pub instance: Option, + pub runtime: Option, + pub tcp_port: Option, + pub sink: Arc, + pub activate: Arc, + pub frames: Arc, +} + +/// Time and count limits on connections. `Limits::default()` is what the apps run. +/// Tests shorten the times through [`Server::start_with_limits`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// A new connection must send a valid `auth` line within this time, or it is closed. + pub auth_deadline: Duration, + /// A request line, once started, must end within this time, or the connection is closed. + pub partial_line: Duration, + /// An authenticated control connection that sends nothing for this long is closed. A + /// connection with an event subscription and a bulk connection are exempt. + pub idle: Duration, + /// Connections that have not authenticated yet. At most `MAX_CONNECTIONS` minus this many + /// slots can be held by them. + pub max_unauthenticated: usize, + /// How long an `Assets` or `Activate` call may run before the caller gets a timeout reply and + /// the server detaches the helper thread that runs it. + pub provider_deadline: Duration, +} + +impl Default for Limits { + fn default() -> Self { + Self { + auth_deadline: Duration::from_secs(2), + partial_line: Duration::from_secs(10), + idle: Duration::from_secs(60 * 60), + max_unauthenticated: crate::wire::MAX_UNAUTHENTICATED, + provider_deadline: Duration::from_secs(crate::wire::REPLY_DEADLINE_SECS), + } + } +} + +pub(super) struct Hub { + pub(super) limits: Limits, + pub(super) pending: AtomicUsize, + pub(super) record: InstanceRecord, + pub(super) sink: Arc, + pub(super) activate: Arc, + pub(super) frames: Arc, + pub(super) assets: Option>, + pub(super) stop: AtomicBool, + pub(super) open: AtomicUsize, + /// Live `Assets` and `Activate` helper threads, stuck ones included. See `provider`. + pub(super) providers: Arc, + pub(super) next_id: AtomicU64, + pub(super) tickets: Mutex>, + pub(in crate::server) subs: Mutex>>, + pub(super) threads: Mutex>>, +} + +/// A running link endpoint. [`Server::shutdown`] removes the instance record. +/// Drop only joins the accept thread and the connection threads. A helper thread that runs a +/// stuck `Assets` or `Activate` call is detached and not joined. +pub struct Server { + discovery: Option, + hub: Arc, + accept: Option>, + tcp_port: Option, + record: InstanceRecord, +} + +impl Server { + /// Bind, publish, then accept. Scan runs before the bind so a dead socket + /// from an earlier process is unlinked. A lock that is still held is left + /// alone. The instance record keeps the local socket as its endpoint. When `tcp_port` is set the + /// record also carries the bound port (`record.tcp_port()`), and the TCP listener requires the + /// same `auth` token line as the socket. + pub fn start(config: ServerConfig) -> Result { + Self::start_inner(config, Extensions::default(), Limits::default()) + } + + /// Like [`Server::start`], plus the optional parts in [`Extensions`]. + pub fn start_with(config: ServerConfig, extensions: Extensions) -> Result { + Self::start_inner(config, extensions, Limits::default()) + } + + /// Like [`Server::start_with`], with connection limits other than the defaults. + pub fn start_with_limits(config: ServerConfig, extensions: Extensions, limits: Limits) -> Result { + Self::start_inner(config, extensions, limits) + } + + fn start_inner(config: ServerConfig, extensions: Extensions, limits: Limits) -> Result { + let runtime = match config.runtime { + Some(dir) => dir, + None => RuntimeDir::resolve()?, + }; + let _existing = Discovery::scan(&runtime)?; + let pid = std::process::id(); + #[cfg(unix)] + let socket_path = runtime.socket_file(&config.app, pid); + let listener = listener::bind(&runtime, &config.app, pid)?; + #[cfg(unix)] + if let Err(err) = crate::discovery::enforce_mode(&socket_path, 0o600) { + drop(listener); + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + let endpoint = listener::endpoint(&runtime, &config.app, pid)?; + let mut record = match InstanceRecord::new(&config.app, &config.app_version, pid, endpoint) { + Ok(record) => record, + Err(err) => { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + }; + let Extensions { assets, caps } = extensions; + let mut wanted = caps; + if assets.is_some() { + wanted.push("assets".to_string()); + } + for cap in &wanted { + if let Err(err) = record.add_cap(cap) { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + } + if let Some(instance) = config.instance { + record.instance = instance; + if let Err(err) = record.validate() { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + } + // Bind TCP before publishing so the record carries the real port (port 0 picks one). + let tcp = match config.tcp_port { + Some(port) => match tcp::bind_loopback(port) { + Ok(bound) => Some(bound), + Err(err) => { + drop(listener); + #[cfg(unix)] + let _ = std::fs::remove_file(&socket_path); + return Err(err); + } + }, + None => None, + }; + let tcp_port = tcp.as_ref().map(|bound| bound.port); + if let Some(port) = tcp_port { + record.set_tcp_port(port); + } + let discovery = match Discovery::publish(&runtime, record.clone()) { + Ok(discovery) => discovery, + Err(err) => { + drop(listener); + return Err(err); + } + }; + let hub = Arc::new(Hub { + limits, + pending: AtomicUsize::new(0), + record: record.clone(), + sink: config.sink, + activate: config.activate, + frames: config.frames, + assets, + stop: AtomicBool::new(false), + open: AtomicUsize::new(0), + providers: Arc::new(AtomicUsize::new(0)), + next_id: AtomicU64::new(1), + tickets: Mutex::new(HashMap::new()), + subs: Mutex::new(Vec::new()), + threads: Mutex::new(Vec::new()), + }); + let accept_hub = Arc::clone(&hub); + let accept = match std::thread::Builder::new().name("craft-ipc-accept".into()).spawn(move || { + listener::accept_loop(listener, tcp, accept_hub); + }) { + Ok(handle) => handle, + Err(err) => { + let _ = discovery.remove(); + return Err(err.into()); + } + }; + Ok(Self { discovery: Some(discovery), hub, accept: Some(accept), tcp_port, record }) + } + + pub fn record(&self) -> &InstanceRecord { + &self.record + } + + /// Queue `event` on subscribed connections. This returns after the enqueue, + /// not after the socket write. A full per-connection queue records one + /// `events.overflow` and then drops until that overflow line is written. + pub fn emit(&self, event: &str, data: Value) -> Result<(), LinkError> { + let subs = lock(&self.hub.subs); + for sub in subs.iter() { + events::enqueue(sub, event, &data); + } + Ok(()) + } + + /// Stop accept, join connection threads, and delete the instance record. A stuck `Assets` or + /// `Activate` call does not delay this. A running `Frames` call does. + pub fn shutdown(mut self) -> Result<(), LinkError> { + self.stop_threads(); + match self.discovery.take() { + Some(discovery) => discovery.remove(), + None => Ok(()), + } + } + + /// Bound loopback port, or `None` when TCP was not requested. + pub fn tcp_port(&self) -> Option { + self.tcp_port + } + + fn stop_threads(&mut self) { + self.hub.stop.store(true, Ordering::SeqCst); + events::wake_all(&self.hub.subs); + if let Some(accept) = self.accept.take() { + let _ = accept.join(); + } + let handles = std::mem::take(&mut *lock(&self.hub.threads)); + for handle in handles { + let _ = handle.join(); + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop_threads(); + } +} + +pub(super) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(err) => err.into_inner(), + } +} diff --git a/crates/ipc/src/server/provider.rs b/crates/ipc/src/server/provider.rs new file mode 100644 index 000000000..f74a62923 --- /dev/null +++ b/crates/ipc/src/server/provider.rs @@ -0,0 +1,71 @@ +//! Calls into app code that must not hold up shutdown. +//! +//! `Assets` and `Activate` are control plane calls, rare and slow enough to pay for a thread. Each +//! call runs on a helper thread while the connection thread waits for the answer, for at most +//! [`super::Limits::provider_deadline`] and not past the stop flag. A call that does not finish in +//! time gets a timeout reply, and the helper thread is detached: nobody joins it, so +//! [`super::Server::shutdown`] and drop do not wait for it. At most [`MAX_HELPERS`] helper +//! threads are alive at once. A new call past that gets a `busy` error until a stuck one returns. +//! +//! `Frames` is not run here. It is the hot path with one call per frame, so it stays on the +//! connection thread (see [`super::Frames`]). + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use super::Hub; +use crate::error::LinkError; +use crate::wire::MAX_CONNECTIONS; + +/// Most provider helper threads alive at once, running or stuck. Every connection can have one +/// call in flight, so this leaves room for the same number of stuck ones. +pub(super) const MAX_HELPERS: usize = 2 * MAX_CONNECTIONS; + +/// How often a waiting connection thread looks at the stop flag. +const STOP_POLL: Duration = Duration::from_millis(100); + +/// Counts one live helper thread. Dropping it, on any path, frees the slot. +struct Slot(Arc); + +impl Drop for Slot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Run `run` on a helper thread and wait for its result. +/// +/// `Err(Timeout)` when it takes longer than the deadline, `Err(Closed)` when the server stops +/// first, `Err(Busy)` when [`MAX_HELPERS`] helpers are already alive, and `Err(Io)` when the +/// helper could not start or panicked. +pub(super) fn call(hub: &Hub, what: &str, run: impl FnOnce() -> T + Send + 'static) -> Result { + if hub.providers.fetch_add(1, Ordering::SeqCst) >= MAX_HELPERS { + hub.providers.fetch_sub(1, Ordering::SeqCst); + return Err(LinkError::Busy(format!("{what}: too many provider calls are stuck"))); + } + let slot = Slot(Arc::clone(&hub.providers)); + let (tx, rx) = mpsc::channel(); + let started = std::thread::Builder::new().name("craft-ipc-provider".into()).spawn(move || { + let _slot = slot; + let _ = tx.send(run()); + }); + if let Err(err) = started { + return Err(LinkError::Io(format!("{what}: cannot start a helper thread: {err}"))); + } + let deadline = Instant::now() + hub.limits.provider_deadline; + loop { + if hub.stop.load(Ordering::SeqCst) { + return Err(LinkError::Closed); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(LinkError::Timeout); + } + match rx.recv_timeout(left.min(STOP_POLL)) { + Ok(value) => return Ok(value), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => return Err(LinkError::Io(format!("{what}: the call ended without an answer"))), + } + } +} diff --git a/crates/ipc/src/server/tcp.rs b/crates/ipc/src/server/tcp.rs new file mode 100644 index 000000000..6c1ae386b --- /dev/null +++ b/crates/ipc/src/server/tcp.rs @@ -0,0 +1,32 @@ +//! Optional loopback TCP. Off unless [`crate::server::ServerConfig::tcp_port`] is set. +//! It binds 127.0.0.1 only and runs the same token handshake, unauthenticated cap and +//! deadlines as the local socket. The 16-connection cap is shared +//! with the local listener. The instance record (0600) carries the port and the token. + +use std::net::{TcpListener, TcpStream}; + +use crate::error::LinkError; + +pub(super) struct BoundTcp { + pub listener: TcpListener, + pub port: u16, +} + +pub(super) fn bind_loopback(port: u16) -> Result { + let listener = TcpListener::bind(("127.0.0.1", port))?; + listener.set_nonblocking(true)?; + let port = listener.local_addr()?.port(); + Ok(BoundTcp { listener, port }) +} + +pub(super) fn poll_accept(listener: &TcpListener) -> std::io::Result> { + match listener.accept() { + Ok((stream, _addr)) => Ok(Some(stream)), + Err(err) if is_idle(&err) => Ok(None), + Err(err) => Err(err), + } +} + +fn is_idle(err: &std::io::Error) -> bool { + matches!(err.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted) +} diff --git a/crates/ipc/src/trust.rs b/crates/ipc/src/trust.rs new file mode 100644 index 000000000..7f57f6d5b --- /dev/null +++ b/crates/ipc/src/trust.rs @@ -0,0 +1,215 @@ +//! Checks before this crate runs a program. +//! +//! The launcher finds an app binary through `CRAFT__BIN`, `installed.json`, `PATH` and a few +//! fixed directories. A same-user process can write most of those places, so a name that resolves +//! is not proof that the file is the app. These checks do not stop a process of the same user, +//! which can do anything the user can. They stop two mistakes: running a file that any local +//! user can replace, and running a file that a relative path or an empty `PATH` entry picked from +//! the current directory. +//! +//! Group write is not checked. A file or directory that is group-writable passes, so every other +//! member of its group can replace the program. The check accepts that because most Linux +//! distributions give each user a private group with no other members, and the check cannot see +//! who is in a group. + +use std::path::Path; + +use crate::error::LinkError; + +/// `Ok` when `path` is an absolute path to a regular, executable file, owned by this user or root +/// and not writable by every user, in a directory with the same owner and mode rules. +/// +/// Symlinks are followed, and the ancestors of both the supplied and resolved paths are checked. +/// A sticky, trusted-owner directory may appear above the immediate directory: it cannot let +/// another user replace the next trusted-owner path component (as with a private folder in /tmp). +/// A group-writable file or directory passes, and the other members of its group can replace it. +/// Most Linux distributions give each user a private group, and the check cannot see group +/// members. A file or directory that every user can write fails. +/// +/// Non-Unix targets only check that the path is an absolute regular file. The Windows ACL on the +/// install directory is the user's check there. +pub fn check_executable(path: &Path) -> Result<(), LinkError> { + if !path.is_absolute() { + return Err(refused(path, "path is not absolute")); + } + #[cfg(unix)] + { + unix::check(path) + } + #[cfg(not(unix))] + { + if std::fs::metadata(path).is_ok_and(|meta| meta.is_file()) { Ok(()) } else { Err(refused(path, "not a file")) } + } +} + +fn refused(path: &Path, reason: &str) -> LinkError { + LinkError::Refused { path: path.display().to_string(), reason: reason.to_string() } +} + +#[cfg(unix)] +mod unix { + use std::os::unix::fs::MetadataExt; + use std::path::Path; + + use super::refused; + use crate::error::LinkError; + + const WORLD_WRITE: u32 = 0o002; + const ANY_EXEC: u32 = 0o111; + const STICKY: u32 = 0o1000; + + pub(super) fn check(path: &Path) -> Result<(), LinkError> { + let euid = crate::discovery::current_euid()?; + let meta = std::fs::metadata(path).map_err(|err| refused(path, &format!("cannot stat: {err}")))?; + if !meta.is_file() { + return Err(refused(path, "not a regular file")); + } + if meta.mode() & ANY_EXEC == 0 { + return Err(refused(path, "not executable")); + } + owner_and_mode(path, &meta, euid, "file")?; + let real = std::fs::canonicalize(path).map_err(|err| refused(path, &format!("cannot resolve: {err}")))?; + for candidate in [path, real.as_path()] { + // Check each named component as well as its resolved directory. In particular a + // foreign-owned symlink in a sticky ancestor can be replaced by that symlink's owner. + for component in candidate.ancestors() { + let named = std::fs::symlink_metadata(component).map_err(|err| refused(component, &format!("cannot stat: {err}")))?; + if named.file_type().is_symlink() && named.uid() != euid && named.uid() != 0 { + return Err(refused(component, "symlink is owned by another user")); + } + } + for (depth, parent) in candidate.ancestors().skip(1).enumerate() { + let dir = std::fs::metadata(parent).map_err(|err| refused(parent, &format!("cannot stat: {err}")))?; + // Sticky ancestors protect the trusted-owned child checked on the preceding + // iteration. The immediate directory still must not be world-writable. + if depth > 0 && dir.mode() & STICKY != 0 { + if dir.uid() != euid && dir.uid() != 0 { + return Err(refused(parent, "sticky ancestor is owned by another user")); + } + } else { + owner_and_mode(parent, &dir, euid, "directory")?; + } + } + } + Ok(()) + } + + fn owner_and_mode(path: &Path, meta: &std::fs::Metadata, euid: u32, what: &str) -> Result<(), LinkError> { + if meta.uid() != euid && meta.uid() != 0 { + return Err(refused(path, &format!("{what} is owned by uid {}, not uid {euid} or root", meta.uid()))); + } + if meta.mode() & WORLD_WRITE != 0 { + return Err(refused(path, &format!("{what} is writable by every user"))); + } + Ok(()) + } +} + +#[cfg(all(test, unix))] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use std::os::unix::fs::PermissionsExt; + use std::path::PathBuf; + + use super::*; + + fn scratch(label: &str) -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let path = std::env::temp_dir().join(format!("cl-trust-{label}-{}", crate::auth::encode_hex(&bytes))); + std::fs::create_dir_all(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + path + } + + fn program(dir: &Path, mode: u32) -> PathBuf { + let file = dir.join("app"); + std::fs::write(&file, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(mode)).unwrap(); + file + } + + #[test] + fn a_private_executable_passes() { + let dir = scratch("ok"); + assert!(check_executable(&program(&dir, 0o755)).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn relative_and_empty_paths_fail() { + assert!(matches!(check_executable(Path::new("app")), Err(LinkError::Refused { .. }))); + assert!(matches!(check_executable(Path::new("")), Err(LinkError::Refused { .. }))); + assert!(matches!(check_executable(Path::new("./app")), Err(LinkError::Refused { .. }))); + } + + #[test] + fn loose_modes_and_non_programs_fail() { + let dir = scratch("modes"); + let plain = program(&dir, 0o644); + assert!(check_executable(&plain).is_err(), "not executable"); + let open = program(&dir, 0o777); + assert!(check_executable(&open).is_err(), "world-writable file"); + std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&open).is_ok()); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&open).is_err(), "world-writable directory"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o1777)).unwrap(); + assert!(check_executable(&open).is_err(), "sticky does not excuse a world-writable directory"); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&dir).is_err(), "a directory is not a program"); + assert!(check_executable(&dir.join("missing")).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_symlink_into_a_loose_directory_fails() { + let good = scratch("linkgood"); + let loose = scratch("linkloose"); + let target = program(&loose, 0o755); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o777)).unwrap(); + let link = good.join("app"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + assert!(check_executable(&link).is_err()); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&link).is_ok()); + std::fs::remove_dir_all(&good).unwrap(); + std::fs::remove_dir_all(&loose).unwrap(); + } + + #[test] + fn system_programs_pass() { + assert!(check_executable(Path::new("/usr/bin/env")).is_ok() || check_executable(Path::new("/bin/sh")).is_ok()); + } + + #[test] + fn writable_ancestor_cannot_replace_a_private_program_directory() { + let ancestor = scratch("ancestor"); + let bin = ancestor.join("install/bin"); + std::fs::create_dir_all(&bin).unwrap(); + let app = program(&bin, 0o755); + assert!(check_executable(&app).is_ok()); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&app).is_err(), "a safe immediate parent does not protect its ancestors"); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o1777)).unwrap(); + assert!(check_executable(&app).is_ok(), "sticky ancestor protects the owned install directory"); + std::fs::remove_dir_all(&ancestor).unwrap(); + } + + #[test] + fn symlink_target_ancestors_are_checked_too() { + let good = scratch("ancestor-link"); + let ancestor = scratch("ancestor-target"); + let bin = ancestor.join("install/bin"); + std::fs::create_dir_all(&bin).unwrap(); + let target = program(&bin, 0o755); + let link = good.join("app"); + std::os::unix::fs::symlink(target, &link).unwrap(); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o777)).unwrap(); + assert!(check_executable(&link).is_err()); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(check_executable(&link).is_ok()); + std::fs::remove_dir_all(good).unwrap(); + std::fs::remove_dir_all(ancestor).unwrap(); + } +} diff --git a/crates/ipc/src/watch.rs b/crates/ipc/src/watch.rs new file mode 100644 index 000000000..cc765314c --- /dev/null +++ b/crates/ipc/src/watch.rs @@ -0,0 +1,397 @@ +//! Scheduled polling of linked files (`PROTOCOL.md` section 10, rule 4). +//! +//! [`crate::refresh::Tracker`] looks at the disk once per call. A [`Watcher`] +//! owns trackers and calls them on a timer: a shallow poll every +//! [`SHALLOW_INTERVAL`] (1 s) and a deep poll every [`DEEP_INTERVAL`] (30 s). +//! A deep poll replaces the shallow poll that falls on the same tick. Nothing +//! here needs the producer, so it works while the producer is offline. +//! +//! # Time +//! +//! The watcher never reads a clock. A [`Ticker`] blocks until the next tick and +//! returns the time elapsed since it started. Two tickers exist: +//! +//! - [`IntervalTicker`] waits on a condition variable with a timeout. It is the +//! native ticker. It uses no CPU between ticks and wakes at once on close. +//! - [`ManualTicker`] moves only when [`ManualTicker::advance`] is called. Tests +//! use it, so they never sleep. +//! +//! # Threads and shutdown +//! +//! [`Watcher::new`] starts one thread. Dropping the watcher closes the ticker +//! and joins the thread. A poll that is running finishes first. A panic in the +//! notice callback ends the thread, and the panic message goes to stderr. +//! +//! # Notices +//! +//! The callback runs on the watcher thread with no lock held, so it may call +//! back into the watcher. It receives [`Notice::Changed`] when a poll bumped the +//! epoch or moved the state (`current`, `modified`, `missing`), and +//! [`Notice::PollFailed`] when a poll hit a read error (rule 6). A repeated +//! identical error is reported once until a poll succeeds. An app turns +//! `Changed` into a `link.changed` event with the new epoch. +//! +//! `std` only. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use crate::error::LinkError; +use crate::refresh::{Change, FileState, Tracker}; + +/// How often a shallow poll runs (size and mtime, hash only on a difference). +pub const SHALLOW_INTERVAL: Duration = Duration::from_secs(1); +/// How often a deep poll runs (always hash). Covers coarse file system mtime. +pub const DEEP_INTERVAL: Duration = Duration::from_secs(30); + +/// Poll periods. [`Intervals::default`] is the contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Intervals { + pub shallow: Duration, + pub deep: Duration, +} + +impl Default for Intervals { + fn default() -> Self { + Self { shallow: SHALLOW_INTERVAL, deep: DEEP_INTERVAL } + } +} + +impl Intervals { + fn validate(self) -> Result<(), LinkError> { + if self.shallow.is_zero() { + return Err(LinkError::Invalid("the shallow interval must be above zero".into())); + } + if self.deep < self.shallow { + return Err(LinkError::Invalid("the deep interval must not be shorter than the shallow interval".into())); + } + Ok(()) + } +} + +/// A source of ticks for a [`Watcher`]. +pub trait Ticker: Send + Sync { + /// Block until the next tick, then return the time since the ticker started. + /// The value never decreases. Return `None` after [`Ticker::close`]. + fn wait(&self) -> Option; + /// Wake a blocked [`Ticker::wait`] and make every later call return `None`. + fn close(&self); +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// The native ticker: one tick per period on the wall clock. +/// +/// A tick that comes late skips the ticks it missed. It does not fire them in a burst. +#[derive(Debug)] +pub struct IntervalTicker { + start: Instant, + period: Duration, + state: Mutex, + wake: Condvar, +} + +#[derive(Debug)] +struct IntervalState { + /// The number of the next tick, counted from 1. + next: u32, + closed: bool, +} + +impl IntervalTicker { + /// `period` must be above zero. + pub fn new(period: Duration) -> Result { + if period.is_zero() { + return Err(LinkError::Invalid("the tick period must be above zero".into())); + } + Ok(Self { start: Instant::now(), period, state: Mutex::new(IntervalState { next: 1, closed: false }), wake: Condvar::new() }) + } +} + +impl Ticker for IntervalTicker { + fn wait(&self) -> Option { + let mut state = lock(&self.state); + loop { + if state.closed { + return None; + } + let deadline = self.start + self.period.saturating_mul(state.next); + let now = Instant::now(); + if now >= deadline { + let elapsed = now.duration_since(self.start); + let passed = elapsed.as_nanos() / self.period.as_nanos().max(1); + state.next = u32::try_from(passed.saturating_add(1)).unwrap_or(u32::MAX); + return Some(elapsed); + } + state = self.wake.wait_timeout(state, deadline - now).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } + + fn close(&self) { + lock(&self.state).closed = true; + self.wake.notify_all(); + } +} + +/// A ticker that moves only when told. For tests and for hosts with their own clock. +#[derive(Debug, Default)] +pub struct ManualTicker { + state: Mutex, + wake: Condvar, +} + +#[derive(Debug, Default)] +struct ManualState { + now: Duration, + delivered: Duration, + /// The watcher thread is blocked in `wait` and has handled every earlier tick. + idle: bool, + closed: bool, +} + +impl ManualTicker { + pub fn new() -> Self { + Self::default() + } + + /// Move time forward by `by` and wake the watcher. The watcher sees one tick + /// at the new time, however large `by` is. + pub fn advance(&self, by: Duration) { + let mut state = lock(&self.state); + let moved = state.now.saturating_add(by); + if moved > state.now { + state.now = moved; + state.idle = false; + } + drop(state); + self.wake.notify_all(); + } + + /// Block until the watcher has handled every tick so far and waits again. + /// Returns `false` when `limit` passes first or the ticker is closed. The + /// limit only guards a test from hanging. A healthy watcher settles at once. + pub fn settle(&self, limit: Duration) -> bool { + let deadline = Instant::now() + limit; + let mut state = lock(&self.state); + loop { + if state.closed { + return false; + } + if state.idle && state.delivered == state.now { + return true; + } + let now = Instant::now(); + if now >= deadline { + return false; + } + state = self.wake.wait_timeout(state, deadline - now).unwrap_or_else(std::sync::PoisonError::into_inner).0; + } + } +} + +impl Ticker for ManualTicker { + fn wait(&self) -> Option { + let mut state = lock(&self.state); + state.idle = state.delivered == state.now; + self.wake.notify_all(); + loop { + if state.closed { + return None; + } + if state.now > state.delivered { + state.delivered = state.now; + state.idle = false; + return Some(state.now); + } + state = self.wake.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + + fn close(&self) { + lock(&self.state).closed = true; + self.wake.notify_all(); + } +} + +/// What a [`Watcher`] tells its callback. +#[derive(Debug, Clone, PartialEq)] +pub enum Notice { + /// A poll bumped the epoch or moved the state of `link_id`. + Changed { link_id: String, change: Change, deep: bool }, + /// A poll could not read the file. The state did not change. + PollFailed { link_id: String, error: LinkError }, +} + +/// How many polls the watcher thread has run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PollCounts { + pub shallow: u64, + pub deep: u64, +} + +type Callback = Box; + +struct Entry { + id: String, + state: Mutex, +} + +struct EntryState { + /// `None` after `unwatch`, for a poll that was already under way. + tracker: Option, + /// The state the callback last heard about. + reported: FileState, + last_error: Option, +} + +struct Shared { + entries: Mutex>>, + notify: Callback, + shallow_polls: AtomicU64, + deep_polls: AtomicU64, +} + +/// Polls a set of trackers on a schedule. See the module docs. +pub struct Watcher { + shared: Arc, + ticker: Arc, + thread: Option>, +} + +impl Watcher { + /// Watch on the wall clock with the contract intervals. + pub fn start(notify: impl Fn(Notice) + Send + Sync + 'static) -> Result { + let ticker = Arc::new(IntervalTicker::new(SHALLOW_INTERVAL)?); + Self::new(ticker, Intervals::default(), notify) + } + + /// Watch with a chosen ticker. The ticker should tick at least as often as + /// `intervals.shallow`, or polls run at the ticker rate. + pub fn new(ticker: Arc, intervals: Intervals, notify: impl Fn(Notice) + Send + Sync + 'static) -> Result { + intervals.validate()?; + let shared = Arc::new(Shared { + entries: Mutex::new(Vec::new()), + notify: Box::new(notify), + shallow_polls: AtomicU64::new(0), + deep_polls: AtomicU64::new(0), + }); + let thread = { + let shared = Arc::clone(&shared); + let ticker = Arc::clone(&ticker); + std::thread::Builder::new().name("craft-ipc-watch".into()).spawn(move || run(&shared, ticker.as_ref(), intervals))? + }; + Ok(Self { shared, ticker, thread: Some(thread) }) + } + + /// Start polling `tracker` under `link_id`. A second tracker for the same id is an error. + pub fn watch(&self, link_id: impl Into, tracker: Tracker) -> Result<(), LinkError> { + let id = link_id.into(); + let mut entries = lock(&self.shared.entries); + if entries.iter().any(|entry| entry.id == id) { + return Err(LinkError::Invalid(format!("{id} is already watched"))); + } + let reported = tracker.state(); + entries.push(Arc::new(Entry { id, state: Mutex::new(EntryState { tracker: Some(tracker), reported, last_error: None }) })); + Ok(()) + } + + /// Stop polling `link_id` and hand its tracker back. + pub fn unwatch(&self, link_id: &str) -> Option { + let entry = { + let mut entries = lock(&self.shared.entries); + let at = entries.iter().position(|entry| entry.id == link_id)?; + entries.remove(at) + }; + // The thread may hold a clone of the entry for one more poll, which skips it now. + lock(&entry.state).tracker.take() + } + + /// Run `act` on the tracker of `link_id`: `accept`, `relink`, `adopt_epoch` + /// or a read. `None` when the id is not watched. The callback does not hear + /// about the state that `act` leaves behind. + pub fn with_tracker(&self, link_id: &str, act: impl FnOnce(&mut Tracker) -> R) -> Option { + let entry = lock(&self.shared.entries).iter().find(|entry| entry.id == link_id).cloned()?; + let mut state = lock(&entry.state); + let tracker = state.tracker.as_mut()?; + let out = act(tracker); + let now = tracker.state(); + state.reported = now; + Some(out) + } + + pub fn poll_counts(&self) -> PollCounts { + PollCounts { shallow: self.shared.shallow_polls.load(Ordering::SeqCst), deep: self.shared.deep_polls.load(Ordering::SeqCst) } + } +} + +impl Drop for Watcher { + fn drop(&mut self) { + self.ticker.close(); + if let Some(thread) = self.thread.take() { + // An Err means the callback panicked. The panic message is already on stderr. + let _ = thread.join(); + } + } +} + +/// Next due time on the grid `first, first + step, ...` that is later than `now`. +fn next_after(due: u128, step: u128, now: u128) -> u128 { + if due > now { due } else { due + ((now - due) / step + 1) * step } +} + +fn run(shared: &Shared, ticker: &dyn Ticker, intervals: Intervals) { + let shallow = intervals.shallow.as_nanos(); + let deep = intervals.deep.as_nanos(); + let mut due_shallow = shallow; + let mut due_deep = deep; + while let Some(elapsed) = ticker.wait() { + let now = elapsed.as_nanos(); + if now >= due_deep { + due_deep = next_after(due_deep, deep, now); + due_shallow = next_after(due_shallow, shallow, now); + shared.deep_polls.fetch_add(1, Ordering::SeqCst); + poll_all(shared, true); + } else if now >= due_shallow { + due_shallow = next_after(due_shallow, shallow, now); + shared.shallow_polls.fetch_add(1, Ordering::SeqCst); + poll_all(shared, false); + } + } +} + +fn poll_all(shared: &Shared, deep: bool) { + let entries: Vec> = lock(&shared.entries).clone(); + for entry in entries { + let notice = poll_entry(&entry, deep); + if let Some(notice) = notice { + (shared.notify)(notice); + } + } +} + +fn poll_entry(entry: &Entry, deep: bool) -> Option { + let mut state = lock(&entry.state); + let polled = state.tracker.as_mut()?.poll(deep); + match polled { + Ok(change) => { + state.last_error = None; + if change.bumped || change.state != state.reported { + state.reported = change.state; + Some(Notice::Changed { link_id: entry.id.clone(), change, deep }) + } else { + None + } + } + Err(error) => { + if state.last_error.as_ref() == Some(&error) { + return None; + } + state.last_error = Some(error.clone()); + Some(Notice::PollFailed { link_id: entry.id.clone(), error }) + } + } +} diff --git a/crates/ipc/src/wire.rs b/crates/ipc/src/wire.rs new file mode 100644 index 000000000..f3dc76621 --- /dev/null +++ b/crates/ipc/src/wire.rs @@ -0,0 +1,561 @@ +//! JSON lines and bulk frame records. No sockets live here. + +use std::io::{BufRead, Read, Write}; +use std::time::{Duration, Instant}; + +use serde_json::{Map, Value, json}; + +use crate::error::LinkError; + +/// Longest accepted control line, without the newline. Matches the app control servers. +pub const MAX_LINE: usize = 4 * 1024 * 1024; +/// Open connections per server, authenticated or not. Bulk connections count. +pub const MAX_CONNECTIONS: usize = 16; +/// Connections that have not authenticated yet. The other `MAX_CONNECTIONS - this` slots are +/// reserved for authenticated peers. +pub const MAX_UNAUTHENTICATED: usize = 4; +pub const EVENT_QUEUE: usize = 256; +pub const REPLY_DEADLINE_SECS: u64 = 60; +/// `link.open` payload ceiling. The control line cap is the real limit: a longer request line +/// closes the connection. This number is the same, so `OpenPayload` gives `payload_too_large` +/// to a caller that does not come through the line reader. +pub const MAX_LINK_PAYLOAD: u64 = MAX_LINE as u64; +pub const BULK_HEADER_MAX: u32 = 1024 * 1024; +pub const FRAME_PAYLOAD_MAX: u64 = 256 * 1024 * 1024; +pub const PROTOCOL_MIN: u32 = 1; +pub const PROTOCOL_MAX: u32 = 1; +pub const AUTH_ERROR: &str = "authentication required"; + +const CFRQ: &[u8; 4] = b"CFRQ"; +const CFRM: &[u8; 4] = b"CFRM"; + +/// One control request. `id` is JSON null when the caller omitted it. +#[derive(Debug, Clone, PartialEq)] +pub struct Request { + pub id: Value, + pub method: String, + pub params: Value, +} + +/// Read one line of at most `max` bytes. `Ok(None)` is a clean end of stream. +/// +/// This keeps no state between calls. A read timeout drops the bytes read so far, so use it only +/// on a stream that cannot time out. A server loop that polls with a receive timeout uses +/// [`LineReader`]. +pub fn read_line(reader: &mut impl BufRead, max: usize) -> Result>, LinkError> { + LineReader::new(max).read(reader) +} + +/// Reads lines from a stream whose reads can time out. Bytes read before a timeout stay in the +/// reader, so a line that arrives in pieces more than one poll apart is not lost. +#[derive(Debug)] +pub struct LineReader { + buf: Vec, + max: usize, + started: Option, + patience: Option, +} + +impl LineReader { + pub fn new(max: usize) -> Self { + Self { buf: Vec::new(), max, started: None, patience: None } + } + + /// How long the current unfinished line has been open. `None` when no line is in progress. + pub fn partial_age(&self) -> Option { + self.started.map(|start| start.elapsed()) + } + + /// Return `Err(Timeout)` once a line has been open this long, even when bytes keep arriving. + /// Without it a peer that sends one byte per poll holds the connection for ever. + pub fn with_patience(mut self, patience: Duration) -> Self { + self.patience = Some(patience); + self + } + + /// Change the limit that [`LineReader::with_patience`] sets. + pub fn set_patience(&mut self, patience: Duration) { + self.patience = Some(patience); + } + + /// Read the next line. On `Err(Timeout)` the partial line stays here and the next call + /// continues it. `Ok(None)` is a clean end of stream with no partial line. + pub fn read(&mut self, reader: &mut impl BufRead) -> Result>, LinkError> { + loop { + let available = match reader.fill_buf() { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) => return Err(err.into()), + }; + if available.is_empty() { + // End of stream. A final line without a newline is still a line. + if self.buf.is_empty() { + return Ok(None); + } + let line = self.finish(); + if line.len() > self.max { + return Err(LinkError::TooLarge); + } + return Ok(Some(line)); + } + let newline = available.iter().position(|byte| *byte == b'\n'); + let take = newline.map_or(available.len(), |at| at + 1); + if self.buf.len().saturating_add(take) > self.max.saturating_add(1) { + return Err(LinkError::TooLarge); + } + self.buf.extend_from_slice(&available[..take]); + reader.consume(take); + if self.started.is_none() { + self.started = Some(Instant::now()); + } + if newline.is_some() { + let mut line = self.finish(); + line.pop(); + if line.last() == Some(&b'\r') { + line.pop(); + } + if line.len() > self.max { + return Err(LinkError::TooLarge); + } + return Ok(Some(line)); + } + if self.patience.zip(self.partial_age()).is_some_and(|(limit, age)| age >= limit) { + return Err(LinkError::Timeout); + } + } + } + + fn finish(&mut self) -> Vec { + self.started = None; + std::mem::take(&mut self.buf) + } +} + +/// Parse one control line. +/// +/// `Ok(None)` is a blank line (the caller skips it). `Err` is the existing +/// control-server reject text: one reply, then close. +pub fn parse_request(bytes: &[u8]) -> Result, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "not a control request (invalid UTF-8); closing the connection".to_string())?; + let text = text.trim(); + if text.is_empty() { + return Ok(None); + } + let msg: Value = + serde_json::from_str(text).map_err(|err| format!("bad JSON ({err}); expected one JSON request object per line; closing the connection"))?; + let Some(obj) = msg.as_object() else { + return Err("not a control request (expected a JSON object); closing the connection".into()); + }; + let Some(method) = obj.get("method").and_then(Value::as_str) else { + return Err("not a control request (missing string `method`); closing the connection".into()); + }; + if method.is_empty() { + return Err("not a control request (missing string `method`); closing the connection".into()); + } + let id = obj.get("id").cloned().unwrap_or(Value::Null); + if !(id.is_null() || id.is_number() || id.is_string()) { + return Err("not a control request (id must be a number, string, or null); closing the connection".into()); + } + let params = obj.get("params").cloned().unwrap_or_else(|| json!({})); + Ok(Some(Request { id, method: method.to_string(), params })) +} + +pub fn params_object(params: &Value) -> Result<&Map, LinkError> { + params.as_object().ok_or_else(|| LinkError::BadParams("params must be an object".into())) +} + +pub fn auth_failure() -> Value { + json!({"id": 0, "ok": false, "error": AUTH_ERROR}) +} + +pub fn auth_success(id: &Value) -> Value { + let id = if id.is_null() { json!(0) } else { id.clone() }; + json!({"id": id, "ok": true, "result": {"authenticated": true}}) +} + +pub fn reply_ok(id: &Value, result: Value) -> Value { + json!({"id": id, "ok": true, "result": result}) +} + +pub fn reply_err_text(id: &Value, error: &str) -> Value { + json!({"id": id, "ok": false, "error": error}) +} + +pub fn reply_err(id: &Value, err: &LinkError) -> Value { + json!({"id": id, "ok": false, "error": err.to_object()}) +} + +pub fn push_event(event: &str, seq: u64, data: Value) -> Value { + json!({"event": event, "seq": seq, "data": data}) +} + +/// Write a reply line that a peer will read with the [`MAX_LINE`] limit. A value longer than the +/// limit is replaced by a `payload_too_large` reply that keeps the request `id`. When even that +/// does not fit, or the value has no `id` to answer, nothing is written and the result is +/// `Err(TooLarge)`. The limit counts the content without the newline, as the readers do. +pub fn write_reply_line(writer: &mut impl Write, value: &Value) -> Result<(), LinkError> { + let mut bytes = serde_json::to_vec(value)?; + if bytes.len() > MAX_LINE { + let id = value.get("id").ok_or(LinkError::TooLarge)?; + bytes = serde_json::to_vec(&reply_err(id, &LinkError::TooLarge))?; + if bytes.len() > MAX_LINE { + return Err(LinkError::TooLarge); + } + } + bytes.push(b'\n'); + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +pub fn write_line(writer: &mut impl Write, value: &Value) -> Result<(), LinkError> { + let mut bytes = serde_json::to_vec(value)?; + bytes.push(b'\n'); + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +pub fn protocols_overlap(min: u32, max: u32) -> bool { + min <= max && min <= PROTOCOL_MAX && max >= PROTOCOL_MIN +} + +pub fn write_cfrq(writer: &mut impl Write, header: &Value) -> Result<(), LinkError> { + write_magic(writer, CFRQ, header)?; + Ok(()) +} + +pub fn read_cfrq(reader: &mut impl Read) -> Result { + read_magic(reader, CFRQ) +} + +/// The checks that [`write_cfrm`] makes before it writes a byte: the header is an object that fits +/// `BULK_HEADER_MAX`, and the payload fits `FRAME_PAYLOAD_MAX`. A record that fails them has put +/// nothing on the wire, so a caller may still send a different record. An error from `write_cfrm` +/// after this check passed came from the stream and may have left part of a record on it. +pub fn check_cfrm(header: &Value, payload: &[u8]) -> Result<(), LinkError> { + cfrm_parts(header, payload).map(|_| ()) +} + +fn cfrm_parts(header: &Value, payload: &[u8]) -> Result<(Vec, u64), LinkError> { + let header_bytes = header_bytes(header)?; + let payload_len = u64::try_from(payload.len()).map_err(|_| LinkError::TooLarge)?; + if payload_len > FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + Ok((header_bytes, payload_len)) +} + +pub fn write_cfrm(writer: &mut impl Write, header: &Value, payload: &[u8]) -> Result<(), LinkError> { + let (header_bytes, payload_len) = cfrm_parts(header, payload)?; + writer.write_all(CFRM)?; + writer.write_all(&u32_len(&header_bytes)?.to_le_bytes())?; + writer.write_all(&payload_len.to_le_bytes())?; + writer.write_all(&header_bytes)?; + writer.write_all(payload)?; + writer.flush()?; + Ok(()) +} + +pub fn read_cfrm(reader: &mut impl Read) -> Result<(Value, Vec), LinkError> { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if &magic != CFRM { + return Err(LinkError::Invalid("bulk reply magic".into())); + } + let header_len = read_u32(reader)?; + if header_len > BULK_HEADER_MAX { + return Err(LinkError::TooLarge); + } + let payload_len = read_u64(reader)?; + if payload_len > FRAME_PAYLOAD_MAX { + return Err(LinkError::TooLarge); + } + let header = read_header(reader, header_len)?; + let mut payload = vec![0u8; usize::try_from(payload_len).map_err(|_| LinkError::TooLarge)?]; + reader.read_exact(&mut payload)?; + Ok((header, payload)) +} + +fn write_magic(writer: &mut impl Write, magic: &[u8; 4], header: &Value) -> Result<(), LinkError> { + let bytes = header_bytes(header)?; + writer.write_all(magic)?; + writer.write_all(&u32_len(&bytes)?.to_le_bytes())?; + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +} + +fn read_magic(reader: &mut impl Read, expect: &[u8; 4]) -> Result { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if &magic != expect { + return Err(LinkError::Invalid("bulk request magic".into())); + } + let len = read_u32(reader)?; + if len > BULK_HEADER_MAX { + return Err(LinkError::TooLarge); + } + read_header(reader, len) +} + +fn header_bytes(header: &Value) -> Result, LinkError> { + if !header.is_object() { + return Err(LinkError::BadParams("bulk header must be an object".into())); + } + let bytes = serde_json::to_vec(header)?; + if bytes.len() > usize::try_from(BULK_HEADER_MAX).unwrap_or(usize::MAX) { + return Err(LinkError::TooLarge); + } + Ok(bytes) +} + +fn u32_len(bytes: &[u8]) -> Result { + u32::try_from(bytes.len()).map_err(|_| LinkError::TooLarge) +} + +fn read_u32(reader: &mut impl Read) -> Result { + let mut buf = [0u8; 4]; + reader.read_exact(&mut buf)?; + Ok(u32::from_le_bytes(buf)) +} + +fn read_u64(reader: &mut impl Read) -> Result { + let mut buf = [0u8; 8]; + reader.read_exact(&mut buf)?; + Ok(u64::from_le_bytes(buf)) +} + +fn read_header(reader: &mut impl Read, len: u32) -> Result { + let mut buf = vec![0u8; len as usize]; + reader.read_exact(&mut buf)?; + let value: Value = serde_json::from_slice(&buf)?; + if !value.is_object() { + return Err(LinkError::BadParams("bulk header must be an object".into())); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::*; + + #[test] + fn blank_and_request_round_trip() { + assert!(parse_request(b" \n").unwrap().is_none()); + let req = parse_request(br#"{"id":1,"method":"engine.execute","params":{"command":"x"}}"#).unwrap().unwrap(); + assert_eq!(req.method, "engine.execute"); + assert_eq!(req.id, json!(1)); + assert!(parse_request(b"[]").is_err()); + assert!(parse_request(b"{").is_err()); + } + + #[test] + fn a_reply_over_the_line_limit_becomes_a_too_large_error_with_the_same_id() { + let mut out = Vec::new(); + let small = reply_ok(&json!(3), json!({"a": 1})); + write_reply_line(&mut out, &small).unwrap(); + assert_eq!(out, [serde_json::to_vec(&small).unwrap(), b"\n".to_vec()].concat(), "a reply inside the limit is written as it is"); + + // The content is exactly MAX_LINE bytes: still inside. + let overhead = serde_json::to_vec(&reply_ok(&json!(3), json!(""))).unwrap().len(); + let exact = reply_ok(&json!(3), json!("x".repeat(MAX_LINE - overhead))); + assert_eq!(serde_json::to_vec(&exact).unwrap().len(), MAX_LINE); + let mut out = Vec::new(); + write_reply_line(&mut out, &exact).unwrap(); + assert_eq!(out.len(), MAX_LINE + 1); + + let big = reply_ok(&json!(3), json!("x".repeat(MAX_LINE - overhead + 1))); + let mut out = Vec::new(); + write_reply_line(&mut out, &big).unwrap(); + let line = read_line(&mut Cursor::new(out), MAX_LINE).unwrap().unwrap(); + let reply: Value = serde_json::from_slice(&line).unwrap(); + assert_eq!(reply["id"], json!(3)); + assert_eq!(reply["ok"], json!(false)); + assert_eq!(reply["error"]["code"], json!("payload_too_large")); + + // No id to answer: nothing is written. + let mut out = Vec::new(); + let anonymous = json!({"ok": true, "result": "x".repeat(MAX_LINE)}); + assert!(matches!(write_reply_line(&mut out, &anonymous), Err(LinkError::TooLarge))); + assert!(out.is_empty()); + // An id so long that the error reply does not fit either. + let huge_id = json!({"id": "i".repeat(MAX_LINE + 1), "ok": true}); + assert!(matches!(write_reply_line(&mut out, &huge_id), Err(LinkError::TooLarge))); + assert!(out.is_empty()); + } + + #[test] + fn a_last_line_without_a_newline_obeys_the_limit() { + let at_limit = vec![b'a'; 8]; + let line = read_line(&mut Cursor::new(at_limit.clone()), 8).unwrap().unwrap(); + assert_eq!(line, at_limit); + let over = vec![b'a'; 9]; + assert!(matches!(read_line(&mut Cursor::new(over), 8), Err(LinkError::TooLarge))); + // The same bytes with a newline are refused too, so the two paths agree. + let mut with_newline = vec![b'a'; 9]; + with_newline.push(b'\n'); + assert!(matches!(read_line(&mut Cursor::new(with_newline), 8), Err(LinkError::TooLarge))); + } + + #[test] + fn line_limit_and_crlf() { + let mut cur = Cursor::new(b"abc\r\n".to_vec()); + let line = read_line(&mut cur, 8).unwrap().unwrap(); + assert_eq!(line, b"abc"); + let mut huge = vec![b'a'; 20]; + huge.push(b'\n'); + let mut cur = Cursor::new(huge); + assert!(matches!(read_line(&mut cur, 8), Err(LinkError::TooLarge))); + } + + /// A reader that hands out `chunks` and fails with a timeout after each one, like a socket + /// with a receive timeout and a slow peer. + struct Stuttering { + chunks: std::collections::VecDeque>, + pending_timeout: bool, + current: Vec, + pos: usize, + } + + impl Stuttering { + fn new(chunks: Vec>) -> Self { + Self { chunks: chunks.into(), pending_timeout: false, current: Vec::new(), pos: 0 } + } + } + + impl Read for Stuttering { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.pos >= self.current.len() { + if self.pending_timeout { + self.pending_timeout = false; + return Err(std::io::ErrorKind::WouldBlock.into()); + } + let Some(next) = self.chunks.pop_front() else { return Ok(0) }; + self.current = next; + self.pos = 0; + if self.current.is_empty() { + return Err(std::io::ErrorKind::WouldBlock.into()); + } + self.pending_timeout = true; + } + let n = out.len().min(self.current.len() - self.pos); + out[..n].copy_from_slice(&self.current[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } + + fn collect_lines(chunks: Vec>, max: usize) -> Vec> { + let mut reader = std::io::BufReader::with_capacity(4, Stuttering::new(chunks)); + let mut lines = LineReader::new(max); + let mut got = Vec::new(); + loop { + match lines.read(&mut reader) { + Ok(Some(line)) => got.push(line), + Ok(None) => return got, + Err(LinkError::Timeout) => {} + Err(other) => panic!("{other:?}"), + } + } + } + + #[test] + fn a_line_split_at_every_offset_survives_a_timeout() { + let text = b"{\"id\":1,\"method\":\"link.hello\"}\r\n{\"id\":2}\n"; + let want = vec![b"{\"id\":1,\"method\":\"link.hello\"}".to_vec(), b"{\"id\":2}".to_vec()]; + for cut in 0..=text.len() { + let chunks = vec![text[..cut].to_vec(), text[cut..].to_vec()]; + assert_eq!(collect_lines(chunks, 1024), want, "cut {cut}"); + } + // Two cuts, and one byte at a time. + for a in 0..text.len() { + for b in a..text.len() { + let chunks = vec![text[..a].to_vec(), text[a..b].to_vec(), text[b..].to_vec()]; + assert_eq!(collect_lines(chunks, 1024), want, "cuts {a} {b}"); + } + } + let bytes: Vec> = text.iter().map(|byte| vec![*byte]).collect(); + assert_eq!(collect_lines(bytes, 1024), want); + } + + #[test] + fn the_line_cap_holds_across_timeouts() { + let mut reader = std::io::BufReader::new(Stuttering::new(vec![vec![b'a'; 6], vec![b'b'; 6], b"\n".to_vec()])); + let mut lines = LineReader::new(8); + let mut result = None; + for _ in 0..8 { + match lines.read(&mut reader) { + Err(LinkError::Timeout) => {} + other => { + result = Some(other); + break; + } + } + } + assert!(matches!(result, Some(Err(LinkError::TooLarge))), "{result:?}"); + } + + /// One byte per read, a short pause each, and never a newline: a slow-loris peer whose reads + /// never time out. + struct Trickle; + + impl Read for Trickle { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + std::thread::sleep(Duration::from_millis(10)); + out[0] = b'a'; + Ok(1) + } + } + + #[test] + fn a_trickling_line_is_cut_by_patience() { + let mut reader = std::io::BufReader::with_capacity(1, Trickle); + let mut lines = LineReader::new(MAX_LINE).with_patience(Duration::from_millis(100)); + let start = Instant::now(); + assert!(matches!(lines.read(&mut reader), Err(LinkError::Timeout))); + assert!(start.elapsed() < Duration::from_secs(2), "{:?}", start.elapsed()); + assert!(lines.partial_age().is_some_and(|age| age >= Duration::from_millis(100))); + } + + #[test] + fn partial_age_tracks_an_open_line() { + let mut reader = std::io::BufReader::new(Stuttering::new(vec![b"abc".to_vec(), b"d\n".to_vec()])); + let mut lines = LineReader::new(64); + assert!(lines.partial_age().is_none()); + assert!(matches!(lines.read(&mut reader), Err(LinkError::Timeout))); + assert!(lines.partial_age().is_some()); + assert_eq!(lines.read(&mut reader).unwrap().unwrap(), b"abcd"); + assert!(lines.partial_age().is_none()); + } + + #[test] + fn bulk_round_trip() { + let header = json!({"req": 7, "time_s": 1.5, "epoch": 3, "region": null}); + let mut buf = Vec::new(); + write_cfrq(&mut buf, &header).unwrap(); + let got = read_cfrq(&mut Cursor::new(&buf)).unwrap(); + assert_eq!(got["req"], json!(7)); + + let mut buf = Vec::new(); + let reply = json!({"req": 7, "error": "no"}); + write_cfrm(&mut buf, &reply, b"").unwrap(); + let (h, payload) = read_cfrm(&mut Cursor::new(&buf)).unwrap(); + assert_eq!(h["error"], json!("no")); + assert!(payload.is_empty()); + + let mut buf = Vec::new(); + write_cfrm(&mut buf, &json!({"req": 1, "w": 2}), &[9, 8, 7]).unwrap(); + let (_h, payload) = read_cfrm(&mut Cursor::new(buf)).unwrap(); + assert_eq!(payload, vec![9, 8, 7]); + } + + #[test] + fn protocol_window() { + assert!(protocols_overlap(1, 1)); + assert!(!protocols_overlap(2, 2)); + assert!(!protocols_overlap(2, 1)); + } +} diff --git a/crates/ipc/tests/bulk.rs b/crates/ipc/tests/bulk.rs new file mode 100644 index 000000000..8cd8295a2 --- /dev/null +++ b/crates/ipc/tests/bulk.rs @@ -0,0 +1,334 @@ +//! Bulk frames over a real local socket with a fake `FrameSource`. +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use craft_ipc::bulk::{BulkClient, FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; +use craft_ipc::client::Client; +use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; +use craft_ipc::wire; +use craft_ipc::{Endpoint, InstanceRecord, LinkError, RuntimeDir}; + +const W: u32 = 4; +const H: u32 = 3; +const STRIDE: u32 = 16; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +/// Pixels encode the request so a wrong frame cannot match by accident. +struct FakeSource { + epoch: AtomicU64, + closed: Mutex>, +} + +/// One row holds `W` pixels: 4 bytes each in rgba8, 8 bytes each in rgba16f. +fn stride_for(format: FrameFormat) -> u32 { + match format { + FrameFormat::Rgba8Premul => STRIDE, + FrameFormat::Rgba16fPremul => W * 8, + } +} + +fn pixels_for(link_id: &str, time_s: f64, region: Option<[u32; 4]>) -> Vec { + let seed = + link_id.bytes().fold(0u8, |acc, b| acc.wrapping_add(b)).wrapping_add((time_s * 10.0) as u8).wrapping_add(region.map_or(0, |r| r[0] as u8)); + (0..(STRIDE * H) as usize).map(|i| seed.wrapping_add((i as u8).wrapping_mul(7))).collect() +} + +impl FrameSource for FakeSource { + fn open(&self, spec: &FrameSpec) -> Result { + if spec.link_id == "missing" { + return Err(LinkError::NotReady("link_unknown".into())); + } + let stride = stride_for(spec.format); + Ok(FrameLayout { w: W, h: H, stride, len: u64::from(stride * H) }) + } + + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { + let stride = stride_for(spec.format); + let mut pixels = pixels_for(&spec.link_id, request.time_s, request.region); + pixels.resize((stride * H) as usize, 0); + Ok(RenderedFrame { epoch: self.epoch.load(Ordering::SeqCst), w: W, h: H, stride, format: spec.format, render_ms: 0.25, pixels }) + } + + fn close(&self, spec: &FrameSpec) { + self.closed.lock().unwrap().push(spec.link_id.clone()); + } +} + +struct Rig { + _tmp: Tmp, + server: Server, + record: InstanceRecord, + source: Arc, + tickets: Arc, +} + +fn boot(epoch: u64) -> Rig { + // Parallel tests can observe the same clock tick on macOS. + static NEXT_RIG: AtomicU64 = AtomicU64::new(0); + let serial = NEXT_RIG.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-blk-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let source = Arc::new(FakeSource { epoch: AtomicU64::new(epoch), closed: Mutex::new(Vec::new()) }); + let tickets = Arc::new(FrameTickets::new(Arc::clone(&source) as Arc)); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(RuntimeDir::at(&path).unwrap()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: Arc::clone(&tickets) as Arc, + }) + .unwrap(); + let record = server.record().clone(); + Rig { _tmp: Tmp(path), server, record, source, tickets } +} + +fn wait_for(what: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while !done() { + assert!(Instant::now() < deadline, "timed out waiting for {what}"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn socket_of(record: &InstanceRecord) -> PathBuf { + match &record.endpoint { + Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + } +} + +#[test] +fn round_trip_returns_the_exact_bytes() { + let rig = boot(3); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + assert_eq!((ticket.w, ticket.h, ticket.stride, ticket.len), (W, H, STRIDE, u64::from(STRIDE * H))); + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + let whole = bulk.request(1.5, 3, None).unwrap(); + assert_eq!(whole.pixels, pixels_for("link-a", 1.5, None)); + assert_eq!((whole.req, whole.epoch, whole.w, whole.h, whole.stride), (1, 3, W, H, STRIDE)); + assert_eq!(whole.format, FrameFormat::Rgba8Premul); + assert!((whole.render_ms - 0.25).abs() < f64::EPSILON); + let part = bulk.request(2.0, 3, Some([5, 0, 2, 2])).unwrap(); + assert_eq!(part.pixels, pixels_for("link-a", 2.0, Some([5, 0, 2, 2]))); + assert_ne!(part.pixels, whole.pixels); + assert_eq!(part.req, 2); + drop(bulk); + drop(control); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_newer_epoch_is_returned_and_an_older_source_is_an_error() { + let rig = boot(7); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba16fPremul, 2.0).unwrap(); + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + let frame = bulk.request(0.0, 2, None).unwrap(); + assert_eq!(frame.epoch, 7, "asked for epoch 2, source is at 7"); + assert_eq!(frame.format, FrameFormat::Rgba16fPremul); + let err = bulk.request(0.0, 9, None).unwrap_err(); + assert!(err.to_string().contains("epoch 7"), "{err}"); + // The error did not poison the connection. + rig.source.epoch.store(9, Ordering::SeqCst); + assert_eq!(bulk.request(0.0, 9, None).unwrap().epoch, 9); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_ticket_is_refused_after_its_control_connection_closes() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut live = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + live.request(0.0, 1, None).unwrap(); + assert_eq!(rig.tickets.open_tickets(), 1); + drop(control); + wait_for("source.close", || !rig.source.closed.lock().unwrap().is_empty()); + assert_eq!(rig.tickets.open_tickets(), 0); + assert!(live.request(0.0, 1, None).is_err(), "an attached connection must lose its frames"); + let again = BulkClient::attach(&rig.record, &ticket.ticket); + assert!(again.is_err(), "reuse after control close must be refused"); + rig.server.shutdown().unwrap(); +} + +#[test] +fn bulk_attach_must_be_first_and_the_ticket_must_exist() { + let rig = boot(1); + let bogus = "0123456789abcdef0123456789abcdef"; + assert!(BulkClient::attach(&rig.record, bogus).is_err()); + let mut control = Client::connect(&rig.record).unwrap(); + assert!(control.frames_open("missing", FrameFormat::Rgba8Premul, 1.0).is_err()); + assert!(control.frames_open("a", FrameFormat::Rgba8Premul, 0.0).is_err()); + assert_eq!(rig.tickets.open_tickets(), 0); + rig.server.shutdown().unwrap(); +} + +fn raw_attach(rig: &Rig, ticket: &str) -> UnixStream { + let mut stream = UnixStream::connect(socket_of(&rig.record)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + wire::write_line(&mut stream, &json!({"id": 0, "method": "auth", "params": {"token": rig.record.token}})).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &json!({"id": 1, "method": "bulk.attach", "params": {"ticket": ticket}})).unwrap(); + let reply: Value = serde_json::from_slice(&read_line(&mut stream)).unwrap(); + assert_eq!(reply["ok"], json!(true), "{reply}"); + stream +} + +fn read_line(stream: &mut UnixStream) -> Vec { + let mut out = Vec::new(); + let mut byte = [0u8; 1]; + while stream.read_exact(&mut byte).is_ok() && byte[0] != b'\n' { + out.push(byte[0]); + } + out +} + +#[test] +fn an_oversized_request_header_closes_that_connection_only() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut raw = raw_attach(&rig, &ticket.ticket); + raw.write_all(b"CFRQ").unwrap(); + raw.write_all(&(wire::BULK_HEADER_MAX + 1).to_le_bytes()).unwrap(); + let mut rest = Vec::new(); + // The server closes without a reply, so the read ends at EOF instead of timing out. + let n = raw.read_to_end(&mut rest).unwrap(); + assert_eq!(n, 0, "no reply expected for an oversized header"); + // The control connection and a fresh bulk connection still work. + let mut bulk = BulkClient::attach(&rig.record, &ticket.ticket).unwrap(); + assert!(bulk.request(0.0, 1, None).is_ok()); + rig.server.shutdown().unwrap(); +} + +#[test] +fn a_bad_magic_and_a_bad_header_do_not_panic_the_server() { + let rig = boot(1); + let mut control = Client::connect(&rig.record).unwrap(); + let ticket = control.frames_open("link-a", FrameFormat::Rgba8Premul, 1.0).unwrap(); + let mut raw = raw_attach(&rig, &ticket.ticket); + raw.write_all(b"XXXX\0\0\0\0").unwrap(); + let mut rest = Vec::new(); + assert_eq!(raw.read_to_end(&mut rest).unwrap(), 0); + // A well-formed record with a bad header gets an error record, not a close. + let mut raw = raw_attach(&rig, &ticket.ticket); + wire::write_cfrq(&mut raw, &json!({"req": 5, "epoch": 0})).unwrap(); + let (header, payload) = wire::read_cfrm(&mut raw).unwrap(); + assert_eq!(header["req"], json!(5)); + assert!(header["error"].as_str().unwrap().contains("time_s"), "{header}"); + assert!(payload.is_empty()); + rig.server.shutdown().unwrap(); +} + +/// A one-shot fake server. Answers auth and attach, then sends `bytes` and closes. +fn fake_server(bytes: Vec) -> (InstanceRecord, std::thread::JoinHandle<()>, Tmp) { + static NEXT_FAKE: AtomicU64 = AtomicU64::new(0); + let serial = NEXT_FAKE.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("cl-fk-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("s.sock"); + let listener = UnixListener::bind(&path).unwrap(); + let record = InstanceRecord::new("fake", "0", std::process::id(), Endpoint::Unix { path: path.to_string_lossy().into_owned() }).unwrap(); + let handle = std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { return }; + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &wire::auth_success(&json!(0))).unwrap(); + read_line(&mut stream); + wire::write_line(&mut stream, &wire::reply_ok(&json!(1), json!({}))).unwrap(); + let mut request = [0u8; 8]; + stream.read_exact(&mut request).unwrap(); + let mut header = vec![0u8; u32::from_le_bytes([request[4], request[5], request[6], request[7]]) as usize]; + stream.read_exact(&mut header).unwrap(); + stream.write_all(&bytes).unwrap(); + }); + (record, handle, Tmp(dir)) +} + +fn cfrm(header: &Value, declared_payload: u64, payload: &[u8]) -> Vec { + let header = serde_json::to_vec(header).unwrap(); + let mut out = b"CFRM".to_vec(); + out.extend((header.len() as u32).to_le_bytes()); + out.extend(declared_payload.to_le_bytes()); + out.extend(header); + out.extend(payload); + out +} + +#[test] +fn truncated_replies_are_errors_not_panics() { + let good_header = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + let full = cfrm(&good_header, 48, &[7u8; 48]); + for cut in [2, 6, 10, 20, full.len() - 1] { + let (record, handle, _tmp) = fake_server(full[..cut].to_vec()); + let mut bulk = BulkClient::attach(&record, &"a".repeat(32)).unwrap(); + let err = bulk.request(0.0, 0, None).unwrap_err(); + assert!(matches!(err, LinkError::Io(_)), "cut {cut}: {err:?}"); + handle.join().unwrap(); + } + // The uncut record is accepted by the same fake, so the cuts above are the cause. + let (record, handle, _tmp) = fake_server(full); + let mut bulk = BulkClient::attach(&record, &"a".repeat(32)).unwrap(); + assert_eq!(bulk.request(0.0, 0, None).unwrap().pixels, vec![7u8; 48]); + handle.join().unwrap(); +} + +#[test] +fn oversized_or_inconsistent_replies_are_rejected_by_the_client() { + let good = json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}); + let mut huge_header = b"CFRM".to_vec(); + huge_header.extend((wire::BULK_HEADER_MAX + 1).to_le_bytes()); + huge_header.extend(0u64.to_le_bytes()); + let huge_payload = cfrm(&good, wire::FRAME_PAYLOAD_MAX + 1, &[]); + let short_payload = cfrm(&good, 47, &[1u8; 47]); + let narrow = cfrm(&json!({"req": 1, "epoch": 1, "w": 4, "h": 3, "stride": 1, "format": "rgba8_premul", "render_ms": 1.0}), 3, &[1u8; 3]); + let wrong_req = cfrm(&json!({"req": 99, "epoch": 1, "w": 4, "h": 3, "stride": 16, "format": "rgba8_premul", "render_ms": 1.0}), 48, &[1u8; 48]); + for (name, bytes, want_too_large) in [ + ("header", huge_header, true), + ("payload", huge_payload, true), + ("short", short_payload, false), + ("narrow", narrow, false), + ("req", wrong_req, false), + ] { + let (record, handle, _tmp) = fake_server(bytes); + let mut bulk = BulkClient::attach(&record, &"b".repeat(32)).unwrap(); + let err = bulk.request(0.0, 0, None).unwrap_err(); + assert_eq!(matches!(err, LinkError::TooLarge), want_too_large, "{name}: {err:?}"); + handle.join().unwrap(); + } +} diff --git a/crates/ipc/tests/connection_limits.rs b/crates/ipc/tests/connection_limits.rs new file mode 100644 index 000000000..5c0888340 --- /dev/null +++ b/crates/ipc/tests/connection_limits.rs @@ -0,0 +1,280 @@ +//! Connection limits and partial-line handling. Each test asserts the secure behavior: silent +//! peers cannot fill the server, and a line that arrives in pieces is not lost. +#![cfg(unix)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, Extensions, FrameOpen, Frames, Limits, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; + +struct Sink; +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({})); + Some(rx) + } +} +struct NoActivate; +impl Activate for NoActivate { + fn activate(&self, _t: Option, _p: Option, _o: Option) -> Activation { + Activation::No + } +} +struct NoFrames; +impl Frames for NoFrames { + fn open(&self, _p: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _t: &str, _h: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _t: &str) {} +} + +fn boot_with(label: &str, limits: Limits) -> (Server, PathBuf, String, PathBuf) { + let dir = std::env::temp_dir().join(format!("cl-ipc-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let runtime = RuntimeDir::at(&dir).unwrap(); + let server = Server::start_with_limits( + ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(runtime), + tcp_port: None, + sink: Arc::new(Sink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }, + Extensions::default(), + limits, + ) + .unwrap(); + let socket = match &server.record().endpoint { + craft_ipc::Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("unix endpoint expected, got {other:?}"), + }; + let token = server.record().token.clone(); + (server, socket, token, dir) +} + +fn boot(label: &str) -> (Server, PathBuf, String, PathBuf) { + boot_with(label, Limits::default()) +} + +fn auth_line(token: &str) -> String { + format!("{{\"id\":0,\"method\":\"auth\",\"params\":{{\"token\":\"{token}\"}}}}\n") +} + +const HELLO: &str = "{\"id\":1,\"method\":\"link.hello\",\"params\":{\"protocol\":{\"min\":1,\"max\":1}}}\n"; + +fn arm_read_timeout(stream: &UnixStream, secs: u64) { + let result = stream.set_read_timeout(Some(Duration::from_secs(secs))); + // Darwin refuses socket options after shutdown. The queued denial/EOF can still be read. + #[cfg(target_os = "macos")] + if result.as_ref().is_err_and(|err| err.raw_os_error() == Some(libc::EINVAL)) { + return; + } + result.unwrap(); +} + +fn read_text(stream: &mut UnixStream, secs: u64) -> String { + arm_read_timeout(stream, secs); + let mut text = String::new(); + let _ = stream.read_to_string(&mut text); + text +} + +fn next_line(reader: &mut BufReader) -> String { + let mut line = String::new(); + arm_read_timeout(reader.get_ref(), 5); + reader.read_line(&mut line).unwrap(); + line +} + +/// 16 silent connections must not lock the legitimate client out. At most 4 unauthenticated +/// connections are admitted, the rest are refused at once, and the admitted ones are closed at the +/// auth deadline. A client that authenticated before the flood keeps working. +#[test] +fn silent_connections_cannot_lock_out_clients() { + let (server, socket, token, dir) = boot("slots"); + let early = UnixStream::connect(&socket).unwrap(); + let mut early_writer = early.try_clone().unwrap(); + let mut early_reader = BufReader::new(early); + early_writer.write_all(auth_line(&token).as_bytes()).unwrap(); + assert!(next_line(&mut early_reader).contains("authenticated")); + + let mut hogs = Vec::new(); + for _ in 0..16 { + hogs.push(UnixStream::connect(&socket).unwrap()); + } + // The authenticated client still gets its answer while the flood is open. + early_writer.write_all(HELLO.as_bytes()).unwrap(); + let line = next_line(&mut early_reader); + assert!(line.contains("\"ok\":true"), "authenticated client lost service: {line:?}"); + + // Past the auth deadline every silent connection is closed and a new client is served. + std::thread::sleep(Duration::from_millis(3500)); + for hog in &mut hogs { + let text = read_text(hog, 1); + assert!(!text.contains("\"authenticated\""), "{text:?}"); + } + let mut legit = UnixStream::connect(&socket).unwrap(); + legit.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(legit); + let line = next_line(&mut reader); + assert!(line.contains("authenticated"), "legitimate client was not served: {line:?}"); + drop(hogs); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// the unauthenticated pool is small, and the refusal for an extra peer is immediate. +#[test] +fn unauthenticated_peers_beyond_the_cap_are_refused_at_once() { + let limits = Limits { max_unauthenticated: 2, ..Limits::default() }; + let (server, socket, token, dir) = boot_with("unauth-cap", limits); + let a = UnixStream::connect(&socket).unwrap(); + let b = UnixStream::connect(&socket).unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let mut third = UnixStream::connect(&socket).unwrap(); + let text = read_text(&mut third, 1); + assert!(text.contains("too many unauthenticated connections"), "{text:?}"); + // Authenticating frees an unauthenticated slot. + let mut a = a; + a.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(a); + assert!(next_line(&mut reader).contains("authenticated")); + let mut fourth = UnixStream::connect(&socket).unwrap(); + fourth.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut reader = BufReader::new(fourth); + assert!(next_line(&mut reader).contains("authenticated")); + drop(b); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a connection that sends no auth line is closed after the deadline (default 2 s). +#[test] +fn silent_connection_is_closed_at_the_auth_deadline() { + let (server, socket, _token, dir) = boot("deadline"); + let mut s = UnixStream::connect(&socket).unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut s, 6); + let waited = start.elapsed(); + assert!(text.contains("authentication required"), "{text:?}"); + assert!(waited < Duration::from_secs(4), "closed after {waited:?}"); + assert!(waited >= Duration::from_millis(1500), "closed too early: {waited:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// an auth line that completes after the deadline does not authenticate, even when it lands +/// between two read polls (250 ms apart) and the poll has not yet seen the deadline pass. +#[test] +fn an_auth_line_that_completes_after_the_deadline_is_refused() { + let limits = Limits { auth_deadline: Duration::from_millis(300), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("lateauth", limits); + let mut s = UnixStream::connect(&socket).unwrap(); + std::thread::sleep(Duration::from_millis(400)); + if let Err(err) = s.write_all(auth_line(&token).as_bytes()) { + assert!( + matches!(err.kind(), std::io::ErrorKind::BrokenPipe | std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::NotConnected), + "{err}" + ); + } + let text = read_text(&mut s, 6); + assert!(text.contains("authentication required"), "{text:?}"); + assert!(!text.contains("\"authenticated\""), "{text:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a peer that starts a line and stalls is cut at the partial-line deadline. +#[test] +fn a_stalled_partial_line_is_closed() { + let limits = Limits { partial_line: Duration::from_millis(800), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("stall", limits); + let mut s = UnixStream::connect(&socket).unwrap(); + s.write_all(auth_line(&token).as_bytes()).unwrap(); + s.write_all(b"{\"id\":1,\"method\":\"link.he").unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut s, 6); + assert!(text.contains("authenticated"), "{text:?}"); + assert!(start.elapsed() < Duration::from_secs(4), "the stalled line stayed open for {:?}", start.elapsed()); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// an authenticated connection with no subscription and no traffic is closed at the idle limit. +/// A subscribed connection stays open. +#[test] +fn idle_connections_are_closed_but_subscribers_stay() { + let limits = Limits { idle: Duration::from_millis(900), ..Limits::default() }; + let (server, socket, token, dir) = boot_with("idle", limits); + let mut idle = UnixStream::connect(&socket).unwrap(); + idle.write_all(auth_line(&token).as_bytes()).unwrap(); + let mut sub = UnixStream::connect(&socket).unwrap(); + sub.write_all(auth_line(&token).as_bytes()).unwrap(); + sub.write_all(b"{\"id\":2,\"method\":\"events.subscribe\",\"params\":{\"events\":[\"doc.changed\"]}}\n").unwrap(); + let start = std::time::Instant::now(); + let text = read_text(&mut idle, 6); + assert!(text.contains("authenticated"), "{text:?}"); + assert!(start.elapsed() < Duration::from_secs(4), "idle connection stayed open for {:?}", start.elapsed()); + // The subscriber is still served after the idle limit passed. + std::thread::sleep(Duration::from_millis(300)); + sub.write_all(HELLO.as_bytes()).unwrap(); + let mut reader = BufReader::new(sub); + let mut seen = String::new(); + for _ in 0..3 { + seen.push_str(&next_line(&mut reader)); + if seen.contains("\"id\":1") { + break; + } + } + assert!(seen.contains("\"id\":1"), "subscriber was cut: {seen:?}"); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// a request split at several points, with a pause longer than the 250 ms read poll, is served. +#[test] +fn a_line_split_across_the_250ms_read_poll_is_kept() { + let (server, socket, token, dir) = boot("split"); + for cut in [1usize, 23, HELLO.len() - 2, HELLO.len() - 1] { + let mut s = UnixStream::connect(&socket).unwrap(); + s.write_all(auth_line(&token).as_bytes()).unwrap(); + s.write_all(&HELLO.as_bytes()[..cut]).unwrap(); + std::thread::sleep(Duration::from_millis(600)); + s.write_all(&HELLO.as_bytes()[cut..]).unwrap(); + let mut reader = BufReader::new(s); + assert!(next_line(&mut reader).contains("authenticated"), "cut {cut}"); + let line = next_line(&mut reader); + assert!(line.contains("\"ok\":true"), "cut {cut}: split request was not served: {line:?}"); + } + drop(server); + let _ = std::fs::remove_dir_all(dir); +} + +/// the auth line itself may arrive in pieces. +#[test] +fn an_auth_line_split_across_the_read_poll_is_kept() { + let (server, socket, token, dir) = boot("authsplit"); + let mut s = UnixStream::connect(&socket).unwrap(); + let line = auth_line(&token); + s.write_all(&line.as_bytes()[..20]).unwrap(); + std::thread::sleep(Duration::from_millis(600)); + s.write_all(&line.as_bytes()[20..]).unwrap(); + let mut reader = BufReader::new(s); + assert!(next_line(&mut reader).contains("authenticated")); + drop(server); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/ipc/tests/discovery.rs b/crates/ipc/tests/discovery.rs new file mode 100644 index 000000000..7d9f4a05f --- /dev/null +++ b/crates/ipc/tests/discovery.rs @@ -0,0 +1,97 @@ +//! A record held by a child dies with the child. +//! +//! `hold_until_killed` returns at once unless `CRAFT_HOLD_DIR` is set. +//! The parent test spawns that child, checks the live record, sends SIGKILL, +//! and checks that `Discovery::scan` removed it. Drop does not run on SIGKILL. + +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +use craft_ipc::{Discovery, InstanceRecord, RuntimeDir}; + +#[test] +fn hold_until_killed() { + let Ok(dir) = std::env::var("CRAFT_HOLD_DIR") else { + return; + }; + let runtime = RuntimeDir::at(&dir).unwrap(); + let app = "effectcraft"; + let pid = std::process::id(); + let endpoint = runtime.unix_endpoint(app, pid).unwrap(); + let record = InstanceRecord::new(app, "0.8.3", pid, endpoint).unwrap(); + let held = Discovery::publish(&runtime, record).unwrap(); + std::fs::write(runtime.path().join("ready"), b"ready").unwrap(); + let _held = held; + // park can wake without an unpark. Loop so Drop does not run until SIGKILL. + loop { + std::thread::park(); + } +} + +#[test] +fn sigkill_drops_the_record() { + let path = hold_dir(); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let exe = std::env::current_exe().unwrap(); + let child = Command::new(exe).args(["--exact", "hold_until_killed", "--test-threads=1"]).env("CRAFT_HOLD_DIR", &path).spawn().unwrap(); + let mut guard = HoldCleanup { dir: path.clone(), child: Some(child) }; + let ready = runtime.path().join("ready"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if ready.is_file() { + break; + } + let child = guard.child.as_mut().unwrap(); + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before ready: {status}"); + } + if Instant::now() >= deadline { + panic!("child did not write ready"); + } + std::thread::sleep(Duration::from_millis(20)); + } + let child_pid = guard.child.as_ref().unwrap().id(); + let live = Discovery::scan(&runtime).unwrap(); + assert_eq!(live.len(), 1, "expected one live record"); + assert_eq!(live[0].pid, child_pid); + assert_eq!(live[0].app, "effectcraft"); + let mut child = guard.child.take().unwrap(); + child.kill().unwrap(); + let status = child.wait().unwrap(); + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + assert_eq!(status.signal(), Some(9), "SIGKILL so the child's Drop does not unlock"); + } + #[cfg(not(unix))] + { + let _ = status; + } + let live = Discovery::scan(&runtime).unwrap(); + assert!(live.is_empty(), "scan should drop the record after SIGKILL, got {live:?}"); + assert!(!runtime.record_file("effectcraft", child_pid).exists(), "record json should be gone"); +} + +struct HoldCleanup { + dir: PathBuf, + child: Option, +} + +impl Drop for HoldCleanup { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn hold_dir() -> PathBuf { + let mut bytes = [0u8; 4]; + getrandom::getrandom(&mut bytes).unwrap(); + let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + std::env::temp_dir().join(format!("cl-hold-{hex}")) +} diff --git a/crates/ipc/tests/fetch.rs b/crates/ipc/tests/fetch.rs new file mode 100644 index 000000000..8e1dcde73 --- /dev/null +++ b/crates/ipc/tests/fetch.rs @@ -0,0 +1,366 @@ +//! `FrameFetcher` against a real server with a fake `FrameSource`: cache, deadline, dedupe, +//! parallel connections, and a peer that goes away. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use craft_ipc::bulk::{FrameFormat, FrameLayout, FrameRequest, FrameSource, FrameSpec, FrameTickets, RenderedFrame}; +use craft_ipc::fetch::{FetchOptions, Fetched, FrameFetcher, FrameKey, QUEUE_MAX}; +use craft_ipc::server::{Activate, Activation, ControlSink, Frames, Server, ServerConfig}; +use craft_ipc::{InstanceRecord, LinkError, RuntimeDir}; + +const W: u32 = 4; +const H: u32 = 3; +const STRIDE: u32 = 16; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +/// The first pixel byte is the frame number asked for, so a wrong frame cannot match by accident. +struct Fake { + epoch: AtomicU64, + delay_ms: AtomicU64, + renders: AtomicUsize, + times: Mutex>, + fail_time: Mutex>, +} + +impl FrameSource for Fake { + fn open(&self, _spec: &FrameSpec) -> Result { + Ok(FrameLayout { w: W, h: H, stride: STRIDE, len: u64::from(STRIDE * H) }) + } + + fn render(&self, spec: &FrameSpec, request: &FrameRequest) -> Result { + self.renders.fetch_add(1, Ordering::SeqCst); + self.times.lock().unwrap().push(request.time_s); + let delay = self.delay_ms.load(Ordering::SeqCst); + if delay > 0 { + std::thread::sleep(Duration::from_millis(delay)); + } + if *self.fail_time.lock().unwrap() == Some(request.time_s) { + return Err(LinkError::NotReady("comp is busy".into())); + } + let mut pixels = vec![0u8; (STRIDE * H) as usize]; + pixels[0] = request.time_s as u8; + Ok(RenderedFrame { + epoch: self.epoch.load(Ordering::SeqCst).max(request.epoch), + w: W, + h: H, + stride: STRIDE, + format: spec.format, + render_ms: 1.0, + pixels, + }) + } + + fn close(&self, _spec: &FrameSpec) {} +} + +struct Rig { + _tmp: Tmp, + server: Option, + record: InstanceRecord, + fake: Arc, +} + +fn boot() -> Rig { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-fch-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let fake = Arc::new(Fake { + epoch: AtomicU64::new(1), + delay_ms: AtomicU64::new(0), + renders: AtomicUsize::new(0), + times: Mutex::new(Vec::new()), + fail_time: Mutex::new(None), + }); + let tickets = Arc::new(FrameTickets::new(Arc::clone(&fake) as Arc)); + let server = Server::start(ServerConfig { + app: format!("ec-{serial}"), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(RuntimeDir::at(&path).unwrap()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: tickets as Arc, + }) + .unwrap(); + let record = server.record().clone(); + Rig { _tmp: Tmp(path), server: Some(server), record, fake } +} + +fn connect(rig: &Rig, connections: usize) -> FrameFetcher { + let options = FetchOptions { connections, io_timeout: Duration::from_secs(2), ..FetchOptions::default() }; + FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap() +} + +const fn key(frame: i64) -> FrameKey { + FrameKey { frame, scale_milli: 1000 } +} + +const WAIT: Duration = Duration::from_secs(2); + +#[test] +fn fetch_returns_the_frame_and_the_second_ask_is_a_cache_hit() { + let rig = boot(); + let fetcher = connect(&rig, 1); + let layout = fetcher.layout(); + assert_eq!((layout.w, layout.h, layout.stride), (W, H, STRIDE)); + let Fetched::Frame(first) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; + assert_eq!(first.pixels[0], 7); + assert_eq!(first.epoch, 1); + let Fetched::Frame(second) = fetcher.fetch(key(7), 7.0, 1, WAIT) else { panic!("no frame") }; + assert!(Arc::ptr_eq(&first, &second), "a hit returns the cached frame"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); + let stats = fetcher.stats(); + assert_eq!((stats.hits, stats.renders), (1, 1)); +} + +#[test] +fn a_late_frame_misses_the_deadline_then_lands_in_the_cache() { + let rig = boot(); + rig.fake.delay_ms.store(300, Ordering::SeqCst); + let fetcher = connect(&rig, 1); + let t0 = Instant::now(); + assert!(matches!(fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(40)), Fetched::TimedOut)); + assert!(t0.elapsed() < Duration::from_millis(200), "the call must not wait for the render: {:?}", t0.elapsed()); + assert!(fetcher.cached(key(3), 1).is_none(), "not there yet"); + let deadline = Instant::now() + Duration::from_secs(3); + while fetcher.cached(key(3), 1).is_none() { + assert!(Instant::now() < deadline, "the late frame never reached the cache"); + std::thread::sleep(Duration::from_millis(10)); + } + let Fetched::Frame(frame) = fetcher.fetch(key(3), 3.0, 1, Duration::from_millis(10)) else { panic!("no frame") }; + assert_eq!(frame.pixels[0], 3); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1, "the timed-out ask and the retry share one render"); + assert_eq!(fetcher.stats().timeouts, 1); +} + +#[test] +fn a_newer_epoch_asked_for_renders_again_and_an_older_cached_one_does_not_count() { + let rig = boot(); + let fetcher = connect(&rig, 1); + let Fetched::Frame(old) = fetcher.fetch(key(1), 1.0, 1, WAIT) else { panic!("no frame") }; + assert_eq!(old.epoch, 1); + rig.fake.epoch.store(5, Ordering::SeqCst); + // Epoch 1 is still good for a caller that wants epoch 1. + assert!(fetcher.cached(key(1), 1).is_some()); + assert!(fetcher.cached(key(1), 2).is_none(), "an epoch 1 frame cannot answer a request for epoch 2"); + let Fetched::Frame(fresh) = fetcher.fetch(key(1), 1.0, 2, WAIT) else { panic!("no frame") }; + assert_eq!(fresh.epoch, 5, "the source may answer with a newer epoch"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2); + // The newer frame answers an older request too. + assert!(fetcher.cached(key(1), 1).is_some_and(|f| f.epoch == 5)); +} + +#[test] +fn two_asks_for_one_key_render_once() { + let rig = boot(); + rig.fake.delay_ms.store(120, Ordering::SeqCst); + let fetcher = Arc::new(connect(&rig, 2)); + let a = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) + }; + let b = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(9), 9.0, 1, WAIT), Fetched::Frame(_))) + }; + assert!(a.join().unwrap() && b.join().unwrap()); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 1); +} + +#[test] +fn four_connections_render_four_frames_at_once() { + let rig = boot(); + rig.fake.delay_ms.store(200, Ordering::SeqCst); + let fetcher = Arc::new(connect(&rig, 4)); + let t0 = Instant::now(); + let handles: Vec<_> = (0..4) + .map(|n| { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))) + }) + .collect(); + for handle in handles { + assert!(handle.join().unwrap()); + } + let took = t0.elapsed(); + assert!(took < Duration::from_millis(600), "four 200 ms renders on four connections took {took:?}"); +} + +#[test] +fn a_frame_error_fails_that_frame_and_the_fetcher_keeps_working() { + let rig = boot(); + *rig.fake.fail_time.lock().unwrap() = Some(4.0); + let fetcher = connect(&rig, 1); + match fetcher.fetch(key(4), 4.0, 1, WAIT) { + Fetched::Failed(message) => assert!(message.contains("comp is busy"), "{message}"), + other => panic!("expected Failed, got {other:?}"), + } + assert!(!fetcher.is_down()); + assert!(matches!(fetcher.fetch(key(5), 5.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn a_peer_that_goes_away_makes_the_fetcher_down_and_asks_return_at_once() { + let mut rig = boot(); + let fetcher = connect(&rig, 2); + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); + rig.server.take().unwrap().shutdown().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !fetcher.is_down() { + // A miss makes the workers touch the dead socket. + let _ = fetcher.fetch(key(100), 100.0, 1, Duration::from_millis(50)); + assert!(Instant::now() < deadline, "the fetcher never noticed the peer is gone"); + } + let t0 = Instant::now(); + assert!(matches!(fetcher.fetch(key(200), 200.0, 1, WAIT), Fetched::Down)); + assert!(t0.elapsed() < Duration::from_millis(100)); + // Frames already in the cache still answer. + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn the_cache_stays_inside_its_byte_budget() { + let rig = boot(); + let options = FetchOptions { connections: 1, cache_bytes: usize::try_from(STRIDE * H).unwrap() * 3, ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + for n in 0..6 { + assert!(matches!(fetcher.fetch(key(n), n as f64, 1, WAIT), Fetched::Frame(_))); + } + let kept = (0..6).filter(|n| fetcher.cached(key(*n), 1).is_some()).count(); + assert_eq!(kept, 3, "a budget of three frames keeps three"); + assert!(fetcher.cached(key(5), 1).is_some(), "the newest frame stays"); +} + +#[test] +fn queued_asks_nobody_waits_for_are_dropped_when_old() { + let rig = boot(); + rig.fake.delay_ms.store(150, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + // Five asks with no patience: frame 0 starts at once, frames 1 to 4 wait behind it. + for n in 0..5 { + assert!(matches!(fetcher.fetch(key(n), n as f64, 1, Duration::from_millis(1)), Fetched::TimedOut)); + } + std::thread::sleep(Duration::from_millis(900)); + let renders = rig.fake.renders.load(Ordering::SeqCst); + assert!(renders < 5, "frames that waited longer than max_age must be skipped, got {renders} renders"); +} + +#[test] +fn a_time_that_is_not_finite_fails_the_ask_and_keeps_the_workers() { + let rig = boot(); + let fetcher = connect(&rig, 1); + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + match fetcher.fetch(key(1), bad, 1, WAIT) { + Fetched::Failed(message) => assert!(message.contains("finite"), "{message}"), + other => panic!("expected Failed for {bad}, got {other:?}"), + } + } + assert!(!fetcher.is_down(), "a bad time must not end the worker"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 0, "nothing was queued"); + assert!(matches!(fetcher.fetch(key(2), 2.0, 1, WAIT), Fetched::Frame(_))); +} + +#[test] +fn an_ask_with_a_waiting_caller_is_not_dropped_for_age() { + let rig = boot(); + rig.fake.delay_ms.store(300, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_millis(100), ..FetchOptions::default() }; + let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); + // Frame 0 keeps the only worker busy for 300 ms. Frame 1 waits in the queue past max_age, + // and its caller is still there when the worker reaches it. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + let waiter = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || fetcher.fetch(key(1), 1.0, 1, WAIT)) + }; + // A second caller for the same key joins the queued job and shares its render. + std::thread::sleep(Duration::from_millis(20)); + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + assert!(matches!(waiter.join().unwrap(), Fetched::Frame(_))); + let stats = fetcher.stats(); + assert_eq!(stats.skipped, 0, "a job a caller waits for is not skipped"); + assert_eq!(rig.fake.renders.load(Ordering::SeqCst), 2, "each frame renders once"); +} + +#[test] +fn the_queue_holds_at_most_queue_max_asks() { + let rig = boot(); + rig.fake.delay_ms.store(500, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; + let fetcher = FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap(); + // Frame 0 holds the only worker. The asks after it only queue. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + std::thread::sleep(Duration::from_millis(100)); + let extra = 40; + for n in 1..=i64::try_from(QUEUE_MAX + extra).unwrap() { + assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); + } + assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap(), "the oldest asks past the cap are dropped"); +} + +#[test] +fn a_caller_waiting_on_a_dropped_ask_gets_failed_and_does_not_queue_it_again() { + let rig = boot(); + rig.fake.delay_ms.store(1500, Ordering::SeqCst); + let options = FetchOptions { connections: 1, max_age: Duration::from_secs(60), ..FetchOptions::default() }; + let fetcher = Arc::new(FrameFetcher::connect(&rig.record, "link-a", FrameFormat::Rgba8Premul, 1.0, options).unwrap()); + // Frame 0 holds the only worker. Frame 1 queues behind it with a caller that waits. + assert!(matches!(fetcher.fetch(key(0), 0.0, 1, Duration::from_millis(1)), Fetched::TimedOut)); + std::thread::sleep(Duration::from_millis(100)); + let (done, finished) = std::sync::mpsc::channel(); + let waiter = { + let fetcher = Arc::clone(&fetcher); + std::thread::spawn(move || { + let started = Instant::now(); + let _ = done.send((fetcher.fetch(key(1), 1.0, 1, Duration::from_secs(20)), started.elapsed())); + }) + }; + std::thread::sleep(Duration::from_millis(100)); + // Fill the queue past its cap: frame 1 is the oldest, so it goes first. + let extra = 5; + for n in 2..=i64::try_from(QUEUE_MAX + extra).unwrap() { + assert!(matches!(fetcher.fetch(key(n), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); + } + let (outcome, waited) = finished.recv_timeout(Duration::from_secs(1)).expect("the dropped caller must return at once"); + assert!(matches!(&outcome, Fetched::Failed(message) if message.contains("dropped")), "{outcome:?}"); + assert!(waited < Duration::from_secs(5), "{waited:?}"); + waiter.join().unwrap(); + std::thread::sleep(Duration::from_millis(100)); + // Frame 1 plus the others make QUEUE_MAX + extra asks for QUEUE_MAX places. A caller that queued + // frame 1 again would evict more. + assert_eq!(fetcher.stats().skipped, u64::try_from(extra).unwrap()); + // A new ask for the dropped frame is a new ask, not a repeat of the failure. + assert!(matches!(fetcher.fetch(key(1), 1.0, 1, Duration::ZERO), Fetched::TimedOut)); +} diff --git a/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json b/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json new file mode 100644 index 000000000..9b0330361 --- /dev/null +++ b/crates/ipc/tests/fixtures/link-record-filmcraft-effectcraft.json @@ -0,0 +1,13 @@ +{ + "v": 1, + "link_id": "550e8400-e29b-41d4-a716-446655440000", + "target_app": "effectcraft", + "target_project": "links/shot.ecproj", + "target_comp": "comp-1", + "nested_sequence": "item-9", + "result_path": "Project Links/550e8400-e29b-41d4-a716-446655440000.mov", + "result_epoch": 7, + "format": { "w": 1920, "h": 1080, "rate_num": 24, "rate_den": 1, "duration_ticks": 240, "alpha": true, "tint": "warm" }, + "created": "2026-10-11T15:04:05Z", + "badge": "kept" +} diff --git a/crates/ipc/tests/fixtures/link-record-still.json b/crates/ipc/tests/fixtures/link-record-still.json new file mode 100644 index 000000000..20cf44e85 --- /dev/null +++ b/crates/ipc/tests/fixtures/link-record-still.json @@ -0,0 +1,15 @@ +{ + "v": 1, + "link_id": "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", + "target_app": "photocraft", + "target_project": "art/poster.photocraft", + "result_path": "Project Links/0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d.png", + "result_epoch": 3, + "asset": { + "kind": "still", + "moniker": "craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/flattened" + }, + "producer": { "app": "photocraft", "app_version": "0.4.2", "doc_id": "9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f" }, + "format": { "w": 4096, "h": 2160, "alpha": false }, + "created": "2026-10-23T09:30:00.250+02:00" +} diff --git a/crates/ipc/tests/hub.rs b/crates/ipc/tests/hub.rs new file mode 100644 index 000000000..0a9482b41 --- /dev/null +++ b/crates/ipc/tests/hub.rs @@ -0,0 +1,334 @@ +//! A hub is a peer. Two fake apps share one runtime directory: +//! `photocraft`, which owns a still and answers the asset methods, and `hub`, +//! which owns nothing, writes `caps: ["hub"]` and receives `app.activate` +//! with `open:{asset}`. A third party finds both through discovery. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use craft_ipc::asset::{Described, Refreshed, Resolved}; +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::moniker::Moniker; +use craft_ipc::record::AssetKind; +use craft_ipc::refresh::{Tracker, write_atomic}; +use craft_ipc::server::{Activate, Activation, Assets, ControlSink, Extensions, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{Discovery, InstanceRecord, LinkError, RuntimeDir}; + +// Both tests bind the same app and process named pipe on Windows, regardless of runtime directory. +static HUB_TEST: Mutex<()> = Mutex::new(()); + +const DOC: &str = "9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f"; +const LINK: &str = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +struct Tmp(PathBuf); + +impl Tmp { + fn new() -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-hub-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[derive(Default)] +struct RecordingSink { + seen: Mutex>, +} + +impl ControlSink for RecordingSink { + fn submit(&self, method: String, _params: Value) -> Option> { + self.seen.lock().unwrap().push(method); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({"forwarded": true})); + Some(rx) + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[derive(Default)] +struct RecordingActivate { + opens: Mutex>>, +} + +impl Activate for RecordingActivate { + fn activate(&self, _token: Option, _from_pid: Option, open: Option) -> Activation { + self.opens.lock().unwrap().push(open); + Activation::Requested + } +} + +/// A producer with one still. The real file watcher answers `link.refresh`. +struct Stills { + asset: Moniker, + file: PathBuf, + thumb: PathBuf, + tracker: Mutex, + links: HashMap, +} + +impl Assets for Stills { + fn resolve(&self, asset: &Moniker) -> Result { + self.known(asset)?; + let fp = craft_ipc::refresh::fingerprint_if_present(&self.file)?.ok_or_else(|| LinkError::Unknown("no result file yet".into()))?; + let epoch = self.tracker.lock().unwrap().epoch(); + Ok(Resolved::from_fingerprint(asset.clone(), &self.file.display().to_string(), epoch, &fp)) + } + + fn describe(&self, asset: &Moniker) -> Result { + self.known(asset)?; + let fp = craft_ipc::refresh::fingerprint_if_present(&self.file)?; + Ok(Described { + asset: asset.clone(), + name: "Poster".into(), + kind: AssetKind::Still, + epoch: self.tracker.lock().unwrap().epoch(), + w: Some(640), + h: Some(480), + hash: fp.as_ref().map(|f| f.hash.clone()), + thumb_path: Some(self.thumb.display().to_string()), + modified_ms: fp.map(|f| f.mtime_ms), + }) + } + + fn refresh(&self, link_id: &str) -> Result { + if !self.links.contains_key(link_id) { + return Err(LinkError::Unknown(format!("link {link_id} is not known"))); + } + let mut tracker = self.tracker.lock().unwrap(); + let change = tracker.poll(true)?; + Ok(Refreshed::new(link_id, change.epoch, change.state, &self.file.display().to_string(), tracker.seen())) + } +} + +impl Stills { + fn known(&self, asset: &Moniker) -> Result<(), LinkError> { + if *asset == self.asset { Ok(()) } else { Err(LinkError::Unknown(format!("{asset} is not an asset of this document"))) } + } +} + +fn config(app: &str, runtime: &RuntimeDir, sink: Arc, activate: Arc) -> ServerConfig { + ServerConfig { + app: app.into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink, + activate, + frames: Arc::new(NoFrames), + } +} + +fn dial(runtime: &RuntimeDir, app: &str) -> (Client, InstanceRecord) { + let found = Discovery::scan(runtime).unwrap(); + let record = found.into_iter().find(|r| r.app == app).unwrap_or_else(|| panic!("no live {app} record")); + let mut client = Client::connect(&record).unwrap(); + let identity = HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440000".into(), + protocol_min: 1, + protocol_max: 1, + }; + let hello = client.hello(&identity).unwrap(); + assert_eq!(hello["app"], json!(app)); + (client, record) +} + +fn caps_of(hello_or_record: &Value) -> Vec { + hello_or_record["caps"].as_array().unwrap().iter().map(|c| c.as_str().unwrap().to_string()).collect() +} + +/// The two fake apps of the hub test and the files behind them. +struct World { + runtime: RuntimeDir, + asset: Moniker, + file: PathBuf, + producer: Server, + producer_sink: Arc, + hub: Server, + hub_activate: Arc, +} + +fn start_world(tmp: &Tmp) -> World { + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let asset = Moniker::parse(&format!("craft://photocraft/{DOC}/image/flattened")).unwrap(); + let file = tmp.0.join("flat.png"); + write_atomic(&file, b"png-v1").unwrap(); + let stills = Arc::new(Stills { + asset: asset.clone(), + file: file.clone(), + thumb: tmp.0.join("thumb.png"), + tracker: Mutex::new(Tracker::new(&file, 3).unwrap()), + links: HashMap::from([(LINK.to_string(), ())]), + }); + + let producer_sink = Arc::new(RecordingSink::default()); + let producer = Server::start_with( + config("photocraft", &runtime, Arc::clone(&producer_sink), Arc::new(RecordingActivate::default())), + Extensions { assets: Some(stills), caps: Vec::new() }, + ) + .unwrap(); + let hub_activate = Arc::new(RecordingActivate::default()); + let hub = Server::start_with( + config("hub", &runtime, Arc::new(RecordingSink::default()), Arc::clone(&hub_activate)), + Extensions { assets: None, caps: vec!["hub".into()] }, + ) + .unwrap(); + World { runtime, asset, file, producer, producer_sink, hub, hub_activate } +} + +/// Discovery: the hub is an ordinary record with app "hub" and caps ["hub"]. `link.hello` +/// advertises the base caps plus the record caps. +fn check_discovery_and_hello(world: &World, hub_client: &mut Client, hub_record: &InstanceRecord, producer_client: &mut Client) { + let live = Discovery::scan(&world.runtime).unwrap(); + let mut names: Vec<&str> = live.iter().map(|r| r.app.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["hub", "photocraft"]); + assert_eq!(world.hub.record().caps(), vec!["hub".to_string()]); + assert_eq!(world.producer.record().caps(), vec!["assets".to_string()]); + + let hello = hub_client.call("link.hello", json!({"protocol": {"min": 1, "max": 1}})).unwrap(); + assert_eq!(hello["app"], json!("hub")); + assert!(caps_of(&hello).contains(&"hub".to_string()), "{hello}"); + assert!(caps_of(&hello).contains(&"activate".to_string()), "{hello}"); + assert_eq!(hub_record.docs.len(), 0, "a hub owns no document"); + + let hello = producer_client.call("link.hello", json!({"protocol": {"min": 1, "max": 1}})).unwrap(); + assert!(caps_of(&hello).contains(&"assets".to_string()), "{hello}"); +} + +/// The hub browses the producer with no document open. Returns the hash of the first version. +fn check_browse(world: &World, producer_client: &mut Client) -> String { + let asset = &world.asset; + let described = producer_client.call("asset.describe", json!({"asset": asset.to_string()})).unwrap(); + assert_eq!(described["name"], json!("Poster")); + assert_eq!(described["kind"], json!("still")); + assert_eq!(described["epoch"], json!(3)); + assert_eq!(described["w"], json!(640)); + assert_eq!(described["asset"], json!(asset.to_string())); + let resolved = producer_client.call("link.resolve", json!({"asset": asset.to_string()})).unwrap(); + assert_eq!(resolved["path"], json!(world.file.display().to_string())); + assert_eq!(resolved["size"], json!(6)); + let v1_hash = resolved["hash"].as_str().unwrap().to_string(); + assert_eq!(described["hash"], json!(v1_hash)); + assert!(v1_hash.starts_with("sha256:") && v1_hash.len() == 71, "{v1_hash}"); + assert!(world.producer_sink.seen.lock().unwrap().is_empty(), "asset methods must not reach the UI sink"); + v1_hash +} + +/// Errors map to the protocol's error codes. +fn check_error_codes(producer_client: &mut Client) { + let other = format!("craft://photocraft/{DOC}/image/other"); + assert!(matches!(producer_client.call("asset.describe", json!({"asset": other})), Err(LinkError::Unknown(_)))); + assert!(matches!(producer_client.call("link.resolve", json!({"asset": "craft://nope"})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.resolve", json!({})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.refresh", json!({"link_id": ""})), Err(LinkError::BadParams(_)))); + assert!(matches!(producer_client.call("link.refresh", json!({"link_id": "7b1e0000-0000-4000-8000-000000000000"})), Err(LinkError::Unknown(_)))); +} + +/// `link.refresh` sees a new version on disk and bumps the epoch once. +fn check_refresh(world: &World, producer_client: &mut Client, v1_hash: &str) { + let file = &world.file; + let current = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(current["state"], json!("current")); + assert_eq!(current["epoch"], json!(3)); + write_atomic(file, b"png-version-two").unwrap(); + let modified = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(modified["state"], json!("modified")); + assert_eq!(modified["epoch"], json!(4)); + assert_ne!(modified["hash"], json!(v1_hash)); + let again = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(again["epoch"], json!(4), "no second bump without a new edit"); + std::fs::remove_file(file).unwrap(); + let missing = producer_client.call("link.refresh", json!({"link_id": LINK})).unwrap(); + assert_eq!(missing["state"], json!("missing")); + assert!(missing.get("hash").is_none() && missing.get("result_path").is_none(), "{missing}"); + assert!(matches!(producer_client.call("link.resolve", json!({"asset": world.asset.to_string()})), Err(LinkError::Unknown(_)))); +} + +/// `app.activate` carries open:{asset} to the hub's Activate. +fn check_activate(world: &World, hub_client: &mut Client) { + let asset = &world.asset; + let reply = hub_client.call("app.activate", json!({"token": null, "from_pid": 4242, "open": {"asset": asset.to_string()}})).unwrap(); + assert_eq!(reply["activated"], json!("requested")); + let reply = hub_client.call("app.activate", json!({"open": {"link_id": LINK, "asset": asset.to_string()}})).unwrap(); + assert_eq!(reply["activated"], json!("requested")); + // Legacy shapes still pass through unchanged. + hub_client.call("app.activate", json!({"open": {"link_id": LINK}})).unwrap(); + hub_client.call("app.activate", json!({})).unwrap(); + // A bad moniker never reaches the app. + assert!(matches!(hub_client.call("app.activate", json!({"open": {"asset": "craft://bad"}})), Err(LinkError::BadParams(_)))); + assert!(matches!(hub_client.call("app.activate", json!({"open": {"asset": 7}})), Err(LinkError::BadParams(_)))); + let opens = world.hub_activate.opens.lock().unwrap().clone(); + assert_eq!( + opens, + vec![ + Some(json!({"asset": asset.to_string()})), + Some(json!({"link_id": LINK, "asset": asset.to_string()})), + Some(json!({"link_id": LINK})), + None, + ] + ); +} + +#[test] +fn hub_is_a_peer_and_producer_answers_asset_methods() { + let _guard = HUB_TEST.lock().unwrap(); + let tmp = Tmp::new(); + let world = start_world(&tmp); + let (mut hub_client, hub_record) = dial(&world.runtime, "hub"); + let (mut producer_client, _) = dial(&world.runtime, "photocraft"); + + check_discovery_and_hello(&world, &mut hub_client, &hub_record, &mut producer_client); + let v1_hash = check_browse(&world, &mut producer_client); + check_error_codes(&mut producer_client); + check_refresh(&world, &mut producer_client, &v1_hash); + check_activate(&world, &mut hub_client); + + world.producer.shutdown().unwrap(); + world.hub.shutdown().unwrap(); +} + +#[test] +fn server_without_a_provider_forwards_asset_methods_to_the_app() { + let _guard = HUB_TEST.lock().unwrap(); + let tmp = Tmp::new(); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let sink = Arc::new(RecordingSink::default()); + let server = Server::start(config("photocraft", &runtime, Arc::clone(&sink), Arc::new(RecordingActivate::default()))).unwrap(); + assert!(server.record().caps().is_empty()); + let (mut client, _) = dial(&runtime, "photocraft"); + let asset = format!("craft://photocraft/{DOC}/image/x"); + for method in ["link.resolve", "asset.describe", "link.refresh"] { + let reply = client.call(method, json!({"asset": asset, "link_id": LINK})).unwrap(); + assert_eq!(reply, json!({"forwarded": true}), "{method}"); + } + assert_eq!(sink.seen.lock().unwrap().as_slice(), ["link.resolve", "asset.describe", "link.refresh"]); + server.shutdown().unwrap(); +} diff --git a/crates/ipc/tests/native_roundtrip.rs b/crates/ipc/tests/native_roundtrip.rs new file mode 100644 index 000000000..7539d91cf --- /dev/null +++ b/crates/ipc/tests/native_roundtrip.rs @@ -0,0 +1,93 @@ +//! Exercise the real native transport: Unix sockets or Windows named pipes. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::PathBuf; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; +use serde_json::{Value, json}; + +struct Cleanup(PathBuf); +impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct Echo; +impl ControlSink for Echo { + fn submit(&self, method: String, params: Value) -> Option> { + let (tx, rx) = mpsc::channel(); + tx.send(json!({"ok": true, "result": {"method": method, "params": params}})).unwrap(); + Some(rx) + } +} + +struct NoActivate; +impl Activate for NoActivate { + fn activate(&self, _token: Option, _pid: Option, _options: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _hints: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[test] +fn native_transport_authenticates_and_delivers_commands() { + let mut nonce = [0u8; 4]; + getrandom::getrandom(&mut nonce).unwrap(); + let path = std::env::temp_dir().join(format!("cl-nr-{}-{}", std::process::id(), u32::from_le_bytes(nonce))); + let cleanup = Cleanup(path.clone()); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "test".into(), + instance: None, + runtime: Some(RuntimeDir::at(path).unwrap()), + tcp_port: None, + sink: Arc::new(Echo), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let mut client = Client::connect(server.record()).unwrap(); + client + .hello(&HelloClient { + app: "ec".into(), + app_version: "test".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440000".into(), + protocol_min: craft_ipc::wire::PROTOCOL_MIN, + protocol_max: craft_ipc::wire::PROTOCOL_MAX, + }) + .unwrap(); + let reply = client.call("engine.execute", json!({"command": "ping"})).unwrap(); + assert_eq!(reply, json!({"method": "engine.execute", "params": {"command": "ping"}})); + + client.subscribe(&["doc.changed"]).unwrap(); + let start = Instant::now(); + assert!(client.next_event(Some(Duration::from_millis(50))).unwrap().is_none()); + assert!(start.elapsed() < Duration::from_secs(2), "quiet native transport must respect its deadline"); + server.emit("doc.changed", json!({"after_timeout": true})).unwrap(); + let event = client.next_event(Some(Duration::from_secs(2))).unwrap().expect("event after timeout"); + assert_eq!(event.data, json!({"after_timeout": true})); + + let mut wrong = server.record().clone(); + let opposite = if wrong.token.starts_with('0') { "1" } else { "0" }; + wrong.token.replace_range(..1, opposite); + assert!(matches!(Client::connect(&wrong), Err(LinkError::Auth))); + drop(client); + drop(server); + drop(cleanup); +} diff --git a/crates/ipc/tests/open_payload.rs b/crates/ipc/tests/open_payload.rs new file mode 100644 index 000000000..1a6f3453b --- /dev/null +++ b/crates/ipc/tests/open_payload.rs @@ -0,0 +1,169 @@ +//! `link.open` payload validation over a real server and a real client. +//! A bad payload gets `payload_invalid` and never reaches the app sink. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{Discovery, LinkError, RuntimeDir}; + +const ASSET: &str = "craft://photocraft/550e8400-e29b-41d4-a716-446655440000/image/flat"; +const LINK: &str = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +struct Tmp(PathBuf); + +impl Tmp { + fn new() -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-open-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Records every forwarded request and answers with a finished reply payload. +#[derive(Default)] +struct RecordingSink { + seen: Mutex>, +} + +impl ControlSink for RecordingSink { + fn submit(&self, method: String, params: Value) -> Option> { + self.seen.lock().unwrap().push((method, params)); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({"comp": "c1"})); + Some(rx) + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +fn native_path(name: &str) -> String { + std::env::temp_dir().join(name).to_str().unwrap().to_owned() +} + +fn file_url(path: &std::path::Path) -> String { + let text = path.to_str().unwrap().replace('\\', "/"); + if text.starts_with('/') { format!("file://{text}") } else { format!("file:///{text}") } +} + +fn open_params(payload: Value) -> Value { + json!({"link_id": LINK, "payload": payload, "result_path": native_path("x.png")}) +} + +#[test] +fn bad_payloads_get_payload_invalid_and_good_ones_reach_the_sink() { + let tmp = Tmp::new(); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let sink = Arc::new(RecordingSink::default()); + let server = Server::start(ServerConfig { + app: "effectcraft".into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink: Arc::clone(&sink) as Arc, + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let record = Discovery::scan(&runtime).unwrap().into_iter().next().unwrap(); + let mut client = Client::connect(&record).unwrap(); + client + .hello(&HelloClient { + app: "filmcraft".into(), + app_version: "0.9.1".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440001".into(), + protocol_min: 1, + protocol_max: 1, + }) + .unwrap(); + + let bad = [ + ("unknown format", json!({"format": "svg", "data": ""})), + ("no format", json!({"data": "{}"})), + ("payload is a string", json!("otio")), + ("otio without data", json!({"format": "otio-json"})), + ("otio empty data", json!({"format": "otio-json", "data": ""})), + ("image-doc relative path", json!({"format": "image-doc", "path": "rel.png", "asset": ASSET})), + ("image-doc no path", json!({"format": "image-doc", "asset": ASSET})), + ("image-doc bad moniker", json!({"format": "image-doc", "path": native_path("a.png"), "asset": "craft://bad"})), + ("image-doc no asset", json!({"format": "image-doc", "path": native_path("a.png")})), + ]; + for (name, payload) in bad { + let err = client.call("link.open", open_params(payload)).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{name}: {err:?}"); + } + let err = client.call("link.open", json!({"link_id": LINK})).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "missing payload: {err:?}"); + // A payload that names a file that is not there gets media_missing with the paths. + let missing_path = tmp.0.join("missing-clip.mov"); + let missing = json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": file_url(&missing_path)}).to_string(); + let err = client.call("link.open", open_params(json!({"format": "otio-json", "data": missing}))).unwrap_err(); + assert_eq!(err, LinkError::MediaMissing { paths: vec![missing_path.to_str().unwrap().replace('\\', "/")] }); + // A path that climbs, is relative or is not a string never reaches the app. + let good = json!({"format": "otio-json", "data": "{}"}); + for result_path in [json!("../x.mov"), json!("/tmp/../etc/x.mov"), json!("rel/x.mov"), json!(7), json!("/tmp/a\u{0}b")] { + let params = json!({"link_id": LINK, "payload": good, "result_path": result_path}); + let err = client.call("link.open", params).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{result_path}: {err:?}"); + } + let climbing = json!({"OTIO_SCHEMA": "ExternalReference.1", "target_url": file_url(&tmp.0.join("..").join("outside.mov"))}).to_string(); + let err = client.call("link.open", open_params(json!({"format": "otio-json", "data": climbing}))).unwrap_err(); + assert!(matches!(err, LinkError::PayloadInvalid(_)), "{err:?}"); + // Params that are not an object are still bad_params. + let err = client.call("link.open", json!("nope")).unwrap_err(); + assert!(matches!(err, LinkError::BadParams(_)), "{err:?}"); + assert!(sink.seen.lock().unwrap().is_empty(), "a bad payload must not reach the app: {:?}", sink.seen.lock().unwrap()); + + let otio = open_params(json!({"format": "otio-json", "data": "{\"OTIO_SCHEMA\":\"Timeline.1\"}"})); + let still = open_params(json!({"format": "image-doc", "path": native_path("a.photocraft"), "asset": ASSET})); + for params in [otio, still] { + let reply = client.call("link.open", params.clone()).unwrap(); + assert_eq!(reply, json!({"comp": "c1"})); + let seen = sink.seen.lock().unwrap(); + let (method, got) = seen.last().unwrap(); + assert_eq!(method, "link.open"); + assert_eq!(got, ¶ms, "the app gets the request unchanged"); + } + assert_eq!(sink.seen.lock().unwrap().len(), 2); + + // The wire carries the contract code, not bad_params. + let err = client.call("link.open", open_params(json!({"format": "svg"}))).unwrap_err(); + assert_eq!(err.code(), "payload_invalid"); + // The connection stays usable after a refusal. + client.call("link.open", open_params(json!({"format": "otio-json", "data": "{}"}))).unwrap(); + assert_eq!(sink.seen.lock().unwrap().len(), 3); + + server.shutdown().unwrap(); +} diff --git a/crates/ipc/tests/protocol.rs b/crates/ipc/tests/protocol.rs new file mode 100644 index 000000000..d77a46017 --- /dev/null +++ b/crates/ipc/tests/protocol.rs @@ -0,0 +1,361 @@ +//! Session behaviour. The tests own the sink, the activate trait, and frames. +//! They do not use the client module. +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Read}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::wire::{self, MAX_LINE}; +use craft_ipc::{Endpoint, LinkError, RuntimeDir}; + +const PAYLOAD: &[u8] = b"craft-frame"; +const TICKET: &str = "00112233445566778899aabbccddeeff"; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct ReplySink { + result: Mutex, + seen: Mutex>, +} + +impl ControlSink for ReplySink { + fn submit(&self, method: String, _params: Value) -> Option> { + self.seen.lock().unwrap().push(method); + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(self.result.lock().unwrap().clone()); + Some(rx) + } +} + +struct SwitchActivate { + mode: Mutex, + seen_token: Mutex>>, + seen_pid: Mutex>>, +} + +impl Activate for SwitchActivate { + fn activate(&self, token: Option, from_pid: Option, _open: Option) -> Activation { + *self.seen_token.lock().unwrap() = Some(token); + *self.seen_pid.lock().unwrap() = Some(from_pid); + *self.mode.lock().unwrap() + } +} + +struct TestFrames { + open_count: Mutex, + closed: Mutex>, +} + +impl Frames for TestFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + *self.open_count.lock().unwrap() += 1; + Ok(FrameOpen { ticket: TICKET.into(), w: 4, h: 2, stride: 16, len: PAYLOAD.len() as u64 }) + } + + fn render(&self, ticket: &str, header: &Value) -> Result<(Value, Vec), LinkError> { + if ticket != TICKET { + return Err(LinkError::BadParams("unknown ticket".into())); + } + let req = header.get("req").cloned().unwrap_or(Value::Null); + Ok((json!({"req": req, "epoch": 3, "w": 4, "h": 2, "stride": 16, "format": "rgba8_premul", "render_ms": 1.5}), PAYLOAD.to_vec())) + } + + fn close_ticket(&self, ticket: &str) { + self.closed.lock().unwrap().push(ticket.to_string()); + } +} + +struct Harness { + tmp: Tmp, + server: Server, + socket: PathBuf, + token: String, + sink: Arc, + activate: Arc, + frames: Arc, +} + +fn boot() -> Harness { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-proto-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let sink = Arc::new(ReplySink { result: Mutex::new(json!({"commands": ["cut"]})), seen: Mutex::new(Vec::new()) }); + let activate = Arc::new(SwitchActivate { mode: Mutex::new(Activation::Requested), seen_token: Mutex::new(None), seen_pid: Mutex::new(None) }); + let frames = Arc::new(TestFrames { open_count: Mutex::new(0), closed: Mutex::new(Vec::new()) }); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.8.3".into(), + instance: None, + runtime: Some(runtime), + tcp_port: None, + sink: Arc::clone(&sink) as Arc, + activate: Arc::clone(&activate) as Arc, + frames: Arc::clone(&frames) as Arc, + }) + .unwrap(); + let socket = match &server.record().endpoint { + Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + }; + let token = server.record().token.clone(); + Harness { tmp: Tmp(path), server, socket, token, sink, activate, frames } +} + +struct Client { + reader: BufReader, +} + +impl Client { + fn connect(socket: &Path, token: &str) -> Self { + let stream = UnixStream::connect(socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut reader = BufReader::new(stream); + let req = json!({"id": 0, "method": "auth", "params": {"token": token}}); + wire::write_line(reader.get_mut(), &req).unwrap(); + let reply = read_value(&mut reader).unwrap_or_else(|| panic!("no auth reply")); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"]["authenticated"], json!(true), "{reply}"); + Self { reader } + } + + fn call(&mut self, id: i64, method: &str, params: Value) -> Value { + let req = json!({"id": id, "method": method, "params": params}); + wire::write_line(self.reader.get_mut(), &req).unwrap(); + read_value(&mut self.reader).unwrap_or_else(|| panic!("no reply for {method}")) + } + + fn set_timeout(&mut self, timeout: Duration) { + self.reader.get_mut().set_read_timeout(Some(timeout)).unwrap(); + } +} + +fn read_value(reader: &mut BufReader) -> Option { + match wire::read_line(reader, MAX_LINE) { + Ok(Some(bytes)) => Some(serde_json::from_slice(&bytes).unwrap()), + Ok(None) => None, + Err(_) => None, + } +} + +fn read_raw_line(stream: &mut impl Read) -> Vec { + let mut out = Vec::new(); + let mut byte = [0u8; 1]; + loop { + stream.read_exact(&mut byte).unwrap(); + if byte[0] == b'\n' { + break; + } + out.push(byte[0]); + } + if out.last() == Some(&b'\r') { + out.pop(); + } + out +} + +#[test] +fn legacy_command_keeps_the_request_id() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let reply = client.call(41, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(41), "{reply}"); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"], json!({"commands": ["cut"]}), "{reply}"); + assert_eq!(harness.sink.seen.lock().unwrap().as_slice(), ["engine.commands"]); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// A sink reply longer than `MAX_LINE` cannot be read by a client. The server answers that request +/// with `payload_too_large` and the same id, and the connection keeps working. +#[test] +fn a_reply_over_the_line_limit_is_answered_with_an_error_and_the_connection_survives() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + *harness.sink.result.lock().unwrap() = json!({"blob": "x".repeat(MAX_LINE + 16)}); + let reply = client.call(5, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(5), "{reply}"); + assert_eq!(reply["ok"], json!(false), "{reply}"); + assert_eq!(reply["error"]["code"], json!("payload_too_large"), "{reply}"); + *harness.sink.result.lock().unwrap() = json!({"commands": ["cut"]}); + let next = client.call(6, "engine.commands", json!({})); + assert_eq!(next["ok"], json!(true), "the connection must still work: {next}"); + assert_eq!(next["id"], json!(6), "{next}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// The server polls each connection with a 250 ms read timeout. On Linux that timeout surfaces as +/// `WouldBlock` ("Resource temporarily unavailable"), which must mean "idle, keep waiting". A +/// connection that stays quiet for longer than one poll must still answer the next request. +#[test] +fn a_connection_idle_for_longer_than_the_read_poll_still_answers() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let first = client.call(1, "engine.commands", json!({})); + assert_eq!(first["ok"], json!(true), "{first}"); + std::thread::sleep(Duration::from_millis(900)); + let second = client.call(2, "engine.commands", json!({})); + assert_eq!(second["id"], json!(2), "the idle connection must still be open: {second}"); + assert_eq!(second["ok"], json!(true), "{second}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +/// A sink value that already has `ok` is the finished reply. The server inserts `id` and does not nest it. +#[test] +fn sink_reply_with_ok_is_not_wrapped() { + let harness = boot(); + *harness.sink.result.lock().unwrap() = json!({"ok": true, "result": null}); + let mut client = Client::connect(&harness.socket, &harness.token); + let reply = client.call(3, "engine.commands", json!({})); + assert_eq!(reply["id"], json!(3), "{reply}"); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"], Value::Null, "{reply}"); + assert!(reply.get("error").is_none(), "{reply}"); + + *harness.sink.result.lock().unwrap() = json!({"ok": false, "error": "missing command"}); + let bad = client.call(4, "engine.commands", json!({})); + assert_eq!(bad["id"], json!(4), "{bad}"); + assert_eq!(bad["ok"], json!(false), "{bad}"); + assert_eq!(bad["error"], json!("missing command"), "{bad}"); + assert!(bad.get("result").is_none(), "an error reply must stay flat: {bad}"); + harness.server.shutdown().unwrap(); +} + +#[test] +fn events_wait_for_subscribe_then_one_event_is_delivered() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + client.set_timeout(Duration::from_millis(200)); + harness.server.emit("doc.changed", json!({"doc_id": "a"})).unwrap(); + assert!(read_value(&mut client.reader).is_none(), "event before subscribe"); + client.set_timeout(Duration::from_secs(2)); + let sub = client.call(2, "events.subscribe", json!({"events": ["doc.changed"]})); + assert_eq!(sub["id"], json!(2), "{sub}"); + assert_eq!(sub["result"], json!({"seq": 0}), "{sub}"); + harness.server.emit("doc.changed", json!({"doc_id": "a"})).unwrap(); + let event = read_value(&mut client.reader).unwrap_or_else(|| panic!("missing event")); + assert_eq!(event["event"], json!("doc.changed"), "{event}"); + assert_eq!(event["seq"], json!(1), "{event}"); + assert_eq!(event["data"], json!({"doc_id": "a"}), "{event}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +#[test] +fn oversized_event_delivers_an_overflow_marker() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let sub = client.call(2, "events.subscribe", json!({"events": ["doc.changed"]})); + assert_eq!(sub["result"]["seq"], json!(0), "{sub}"); + harness.server.emit("doc.changed", json!({"blob": "x".repeat(MAX_LINE)})).unwrap(); + let event = read_value(&mut client.reader).expect("missing overflow event"); + assert_eq!(event["event"], json!("events.overflow")); + assert!(event["data"]["since"].is_number()); + harness.server.shutdown().unwrap(); +} + +#[test] +fn bulk_round_trip_returns_the_frame_bytes() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let hello = client.call( + 1, + "link.hello", + json!({"client": {"app": "fc", "app_version": "0.9.1", "pid": 7, "instance": "00000000-0000-4000-8000-000000000001"}, "protocol": {"min": 1, "max": 1}}), + ); + assert_eq!(hello["ok"], json!(true), "{hello}"); + assert_eq!(hello["result"]["app"], json!("ec"), "{hello}"); + assert_eq!(hello["result"]["app_version"], json!("0.8.3"), "{hello}"); + assert_eq!(hello["result"]["protocol"], json!(1), "{hello}"); + assert_eq!(hello["result"]["caps"], json!(["events", "activate", "link.receive", "frames"]), "{hello}"); + let opened = client.call(8, "frames.open", json!({"link_id": "L", "format": "rgba8_premul", "scale": 1.0})); + assert_eq!(opened["ok"], json!(true), "{opened}"); + assert_eq!(opened["result"]["ticket"], json!(TICKET), "{opened}"); + assert_eq!(opened["result"]["w"], json!(4), "{opened}"); + assert_eq!(opened["result"]["len"], json!(PAYLOAD.len()), "{opened}"); + + let mut bulk = UnixStream::connect(&harness.socket).unwrap(); + bulk.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + bulk.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + let auth = json!({"id": 0, "method": "auth", "params": {"token": harness.token}}); + wire::write_line(&mut bulk, &auth).unwrap(); + let auth_reply: Value = serde_json::from_slice(&read_raw_line(&mut bulk)).unwrap(); + assert_eq!(auth_reply["ok"], json!(true), "{auth_reply}"); + let attach = json!({"id": 9, "method": "bulk.attach", "params": {"ticket": TICKET}}); + wire::write_line(&mut bulk, &attach).unwrap(); + let attach_reply: Value = serde_json::from_slice(&read_raw_line(&mut bulk)).unwrap(); + assert_eq!(attach_reply["ok"], json!(true), "{attach_reply}"); + assert_eq!(attach_reply["id"], json!(9), "{attach_reply}"); + + let header = json!({"req": 7, "time_s": 1.5, "epoch": 3, "region": null}); + wire::write_cfrq(&mut bulk, &header).unwrap(); + let (got, payload) = wire::read_cfrm(&mut bulk).unwrap(); + assert_eq!(got["req"], json!(7), "{got}"); + assert_eq!(payload, PAYLOAD); + + drop(bulk); + drop(client); + harness.server.shutdown().unwrap(); + assert_eq!(harness.frames.closed.lock().unwrap().as_slice(), [TICKET]); + drop(harness.tmp); +} + +#[test] +fn activate_returns_requested_from_the_trait() { + let harness = boot(); + let mut client = Client::connect(&harness.socket, &harness.token); + let params = json!({"token": "wayland-token", "from_pid": 4242, "open": {"link_id": "abc"}}); + let reply = client.call(3, "app.activate", params); + assert_eq!(reply["ok"], json!(true), "{reply}"); + assert_eq!(reply["result"]["activated"], json!("requested"), "{reply}"); + assert_eq!(harness.activate.seen_token.lock().unwrap().clone().unwrap().as_deref(), Some("wayland-token")); + assert_eq!(harness.activate.seen_pid.lock().unwrap().unwrap(), Some(4242)); + + *harness.activate.mode.lock().unwrap() = Activation::No; + let reply = client.call(4, "app.activate", json!({"token": null})); + assert_eq!(reply["result"]["activated"], json!(false), "{reply}"); + + *harness.activate.mode.lock().unwrap() = Activation::Yes; + let reply = client.call(5, "app.activate", json!({})); + assert_eq!(reply["result"]["activated"], json!(true), "{reply}"); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} + +#[test] +fn the_17th_connection_is_refused_and_does_not_stay_open() { + let harness = boot(); + let mut held = Vec::new(); + // Authenticated connections fill the cap. Unauthenticated ones are limited to a smaller pool. + for _ in 0..wire::MAX_CONNECTIONS { + held.push(Client::connect(&harness.socket, &harness.token)); + } + let mut extra = UnixStream::connect(&harness.socket).unwrap(); + extra.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut line = String::new(); + BufReader::new(&mut extra).read_line(&mut line).unwrap(); + let value: Value = serde_json::from_str(line.trim()).unwrap(); + assert_eq!(value["ok"], json!(false), "{line}"); + assert_eq!(value["error"], json!("too many control connections"), "{line}"); + drop(held); + harness.server.shutdown().unwrap(); + drop(harness.tmp); +} diff --git a/crates/ipc/tests/provider_stall.rs b/crates/ipc/tests/provider_stall.rs new file mode 100644 index 000000000..3407df05d --- /dev/null +++ b/crates/ipc/tests/provider_stall.rs @@ -0,0 +1,248 @@ +//! A provider that never returns must not hold up shutdown. `Assets` and `Activate` calls run on +//! helper threads with a deadline: the caller gets a timeout reply, the helper is detached, and a +//! normal call still works afterwards. +#![cfg(unix)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use craft_ipc::asset::{Described, Refreshed, Resolved}; +use craft_ipc::moniker::Moniker; +use craft_ipc::record::AssetKind; +use craft_ipc::refresh::FileState; +use craft_ipc::server::{Activate, Activation, Assets, ControlSink, Extensions, FrameOpen, Frames, Limits, Server, ServerConfig}; +use craft_ipc::{Endpoint, LinkError, RuntimeDir}; + +const DEADLINE: Duration = Duration::from_millis(400); +/// `link.refresh` with this link id blocks until the test releases it. `app.activate` with this +/// token blocks the same way. +const STUCK: &str = "stuck"; + +/// Blocks while `release` is false, so a test can end its helper threads when it is done. +struct Gate { + release: AtomicBool, + entered: AtomicUsize, +} + +impl Gate { + fn wait(&self) { + self.entered.fetch_add(1, Ordering::SeqCst); + while !self.release.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +struct StuckAssets(Arc); + +impl Assets for StuckAssets { + fn resolve(&self, _asset: &Moniker) -> Result { + Err(LinkError::Unknown("no such asset".into())) + } + + fn describe(&self, asset: &Moniker) -> Result { + Ok(Described { + asset: asset.clone(), + name: "x".into(), + kind: AssetKind::Still, + epoch: 1, + w: None, + h: None, + hash: None, + thumb_path: None, + modified_ms: None, + }) + } + + fn refresh(&self, link_id: &str) -> Result { + if link_id == STUCK { + self.0.wait(); + } + Ok(Refreshed::new(link_id, 1, FileState::Current, "/tmp/x", None)) + } +} + +struct StuckActivate(Arc); + +impl Activate for StuckActivate { + fn activate(&self, token: Option, _from_pid: Option, _open: Option) -> Activation { + if token.as_deref() == Some(STUCK) { + self.0.wait(); + } + Activation::Yes + } +} + +struct Sink; + +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + None + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +struct Rig { + server: Server, + gate: Arc, + reader: BufReader, + dir: PathBuf, +} + +impl Rig { + fn boot(label: &str, deadline: Duration) -> Self { + let dir = std::env::temp_dir().join(format!("cl-stall-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let gate = Arc::new(Gate { release: AtomicBool::new(false), entered: AtomicUsize::new(0) }); + let server = Server::start_with_limits( + ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(RuntimeDir::at(&dir).unwrap()), + tcp_port: None, + sink: Arc::new(Sink), + activate: Arc::new(StuckActivate(Arc::clone(&gate))), + frames: Arc::new(NoFrames), + }, + Extensions { assets: Some(Arc::new(StuckAssets(Arc::clone(&gate)))), caps: Vec::new() }, + Limits { provider_deadline: deadline, ..Limits::default() }, + ) + .unwrap(); + let Endpoint::Unix { path } = &server.record().endpoint else { panic!("unix endpoint expected") }; + let stream = UnixStream::connect(path).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + let mut reader = BufReader::new(stream); + let token = server.record().token.clone(); + send(&mut reader, &json!({"id": 0, "method": "auth", "params": {"token": token}})); + assert_eq!(read(&mut reader)["ok"], json!(true)); + Self { server, gate, reader, dir } + } + + fn call(&mut self, id: u64, method: &str, params: Value) -> Value { + send(&mut self.reader, &json!({"id": id, "method": method, "params": params})); + read(&mut self.reader) + } + + /// Shut down on a thread and return how long it took. A hang fails the test after `limit`. + fn shutdown_within(self, limit: Duration) -> Duration { + let Self { server, gate, reader, dir } = self; + let (done, finished) = mpsc::channel(); + let started = Instant::now(); + std::thread::spawn(move || { + let result = server.shutdown(); + let _ = done.send(result.is_ok()); + }); + let waited = finished.recv_timeout(limit); + // Let the detached helpers end, so the test process leaves no spinning threads. + gate.release.store(true, Ordering::SeqCst); + drop(reader); + let _ = std::fs::remove_dir_all(dir); + assert!(waited.expect("shutdown did not finish: a stuck provider held it up"), "shutdown failed"); + started.elapsed() + } +} + +fn send(reader: &mut BufReader, value: &Value) { + let mut bytes = serde_json::to_vec(value).unwrap(); + bytes.push(b'\n'); + reader.get_mut().write_all(&bytes).unwrap(); +} + +fn read(reader: &mut BufReader) -> Value { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + serde_json::from_str(&line).unwrap_or_else(|err| panic!("reply {line:?}: {err}")) +} + +fn timed_out(reply: &Value) -> bool { + reply["ok"] == json!(false) && reply["error"]["message"] == json!("timeout") +} + +#[test] +fn a_stuck_asset_call_times_out_and_the_connection_keeps_working() { + let mut rig = Rig::boot("assets", DEADLINE); + let started = Instant::now(); + let reply = rig.call(1, "link.refresh", json!({"link_id": STUCK})); + let waited = started.elapsed(); + assert!(timed_out(&reply), "{reply}"); + assert_eq!(reply["id"], json!(1)); + assert!(waited >= DEADLINE && waited < DEADLINE * 5, "{waited:?}"); + // Other calls on the same connection still work, asset calls included. + let fine = rig.call(2, "link.refresh", json!({"link_id": "ok"})); + assert_eq!(fine["ok"], json!(true), "{fine}"); + assert_eq!(fine["result"]["link_id"], json!("ok")); + let hello = rig.call(3, "link.hello", json!({"protocol": {"min": 1, "max": 1}})); + assert_eq!(hello["ok"], json!(true), "{hello}"); + assert_eq!(rig.gate.entered.load(Ordering::SeqCst), 1); + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn shutdown_does_not_wait_for_a_stuck_asset_call() { + // The deadline is long, so shutdown meets the call still in flight. + let mut rig = Rig::boot("assets-stop", Duration::from_secs(60)); + send(&mut rig.reader, &json!({"id": 1, "method": "link.refresh", "params": {"link_id": STUCK}})); + while rig.gate.entered.load(Ordering::SeqCst) == 0 { + std::thread::sleep(Duration::from_millis(10)); + } + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn a_stuck_activate_call_times_out_and_shutdown_does_not_wait_for_it() { + let mut rig = Rig::boot("activate", DEADLINE); + let reply = rig.call(1, "app.activate", json!({"token": STUCK})); + assert!(timed_out(&reply), "{reply}"); + let fine = rig.call(2, "app.activate", json!({"token": "ok"})); + assert_eq!(fine["result"]["activated"], json!(true), "{fine}"); + // A second stuck call, then shut down while it is in flight. + send(&mut rig.reader, &json!({"id": 3, "method": "app.activate", "params": {"token": STUCK}})); + while rig.gate.entered.load(Ordering::SeqCst) < 2 { + std::thread::sleep(Duration::from_millis(10)); + } + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} + +#[test] +fn stuck_helpers_are_capped_and_the_server_says_busy() { + let mut rig = Rig::boot("cap", Duration::from_millis(60)); + let mut busy_at = None; + for id in 1..=40u64 { + let reply = rig.call(id, "link.refresh", json!({"link_id": STUCK})); + if reply["error"]["code"] == json!("busy") { + busy_at = Some(id); + break; + } + assert!(timed_out(&reply), "call {id}: {reply}"); + } + let busy_at = busy_at.expect("the server kept starting helper threads past its cap"); + assert_eq!(busy_at, 33, "32 stuck helpers are allowed, the next call is refused"); + assert_eq!(rig.gate.entered.load(Ordering::SeqCst), 32, "a refused call starts no thread"); + // A call that needs no helper still works while the cap is full. + let hello = rig.call(100, "link.hello", json!({"protocol": {"min": 1, "max": 1}})); + assert_eq!(hello["ok"], json!(true), "{hello}"); + let took = rig.shutdown_within(Duration::from_secs(3)); + assert!(took < Duration::from_secs(2), "{took:?}"); +} diff --git a/crates/ipc/tests/schemas.rs b/crates/ipc/tests/schemas.rs new file mode 100644 index 000000000..4dbcd251b --- /dev/null +++ b/crates/ipc/tests/schemas.rs @@ -0,0 +1,318 @@ +//! The JSON schemas in `schemas/` and the Rust types +//! must accept the same documents. Each table row is checked against both. +//! The fixtures are stored records. The first one is the FilmCraft to +//! EffectCraft shape written before the asset fields existed. + +use std::path::PathBuf; + +use jsonschema::Validator; +use serde_json::{Value, json}; + +use craft_ipc::InstanceRecord; +use craft_ipc::record::{AssetKind, LinkRecord}; + +fn schema_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schemas") +} + +fn validator(name: &str) -> Validator { + let path = schema_dir().join(name); + let schema: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + jsonschema::options().should_validate_formats(true).build(&schema).unwrap() +} + +fn fixture(name: &str) -> Value { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures").join(name); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +} + +/// Schema verdict and Rust verdict for one document. Both must equal `want`. +fn check_link(schema: &Validator, doc: &Value, want: bool, why: &str) { + let by_schema = schema.is_valid(doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, want, "schema: {why}: {doc}"); + assert_eq!(by_rust, want, "rust: {why}: {doc}"); +} + +#[test] +fn stored_filmcraft_effectcraft_record_stays_valid() { + let schema = validator("link-record.schema.json"); + let doc = fixture("link-record-filmcraft-effectcraft.json"); + check_link(&schema, &doc, true, "legacy fixture"); + let record: LinkRecord = serde_json::from_value(doc.clone()).unwrap(); + assert_eq!(record.target_app, "effectcraft"); + assert_eq!(record.target_comp.as_deref(), Some("comp-1")); + assert_eq!(record.nested_sequence.as_deref(), Some("item-9")); + assert_eq!(record.format.rate_num, Some(24)); + assert_eq!(record.format.duration_ticks, Some(240)); + assert!(record.asset.is_none() && record.producer.is_none()); + // Unknown fields survive a load and save, byte-for-byte as JSON values. + assert_eq!(serde_json::to_value(&record).unwrap(), doc); +} + +#[test] +fn still_record_has_no_video_fields_and_round_trips() { + let schema = validator("link-record.schema.json"); + let doc = fixture("link-record-still.json"); + check_link(&schema, &doc, true, "still fixture"); + let record: LinkRecord = serde_json::from_value(doc.clone()).unwrap(); + assert_eq!(record.target_app, "photocraft"); + assert!(record.nested_sequence.is_none() && record.target_comp.is_none()); + assert_eq!(record.format.rate_num, None); + assert_eq!(record.format.rate_den, None); + assert_eq!(record.format.duration_ticks, None); + assert_eq!(record.asset.as_ref().map(|a| a.kind), Some(AssetKind::Still)); + let out = serde_json::to_value(&record).unwrap(); + assert_eq!(out, doc); + let format = out["format"].as_object().unwrap(); + for key in ["rate_num", "rate_den", "duration_ticks"] { + assert!(!format.contains_key(key), "{key} must not be written as null"); + } +} + +#[test] +fn schema_and_rust_agree_on_edge_cases() { + let schema = validator("link-record.schema.json"); + let legacy = fixture("link-record-filmcraft-effectcraft.json"); + let still = fixture("link-record-still.json"); + + let edit = |base: &Value, f: &dyn Fn(&mut Value)| { + let mut doc = base.clone(); + f(&mut doc); + doc + }; + let cases: Vec<(Value, bool, &str)> = vec![ + (edit(&legacy, &|d| d["target_app"] = json!("photocraft")), true, "any app id"), + (edit(&legacy, &|d| d["target_app"] = json!("Film")), false, "uppercase app id"), + (edit(&legacy, &|d| d["target_app"] = json!("")), false, "empty app id"), + (edit(&legacy, &|d| d["v"] = json!(2)), false, "v 2"), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("nested_sequence"); + }), + true, + "nested_sequence optional", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("duration_ticks"); + }), + true, + "duration optional", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("rate_num"); + }), + false, + "rate_den without rate_num", + ), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("rate_den"); + }), + false, + "rate_num without rate_den", + ), + (edit(&legacy, &|d| d["format"]["rate_num"] = json!(0)), false, "rate_num 0"), + (edit(&legacy, &|d| d["format"]["w"] = json!(0)), false, "w 0"), + ( + edit(&legacy, &|d| { + d["format"].as_object_mut().unwrap().remove("alpha"); + }), + false, + "alpha stays required", + ), + (edit(&legacy, &|d| d["result_epoch"] = json!(0)), true, "epoch 0"), + (edit(&legacy, &|d| d["created"] = json!("yesterday")), false, "bad date"), + (edit(&legacy, &|d| d["created"] = json!("2026-02-31T09:30:00Z")), false, "31 February"), + (edit(&legacy, &|d| d["created"] = json!("2026-02-29T09:30:00Z")), false, "29 February in a common year"), + (edit(&legacy, &|d| d["created"] = json!("2026-04-31T09:30:00+02:00")), false, "31 April"), + (edit(&legacy, &|d| d["created"] = json!("1900-02-29T00:00:00Z")), false, "29 February in a century year"), + (edit(&legacy, &|d| d["created"] = json!("2024-02-29T09:30:00Z")), true, "29 February in a leap year"), + (edit(&legacy, &|d| d["created"] = json!("2000-02-29T09:30:00.250Z")), true, "29 February in a 400 year"), + (edit(&legacy, &|d| d["created"] = json!("2026-12-31T23:59:59-05:30")), true, "last day of the year"), + (edit(&legacy, &|d| d["link_id"] = json!("nope")), false, "bad link id"), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("target_comp"); + }), + false, + "no comp and no asset", + ), + ( + edit(&legacy, &|d| { + d.as_object_mut().unwrap().remove("target_project"); + }), + false, + "target_project required", + ), + ( + edit(&still, &|d| { + d.as_object_mut().unwrap().remove("asset"); + }), + false, + "still without asset or comp", + ), + (edit(&still, &|d| d["asset"]["kind"] = json!("video")), false, "unknown asset kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("vector")), true, "vector kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("sequence")), true, "sequence kind"), + (edit(&still, &|d| d["asset"]["kind"] = json!("comp")), true, "comp kind"), + (edit(&still, &|d| d["asset"]["moniker"] = json!("craft://photocraft/not-a-uuid/image/x")), false, "bad moniker doc"), + (edit(&still, &|d| d["asset"]["moniker"] = json!("photocraft/x")), false, "moniker without scheme"), + ( + edit(&still, &|d| d["asset"]["moniker"] = json!("craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/a/b")), + false, + "moniker with extra segment", + ), + (edit(&still, &|d| d["producer"]["app"] = json!("Bad App")), false, "bad producer app"), + ( + edit(&still, &|d| { + d.as_object_mut().unwrap().remove("producer"); + }), + true, + "producer optional", + ), + (edit(&still, &|d| d["asset"]["extra"] = json!(1)), true, "unknown asset field kept"), + ]; + for (doc, want, why) in &cases { + check_link(&schema, doc, *want, why); + } +} + +const MONIKER_BASE: &str = "craft://photocraft/9f8e7d6c-5b4a-4c3d-8e2f-1a0b9c8d7e6f/image/"; + +fn with_moniker(still: &Value, id: &str) -> Value { + let mut doc = still.clone(); + doc["asset"]["moniker"] = json!(format!("{MONIKER_BASE}{id}")); + doc +} + +/// The moniker id rules of `Moniker::parse` as the schema states them: unreserved characters and +/// two-digit escapes only, 1 to 512 decoded bytes, and decoded bytes that are valid UTF-8. +#[test] +fn schema_and_rust_agree_on_moniker_ids() { + let schema = validator("link-record.schema.json"); + let still = fixture("link-record-still.json"); + let escaped = |count: usize| "%41".repeat(count); + let cases: Vec<(String, bool, &str)> = vec![ + ("layer-3".into(), true, "plain"), + ("~._-AZaz09".into(), true, "every unreserved class"), + ("a%20b".into(), true, "escaped space"), + ("a%2fb".into(), true, "lowercase escape"), + ("%C3%A9".into(), true, "two byte character"), + ("%c3%a9".into(), true, "two byte character in lowercase hex"), + ("%E2%82%AC".into(), true, "three byte character"), + ("%F0%9F%98%80".into(), true, "four byte character"), + ("%F4%8F%BF%BF".into(), true, "highest scalar"), + ("%ED%9F%BF".into(), true, "just below the surrogates"), + ("%00".into(), true, "escaped NUL is a byte like any other"), + ("%GG".into(), false, "bad escape"), + ("%4".into(), false, "truncated escape"), + ("%".into(), false, "lone percent"), + ("a b".into(), false, "raw space"), + ("caf\u{e9}".into(), false, "raw non-ASCII"), + ("a:b".into(), false, "raw colon"), + ("a%2".into(), false, "truncated escape after text"), + ("".into(), false, "empty id"), + ("%FF".into(), false, "invalid UTF-8 byte"), + ("%C3".into(), false, "truncated two byte character"), + ("%80".into(), false, "lone continuation byte"), + ("%C0%80".into(), false, "overlong two byte form"), + ("%E0%80%80".into(), false, "overlong three byte form"), + ("%ED%A0%80".into(), false, "surrogate"), + ("%F0%80%80%80".into(), false, "overlong four byte form"), + ("%F4%90%80%80".into(), false, "above U+10FFFF"), + ("%F5%80%80%80".into(), false, "lead byte F5"), + ("a/b".into(), false, "raw slash"), + ("a?b".into(), false, "raw query"), + ("a#b".into(), false, "raw fragment"), + ("a".repeat(512), true, "512 raw bytes"), + ("a".repeat(513), false, "513 raw bytes"), + (escaped(512), true, "512 escaped bytes"), + (escaped(513), false, "513 escaped bytes"), + (format!("{}%C3%A9", "a".repeat(510)), true, "512 decoded bytes ending in a two byte character"), + (format!("{}%C3%A9", "a".repeat(511)), false, "513 decoded bytes ending in a two byte character"), + ("%C3%A9".repeat(256), true, "256 two byte characters make 512 bytes"), + ("%C3%A9".repeat(257), false, "257 two byte characters make 514 bytes"), + ]; + for (id, want, why) in &cases { + check_link(&schema, &with_moniker(&still, id), *want, why); + } + + // Every lead byte against a spread of second and third bytes: the two verdicts must match. + let probes = [0x00u8, 0x41, 0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0]; + let mut seen_valid = 0; + for lead in 0u8..=0xFF { + for second in probes { + for tail in [&[][..], &[0x80][..], &[0x80, 0x80][..]] { + for hex in [|b: u8| format!("%{b:02X}"), |b: u8| format!("%{b:02x}")] { + let id: String = [lead, second].iter().chain(tail).map(|byte| hex(*byte)).collect(); + let doc = with_moniker(&still, &id); + let by_schema = schema.is_valid(&doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, by_rust, "bytes {lead:02X} {second:02X} {tail:02X?} as {id}"); + seen_valid += usize::from(by_schema); + } + } + } + } + assert!(seen_valid > 100, "the probe must reach valid sequences too, got {seen_valid}"); +} + +fn instance_doc() -> Value { + let endpoint = if cfg!(windows) { + json!({"kind": "pipe", "name": "craft-hub-4242"}) + } else { + json!({"kind": "unix", "path": "/run/user/1000/craft/hub-4242.sock"}) + }; + json!({ + "v": 1, + "app": "hub", + "app_version": "0.1.0", + "pid": 4242, + "instance": "550e8400-e29b-41d4-a716-446655440000", + "endpoint": endpoint, + "token": "ab".repeat(32), + "started": 1_760_000_000u64, + "protocol": {"link": 1}, + "docs": [] + }) +} + +fn check_instance(schema: &Validator, doc: &Value, want: bool, why: &str) { + let by_schema = schema.is_valid(doc); + let by_rust = serde_json::from_value::(doc.clone()).is_ok_and(|r| r.validate().is_ok()); + assert_eq!(by_schema, want, "schema: {why}: {doc}"); + assert_eq!(by_rust, want, "rust: {why}: {doc}"); +} + +#[test] +fn instance_record_caps_agree_between_schema_and_rust() { + let schema = validator("instance-record.schema.json"); + let base = instance_doc(); + check_instance(&schema, &base, true, "no caps (existing records)"); + for (caps, want, why) in [ + (json!(["hub"]), true, "hub"), + (json!(["hub", "assets"]), true, "two caps"), + (json!([]), true, "empty caps"), + (json!(["Hub"]), false, "uppercase"), + (json!([""]), false, "empty cap"), + (json!(["hub", "hub"]), false, "duplicate"), + (json!("hub"), false, "not an array"), + (json!([1]), false, "not strings"), + ] { + let mut doc = base.clone(); + doc["caps"] = caps; + check_instance(&schema, &doc, want, why); + } + let mut hub: InstanceRecord = serde_json::from_value(base).unwrap(); + assert!(hub.caps().is_empty()); + hub.add_cap("hub").unwrap(); + hub.add_cap("hub").unwrap(); + assert_eq!(hub.caps(), vec!["hub".to_string()]); + assert!(hub.add_cap("Not Valid").is_err()); + let value = serde_json::to_value(&hub).unwrap(); + assert_eq!(value["caps"], json!(["hub"])); + assert!(schema.is_valid(&value)); +} diff --git a/crates/ipc/tests/security.rs b/crates/ipc/tests/security.rs new file mode 100644 index 000000000..0c42c599a --- /dev/null +++ b/crates/ipc/tests/security.rs @@ -0,0 +1,290 @@ +//! Auth refusal on the local socket. These tests speak bytes themselves. +//! They do not use the client module. +#![cfg(unix)] + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::{Value, json}; + +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::{LinkError, RuntimeDir}; + +struct Tmp(PathBuf); + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct Sink; + +impl ControlSink for Sink { + fn submit(&self, _method: String, _params: Value) -> Option> { + let (tx, rx) = std::sync::mpsc::channel(); + let _ = tx.send(json!({})); + Some(rx) + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + + fn render(&self, _ticket: &str, _header: &Value) -> Result<(Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + + fn close_ticket(&self, _ticket: &str) {} +} + +struct Running { + _tmp: Tmp, + server: Server, + socket: PathBuf, + token: String, +} + +fn boot() -> Running { + boot_with(None) +} + +fn boot_with(tcp_port: Option) -> Running { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-sec-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + let runtime = RuntimeDir::at(&path).unwrap(); + let server = Server::start(ServerConfig { + app: "ec".into(), + app_version: "0.0.1".into(), + instance: None, + runtime: Some(runtime), + tcp_port, + sink: Arc::new(Sink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(); + let socket = match &server.record().endpoint { + craft_ipc::Endpoint::Unix { path } => PathBuf::from(path), + other => panic!("expected a unix endpoint, got {other:?}"), + }; + let token = server.record().token.clone(); + Running { _tmp: Tmp(path), server, socket, token } +} + +fn read_closed_line(socket: &Path, payload: &[u8]) -> Value { + let mut stream = std::os::unix::net::UnixStream::connect(socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.write_all(payload).unwrap(); + let mut text = String::new(); + stream.read_to_string(&mut text).unwrap_or_else(|err| panic!("socket did not close after auth failure: {err}")); + let lines: Vec<_> = text.lines().filter(|line| !line.is_empty()).collect(); + assert_eq!(lines.len(), 1, "{text}"); + assert!(!lines[0].contains("bad JSON"), "{text}"); + serde_json::from_str(lines[0]).unwrap() +} + +fn assert_auth_failure(value: &Value) { + assert_eq!(value["id"], json!(0), "{value}"); + assert_eq!(value["ok"], json!(false), "{value}"); + assert_eq!(value["error"], json!("authentication required"), "{value}"); +} + +#[test] +fn no_token_is_auth_failure_not_bad_json() { + let running = boot(); + let value = read_closed_line(&running.socket, b"{\"id\":1,\"method\":\"engine.commands\",\"params\":{}}\n"); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn wrong_token_is_auth_failure() { + let running = boot(); + let mut bad = "ab".repeat(32); + if bad == running.token { + bad = "cd".repeat(32); + } + let line = json!({"id": 4, "method": "auth", "params": {"token": bad}}); + let mut bytes = serde_json::to_vec(&line).unwrap(); + bytes.push(b'\n'); + let value = read_closed_line(&running.socket, &bytes); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn malformed_first_line_is_auth_failure() { + let running = boot(); + let value = read_closed_line(&running.socket, b"{\n"); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_is_off_when_the_port_is_none() { + let running = boot(); + assert_eq!(running.server.tcp_port(), None); + running.server.shutdown().unwrap(); +} + +fn tcp_exchange(port: u16, payload: &[u8]) -> String { + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(2))).unwrap(); + stream.write_all(payload).unwrap(); + let mut text = String::new(); + let _ = stream.read_to_string(&mut text); + text +} + +#[test] +fn tcp_without_a_token_is_refused_and_runs_nothing() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + assert_ne!(port, 0); + let text = tcp_exchange(port, b"{\"id\":7,\"method\":\"engine.commands\",\"params\":{}}\n"); + let first = text.lines().find(|line| !line.is_empty()).unwrap_or_else(|| panic!("tcp closed with no refusal line")); + assert_auth_failure(&serde_json::from_str(first).unwrap()); + assert!(!text.contains("\"result\""), "{text}"); + let bad_json = tcp_exchange(port, b"{\n"); + assert!(!bad_json.contains("bad JSON"), "{bad_json}"); + assert!(bad_json.contains("authentication required"), "{bad_json}"); + let mut wrong = "ab".repeat(32); + if wrong == running.token { + wrong = "cd".repeat(32); + } + let line = json!({"id": 0, "method": "auth", "params": {"token": wrong}}); + let mut bytes = serde_json::to_vec(&line).unwrap(); + bytes.push(b'\n'); + assert_auth_failure(&serde_json::from_str(tcp_exchange(port, &bytes).lines().next().unwrap()).unwrap()); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_with_the_record_token_works_and_the_record_publishes_the_port() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + match &running.server.record().endpoint { + craft_ipc::Endpoint::Unix { .. } => {} + other => panic!("the instance record must keep the unix socket, got {other:?}"), + } + assert_eq!(running.server.record().tcp_port(), Some(port)); + let mut record = running.server.record().clone(); + record.endpoint = craft_ipc::Endpoint::Tcp { port }; + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(3))).unwrap(); + let auth = json!({"id": 0, "method": "auth", "params": {"token": running.token}}); + let mut bytes = serde_json::to_vec(&auth).unwrap(); + bytes.extend_from_slice(b"\n{\"id\":7,\"method\":\"engine.commands\",\"params\":{}}\n"); + stream.write_all(&bytes).unwrap(); + let mut buf = Vec::new(); + while buf.iter().filter(|b| **b == b'\n').count() < 2 { + let mut byte = [0u8; 1]; + assert_ne!(stream.read(&mut byte).unwrap(), 0, "tcp closed before a reply"); + buf.push(byte[0]); + } + let text = String::from_utf8(buf).unwrap(); + let mut lines = text.lines(); + let ok: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + assert_eq!(ok["ok"], json!(true), "{ok}"); + let value: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + assert_eq!(value["id"], json!(7), "{value}"); + assert_eq!(value["ok"], json!(true), "{value}"); + assert_eq!(value["result"], json!({}), "{value}"); + // The library client does the same from the published record. + let mut client = craft_ipc::client::Client::connect_tcp(running.server.record()).unwrap(); + assert_eq!(client.call("engine.commands", json!({})).unwrap(), json!({})); + running.server.shutdown().unwrap(); +} + +#[test] +fn tcp_is_not_published_when_it_is_off() { + let running = boot(); + assert_eq!(running.server.record().tcp_port(), None); + assert!(craft_ipc::client::Client::connect_tcp(running.server.record()).is_err()); + running.server.shutdown().unwrap(); +} + +#[test] +fn silent_tcp_connections_hit_the_unauthenticated_cap() { + let running = boot_with(Some(0)); + let port = running.server.tcp_port().expect("tcp port"); + let silent: Vec<_> = (0..craft_ipc::wire::MAX_UNAUTHENTICATED).map(|_| std::net::TcpStream::connect(("127.0.0.1", port)).unwrap()).collect(); + std::thread::sleep(Duration::from_millis(400)); + let text = tcp_exchange(port, b""); + assert!(text.contains("too many unauthenticated connections"), "{text}"); + drop(silent); + running.server.shutdown().unwrap(); +} + +#[test] +fn other_uid_is_refused_when_the_env_is_set() { + let Ok(user) = std::env::var("CRAFT_TEST_OTHER_USER") else { + return; + }; + let running = boot(); + // 0600 blocks a non-owner before the peer check. Loosen it so this user + // can connect. The assertion is still the auth failure line. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&running.socket, std::fs::Permissions::from_mode(0o666)).unwrap(); + } + let script = r#" +import socket, sys +s = socket.socket(socket.AF_UNIX) +s.settimeout(5) +s.connect(sys.argv[1]) +token = sys.argv[2] +msg = '{"id":1,"method":"auth","params":{"token":"%s"}}\n' % token +s.sendall(msg.encode()) +data = b"" +while b"\n" not in data: + chunk = s.recv(4096) + if not chunk: + break + data += chunk +sys.stdout.buffer.write(data) +"#; + let output = run_as(&user, script, &running.socket, &running.token); + if !output.status.success() { + panic!("run as {user} failed: status {:?} stderr {}", output.status.code(), String::from_utf8_lossy(&output.stderr)); + } + let text = String::from_utf8_lossy(&output.stdout); + assert!(!text.contains("bad JSON"), "{text}"); + let value: Value = serde_json::from_str(text.lines().next().unwrap()).unwrap(); + assert_auth_failure(&value); + running.server.shutdown().unwrap(); +} + +fn run_as(user: &str, script: &str, socket: &Path, token: &str) -> std::process::Output { + let args = ["-u", user, "--", "python3", "-c", script]; + match std::process::Command::new("runuser").args(args).arg(socket).arg(token).output() { + Ok(output) => output, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + std::process::Command::new("sudo").args(args).arg(socket).arg(token).output().unwrap_or_else(|err| { + panic!("sudo failed to start: {err}"); + }) + } + Err(err) => panic!("runuser failed to start: {err}"), + } +} diff --git a/crates/ipc/tests/watch.rs b/crates/ipc/tests/watch.rs new file mode 100644 index 000000000..185990e92 --- /dev/null +++ b/crates/ipc/tests/watch.rs @@ -0,0 +1,361 @@ +//! Scheduled polling. A `ManualTicker` stands in for the clock, so no test +//! sleeps to wait for an interval. Real-time waits only bound a failure. + +use std::fs::OpenOptions; +use std::path::PathBuf; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde_json::json; + +use craft_ipc::client::{Client, HelloClient}; +use craft_ipc::refresh::{Change, FileState, Tracker, write_atomic}; +use craft_ipc::server::{Activate, Activation, ControlSink, FrameOpen, Frames, Server, ServerConfig}; +use craft_ipc::watch::{DEEP_INTERVAL, IntervalTicker, Intervals, ManualTicker, Notice, SHALLOW_INTERVAL, Ticker, Watcher}; +use craft_ipc::{Discovery, LinkError, RuntimeDir}; + +const LIMIT: Duration = Duration::from_secs(10); +const SECOND: Duration = Duration::from_secs(1); + +struct Tmp(PathBuf); + +impl Tmp { + fn new(label: &str) -> Self { + // Clock ticks need not be distinct across parallel tests. + static NEXT_TMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let serial = NEXT_TMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("cl-watch-{label}-{}-{serial}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn forward(tx: Sender) -> impl Fn(Notice) + Send + Sync + 'static { + let tx = Mutex::new(tx); + move |notice| { + let _ = tx.lock().unwrap().send(notice); + } +} + +struct Rig { + _tmp: Tmp, + file: PathBuf, + clock: Arc, + notices: Receiver, + watcher: Watcher, +} + +impl Rig { + /// A watcher on `a.png` with content "one" and epoch 4, on the contract intervals. + fn new(label: &str) -> Self { + let tmp = Tmp::new(label); + let file = tmp.0.join("a.png"); + std::fs::write(&file, b"one").unwrap(); + let clock = Arc::new(ManualTicker::new()); + let (tx, notices) = channel(); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), forward(tx)).unwrap(); + watcher.watch("link-1", Tracker::new(&file, 4).unwrap()).unwrap(); + Self { _tmp: tmp, file, clock, notices, watcher } + } + + /// Move time and wait until the watcher thread has handled it. + fn advance(&self, by: Duration) { + self.clock.advance(by); + assert!(self.clock.settle(LIMIT), "the watcher did not settle"); + } + + fn quiet(&self) { + let got = self.notices.try_recv(); + assert!(got.is_err(), "unexpected notice {got:?}"); + } + + fn next(&self) -> Notice { + self.notices.try_recv().expect("a notice was due") + } +} + +fn changed(link: &str, state: FileState, epoch: u64, bumped: bool, deep: bool) -> Notice { + Notice::Changed { link_id: link.into(), change: Change { state, epoch, bumped }, deep } +} + +#[test] +fn contract_intervals_are_one_and_thirty_seconds() { + assert_eq!(SHALLOW_INTERVAL, Duration::from_secs(1)); + assert_eq!(DEEP_INTERVAL, Duration::from_secs(30)); + assert_eq!(Intervals::default(), Intervals { shallow: SHALLOW_INTERVAL, deep: DEEP_INTERVAL }); +} + +#[test] +fn edit_on_disk_bumps_the_epoch_on_the_first_shallow_tick_and_not_before() { + let rig = Rig::new("shallow"); + write_atomic(&rig.file, b"two!").unwrap(); + rig.advance(Duration::from_millis(500)); + rig.advance(Duration::from_millis(499)); + rig.quiet(); + assert_eq!(rig.watcher.poll_counts().shallow, 0, "no poll before the 1 s interval"); + rig.advance(Duration::from_millis(1)); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, false)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (1, 0)); + // No further edit, no further notice, and the next poll comes one interval later. + rig.advance(Duration::from_millis(999)); + assert_eq!(rig.watcher.poll_counts().shallow, 1); + rig.advance(Duration::from_millis(1)); + assert_eq!(rig.watcher.poll_counts().shallow, 2); + rig.quiet(); + // A second edit bumps again. + write_atomic(&rig.file, b"three").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 6, true, false)); +} + +#[test] +fn deep_poll_runs_at_thirty_seconds_and_finds_an_edit_that_keeps_size_and_mtime() { + let rig = Rig::new("deep"); + let mtime = std::fs::metadata(&rig.file).unwrap().modified().unwrap(); + std::fs::write(&rig.file, b"two").unwrap(); + OpenOptions::new().write(true).open(&rig.file).unwrap().set_modified(mtime).unwrap(); + for _ in 1..30 { + rig.advance(SECOND); + } + rig.quiet(); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (29, 0), "shallow polls are blind to this edit"); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, true)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (29, 1), "the deep poll replaces the shallow poll on its tick"); + // The schedule goes on: shallow at 31 s, deep again at 60 s. + rig.advance(SECOND); + assert_eq!(rig.watcher.poll_counts().shallow, 30); + rig.advance(Duration::from_secs(29)); + assert_eq!(rig.watcher.poll_counts().deep, 2); +} + +#[test] +fn a_large_time_jump_is_one_deep_poll_and_the_grid_stays_aligned() { + let rig = Rig::new("jump"); + rig.advance(Duration::from_secs(100)); + let counts = rig.watcher.poll_counts(); + assert_eq!((counts.shallow, counts.deep), (0, 1)); + rig.advance(Duration::from_millis(500)); + assert_eq!(rig.watcher.poll_counts().shallow, 0, "100.5 s is before the next grid point at 101 s"); + rig.advance(Duration::from_millis(500)); + assert_eq!(rig.watcher.poll_counts().shallow, 1, "101 s is the next grid point"); +} + +#[test] +fn producer_offline_missing_and_return_are_reported_without_a_bump() { + let rig = Rig::new("offline"); + // A touch alone is silent. + let file = OpenOptions::new().write(true).open(&rig.file).unwrap(); + file.set_modified(std::time::SystemTime::now() + Duration::from_secs(60)).unwrap(); + drop(file); + rig.advance(SECOND); + rig.quiet(); + std::fs::remove_file(&rig.file).unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Missing, 4, false, false)); + rig.advance(SECOND); + rig.quiet(); + std::fs::write(&rig.file, b"one").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Current, 4, false, false)); + rig.quiet(); +} + +#[test] +fn read_errors_are_reported_once_and_recover() { + let rig = Rig::new("error"); + std::fs::remove_file(&rig.file).unwrap(); + std::fs::create_dir(&rig.file).unwrap(); + rig.advance(SECOND); + let Notice::PollFailed { link_id, error } = rig.next() else { + panic!("expected a failure notice"); + }; + assert_eq!(link_id, "link-1"); + assert!(matches!(error, LinkError::Invalid(_)), "{error:?}"); + rig.advance(SECOND); + rig.quiet(); + std::fs::remove_dir(&rig.file).unwrap(); + std::fs::write(&rig.file, b"different").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-1", FileState::Modified, 5, true, false)); +} + +#[test] +fn with_tracker_accept_silences_the_next_notice_and_unwatch_stops_polling() { + let rig = Rig::new("accept"); + write_atomic(&rig.file, b"two!").unwrap(); + rig.advance(SECOND); + rig.next(); + let state = rig.watcher.with_tracker("link-1", |tracker| { + tracker.accept(); + tracker.state() + }); + assert_eq!(state, Some(FileState::Current)); + rig.advance(SECOND); + rig.quiet(); + assert!(rig.watcher.with_tracker("nope", |_| ()).is_none()); + assert!(rig.watcher.watch("link-1", Tracker::new(&rig.file, 0).unwrap()).is_err(), "duplicate id"); + let tracker = rig.watcher.unwatch("link-1").unwrap(); + assert_eq!(tracker.epoch(), 5); + write_atomic(&rig.file, b"after unwatch").unwrap(); + rig.advance(SECOND); + rig.quiet(); + assert!(rig.watcher.unwatch("link-1").is_none()); +} + +#[test] +fn several_links_are_polled_together() { + let rig = Rig::new("many"); + let other = rig._tmp.0.join("b.png"); + std::fs::write(&other, b"b").unwrap(); + rig.watcher.watch("link-2", Tracker::new(&other, 0).unwrap()).unwrap(); + write_atomic(&other, b"bb").unwrap(); + rig.advance(SECOND); + assert_eq!(rig.next(), changed("link-2", FileState::Modified, 1, true, false)); + rig.quiet(); +} + +#[test] +fn bad_intervals_are_refused() { + let clock = Arc::new(ManualTicker::new()); + let zero = Intervals { shallow: Duration::ZERO, deep: SECOND }; + assert!(Watcher::new(Arc::clone(&clock) as Arc, zero, |_| ()).is_err()); + let inverted = Intervals { shallow: Duration::from_secs(5), deep: SECOND }; + assert!(Watcher::new(clock as Arc, inverted, |_| ()).is_err()); + assert!(IntervalTicker::new(Duration::ZERO).is_err()); +} + +#[test] +fn drop_stops_the_thread_at_once_even_with_a_slow_ticker() { + let ticker = Arc::new(IntervalTicker::new(Duration::from_secs(3600)).unwrap()); + let watcher = + Watcher::new(ticker as Arc, Intervals { shallow: Duration::from_secs(3600), deep: Duration::from_secs(7200) }, |_| ()).unwrap(); + let (done, finished) = channel(); + std::thread::spawn(move || { + drop(watcher); + let _ = done.send(()); + }); + finished.recv_timeout(LIMIT).expect("drop must not wait for the next tick"); + + let clock = Arc::new(ManualTicker::new()); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), |_| ()).unwrap(); + assert!(clock.settle(LIMIT)); + drop(watcher); + assert!(clock.wait().is_none(), "a closed ticker stays closed"); + assert!(!clock.settle(Duration::from_millis(1))); +} + +#[test] +fn the_native_ticker_drives_a_watcher_without_a_manual_clock() { + let tmp = Tmp::new("native"); + let file = tmp.0.join("a.png"); + std::fs::write(&file, b"one").unwrap(); + let (tx, notices) = channel(); + let ticker = Arc::new(IntervalTicker::new(Duration::from_millis(10)).unwrap()); + let intervals = Intervals { shallow: Duration::from_millis(10), deep: Duration::from_millis(50) }; + let watcher = Watcher::new(ticker as Arc, intervals, forward(tx)).unwrap(); + watcher.watch("link-1", Tracker::new(&file, 0).unwrap()).unwrap(); + write_atomic(&file, b"two!").unwrap(); + let got = notices.recv_timeout(LIMIT).expect("a notice within the limit"); + // The first poll that sees the edit is shallow or deep, depending on when the 50 ms deep deadline falls. + let deep = matches!(got, Notice::Changed { deep: true, .. }); + assert_eq!(got, changed("link-1", FileState::Modified, 1, true, deep)); + drop(watcher); +} + +struct NoSink; + +impl ControlSink for NoSink { + fn submit(&self, _method: String, _params: serde_json::Value) -> Option> { + None + } +} + +struct NoActivate; + +impl Activate for NoActivate { + fn activate(&self, _token: Option, _from_pid: Option, _open: Option) -> Activation { + Activation::No + } +} + +struct NoFrames; + +impl Frames for NoFrames { + fn open(&self, _params: &serde_json::Map) -> Result { + Err(LinkError::NotReady("no frames".into())) + } + fn render(&self, _ticket: &str, _header: &serde_json::Value) -> Result<(serde_json::Value, Vec), LinkError> { + Err(LinkError::NotReady("no frames".into())) + } + fn close_ticket(&self, _ticket: &str) {} +} + +#[test] +fn a_disk_edit_reaches_a_subscribed_client_as_link_changed() { + let tmp = Tmp::new("wire"); + let runtime = RuntimeDir::at(tmp.0.join("rt")).unwrap(); + let server = Arc::new( + Server::start(ServerConfig { + app: "filmcraft".into(), + app_version: "0.1.0".into(), + instance: None, + runtime: Some(runtime.clone()), + tcp_port: None, + sink: Arc::new(NoSink), + activate: Arc::new(NoActivate), + frames: Arc::new(NoFrames), + }) + .unwrap(), + ); + let record = Discovery::scan(&runtime).unwrap().into_iter().next().unwrap(); + let mut client = Client::connect(&record).unwrap(); + client + .hello(&HelloClient { + app: "hub".into(), + app_version: "0.1.0".into(), + pid: std::process::id(), + instance: "550e8400-e29b-41d4-a716-446655440002".into(), + protocol_min: 1, + protocol_max: 1, + }) + .unwrap(); + client.subscribe(&["link.changed"]).unwrap(); + + let file = tmp.0.join("still.png"); + std::fs::write(&file, b"v1").unwrap(); + let clock = Arc::new(ManualTicker::new()); + let emitter = Arc::clone(&server); + let watcher = Watcher::new(Arc::clone(&clock) as Arc, Intervals::default(), move |notice| { + if let Notice::Changed { link_id, change, .. } = notice + && change.bumped + { + emitter.emit("link.changed", json!({"link_id": link_id, "epoch": change.epoch})).unwrap(); + } + }) + .unwrap(); + watcher.watch("link-9", Tracker::new(&file, 7).unwrap()).unwrap(); + + clock.advance(SECOND); + assert!(clock.settle(LIMIT)); + assert!(client.next_event(Some(Duration::from_millis(50))).unwrap().is_none(), "no edit, no event"); + write_atomic(&file, b"v2 is longer").unwrap(); + clock.advance(SECOND); + assert!(clock.settle(LIMIT)); + let event = client.next_event(Some(LIMIT)).unwrap().expect("a link.changed event"); + assert_eq!(event.event, "link.changed"); + assert_eq!(event.data, json!({"link_id": "link-9", "epoch": 8})); + + drop(watcher); + Arc::try_unwrap(server).ok().expect("the watcher released the server").shutdown().unwrap(); +} diff --git a/crates/odf/src/text/shapes.rs b/crates/odf/src/text/shapes.rs index 7c08151aa..eb184f5c9 100644 --- a/crates/odf/src/text/shapes.rs +++ b/crates/odf/src/text/shapes.rs @@ -124,7 +124,8 @@ pub(super) fn element_of(kind: ShapeKind) -> (&'static str, Option<&'static str> ShapeKind::ArrowDown => custom("down-arrow"), ShapeKind::Star => custom("star5"), ShapeKind::Heart => custom("heart"), - ShapeKind::Freeform => ("draw:path", None), + // A frame reaches the writer as the freeform it draws (see `model::from_doc`). + ShapeKind::Freeform | ShapeKind::Frame => ("draw:path", None), ShapeKind::Rectangle | ShapeKind::TextBox => ("draw:rect", None), } } diff --git a/crates/odf/src/text/tests.rs b/crates/odf/src/text/tests.rs index 8d7af18dc..da00b350c 100644 --- a/crates/odf/src/text/tests.rs +++ b/crates/odf/src/text/tests.rs @@ -442,6 +442,41 @@ fn table_column_widths_survive_a_round_trip() { assert_eq!(grid(&back), widths); } +#[test] +fn narrow_table_columns_keep_their_width() { + // Columns of a few points stay as the file gives them, so the table stays as wide as its source. + let content = format!( + r##"AB"## + ); + let doc = import(&package(&content, None)).unwrap(); + let grid_of = |doc: &craft_wordmodel::Document| { + doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }) + }; + assert_eq!(grid_of(&doc), Some(vec![3.0, 100.0])); + // Saved and opened again, the narrow column is still the file's width. + let back = import(&export(&doc).unwrap()).unwrap(); + let grid = grid_of(&back).unwrap(); + assert_eq!(grid.len(), 2, "{grid:?}"); + assert!((grid[0] - 3.0).abs() < 0.01 && (grid[1] - 100.0).abs() < 0.01, "{grid:?}"); +} + +#[test] +fn table_columns_of_a_point_or_less_give_equal_columns() { + // A column of a point or less, nothing or NaN (a damaged or hostile file) is no width to + // lay out: the whole grid falls back to equal columns, and nothing reaches layout below a + // point, negative or not a number. + for bad in ["1pt", "0.5pt", "0pt", "-3pt", "NaNpt"] { + let content = format!( + r##"AB"## + ); + let doc = import(&package(&content, None)).unwrap(); + let grid = doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }).unwrap(); + assert_eq!(grid.len(), 2, "{bad}: {grid:?}"); + assert!(grid.iter().all(|w| w.is_finite() && *w > 1.0), "{bad}: {grid:?}"); + assert_eq!(grid[0], grid[1], "{bad}: equal columns, {grid:?}"); + } +} + #[test] fn table_without_column_widths_keeps_the_equal_split() { let content = format!( diff --git a/crates/odf/src/text/tests_more/shapes.rs b/crates/odf/src/text/tests_more/shapes.rs index f6bbe9ae8..8d1a9d5fd 100644 --- a/crates/odf/src/text/tests_more/shapes.rs +++ b/crates/odf/src/text/tests_more/shapes.rs @@ -188,3 +188,44 @@ fn a_shape_with_only_an_empty_paragraph_has_no_text() { let stories: Vec = d.body.iter().filter_map(|b| b.as_para()).flat_map(|p| p.objects.iter()).map(|o| o.text_box().is_some()).collect(); assert_eq!(stories, [false, false, true, false, false, false]); } + +/// A frame shape (a letterhead's page border, from DOCX `frame`), which ODT has no shape for, +/// is written as the outline it draws: its outer edge and its hole, so LibreOffice doesn't fill +/// the whole box; read back, it is that freeform, hole and all. +#[test] +fn a_frame_shape_is_written_as_a_border_with_a_hole() { + let mut p = Paragraph::new(); + p.text.push(OBJ); + p.objects.push(InlineObject::Shape { + kind: ShapeKind::Frame, + w: 200.0, + h: 100.0, + fill: Some(craft_wordmodel::fill::Fill::solid(Rgb(1, 2, 3))), + stroke: None, + stroke_width: 0.0, + float: Default::default(), + story: None, + freeform: None, + effects: Default::default(), + // A border a tenth of the shorter side: 10 pt. + extra: craft_wordmodel::connector::ShapeExtra { corner: Some(10_000), ..Default::default() }, + }); + p.runs = vec![craft_wordmodel::para::Run { len: p.text.len(), props: Default::default() }]; + let mut d = Document::new(); + d.body = vec![para_block(p)]; + let odt = export(&d).unwrap(); + let content = content_of(&odt); + assert!(content.contains("= 9.5 * sx && *x <= 190.5 * sx && *y >= 9.5 * sy && *y <= 90.5 * sy), "{hole:?}"); +} diff --git a/crates/psd/src/compression.rs b/crates/psd/src/compression.rs index b41936a04..76891a421 100644 --- a/crates/psd/src/compression.rs +++ b/crates/psd/src/compression.rs @@ -206,10 +206,34 @@ pub mod packbits { /// `-128` no-op bytes (which are always permitted). Errors, rather than /// overflowing, when `out.len() + expected` does not fit in `usize`. pub(crate) fn decode_into(src: &[u8], expected: usize, out: &mut Vec) -> Result<()> { + decode_row(src, expected, out, false) + } + + /// Appends a recovery row: complete oversized packets are clipped and a short + /// row is zero-padded. Truncated packet payloads still fail; trailing packets + /// after the row is full are ignored. Strict decoding remains the default. + /// + /// Appended row bytes are bounded by `max_row_bytes` and the PSD decoded-size + /// limit; callers enforce their aggregate image budget. Allocation is fallible. + /// Existing output is retained; an error may leave a partially appended row. + pub fn decode_padded_into(src: &[u8], expected: usize, max_row_bytes: usize, out: &mut Vec) -> Result<()> { + if expected > max_row_bytes || expected as u64 > super::MAX_DECODED_BYTES { + return Err(PsdError::LimitExceeded("PackBits recovery row exceeds limit")); + } + out.len().checked_add(expected).ok_or(PsdError::LimitExceeded("PackBits output length overflows usize"))?; + out.try_reserve_exact(expected).map_err(|_| PsdError::OutOfMemory("PackBits recovery row allocation failed"))?; + decode_row(src, expected, out, true) + } + + fn decode_row(src: &[u8], expected: usize, out: &mut Vec, padded: bool) -> Result<()> { let target = out.len().checked_add(expected).ok_or(PsdError::LimitExceeded("PackBits output length overflows usize"))?; let mut i = 0; while out.len() < target { let Some(&h) = src.get(i) else { + if padded { + out.resize(target, 0); + break; + } return Err(PsdError::Decompress("PackBits row ended early".into())); }; i += 1; @@ -219,10 +243,10 @@ pub mod packbits { let Some(lit) = src.get(i..i + len) else { return Err(PsdError::Decompress("PackBits literal truncated".into())); }; - if len > target - out.len() { + if !padded && len > target - out.len() { return Err(PsdError::Decompress("PackBits literal overflows row".into())); } - out.extend_from_slice(lit); + out.extend_from_slice(&lit[..len.min(target - out.len())]); i += len; } else if h != -128 { let len = 1 - h as isize; @@ -231,10 +255,10 @@ pub mod packbits { return Err(PsdError::Decompress("PackBits run truncated".into())); }; i += 1; - if len > target - out.len() { + if !padded && len > target - out.len() { return Err(PsdError::Decompress("PackBits run overflows row".into())); } - out.extend(std::iter::repeat_n(b, len)); + out.extend(std::iter::repeat_n(b, len.min(target - out.len()))); } } Ok(()) diff --git a/crates/psd/src/compression/tests.rs b/crates/psd/src/compression/tests.rs index 6e0d86a7e..7cf876321 100644 --- a/crates/psd/src/compression/tests.rs +++ b/crates/psd/src/compression/tests.rs @@ -771,3 +771,25 @@ fn rle_chunks_use_the_callers_pool_and_report_panics() { assert!(map_chunks(2, vec![0, 1], |_| panic!("worker")).is_err()); }); } + +#[test] +fn padded_packbits_preserves_recovery_without_weakening_strict_decode() { + for (input, expected) in + [(&[0, 7][..], vec![7, 0, 0]), (&[3, 1, 2, 3, 4][..], vec![1, 2, 3]), (&[253, 9][..], vec![9, 9, 9]), (&[128][..], vec![0, 0, 0])] + { + let mut out = vec![42]; + super::packbits::decode_padded_into(input, 3, 3, &mut out).unwrap(); + assert_eq!(&out[1..], expected); + assert_eq!(out[0], 42); + assert!(super::packbits::decode(input, 3).is_err()); + } + for input in [&[3, 1][..], &[255][..]] { + assert!(super::packbits::decode_padded_into(input, 1, 1, &mut vec![]).is_err()); + } + let mut out = vec![42]; + assert!(super::packbits::decode_padded_into(&[], 4, 3, &mut out).is_err()); + assert_eq!(out, [42]); + super::packbits::decode_padded_into(&[255], 0, 0, &mut out).unwrap(); + super::packbits::decode_padded_into(&[0, 7, 255], 1, 1, &mut out).unwrap(); + assert_eq!(out, [42, 7]); +} diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs index f3afae64a..96ad47a7b 100644 --- a/crates/psd/src/descriptor.rs +++ b/crates/psd/src/descriptor.rs @@ -300,6 +300,15 @@ impl Descriptor { Ok(d) } + /// Parses a descriptor at the start of `data`, returning it and the number of bytes consumed + /// (trailing data is allowed). Mid-stream reads of layer effects, fill and adjustment + /// descriptors use this; [`from_bytes`] still requires the slice to be fully consumed. + pub fn parse_prefix(data: &[u8]) -> Result<(Self, usize)> { + let mut r = Reader::new(data); + let d = Self::read(&mut r)?; + Ok((d, r.pos())) + } + /// Serializes the descriptor. pub fn to_bytes(&self) -> Vec { let mut v = Vec::new(); @@ -705,6 +714,18 @@ mod tests { assert_eq!(VersionedDescriptor::from_bytes(&b).unwrap(), v); } + #[test] + fn parse_prefix() { + let d = sample(); + let mut b = d.to_bytes(); + let n = b.len(); + b.extend_from_slice(&[1, 2, 3]); + let (p, used) = Descriptor::parse_prefix(&b).unwrap(); + assert_eq!(p, d); + assert_eq!(used, n); + assert!(Descriptor::from_bytes(&b).is_err(), "from_bytes still refuses trailing bytes"); + } + #[test] fn versioned_parse_prefix() { let v = VersionedDescriptor::new(sample()); diff --git a/crates/raw/src/dng_lines.rs b/crates/raw/src/dng_lines.rs index 83877492e..8da1f45d8 100644 --- a/crates/raw/src/dng_lines.rs +++ b/crates/raw/src/dng_lines.rs @@ -342,7 +342,13 @@ pub fn read(t: &Tiff, ifd: &Ifd) -> Result { for c in &chunks { let n = c.w.checked_mul(c.h).and_then(|v| v.checked_mul(spp)).ok_or(Error::TooLarge)?; let budget = decode_budget(c, &l).filter(|&b| b <= MAX_SAMPLES).ok_or(Error::TooLarge)?; - decoded_samples = decoded_samples.checked_add(budget).ok_or(Error::TooLarge)?; + // the chunk's own `w` × `h` × `spp` output is allocated whatever its coded frame declares + // (a lossy JPEG smaller than its tile still fills a whole tile), so it is charged too + let alloc = budget.max(n); + if alloc > MAX_SAMPLES { + return Err(Error::TooLarge); + } + decoded_samples = decoded_samples.checked_add(alloc).ok_or(Error::TooLarge)?; if decoded_samples > MAX_DECODED_TILE_SAMPLES { return Err(Error::TooLarge); } diff --git a/crates/raw/tests/it/dng_lines.rs b/crates/raw/tests/it/dng_lines.rs index 1cf266d67..dd10c763c 100644 --- a/crates/raw/tests/it/dng_lines.rs +++ b/crates/raw/tests/it/dng_lines.rs @@ -277,3 +277,25 @@ fn short_lossy_jpeg_strips_are_charged_their_own_rows() { let band = w * rows * 3; assert_eq!(&img.data[..band], &img.data[img.data.len() - band..]); } + +#[test] +fn a_small_lossy_jpeg_in_a_huge_tile_is_charged_the_whole_tile() { + // a 256 x 256 RGB image in one 16384 x 16384 tile whose JPEG is only 256 x 256 (padded with + // comments to pass the bytes-per-sample guard): the JPEG's frame is small, but decoding fills + // the whole tile (about 1.5 GiB of samples), so it is refused before anything is allocated + let (w, tile) = (256usize, 16384usize); + let px: Vec = (0..w * w).flat_map(|i| [(i % 251) as u8, 90, 160]).collect(); + let mut jpeg = Vec::new(); + jpeg_encoder::Encoder::new(&mut jpeg, 90).encode(&px, w as u16, w as u16, jpeg_encoder::ColorType::Rgb).unwrap(); + let mut padded = jpeg[..2].to_vec(); + let need = tile * tile * 3 / 768 + 1; + while padded.len() < need { + padded.extend_from_slice(&[0xFF, 0xFE, 0xFF, 0xFF]); + padded.extend(std::iter::repeat_n(0u8, 0xFFFF - 2)); + } + padded.extend_from_slice(&jpeg[2..]); + let mut raw = base_ifd(w, w, 8, 3, false); + raw.set(t::COMPRESSION, Value::Short(vec![34892])); + raw.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles: vec![padded] }); + assert_eq!(lines(&dng(raw, ByteOrder::Little)).unwrap_err(), Error::TooLarge); +} diff --git a/crates/render-parity/Cargo.toml b/crates/render-parity/Cargo.toml new file mode 100644 index 000000000..087ff748b --- /dev/null +++ b/crates/render-parity/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-render-parity" +description = "Compare two renders of the same photo (per-channel error and CIEDE2000) against a stated tolerance: the parity check for the shared develop pipeline. Test support only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-cms = { workspace = true } diff --git a/crates/render-parity/README.md b/crates/render-parity/README.md new file mode 100644 index 000000000..0924d891a --- /dev/null +++ b/crates/render-parity/README.md @@ -0,0 +1,37 @@ +# craft-render-parity + +Test support for the shared develop pipeline ([#587](https://github.com/storytold/craft-libs/issues/587)): +how far two renders of the same photo are apart, and whether that is within a stated tolerance. +A develop stage that moves from an app, or an app that switches to the shared pipeline, renders +the same fixture both ways and compares the two here. + +```toml +[dev-dependencies] +craft-render-parity = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +```rust +use craft_render_parity::{Image, Tolerance, compare}; + +let expected = Image::new(width, height, &app_render)?; +let actual = Image::new(width, height, &shared_render)?; +let report = compare(&expected, &actual)?; +assert!(report.within(&Tolerance::BASELINE), "{report}: {:?}", report.exceeded(&Tolerance::BASELINE)); +``` + +## Input + +Scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, three interleaved `f32` samples +per pixel, row by row, with 1.0 as diffuse white. Values above 1.0 are compared as they are. A NaN +or infinite sample is an error, never a pass. + +## Measures + +| Measure | Meaning | +|---|---| +| `max_delta_e`, `mean_delta_e` | CIEDE2000 (kL = kC = kH = 1) in CIELAB relative to D65, from `craft_cms::math` | +| `max_channel`, `mean_channel` | Absolute difference of the linear samples, overall and per channel | +| `worst` | The first pixel with the largest CIEDE2000 difference, or the first that differs at all | + +`Tolerance::BASELINE`, the acceptance bar of #587, is CIEDE2000 at most 0.5 at any pixel and 0.05 +on average (1.0 is about the smallest visible difference), and at most 1/1024 in any linear sample. diff --git a/crates/render-parity/src/lib.rs b/crates/render-parity/src/lib.rs new file mode 100644 index 000000000..aedac7f0f --- /dev/null +++ b/crates/render-parity/src/lib.rs @@ -0,0 +1,205 @@ +//! Render parity: how far two renders of the same photo are apart, and whether that is within a +//! stated [`Tolerance`]. +//! +//! The shared develop pipeline (craft-libs#587) has to give LightCraft and PhotoCraft the same +//! picture. A stage moved from an app, or an app switched to the shared crate, is checked by +//! rendering the same fixture both ways and comparing them here. The [`Report`] names the worst +//! pixel, so a failure points at where to look. +//! +//! Renders are scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, interleaved, with +//! 1.0 as diffuse white. Colour difference is CIEDE2000 in CIELAB relative to D65 (from +//! [`craft_cms::math`]); values above 1.0 are allowed and compared as they are. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::fmt; + +use craft_cms::math::{delta_e2000, xyz_to_lab}; + +/// Linear Rec. 709 RGB → CIE XYZ (D65), IEC 61966-2-1. +const RGB_TO_XYZ: [[f64; 3]; 3] = [[0.4124, 0.3576, 0.1805], [0.2126, 0.7152, 0.0722], [0.0193, 0.1192, 0.9505]]; +/// The D65 white in XYZ, from the matrix above applied to RGB (1, 1, 1). +const D65: [f64; 3] = [0.9505, 1.0, 1.089]; + +/// A render to compare: `width` × `height` interleaved RGB samples, row by row. +#[derive(Clone, Copy, Debug)] +pub struct Image<'a> { + width: u32, + height: u32, + rgb: &'a [f32], +} + +impl<'a> Image<'a> { + /// A render of `width` × `height` pixels whose `rgb` holds exactly three samples per pixel. + pub fn new(width: u32, height: u32, rgb: &'a [f32]) -> Result { + let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(3)); + if expected != Some(rgb.len()) { + return Err(ParityError::Length { width, height, samples: rgb.len() }); + } + Ok(Self { width, height, rgb }) + } + + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } +} + +/// Why two renders could not be compared. +#[derive(Clone, Debug, PartialEq)] +pub enum ParityError { + /// The sample count is not three per pixel. + Length { width: u32, height: u32, samples: usize }, + /// The renders differ in size, so a pixel of one has no counterpart in the other. + Size { expected: (u32, u32), actual: (u32, u32) }, + /// A sample is NaN or infinite: no difference can be stated for it. + NotFinite { x: u32, y: u32 }, +} + +impl fmt::Display for ParityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Length { width, height, samples } => write!(f, "{samples} samples is not RGB for {width} × {height} pixels"), + Self::Size { expected, actual } => write!(f, "render is {} × {}, expected {} × {}", actual.0, actual.1, expected.0, expected.1), + Self::NotFinite { x, y } => write!(f, "pixel ({x}, {y}) is not a finite colour"), + } + } +} + +impl std::error::Error for ParityError {} + +/// How far two renders may be apart and still count as the same picture. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Tolerance { + /// The largest CIEDE2000 difference at any pixel. + pub max_delta_e: f64, + /// The largest mean CIEDE2000 difference over the picture. + pub mean_delta_e: f64, + /// The largest difference in any one linear channel sample. + pub max_channel: f32, +} + +impl Tolerance { + /// The baseline parity of craft-libs#587: well below a visible difference (CIEDE2000 1.0), and + /// within a 10-bit code value in any channel. + pub const BASELINE: Self = Self { max_delta_e: 0.5, mean_delta_e: 0.05, max_channel: 1.0 / 1024.0 }; +} + +impl Default for Tolerance { + fn default() -> Self { + Self::BASELINE + } +} + +/// The difference between two renders of the same size. +#[derive(Clone, Debug, PartialEq)] +pub struct Report { + /// Pixels compared. + pub pixels: u64, + /// The largest CIEDE2000 difference, at [`Self::worst`]. + pub max_delta_e: f64, + /// The mean CIEDE2000 difference. + pub mean_delta_e: f64, + /// The largest absolute difference in any channel sample. + pub max_channel: f32, + /// The mean absolute difference per channel (R, G, B). + pub mean_channel: [f64; 3], + /// The first pixel with the largest CIEDE2000 difference, when any pixel differs. + pub worst: Option<(u32, u32)>, +} + +impl Report { + /// Whether every measure is within `tolerance`. A NaN limit is never met. + pub fn within(&self, tolerance: &Tolerance) -> bool { + self.exceeded(tolerance).is_empty() + } + + /// The measures over `tolerance`, each with its value and limit, for a test failure message. + pub fn exceeded(&self, tolerance: &Tolerance) -> Vec { + let mut over = Vec::new(); + if exceeds(self.max_delta_e, tolerance.max_delta_e) { + over.push(format!("max ΔE2000 {:.4} > {}", self.max_delta_e, tolerance.max_delta_e)); + } + if exceeds(self.mean_delta_e, tolerance.mean_delta_e) { + over.push(format!("mean ΔE2000 {:.4} > {}", self.mean_delta_e, tolerance.mean_delta_e)); + } + if exceeds(f64::from(self.max_channel), f64::from(tolerance.max_channel)) { + over.push(format!("max channel difference {:.6} > {}", self.max_channel, tolerance.max_channel)); + } + over + } +} + +impl fmt::Display for Report { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} px: ΔE2000 max {:.4} mean {:.4}, channel max {:.6}", self.pixels, self.max_delta_e, self.mean_delta_e, self.max_channel)?; + match self.worst { + Some((x, y)) => write!(f, ", worst at ({x}, {y})"), + None => write!(f, ", identical"), + } + } +} + +/// Compare `actual` with `expected`, pixel by pixel. +pub fn compare(expected: &Image<'_>, actual: &Image<'_>) -> Result { + if (expected.width, expected.height) != (actual.width, actual.height) { + return Err(ParityError::Size { expected: (expected.width, expected.height), actual: (actual.width, actual.height) }); + } + let width = expected.width.max(1) as usize; + let mut report = Report { pixels: 0, max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0, mean_channel: [0.0; 3], worst: None }; + let mut sum_delta_e = 0.0; + for (i, (e, a)) in expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).enumerate() { + let (x, y) = ((i % width) as u32, (i / width) as u32); + let (Some(e), Some(a)) = (finite(*e), finite(*a)) else { return Err(ParityError::NotFinite { x, y }) }; + for c in 0..3 { + let d = (e[c] - a[c]).abs(); + report.max_channel = report.max_channel.max(d); + report.mean_channel[c] += f64::from(d); + } + let delta_e = delta_e2000(lab(e), lab(a)); + sum_delta_e += delta_e; + if delta_e > report.max_delta_e { + report.max_delta_e = delta_e; + report.worst = Some((x, y)); + } + report.pixels += 1; + } + if report.pixels > 0 { + let n = report.pixels as f64; + report.mean_delta_e = sum_delta_e / n; + report.mean_channel = report.mean_channel.map(|s| s / n); + } + if report.worst.is_none() && report.max_channel > 0.0 { + // channel differences too small to move CIELAB still name a pixel + report.worst = first_difference(expected, actual); + } + Ok(report) +} + +/// Whether `value` exceeds `limit`. A NaN limit (or value) exceeds it: an unordered comparison never +/// passes, so an invalid tolerance cannot switch a check off. +fn exceeds(value: f64, limit: f64) -> bool { + !matches!(value.partial_cmp(&limit), Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal)) +} + +fn finite(px: [f32; 3]) -> Option<[f32; 3]> { + px.iter().all(|v| v.is_finite()).then_some(px) +} + +fn first_difference(expected: &Image<'_>, actual: &Image<'_>) -> Option<(u32, u32)> { + let width = expected.width.max(1) as usize; + let i = expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).position(|(e, a)| e != a)?; + Some(((i % width) as u32, (i / width) as u32)) +} + +fn lab(rgb: [f32; 3]) -> [f64; 3] { + let rgb = rgb.map(f64::from); + let xyz = RGB_TO_XYZ.map(|row| row[0] * rgb[0] + row[1] * rgb[1] + row[2] * rgb[2]); + xyz_to_lab(xyz, D65) +} + +#[cfg(test)] +mod tests; diff --git a/crates/render-parity/src/tests.rs b/crates/render-parity/src/tests.rs new file mode 100644 index 000000000..a5c81f24f --- /dev/null +++ b/crates/render-parity/src/tests.rs @@ -0,0 +1,100 @@ +use super::*; + +fn gray_ramp(width: u32, height: u32) -> Vec { + (0..width * height).flat_map(|i| [i as f32 / (width * height) as f32; 3]).collect() +} + +#[test] +fn identical_renders_are_within_any_tolerance() { + let rgb = gray_ramp(4, 3); + let image = Image::new(4, 3, &rgb).unwrap(); + let report = compare(&image, &image).unwrap(); + assert_eq!((report.pixels, report.max_delta_e, report.max_channel, report.worst), (12, 0.0, 0.0, None)); + assert!(report.within(&Tolerance { max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0 })); + assert_eq!(report.to_string(), "12 px: ΔE2000 max 0.0000 mean 0.0000, channel max 0.000000, identical"); +} + +#[test] +fn white_in_the_matrix_is_neutral() { + // RGB (1, 1, 1) is the D65 white: L* 100, a* = b* = 0 + let [l, a, b] = lab([1.0; 3]); + assert!((l - 100.0).abs() < 1e-9 && a.abs() < 1e-9 && b.abs() < 1e-9, "{l} {a} {b}"); +} + +#[test] +fn the_worst_pixel_and_the_means_are_reported() { + let expected = gray_ramp(3, 2); + let mut actual = expected.clone(); + // pixel (1, 1): red up by 0.1, a visible difference + actual[4 * 3] += 0.1; + let report = compare(&Image::new(3, 2, &expected).unwrap(), &Image::new(3, 2, &actual).unwrap()).unwrap(); + assert_eq!(report.worst, Some((1, 1))); + assert!(report.max_delta_e > 1.0, "{report}"); + assert!((report.mean_delta_e - report.max_delta_e / 6.0).abs() < 1e-9); + assert!((report.max_channel - 0.1).abs() < 1e-6); + assert!((report.mean_channel[0] - 0.1 / 6.0).abs() < 1e-6 && report.mean_channel[1] == 0.0); + let over = report.exceeded(&Tolerance::BASELINE); + assert_eq!(over.len(), 3, "{over:?}"); + assert!(over[0].starts_with("max ΔE2000") && over[1].starts_with("mean ΔE2000") && over[2].starts_with("max channel")); + assert!(!report.within(&Tolerance::default())); + assert!(report.to_string().ends_with("worst at (1, 1)")); +} + +#[test] +fn a_tenth_of_a_10_bit_code_value_passes_the_baseline() { + let expected = vec![0.18f32; 2 * 2 * 3]; + let actual: Vec = expected.iter().map(|v| v + 0.1 / 1024.0).collect(); + let report = compare(&Image::new(2, 2, &expected).unwrap(), &Image::new(2, 2, &actual).unwrap()).unwrap(); + assert!(report.within(&Tolerance::BASELINE), "{report}"); + assert!(report.worst.is_some()); +} + +#[test] +fn a_channel_difference_too_small_for_cielab_still_names_a_pixel() { + let expected = vec![0.5f32; 6]; + let mut actual = expected.clone(); + actual[5] = f32::from_bits(0.5f32.to_bits() + 1); + let report = compare(&Image::new(2, 1, &expected).unwrap(), &Image::new(2, 1, &actual).unwrap()).unwrap(); + assert!(report.max_channel > 0.0); + assert_eq!(report.worst, Some((1, 0))); +} + +#[test] +fn an_empty_render_compares_as_identical() { + let image = Image::new(0, 5, &[]).unwrap(); + let report = compare(&image, &image).unwrap(); + assert_eq!((report.pixels, report.mean_delta_e, report.worst), (0, 0.0, None)); + assert_eq!((image.width(), image.height()), (0, 5)); +} + +#[test] +fn bad_inputs_are_errors_not_panics() { + let e = Image::new(2, 2, &[0.0; 11]).unwrap_err(); + assert_eq!(e, ParityError::Length { width: 2, height: 2, samples: 11 }); + assert_eq!(e.to_string(), "11 samples is not RGB for 2 × 2 pixels"); + assert!(Image::new(u32::MAX, u32::MAX, &[]).is_err()); + + let a = vec![0.0f32; 6]; + let b = vec![0.0f32; 6]; + let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(1, 2, &b).unwrap()).unwrap_err(); + assert_eq!(e.to_string(), "render is 1 × 2, expected 2 × 1"); + + let mut nan = a.clone(); + nan[4] = f32::NAN; + let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(2, 1, &nan).unwrap()).unwrap_err(); + assert_eq!(e, ParityError::NotFinite { x: 1, y: 0 }); + assert_eq!(e.to_string(), "pixel (1, 0) is not a finite colour"); + let e: Box = Box::new(e); + assert!(e.source().is_none()); +} + +#[test] +fn a_nan_tolerance_fails_every_check() { + let rgb = gray_ramp(2, 2); + let image = Image::new(2, 2, &rgb).unwrap(); + let report = compare(&image, &image).unwrap(); + let nan = Tolerance { max_delta_e: f64::NAN, mean_delta_e: f64::NAN, max_channel: f32::NAN }; + assert_eq!(report.exceeded(&nan).len(), 3); + assert!(!report.within(&nan)); + assert!(report.within(&Tolerance { max_delta_e: f64::INFINITY, ..Tolerance::BASELINE })); +} diff --git a/crates/rtf/src/doc/parse.rs b/crates/rtf/src/doc/parse.rs index 60a65740a..01978da1e 100644 --- a/crates/rtf/src/doc/parse.rs +++ b/crates/rtf/src/doc/parse.rs @@ -1,3 +1,6 @@ +use std::cmp::Reverse; +use std::collections::BinaryHeap; + use super::reader::Reader; use super::state::{CommentData, Dest, HeaderFooter, HfPages, MARK_CLOSE, MARK_OPEN, MAX_PICT, NoteData, PageGeom}; use super::{Arc, Block, Blocks, Cell, Document, FBlock, Flow, HashMap, InlineObject, Para, Paragraph, model}; @@ -262,43 +265,137 @@ fn finalize(mut r: Reader) -> Parsed { Parsed { flow: Flow { blocks, meta: r.meta }, page: r.page, last_section, default_font, headers_footers, notes: r.notes, comments: r.comments } } +/// Most grid columns a table gets (the document model's own cap). +const MAX_GRID_COLS: usize = 63; +/// Grid columns of this width or less, in twips (1 pt), always merge: the document model refuses +/// them, and would then give the table equal columns. +const NO_GRID_GAP: i64 = 20; + /// The column grid of a table from its rows' cell right edges: every distinct edge starts a grid /// column, and each cell spans the grid columns between its left and right edges, as Word lays /// out RTF rows whose cells don't line up. Returns the widths in points, or none (equal columns) -/// when the edges don't make a grid. +/// when no grid fits, which only a row of more cells than grid columns can cause. +/// +/// One odd row never costs the whole table its grid (a form exported from a spreadsheet, with +/// many narrow columns, otherwise had its wide title squeezed into one of four equal columns): +/// - Over [`MAX_GRID_COLS`] edges, or edges [`NO_GRID_GAP`] or less apart, merge +/// ([`coarse_edges`]); each cell edge moves to the nearest grid edge. Narrow columns over a +/// point stay: import keeps them as narrow as the file gives them. +/// - A cell that ends at or before the previous one (a zero-width cell, or a cell with no +/// `\cellx` of its own) still gets a grid column of its own; cells pushed past the table's +/// right edge move back. pub(super) fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec { + if rights.len() != rows.len() { + return Vec::new(); + } let mut edges: Vec = rights.iter().flatten().copied().filter(|&x| x > 0).collect(); edges.sort_unstable(); edges.dedup(); - let ok = rights.len() == rows.len() - && rows - .iter() - .zip(rights) - .all(|(r, x)| r.len() == x.len() && x.windows(2).all(|w| w.first() < w.get(1)) && x.first().is_some_and(|&v| v > 0)) - && !edges.is_empty() - && edges.len() <= 63; - if !ok { + if edges.is_empty() { return Vec::new(); } - for (row, x) in rows.iter_mut().zip(rights) { + let grid = coarse_edges(&edges); + // Each cell's right edge as a grid edge number (1 = the first grid column's right edge). + let mut ends: Vec> = Vec::with_capacity(rows.len()); + for (row, x) in rows.iter().zip(rights) { + if row.len() > grid.len() { + return Vec::new(); + } + let mut at: Vec = Vec::with_capacity(row.len()); + let mut prev = 0; + for i in 0..row.len() { + let end = x.get(i).map_or(0, |&r| nearest_edge(&grid, r)).max(prev + 1); + at.push(end); + prev = end; + } + // Cells pushed past the right edge move back, each keeping a column of its own. + let mut limit = grid.len(); + for end in at.iter_mut().rev() { + *end = (*end).min(limit); + limit = end.saturating_sub(1); + } + ends.push(at); + } + for (row, at) in rows.iter_mut().zip(&ends) { let mut left = 0; - for (c, &right) in row.iter_mut().zip(x) { - let span = edges.iter().filter(|&&e| e > left && e <= right).count().max(1); - c.colspan = span as u32; - left = right; + for (c, &end) in row.iter_mut().zip(at) { + c.colspan = u32::try_from(end.saturating_sub(left)).unwrap_or(1).max(1); + left = end; } } - let mut prev = 0; - edges - .iter() + let mut prev = 0i64; + grid.iter() .map(|&e| { - let w = (e - prev) as f32 / 20.0; - prev = e; + let w = (i64::from(e) - prev) as f32 / 20.0; + prev = i64::from(e); w }) .collect() } +/// The grid edges kept from the sorted, distinct, positive `edges`. The narrowest column merges +/// into the next, one at a time, so an edge moves only when it has to: while there are over +/// [`MAX_GRID_COLS`] columns, or columns of [`NO_GRID_GAP`] or less. The table's right edge is +/// always kept. +fn coarse_edges(edges: &[i32]) -> Vec { + let n = edges.len(); + // Each edge's neighbours among the edges still kept. + let mut left: Vec> = (0..n).map(|i| i.checked_sub(1)).collect(); + let mut right: Vec> = (1..=n).map(|i| (i < n).then_some(i)).collect(); + let mut kept = vec![true; n]; + let at = |i: Option| i.and_then(|i| edges.get(i)).map_or(0, |&e| i64::from(e)); + let width = |i: usize, left: &[Option]| at(Some(i)) - at(left.get(i).copied().flatten()); + let mut narrowest: BinaryHeap> = (0..n).map(|i| Reverse((width(i, &left), i))).collect(); + let mut count = n; + while count > 1 { + let Some(Reverse((w, i))) = narrowest.pop() else { + break; + }; + // A column that has since grown, or gone, has a fresher entry or none. + if !kept.get(i).copied().unwrap_or(false) || w != width(i, &left) { + continue; + } + if count <= MAX_GRID_COLS && w > NO_GRID_GAP { + break; + } + // The column's right edge goes, unless it is the table's: then its left edge does. + let last = right.get(i).copied().flatten().is_none(); + let Some(gone) = (if last { left.get(i).copied().flatten() } else { Some(i) }) else { + break; + }; + let before = left.get(gone).copied().flatten(); + let after = right.get(gone).copied().flatten(); + if let Some(slot) = before.and_then(|b| right.get_mut(b)) { + *slot = after; + } + if let Some(a) = after { + if let Some(slot) = left.get_mut(a) { + *slot = before; + } + narrowest.push(Reverse((width(a, &left), a))); + } + if let Some(slot) = kept.get_mut(gone) { + *slot = false; + } + count -= 1; + } + edges.iter().zip(&kept).filter(|&(_, &k)| k).map(|(&e, _)| e).collect() +} + +/// The number of the grid edge nearest `x` (1-based; 0 is the table's left edge). +fn nearest_edge(grid: &[i32], x: i32) -> usize { + if x <= 0 { + return 0; + } + let p = grid.partition_point(|&e| e < x); + let below = if p == 0 { 0 } else { grid.get(p - 1).copied().unwrap_or(0) }; + match grid.get(p) { + Some(&above) if i64::from(above) - i64::from(x) <= i64::from(x) - i64::from(below) => p + 1, + Some(_) => p, + None => grid.len(), + } +} + /// Parse RTF bytes into a document. pub fn import(bytes: &[u8]) -> Result { model::begin_import(); diff --git a/crates/rtf/src/doc/tests.rs b/crates/rtf/src/doc/tests.rs index 2a4103eb1..fddd198da 100644 --- a/crates/rtf/src/doc/tests.rs +++ b/crates/rtf/src/doc/tests.rs @@ -389,6 +389,26 @@ fn page_size_and_margins() { assert_eq!(d.last_section, craft_wordmodel::SectionProps::default()); } +#[test] +fn narrow_grid_columns_keep_their_width() { + // Rows' edges a few points apart, as in a form exported from a spreadsheet. The table stays as + // wide as its source, through an export and back too. + let rtf = br"{\rtf1\ansi +\trowd\cellx400\cellx800 \pard\intbl a\cell b\cell\row +\trowd\cellx80\cellx480\cellx800 \pard\intbl c\cell d\cell e\cell\row +\pard after\par}"; + let table = |doc: &Document| match doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.clone()) } else { None }) + { + Some(t) => t, + None => panic!("no table"), + }; + let doc = import(rtf).unwrap(); + let t = table(&doc); + assert_eq!(t.grid, [4.0, 16.0, 4.0, 16.0]); + assert_eq!(t.rows[0].cells.iter().map(|c| c.props.width).collect::>(), [Some(20.0), Some(20.0)]); + assert_eq!(table(&import(export(&doc).as_bytes()).unwrap()).grid, t.grid); +} + #[test] fn table_grid_from_cell_edges_and_borders() { // Rows whose cells don't line up share one grid: edges 2000, 3000, 9000. @@ -460,6 +480,141 @@ fn table_grid_from_cell_edges_and_borders() { assert_eq!(t.grid, [240.9, 241.0]); } +/// The table of `rtf`'s flow and every row's column spans; checks the spans fit the grid. +fn grid_of(rtf: &[u8]) -> (Vec, Vec>) { + let f = parse(rtf).unwrap(); + let FBlock::Table(t) = &f.blocks[0] else { panic!("no table") }; + let spans: Vec> = t.rows.iter().map(|r| r.iter().map(|c| c.colspan).collect()).collect(); + assert!(!t.widths.is_empty(), "the table kept no grid"); + assert_eq!(t.cols(), t.widths.len(), "spans {spans:?} don't fit {} grid columns", t.widths.len()); + assert!(t.widths.iter().all(|&w| w > 1.0), "a grid column under a point: {:?}", t.widths); + (t.widths.clone(), spans) +} + +/// Width in points of the first `n` grid columns. +fn first_cols(widths: &[f32], n: u32) -> f32 { + widths.iter().take(n as usize).sum() +} + +#[test] +fn table_grid_survives_more_than_63_edges() { + // A form exported from a spreadsheet: a wide title over rows whose middle edge moves by + // 100 twips, 80 distinct edges in all. The grid used to be dropped, giving equal columns. + let mut rtf = String::from(r"{\rtf1\ansi\trowd\cellx6000\cellx8000\pard\intbl Title\cell Codes\cell\row"); + for k in 1..80 { + rtf.push_str(&format!(r"\trowd\cellx{}\cellx8000\pard\intbl a\cell b\cell\row", k * 100)); + } + rtf.push_str(r"\pard\par}"); + let (widths, spans) = grid_of(rtf.as_bytes()); + assert!(widths.len() <= 63, "{} grid columns", widths.len()); + assert!((widths.iter().sum::() - 400.0).abs() < 0.01, "the table keeps its 400 pt"); + // The title keeps its 300 pt, within a grid column's merge. + let title = first_cols(&widths, spans[0][0]); + assert!((title - 300.0).abs() <= 5.0, "title {title} pt"); + // The staircase still climbs: the 60th row's first cell is about 300 pt wide too. + assert!((first_cols(&widths, spans[60][0]) - 300.0).abs() <= 5.0); + + // The same through import: the document table has the grid, not equal columns. + let d = import(rtf.as_bytes()).unwrap(); + let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); + assert_eq!(t.grid.len(), widths.len()); + let w = t.rows[0].cells[0].props.width.unwrap(); + assert!((w - 300.0).abs() <= 5.0, "title cell {w} pt"); +} + +#[test] +fn table_grid_survives_a_zero_width_cell_and_a_missing_cellx() { + // A zero-width cell (two edges at 3000) in one row: the other rows keep their grid. + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx6000\cellx9000 \pard\intbl Title\cell b\cell\row +\trowd\cellx2000\cellx2000\cellx9000 \pard\intbl a\cell\cell c\cell\row +\pard\par}", + ); + assert_eq!(first_cols(&widths, spans[0][0]), 300.0); + assert_eq!(spans[1].len(), 3, "the zero-width cell is kept: {spans:?}"); + assert_eq!(first_cols(&widths, spans[1][0]), 100.0, "the cell before it keeps its width"); + + // A row with a third cell but only two `\cellx`. + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx6000\cellx9000 \pard\intbl Title\cell b\cell\row +\trowd\cellx2000\cellx6000 \pard\intbl a\cell b\cell extra\cell\row +\pard\par}", + ); + assert_eq!(first_cols(&widths, spans[0][0]), 300.0); + assert_eq!(spans[1].len(), 3); + assert_eq!(first_cols(&widths, spans[1][0]), 100.0); +} + +#[test] +fn table_grid_merges_edges_a_few_twips_apart() { + // Two rows that mean the same 3000-twip edge, 5 twips apart: one grid column, not one of + // a quarter point (which the document model refuses, so the table got equal columns). + let (widths, spans) = grid_of( + br"{\rtf1\ansi +\trowd\cellx3000\cellx9000 \pard\intbl a\cell b\cell\row +\trowd\cellx3005\cellx9000 \pard\intbl c\cell d\cell\row +\pard\par}", + ); + assert_eq!(widths.len(), 2, "{widths:?}"); + assert_eq!(spans, [[1, 1], [1, 1]]); + assert_eq!(widths.iter().sum::(), 450.0); +} + +/// `edges` as one table row, a cell for each. +fn rtf_row(edges: &[usize]) -> String { + let cells: String = edges.iter().map(|e| format!(r"\cellx{e}")).collect(); + format!(r"\trowd{cells}\pard\intbl {}\row", r"x\cell ".repeat(edges.len())) +} + +#[test] +fn table_grid_keeps_a_column_for_each_cell_of_the_widest_row() { + // 64 evenly spaced edges: merging by a common gap halves them to 32 columns, too few for a + // row of 33 or more cells, and the table lost its grid. + let edges: Vec = (1..=64).map(|k| k * 100).collect(); + for cells in [33, 40, 50, 62] { + let wide: Vec = edges.iter().copied().take(cells).chain([6400]).collect(); + let rest: Vec = edges.iter().copied().skip(cells - 1).collect(); + let rtf = format!(r"{{\rtf1\ansi {}{}\pard\par}}", rtf_row(&wide), rtf_row(&rest)); + let (widths, spans) = grid_of(rtf.as_bytes()); + assert!((cells + 1..=63).contains(&widths.len()), "{cells} cells, {} grid columns", widths.len()); + assert_eq!(spans[0].len(), cells + 1, "every cell is kept"); + assert_eq!(widths.iter().sum::(), 320.0, "the table keeps its 320 pt"); + } +} + +#[test] +fn table_grid_keeps_narrow_columns_at_their_width() { + // Columns under 6 pt that no row needs keep their 3 pt: import no longer widens them, so the + // table keeps its 60 pt. + let mut rtf = String::from(r"{\rtf1\ansi "); + for k in 1..=20 { + rtf.push_str(&rtf_row(&[k * 60, 1200])); + } + rtf.push_str(r"\pard\par}"); + let (widths, _) = grid_of(rtf.as_bytes()); + assert_eq!(widths, [3.0; 20], "{widths:?}"); + let d = import(rtf.as_bytes()).unwrap(); + let t = d.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t) } else { None }).unwrap(); + assert_eq!(t.grid.iter().sum::(), 60.0, "{:?}", t.grid); + + // A row that needs them keeps them, narrow as they are. + let (widths, spans) = grid_of(format!(r"{{\rtf1\ansi {}\pard\par}}", rtf_row(&[60, 120, 180])).as_bytes()); + assert_eq!(widths, [3.0, 3.0, 3.0]); + assert_eq!(spans, [[1, 1, 1]]); +} + +#[test] +fn table_grid_never_keeps_a_column_the_document_model_refuses() { + // Three cells within a point of each other: too narrow to keep even for a row that needs + // them. The row can't fit, so the table has no grid, and nothing panics. + let f = parse(format!(r"{{\rtf1\ansi {}\pard\par}}", rtf_row(&[5, 10, 15])).as_bytes()).unwrap(); + let FBlock::Table(t) = &f.blocks[0] else { panic!("no table") }; + assert!(t.widths.is_empty(), "{:?}", t.widths); + assert_eq!(t.rows[0].len(), 3); +} + #[test] fn code_pages_from_ansicpg_and_font_charsets() { // The document's code page: Russian RTF as accounting systems write it. diff --git a/crates/settings/src/recovery.rs b/crates/settings/src/recovery.rs index 074a1b4a8..1435797e0 100644 --- a/crates/settings/src/recovery.rs +++ b/crates/settings/src/recovery.rs @@ -37,23 +37,28 @@ pub fn backup_path(path: &Path) -> PathBuf { /// Writes `text`, the last version that parsed, to the backup file atomically. /// /// The caller decides what is safe to keep: a blank or unparsable file must never replace the -/// previous backup. On Unix the backup always gets the same mode as `path` (`0600` when there is -/// none), so it is never readable by more people than the settings themselves. +/// previous backup. On Unix the backup always gets the same mode and group as `path` (`0600` +/// when there is none), so it is never readable by more people than the settings themselves; +/// when it cannot be put in the config's group, it keeps the owner bits only. pub fn write_backup(path: &Path, text: &str) -> io::Result<()> { let backup = backup_path(path); #[cfg(unix)] { - use std::os::unix::fs::PermissionsExt; - let mode = std::fs::metadata(path).map(|m| m.permissions().mode() & 0o777).unwrap_or(0o600); + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let config = std::fs::metadata(path).ok(); + let mode = config.as_ref().map(|m| m.permissions().mode() & 0o777).unwrap_or(0o600); // An atomic replace keeps the existing file's mode, so narrow an old, broader backup // before new contents go into it, then match the config once written. if backup.exists() { - std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode))?; + std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode & 0o700))?; } craft_atomic::WriteOptions::default() .with_new_file_permissions(craft_atomic::NewFilePermissions::Private) .atomic_write(&backup, text.as_bytes())?; - std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(mode)) + // The group bits were granted to the config's group; the new file may have been created + // in another (a setgid directory, a different primary group). + let grouped = config.as_ref().is_some_and(|m| std::os::unix::fs::chown(&backup, None, Some(m.gid())).is_ok()); + std::fs::set_permissions(&backup, std::fs::Permissions::from_mode(if grouped { mode } else { mode & 0o700 })) } #[cfg(not(unix))] { diff --git a/crates/settings/tests/store.rs b/crates/settings/tests/store.rs index bc55d97cb..a9c4b9553 100644 --- a/crates/settings/tests/store.rs +++ b/crates/settings/tests/store.rs @@ -289,4 +289,8 @@ fn backup_follows_a_mode_the_user_chose_for_the_config() { store.update(|p| p.general.language = "fr".into()); store.flush().unwrap(); assert_eq!(mode(&tmp.path().join("config.toml.bak")), 0o640); + // The group bits authorize the config's group, which the backup is in. + use std::os::unix::fs::MetadataExt; + let gid = |p: &std::path::Path| std::fs::metadata(p).unwrap().gid(); + assert_eq!(gid(&tmp.path().join("config.toml.bak")), gid(&config_path(tmp.path()))); } diff --git a/crates/ssip/Cargo.toml b/crates/ssip/Cargo.toml new file mode 100644 index 000000000..5ff35544f --- /dev/null +++ b/crates/ssip/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-ssip" +description = "Bounded Speech Dispatcher SSIP wire framing and message-data encoding" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true diff --git a/crates/ssip/README.md b/crates/ssip/README.md new file mode 100644 index 000000000..7fc98924e --- /dev/null +++ b/crates/ssip/README.md @@ -0,0 +1,19 @@ +# craft-ssip + +A focused Speech Dispatcher SSIP wire layer: sentence-as-data encoding, final response codes, +and a bounded incremental reply reader. No sockets, process launch, cancellation commands, +voice discovery, retry or playback scheduling: those policies remain with the application. + +Derived from Skaywalker's WordCraft SSIP implementation in +`crates/engine/src/speech/ssip.rs` at `e9750c5cb75d577a4669059d7bf6f9874e945839` +(MIT OR Apache-2.0). WordCraft #2937 (`65d7999eddebf120ca9fde523b01156b3e1f5fce`) +adds voice selection on this same core; this extraction does not duplicate those application changes. + +Unlike the original reader, a complete line that becomes overlong in the final read is refused. +The 4096-byte limit counts bytes before LF, including an optional CR. Reply buffering holds +at most that limit plus one 512-byte read. Invalid UTF-8 is replaced, preserving WordCraft behavior. +Message input is bounded by the caller; ASCII three-digit response codes are enforced. +Caller-provided `Read` implementations must obey Rust's Read contract; callers remain responsible +for transport timeouts, cancellation and an overall deadline between `next_line` calls. + +This is not a complete SSIP client and does not establish WordCraft consumer cutover. diff --git a/crates/ssip/src/lib.rs b/crates/ssip/src/lib.rs new file mode 100644 index 000000000..8bb4cce37 --- /dev/null +++ b/crates/ssip/src/lib.rs @@ -0,0 +1,140 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Speech Dispatcher SSIP wire helpers extracted from WordCraft's contributor implementation. +//! Socket discovery, deadlines, voice selection, retry and cancellation policy stay in the app. +use std::io::{self, ErrorKind, Read}; + +/// Maximum reply bytes before LF, including an optional CR. +pub const MAX_LINE: usize = 4096; + +/// Encode one sentence as data, never as protocol commands. CR/LF become spaces; a leading +/// dot is doubled and the message terminator is appended. The caller bounds input bytes. +pub fn message_data(text: &str, max_input_bytes: usize) -> io::Result { + if text.len() > max_input_bytes { + return Err(ErrorKind::InvalidInput.into()); + } + let size = text.len().checked_add(6).ok_or(ErrorKind::InvalidInput)?; + let mut out = String::new(); + out.try_reserve_exact(size).map_err(|_| io::Error::other("SSIP message allocation failed"))?; + if text.starts_with('.') { + out.push('.'); + } + out.extend(text.chars().map(|c| if c == '\r' || c == '\n' { ' ' } else { c })); + out.push_str("\r\n.\r\n"); + Ok(out) +} + +/// The three ASCII digits of a final response; continuation (`225-...`) is not final. +pub fn final_code(line: &str) -> Option { + let digits = line.as_bytes().get(..3)?; + if !digits.iter().all(u8::is_ascii_digit) || !(line.len() == 3 || line.as_bytes().get(3) == Some(&b' ')) { + return None; + } + line.get(..3)?.parse().ok() +} + +/// Incremental replies. At most one read per call lets the app enforce its own deadline. +/// Invalid UTF-8 retains WordCraft's replacement-character behavior. Overlong replies fail, +/// including those whose terminating newline arrives in the same read that exceeds the cap. +pub struct Lines { + reader: R, + pending: Vec, +} + +impl Lines { + pub fn new(reader: R) -> Self { + Self { reader, pending: Vec::new() } + } + + /// Complete next line without CR/LF, or None after a partial read or transient timeout. + /// The caller must set transport timeouts and enforce cancellation/deadlines. + pub fn next_line(&mut self) -> io::Result> { + if let Some(line) = self.complete_line()? { + return Ok(Some(line)); + } + let mut buf = [0u8; 512]; + match self.reader.read(&mut buf) { + Ok(0) => Err(ErrorKind::UnexpectedEof.into()), + Ok(n) => { + let bytes = buf.get(..n).ok_or(ErrorKind::InvalidData)?; + // Invariant: pending <= MAX_LINE before this read, so storage <= MAX_LINE+512. + self.pending.try_reserve(bytes.len()).map_err(|_| io::Error::other("SSIP reply allocation failed"))?; + self.pending.extend_from_slice(bytes); + self.complete_line() + } + Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted) => Ok(None), + Err(e) => Err(e), + } + } + + fn complete_line(&mut self) -> io::Result> { + let end = self.pending.iter().position(|&b| b == b'\n'); + if end.unwrap_or(self.pending.len()) > MAX_LINE { + return Err(ErrorKind::InvalidData.into()); + } + let Some(end) = end else { return Ok(None) }; + let bytes = self.pending.get(..end).ok_or(ErrorKind::InvalidData)?; + let line = String::from_utf8_lossy(bytes).trim_end_matches('\r').to_string(); + self.pending.drain(..=end); + Ok(Some(line)) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + use std::collections::VecDeque; + struct Chunks(VecDeque>>); + impl Read for Chunks { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let bytes = self.0.pop_front().unwrap_or(Ok(Vec::new()))?; + out[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + #[test] + fn message_text_cannot_inject_commands() { + assert_eq!(message_data(".\r\nCANCEL ALL\n!-!", 64).unwrap(), ".. CANCEL ALL !-!\r\n.\r\n"); + assert_eq!(message_data("你好", 6).unwrap(), "你好\r\n.\r\n"); + assert!(message_data("你好", 5).is_err()); + } + #[test] + fn final_codes_are_ascii_and_not_continuations() { + assert_eq!(final_code("225 OK"), Some(225)); + assert_eq!(final_code("701"), Some(701)); + for bad in ["225-12", "+22 OK", "22 OK", "225 OK", "225x", ""] { + assert_eq!(final_code(bad), None); + } + } + #[test] + fn fragmented_utf8_timeout_and_buffered_lines() { + let mut lines = Lines::new(Chunks(VecDeque::from([ + Ok(vec![b'2', b'2', b'5', b' ', 0xc3]), + Err(ErrorKind::TimedOut.into()), + Ok(vec![0xa9, b'\r', b'\n', b'7', b'0', b'2', b'\n']), + ]))); + assert!(lines.next_line().unwrap().is_none()); + assert!(lines.next_line().unwrap().is_none()); + assert_eq!(lines.next_line().unwrap().as_deref(), Some("225 é")); + assert_eq!(lines.next_line().unwrap().as_deref(), Some("702")); + assert_eq!(lines.next_line().unwrap_err().kind(), ErrorKind::UnexpectedEof); + } + #[test] + fn completed_overlong_line_is_rejected_at_boundary() { + for extra in [0, 1, 511] { + let mut chunks: VecDeque<_> = (0..8).map(|_| Ok(vec![b'x'; 512])).collect(); + let mut last = vec![b'x'; extra]; + last.push(b'\n'); + chunks.push_back(Ok(last)); + let mut lines = Lines::new(Chunks(chunks)); + for _ in 0..8 { + assert!(lines.next_line().unwrap().is_none()); + } + if extra == 0 { + assert_eq!(lines.next_line().unwrap().unwrap().len(), MAX_LINE); + } else { + assert_eq!(lines.next_line().unwrap_err().kind(), ErrorKind::InvalidData); + } + } + } +} diff --git a/crates/svg/src/guard.rs b/crates/svg/src/guard.rs index f3573e443..9601b2a13 100644 --- a/crates/svg/src/guard.rs +++ b/crates/svg/src/guard.rs @@ -21,22 +21,56 @@ const MAX_NESTING: u32 = 16; const MAX_NAME: usize = 256; /// The internal general entities declared in `svg`: name → every value declared for it. Text that -/// only looks like a declaration (in a comment, or in another entity's value) is kept too: the -/// largest value counts, so it can only make the estimate larger. +/// only looks like a declaration in another entity's value is kept too: the largest value counts, +/// so it can only make the estimate larger. Comments and CDATA are skipped, so markers there are +/// not treated as declarations. fn declared(svg: &str) -> HashMap<&str, Vec<&str>> { let mut map: HashMap<&str, Vec<&str>> = HashMap::new(); - for (at, _) in svg.match_indices("") { + Some(n) => at + 1 + 3 + n + 3, + None => svg.len(), + }; continue; } - let Some(end) = rest.find(char::is_whitespace) else { continue }; - let (name, rest) = rest.split_at(end); - let rest = rest.trim_start(); - let Some(quote) = rest.chars().next().filter(|q| matches!(q, '"' | '\'')) else { continue }; - let Some(value) = rest.get(1..).and_then(|r| r.split_once(quote)).map(|(v, _)| v) else { continue }; - map.entry(name).or_default().push(value); + if let Some(body) = after.strip_prefix("![CDATA[") { + i = match body.find("]]>") { + Some(n) => at + 1 + 8 + n + 3, + None => svg.len(), + }; + continue; + } + if let Some(rest) = after.strip_prefix("!ENTITY") { + let rest = rest.trim_start(); + // A parameter entity (`%`) isn't referenced as `&name;`. + if rest.starts_with('%') { + i = at + 1; + continue; + } + // Bound the name scan: a comment full of ` MAX_NAME { + // A name longer than MAX_NAME cannot be referenced, but it must still be + // recorded so `entities_expand_past` can refuse the document. + map.entry(window).or_default().push(""); + } + i = at + 1; + continue; + } + i = at + 1; } map } @@ -99,7 +133,8 @@ pub fn entities_expand_past(svg: &str, most: usize) -> bool { /// an entity becomes elements where it is referenced, so what the text shows of the tree is not /// what a reader builds; drawing apps declare entities for namespace names only. A caller may /// refuse these outright instead of counting them ([`prescan`] counts them ten times). Text that -/// only looks like a declaration (in a comment) counts too. `<` is text, not markup. +/// only looks like a declaration in another entity's value counts too; comments are skipped. +/// `<` is text, not markup. pub fn entity_holds_markup(svg: &str) -> bool { declared(svg).values().flatten().any(|value| value.contains('<')) } @@ -396,6 +431,16 @@ mod tests { assert!(expansion_too_large(&long_name)); } + #[test] + fn entity_markers_in_a_comment_are_scanned_in_linear_time() { + // 100,000 `"); + assert!(!entities_expand_past(&svg, MAX_ENTITY_EXPANSION)); + } + fn svg(body: &str) -> String { format!(r#"{body}"#) } diff --git a/crates/svg/src/path_data.rs b/crates/svg/src/path_data.rs index 96d0db771..fc31fbaf2 100644 --- a/crates/svg/src/path_data.rs +++ b/crates/svg/src/path_data.rs @@ -214,6 +214,10 @@ pub fn parse_path(d: &str) -> Result> { } cur = start; cmd = 0; + // Z is not a cubic or quadratic, so a following S or T starts from the + // current point rather than reflecting the pre-close control. + prev_cubic = None; + prev_quad = None; continue; } cmd = c; @@ -434,6 +438,15 @@ mod tests { assert_eq!(paths[0].segs[1], Segment::Cubic(Point::new(8.0, -2.0), Point::new(10.0, -2.0), Point::new(12.0, 0.0))); } + #[test] + fn close_resets_smooth_control_history() { + // After Z, S must not reflect the preceding C's control (20,20) → (0,0). + let paths = parse_path("M10 10 C10 20 20 20 20 10 Z S30 0 30 10").unwrap(); + assert_eq!(paths.len(), 2); + assert!(paths[0].closed); + assert_eq!(paths[1].segs[0], Segment::Cubic(Point::new(10.0, 10.0), Point::new(30.0, 0.0), Point::new(30.0, 10.0))); + } + #[test] fn arc_centers_keep_radius_correction_and_sweep() { let (center, delta) = arc_center_circular(Point::new(0.0, 0.0), Point::new(10.0, 0.0), 2.0, false, true).unwrap(); diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 99b6f73a4..ab04a3002 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -174,12 +174,49 @@ impl Preflight<'_> { self.depth } + /// The SVG elements in the document that the parsed tree leaves out, by name and count, + /// sorted by name: animation, scripts, embedded HTML and media, and text when this build + /// converts no text (neither `tree-text` nor `tree-render`). For an import's "not imported" notes. + pub fn unsupported_elements(&self) -> Vec<(&'static str, usize)> { + const SVG_NS: &str = "http://www.w3.org/2000/svg"; + let mut counts: std::collections::BTreeMap<&'static str, usize> = std::collections::BTreeMap::new(); + for node in self.xml.descendants().filter(roxmltree::Node::is_element) { + if !matches!(node.tag_name().namespace(), None | Some(SVG_NS)) { + continue; + } + if let Some(name) = UNSUPPORTED.iter().copied().find(|name| *name == node.tag_name().name()) { + *counts.entry(name).or_default() += 1; + } + } + counts.into_iter().collect() + } + /// Converts this already-checked XML into the configured usvg tree. pub fn parse(self, options: &usvg::Options<'_>) -> Result { usvg::Tree::from_xmltree(&self.xml, options).map_err(|e| Error::Parse(e.to_string())) } } +/// Elements the parsed tree never holds: usvg reads static content only, and text needs fonts. +const UNSUPPORTED: &[&str] = &[ + "animate", + "animateColor", + "animateMotion", + "animateTransform", + "audio", + "canvas", + "discard", + "foreignObject", + "iframe", + "script", + "set", + // usvg converts text when built with its `text` feature: `tree-text`, or `tree-render` + // through resvg. + #[cfg(not(any(feature = "tree-text", feature = "tree-render")))] + "text", + "video", +]; + /// Checks already-decoded SVG text once, retaining the parsed XML for both depth policy and tree conversion. pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { let preflight_limits = crate::preflight::Limits { diff --git a/crates/svg/src/tree/document.rs b/crates/svg/src/tree/document.rs index 838f62e98..19f594173 100644 --- a/crates/svg/src/tree/document.rs +++ b/crates/svg/src/tree/document.rs @@ -1,8 +1,12 @@ //! A format-neutral vector scene made from a bounded SVG tree. //! //! The scene preserves paths, groups, paints, transforms, opacity, strokes and image sources -//! without depending on an application's document model. Coordinates and stroke widths use the -//! SVG tree's CSS-pixel coordinate system. +//! without depending on an application's document model. +//! +//! [`Document::from_tree`] gives every coordinate and stroke width in the document's CSS pixels +//! ([`Coordinates::Absolute`]). [`Document::from_tree_local`] keeps each node in its own user +//! space with its transform relative to its parent ([`Coordinates::Local`]), for an app that +//! keeps a node's geometry editable apart from where it is placed. #[cfg(feature = "tree-render")] use super::draw::Refusal; @@ -18,9 +22,42 @@ pub struct Document { pub images: usize, /// True if nodes deeper than the requested depth were omitted. pub too_deep: bool, + /// The space segments, transforms, bounds and stroke widths are given in. + pub coordinates: Coordinates, + /// Group effects the scene flags ([`GroupFeatures`]) but doesn't carry, counted. + pub omitted: Omitted, source: Option>, } +/// How many groups have an effect the scene doesn't carry: an app that draws only the scene +/// draws them without it, and can say so. Elements usvg itself leaves out (animation, scripts…) +/// are listed by [`super::Preflight::unsupported_elements`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Omitted { + /// Groups with `filter`. + pub filters: usize, + /// Groups with `clip-path`. + pub clip_paths: usize, + /// Groups with `mask`. + pub masks: usize, + /// Groups with a `mix-blend-mode` other than normal. + pub blends: usize, +} + +/// Where a scene's geometry is given. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Coordinates { + /// Document CSS pixels: path segments, stroke widths and dashes are already transformed, a + /// path's `transform` is the identity, and a group's or image's `transform` is its full + /// transform to the document. + #[default] + Absolute, + /// Each node in its own user space: path segments, stroke widths, dashes, gradients and + /// bounds are as the SVG gives them, and every `transform` maps the node into its parent + /// (the document for top-level nodes). + Local, +} + impl PartialEq for Document { fn eq(&self, other: &Self) -> bool { self.width == other.width @@ -28,6 +65,8 @@ impl PartialEq for Document { && self.nodes == other.nodes && self.images == other.images && self.too_deep == other.too_deep + && self.coordinates == other.coordinates + && self.omitted == other.omitted } } @@ -44,7 +83,16 @@ impl NodePath { /// A group or path in document order. #[derive(Clone, Debug, PartialEq)] pub enum Node { - Group { children: Vec, opacity: f32, transform: [f64; 6], bounds: [f64; 4], features: GroupFeatures, source_path: NodePath }, + Group { + children: Vec, + opacity: f32, + transform: [f64; 6], + bounds: [f64; 4], + features: GroupFeatures, + source_path: NodePath, + /// The element's `id`, empty when it has none. + id: String, + }, Path(Box), Image(Image), } @@ -80,13 +128,27 @@ pub enum ImageKind { Svg, } -/// A transformed path with its resolved fill and stroke. +/// A path with its resolved fill and stroke. #[derive(Clone, Debug, PartialEq)] pub struct Path { pub segments: Vec, pub fill: Option, + /// How the fill decides what is inside (`fill-rule`); `NonZero` when there is no fill. + pub fill_rule: FillRule, pub stroke: Option, pub source_path: NodePath, + /// The element's `id`, empty when it has none. + pub id: String, + /// Maps the segments into the parent ([`Coordinates`]); the identity in absolute scenes. + pub transform: [f64; 6], +} + +/// The `fill-rule` property. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FillRule { + #[default] + NonZero, + EvenOdd, } impl Path { @@ -97,7 +159,7 @@ impl Path { } } -/// A path segment in CSS pixels after its absolute transform. +/// A path segment, in the space the scene's [`Coordinates`] name. #[derive(Clone, Copy, Debug, PartialEq)] pub enum Segment { MoveTo([f64; 2]), @@ -107,12 +169,42 @@ pub enum Segment { Close, } -/// A resolved color or an unsupported SVG paint server. +/// A resolved color, gradient or pattern. #[derive(Clone, Debug, PartialEq)] pub enum Paint { - Color { rgb: [u8; 3], opacity: f32 }, - Gradient { kind: GradientKind, stops: Vec, transform: [f64; 6], spread: Spread, opacity: f32 }, - Pattern, + Color { + rgb: [u8; 3], + opacity: f32, + }, + Gradient { + kind: GradientKind, + stops: Vec, + transform: [f64; 6], + spread: Spread, + opacity: f32, + }, + /// A ``; `None` when its content is past the depth limit or refers back to the + /// pattern itself. + Pattern { + pattern: Option, + opacity: f32, + }, +} + +/// A resolved `` tile. usvg has already applied `patternUnits`, +/// `patternContentUnits` and `viewBox`, so the tile and its content share one space. +#[derive(Clone, Debug, PartialEq)] +pub struct Pattern { + /// The element's `id`. + pub id: String, + /// The tile `[x, y, width, height]` in tile space. + pub rect: [f64; 4], + /// Maps tile space into the painted path's space: `patternTransform` in a local scene; in + /// an absolute scene, `patternTransform` followed by the path's transform to the document. + pub transform: [f64; 6], + /// What one tile draws, in tile space ([`Coordinates::Local`] whatever the scene's + /// coordinates). Shared by every use of the same pattern. + pub content: std::sync::Arc>, } #[derive(Clone, Debug, PartialEq)] @@ -162,20 +254,32 @@ pub enum LineJoin { } impl Document { - /// Convert a bounded usvg tree to a scene, omitting nodes beyond `max_depth`. + /// Convert a bounded usvg tree to a scene in document pixels ([`Coordinates::Absolute`]), + /// omitting nodes beyond `max_depth`. pub fn from_tree(tree: &usvg::Tree, max_depth: usize) -> Self { - let size = tree.size(); - let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), ..Self::default() }; - document.nodes = convert_group(tree.root(), 0, max_depth, &mut document, &[]); + Self::convert(tree, max_depth, Coordinates::Absolute) + } + + /// Convert a bounded usvg tree to a scene that keeps every node in its own user space with a + /// transform relative to its parent ([`Coordinates::Local`]), omitting nodes beyond + /// `max_depth`. + pub fn from_tree_local(tree: &usvg::Tree, max_depth: usize) -> Self { + Self::convert(tree, max_depth, Coordinates::Local) + } + + /// Build a scene in document pixels that retains its source tree for fallback rendering. + pub fn from_owned_tree(tree: usvg::Tree, max_depth: usize) -> Self { + let mut document = Self::convert(&tree, max_depth, Coordinates::Absolute); + document.source = Some(Box::new(tree)); document } - /// Build a scene that retains its source tree for fallback rendering. - pub fn from_owned_tree(tree: usvg::Tree, max_depth: usize) -> Self { + fn convert(tree: &usvg::Tree, max_depth: usize, coordinates: Coordinates) -> Self { let size = tree.size(); - let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), ..Self::default() }; - document.nodes = convert_group(tree.root(), 0, max_depth, &mut document, &[]); - document.source = Some(Box::new(tree)); + let mut document = Self { width: f64::from(size.width()), height: f64::from(size.height()), coordinates, ..Self::default() }; + let mut cx = Convert { doc: &mut document, max_depth, local: coordinates == Coordinates::Local, patterns: std::collections::HashMap::new() }; + let nodes = cx.group(tree.root(), tree.root().abs_transform(), 0, &[]); + document.nodes = nodes; document } @@ -231,7 +335,7 @@ fn spread(s: usvg::SpreadMethod) -> Spread { } } -fn color(paint: &usvg::Paint, opacity: f32) -> Paint { +fn color(paint: &usvg::Paint, opacity: f32, pattern: impl FnOnce(&std::sync::Arc) -> Option) -> Paint { match paint { usvg::Paint::Color(c) => Paint::Color { rgb: [c.red, c.green, c.blue], opacity }, usvg::Paint::LinearGradient(g) => Paint::Gradient { @@ -269,7 +373,7 @@ fn color(paint: &usvg::Paint, opacity: f32) -> Paint { spread: spread(g.spread_method()), opacity, }, - usvg::Paint::Pattern(_) => Paint::Pattern, + usvg::Paint::Pattern(source) => Paint::Pattern { pattern: pattern(source), opacity }, } } @@ -279,122 +383,213 @@ fn point(transform: usvg::Transform, p: usvg::tiny_skia_path::Point) -> [f64; 2] [f64::from(p.x), f64::from(p.y)] } -fn convert_group(group: &usvg::Group, depth: usize, max_depth: usize, doc: &mut Document, parent_path: &[usize]) -> Vec { - let mut result = Vec::new(); - for (index, node) in group.children().iter().enumerate() { - let mut source_path = parent_path.to_vec(); - source_path.push(index); - match node { - usvg::Node::Path(path) => { - if !path.is_visible() || path.data().is_empty() { - continue; +const IDENTITY: [f64; 6] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + +/// A node's transform relative to `parent` (both absolute): `parent⁻¹ · node`. The identity +/// when `parent` cannot be inverted (a degenerate group draws nothing anyway). +fn relative(parent: usvg::Transform, node: usvg::Transform) -> usvg::Transform { + parent.invert().map_or(usvg::Transform::identity(), |inverse| inverse.pre_concat(node)) +} + +struct Convert<'d> { + doc: &'d mut Document, + max_depth: usize, + local: bool, + /// Pattern content already converted, by the address of usvg's shared pattern; `None` while + /// a pattern is being converted, so a pattern that refers back to itself gets no content. + patterns: std::collections::HashMap>>>, +} + +impl Convert<'_> { + /// The transform a node with absolute transform `node` gets under a group whose absolute + /// transform is `parent`. + fn placement(&self, parent: usvg::Transform, node: usvg::Transform) -> [f64; 6] { + if self.local { transform(relative(parent, node)) } else { transform(node) } + } + + /// `parent` is the absolute transform `group`'s leaves are placed against: its own, except + /// for flattened text, whose outline paths carry the text's while the group has none. + fn group(&mut self, group: &usvg::Group, parent: usvg::Transform, depth: usize, parent_path: &[usize]) -> Vec { + let mut result = Vec::new(); + for (index, node) in group.children().iter().enumerate() { + let mut source_path = parent_path.to_vec(); + source_path.push(index); + match node { + usvg::Node::Path(path) => { + if let Some(path) = self.path(path, parent, source_path, depth) { + result.push(Node::Path(Box::new(path))); + } } - let transform = path.abs_transform(); - let segments = path - .data() - .segments() - .map(|segment| { - use usvg::tiny_skia_path::PathSegment as S; - match segment { - S::MoveTo(a) => Segment::MoveTo(point(transform, a)), - S::LineTo(a) => Segment::LineTo(point(transform, a)), - S::QuadTo(a, b) => Segment::QuadTo(point(transform, a), point(transform, b)), - S::CubicTo(a, b, c) => Segment::CubicTo(point(transform, a), point(transform, b), point(transform, c)), - S::Close => Segment::Close, + usvg::Node::Group(child) => { + if depth >= self.max_depth { + self.doc.too_deep = true; + continue; + } + let children = self.group(child, child.abs_transform(), depth + 1, &source_path); + if !children.is_empty() { + let omitted = &mut self.doc.omitted; + omitted.filters += usize::from(!child.filters().is_empty()); + omitted.clip_paths += usize::from(child.clip_path().is_some()); + omitted.masks += usize::from(child.mask().is_some()); + omitted.blends += usize::from(child.blend_mode() != usvg::BlendMode::Normal); + let bounds = if self.local { child.layer_bounding_box().to_rect() } else { child.abs_layer_bounding_box().to_rect() }; + result.push(Node::Group { + children, + opacity: child.opacity().get(), + // Locally, the group's own transform: usvg leaves the absolute one at + // the identity for colour and bitmap glyph groups in text. + transform: if self.local { transform(child.transform()) } else { transform(child.abs_transform()) }, + bounds: rect(bounds), + features: GroupFeatures { + filters: !child.filters().is_empty(), + clip_path: child.clip_path().is_some(), + mask: child.mask().is_some(), + non_normal_blend: child.blend_mode() != usvg::BlendMode::Normal, + isolation: child.should_isolate(), + }, + source_path: NodePath(source_path), + id: child.id().to_string(), + }); + } + } + usvg::Node::Text(text) => { + if depth >= self.max_depth { + self.doc.too_deep = true; + continue; + } + let flat = text.flattened(); + // The flattened group has no transform of its own, but its outline paths + // have the text's: place its leaves, and the wrapper, in the text's frame. + let frame = if self.local { text.abs_transform() } else { flat.abs_transform() }; + let children = self.group(flat, frame, depth + 1, &source_path); + if !children.is_empty() { + let bounds = if self.local { text.bounding_box() } else { text.abs_bounding_box() }; + result.push(Node::Group { + children, + opacity: 1.0, + transform: self.placement(parent, frame), + bounds: rect(bounds), + features: GroupFeatures::default(), + source_path: NodePath(source_path), + id: text.id().to_string(), + }); + } + } + usvg::Node::Image(image) => { + self.doc.images += 1; + let kind = match image.kind() { + usvg::ImageKind::JPEG(_) => ImageKind::Jpeg, + usvg::ImageKind::PNG(_) => ImageKind::Png, + usvg::ImageKind::GIF(_) => ImageKind::Gif, + usvg::ImageKind::WEBP(_) => ImageKind::WebP, + usvg::ImageKind::SVG(_) => ImageKind::Svg, + }; + let (encoded, svg) = match image.kind() { + usvg::ImageKind::JPEG(data) | usvg::ImageKind::PNG(data) | usvg::ImageKind::GIF(data) | usvg::ImageKind::WEBP(data) => { + (Some(data.clone()), None) } - }) - .collect(); - let fill = path.fill().map(|f| color(f.paint(), f.opacity().get())); - let stroke = path.stroke().map(|s| { - let scale = f64::from((transform.sx * transform.sy - transform.kx * transform.ky).abs().sqrt()); - Stroke { - paint: color(s.paint(), s.opacity().get()), - width: f64::from(s.width().get()) * scale, - cap: match s.linecap() { - usvg::LineCap::Butt => LineCap::Butt, - usvg::LineCap::Round => LineCap::Round, - usvg::LineCap::Square => LineCap::Square, - }, - join: match s.linejoin() { - usvg::LineJoin::Round => LineJoin::Round, - usvg::LineJoin::Bevel => LineJoin::Bevel, - usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => LineJoin::Miter, - }, - miter_limit: f64::from(s.miterlimit().get()), - dash_array: s.dasharray().map(|d| d.iter().map(|v| f64::from(*v) * scale).collect()), - dash_offset: f64::from(s.dashoffset()) * scale, - } - }); - result.push(Node::Path(Box::new(Path { segments, fill, stroke, source_path: NodePath(source_path) }))); - } - usvg::Node::Group(child) => { - if depth >= max_depth { - doc.too_deep = true; - continue; - } - let children = convert_group(child, depth + 1, max_depth, doc, &source_path); - if !children.is_empty() { - result.push(Node::Group { - children, - opacity: child.opacity().get(), - transform: transform(child.abs_transform()), - bounds: rect(child.abs_layer_bounding_box().to_rect()), - features: GroupFeatures { - filters: !child.filters().is_empty(), - clip_path: child.clip_path().is_some(), - mask: child.mask().is_some(), - non_normal_blend: child.blend_mode() != usvg::BlendMode::Normal, - isolation: child.should_isolate(), - }, + usvg::ImageKind::SVG(tree) => { + let max_depth = self.max_depth.saturating_sub(depth + 1); + let nested = if self.local { + Document::from_tree_local(tree, max_depth) + } else { + Document::from_owned_tree(tree.clone(), max_depth) + }; + (None, Some(Box::new(nested))) + } + }; + let bounds = if self.local { image.bounding_box() } else { image.abs_bounding_box() }; + result.push(Node::Image(Image { + kind, + visible: image.is_visible(), + transform: self.placement(parent, image.abs_transform()), + bounds: rect(bounds), + encoded, + svg, source_path: NodePath(source_path), - }); + })); } } - usvg::Node::Text(text) => { - if depth >= max_depth { - doc.too_deep = true; - continue; - } - let children = convert_group(text.flattened(), depth + 1, max_depth, doc, &source_path); - if !children.is_empty() { - result.push(Node::Group { - children, - opacity: 1.0, - transform: transform(text.abs_transform()), - bounds: rect(text.abs_bounding_box()), - features: GroupFeatures::default(), - source_path: NodePath(source_path), - }); - } - } - usvg::Node::Image(image) => { - doc.images += 1; - let kind = match image.kind() { - usvg::ImageKind::JPEG(_) => ImageKind::Jpeg, - usvg::ImageKind::PNG(_) => ImageKind::Png, - usvg::ImageKind::GIF(_) => ImageKind::Gif, - usvg::ImageKind::WEBP(_) => ImageKind::WebP, - usvg::ImageKind::SVG(_) => ImageKind::Svg, - }; - let (encoded, svg) = match image.kind() { - usvg::ImageKind::JPEG(data) | usvg::ImageKind::PNG(data) | usvg::ImageKind::GIF(data) | usvg::ImageKind::WEBP(data) => { - (Some(data.clone()), None) - } - usvg::ImageKind::SVG(tree) => { - (None, Some(Box::new(Document::from_owned_tree(tree.clone(), max_depth.saturating_sub(depth + 1))))) - } - }; - result.push(Node::Image(Image { - kind, - visible: image.is_visible(), - transform: transform(image.abs_transform()), - bounds: rect(image.abs_bounding_box()), - encoded, - svg, - source_path: NodePath(source_path), - })); - } } + result + } + + /// A visible, non-empty path. Its own absolute transform is its group's in usvg, so in a + /// local scene its segments stay as they are and its transform is normally the identity. + /// The tile of `pattern` painted on a path at `depth` whose own transform to the document is + /// `abs`. + fn pattern(&mut self, pattern: &std::sync::Arc, abs: usvg::Transform, depth: usize) -> Option { + let key = std::sync::Arc::as_ptr(pattern) as usize; + let content = match self.patterns.get(&key) { + Some(done) => done.clone(), + None if depth >= self.max_depth => { + self.doc.too_deep = true; + None + } + None => { + self.patterns.insert(key, None); + let local = std::mem::replace(&mut self.local, true); + let nodes = self.group(pattern.root(), pattern.root().abs_transform(), depth + 1, &[]); + self.local = local; + let content = Some(std::sync::Arc::new(nodes)); + self.patterns.insert(key, content.clone()); + content + } + }?; + let placement = if self.local { pattern.transform() } else { abs.pre_concat(pattern.transform()) }; + Some(Pattern { id: pattern.id().to_string(), rect: rect(pattern.rect().to_rect()), transform: transform(placement), content }) + } + + fn path(&mut self, path: &usvg::Path, parent: usvg::Transform, source_path: Vec, depth: usize) -> Option { + if !path.is_visible() || path.data().is_empty() { + return None; + } + let abs = path.abs_transform(); + let (to_points, scale, placement) = if self.local { + (usvg::Transform::identity(), 1.0, transform(relative(parent, abs))) + } else { + (abs, f64::from((abs.sx * abs.sy - abs.kx * abs.ky).abs().sqrt()), IDENTITY) + }; + let segments = path + .data() + .segments() + .map(|segment| { + use usvg::tiny_skia_path::PathSegment as S; + match segment { + S::MoveTo(a) => Segment::MoveTo(point(to_points, a)), + S::LineTo(a) => Segment::LineTo(point(to_points, a)), + S::QuadTo(a, b) => Segment::QuadTo(point(to_points, a), point(to_points, b)), + S::CubicTo(a, b, c) => Segment::CubicTo(point(to_points, a), point(to_points, b), point(to_points, c)), + S::Close => Segment::Close, + } + }) + .collect(); + let fill = path.fill().map(|f| color(f.paint(), f.opacity().get(), |p| self.pattern(p, abs, depth))); + let fill_rule = match path.fill().map(usvg::Fill::rule) { + Some(usvg::FillRule::EvenOdd) => FillRule::EvenOdd, + _ => FillRule::NonZero, + }; + let stroke = path.stroke().map(|s| Stroke { + paint: color(s.paint(), s.opacity().get(), |p| self.pattern(p, abs, depth)), + width: f64::from(s.width().get()) * scale, + cap: match s.linecap() { + usvg::LineCap::Butt => LineCap::Butt, + usvg::LineCap::Round => LineCap::Round, + usvg::LineCap::Square => LineCap::Square, + }, + join: match s.linejoin() { + usvg::LineJoin::Round => LineJoin::Round, + usvg::LineJoin::Bevel => LineJoin::Bevel, + usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => LineJoin::Miter, + }, + miter_limit: f64::from(s.miterlimit().get()), + dash_array: s.dasharray().map(|d| d.iter().map(|v| f64::from(*v) * scale).collect()), + dash_offset: f64::from(s.dashoffset()) * scale, + }); + Some(Path { segments, fill, fill_rule, stroke, source_path: NodePath(source_path), id: path.id().to_string(), transform: placement }) } - result } + +#[cfg(test)] +#[path = "document_tests.rs"] +mod tests; diff --git a/crates/svg/src/tree/document_tests.rs b/crates/svg/src/tree/document_tests.rs new file mode 100644 index 000000000..a130a6f6f --- /dev/null +++ b/crates/svg/src/tree/document_tests.rs @@ -0,0 +1,349 @@ +#![allow(clippy::unwrap_used, clippy::panic, clippy::indexing_slicing, clippy::float_cmp)] + +use super::*; +use crate::tree::{Limits, parse}; + +fn tree(body: &str) -> usvg::Tree { + let svg = format!(r#"{body}"#); + parse(svg.as_bytes(), &Limits::default(), &usvg::Options::default()).unwrap() +} + +fn apply(m: [f64; 6], p: [f64; 2]) -> [f64; 2] { + [m[0] * p[0] + m[2] * p[1] + m[4], m[1] * p[0] + m[3] * p[1] + m[5]] +} + +fn then(outer: [f64; 6], inner: [f64; 6]) -> [f64; 6] { + let [a, b, c, d, e, f] = outer; + let [p, q, r, s, t, u] = inner; + [a * p + c * q, b * p + d * q, a * r + c * s, b * r + d * s, a * t + c * u + e, b * t + d * u + f] +} + +fn close(a: [f64; 2], b: [f64; 2]) -> bool { + (a[0] - b[0]).abs() < 1e-3 && (a[1] - b[1]).abs() < 1e-3 +} + +/// Every path's points, mapped to document pixels through the scene's transforms. +fn document_points(nodes: &[Node], outer: [f64; 6], local: bool, out: &mut Vec<[f64; 2]>) { + for node in nodes { + match node { + Node::Group { children, transform, .. } => { + let next = if local { then(outer, *transform) } else { outer }; + document_points(children, next, local, out); + } + Node::Path(path) => { + let m = then(outer, path.transform); + for segment in &path.segments { + match segment { + Segment::MoveTo(p) | Segment::LineTo(p) => out.push(apply(m, *p)), + Segment::QuadTo(a, b) => out.extend([apply(m, *a), apply(m, *b)]), + Segment::CubicTo(a, b, c) => out.extend([apply(m, *a), apply(m, *b), apply(m, *c)]), + Segment::Close => {} + } + } + } + Node::Image(_) => {} + } + } +} + +fn paths(nodes: &[Node]) -> Vec<&Path> { + let mut out = Vec::new(); + for node in nodes { + match node { + Node::Group { children, .. } => out.extend(paths(children)), + Node::Path(path) => out.push(path), + Node::Image(_) => {} + } + } + out +} + +fn group_ids(nodes: &[Node]) -> Vec<(String, [f64; 6])> { + let mut out = Vec::new(); + for node in nodes { + if let Node::Group { children, id, transform, .. } = node { + out.push((id.clone(), *transform)); + out.extend(group_ids(children)); + } + } + out +} + +const NESTED: &str = r##" + + + + +"##; + +#[test] +fn a_local_scene_keeps_user_space_geometry_and_relative_transforms() { + let tree = tree(NESTED); + let local = Document::from_tree_local(&tree, 64); + assert_eq!(local.coordinates, Coordinates::Local); + let groups = group_ids(&local.nodes); + let outer = groups.iter().find(|(id, _)| id == "outer").unwrap(); + assert!(close([outer.1[0], outer.1[3]], [2.0, 2.0]) && close([outer.1[4], outer.1[5]], [10.0, 20.0]), "{outer:?}"); + let inner = groups.iter().find(|(id, _)| id == "inner").unwrap(); + assert!(close([inner.1[0], inner.1[1]], [0.0, 1.0]) && close([inner.1[4], inner.1[5]], [0.0, 0.0]), "relative, not absolute: {inner:?}"); + + let all = paths(&local.nodes); + let rect = all.iter().find(|p| p.id == "box").unwrap(); + let Segment::MoveTo(start) = rect.segments[0] else { panic!("{:?}", rect.segments) }; + assert!(close(start, [1.0, 2.0]), "user space: {start:?}"); + let stroke = rect.stroke.as_ref().unwrap(); + assert_eq!(stroke.width, 2.0, "not scaled by the groups"); + assert_eq!(stroke.dash_array.as_deref(), Some([1.0, 3.0].as_slice())); +} + +#[test] +fn an_absolute_scene_is_unchanged_by_the_local_option() { + let tree = tree(NESTED); + let absolute = Document::from_tree(&tree, 64); + assert_eq!(absolute.coordinates, Coordinates::Absolute); + let rect = paths(&absolute.nodes).into_iter().find(|p| p.id == "box").unwrap(); + assert_eq!(rect.transform, IDENTITY); + // (1, 2) rotated 90° is (-2, 1), then scaled by 2 and moved by (10, 20). + let Segment::MoveTo(start) = rect.segments[0] else { panic!("{:?}", rect.segments) }; + assert!(close(start, [6.0, 22.0]), "{start:?}"); + assert_eq!(rect.stroke.as_ref().unwrap().width, 4.0); +} + +#[test] +fn local_and_absolute_scenes_put_every_point_in_the_same_place() { + let tree = tree(NESTED); + let (mut local, mut absolute) = (Vec::new(), Vec::new()); + document_points(&Document::from_tree_local(&tree, 64).nodes, IDENTITY, true, &mut local); + document_points(&Document::from_tree(&tree, 64).nodes, IDENTITY, false, &mut absolute); + assert_eq!(local.len(), absolute.len()); + assert!(!local.is_empty()); + for (l, a) in local.iter().zip(&absolute) { + assert!(close(*l, *a), "{l:?} vs {a:?}"); + } +} + +#[test] +fn fill_rule_and_ids_are_kept() { + let tree = + tree(r##""##); + for document in [Document::from_tree(&tree, 64), Document::from_tree_local(&tree, 64)] { + let all = paths(&document.nodes); + assert_eq!(all.len(), 2); + assert_eq!((all[0].id.as_str(), all[0].fill_rule), ("hole", FillRule::EvenOdd)); + assert_eq!((all[1].id.as_str(), all[1].fill_rule), ("", FillRule::NonZero), "no fill: the default"); + assert!(all[1].fill.is_none()); + } +} + +#[test] +fn a_local_scene_respects_the_depth_limit() { + let deep = format!("{}{}", "".repeat(6), "".repeat(6)); + let tree = tree(&deep); + let shallow = Document::from_tree_local(&tree, 2); + assert!(shallow.too_deep); + assert!(paths(&shallow.nodes).is_empty()); + assert!(!Document::from_tree_local(&tree, 64).too_deep); +} + +#[test] +fn an_embedded_svg_in_a_local_scene_is_local_too() { + use std::io::Write as _; + let inner = r##""##; + let mut b64 = Vec::new(); + { + // A tiny base64 encoder keeps the test free of extra dependencies. + const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + for chunk in inner.as_bytes().chunks(3) { + let n = chunk.iter().enumerate().fold(0u32, |n, (i, b)| n | (u32::from(*b) << (16 - 8 * i))); + for i in 0..4 { + if i <= chunk.len() { + b64.push(A[((n >> (18 - 6 * i)) & 63) as usize]); + } else { + b64.push(b'='); + } + } + } + } + let mut body = Vec::new(); + write!(body, r#""#, String::from_utf8(b64).unwrap()).unwrap(); + let svg = format!(r#"{}"#, String::from_utf8(body).unwrap()); + let options = usvg::Options { image_href_resolver: crate::tree::image_resolver(Limits::default()), ..usvg::Options::default() }; + let tree = parse(svg.as_bytes(), &Limits::default(), &options).unwrap(); + let local = Document::from_tree_local(&tree, 64); + let image = local + .nodes + .iter() + .find_map(|n| match n { + Node::Image(image) => Some(image), + Node::Group { children, .. } => children.iter().find_map(|c| if let Node::Image(image) = c { Some(image) } else { None }), + Node::Path(_) => None, + }) + .unwrap(); + let nested = image.svg.as_ref().unwrap(); + assert_eq!(nested.coordinates, Coordinates::Local); + let rect = paths(&nested.nodes)[0]; + let Segment::LineTo(p) = rect.segments[1] else { panic!("{:?}", rect.segments) }; + assert!(close(p, [1.0, 0.0]), "inside its own scaled group: {p:?}"); +} + +fn fill_pattern(path: &Path) -> Option<&Pattern> { + match path.fill.as_ref()? { + Paint::Pattern { pattern, .. } => pattern.as_ref(), + _ => None, + } +} + +const PATTERN: &str = r##" + + +"##; + +#[test] +fn a_pattern_paint_carries_its_tile_and_content() { + let tree = tree(PATTERN); + let local = Document::from_tree_local(&tree, 64); + let all = paths(&local.nodes); + let a = all.iter().find(|p| p.id == "a").unwrap(); + let pattern = fill_pattern(a).unwrap(); + assert_eq!(pattern.id, "dots"); + assert_eq!(pattern.rect, [0.0, 0.0, 4.0, 4.0]); + assert!(close([pattern.transform[4], pattern.transform[5]], [1.0, 2.0]) && close([pattern.transform[0], pattern.transform[3]], [1.0, 1.0])); + let tile = paths(&pattern.content); + assert_eq!(tile.len(), 1); + assert_eq!(tile[0].fill, Some(Paint::Color { rgb: [255, 0, 0], opacity: 1.0 })); + + // Every use shares one converted tile, and keeps its own opacity. + let b = all.iter().find(|p| p.id == "b").unwrap(); + assert!(std::sync::Arc::ptr_eq(&pattern.content, &fill_pattern(b).unwrap().content)); + assert!(matches!(b.fill, Some(Paint::Pattern { opacity, .. }) if (opacity - 0.5).abs() < 1e-6)); +} + +#[test] +fn an_absolute_scene_maps_the_tile_to_the_document() { + let tree = tree(PATTERN); + let absolute = Document::from_tree(&tree, 64); + let a = paths(&absolute.nodes).into_iter().find(|p| p.id == "a").unwrap(); + let pattern = fill_pattern(a).unwrap(); + // translate(1 2), then the group's scale(2). + assert!(close([pattern.transform[0], pattern.transform[3]], [2.0, 2.0]), "{:?}", pattern.transform); + assert!(close([pattern.transform[4], pattern.transform[5]], [2.0, 4.0]), "{:?}", pattern.transform); + // The tile's content stays in tile space. + let Segment::LineTo(p) = paths(&pattern.content)[0].segments[1] else { panic!() }; + assert!(close(p, [2.0, 0.0]), "{p:?}"); +} + +#[test] +fn bounding_box_pattern_units_are_resolved() { + let tree = tree( + r##""##, + ); + let document = Document::from_tree_local(&tree, 64); + let pattern = fill_pattern(paths(&document.nodes)[0]).unwrap(); + assert_eq!(pattern.rect, [0.0, 0.0, 5.0, 10.0]); +} + +#[test] +fn nested_patterns_convert_within_the_depth_limit() { + let svg = r##" + + + "##; + let tree = tree(svg); + let deep = Document::from_tree_local(&tree, 64); + let outer = fill_pattern(paths(&deep.nodes)[0]).unwrap(); + let inner = fill_pattern(paths(&outer.content)[0]).unwrap(); + assert_eq!(inner.id, "inner"); + assert!(!deep.too_deep); + + let shallow = Document::from_tree_local(&tree, 1); + assert!(shallow.too_deep); + let outer = fill_pattern(paths(&shallow.nodes)[0]).unwrap(); + assert!(fill_pattern(paths(&outer.content)[0]).is_none(), "past the limit: no content"); +} + +#[test] +fn a_pattern_that_refers_to_itself_ends() { + let tree = tree( + r##""##, + ); + let document = Document::from_tree_local(&tree, 64); + // usvg may drop the reference; if it keeps it, the inner use has no content. + for path in paths(&document.nodes) { + if let Some(pattern) = fill_pattern(path) { + for inner in paths(&pattern.content) { + assert!(fill_pattern(inner).is_none()); + } + } + } +} + +#[test] +fn group_effects_the_scene_leaves_out_are_counted() { + let tree = tree( + r##" + + + + + + "##, + ); + for document in [Document::from_tree(&tree, 64), Document::from_tree_local(&tree, 64)] { + assert_eq!(document.omitted, Omitted { filters: 1, clip_paths: 2, masks: 1, blends: 1 }); + } + assert_eq!(Document::from_tree(&self::tree(r#""#), 64).omitted, Omitted::default()); +} + +#[test] +fn elements_usvg_leaves_out_are_listed() { + let svg = r##" + + + + + "##; + let checked = crate::tree::preflight(svg, &Limits::default()).unwrap(); + assert_eq!( + checked.unsupported_elements(), + vec![("animate", 2), ("animateTransform", 1), ("foreignObject", 1), ("script", 1), ("set", 1)], + "the XHTML div is not an SVG element" + ); + // The same checked XML still parses. + assert!(checked.parse(&usvg::Options::default()).is_ok()); + let plain = crate::tree::preflight(r#""#, &Limits::default()).unwrap(); + assert!(plain.unsupported_elements().is_empty()); +} + +#[cfg(feature = "tree-text")] +#[test] +fn local_text_is_placed_and_bounded_in_its_parents_frame() { + let mut options = usvg::Options::default(); + let fonts = options.fontdb_mut(); + fonts.load_font_data(include_bytes!("../../../ui/tests/fonts/NotoSans-test.ttf").to_vec()); + let family = fonts.faces().next().unwrap().families[0].0.clone(); + let svg = format!( + r#"Hi"# + ); + let tree = parse(svg.as_bytes(), &Limits::default(), &options).unwrap(); + let local = Document::from_tree_local(&tree, 64); + let absolute = Document::from_tree(&tree, 64); + + let groups = group_ids(&local.nodes); + let text = groups.iter().find(|(id, _)| id == "t").unwrap(); + assert!(close([text.1[0], text.1[3]], [1.0, 1.0]) && close([text.1[4], text.1[5]], [0.0, 0.0]), "text sits in its group's frame: {text:?}"); + + let bounds = |nodes: &[Node]| -> [f64; 4] { + let Node::Group { children, .. } = &nodes[0] else { panic!() }; + let Node::Group { bounds, .. } = &children[0] else { panic!() }; + *bounds + }; + let (l, a) = (bounds(&local.nodes), bounds(&absolute.nodes)); + assert!(close([l[0] + 50.0, l[1]], [a[0], a[1]]) && close([l[2], l[3]], [a[2], a[3]]), "local {l:?} vs absolute {a:?}"); + assert!(l[0] > 0.0 && l[0] < 10.0, "near x=5 in the group's frame: {l:?}"); + + let (mut lp, mut ap) = (Vec::new(), Vec::new()); + document_points(&local.nodes, IDENTITY, true, &mut lp); + document_points(&absolute.nodes, IDENTITY, false, &mut ap); + assert!(!lp.is_empty() && lp.len() == ap.len()); + assert!(lp.iter().zip(&ap).all(|(l, a)| close(*l, *a))); +} diff --git a/crates/svg/src/tree/primitive_tests.rs b/crates/svg/src/tree/primitive_tests.rs index 81f4c0979..ac417eb6c 100644 --- a/crates/svg/src/tree/primitive_tests.rs +++ b/crates/svg/src/tree/primitive_tests.rs @@ -109,8 +109,11 @@ fn hand_written_segments_are_recognised_too() { Segment::Close, ], fill: None, + fill_rule: crate::tree::document::FillRule::NonZero, stroke: None, source_path: crate::tree::document::NodePath::default(), + id: String::new(), + transform: [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], }; assert_eq!(path.primitive(), Some(Primitive::Ellipse { cx: 5.0, cy: 5.0, rx: 5.0, ry: 5.0 })); } diff --git a/crates/svg/src/write.rs b/crates/svg/src/write.rs index 658454ab2..af3eec992 100644 --- a/crates/svg/src/write.rs +++ b/crates/svg/src/write.rs @@ -258,6 +258,42 @@ pub fn path_data_fixed(path: &BezPath, decimals: usize) -> String { #[derive(Clone, Debug, Default)] pub struct SvgWriter(craft_xml::Writer); +// A subset ends with `] S? >`; quoted entity values, comments and PIs may contain that text. +// Scan once without allocations; malformed unterminated constructs remain the caller's error. +fn closes_internal_subset(internal: &str) -> bool { + let bytes = internal.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i..].starts_with(b"") } else { (2, b"?>") }; + let Some(offset) = bytes[i + start..].windows(end.len()).position(|part| part == end) else { + return false; + }; + i += start + offset + end.len(); + } else if matches!(bytes[i], b'\'' | b'"') { + let quote = bytes[i]; + i += 1; + while i < bytes.len() && bytes[i] != quote { + i += 1; + } + if i < bytes.len() { + i += 1; + } + } else if bytes[i] == b']' { + i += 1; + while i < bytes.len() && matches!(bytes[i], b' ' | b'\t' | b'\r' | b'\n') { + i += 1; + } + if bytes.get(i) == Some(&b'>') { + return true; + } + } else { + i += 1; + } + } + false +} + impl SvgWriter { /// A document: starts with the XML declaration. pub fn new() -> SvgWriter { @@ -270,25 +306,28 @@ impl SvgWriter { SvgWriter::default() } /// A document that starts with an XML declaration naming `encoding` (for example `UTF-8` or - /// `ISO-8859-1`) and a newline. Nothing is written if `encoding` is not an XML name. + /// `ISO-8859-1`) and a newline. Nothing is written for an invalid XML encoding name. pub fn with_encoding(encoding: &str) -> SvgWriter { let mut w = SvgWriter::fragment(); w.decl(encoding, "\n"); w } /// Writes `` followed by `newline`. Nothing is - /// written if `encoding` is not an XML name. + /// written unless `encoding` starts with an ASCII letter and continues with ASCII letters, + /// digits, `.`, `_`, or `-` (XML EncName). pub fn decl(&mut self, encoding: &str, newline: &str) { - if craft_xml::is_name(encoding) { + let mut bytes = encoding.bytes(); + if bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { self.0.raw(&format!("{newline}")); } } /// Writes `` + `newline`. When /// `internal` is empty, the inner newline is omitted, so a minified document can be /// `` with no extra breaks. Nothing is written if `name` is not an XML name - /// or `internal` contains `]]>`. + /// or `internal` closes the subset outside a quoted value, comment or processing instruction. + /// The caller still supplies valid DTD declarations; this is not a DTD validator. pub fn doctype_internal(&mut self, name: &str, internal: &str, newline: &str) { - if !craft_xml::is_name(name) || internal.contains("]]>") { + if !craft_xml::is_name(name) || closes_internal_subset(internal) { return; } let mut out = format!("\n"); assert!(SvgWriter::with_encoding("UTF-8\"").is_empty()); assert!(SvgWriter::with_encoding("").is_empty()); + for valid in ["A", "a0._-"] { + assert_eq!(SvgWriter::with_encoding(valid).finish(), format!("\n")); + } + for invalid in [":UTF-8", "_UTF8", "1UTF8", "é", "UTF:8", "UTFé", "UTF 8"] { + assert!(SvgWriter::with_encoding(invalid).is_empty(), "{invalid}"); + } let mut compact = SvgWriter::fragment(); compact.decl("ISO-8859-1", ""); compact.root().open(); @@ -479,7 +524,22 @@ mod tests { assert_eq!(empty.finish(), "\n"); let mut refused = SvgWriter::fragment(); refused.doctype_internal("svg", "]]>", "\n"); + refused.doctype_internal("svg", "]>", "?>"] { + let mut writer = SvgWriter::fragment(); + writer.doctype_internal("svg", internal, ""); + assert_eq!(writer.finish(), format!("")); + } + for internal in ["]>", "] >", "]>"] { + let mut writer = SvgWriter::fragment(); + writer.doctype_internal("svg", internal, ""); + assert!(writer.is_empty()); + } + } } diff --git a/crates/wordmodel/src/connector.rs b/crates/wordmodel/src/connector.rs index 366679070..2233b67f6 100644 --- a/crates/wordmodel/src/connector.rs +++ b/crates/wordmodel/src/connector.rs @@ -201,6 +201,8 @@ pub struct ShapeExtra { pub end_style: ArrowStyle, /// A rounded rectangle's corner radius (`roundRect`'s `adj`, ECMA-376 §20.1.10.56): in /// 100000ths of the shorter side, at most [`MAX_CORNER`]; `None` draws [`DEFAULT_CORNER`]. + /// A [`ShapeKind::Frame`]'s border width (`frame`'s `adj1`), in the same units; `None` + /// draws [`DEFAULT_FRAME`]. #[serde(skip_serializing_if = "Option::is_none")] pub corner: Option, /// The outline dash pattern; absent in older saved documents means solid. @@ -273,6 +275,16 @@ pub const MAX_CORNER: u32 = 50_000; /// A rounded rectangle's corner radius without its own: 16% of the shorter side. pub const DEFAULT_CORNER: f32 = 0.16; +/// A frame's border width without its own: `frame`'s default `adj1`, 12.5% of the shorter side. +pub const DEFAULT_FRAME: f32 = 0.125; + +/// The border width, in points, of a `w` × `h` [`ShapeKind::Frame`] with `corner` (see +/// [`ShapeExtra::corner`]): at most half the shorter side, which leaves no hole. +pub fn frame_width(corner: Option, w: f32, h: f32) -> f32 { + let side = crate::geom::finite(w).min(crate::geom::finite(h)).max(0.0); + side * corner.map_or(DEFAULT_FRAME, |c| c.min(MAX_CORNER) as f32 / 100_000.0) +} + /// The corner radius, in points, of a `w` × `h` rounded rectangle with `corner` (see /// [`ShapeExtra::corner`]). pub fn corner_radius(corner: Option, w: f32, h: f32) -> f32 { diff --git a/crates/wordmodel/src/para/inline_kinds.rs b/crates/wordmodel/src/para/inline_kinds.rs index 686089f7f..c36bd8fe6 100644 --- a/crates/wordmodel/src/para/inline_kinds.rs +++ b/crates/wordmodel/src/para/inline_kinds.rs @@ -24,6 +24,9 @@ pub enum ShapeKind { ArrowDown, Star, Heart, + /// A picture frame (DrawingML's `frame`): a rectangle with a rectangular hole, its border as + /// wide as [`crate::connector::frame_width`] says. + Frame, /// A text box (rectangle with a text story). TextBox, /// A shape drawn point by point, such as ink (its geometry is the shape's `freeform`). diff --git a/crates/wordmodel/src/props.rs b/crates/wordmodel/src/props.rs index 65a59592c..318779acb 100644 --- a/crates/wordmodel/src/props.rs +++ b/crates/wordmodel/src/props.rs @@ -27,7 +27,7 @@ pub use char_props::{ }; pub use color::{Rgb, TextColor}; pub use para_props::{ - Align, Border, BorderStyle, Borders, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, + Align, Border, BorderStyle, Borders, Kashida, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, TextAlignment, TextFrame, }; pub use table_props::{ diff --git a/crates/wordmodel/src/props/para_props.rs b/crates/wordmodel/src/props/para_props.rs index a48540248..b3dec00ad 100644 --- a/crates/wordmodel/src/props/para_props.rs +++ b/crates/wordmodel/src/props/para_props.rs @@ -26,6 +26,45 @@ impl Align { } } +/// The kashida (tatweel) length a justified paragraph uses (`ST_Jc` in ECMA-376 §17.18.44). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Kashida { + /// No kashidas: justified lines widen their word spaces (`both`, `justify`). + #[default] + None, + /// `lowKashida`. + Short, + /// `mediumKashida`. + Medium, + /// `highKashida`. + Long, +} + +impl Kashida { + /// The length named by a `w:jc` value, if that value is a kashida or ordinary justify form. + /// Other alignments return `None` so they leave an inherited length. + pub fn from_jc(value: &str) -> Option { + match value { + "both" | "justify" => Some(Kashida::None), + "lowKashida" => Some(Kashida::Short), + "mediumKashida" => Some(Kashida::Medium), + "highKashida" => Some(Kashida::Long), + _ => None, + } + } + + /// The `w:jc` value for a justified paragraph of this length. + pub fn jc(self) -> &'static str { + match self { + Kashida::None => "both", + Kashida::Short => "lowKashida", + Kashida::Medium => "mediumKashida", + Kashida::Long => "highKashida", + } + } +} + /// Paragraph-level text flow (`w:pPr/w:textDirection`, ECMA-376 §17.3.1.41). /// /// Kept separate from [`super::TextDirection`], whose three values model the table-cell direction @@ -383,6 +422,10 @@ pub struct ParaProps { pub style: Option, #[serde(skip_serializing_if = "Option::is_none")] pub align: Option, + /// Kashida length for a justified paragraph. `None` inherits; `Some(Kashida::None)` is + /// ordinary justification (`both` / `justify`) and cancels an inherited length. + #[serde(skip_serializing_if = "Option::is_none")] + pub kashida: Option, /// Left indent, points. #[serde(skip_serializing_if = "Option::is_none")] pub indent_left: Option, @@ -516,7 +559,7 @@ impl ParaProps { } fn overlay_layout(&mut self, patch: &ParaProps) { - overlay_fields!(self, patch; style, align, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, + overlay_fields!(self, patch; style, align, kashida, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, space_before, space_after, space_before_auto, space_after_auto, line_spacing, contextual_spacing, mirror_indents, adjust_right_indent, tabs, shading, borders); } diff --git a/crates/wordmodel/src/resolve.rs b/crates/wordmodel/src/resolve.rs index 7ef12ccdc..85a7a1550 100644 --- a/crates/wordmodel/src/resolve.rs +++ b/crates/wordmodel/src/resolve.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::props::{ - Align, Border, Borders, CharProps, Highlight, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, + Align, Border, Borders, CharProps, Highlight, Kashida, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, }; use crate::styles::{StyleKind, StyleSheet}; @@ -173,6 +173,9 @@ fn default_adjust_right_indent() -> bool { pub struct ResolvedPara { pub style: String, pub align: Align, + /// Kashida length when `align` is Justify; ignored for other alignments. + #[serde(default)] + pub kashida: Kashida, pub indent_left: f32, pub indent_right: f32, pub indent_first: f32, @@ -268,6 +271,7 @@ impl ResolvedPara { ResolvedPara { style: style.to_string(), align: p.align.unwrap_or_default(), + kashida: p.kashida.unwrap_or_default(), indent_left: clampi(p.indent_left), indent_right: clampi(p.indent_right), indent_first: clampi(p.indent_first), @@ -634,4 +638,33 @@ mod tests { assert_eq!(back.padding, [0.0; 4]); assert_eq!(back, ResolvedPara { padding: [0.0; 4], ..p }); } + + /// A style's kashida length is inherited, and a direct `both` / `justify` cancels it. + #[test] + fn kashida_length_follows_the_style_chain_and_direct_formatting() { + use crate::styles::{Style, StyleKind}; + let mut styles = StyleSheet::builtin(); + styles.upsert(Style { + id: "Body".into(), + name: "Body".into(), + kind: StyleKind::Paragraph, + based_on: Some("Normal".into()), + para: ParaProps { align: Some(Align::Justify), kashida: Some(Kashida::Short), ..Default::default() }, + ..Default::default() + }); + let inherited = styles.resolve_para(&ParaProps { style: Some("Body".into()), ..Default::default() }); + assert_eq!(inherited.align, Align::Justify); + assert_eq!(inherited.kashida, Kashida::Short); + + let cleared = styles.resolve_para(&ParaProps { + style: Some("Body".into()), + align: Some(Align::Justify), + kashida: Some(Kashida::None), + ..Default::default() + }); + assert_eq!(cleared.kashida, Kashida::None); + + let longer = styles.resolve_para(&ParaProps { style: Some("Body".into()), kashida: Some(Kashida::Long), ..Default::default() }); + assert_eq!(longer.kashida, Kashida::Long); + } } diff --git a/crates/wordmodel/src/wrap.rs b/crates/wordmodel/src/wrap.rs index 1c235d73e..9b305fafe 100644 --- a/crates/wordmodel/src/wrap.rs +++ b/crates/wordmodel/src/wrap.rs @@ -70,6 +70,7 @@ pub fn shape_outline(kind: ShapeKind, w: f32, h: f32, corner: Option) -> Op | ShapeKind::TextBox | ShapeKind::Line | ShapeKind::Freeform + | ShapeKind::Frame | ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector => return None, diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 998f8ed88..3d51a65a8 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -92,16 +92,25 @@ empty settings left out, a cell style format's `apply*="0"`, a number format wit in its Excel 2007 spelling, as `formatCode` has it). Custom table styles and recent colours are not read; a damaged styles part is a warning and no styles, while one past a cap (65,536 entries in a table, 1,024 gradient stops) or past `binary_text` is too large. The descent Excel records for rows -and the sheet (`x14ac:dyDescent`) is in alternate-content blocks, which are skipped. Its cell formulas are decoded from -their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from -the four ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks -typed in the formula, numbers as Excel spells them), with `FormulaKind::Shared` or -`FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a -shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A -formula whose decoding needs something the reader does not read keeps only its saved result, with a -warning saying why: user defined names, table references, other workbooks, Python code stored -outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer -are this crate's own, apart from `craft-xls`'s. +and the sheet (`x14ac:dyDescent`) is in alternate-content blocks, which are skipped. Its defined names are those of `definedNames` in the `.xlsx` +Excel writes: built-in names as `_xlnm.Print_Area`, sheet-scoped names with their `localSheetId`, +relative references as the cell they reach from A1, macro names with their `function`, `xlm` or +`vbProcedure`, `functionGroupId`, `shortcutKey`, `description` and `help`. Excel's hidden names for +newer functions and LET parameters are left out, as in `.xlsx`, and so is the formula-less +workbook-level built-in Excel writes beside a sheet's (`Database`), which formulas still spell +`_xlnm.Database`, and the formula-less name of a VBA function a cell calls (`.xlsm` lists none; the +cell spells it); a name whose formula can't be read is left out with a warning, and so is every name +whose formula uses a left-out name. A reference may name an XLM macro sheet (`Macro1!$A$1`, where +macro names point), and a 3-D span is spelled by its ends whatever chart sheets lie between them. +Its cell formulas are decoded from their tokens into the text an `.xlsx` file holds (`_xlfn.` and +`_xlpm.` names as written, apart from the four ISO/IEC 29500 functions `.xlsx` names bare, no +implicit `@`, the spaces and line breaks typed in the formula, numbers as Excel spells them), with +`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell +metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the +file keeps the tokens once. A formula whose decoding needs something the reader does not read keeps +only its saved result, with a warning saying why: a defined name left out, table references, other +workbooks, Python code stored outside the cell, data tables. The XLSB function table +(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index de623537a..02087ed4a 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,15 +1,16 @@ -//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheets (rows, cells, -//! columns, default sizes, merged cells), from their records. Cell formulas are decoded from -//! their tokens ([`formula`]) into the text an `.xlsx` file holds; one that can't be keeps only -//! its saved result, with a warning saying why. The styles part is read into the same -//! [`crate::styles::Styles`] ([`styles`]). A record that does not parse fails the read. +//! Binary workbooks ([MS-XLSB]): the workbook part with its defined names, shared strings and +//! worksheets (rows, cells, columns, default sizes, merged cells), from their records. Cell +//! formulas are decoded from their tokens ([`formula`]) into the text an `.xlsx` file holds; +//! one that can't be keeps only its saved result, with a warning saying why. The styles part +//! is read into the same [`crate::styles::Styles`] ([`styles`]). A record that does not parse +//! fails the read. mod expr; mod formula; mod ftab; mod styles; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use craft_opc::{ContentTypes, Rels}; @@ -18,7 +19,7 @@ use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS, Range, Visibility}; use crate::features::SheetFormat; use crate::sheet::{Cell, CellFormula, CellValue, Column, FormulaKind, Row, SheetPart}; use crate::strings::StringItem; -use crate::workbook::{SheetEntry, SheetKind, Workbook}; +use crate::workbook::{DefinedName, SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; @@ -293,6 +294,9 @@ pub(crate) fn workbook( records.first(131, warnings)?; let mut wb = Workbook::default(); let mut links = Some(Links::default()); + let mut pending = vec![]; + // Each BrtBundleSh's place in `wb.sheets`, `None` for a skipped one. + let mut bundle_sheets: Vec> = vec![]; let mut bundle = false; let mut bundle_seen = false; // Every BrtBundleSh counts toward the cap, a skipped module sheet too: each keeps a place @@ -309,14 +313,15 @@ pub(crate) fn workbook( if wb.sheets.is_empty() { return Err(invalid("no supported worksheets")); } - if links.is_none() { - warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"); + match links.as_mut() { + Some(l) => wb.defined_names = defined_names(pending, &bundle_sheets, l, limits, budget, warnings), + None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), } return Ok((wb, links)); } 39 | 355 | 357 | 358 | 362 | 667 => { if let Some(l) = links.as_mut() - && link_record(id, payload, l).is_err() + && link_record(id, payload, l, &mut pending).is_err() { links = None; } @@ -372,6 +377,7 @@ pub(crate) fn workbook( if let Some(l) = links.as_mut() { l.sheets.push(None); } + bundle_sheets.push(None); continue; }; let rel = rels.by_id(&rel_id).ok_or_else(|| invalid("worksheet relationship is missing"))?; @@ -387,6 +393,7 @@ pub(crate) fn workbook( if let Some(l) = links.as_mut() { l.sheets.push(matches!(kind, SheetKind::Worksheet | SheetKind::Macrosheet).then(|| name.clone())); } + bundle_sheets.push(u32::try_from(wb.sheets.len()).ok()); wb.sheets.push(SheetEntry { name, sheet_id, visibility, rel_id, kind, part: Some(rel.target.clone()) }); } _ => {} @@ -395,9 +402,184 @@ pub(crate) fn workbook( Err(invalid("missing workbook end record")) } +/// A `BrtName`'s parts that only its [`DefinedName`] needs, until the workbook part is read. +struct PendingName { + /// Its place in [`Links`]'s names. + index: usize, + flags: u32, + /// `chKey`: a macro's shortcut key. + key: u8, + rgce: Vec, + rgcb: Vec, + comment: Option, + description: Option, + help: Option, +} + +/// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the +/// hidden ones for newer functions and LET parameters, which Excel doesn't write there. `sheets` +/// maps a name's `itab` (its sheet's place in `BrtBundleSh` order) to [`Workbook::sheets`]. +/// A name whose formula can't be read, or that belongs to a sheet the reader skips, is left out +/// with a warning, and a cell formula using it keeps its saved result. +fn defined_names( + pending: Vec, + sheets: &[Option], + links: &mut Links, + limits: &Limits, + budget: &Budget, + warnings: &mut Warnings, +) -> Vec { + // Each name's formula, and the names it uses. A name whose formula uses a name that is left + // out is left out too, whatever their order, so a cell using it keeps its saved result. + let mut omitted = vec![false; links.names.len()]; + let mut kept = vec![]; + // PtgName identifies a record, but a bare workbook name in a sheet-scoped definition + // would instead bind to a same-spelled local name. Without a workbook qualifier, keep + // the cached results rather than silently changing that identity. Index once so hostile + // files with many names and references cannot turn this check into a quadratic scan. + // Ordinary shadows still count if their own formulas are omitted: do not infer a new + // binding from a partially emitted name environment. Synthetic names are not definitions. + let local_names: HashSet<_> = + links.names.iter().filter(|n| !n.future && !formula::is_param(&n.text)).filter_map(|n| n.scope.map(|s| (s, n.text.to_lowercase()))).collect(); + for p in pending { + let Some(name) = links.names.get(p.index) else { continue }; + if name.future || formula::is_param(&name.text) { + continue; + } + let local_sheet_id = match name.scope { + None => None, + Some(itab) => match usize::try_from(itab).ok().and_then(|i| sheets.get(i)) { + Some(Some(i)) => Some(*i), + _ => { + warnings.warn("XLSB: some defined names were left out (names of sheets the reader skips)"); + omit(&mut omitted, p.index); + continue; + } + }, + }; + // Excel omits the formula for a workbook-level Database built-in and a VBA function + // (fFunc, fOB, fProc). They remain usable by formulas without a definedNames entry. + // Other empty names are malformed: omitting them must also omit their dependants. + if p.rgce.is_empty() { + let database = p.flags & 0x20 != 0 && name.scope.is_none() && name.text == "_xlnm.Database"; + let vba_function = p.flags & 0x0E == 0x0E; + if database || vba_function { + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + } else { + warnings.warn("XLSB: some defined names were left out (empty formula)"); + omit(&mut omitted, p.index); + } + continue; + } + let mut used = vec![]; + let formula = match formula::decode_using(&p.rgce, &p.rgcb, links, Mode::Name, &mut used) { + Ok(e) => expr::print(&e), + Err(formula::Unsupported(why)) => { + warnings.warn(format!("XLSB: some defined names were left out ({why})")); + omit(&mut omitted, p.index); + continue; + } + }; + if name.scope.is_some_and(|scope| { + used.iter().filter_map(|&i| links.names.get(i)).any(|target| { + target.scope.is_none() + && !target.future + && !formula::is_param(&target.text) + && local_names.contains(&(scope, target.text.to_lowercase())) + }) + }) { + warnings.warn("XLSB: some defined names were left out (workbook names shadowed by local names)"); + omit(&mut omitted, p.index); + continue; + } + if !budget.try_formula_text(formula.len(), limits) { + warnings.warn("XLSB: some defined names were left out (more formula text than the limits allow)"); + omit(&mut omitted, p.index); + continue; + } + kept.push((p, local_sheet_id, formula, used)); + } + // Who uses each name, then every name reached from a left-out one: once each, so a chain + // of any length costs what its uses do. + let mut users: HashMap> = HashMap::new(); + for (k, (.., used)) in kept.iter().enumerate() { + for &u in used { + users.entry(u).or_default().push(k); + } + } + let mut queue: Vec = (0..omitted.len()).filter(|&i| omitted[i]).collect(); + while let Some(i) = queue.pop() { + for &k in users.get(&i).map_or(&[][..], Vec::as_slice) { + let Some((p, ..)) = kept.get(k) else { continue }; + if omit(&mut omitted, p.index) { + warnings.warn("XLSB: some defined names were left out (names using a name that was left out)"); + queue.push(p.index); + } + } + } + let mut out = vec![]; + for (p, local_sheet_id, formula, _) in kept { + if omitted.get(p.index).copied().unwrap_or(true) { + continue; + } + let Some(name) = links.names.get(p.index) else { continue }; + let mut attrs = vec![]; + let mut put = |k: &str, v: String| attrs.push((k.to_string(), v)); + // In `CT_DefinedName`'s order. + if let Some(d) = p.description { + put("description", d); + } + if let Some(h) = p.help { + put("help", h); + } + let (func, vba, proc) = (p.flags & 0x02 != 0, p.flags & 0x04 != 0, p.flags & 0x08 != 0); + if func { + put("function", "1".into()); + } + if vba { + put("vbProcedure", "1".into()); + } else if proc { + put("xlm", "1".into()); + } + let group = (p.flags >> 6) & 0x1FF; + if proc && group != 0 { + put("functionGroupId", group.to_string()); + } + if p.key != 0 { + put("shortcutKey", char::from(p.key).to_string()); + } + if p.flags & (1 << 15) != 0 { + put("publishToServer", "1".into()); + } + if p.flags & (1 << 16) != 0 { + put("workbookParameter", "1".into()); + } + out.push(DefinedName { + name: name.text.clone(), + local_sheet_id, + formula, + comment: p.comment.unwrap_or_default(), + hidden: p.flags & 1 != 0, + attrs, + }); + if let Some(n) = links.names.get_mut(p.index) { + n.read = true; + } + } + out +} + +/// Marks name `i` left out; whether it wasn't already. +fn omit(omitted: &mut [bool], i: usize) -> bool { + omitted.get_mut(i).is_some_and(|o| !std::mem::replace(o, true)) +} + /// A supporting link (`BrtSupSelf`, `BrtSupSame`, `BrtSupAddin`, `BrtSupBookSrc`), the sheet links -/// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`. -fn link_record(id: u16, payload: &[u8], links: &mut Links) -> Result<(), Error> { +/// (`BrtExternSheet`) or a defined name (`BrtName`), added to `links`; a name's other parts go to +/// `pending`. +fn link_record(id: u16, payload: &[u8], links: &mut Links, pending: &mut Vec) -> Result<(), Error> { let mut b = Bytes::new(payload); if matches!(id, 355 | 357 | 358 | 667) && links.books.len() >= MAX_BOOKS { return Err(invalid("too many supporting links")); @@ -426,15 +608,40 @@ fn link_record(id: u16, payload: &[u8], links: &mut Links) -> Result<(), Error> b.finish()?; } _ => { - // BrtName ([MS-XLSB] 2.4.718): flags, chKey, itab, then the name; the formula and the - // rest aren't needed to call a function or tell a name from one. + // BrtName ([MS-XLSB] 2.4.718). Its formula is decoded once the workbook part is read. if links.names.len() >= MAX_NAMES { return Err(invalid("too many defined names")); } let flags = b.u32()?; - b.take(5)?; + let key = b.u8()?; + let itab = b.u32()?; let text = b.text(255)?; - links.names.push(Name { text, future: flags & (1 << 17) != 0 }); + let cce = b.u32()? as usize; + let rgce = b.take(cce)?.to_vec(); + let cb = b.u32()? as usize; + let rgcb = b.take(cb)?.to_vec(); + let comment = b.string(255, true)?; + // A macro's name (fProc) has four more strings: one unused, its description, its help + // topic and another unused. ([MS-XLSB] calls the second the custom menu and the third + // the description; Excel writes them as `.xlsx`'s `description` and `help`.) Excel + // leaves them out of a VBA function's name (fOB), which ends after its comment. + let vba_without_strings = flags & 0x04 != 0 && b.rest.is_empty(); + let (description, help) = if flags & 0x08 != 0 && !vba_without_strings { + b.string(32767, true)?; + let description = b.string(32767, true)?; + let help = b.string(32767, true)?; + b.string(32767, true)?; + (description, help) + } else { + (None, None) + }; + b.finish()?; + let scope = (itab != u32::MAX).then_some(itab); + // A built-in name (fBuiltin) is stored bare; `.xlsx` spells it `_xlnm.Print_Area`, in + // `definedNames` and in formulas alike. + let text = if flags & 0x20 != 0 { format!("_xlnm.{text}") } else { text }; + pending.push(PendingName { index: links.names.len(), flags, key, rgce, rgcb, comment, description, help }); + links.names.push(Name { text, future: flags & (1 << 17) != 0, scope, read: false }); } } Ok(()) diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index 197644a33..4c5efe4f7 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -24,6 +24,8 @@ pub(crate) enum RefKind { Rows(u32, bool, u32, bool), /// Whole columns `A:C`. Cols(u32, bool, u32, bool), + /// A deleted cell or range on a named sheet: `Sheet2!#REF!`. + Deleted, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -176,6 +178,7 @@ fn push_reference(s: &mut String, r: &Reference) { s.push_str(dollar(*a1)); push_col(s, *c1); } + RefKind::Deleted => s.push_str(CellError::Ref.as_str()), } } @@ -280,6 +283,20 @@ fn write_args(s: &mut String, args: &[Expr], sep: char) { } } +/// A function's name; a sheet-scoped one (`My Sheet!Fn`) is quoted as a sheet-qualified name is. +/// Kept out of [`write_expr`], whose frame every nesting level of a formula repeats. +#[inline(never)] +fn push_function_name(s: &mut String, name: &str) { + match sheet_split(name) { + Some((sheet, name)) => { + s.push_str("e_sheet(sheet)); + s.push('!'); + s.push_str(name); + } + None => s.push_str(name), + } +} + fn write_expr(s: &mut String, e: &Expr) { match e { Expr::Number(n) => s.push_str(&number(*n)), @@ -328,7 +345,7 @@ fn write_expr(s: &mut String, e: &Expr) { write_expr(s, b); } Expr::Call(name, args) => { - s.push_str(name); + push_function_name(s, name); s.push('('); write_args(s, args, ','); s.push(')'); @@ -374,4 +391,12 @@ mod tests { assert_eq!(number_to_text(1e100), "1E+100"); assert!(number(f64::MAX).parse::().is_ok_and(f64::is_finite)); } + + #[test] + fn a_sheet_scoped_function_name_is_quoted() { + let call = |name: &str| print(&Expr::Call(name.into(), vec![Expr::Number(1.0)])); + assert_eq!(call("My Sheet!Fn"), "'My Sheet'!Fn(1)"); + assert_eq!(call("Data!Fn"), "Data!Fn(1)"); + assert_eq!(call("SUM"), "SUM(1)"); + } } diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index a7293c0df..7e628b197 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -4,8 +4,9 @@ //! called through hidden names), so this is its own decoder, with its own function table //! ([`super::ftab`]) and its own copies of craft-xls's depth guards and limits. Anything outside the //! token set it knows is reported as unsupported, and the caller keeps the cell's cached value. -//! So is a formula that needs something the reader doesn't read: a user's defined name (XLSB -//! names are not read yet) or Python code stored outside the cell. +//! So is a formula that needs something the reader doesn't read: a defined name whose own +//! formula could not be read, or Python code stored outside the cell. Defined names' formulas +//! are decoded here too ([`Mode::Name`]). //! //! Names keep their file spelling (`_xlfn.CONCAT`, `_xlpm.x`), as `.xlsx` formula text has them. @@ -136,13 +137,26 @@ pub(super) struct Xti { pub last: i32, } -/// A `BrtName` as far as cell formulas need it. +/// A `BrtName` as far as formulas need it. #[derive(Clone, Debug)] pub(super) struct Name { - /// The name as written (`_xlfn.CONCAT`, `_xlpm.x`, `Rate`). + /// The name as `.xlsx` spells it (`_xlfn.CONCAT`, `_xlpm.x`, `_xlnm.Print_Area`, `Rate`). pub text: String, /// `fFutureFunction`: a hidden name standing for a function newer than the file format. pub future: bool, + /// `itab`: the sheet a sheet-scoped name belongs to, by its place in `BrtBundleSh` order. + pub scope: Option, + /// Its own formula was read, so it is one of the workbook's defined names. + pub read: bool, +} + +impl Name { + /// Whether a formula may use the name: one the formula text itself defines (a newer + /// function, a LET parameter), or a defined name the reader gives. A name's formula may use + /// any other name: its text is the same either way. + fn usable(&self, mode: Mode) -> bool { + self.future || is_param(&self.text) || self.read || mode == Mode::Name + } } /// Workbook-level tables formulas refer to. @@ -158,20 +172,37 @@ pub(crate) struct Links { } /// Whether a name stands for a LET parameter (`_xlpm.x`), which only the formula itself defines. -fn is_param(name: &str) -> bool { +pub(super) fn is_param(name: &str) -> bool { name.get(..6).is_some_and(|p| p.eq_ignore_ascii_case("_xlpm.")) } impl Links { - /// The sheets a 3-D reference names, or `None` for a deleted sheet (the reference is `#REF!`). - fn sheet_sel(&self, ixti: u16) -> Result, Unsupported> { + /// `PtgName`'s and `PtgNameX`'s name: a 1-based index into every `BrtName`. A cell formula + /// using a defined name the reader doesn't give would calculate to `#NAME?`, so it keeps its + /// saved result instead. + fn name(&self, index: u32, mode: Mode) -> Result<&Name, Unsupported> { + let i = usize::try_from(index).ok().and_then(|i| i.checked_sub(1)); + let name = i.and_then(|i| self.names.get(i)).ok_or(Unsupported("reference to a missing defined name"))?; + if !name.usable(mode) { + return Err(Unsupported("defined names whose formula could not be read")); + } + Ok(name) + } + + /// Sheet link `ixti`, which must be through this workbook's own supporting link. + fn this_book(&self, ixti: u16) -> Result<&Xti, Unsupported> { let x = self.xti.get(usize::from(ixti)).ok_or(Unsupported("reference to a missing sheet link"))?; match usize::try_from(x.book).ok().and_then(|b| self.books.get(b)) { - Some(Book::This) => {} - Some(Book::External) => return Err(Unsupported("references to other workbooks (external links)")), - Some(_) => return Err(Unsupported("references through add-in or same-sheet links")), - None => return Err(Unsupported("reference to a missing workbook link")), + Some(Book::This) => Ok(x), + Some(Book::External) => Err(Unsupported("references to other workbooks (external links)")), + Some(_) => Err(Unsupported("references through add-in or same-sheet links")), + None => Err(Unsupported("reference to a missing workbook link")), } + } + + /// The sheets a 3-D reference names, or `None` for a deleted sheet (the reference is `#REF!`). + fn sheet_sel(&self, ixti: u16) -> Result, Unsupported> { + let x = self.this_book(ixti)?; // -1: "the first sheet in the reference cannot be found" (a deleted sheet). if x.first == -1 || x.last == -1 { return Ok(None); @@ -235,10 +266,15 @@ const LAST_COL: u32 = MAX_COLS - 1; /// Where a formula lives: a cell formula's references hold coordinates; a shared formula's /// relative tokens (`PtgRefN`, `PtgAreaN`) hold offsets from the shared formula's first cell. +/// +/// A defined name's relative references are offsets from A1 that wrap around the grid +/// (`RgceLocRel`), in its 3-D references too: `.xlsx` spells them as the cell they reach from A1 +/// (one column left is `Data!XFD1`). #[derive(Clone, Copy, PartialEq)] pub(super) enum Mode { Cell, Shared(CellRef), + Name, } /// RgceLoc ([MS-XLSB] 2.5.98.91): a row and a `ColRelShort` (14-bit column, then the @@ -266,9 +302,15 @@ fn loc(row: u32, col: u16, mode: Mode) -> Result { match mode { Mode::Cell => anchor(row, col), Mode::Shared(base) => relative_anchor(row, col, base), + Mode::Name => relative_anchor(row, col, CellRef::new(0, 0)), } } +/// A 3-D reference's cell: a coordinate, except in a defined name. +fn loc3d(row: u32, col: u16, mode: Mode) -> Result { + if mode == Mode::Name { loc(row, col, mode) } else { anchor(row, col) } +} + /// RgceArea: a range spanning every row (or column) is a whole-column (row) reference. fn area(r0: u32, r1: u32, c0: u16, c1: u16) -> Result { area_of(anchor(r0, c0)?, anchor(r1, c1)?) @@ -296,6 +338,11 @@ fn reference(sheet: Option, kind: RefKind) -> Expr { /// represented. A shared formula's tokens are decoded once for each of its cells /// (`Mode::Shared` with that cell), whose relative references are offsets from it. pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Result { + decode_using(rgce, rgcb, links, mode, &mut Vec::new()) +} + +/// [`decode`], adding to `used` the 0-based index of every defined name the formula uses. +pub(super) fn decode_using(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode, used: &mut Vec) -> Result { if rgce.len() > MAX_RGCE { return Err(Unsupported("formula too long")); } @@ -470,14 +517,9 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res } 0x23 => { // PtgName: a 1-based index into every BrtName, hidden ones included. - let i = usize::try_from(t.u32()?).ok().and_then(|i| i.checked_sub(1)); - let name = i.and_then(|i| links.names.get(i)).ok_or(Unsupported("reference to a missing defined name"))?; - // Defined names are not imported from XLSB yet: a cell using one would calculate - // to #NAME?, so it keeps its saved result. Newer functions and LET parameters are - // hidden names that the formula text itself defines. - if !name.future && !is_param(&name.text) { - return Err(Unsupported("defined names")); - } + let index = t.u32()?; + let name = links.name(index, mode)?; + used.extend(usize::try_from(index).ok().and_then(|i| i.checked_sub(1))); push(&mut stack, kids, Expr::Name(name.text.clone()))?; } 0x24 => { @@ -510,36 +552,53 @@ pub(super) fn decode(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode) -> Res t.take(12)?; push(&mut stack, kids, Expr::Error(CellError::Ref))?; } - 0x39 => return Err(Unsupported("names in other workbooks or on other sheets")), + 0x39 => { + // PtgNameX: a name through a supporting link. Through this workbook's own link + // it is a sheet-scoped name used from another sheet, which `.xlsx` spells + // `'Other Sheet'!Local`: Excel's files show it, [MS-XLSB] doesn't say. + let ixti = t.u16()?; + let index = t.u32()?; + links.this_book(ixti)?; + let name = links.name(index, mode)?; + used.extend(usize::try_from(index).ok().and_then(|i| i.checked_sub(1))); + let sheet = name.scope.and_then(|s| links.sheets.get(usize::try_from(s).ok()?).cloned().flatten()); + let sheet = sheet.ok_or(Unsupported("names in other workbooks or on other sheets"))?; + push(&mut stack, kids, Expr::Name(format!("{sheet}!{}", name.text)))?; + } 0x3A => { let ixti = t.u16()?; let (r, c) = (t.u32()?, t.u16()?); - let kind = RefKind::Cell(anchor(r, c)?); + let kind = RefKind::Cell(loc3d(r, c, mode)?); push(&mut stack, kids, reference(links.sheet_sel(ixti)?, kind))?; } 0x3B => { let ixti = t.u16()?; let (r0, r1, c0, c1) = (t.u32()?, t.u32()?, t.u16()?, t.u16()?); - let kind = area(r0, r1, c0, c1)?; + let kind = area_of(loc3d(r0, c0, mode)?, loc3d(r1, c1, mode)?)?; push(&mut stack, kids, reference(links.sheet_sel(ixti)?, kind))?; } 0x3C | 0x3D => { + // PtgRefErr3d, PtgAreaErr3d: a deleted cell or range, which `.xlsx` spells + // `Sheet2!#REF!` (`#REF!` when the sheet is gone too). The sheet link must still + // be one the reader follows: an external #REF! stays cached. let ixti = t.u16()?; t.take(if id == 0x3C { 6 } else { 12 })?; - // The sheet link must still be one the reader follows: an external #REF! stays cached. - links.sheet_sel(ixti)?; - push(&mut stack, kids, Expr::Error(CellError::Ref))?; + push(&mut stack, kids, reference(links.sheet_sel(ixti)?, RefKind::Deleted))?; } 0x01 => return Err(Unsupported("the shared or array formula form")), 0x02 => return Err(Unsupported("what-if data tables")), 0x18 => return Err(Unsupported("structured table references")), 0x2C => { - let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; + if mode == Mode::Cell { + return Err(Unsupported("relative references in a cell formula")); + } let (r, c) = (t.u32()?, t.u16()?); push(&mut stack, kids, Expr::Ref(Reference { sheet: SheetSel::Current, kind: RefKind::Cell(loc(r, c, mode)?) }))?; } 0x2D => { - let Mode::Shared(_) = mode else { return Err(Unsupported("relative references in a cell formula")) }; + if mode == Mode::Cell { + return Err(Unsupported("relative references in a cell formula")); + } let (r0, r1, c0, c1) = (t.u32()?, t.u32()?, t.u16()?, t.u16()?); let kind = area_of(loc(r0, c0, mode)?, loc(r1, c1, mode)?)?; push(&mut stack, kids, Expr::Ref(Reference { sheet: SheetSel::Current, kind }))?; diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 1205c1e68..96c7cff5d 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -466,6 +466,9 @@ fn name(flags: u32, text: &str) -> Vec { data.extend(wide(text)); data.extend([2, 0, 0, 0, 0x1c, 0x1d, 0, 0, 0, 0]); // Formula `#NAME?`, no extra data. data.extend(u32::MAX.to_le_bytes()); // No comment. + if flags & 0x08 != 0 { + data.extend(u32::MAX.to_le_bytes().repeat(4)); // A macro's four strings, all null. + } record(39, &data) } @@ -546,8 +549,8 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { ), (9, ref3d(3, 0, 0), vec![]), (10, [&[0x25u8][..], &0u32.to_le_bytes(), &1u32.to_le_bytes(), &REL.to_le_bytes(), &REL.to_le_bytes(), &[0x41, 20, 0]].concat(), vec![]), - // Kept as their saved results: (11, vec![0x43, 4, 0, 0, 0, 0x1E, 2, 0, 0x05], vec![]), + // Kept as their saved results: (12, ref3d(4, 0, REL), vec![]), (13, vec![0x01, 0, 0, 0, 0], vec![0, 0, 0, 0]), // Part of a shared formula at A1 that isn't one. ]; @@ -575,7 +578,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { Some("_xlfn.LET(_xlpm.x,1,_xlpm.x+1)"), Some("#REF!"), Some("SQRT(A1:A2)"), - None, + Some("Rate*2"), None, None, // A BrtCellMeta before a cell gives it cell metadata, as `cm` does in .xlsx. @@ -587,7 +590,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&CellRef::new(0, 14), &1)]); // Every formula cell keeps its saved result as its value. assert!((1..=15).all(|c| sheet.value(CellRef::new(0, c)) == Value::Number(0.0))); - for why in ["defined names", "other workbooks", "missing from the file", "structured table references"] { + for why in ["other workbooks", "missing from the file", "structured table references"] { assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); } } @@ -934,6 +937,176 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() { assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings); } +// Defined names ([MS-XLSB] 2.4.718): `BrtName` records, and the `definedNames` Excel writes for +// the same names when it saves the workbook as `.xlsx`. + +/// `BrtName`. A macro's name (`fProc`, 0x08) has four more strings, null here. +fn brt_name(flags: u32, key: u8, itab: u32, text: &str, rgce: &[u8], comment: Option<&str>) -> Vec { + let mut data = flags.to_le_bytes().to_vec(); + data.push(key); + data.extend(itab.to_le_bytes()); + data.extend(wide(text)); + data.extend((rgce.len() as u32).to_le_bytes()); + data.extend(rgce); + data.extend(0u32.to_le_bytes()); + data.extend(comment.map_or_else(|| u32::MAX.to_le_bytes().to_vec(), wide)); + if flags & 0x08 != 0 { + data.extend(u32::MAX.to_le_bytes().repeat(4)); + } + record(39, &data) +} +fn ref3d_name(ixti: u16, row: u32, col: u16) -> Vec { + [&[0x3Au8][..], &ixti.to_le_bytes(), &loc(row, col)].concat() +} +const WORKBOOK: u32 = u32::MAX; + +/// `book(false)` with the supporting links of `book_with_links` (sheet links Data, Hidden, +/// Data:Hidden, a deleted sheet, the other workbook, and this workbook as a whole, which +/// `PtgNameX` uses), and `names`. +fn book_with_names(names: &[Vec]) -> Vec { + let plain = book(false); + let end = record(132, &[]); + let mut xti = 6u32.to_le_bytes().to_vec(); + for (book, first, last) in [(0u32, 0i32, 0i32), (0, 1, 1), (0, 0, 1), (0, -1, -1), (1, 0, 0), (0, -2, -2)] { + xti.extend([book.to_le_bytes(), first.to_le_bytes(), last.to_le_bytes()].concat()); + } + let links = [record(353, &[]), record(357, &[]), record(355, &wide("rId9")), record(362, &xti), record(354, &[])]; + [&plain[..plain.len() - end.len()], &links.concat(), &names.concat(), &end].concat() +} + +/// The names below, in record order, as `PtgName` numbers them from 1. +fn names() -> Vec> { + vec![ + // 1: a newer function's hidden name: not a defined name in `.xlsx`. + brt_name(0x0002_000b, 0, WORKBOOK, "_xlfn.CONCAT", &[0x1C, 0x1D], None), + // 2: built-in and hidden, on Data. + brt_name(0x21, 0, 0, "_FilterDatabase", &[&[0x3Bu8, 0, 0][..], &0u32.to_le_bytes(), &9u32.to_le_bytes(), &[0, 0, 2, 0]].concat(), None), + // 3: refers to another workbook: left out. + brt_name(0, 0, WORKBOOK, "External", &ref3d_name(4, 0, 0), None), + // 4: a deleted cell. + brt_name(0, 0, WORKBOOK, "Gone", &[0x3C, 0, 0, 0, 0, 0, 0, 0, 0], None), + brt_name(1, 0, WORKBOOK, "Hid", &[0x1E, 42, 0], None), + // 6, 7: the same name on two sheets. + brt_name(0, 0, 0, "Local", &ref3d_name(0, 1, 1), None), + brt_name(0, 0, 1, "Local", &ref3d_name(1, 0, 1), None), + // 8: an XLM macro with a shortcut key, in the "User Defined" group (14). + brt_name(0x388, b'k', WORKBOOK, "MacroName", &ref3d_name(0, 0, 0), None), + // 9 + brt_name(0, 0, WORKBOOK, "Rate", &[&[0x1Fu8][..], &0.05f64.to_le_bytes()].concat(), Some("a note")), + // 10, 11: relative references are offsets from A1, wrapping around the grid. + brt_name(0, 0, WORKBOOK, "Rel", &ref3d_name(0, 0, 0xFFFF), None), + brt_name(0, 0, WORKBOOK, "RelN", &[&[0x2Cu8][..], &loc(u32::MAX, REL)].concat(), None), + // 12: on a sheet the workbook doesn't have: left out. + brt_name(0, 0, 5, "Skipped", &[0x1E, 1, 0], None), + // 13: a name of a name. + brt_name(0, 0, WORKBOOK, "Twice", &[0x23, 9, 0, 0, 0, 0x1E, 2, 0, 0x05], None), + ] +} + +const NAMES_XML: &str = r#"Data!#REF!Data!$B$2Hidden!$B$1Data!$A$10.05Data!XFD1A1048576Rate*2"#; + +fn xml_names(xml: &str) -> Vec { + let types = r#""#; + let root = r#""#; + let bytes = zip(&[("[Content_Types].xml", types.as_bytes()), ("_rels/.rels", root.as_bytes()), ("xl/workbook.xml", xml.as_bytes())]); + Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap().defined_names +} + +#[test] +fn defined_names_are_read_as_their_xlsx_spelling() { + let cells: Vec> = vec![ + vec![0x23, 9, 0, 0, 0, 0x1E, 100, 0, 0x05], + vec![0x23, 6, 0, 0, 0], + // PtgNameX through this workbook's own link: a sheet-scoped name used from another sheet. + vec![0x39, 5, 0, 7, 0, 0, 0], + vec![0x23, 2, 0, 0, 0], + vec![0x3C, 1, 0, 0, 0, 0, 0, 0, 0], + // Kept as their saved results: a name left out, a name in another workbook, and a + // workbook name through PtgNameX. + vec![0x23, 3, 0, 0, 0], + vec![0x39, 4, 0, 9, 0, 0, 0], + vec![0x39, 5, 0, 9, 0, 0, 0], + ]; + let mut data = row(0); + for (i, rgce) in cells.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, rgce, &[])); + } + let bytes = package(&book_with_names(&names()), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert_eq!(wb.defined_names, xml_names(NAMES_XML)); + let book = read(&bytes).unwrap(); + let got: Vec<_> = (1..=8).map(|c| book.sheets[0].formula(CellRef::new(0, c))).collect(); + let want = [Some("Rate*100"), Some("Local"), Some("Hidden!Local"), Some("_xlnm._FilterDatabase"), Some("Hidden!#REF!"), None, None, None]; + assert_eq!(got, want); + for why in ["defined names whose formula could not be read", "other workbooks", "names in other workbooks or on other sheets"] { + assert!(warned(&book, why), "{why}: {:?}", book.warnings); + } + for why in ["(references to other workbooks (external links))", "(names of sheets the reader skips)"] { + assert!(book.warnings.iter().any(|w| w.contains("defined names were left out") && w.contains(why)), "{why}: {:?}", book.warnings); + } +} + +#[test] +fn a_damaged_name_leaves_names_and_formulas_as_values() { + // A truncated BrtName: the workbook opens with no names, and every formula keeps its value. + let mut list = names(); + let full = brt_name(0, 0, WORKBOOK, "Rate", &[0x1E, 1, 0], None); + list[8] = record(39, &full[2..full.len() - 2]); // The payload without its record header and last two bytes. + let data = [row(0), formula_cell(1, &[0x1E, 1, 0], &[])].concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert!(wb.defined_names.is_empty()); + let book = read(&bytes).unwrap(); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 1)), None); + assert!(book.warnings.iter().any(|w| w.contains("names could not be read")), "{:?}", book.warnings); +} + +#[test] +fn names_on_macro_sheets_spans_over_chart_sheets_and_empty_built_ins() { + // Sheets Data, a chart sheet, an XLM macro sheet, Hidden; sheet links Macro1, Data:Hidden + // (over both) and Data. + let mut book = record(131, &[]); + book.extend(record(153, &[[0u8; 8].as_slice(), &wide("")].concat())); + book.extend(record(143, &[])); + for (id, rel, name) in [(1u32, "one", "Data"), (3, "c", "Chart1"), (4, "m", "Macro1"), (2, "two", "Hidden")] { + book.extend(record(156, &[&0u32.to_le_bytes()[..], &id.to_le_bytes(), &wide(rel), &wide(name)].concat())); + } + book.extend(record(144, &[])); + let mut xti = 3u32.to_le_bytes().to_vec(); + for (first, last) in [(2i32, 2i32), (0, 3), (0, 0)] { + xti.extend([0u32.to_le_bytes(), first.to_le_bytes(), last.to_le_bytes()].concat()); + } + book.extend([record(353, &[]), record(357, &[]), record(362, &xti), record(354, &[])].concat()); + let area = [&[0x3Bu8][..], &2u16.to_le_bytes(), &0u32.to_le_bytes(), &1u32.to_le_bytes(), &0u16.to_le_bytes(), &1u16.to_le_bytes()].concat(); + // A sheet's built-in Database, then the workbook-level one Excel writes with no formula. + book.extend(brt_name(0x20, 0, 0, "Database", &area, None)); + book.extend(brt_name(0x20, 0, WORKBOOK, "Database", &[], None)); + book.extend(brt_name(0x388, b'k', WORKBOOK, "Cmd", &ref3d_name(0, 0, 0), None)); + book.extend(brt_name(0, 0, WORKBOOK, "Span", &ref3d_name(1, 0, 0), None)); + book.extend(record(132, &[])); + let rels = RELS.replace( + "", + r#""#, + ); + let ref3d_cell = |ixti: u16| [&[0x5Au8][..], &ixti.to_le_bytes(), &loc(0, 0)].concat(); + let data = [ + row(0), + formula_cell(1, &[0x23, 2, 0, 0, 0], &[]), + formula_cell(2, &[0x23, 4, 0, 0, 0], &[]), + formula_cell(3, &ref3d_cell(0), &[]), + formula_cell(4, &ref3d_cell(1), &[]), + ] + .concat(); + let bytes = package(&book, &worksheet(&data, &[]), &sst(), ROOT, &rels, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"Data!$A$1:$B$2Macro1!$A$1Data:Hidden!$A$1"#; + assert_eq!(wb.defined_names, xml_names(xml)); + let read = read(&bytes).unwrap(); + let got: Vec<_> = (1..=4).map(|c| read.sheets[0].formula(CellRef::new(0, c))).collect(); + assert_eq!(got, [Some("_xlnm.Database"), Some("Span"), Some("Macro1!$A$1"), Some("Data:Hidden!$A$1")]); + assert!(read.warnings.iter().all(|w| !w.contains("left out") && !w.contains("saved results")), "{:?}", read.warnings); +} + /// `BrtRowHdr` with its style, height and three flag bytes, and no column spans. fn row_hdr(r: u32, ixfe: u32, twips: u16, flags: [u8; 3]) -> Vec { record(0, &[&r.to_le_bytes()[..], &ixfe.to_le_bytes(), &twips.to_le_bytes(), &flags, &0u32.to_le_bytes()].concat()) @@ -1443,6 +1616,51 @@ fn references_to_macro_sheets_and_spans_over_chart_sheets_are_named() { assert!(warned(&wb, "references to sheets with no cells"), "{:?}", wb.warnings); } +#[test] +fn vba_function_names_without_macro_strings_are_read() { + // Excel writes a VBA function a cell calls as a BrtName with fFunc, fOB and + // fProc, no formula, and none of the four strings fProc names otherwise carry. `.xlsm` lists + // no such name; a cell calling the function spells it. + let vba = |text: &str| { + let data = [ + &0x0Eu32.to_le_bytes()[..], + &[0], + &u32::MAX.to_le_bytes(), + &wide(text), + &0u32.to_le_bytes(), + &0u32.to_le_bytes(), + &u32::MAX.to_le_bytes(), + ] + .concat(); + record(39, &data) + }; + let mut list = names(); + list.push(vba("Twice")); + let twice = [&[0x23u8, 14, 0, 0, 0][..], &[0x1E, 4, 0], &[0x42, 2, 0xFF, 0]].concat(); + let data = [row(0), formula_cell(1, &twice, &[]), formula_cell(2, &[0x23, 9, 0, 0, 0], &[])].concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + assert_eq!(wb.defined_names, xml_names(NAMES_XML)); + let book = read(&bytes).unwrap(); + assert_eq!([1, 2].map(|c| book.sheets[0].formula(CellRef::new(0, c))), [Some("Twice(4)"), Some("Rate")]); +} + +#[test] +fn a_macro_names_description_and_help_topic_are_read_as_excel_writes_them() { + // An XLM function in category 3 with the strings Excel writes after its comment: unused, the + // description, the help topic, unused. [MS-XLSB] labels the middle two the custom menu and the + // description; Excel's `.xlsm` has them as `description` and `help`. + let flags: u32 = 0x02 | 0x08 | (3 << 6); + let mut data = [&flags.to_le_bytes()[..], &[0], &u32::MAX.to_le_bytes(), &wide("FnA")].concat(); + let rgce = ref3d_name(0, 0, 0); + data.extend([&(rgce.len() as u32).to_le_bytes()[..], &rgce, &0u32.to_le_bytes(), &u32::MAX.to_le_bytes()].concat()); + data.extend([u32::MAX.to_le_bytes().to_vec(), wide("Function A"), wide("!0"), u32::MAX.to_le_bytes().to_vec()].concat()); + let bytes = package(&book_with_names(&[record(39, &data)]), &worksheet(&row(0), &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"Data!$A$1"#; + assert_eq!(wb.defined_names, xml_names(xml)); +} + #[test] fn a_formulas_array_constants_share_one_allowance() { // Two arrays of 40,000 Boolean values, each within what one array may hold, added together: @@ -1470,6 +1688,28 @@ fn a_spent_formula_allowance_skips_decoding() { assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings); } +#[test] +fn a_name_using_a_left_out_name_is_left_out_too() { + // 14 Early = Later (a later name), 15 Alias = External (left out: another workbook), + // 16 Later = Alias, 17 Fine = Rate. Early, Alias and Later are left out whatever their order, + // and cells using them keep their saved results; Fine is read. + let mut list = names(); + for (text, index) in [("Early", 16u32), ("Alias", 3), ("Later", 15), ("Fine", 9)] { + list.push(brt_name(0, 0, WORKBOOK, text, &[&[0x23u8][..], &index.to_le_bytes()].concat(), None)); + } + let cell = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + let data = + [row(0), formula_cell(1, &cell(14), &[]), formula_cell(2, &cell(15), &[]), formula_cell(3, &cell(16), &[]), formula_cell(4, &cell(17), &[])] + .concat(); + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = NAMES_XML.replace("", r#"Rate"#); + assert_eq!(wb.defined_names, xml_names(&xml)); + let book = read(&bytes).unwrap(); + assert_eq!((1..=4).map(|c| book.sheets[0].formula(CellRef::new(0, c))).collect::>(), [None, None, None, Some("Fine")]); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} + #[test] fn differential_format_strings_are_charged_to_the_text_allowance() { // A DXF's font name and number format are text the styles part keeps, charged like a font's. @@ -1497,3 +1737,75 @@ fn module_sheets_and_supporting_links_are_bounded() { assert_eq!(wb.sheets[0].formula(CellRef::new(0, 0)), None); assert!(wb.warnings.iter().any(|w| w.contains("sheet links or names could not be read")), "{:?}", wb.warnings); } + +#[test] +fn empty_ordinary_names_and_their_dependants_keep_cached_values() { + // Forward and transitive references to an empty ordinary name must not become dangling + // formulas. An empty sheet-scoped Database or another built-in is not the observed + // workbook-level Database exception either. + let mut list = names(); + let cell = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + list.push(brt_name(0, 0, WORKBOOK, "Early", &cell(16), None)); // 14 + list.push(brt_name(0, 0, WORKBOOK, "Empty", &[], None)); // 15 + list.push(brt_name(0, 0, WORKBOOK, "Alias", &cell(15), None)); // 16 + list.push(brt_name(0x20, 0, 0, "Database", &[], None)); // 17 + list.push(brt_name(0x20, 0, WORKBOOK, "Print_Area", &[], None)); // 18 + list.push(brt_name(0, 0, WORKBOOK, "Fine", &cell(9), None)); // 19 + let mut data = row(0); + for i in 14..=19 { + data.extend(formula_cell(i - 13, &cell(i), &[])); + } + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = NAMES_XML.replace("", r#"Rate"#); + assert_eq!(wb.defined_names, xml_names(&xml)); + let book = read(&bytes).unwrap(); + for col in 1..=5 { + assert_eq!(book.sheets[0].formula(CellRef::new(0, col)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, col)), Value::Number(0.0)); + } + assert_eq!(book.sheets[0].formula(CellRef::new(0, 6)), Some("Fine")); + assert!(book.warnings.iter().any(|w| w.contains("empty formula")), "{:?}", book.warnings); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} + +#[test] +fn scoped_names_do_not_rebind_shadowed_workbook_names() { + let token = |i: u32| [&[0x23u8][..], &i.to_le_bytes()].concat(); + let list = vec![ + brt_name(0, 0, WORKBOOK, "Rate", &[0x1E, 1, 0], None), // 1 + brt_name(0, 0, 1, "rAtE", &[0x1E, 2, 0], None), // 2: Hidden shadows Rate case-insensitively. + brt_name(0, 0, 1, "Alias", &token(1), None), // 3: must not become Hidden!rAtE. + brt_name(0, 0, 1, "Forward", &token(5), None), // 4 + brt_name(0, 0, 1, "Dependent", &token(3), None), // 5 + brt_name(0, 0, 0, "Fine", &token(1), None), // 6: no shadow on Data. + brt_name(0, 0, 1, "LocalAlias", &token(2), None), // 7: intentionally uses Hidden!rAtE. + brt_name(0, 0, WORKBOOK, "Other", &[0x1E, 3, 0], None), // 8 + brt_name(0, 0, 1, "Other", &[], None), // 9: malformed, but still an ordinary source shadow. + brt_name(0, 0, 1, "Conservative", &token(8), None), // 10 + brt_name(0, 0, WORKBOOK, "Synthetic", &[0x1E, 4, 0], None), // 11 + brt_name(0x0002_0000, 0, 1, "Synthetic", &[0x1C, 0x1D], None), // 12: future-function metadata. + brt_name(0, 0, 1, "SyntheticControl", &token(11), None), // 13: synthetic name is not a local definition. + ]; + let mut data = row(0); + for (col, index) in [3u32, 4, 5, 6, 10, 13].into_iter().enumerate() { + // PtgNameX qualifies these names on their own sheets. + let formula = [&[0x39u8, 0, 0][..], &index.to_le_bytes()].concat(); + data.extend(formula_cell(col as u32 + 1, &formula, &[])); + } + let bytes = package(&book_with_names(&list), &worksheet(&data, &[]), &sst(), ROOT, RELS, TYPES); + let wb = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap(); + let xml = r#"12RaterAtE34Synthetic"#; + assert_eq!(wb.defined_names, xml_names(xml)); + let book = read(&bytes).unwrap(); + for col in 1..=3 { + assert_eq!(book.sheets[0].formula(CellRef::new(0, col)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, col)), Value::Number(0.0)); + } + assert_eq!(book.sheets[0].formula(CellRef::new(0, 4)), Some("Data!Fine")); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 5)), None); + assert_eq!(book.sheets[0].value(CellRef::new(0, 5)), Value::Number(0.0)); + assert_eq!(book.sheets[0].formula(CellRef::new(0, 6)), Some("Hidden!SyntheticControl")); + assert!(book.warnings.iter().any(|w| w.contains("workbook names shadowed by local names")), "{:?}", book.warnings); + assert!(book.warnings.iter().any(|w| w.contains("names using a name that was left out")), "{:?}", book.warnings); +} diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 6d50b5352..89cb2d1a7 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -17,10 +17,28 @@ thiserror = { workspace = true } zip = { version = "9", default-features = false, features = ["deflate-flate2"] } # The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +# `lenient`: DEFLATE for the raw-bytes reader. +miniz_oxide = { workspace = true, optional = true } +# `lenient` entry methods beyond stored and DEFLATE. All pure Rust and wasm32-safe. +bzip2-rs = { version = "0.1", optional = true } # MIT OR Apache-2.0 +lzma-rs = { version = "0.3", optional = true } # MIT +ruzstd = { version = "0.9.1", default-features = false, features = ["std"], optional = true } # MIT +# `serde`: `Serialize` on the `lenient` listing types, for apps that report listings as JSON. +serde = { workspace = true, optional = true } [features] # `zip64_entry_count_sample`, the regression input for a reader's own entry-count test. test-support = [] +# `craft_zip::lenient`: the raw-bytes reader for inspectors (from Craft Anything), without `zip`. +lenient = ["dep:miniz_oxide"] +# Method 12. +lenient-bzip2 = ["lenient", "dep:bzip2-rs"] +# Methods 14 (ZIP LZMA) and 95 (XZ). +lenient-lzma = ["lenient", "dep:lzma-rs"] +# Method 93. +lenient-zstd = ["lenient", "dep:ruzstd"] +serde = ["dep:serde"] [dev-dependencies] -craft-zip = { path = ".", features = ["test-support"] } +craft-zip = { path = ".", features = ["test-support", "lenient-bzip2", "lenient-lzma", "lenient-zstd", "serde"] } +serde_json = { workspace = true } diff --git a/crates/zip/README.md b/crates/zip/README.md index 6cffa7786..071d2124d 100644 --- a/crates/zip/README.md +++ b/crates/zip/README.md @@ -79,3 +79,13 @@ bytes and charges repeated reads against twice that number. Call `Indexing::fini returned handle after the ZIP index has been parsed to permit normal streaming payload reads. The caller must still enforce entry-count, declared-size and overlap limits: the wrapper only bounds metadata reads and does not validate the archive. + +## Lenient reading (feature `lenient`) + +`craft_zip::lenient` reads an archive from its raw bytes without the `zip` crate, for inspectors +(Craft Anything) that must list and describe damaged, prepended, split or oddly encoded archives +instead of refusing them. Only the central directory is trusted for the listing; entry reads are +bounded, checked against the declared size and CRC-32, and decode stored and DEFLATE entries, plus +bzip2 (`lenient-bzip2`), ZIP LZMA and XZ (`lenient-lzma`) and Zstandard (`lenient-zstd`). +`lenient::file::read_named_entry` reads one entry from a seekable file without loading the +archive. With `serde`, the listing types are `Serialize`. diff --git a/crates/zip/src/lenient.rs b/crates/zip/src/lenient.rs new file mode 100644 index 000000000..c9bed3764 --- /dev/null +++ b/crates/zip/src/lenient.rs @@ -0,0 +1,552 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +//! ZIP archives (also APK, JAR, IPA, MSIX, HAP…) read leniently from the raw bytes, for +//! inspectors that must list and describe damaged, prepended, split or oddly encoded archives +//! rather than refuse them (feature `lenient`). The rest of this crate is the strict side: it +//! refuses what fails [`crate::preflight`] and reads through the `zip` crate; nothing here uses +//! `zip`. +//! +//! Only the central directory is trusted for the listing. Entry data is located through the local +//! header, decompressed with a hard output cap, and checked against the recorded CRC-32. Stored, +//! DEFLATE, bzip2, ZIP LZMA, XZ and Zstandard entries are supported; other methods and encrypted entries are listed. +//! +//! The bzip2, LZMA/XZ and Zstandard methods sit behind the `lenient-bzip2`, `lenient-lzma` and +//! `lenient-zstd` features; without one, entries using it are listed and reading them is +//! [`ZipError::Unsupported`]. [`file::read_named_entry`] reads one entry from a seekable file +//! without loading the archive. With the `serde` feature the listing types are `Serialize`. +//! +//! From Craft Anything (`crates/formats/src/zip.rs` and `zip_file.rs`), with its hardening: +//! directory and extra-field bounds, ZIP64 end-record recovery after a prepended stub, refusal of +//! multi-volume archives, Info-ZIP Unicode Path names checked by CRC, decoded sizes checked +//! against the directory, and CRC-32 verification. + +mod bytes; +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma", feature = "lenient-zstd"))] +mod decode; +pub mod file; + +use crate::lenient::bytes::Reader; + +/// Biggest tail we search for the end-of-central-directory record (64 KiB comment + record). +pub const EOCD_SEARCH: usize = 22 + 0xffff; +const ZIP64_FIXED_SIZE: usize = 56; +/// Bound the search for a record shifted by a prepended stub, including extensible data. +const ZIP64_RECOVERY_SEARCH: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ZipError { + #[error("not a ZIP archive (no end-of-central-directory record)")] + NotZip, + #[error("ZIP is truncated: {0}")] + Truncated(&'static str), + #[error("ZIP is corrupt: {0}")] + Corrupt(String), + #[error("unsupported ZIP feature: {0}")] + Unsupported(String), + #[error("entry is larger than the {0}-byte limit")] + TooLarge(u64), + #[error("CRC-32 mismatch (expected {expected:08x}, found {found:08x})")] + CrcMismatch { expected: u32, found: u32 }, +} + +/// One central-directory entry. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct ZipEntry { + /// Entry name as stored (forward slashes). Bit 11 selects lossy UTF-8; otherwise + /// a valid Unicode Path extra field takes precedence over the UTF-8/CP437 fallback. + pub name: String, + pub method: u16, + pub flags: u16, + pub crc32: u32, + pub compressed_size: u64, + pub uncompressed_size: u64, + /// Offset of the local header, already adjusted for any prepended data. + pub local_header_offset: u64, + pub is_dir: bool, + pub encrypted: bool, + /// Unix mode from the external attributes (when stored by Unix or macOS). + #[cfg_attr(feature = "serde", serde(skip))] + pub unix_mode: Option, +} + +impl ZipEntry { + /// Why this entry is not a regular file or directory (symlink, device, FIFO, socket), if so. + pub fn special_kind(&self) -> Option<&'static str> { + match self.unix_mode? & 0o170000 { + 0o120000 => Some("symlink"), + 0o020000 => Some("character device"), + 0o060000 => Some("block device"), + 0o010000 => Some("FIFO"), + 0o140000 => Some("socket"), + _ => None, + } + } + + /// Human name of the compression method. + pub fn method_name(&self) -> &'static str { + match self.method { + 0 => "stored", + 8 => "deflate", + 9 => "deflate64", + 12 => "bzip2", + 14 => "lzma", + 93 => "zstd", + 95 => "xz", + 99 => "aes", + _ => "other", + } + } +} + +/// The parsed central directory. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub struct ZipArchive { + pub entries: Vec, + /// Entries the directory declares (may exceed `entries.len()` when truncated). + pub declared_entries: u64, + /// True when `max_entries` stopped the listing early. + pub truncated: bool, + pub zip64: bool, + /// Bytes before the archive proper (self-extracting stubs). + pub prefix_bytes: u64, + /// Inconsistent end-record metadata retained without hiding a readable directory. + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Vec::is_empty"))] + pub warnings: Vec, +} + +/// True when the bytes start like a ZIP (local header or empty archive). +pub fn looks_like_zip(data: &[u8]) -> bool { + data.starts_with(b"PK\x03\x04") || data.starts_with(b"PK\x05\x06") || data.starts_with(b"PK\x07\x08") +} + +/// Recover a shifted end record only when its fixed fields still describe the classic +/// directory. Longer records must frame their extension exactly; the adjacent fixed +/// record can still report an inconsistent size through the normal warning path. +fn recover_zip64_record(data: &[u8], loc: usize, disk_entries: u64, total: u64, cd_size: u64, cd_offset: u64) -> Option { + let r = Reader::new(data); + let last = loc.checked_sub(ZIP64_FIXED_SIZE)?; + let first = loc.saturating_sub(ZIP64_RECOVERY_SEARCH); + let candidates = data.get(first..last.checked_add(4)?)?; + candidates.windows(4).enumerate().rev().find_map(|(relative, signature)| { + if signature != b"PK\x06\x06" { + return None; + } + let rec = first.checked_add(relative)?; + let fixed = Reader::new(r.slice(rec, ZIP64_FIXED_SIZE)?); + let payload_size = fixed.u64_le(4)?; + if payload_size < 44 { + return None; + } + let declared_end = (rec as u64).checked_add(12)?.checked_add(payload_size)?; + if rec != last && declared_end != loc as u64 { + return None; + } + let record_disk_entries = fixed.u64_le(24)?; + let record_total = fixed.u64_le(32)?; + let record_cd_size = fixed.u64_le(40)?; + let record_cd_offset = fixed.u64_le(48)?; + if fixed.u32_le(16)? != 0 + || fixed.u32_le(20)? != 0 + || record_disk_entries != record_total + || (disk_entries != 0xffff && disk_entries != record_disk_entries) + || (total != 0xffff && total != record_total) + || (cd_size != 0xffff_ffff && cd_size != record_cd_size) + || (cd_offset != 0xffff_ffff && cd_offset != record_cd_offset) + || record_cd_offset.checked_add(record_cd_size)? > rec as u64 + { + return None; + } + if record_total > 0 { + let directory = rec.checked_sub(usize::try_from(record_cd_size).ok()?)?; + if record_cd_size < 46 || r.slice(directory, 4) != Some(b"PK\x01\x02") { + return None; + } + } + Some(rec) + }) +} + +/// Parse the central directory, keeping at most `max_entries`. +pub fn parse(data: &[u8], max_entries: usize) -> Result { + parse_directory(data, max_entries, None) +} + +/// Search the validated directory while retaining only the requested entry. +pub fn has_entry(data: &[u8], name: &str) -> bool { + parse_directory(data, 1, Some(name)).is_ok_and(|archive| !archive.entries.is_empty()) +} + +/// List directory entries, optionally retaining only a matching decoded name. +fn parse_directory(data: &[u8], max_entries: usize, name_filter: Option<&str>) -> Result { + let r = Reader::new(data); + let eocd = find_eocd(data).ok_or(ZipError::NotZip)?; + let mut disk = u32::from(r.u16_le(eocd + 4).ok_or(ZipError::Truncated("end record"))?); + let mut directory_disk = u32::from(r.u16_le(eocd + 6).ok_or(ZipError::Truncated("end record"))?); + let mut disk_entries = u64::from(r.u16_le(eocd + 8).ok_or(ZipError::Truncated("end record"))?); + if !matches!(disk, 0 | 0xffff) || !matches!(directory_disk, 0 | 0xffff) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let mut total = u64::from(r.u16_le(eocd + 10).ok_or(ZipError::Truncated("end record"))?); + let mut cd_size = u64::from(r.u32_le(eocd + 12).ok_or(ZipError::Truncated("end record"))?); + let mut cd_offset = u64::from(r.u32_le(eocd + 16).ok_or(ZipError::Truncated("end record"))?); + let mut zip64 = false; + let mut warnings = Vec::new(); + let mut eocd_record_start = eocd as u64; + if total == 0xffff || disk_entries == 0xffff || disk == 0xffff || directory_disk == 0xffff || cd_size == 0xffff_ffff || cd_offset == 0xffff_ffff { + // ZIP64 locator sits right before the classic record. + if let Some(loc) = eocd.checked_sub(20) + && r.slice(loc, 4) == Some(b"PK\x06\x07") + { + if r.u32_le(loc + 4) != Some(0) || r.u32_le(loc + 16) != Some(1) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let rec = r.u64_le(loc + 8).ok_or(ZipError::Truncated("zip64 locator"))?; + let rec = usize::try_from(rec).ok().filter(|&rec| r.slice(rec, 4) == Some(b"PK\x06\x06")); + let rec = rec.or_else(|| { + let recovered = recover_zip64_record(data, loc, disk_entries, total, cd_size, cd_offset); + if recovered.is_some() { + warnings.push("ZIP64 locator offset does not identify its end record; recovered the record before the locator".into()); + } + recovered + }); + if let Some(rec) = rec { + let payload_size = r.u64_le(rec + 4).ok_or(ZipError::Truncated("zip64 record size"))?; + if payload_size < 44 { + return Err(ZipError::Corrupt("ZIP64 end record is smaller than its fixed fields".into())); + } + let declared_end = (rec as u64) + .checked_add(12) + .and_then(|start| start.checked_add(payload_size)) + .ok_or_else(|| ZipError::Corrupt("ZIP64 end record extent overflows".into()))?; + // Fixed fields must be present before the locator. A bad declared extension + // length cannot make the reader borrow locator or classic end-record bytes. + if rec.checked_add(ZIP64_FIXED_SIZE).is_none_or(|end| end > loc) { + return Err(ZipError::Truncated("zip64 record fixed fields")); + } + if declared_end != loc as u64 { + warnings.push("ZIP64 end record's declared extent does not end at its locator; using bounded fixed fields".into()); + } + disk = r.u32_le(rec + 16).ok_or(ZipError::Truncated("zip64 record"))?; + directory_disk = r.u32_le(rec + 20).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_disk_entries = r.u64_le(rec + 24).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_total = r.u64_le(rec + 32).ok_or(ZipError::Truncated("zip64 record"))?; + if (disk_entries != 0xffff && disk_entries != zip64_disk_entries) || (total != 0xffff && total != zip64_total) { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + disk_entries = zip64_disk_entries; + total = zip64_total; + let zip64_cd_size = r.u64_le(rec + 40).ok_or(ZipError::Truncated("zip64 record"))?; + let zip64_cd_offset = r.u64_le(rec + 48).ok_or(ZipError::Truncated("zip64 record"))?; + if (cd_size != 0xffff_ffff && cd_size != zip64_cd_size) || (cd_offset != 0xffff_ffff && cd_offset != zip64_cd_offset) { + return Err(ZipError::Corrupt("ZIP64 and classic directory bounds disagree".into())); + } + cd_size = zip64_cd_size; + cd_offset = zip64_cd_offset; + zip64 = true; + eocd_record_start = rec as u64; + } else { + // A prepended stub can invalidate the locator's absolute offset. Keep the + // classic-directory fallback, including its existing prefix adjustment. + warnings.push("ZIP64 locator does not point to an end record; using the classic end record".into()); + } + } + } + if disk != 0 || directory_disk != 0 || disk_entries != total { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + // Self-extracting archives: the directory sits later than it claims by the stub length. + let expected_end = cd_offset.checked_add(cd_size).ok_or_else(|| ZipError::Corrupt("directory bounds overflow".into()))?; + let prefix = eocd_record_start.saturating_sub(expected_end); + let start = usize::try_from(cd_offset.saturating_add(prefix)).map_err(|_| ZipError::Corrupt("directory offset".into()))?; + let directory_end = (start as u64) + .checked_add(cd_size) + .filter(|&end| end <= eocd_record_start) + .and_then(|end| usize::try_from(end).ok()) + .ok_or_else(|| ZipError::Corrupt("directory extends into its end record".into()))?; + let mut entries = Vec::new(); + let mut pos = start; + let mut truncated = false; + let mut seen = 0u64; + while seen < total { + if entries.len() >= max_entries { + truncated = true; + break; + } + if pos.checked_add(46).is_none_or(|end| end > directory_end) { + return Err(ZipError::Truncated("directory entry")); + } + if r.slice(pos, 4) != Some(b"PK\x01\x02") { + if entries.is_empty() { + return Err(ZipError::Corrupt(format!("no central directory header at offset {pos}"))); + } + truncated = true; + break; + } + let flags = r.u16_le(pos + 8).ok_or(ZipError::Truncated("directory entry"))?; + let method = r.u16_le(pos + 10).ok_or(ZipError::Truncated("directory entry"))?; + let crc32 = r.u32_le(pos + 16).ok_or(ZipError::Truncated("directory entry"))?; + let mut csize = u64::from(r.u32_le(pos + 20).ok_or(ZipError::Truncated("directory entry"))?); + let mut usize_ = u64::from(r.u32_le(pos + 24).ok_or(ZipError::Truncated("directory entry"))?); + let name_len = usize::from(r.u16_le(pos + 28).ok_or(ZipError::Truncated("directory entry"))?); + let extra_len = usize::from(r.u16_le(pos + 30).ok_or(ZipError::Truncated("directory entry"))?); + let comment_len = usize::from(r.u16_le(pos + 32).ok_or(ZipError::Truncated("directory entry"))?); + let entry_end = pos + .checked_add(46 + name_len + extra_len + comment_len) + .filter(|&end| end <= directory_end) + .ok_or(ZipError::Truncated("directory entry fields"))?; + let host = r.slice(pos + 5, 1).and_then(|b| b.first().copied()).unwrap_or(0); + let attrs = r.u32_le(pos + 38).ok_or(ZipError::Truncated("entry attributes"))?; + let unix_mode = matches!(host, 3 | 19).then_some(attrs >> 16); + let mut local = u64::from(r.u32_le(pos + 42).ok_or(ZipError::Truncated("directory entry"))?); + let mut entry_disk = u32::from(r.u16_le(pos + 34).ok_or(ZipError::Truncated("directory entry"))?); + let name_bytes = r.slice(pos + 46, name_len).ok_or(ZipError::Truncated("entry name"))?; + let extra = r.slice(pos + 46 + name_len, extra_len).ok_or(ZipError::Truncated("entry extra fields"))?; + // Each extra field owns its payload: ZIP64 values cannot borrow a following field's bytes. + let needs_zip64 = usize_ == 0xffff_ffff || csize == 0xffff_ffff || local == 0xffff_ffff || entry_disk == 0xffff; + let mut found_zip64 = false; + let mut unicode_name = None; + let mut name_crc = None; + let mut x = 0usize; + let er = Reader::new(extra); + // Trailing bytes too short for a field header are padding, not truncation. + while x.saturating_add(4) <= extra.len() { + let id = er.u16_le(x).ok_or(ZipError::Truncated("extra field header"))?; + let len = usize::from(er.u16_le(x + 2).ok_or(ZipError::Truncated("extra field header"))?); + let body = x + 4; + let field = er.slice(body, len).ok_or(ZipError::Truncated("extra field payload"))?; + if id == 0x0001 && !found_zip64 { + let values = Reader::new(field); + let mut f = 0; + if usize_ == 0xffff_ffff { + usize_ = values.u64_le(f).ok_or(ZipError::Truncated("zip64 uncompressed size"))?; + f += 8; + } + if csize == 0xffff_ffff { + csize = values.u64_le(f).ok_or(ZipError::Truncated("zip64 compressed size"))?; + f += 8; + } + if local == 0xffff_ffff { + local = values.u64_le(f).ok_or(ZipError::Truncated("zip64 local header offset"))?; + f += 8; + } + if entry_disk == 0xffff { + entry_disk = values.u32_le(f).ok_or(ZipError::Truncated("zip64 entry disk"))?; + } + found_zip64 = true; + } else if id == 0x7075 && flags & (1 << 11) == 0 && unicode_name.is_none() { + let field = Reader::new(field); + if field.u8(0) == Some(1) && field.u32_le(1) == Some(*name_crc.get_or_insert_with(|| self::crc32(name_bytes))) { + unicode_name = field.data().get(5..).and_then(|bytes| std::str::from_utf8(bytes).ok()).filter(|s| !s.is_empty()); + } + } + x = body + len; + } + if needs_zip64 && !found_zip64 { + return Err(ZipError::Corrupt("missing ZIP64 extended information".into())); + } + if entry_disk != 0 { + return Err(ZipError::Unsupported("multi-volume ZIP archives".into())); + } + let name = unicode_name.map(str::to_owned).unwrap_or_else(|| decode_name(name_bytes, flags)).replace('\\', "/"); + if name_filter.is_none_or(|wanted| wanted == name) { + entries.push(ZipEntry { + is_dir: name.ends_with('/') + || unix_mode.is_some_and(|m| m & 0o170000 == 0o040000) + || (matches!(host, 0 | 10 | 14) && attrs & 0x10 != 0), + name, + method, + flags, + crc32, + compressed_size: csize, + uncompressed_size: usize_, + local_header_offset: local.saturating_add(prefix), + encrypted: flags & 1 != 0, + unix_mode, + }); + } + seen += 1; + pos = entry_end; + } + Ok(ZipArchive { entries, declared_entries: total, truncated, zip64, prefix_bytes: prefix, warnings }) +} + +fn decode_name(bytes: &[u8], flags: u16) -> String { + if flags & 0x0800 != 0 || bytes.is_ascii() { + return String::from_utf8_lossy(bytes).into_owned(); + } + if let Ok(name) = std::str::from_utf8(bytes) { + return name.to_owned(); + } + const CP437: [char; 128] = [ + 'Ç', 'ü', 'é', 'â', 'ä', 'à', 'å', 'ç', 'ê', 'ë', 'è', 'ï', 'î', 'ì', 'Ä', 'Å', 'É', 'æ', 'Æ', 'ô', 'ö', 'ò', 'û', 'ù', 'ÿ', 'Ö', 'Ü', '¢', + '£', '¥', '₧', 'ƒ', 'á', 'í', 'ó', 'ú', 'ñ', 'Ñ', 'ª', 'º', '¿', '⌐', '¬', '½', '¼', '¡', '«', '»', '░', '▒', '▓', '│', '┤', '╡', '╢', '╖', + '╕', '╣', '║', '╗', '╝', '╜', '╛', '┐', '└', '┴', '┬', '├', '─', '┼', '╞', '╟', '╚', '╔', '╩', '╦', '╠', '═', '╬', '╧', '╨', '╤', '╥', '╙', + '╘', '╒', '╓', '╫', '╪', '┘', '┌', '█', '▄', '▌', '▐', '▀', 'α', 'ß', 'Γ', 'π', 'Σ', 'σ', 'µ', 'τ', 'Φ', 'Θ', 'Ω', 'δ', '∞', 'φ', 'ε', '∩', + '≡', '±', '≥', '≤', '⌠', '⌡', '÷', '≈', '°', '∙', '·', '√', 'ⁿ', '²', '■', '\u{00a0}', + ]; + bytes + .iter() + .map(|&byte| if byte.is_ascii() { char::from(byte) } else { CP437.get(usize::from(byte - 0x80)).copied().unwrap_or('\u{fffd}') }) + .collect() +} + +/// Where the end-of-central-directory record starts, searching backwards through the last +/// [`EOCD_SEARCH`] bytes. +pub fn find_eocd(data: &[u8]) -> Option { + if data.len() < 22 { + return None; + } + let from = data.len().saturating_sub(EOCD_SEARCH); + let tail = data.get(from..)?; + // Search backwards so a stray signature in the comment can't shadow the real record. + let mut i = tail.len().checked_sub(22)?; + let mut found = None; + let mut last_signature = None; + loop { + if tail.get(i..i + 4) == Some(b"PK\x05\x06") { + last_signature.get_or_insert(from + i); + if let Some(comment_len) = tail.get(i + 20..i + 22).and_then(|bytes| <[u8; 2]>::try_from(bytes).ok()).map(u16::from_le_bytes) + && i + 22 + usize::from(comment_len) == tail.len() + { + found = Some(from + i); + } + } + if i == 0 { + return found.or(last_signature); + } + i -= 1; + } +} + +/// The raw (possibly compressed) bytes of an entry. +pub fn raw_entry_data<'a>(data: &'a [u8], e: &ZipEntry) -> Result<&'a [u8], ZipError> { + let r = Reader::new(data); + let off = usize::try_from(e.local_header_offset).map_err(|_| ZipError::Corrupt("local header offset".into()))?; + if r.slice(off, 4) != Some(b"PK\x03\x04") { + return Err(ZipError::Corrupt(format!("no local header for {}", e.name))); + } + let n = usize::from(r.u16_le(off + 26).ok_or(ZipError::Truncated("local header"))?); + let x = usize::from(r.u16_le(off + 28).ok_or(ZipError::Truncated("local header"))?); + let start = off.checked_add(30 + n + x).ok_or(ZipError::Truncated("local header"))?; + let len = usize::try_from(e.compressed_size).map_err(|_| ZipError::TooLarge(e.compressed_size))?; + r.slice(start, len).ok_or(ZipError::Truncated("entry data")) +} + +/// Read and decompress one entry, refusing to produce more than `max_bytes`. +pub fn read_entry(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let out = read_entry_unverified(data, e, max_bytes)?; + let found = crc32(&out); + if found != e.crc32 { + return Err(ZipError::CrcMismatch { expected: e.crc32, found }); + } + Ok(out) +} + +/// Like [`read_entry`] but without comparing the stored CRC-32, so a caller that accepts a mismatch can still get the bytes. +pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + if e.encrypted { + return Err(ZipError::Unsupported(format!("{} is encrypted", e.name))); + } + if e.uncompressed_size > max_bytes { + return Err(ZipError::TooLarge(max_bytes)); + } + let raw = raw_entry_data(data, e)?; + let cap = usize::try_from(max_bytes).unwrap_or(usize::MAX); + let out = match e.method { + 0 if raw.len() as u64 > max_bytes => return Err(ZipError::TooLarge(max_bytes)), + 0 => raw.to_vec(), + 8 => miniz_oxide::inflate::decompress_to_vec_with_limit(raw, cap).map_err(|err| match err.status { + miniz_oxide::inflate::TINFLStatus::HasMoreOutput => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("deflate stream for {} is damaged", e.name)), + })?, + 12 | 95 => decode_bzip2_or_xz(raw, e, max_bytes)?, + 14 => decode_lzma(raw, e, max_bytes)?, + 93 => decode_zstd(raw, e, max_bytes)?, + m => return Err(ZipError::Unsupported(format!("compression method {m} ({}) for {}", e.method_name(), e.name))), + }; + if out.len() as u64 != e.uncompressed_size { + return Err(ZipError::Corrupt(format!( + "decoded size for {} is {}, but its directory entry declares {}", + e.name, + out.len(), + e.uncompressed_size + ))); + } + Ok(out) +} + +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma"))] +fn decode_bzip2_or_xz(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let decoded = match e.method { + #[cfg(feature = "lenient-bzip2")] + 12 => decode::bzip2(raw, max_bytes), + #[cfg(feature = "lenient-lzma")] + 95 => decode::xz(raw, max_bytes), + _ => return Err(not_built(e)), + }; + decoded.map_err(|error| match error { + decode::DecompressError::TooLarge => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("{} stream for {} is damaged: {error}", e.method_name(), e.name)), + }) +} + +#[cfg(not(any(feature = "lenient-bzip2", feature = "lenient-lzma")))] +fn decode_bzip2_or_xz(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +#[cfg(feature = "lenient-lzma")] +fn decode_lzma(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + let header = Reader::new(raw); + let properties_size = header.u16_le(2).ok_or(ZipError::Truncated("LZMA properties header"))?; + if properties_size != 5 { + return Err(ZipError::Unsupported(format!("LZMA properties size {properties_size} for {}", e.name))); + } + let stream = raw.get(4..).ok_or(ZipError::Truncated("LZMA properties"))?; + decode::lzma_zip(stream, e.uncompressed_size, e.flags & 2 != 0, max_bytes).map_err(|error| match error { + decode::DecompressError::TooLarge => ZipError::TooLarge(max_bytes), + _ => ZipError::Corrupt(format!("LZMA stream for {} is damaged: {error}", e.name)), + }) +} + +#[cfg(not(feature = "lenient-lzma"))] +fn decode_lzma(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +#[cfg(feature = "lenient-zstd")] +fn decode_zstd(raw: &[u8], e: &ZipEntry, max_bytes: u64) -> Result, ZipError> { + decode::zstd(raw, max_bytes).map_err(|err| ZipError::Corrupt(format!("zstd stream for {} is damaged: {err}", e.name))) +} + +#[cfg(not(feature = "lenient-zstd"))] +fn decode_zstd(_: &[u8], e: &ZipEntry, _: u64) -> Result, ZipError> { + Err(not_built(e)) +} + +/// An entry whose method this build leaves out (see the crate features). +#[cfg_attr(all(feature = "lenient-bzip2", feature = "lenient-lzma", feature = "lenient-zstd"), allow(dead_code))] +fn not_built(e: &ZipEntry) -> ZipError { + ZipError::Unsupported(format!("compression method {} ({}) for {} is not built in", e.method, e.method_name(), e.name)) +} + +/// CRC-32 (IEEE, as used by ZIP and PNG). +pub fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + for &b in data { + crc ^= u32::from(b); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc +} + +#[cfg(test)] +mod testkit; + +#[cfg(test)] +mod tests; diff --git a/crates/zip/src/lenient/bytes.rs b/crates/zip/src/lenient/bytes.rs new file mode 100644 index 000000000..be7aa3ebe --- /dev/null +++ b/crates/zip/src/lenient/bytes.rs @@ -0,0 +1,41 @@ +//! Checked little-endian reads over untrusted bytes. Every read returns `None` when out of range. + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Reader<'a> { + data: &'a [u8], +} + +impl<'a> Reader<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Reader { data } + } + + pub(crate) fn data(&self) -> &'a [u8] { + self.data + } + + pub(crate) fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + let end = off.checked_add(len)?; + self.data.get(off..end) + } + + pub(crate) fn u8(&self, off: usize) -> Option { + self.data.get(off).copied() + } + + fn arr(&self, off: usize) -> Option<[u8; N]> { + self.slice(off, N)?.try_into().ok() + } + + pub(crate) fn u16_le(&self, off: usize) -> Option { + self.arr(off).map(u16::from_le_bytes) + } + + pub(crate) fn u32_le(&self, off: usize) -> Option { + self.arr(off).map(u32::from_le_bytes) + } + + pub(crate) fn u64_le(&self, off: usize) -> Option { + self.arr(off).map(u64::from_le_bytes) + } +} diff --git a/crates/zip/src/lenient/decode.rs b/crates/zip/src/lenient/decode.rs new file mode 100644 index 000000000..1189451b4 --- /dev/null +++ b/crates/zip/src/lenient/decode.rs @@ -0,0 +1,352 @@ +//! Bounded decoders for the bzip2, ZIP LZMA, XZ and Zstandard entry methods, from Craft +//! Anything's `decompress.rs`. Each caps its output at `max_out` bytes (`TooLarge` rather than +//! allocating without limit) and never panics on a corrupt or truncated stream. + +use std::io; +#[cfg(any(feature = "lenient-bzip2", feature = "lenient-lzma"))] +use std::io::Read; +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] +use std::io::Write; + +/// Why a decode stopped. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub(crate) enum DecompressError { + #[error("the compressed stream is damaged")] + Corrupt, + #[error("the compressed stream ends early")] + #[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] + Truncated, + #[error("the output would exceed the size limit")] + TooLarge, +} + +/// A `Write` sink that collects at most `cap` bytes, then refuses the rest. The `exceeded` flag lets +/// the caller tell a size-limit stop from a genuine decode error. +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] +struct CapWrite { + buf: Vec, + cap: usize, + exceeded: bool, +} + +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] +impl CapWrite { + fn new(cap: usize) -> Self { + CapWrite { buf: Vec::new(), cap, exceeded: false } + } +} + +#[cfg(any(feature = "lenient-lzma", feature = "lenient-zstd"))] +impl Write for CapWrite { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.buf.len().saturating_add(data.len()) > self.cap { + self.exceeded = true; + return Err(io::Error::other("output size limit reached")); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn cap_of(max_out: u64) -> usize { + usize::try_from(max_out).unwrap_or(usize::MAX) +} + +/// Read a `Read` source into a `Vec`, capped at `cap` bytes. +#[cfg(feature = "lenient-bzip2")] +fn read_capped(mut src: impl Read, cap: usize) -> Result, DecompressError> { + let mut out = Vec::new(); + let mut chunk = [0u8; 64 * 1024]; + loop { + match src.read(&mut chunk) { + Ok(0) => return Ok(out), + Ok(n) => { + let slice = chunk.get(..n).unwrap_or(&[]); + if out.len().saturating_add(slice.len()) > cap { + return Err(DecompressError::TooLarge); + } + out.extend_from_slice(slice); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return Err(DecompressError::Corrupt), + } + } +} + +#[cfg(feature = "lenient-lzma")] +/// Decompress every xz stream, including standard stream padding, within one output budget. +pub(crate) fn xz(input: &[u8], max_out: u64) -> Result, DecompressError> { + let ranges = xz_stream_ranges(input, max_out)?; + let mut out = CapWrite::new(cap_of(max_out)); + for range in ranges.into_iter().rev() { + let stream = input.get(range).ok_or(DecompressError::Truncated)?; + let mut cursor = io::Cursor::new(stream); + match lzma_rs::xz_decompress(&mut cursor, &mut out) { + Ok(()) => {} + Err(_) if out.exceeded => return Err(DecompressError::TooLarge), + Err(lzma_rs::error::Error::HeaderTooShort(_)) => return Err(DecompressError::Truncated), + Err(_) => return Err(DecompressError::Corrupt), + } + } + Ok(out.buf) +} + +#[cfg(feature = "lenient-lzma")] +/// Find stream boundaries backwards through their checked footer/index sizes, without +/// repeatedly trying the decoder at candidate magic bytes inside compressed data. +fn xz_stream_ranges(input: &[u8], max_out: u64) -> Result>, DecompressError> { + let mut ranges = Vec::new(); + let mut end = input.len(); + let mut declared = 0u64; + while end > 0 { + let padded_end = end; + while end > 0 && input.get(end - 1) == Some(&0) { + end -= 1; + } + if end == 0 || !(padded_end - end).is_multiple_of(4) { + return Err(DecompressError::Corrupt); + } + let footer_at = end.checked_sub(12).ok_or(DecompressError::Truncated)?; + let footer = input.get(footer_at..end).ok_or(DecompressError::Truncated)?; + let read = crate::lenient::bytes::Reader::new(footer); + let footer_fields = footer.get(4..10).ok_or(DecompressError::Truncated)?; + if footer.get(10..) != Some(b"YZ") || read.u32_le(0) != Some(crate::lenient::crc32(footer_fields)) { + return Err(DecompressError::Corrupt); + } + let backward = u64::from(read.u32_le(4).ok_or(DecompressError::Truncated)?); + let index_len = usize::try_from((backward + 1) * 4).map_err(|_| DecompressError::Corrupt)?; + let index_at = footer_at.checked_sub(index_len).ok_or(DecompressError::Truncated)?; + let index = input.get(index_at..footer_at).ok_or(DecompressError::Truncated)?; + let crc_at = index.len().checked_sub(4).ok_or(DecompressError::Truncated)?; + let index_read = crate::lenient::bytes::Reader::new(index); + let index_fields = index.get(..crc_at).ok_or(DecompressError::Truncated)?; + if index.first() != Some(&0) || index_read.u32_le(crc_at) != Some(crate::lenient::crc32(index_fields)) { + return Err(DecompressError::Corrupt); + } + let mut records = index.get(1..crc_at).ok_or(DecompressError::Corrupt)?; + let count = xz_vli(&mut records)?; + if count > (records.len() / 2) as u64 { + return Err(DecompressError::Corrupt); + } + let mut block_bytes = 0usize; + for _ in 0..count { + let unpadded = xz_vli(&mut records)?; + if unpadded == 0 { + return Err(DecompressError::Corrupt); + } + let padded = usize::try_from(unpadded.checked_add(3).ok_or(DecompressError::Corrupt)? & !3).map_err(|_| DecompressError::Corrupt)?; + block_bytes = block_bytes.checked_add(padded).filter(|bytes| *bytes <= index_at).ok_or(DecompressError::Corrupt)?; + declared = declared.checked_add(xz_vli(&mut records)?).filter(|size| *size <= max_out).ok_or(DecompressError::TooLarge)?; + } + if records.len() > 3 || records.iter().any(|&byte| byte != 0) { + return Err(DecompressError::Corrupt); + } + let start = index_at.checked_sub(block_bytes).and_then(|at| at.checked_sub(12)).ok_or(DecompressError::Corrupt)?; + let magic_end = start.checked_add(6).ok_or(DecompressError::Corrupt)?; + if input.get(start..magic_end) != Some(b"\xfd7zXZ\0") { + return Err(DecompressError::Corrupt); + } + ranges.push(start..end); + end = start; + } + if ranges.is_empty() { + return Err(DecompressError::Corrupt); + } + Ok(ranges) +} + +#[cfg(feature = "lenient-lzma")] +/// XZ variable-length integers have at most nine bytes and no redundant final zero. +fn xz_vli(input: &mut &[u8]) -> Result { + let mut value = 0u64; + for shift in (0..63).step_by(7) { + let (&byte, rest) = input.split_first().ok_or(DecompressError::Truncated)?; + *input = rest; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return if shift != 0 && byte == 0 { Err(DecompressError::Corrupt) } else { Ok(value) }; + } + } + Err(DecompressError::Corrupt) +} + +#[cfg(feature = "lenient-lzma")] +/// Decode the property bytes and raw stream of a ZIP method-14 member. +pub(crate) fn lzma_zip(input: &[u8], expected_size: u64, has_end_marker: bool, max_out: u64) -> Result, DecompressError> { + let cap = cap_of(max_out).min(isize::MAX as usize); + if expected_size > cap as u64 { + return Err(DecompressError::TooLarge); + } + let (properties, stream) = input.split_at_checked(5).ok_or(DecompressError::Truncated)?; + let [property, d0, d1, d2, d3] = <[u8; 5]>::try_from(properties).map_err(|_| DecompressError::Truncated)?; + let window_limit = u32::try_from(cap).unwrap_or(u32::MAX).max(4096); + let dictionary = u32::from_le_bytes([d0, d1, d2, d3]).max(4096).min(window_limit); + let [d0, d1, d2, d3] = dictionary.to_le_bytes(); + let properties = [property, d0, d1, d2, d3]; + let size = if has_end_marker { None } else { Some(expected_size) }; + let options = lzma_rs::decompress::Options { + unpacked_size: lzma_rs::decompress::UnpackedSize::UseProvided(size), + memlimit: Some(dictionary as usize), + ..Default::default() + }; + let mut out = CapWrite::new(cap); + let mut cursor = io::Cursor::new(properties).chain(io::Cursor::new(stream)); + match lzma_rs::lzma_decompress_with_options(&mut cursor, &mut out, &options) { + Ok(()) if cursor.get_ref().1.position() == stream.len() as u64 => Ok(out.buf), + Ok(()) => Err(DecompressError::Corrupt), + Err(_) if out.exceeded => Err(DecompressError::TooLarge), + Err(lzma_rs::error::Error::HeaderTooShort(_)) => Err(DecompressError::Truncated), + Err(_) => Err(DecompressError::Corrupt), + } +} + +#[cfg(feature = "lenient-bzip2")] +/// Decompress a bzip2 stream, producing at most `max_out` bytes. +pub(crate) fn bzip2(input: &[u8], max_out: u64) -> Result, DecompressError> { + read_capped(bzip2_rs::DecoderReader::new(input), cap_of(max_out)) +} + +#[cfg(feature = "lenient-zstd")] +/// Decompress a Zstandard stream, producing at most `max_out` bytes. +pub(crate) fn zstd(mut input: &[u8], max_out: u64) -> Result, DecompressError> { + if input.is_empty() { + return Err(DecompressError::Corrupt); + } + let mut out = CapWrite::new(cap_of(max_out)); + while !input.is_empty() { + if matches!(input.get(..4), Some([0x50..=0x5f, 0x2a, 0x4d, 0x18])) { + let header = input.get(4..8).and_then(|s| s.try_into().ok()).ok_or(DecompressError::Truncated)?; + let len = u32::from_le_bytes(header) as usize; + input = input.get(8..).and_then(|body| body.get(len..)).ok_or(DecompressError::Truncated)?; + continue; + } + let mut decoder = ruzstd::decoding::StreamingDecoder::new(&mut input).map_err(|_| DecompressError::Corrupt)?; + if io::copy(&mut decoder, &mut out).is_err() { + return Err(if out.exceeded { DecompressError::TooLarge } else { DecompressError::Corrupt }); + } + } + Ok(out.buf) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lenient::testkit as fx; + + #[cfg(feature = "lenient-lzma")] + #[test] + fn xz_round_trip() { + let c = fx::XZ_FIXTURE; + assert_eq!(xz(c, 1 << 20).unwrap(), fx::XZ_PLAIN); + // A tiny cap reports TooLarge, never a partial lie. + assert_eq!(xz(c, 4), Err(DecompressError::TooLarge)); + // Truncation never panics. + for len in 0..c.len() { + let _ = xz(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "lenient-lzma")] + #[test] + fn xz_concatenated_streams_and_padding_share_one_budget() { + let input = [fx::XZ_FIXTURE, &[0; 4], fx::XZ_FIXTURE, &[0; 8]].concat(); + let expected = [fx::XZ_PLAIN, fx::XZ_PLAIN].concat(); + assert_eq!(xz(&input, expected.len() as u64).unwrap(), expected); + assert_eq!(xz(&input, expected.len() as u64 - 1), Err(DecompressError::TooLarge)); + let mut empty = Vec::new(); + lzma_rs::xz_compress(&mut io::Cursor::new(b""), &mut empty).unwrap(); + let with_empty = [&empty[..], fx::XZ_FIXTURE, &empty[..]].concat(); + assert_eq!(xz(&with_empty, fx::XZ_PLAIN.len() as u64).unwrap(), fx::XZ_PLAIN); + assert!(xz(&empty, 0).unwrap().is_empty()); + } + + #[cfg(feature = "lenient-lzma")] + #[test] + fn xz_rejects_incomplete_members_and_nonstandard_padding() { + for padding in [1, 2, 3, 5] { + let input = [fx::XZ_FIXTURE, &vec![0; padding]].concat(); + assert!(xz(&input, 1000).is_err()); + } + assert!(xz(&[&[0; 4][..], fx::XZ_FIXTURE].concat(), 1000).is_err()); + assert!(xz(&[fx::XZ_FIXTURE, b"junk"].concat(), 1000).is_err()); + for missing in 1..fx::XZ_FIXTURE.len() { + let input = [fx::XZ_FIXTURE, &fx::XZ_FIXTURE[..fx::XZ_FIXTURE.len() - missing]].concat(); + assert!(xz(&input, 1000).is_err(), "missing {missing} bytes"); + } + for at in [fx::XZ_FIXTURE.len() - 13, fx::XZ_FIXTURE.len() - 12] { + let mut damaged = fx::XZ_FIXTURE.to_vec(); + damaged[at] ^= 1; + assert!(xz(&damaged, 1000).is_err()); + } + } + + #[cfg(feature = "lenient-lzma")] + #[test] + fn xz_index_sizes_and_integer_encodings_are_checked() { + assert_eq!(xz_vli(&mut &[0x80, 0][..]), Err(DecompressError::Corrupt)); + assert_eq!(xz_vli(&mut &[0xff; 10][..]), Err(DecompressError::Corrupt)); + assert_eq!(xz_vli(&mut &[0x81, 1][..]), Ok(129)); + let mut damaged = fx::XZ_FIXTURE.to_vec(); + let footer = damaged.len() - 12; + let backward = u32::from_le_bytes(damaged[footer + 4..footer + 8].try_into().unwrap()); + let index = footer - (backward as usize + 1) * 4; + damaged[index + 2] = 0; + let checksum = crate::lenient::crc32(&damaged[index..footer - 4]); + damaged[footer - 4..footer].copy_from_slice(&checksum.to_le_bytes()); + assert!(xz(&damaged, 1000).is_err()); + } + + #[cfg(feature = "lenient-bzip2")] + #[test] + fn bzip2_round_trip() { + let c = fx::BZIP2_FIXTURE; + assert_eq!(bzip2(c, 1 << 20).unwrap(), fx::BZIP2_PLAIN); + assert_eq!(bzip2(c, 2), Err(DecompressError::TooLarge)); + for len in 0..c.len() { + let _ = bzip2(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "lenient-zstd")] + #[test] + fn zstd_round_trip() { + let c = fx::ZSTD_FIXTURE; + assert_eq!(zstd(c, 1 << 20).unwrap(), fx::ZSTD_PLAIN); + assert_eq!(zstd(c, 2), Err(DecompressError::TooLarge)); + for len in 0..c.len() { + let _ = zstd(&c[..len], 1 << 20); + } + } + + #[cfg(feature = "lenient-zstd")] + #[test] + fn zstandard_members_can_contain_multiple_frames() { + let frame = fx::ZSTD_FIXTURE; + let plain = fx::ZSTD_PLAIN; + let skip = [0x50, 0x2a, 0x4d, 0x18, 3, 0, 0, 0, 1, 2, 3]; + let input = [frame, &skip, frame].concat(); + let expected = [plain, plain].concat(); + assert_eq!(zstd(&input, expected.len() as u64).unwrap(), expected); + assert_eq!(zstd(&input, plain.len() as u64), Err(DecompressError::TooLarge)); + assert!(zstd(&[frame, &skip[..9]].concat(), u64::MAX).is_err()); + assert!(zstd(&[frame, &frame[..5]].concat(), u64::MAX).is_err()); + } + + #[test] + fn garbage_never_panics() { + for seed in 0..64u64 { + let n = fx::noise(seed, 400); + #[cfg(feature = "lenient-lzma")] + let _ = xz(&n, 1 << 20); + #[cfg(feature = "lenient-bzip2")] + let _ = bzip2(&n, 1 << 20); + #[cfg(feature = "lenient-zstd")] + let _ = zstd(&n, 1 << 20); + let _ = &n; + } + } +} diff --git a/crates/zip/src/lenient/file.rs b/crates/zip/src/lenient/file.rs new file mode 100644 index 000000000..fc50dfd26 --- /dev/null +++ b/crates/zip/src/lenient/file.rs @@ -0,0 +1,262 @@ +//! One entry read straight from a ZIP file without loading the file: only its end records, its +//! central directory and that entry's bytes are read. For archives too big to hold in memory just +//! to look at one small entry (a JAR's manifest). + +use crate::lenient::bytes::Reader; +use crate::lenient::{self as zip, ZipError}; +use std::io::{Read, Seek, SeekFrom}; + +/// Most central directory read (some hundreds of thousands of entries); a bigger one is refused. +const MAX_DIRECTORY: u64 = 16 << 20; +/// The ZIP64 end record as `zip::parse` reads it, without any extensible data after it. +const ZIP64_RECORD: u64 = 56; + +/// The entry named `name` (not a folder), decompressed to at most `max_bytes` and checked against +/// its CRC-32. `None`: the archive has no such entry. +pub fn read_named_entry(file: &mut (impl Read + Seek), name: &str, max_bytes: u64) -> Result>, ZipError> { + let len = file.seek(SeekFrom::End(0)).map_err(io)?; + let tail_start = len.saturating_sub(zip::EOCD_SEARCH as u64); + let tail = read_at(file, tail_start, len - tail_start)?; + let eocd = zip::find_eocd(&tail).ok_or(ZipError::NotZip)?; + let mut end = tail.get(eocd..).ok_or(ZipError::Truncated("end record"))?.to_vec(); + let r = Reader::new(&end); + let field = |at| r.u16_le(at).ok_or(ZipError::Truncated("end record")); + let (size, classic_offset) = (r.u32_le(12).ok_or(ZipError::Truncated("end record"))?, r.u32_le(16).ok_or(ZipError::Truncated("end record"))?); + // Any of these saturated makes `zip::parse` read the ZIP64 records, as below. + let saturated = [field(4)?, field(6)?, field(8)?, field(10)?].contains(&0xffff) || size == 0xffff_ffff || classic_offset == 0xffff_ffff; + let end_at = tail_start + eocd as u64; + // A ZIP64 archive puts its own end record and a locator between the directory and the classic + // end record, whether or not the classic fields are saturated. + let locator_at = end_at.checked_sub(20); + let locator = match locator_at { + Some(at) => read_at(file, at, 20)?, + None => Vec::new(), + }; + // Its stated offset doesn't count a prepended stub; it then sits right before the locator. A + // locator signature with no record at either place is other data (an entry's comment). + let record_at = match (locator_at, locator.starts_with(b"PK\x06\x07")) { + (Some(at), true) => { + let declared = Reader::new(&locator).u64_le(8).ok_or(ZipError::Truncated("zip64 locator"))?; + [declared, at.saturating_sub(ZIP64_RECORD)].into_iter().find(|&at| read_at(file, at, 4).is_ok_and(|sig| sig == b"PK\x06\x06")) + } + _ => None, + }; + let zip64 = match record_at { + Some(record_at) => { + let record = read_at(file, record_at, ZIP64_RECORD)?; + let rr = Reader::new(&record); + let size = rr.u64_le(40).ok_or(ZipError::Truncated("zip64 record"))?; + let offset = rr.u64_le(48).ok_or(ZipError::Truncated("zip64 record"))?; + Some((record_at, size, offset, record)) + } + None => None, + }; + // Unsaturated classic fields must agree with a real ZIP64 record, as `zip::parse` requires; + // one that disagrees is other data shaped like it. + let zip64 = zip64.filter(|&(_, z_size, z_offset, _)| saturated || (z_size == u64::from(size) && z_offset == u64::from(classic_offset))); + let (directory_end, size, offset, zip64) = match zip64 { + Some((record_at, size, offset, record)) => (record_at, size, offset, saturated.then_some((record, locator))), + None => (end_at, u64::from(size), u64::from(classic_offset), None), + }; + if size > MAX_DIRECTORY { + return Err(ZipError::TooLarge(MAX_DIRECTORY)); + } + // The directory ends where the end records start; anything before its stated offset is a + // prepended stub, as `zip::parse` allows. + let start = directory_end.checked_sub(size).ok_or(ZipError::Corrupt("directory starts before the file".into()))?; + let prefix = start.checked_sub(offset).ok_or(ZipError::Corrupt("directory sits before its stated offset".into()))?; + // The directory and its end records alone, their offsets rewritten for that buffer. The + // ZIP64 records go along only when the classic fields need them, as `zip::parse` reads them. + let mut directory = read_at(file, start, size)?; + if let Some((mut record, mut locator)) = zip64 { + put(&mut record, 4, &(ZIP64_RECORD - 12).to_le_bytes())?; + put(&mut record, 48, &0u64.to_le_bytes())?; + put(&mut locator, 8, &size.to_le_bytes())?; + directory.extend_from_slice(&record); + directory.extend_from_slice(&locator); + } + // A saturated classic offset stays so: `zip::parse` then takes the rewritten ZIP64 one. + if classic_offset != 0xffff_ffff { + put(&mut end, 16, &[0; 4])?; + } + directory.extend_from_slice(&end); + // `MAX_DIRECTORY` already bounds how many entries there can be. + let archive = zip::parse(&directory, usize::MAX)?; + let Some(mut entry) = archive.entries.into_iter().find(|e| e.name == name && !e.is_dir) else { return Ok(None) }; + if entry.compressed_size > max_bytes.saturating_add(1 << 16) { + return Err(ZipError::TooLarge(max_bytes)); + } + let at = entry.local_header_offset.checked_add(prefix).ok_or(ZipError::Corrupt("local header offset".into()))?; + let header = read_at(file, at, 30)?; + let fields = Reader::new(&header); + let name_len = u64::from(fields.u16_le(26).ok_or(ZipError::Truncated("local header"))?); + let extra_len = u64::from(fields.u16_le(28).ok_or(ZipError::Truncated("local header"))?); + let local_len = (30 + name_len + extra_len).checked_add(entry.compressed_size).ok_or(ZipError::Corrupt("entry size".into()))?; + let local = read_at(file, at, local_len)?; + entry.local_header_offset = 0; + zip::read_entry(&local, &entry, max_bytes).map(Some) +} + +/// Overwrite `bytes` at `at` in a record read from the file. +fn put(record: &mut [u8], at: usize, bytes: &[u8]) -> Result<(), ZipError> { + record.get_mut(at..at + bytes.len()).ok_or(ZipError::Truncated("end record"))?.copy_from_slice(bytes); + Ok(()) +} + +/// Exactly `len` bytes from `at`; a file that ends sooner is truncated. +fn read_at(file: &mut (impl Read + Seek), at: u64, len: u64) -> Result, ZipError> { + file.seek(SeekFrom::Start(at)).map_err(io)?; + let mut out = Vec::new(); + file.take(len).read_to_end(&mut out).map_err(io)?; + if (out.len() as u64) < len { + return Err(ZipError::Truncated("file")); + } + Ok(out) +} + +fn io(e: std::io::Error) -> ZipError { + ZipError::Corrupt(format!("couldn't read it: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + const MANIFEST: &[u8] = b"Manifest-Version: 1.0\r\nMain-Class: app.Main\r\n\r\n"; + + fn read(bytes: &[u8], name: &str) -> Result>, ZipError> { + read_named_entry(&mut Cursor::new(bytes.to_vec()), name, 1 << 20) + } + + /// The same archive with a ZIP64 end record and locator before its classic end record, whose + /// fields at these offsets are saturated (entry counts and disk numbers 2 bytes, others 4). + fn as_zip64(archive: &[u8], saturate: &[usize]) -> Vec { + let end = archive.len() - 22; + let r = Reader::new(archive); + let (entries, size, offset) = + (u64::from(r.u16_le(end + 10).unwrap()), u64::from(r.u32_le(end + 12).unwrap()), u64::from(r.u32_le(end + 16).unwrap())); + let mut out = archive[..end].to_vec(); + let record_at = out.len() as u64; + out.extend_from_slice(b"PK\x06\x06"); + out.extend_from_slice(&44u64.to_le_bytes()); + out.extend_from_slice(&[45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + for v in [entries, entries, size, offset] { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(b"PK\x06\x07\0\0\0\0"); + out.extend_from_slice(&record_at.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + let mut classic = archive[end..].to_vec(); + for &at in saturate { + let width = if at < 12 { 2 } else { 4 }; + classic[at..at + width].fill(0xff); + } + out.extend_from_slice(&classic); + out + } + + #[test] + fn an_entry_is_read_from_the_directory_without_the_whole_file() { + let filler = vec![7u8; 200_000]; + let archive = crate::lenient::testkit::zip_stored(&[("app/Big.class", &filler), ("META-INF/MANIFEST.MF", MANIFEST), ("META-INF/", b"")]); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + assert_eq!(read(&archive, "META-INF/MISSING").unwrap(), None); + assert_eq!(read(&archive, "META-INF/").unwrap(), None, "a folder isn't an entry to read"); + + let mut stubbed = b"#!/bin/sh\nexec java -jar \"$0\" \"$@\"\n".to_vec(); + stubbed.extend_from_slice(&archive); + assert_eq!(read(&stubbed, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "a launcher stub before the archive"); + + assert_eq!(read(b"not a zip", "META-INF/MANIFEST.MF"), Err(ZipError::NotZip)); + assert!(read(&archive[..archive.len() - 30], "META-INF/MANIFEST.MF").is_err(), "a cut-off file"); + assert_eq!(read_named_entry(&mut Cursor::new(archive), "app/Big.class", 1000), Err(ZipError::TooLarge(1000))); + } + + #[test] + fn a_directory_bigger_than_the_searched_tail_is_read() { + let names: Vec = (0..3000).map(|i| format!("com/example/generated/Class{i:05}.class")).collect(); + let mut files: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b"x"[..])).collect(); + files.push(("META-INF/MANIFEST.MF", MANIFEST)); + let archive = crate::lenient::testkit::zip_stored(&files); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// A one-entry archive whose entry is deflated, as `jar` writes a manifest. + fn zip_deflated(name: &str, body: &[u8]) -> Vec { + let packed = miniz_oxide::deflate::compress_to_vec(body, 6); + let fixed = |out: &mut Vec| { + out.extend_from_slice(&[20, 0, 0, 0, 8, 0, 0, 0, 0, 0]); + out.extend_from_slice(&zip::crc32(body).to_le_bytes()); + out.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&[0, 0]); + }; + let mut out = b"PK\x03\x04".to_vec(); + fixed(&mut out); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&packed); + let directory_at = out.len() as u32; + out.extend_from_slice(b"PK\x01\x02\x14\0"); + fixed(&mut out); + out.extend_from_slice(&[0; 14]); + out.extend_from_slice(name.as_bytes()); + let directory_len = out.len() as u32 - directory_at; + out.extend_from_slice(b"PK\x05\x06\0\0\0\0\x01\0\x01\0"); + out.extend_from_slice(&directory_len.to_le_bytes()); + out.extend_from_slice(&directory_at.to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out + } + + #[test] + fn a_deflated_entry_and_a_comment_that_looks_like_an_end_record() { + let deflated = zip_deflated("META-INF/MANIFEST.MF", MANIFEST); + assert_eq!(read(&deflated, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + + let mut commented = crate::lenient::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let comment = b"built by PK\x05\x06 tools"; + let at = commented.len() - 2; + commented[at..].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + commented.extend_from_slice(comment); + assert_eq!(read(&commented, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// The last entry's comment ends like a ZIP64 record and locator, saying the directory is huge; + /// the classic fields disagree with it and win, as they do for `zip::parse`. + #[test] + fn a_comment_shaped_like_zip64_records_is_not_read_as_them() { + let mut archive = crate::lenient::testkit::zip_stored(&[("META-INF/MANIFEST.MF", MANIFEST)]); + let end = archive.len() - 22; + let last = archive.windows(4).rposition(|w| w == b"PK\x01\x02").unwrap(); + let mut fake = b"PK\x06\x06".to_vec(); + fake.extend_from_slice(&44u64.to_le_bytes()); + fake.extend_from_slice(&[0; 28]); + fake.extend_from_slice(&(MAX_DIRECTORY + 1).to_le_bytes()); + fake.extend_from_slice(&0u64.to_le_bytes()); + fake.extend_from_slice(b"PK\x06\x07\0\0\0\0"); + fake.extend_from_slice(&0u64.to_le_bytes()); + fake.extend_from_slice(&1u32.to_le_bytes()); + archive[last + 32..last + 34].copy_from_slice(&(fake.len() as u16).to_le_bytes()); + let size = Reader::new(&archive).u32_le(end + 12).unwrap() + fake.len() as u32; + archive[end + 12..end + 16].copy_from_slice(&size.to_le_bytes()); + archive.splice(end..end, fake); + assert!(zip::parse(&archive, 10).is_ok(), "precondition: a valid archive"); + assert_eq!(read(&archive, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST)); + } + + /// ZIP64 end records are read with any classic fields saturated or none (`zip -fz` saturates + /// only the offset), after a stub too. + #[test] + fn zip64_archives_are_read_in_parts() { + let archive = crate::lenient::testkit::zip_stored(&[("app/Main.class", b"x"), ("META-INF/MANIFEST.MF", MANIFEST)]); + for saturate in [&[][..], &[8, 10, 12, 16], &[16], &[12], &[10], &[4]] { + let zip64 = as_zip64(&archive, saturate); + assert_eq!(read(&zip64, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}"); + let mut stubbed = b"#!/bin/sh\n".to_vec(); + stubbed.extend_from_slice(&zip64); + assert_eq!(read(&stubbed, "META-INF/MANIFEST.MF").unwrap().as_deref(), Some(MANIFEST), "saturated {saturate:?}, after a stub"); + } + } +} diff --git a/crates/zip/src/lenient/testkit.rs b/crates/zip/src/lenient/testkit.rs new file mode 100644 index 000000000..ca87510e9 --- /dev/null +++ b/crates/zip/src/lenient/testkit.rs @@ -0,0 +1,80 @@ +//! Test fixtures from Craft Anything's testkit: known-good compressed streams built with the +//! reference encoders (`xz`, `bzip2`, `zstd`), a stored-only ZIP writer and a noise generator. + +#![allow(clippy::cast_possible_truncation, dead_code)] + +use crate::lenient::crc32; + +/// Plaintext of [`XZ_FIXTURE`] and [`LZMA_FIXTURE`]. +pub const XZ_PLAIN: &[u8] = b"Hello, xz compression world!\n"; +/// An xz stream of [`XZ_PLAIN`]. +pub const XZ_FIXTURE: &[u8] = &[ + 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, + 0x01, 0x00, 0x1c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x78, 0x7a, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x5f, 0x47, 0xab, 0x2a, 0x4c, 0xd2, 0xb2, 0x00, 0x01, 0x35, 0x1d, + 0x44, 0x1b, 0xb1, 0xe1, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x04, 0x59, 0x5a, +]; +/// Plaintext of [`BZIP2_FIXTURE`]. +pub const BZIP2_PLAIN: &[u8] = b"Hello, bzip2!\n"; +/// A bzip2 stream of [`BZIP2_PLAIN`]. +pub const BZIP2_FIXTURE: &[u8] = &[ + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0x04, 0xad, 0x5a, 0x6b, 0x00, 0x00, 0x02, 0xdd, 0x80, 0x00, 0x10, 0x60, 0x04, 0x10, + 0x00, 0x00, 0x40, 0x12, 0x24, 0xc0, 0x10, 0x20, 0x00, 0x22, 0x00, 0x03, 0x42, 0x01, 0xa0, 0x05, 0x67, 0xde, 0x89, 0xa1, 0xa0, 0x80, 0xf1, 0x77, + 0x24, 0x53, 0x85, 0x09, 0x00, 0x4a, 0xd5, 0xa6, 0xb0, +]; +/// Plaintext of [`ZSTD_FIXTURE`]. +pub const ZSTD_PLAIN: &[u8] = b"Hello, zstd!\n"; +/// A Zstandard frame of [`ZSTD_PLAIN`]. +pub const ZSTD_FIXTURE: &[u8] = &[ + 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x0d, 0x69, 0x00, 0x00, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x7a, 0x73, 0x74, 0x64, 0x21, 0x0a, 0x41, 0xb4, + 0x53, 0xb2, +]; + +/// A stored-only ZIP of `files`. +pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + let len = body.len() as u32; + out.extend_from_slice(b"PK\x03\x04\x14\0\0\0\0\0\0\0\0\0"); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02\x14\0\x14\0\0\0\0\0\0\0\0\0"); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops. +pub fn noise(seed: u64, len: usize) -> Vec { + let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() +} diff --git a/crates/zip/src/lenient/tests.rs b/crates/zip/src/lenient/tests.rs new file mode 100644 index 000000000..c2ca97394 --- /dev/null +++ b/crates/zip/src/lenient/tests.rs @@ -0,0 +1,393 @@ +//! The reader's tests, from Craft Anything's `zip.rs`. + +use super::*; + +/// Minimal stored-only ZIP writer for tests. +pub(crate) fn build(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + out.extend_from_slice(b"PK\x03\x04"); + out.extend_from_slice(&[20, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02"); + cd.extend_from_slice(&[20, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&(body.len() as u32).to_le_bytes()); + cd.extend_from_slice(&(body.len() as u32).to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +#[cfg(all(feature = "lenient-bzip2", feature = "lenient-lzma"))] +fn build_codec_entry(method: u16, compressed: &[u8], plain: &[u8]) -> Vec { + let mut file = build(&[("entry", compressed)]); + let central = 30 + "entry".len() + compressed.len(); + let version: u16 = if method == 12 { 46 } else { 63 }; + file[4..6].copy_from_slice(&version.to_le_bytes()); + file[8..10].copy_from_slice(&method.to_le_bytes()); + file[14..18].copy_from_slice(&crc32(plain).to_le_bytes()); + file[22..26].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + file[central + 6..central + 8].copy_from_slice(&version.to_le_bytes()); + file[central + 10..central + 12].copy_from_slice(&method.to_le_bytes()); + file[central + 16..central + 20].copy_from_slice(&crc32(plain).to_le_bytes()); + file[central + 24..central + 28].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + file +} + +#[cfg(all(feature = "lenient-bzip2", feature = "lenient-lzma"))] +#[test] +fn bzip2_and_xz_zip_entries_keep_size_crc_and_output_checks() { + use crate::lenient::testkit::{BZIP2_FIXTURE, BZIP2_PLAIN, XZ_FIXTURE, XZ_PLAIN}; + for (method, compressed, plain) in [(12, BZIP2_FIXTURE, BZIP2_PLAIN), (95, XZ_FIXTURE, XZ_PLAIN)] { + let file = build_codec_entry(method, compressed, plain); + let original = parse(&file, 1).unwrap().entries.remove(0); + assert_eq!(read_entry(&file, &original, plain.len() as u64).unwrap(), plain); + let mut entry = original.clone(); + entry.crc32 ^= 1; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::CrcMismatch { .. }))); + assert_eq!(read_entry_unverified(&file, &entry, 100).unwrap(), plain); + entry = original.clone(); + entry.uncompressed_size = 0; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::Corrupt(_)))); + assert!(matches!(read_entry(&file, &entry, plain.len() as u64 - 1), Err(ZipError::TooLarge(_)))); + let mut damaged = file.clone(); + damaged[30 + "entry".len()] ^= 0xff; + assert!(matches!(read_entry(&damaged, &original, 100), Err(ZipError::Corrupt(_)))); + entry = original; + entry.encrypted = true; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::Unsupported(_)))); + } +} + +#[cfg(feature = "lenient-lzma")] +fn zip_lzma_member(plain: &[u8], end_marker: bool) -> Vec { + let mut encoded = Vec::new(); + let size = if end_marker { None } else { Some(plain.len() as u64) }; + let options = lzma_rs::compress::Options { unpacked_size: lzma_rs::compress::UnpackedSize::WriteToHeader(size) }; + lzma_rs::lzma_compress_with_options(&mut std::io::Cursor::new(plain), &mut encoded, &options).unwrap(); + let mut compressed = vec![9, 4, 5, 0]; + compressed.extend_from_slice(&encoded[..5]); + compressed.extend_from_slice(&encoded[13..]); + let mut file = build(&[("entry", &compressed)]); + let central = 30 + "entry".len() + compressed.len(); + let flags: u16 = if end_marker { 2 } else { 0 }; + for (version_at, flags_at, method_at, crc_at, size_at) in + [(4, 6, 8, 14, 22), (central + 6, central + 8, central + 10, central + 16, central + 24)] + { + file[version_at..version_at + 2].copy_from_slice(&63u16.to_le_bytes()); + file[flags_at..flags_at + 2].copy_from_slice(&flags.to_le_bytes()); + file[method_at..method_at + 2].copy_from_slice(&14u16.to_le_bytes()); + file[crc_at..crc_at + 4].copy_from_slice(&crc32(plain).to_le_bytes()); + file[size_at..size_at + 4].copy_from_slice(&(plain.len() as u32).to_le_bytes()); + } + file +} + +#[cfg(feature = "lenient-lzma")] +#[test] +fn zip_lzma_reads_size_delimited_and_end_marked_streams() { + for end_marker in [false, true] { + for plain in [&b"ZIP LZMA contents"[..], &[0; 23][..]] { + let file = zip_lzma_member(plain, end_marker); + let original = parse(&file, 1).unwrap().entries.remove(0); + assert_eq!(read_entry(&file, &original, plain.len() as u64).unwrap(), plain); + assert!(matches!(read_entry(&file, &original, plain.len() as u64 - 1), Err(ZipError::TooLarge(_)))); + let mut entry = original.clone(); + entry.crc32 ^= 1; + assert!(matches!(read_entry(&file, &entry, 100), Err(ZipError::CrcMismatch { .. }))); + entry = original.clone(); + entry.uncompressed_size -= 1; + assert!(read_entry(&file, &entry, 100).is_err()); + entry = original.clone(); + entry.compressed_size -= 1; + assert!(read_entry(&file, &entry, 100).is_err()); + entry = original; + entry.compressed_size += 1; + assert!(read_entry(&file, &entry, 100).is_err()); + } + } +} + +#[cfg(feature = "lenient-lzma")] +#[test] +fn zip_lzma_rejects_bad_properties_without_unbounded_dictionaries() { + let mut file = zip_lzma_member(b"small", true); + let entry = parse(&file, 1).unwrap().entries.remove(0); + let body = 30 + "entry".len(); + file[body + 5..body + 9].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(read_entry(&file, &entry, 5).unwrap(), b"small"); + file[body + 4] = 255; + assert!(read_entry(&file, &entry, 5).is_err()); + file[body + 2..body + 4].copy_from_slice(&6u16.to_le_bytes()); + assert!(matches!(read_entry(&file, &entry, 5), Err(ZipError::Unsupported(_)))); +} + +fn build_named_entry(name: &[u8], flags: u16) -> Vec { + let mut zip = build(&[(&"x".repeat(name.len()), b"")]); + let central = 30 + name.len(); + zip[6..8].copy_from_slice(&flags.to_le_bytes()); + zip[30..central].copy_from_slice(name); + zip[central + 8..central + 10].copy_from_slice(&flags.to_le_bytes()); + zip[central + 46..central + 46 + name.len()].copy_from_slice(name); + zip +} + +#[test] +fn verified_unicode_path_overrides_a_legacy_zip_name() { + let raw = b"notes.txt"; + let unicode = "笔记.txt"; + for (flags, version, checksum, expected) in + [(0, 1, crc32(raw), unicode), (0, 1, crc32(b"old-name"), "notes.txt"), (0, 2, crc32(raw), "notes.txt"), (1 << 11, 1, crc32(raw), "notes.txt")] + { + let mut zip = build_named_entry(raw, flags); + let central = 30 + raw.len(); + let end = central + 46 + raw.len(); + let mut extra = 0x7075_u16.to_le_bytes().to_vec(); + extra.extend_from_slice(&((5 + unicode.len()) as u16).to_le_bytes()); + extra.push(version); + extra.extend_from_slice(&checksum.to_le_bytes()); + extra.extend_from_slice(unicode.as_bytes()); + zip[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + zip[end + 12..end + 16].copy_from_slice(&((end - central + extra.len()) as u32).to_le_bytes()); + zip.splice(end..end, extra); + assert_eq!(parse(&zip, 1).unwrap().entries[0].name, expected); + } +} + +#[test] +fn directory_markers_do_not_require_a_trailing_slash() { + let sample = build(&[("folder", b"")]); + let central = 30 + "folder".len(); + for (host, attrs, directory, special) in [ + (0, 0x10u32, true, None), + (10, 0x10, true, None), + (3, 0o040755 << 16, true, None), + (3, 0o120777 << 16, false, Some("symlink")), + (0, 0x20, false, None), + (14, 0x10, true, None), + (14, 0x20, false, None), + (19, 0o040755 << 16, true, None), + (19, 0o100644 << 16, false, None), + (19, 0o120777 << 16, false, Some("symlink")), + (42, 0x10, false, None), + ] { + let mut file = sample.clone(); + file[central + 5] = host; + file[central + 38..central + 42].copy_from_slice(&attrs.to_le_bytes()); + let archive = parse(&file, 1).unwrap(); + let entry = &archive.entries[0]; + assert_eq!((entry.is_dir, entry.special_kind()), (directory, special), "host {host}, attributes {attrs:#x}"); + } +} + +#[test] +fn a_cp437_name_without_the_utf8_flag_is_decoded() { + let zip = build_named_entry(b"caf\x82.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "café.txt"); +} + +#[test] +fn a_cp437_box_drawing_name_without_the_utf8_flag_is_decoded() { + let zip = build_named_entry(b"\xc9.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "╔.txt"); +} + +#[test] +fn a_utf8_name_with_the_utf8_flag_is_preserved() { + let zip = build_named_entry("한글/é.txt".as_bytes(), 0x0800); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "한글/é.txt"); +} + +#[test] +fn a_valid_utf8_name_without_the_utf8_flag_is_preserved() { + let zip = build_named_entry("é.txt".as_bytes(), 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "é.txt"); +} + +#[test] +fn an_ascii_name_is_preserved() { + let zip = build_named_entry(b"dir/file.txt", 0); + assert_eq!(parse(&zip, 10).unwrap().entries[0].name, "dir/file.txt"); +} + +#[test] +fn valid_entry_survives_eocd_signature_in_comment() { + let mut zip = build(&[("a.txt", b"hello")]); + let comment = b"PK\x05\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + let eocd = zip.len() - 22; + zip[eocd + 20..eocd + 22].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + zip.extend_from_slice(comment); + + assert_eq!(parse(&zip, 10).unwrap().entries.len(), 1); +} + +#[test] +fn valid_entry_survives_bytes_after_eocd_comment() { + let mut zip = build(&[("a.txt", b"hello")]); + zip.extend_from_slice(b"tail"); + + assert_eq!(parse(&zip, 10).unwrap().entries.len(), 1); +} + +#[test] +fn checks_the_zip64_record_extent_including_extension_bytes() { + let mut record = vec![0; 56]; + record[..4].copy_from_slice(b"PK\x06\x06"); + record[4..12].copy_from_slice(&46u64.to_le_bytes()); + record.extend([1, 2]); // extensible data + record.extend(b"PK\x06\x07"); + record.extend([0; 12]); + record.extend(1u32.to_le_bytes()); + record.extend(b"PK\x05\x06"); + record.extend([0; 4]); + record.extend([0xff; 4]); + record.extend([0; 10]); + assert!(parse(&record, 1).unwrap().zip64); + for len in [44u64, 47] { + record[4..12].copy_from_slice(&len.to_le_bytes()); + let archive = parse(&record, 1).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.warnings.len(), 1); + } + for len in [43u64, u64::MAX] { + record[4..12].copy_from_slice(&len.to_le_bytes()); + assert!(parse(&record, 1).is_err(), "{len}"); + } +} + +#[test] +fn lists_and_reads_stored_entries() { + let z = build(&[("a.txt", b"hello"), ("dir/b.js", b"console.log(1)")]); + let a = parse(&z, 100).unwrap(); + assert_eq!(a.entries.len(), 2); + assert_eq!(a.entries[1].name, "dir/b.js"); + assert_eq!(read_entry(&z, &a.entries[0], 100).unwrap(), b"hello"); + assert!(matches!(read_entry(&z, &a.entries[1], 3), Err(ZipError::TooLarge(3)))); +} + +#[cfg(feature = "lenient-zstd")] +#[test] +fn zstandard_members_are_read_with_their_limits_and_checks() { + // A raw Zstandard frame holding "hello zip" (single segment, one raw block). + let body = b"hello zip"; + let mut frame = vec![0x28, 0xb5, 0x2f, 0xfd, 0x20, body.len() as u8]; + let block_header = ((body.len() as u32) << 3) | 1; + frame.extend(&block_header.to_le_bytes()[..3]); + frame.extend(body); + let mut file = vec![0u8; 30]; + file[..4].copy_from_slice(b"PK\x03\x04"); + file.extend(&frame); + let mut e = ZipEntry { + name: "x".into(), + method: 93, + flags: 0, + crc32: crc32(body), + compressed_size: frame.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + assert_eq!(read_entry(&file, &e, 100).unwrap(), body); + assert!(matches!(read_entry(&file, &e, 3), Err(ZipError::TooLarge(3)))); + e.crc32 ^= 1; + assert!(matches!(read_entry(&file, &e, 100), Err(ZipError::CrcMismatch { .. }))); +} + +#[test] +fn deflate_round_trip_and_crc_check() { + let body = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".repeat(20); + let comp = miniz_oxide::deflate::compress_to_vec(&body, 6); + let e = ZipEntry { + name: "x".into(), + method: 8, + flags: 0, + crc32: crc32(&body), + compressed_size: comp.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + let mut z = b"PK\x03\x04".to_vec(); + z.extend_from_slice(&[0; 26]); + z.extend_from_slice(&comp); + assert_eq!(read_entry(&z, &e, 1 << 20).unwrap(), body); + // A lying uncompressed size can't make us exceed the cap. + let liar = ZipEntry { uncompressed_size: 10, ..e.clone() }; + assert!(matches!(read_entry(&z, &liar, 10), Err(ZipError::TooLarge(10)))); + let bad = ZipEntry { crc32: 1, ..e }; + assert!(matches!(read_entry(&z, &bad, 1 << 20), Err(ZipError::CrcMismatch { .. }))); +} + +#[test] +fn hostile_inputs_do_not_panic() { + let z = build(&[("a", b"1"), ("b", b"22")]); + for cut in 0..z.len() { + if let Ok(a) = parse(&z[..cut], 10) { + for e in &a.entries { + let _ = read_entry(&z[..cut], e, 100); + } + } + } + for i in 0..z.len() { + let mut m = z.clone(); + m[i] ^= 0xff; + if let Ok(a) = parse(&m, 10) { + for e in &a.entries { + let _ = read_entry(&m, e, 100); + } + } + } + assert_eq!(parse(b"", 1), Err(ZipError::NotZip)); +} + +#[test] +fn crc_known_value() { + assert_eq!(crc32(b"123456789"), 0xcbf4_3926); +} + +#[cfg(feature = "serde")] +#[test] +fn listings_serialize_without_empty_warnings_or_unix_mode() { + let data = crate::lenient::testkit::zip_stored(&[("a.txt", b"hi")]); + let archive = parse(&data, 10).unwrap(); + let value = serde_json::to_value(&archive).unwrap(); + assert!(value.get("warnings").is_none()); + assert_eq!(value["entries"][0]["name"], "a.txt"); + assert!(value["entries"][0].get("unix_mode").is_none()); +} + +#[cfg(not(feature = "lenient-zstd"))] +#[test] +fn a_method_left_out_of_the_build_is_unsupported() { + let mut data = crate::lenient::testkit::zip_stored(&[("a", b"abc")]); + let archive = parse(&data, 10).unwrap(); + let mut entry = archive.entries[0].clone(); + entry.method = 93; + data[8..10].copy_from_slice(&93u16.to_le_bytes()); + assert!(matches!(read_entry_unverified(&data, &entry, 100), Err(ZipError::Unsupported(m)) if m.contains("not built in"))); +} diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index 0a348dc1f..ff8ce17a6 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -13,6 +13,9 @@ //! timestamp on every entry so equal input gives equal bytes. //! - [`preflight`] is a stricter check of the whole end-record and central-directory structure //! on the raw bytes, for a caller that wants nothing ambiguous to reach a ZIP reader at all. +//! - `lenient` (feature `lenient`) is the other way round: a reader of the raw bytes without +//! `zip`, for inspectors that list and describe damaged or unusual archives instead of refusing +//! them, with bounded, CRC-checked entry reads. //! //! Every limit is the caller's number. Entry-name rules (which names are safe, which spellings //! are one part) stay with the format that knows them. @@ -23,6 +26,8 @@ struct ReadmeExamples; mod archive; mod guard; +#[cfg(feature = "lenient")] +pub mod lenient; mod metadata_reader; pub mod preflight; mod seekable; diff --git a/crates/zip/tests/lenient_common/mod.rs b/crates/zip/tests/lenient_common/mod.rs new file mode 100644 index 000000000..05d8c5fff --- /dev/null +++ b/crates/zip/tests/lenient_common/mod.rs @@ -0,0 +1,41 @@ +//! The stored-only ZIP writer the integration tests share, as in Craft Anything's testkit. + +#![allow(dead_code, clippy::cast_possible_truncation)] + +use craft_zip::lenient::crc32; + +/// A stored-only ZIP of `files`. +pub fn zip_stored(files: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut cd = Vec::new(); + for (name, body) in files { + let off = out.len() as u32; + let crc = crc32(body); + let len = body.len() as u32; + out.extend_from_slice(b"PK\x03\x04\x14\0\0\0\0\0\0\0\0\0"); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(body); + cd.extend_from_slice(b"PK\x01\x02\x14\0\x14\0\0\0\0\0\0\0\0\0"); + cd.extend_from_slice(&crc.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&len.to_le_bytes()); + cd.extend_from_slice(&(name.len() as u16).to_le_bytes()); + cd.extend_from_slice(&[0; 12]); + cd.extend_from_slice(&off.to_le_bytes()); + cd.extend_from_slice(name.as_bytes()); + } + let cd_off = out.len() as u32; + out.extend_from_slice(&cd); + out.extend_from_slice(b"PK\x05\x06\0\0\0\0"); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(cd.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_off.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} diff --git a/crates/zip/tests/lenient_zip64_end_record_compatibility.rs b/crates/zip/tests/lenient_zip64_end_record_compatibility.rs new file mode 100644 index 000000000..ed599fbda --- /dev/null +++ b/crates/zip/tests/lenient_zip64_end_record_compatibility.rs @@ -0,0 +1,171 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip64_end_record_compatibility.rs. +use craft_zip::lenient::{self as zip, ZipArchive, ZipError}; +#[path = "lenient_common/mod.rs"] +mod common; +use common::zip_stored; + +fn warning_contains(archive: &ZipArchive, text: &str) -> bool { + archive.warnings.iter().any(|warning| warning.contains(text)) +} + +fn zip64_with_extension(extension: &[u8]) -> (Vec, usize) { + let mut data = zip_stored(&[("a.txt", b"retained payload")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let offset = u32::from_le_bytes(data[end + 16..end + 20].try_into().unwrap()); + let mut record = b"PK\x06\x06".to_vec(); + record.extend((44 + extension.len() as u64).to_le_bytes()); + record.extend([45, 0, 45, 0]); + record.extend([0; 8]); + record.extend(1u64.to_le_bytes()); + record.extend(1u64.to_le_bytes()); + record.extend(u64::from(size).to_le_bytes()); + record.extend(u64::from(offset).to_le_bytes()); + record.extend(extension); + record.extend(b"PK\x06\x07"); + record.extend(0u32.to_le_bytes()); + record.extend((end as u64).to_le_bytes()); + record.extend(1u32.to_le_bytes()); + data[end + 8..end + 12].fill(0xff); + data.splice(end..end, record); + (data, end) +} + +#[test] +fn readable_fixed_fields_survive_inconsistent_declared_extents() { + let (original, record) = zip64_with_extension(&[1, 2, 3, 4]); + for declared in [44u64, 47, 49, 4096] { + let mut data = original.clone(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.declared_entries, 1); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.prefix_bytes, 0); + assert!(warning_contains(&archive, "extent"), "declared payload {declared}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn correctly_sized_extensions_need_no_warning() { + for extension in [&[][..], &[1, 2, 3, 4][..]] { + let (data, _) = zip64_with_extension(extension); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert!(archive.warnings.is_empty()); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn too_small_and_overflowing_record_extents_remain_untrusted() { + let (original, record) = zip64_with_extension(&[]); + for declared in [0u64, 43, u64::MAX - 12, u64::MAX] { + let mut data = original.clone(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Corrupt(_))), "declared payload {declared}"); + } +} + +#[test] +fn fixed_fields_cannot_borrow_bytes_from_the_locator() { + let (mut data, record) = zip64_with_extension(&[]); + data.drain(record + 48..record + 56); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn prepended_stubs_keep_real_zip64_directories_and_entry_offsets() { + for extension in [&[][..], &[1, 2, 3, 4][..], &[0; 1024][..]] { + let (original, _) = zip64_with_extension(extension); + for prefix in [&b"MZ"[..], &b"MZ synthetic self-extracting prefix"[..]] { + for saturated_bounds in [false, true] { + let mut data = prefix.to_vec(); + data.extend(&original); + if saturated_bounds { + let end = data.len() - 22; + data[end + 12..end + 20].fill(0xff); + } + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64); + assert_eq!(archive.declared_entries, 1); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert_eq!(archive.entries[0].local_header_offset, prefix.len() as u64); + assert!(warning_contains(&archive, "recovered")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } + } + } +} + +#[test] +fn shifted_fixed_records_keep_extent_warnings() { + let (original, record) = zip64_with_extension(&[]); + let prefix = b"MZ synthetic self-extracting prefix"; + for declared in [45u64, 4096] { + let mut data = prefix.to_vec(); + data.extend(&original); + let record = record + prefix.len(); + data[record + 4..record + 12].copy_from_slice(&declared.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert!(warning_contains(&archive, "recovered")); + assert!(warning_contains(&archive, "extent")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } +} + +#[test] +fn shifted_record_recovery_has_a_bounded_lookbehind() { + let (original, _) = zip64_with_extension(&[0; 65_536]); + let mut data = b"MZ synthetic self-extracting prefix".to_vec(); + data.extend(&original); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn an_unusable_locator_keeps_the_classic_directory_and_prefix_adjustment() { + // 65,535 is a valid classic count as well as a ZIP64 sentinel. A final entry comment + // can look like a locator without actually pointing to a ZIP64 end record. + let files = vec![("a.txt", &b"retained payload"[..]); 65_535]; + let original = zip_stored(&files); + let end = original.len() - 22; + let size = u32::from_le_bytes(original[end + 12..end + 16].try_into().unwrap()); + let last_entry = end - 46 - "a.txt".len(); + let mut locator = b"PK\x06\x07".to_vec(); + locator.extend(0u32.to_le_bytes()); + locator.extend(1u64.to_le_bytes()); + locator.extend(1u32.to_le_bytes()); + // A nearby record-shaped comment is still not a ZIP64 directory when its bounds + // disagree with the explicit classic fields. + let mut fake_record = [0; 56]; + fake_record[..4].copy_from_slice(b"PK\x06\x06"); + fake_record[4..12].copy_from_slice(&44u64.to_le_bytes()); + for record_shaped_comment in [&[][..], &fake_record[..]] { + let mut comment = record_shaped_comment.to_vec(); + comment.extend(&locator); + let mut archive_data = original.clone(); + archive_data[last_entry + 32..last_entry + 34].copy_from_slice(&(comment.len() as u16).to_le_bytes()); + archive_data[end + 12..end + 16].copy_from_slice(&(size + comment.len() as u32).to_le_bytes()); + archive_data.splice(end..end, comment); + for prefix in [&[][..], &b"MZ synthetic self-extracting prefix"[..]] { + for offset in [1u64, u64::MAX] { + let mut data = prefix.to_vec(); + data.extend(&archive_data); + let locator = data.len() - 22 - 20; + data[locator + 8..locator + 16].copy_from_slice(&offset.to_le_bytes()); + let archive = zip::parse(&data, 2).unwrap(); + assert!(!archive.zip64); + assert_eq!(archive.declared_entries, 65_535); + assert_eq!(archive.entries.len(), 2); + assert!(archive.truncated); + assert_eq!(archive.prefix_bytes, prefix.len() as u64); + assert!(warning_contains(&archive, "classic end record")); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 100).unwrap(), b"retained payload"); + } + } + } +} diff --git a/crates/zip/tests/lenient_zip64_extra_boundaries.rs b/crates/zip/tests/lenient_zip64_extra_boundaries.rs new file mode 100644 index 000000000..fae4fd5f9 --- /dev/null +++ b/crates/zip/tests/lenient_zip64_extra_boundaries.rs @@ -0,0 +1,66 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip64_extra_boundaries.rs. + +#[path = "lenient_common/mod.rs"] +mod common; +use craft_zip::lenient as zip; + +fn archive(extra: &[u8], sentinel: usize) -> Vec { + let mut data = common::zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + data[central + sentinel..central + sentinel + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + data[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + data.splice(end..end, extra.iter().copied()); + let end = data.len() - 22; + data[end + 12..end + 16].copy_from_slice(&((47 + extra.len()) as u32).to_le_bytes()); + data +} + +#[test] +fn a_short_zip64_field_cannot_borrow_values_from_its_neighbor() { + for field in [20, 24, 42] { + let mut extra = vec![1, 0, 0, 0, 2, 0, 8, 0]; + extra.extend_from_slice(&1u64.to_le_bytes()); + assert!(zip::parse(&archive(&extra, field), 10).is_err(), "field {field}"); + } +} + +#[test] +fn zip64_sentinels_require_complete_values() { + for field in [20, 24, 42] { + assert!(zip::parse(&archive(&[], field), 10).is_err()); + assert!(zip::parse(&archive(&[1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0], field), 10).is_err()); + } +} + +#[test] +fn a_complete_zip64_field_preserves_the_value() { + for field in [20, 24, 42] { + let value = if field == 42 { 0u64 } else { 1u64 }; + let mut extra = vec![1, 0, 8, 0]; + extra.extend_from_slice(&value.to_le_bytes()); + let data = archive(&extra, field); + let parsed = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &parsed.entries[0], 1).unwrap(), b"x"); + } +} + +#[test] +fn a_legitimate_64_bit_value_equal_to_the_32_bit_sentinel_is_preserved() { + let mut extra = vec![1, 0, 8, 0]; + extra.extend_from_slice(&u64::from(u32::MAX).to_le_bytes()); + let archive = zip::parse(&archive(&extra, 24), 10).unwrap(); + assert_eq!(archive.entries[0].uncompressed_size, u64::from(u32::MAX)); +} + +#[test] +fn trailing_bytes_too_short_for_a_field_header_are_padding() { + for stray in 1..=3usize { + let mut extra = vec![0xaa, 0xbb, 0, 0]; + extra.extend(std::iter::repeat_n(0u8, stray)); + let data = archive(&extra, 36); + let parsed = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &parsed.entries[0], 1).unwrap(), b"x", "stray {stray}"); + } +} diff --git a/crates/zip/tests/lenient_zip_decoded_size.rs b/crates/zip/tests/lenient_zip_decoded_size.rs new file mode 100644 index 000000000..9db3ec80e --- /dev/null +++ b/crates/zip/tests/lenient_zip_decoded_size.rs @@ -0,0 +1,45 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip_decoded_size.rs. + +#[path = "lenient_common/mod.rs"] +mod common; +use craft_zip::lenient::{self as zip, ZipEntry, ZipError}; + +fn deflated(body: &[u8]) -> (Vec, ZipEntry) { + let compressed = miniz_oxide::deflate::compress_to_vec(body, 6); + let mut data = vec![0; 30]; + data[..4].copy_from_slice(b"PK\x03\x04"); + data.extend_from_slice(&compressed); + let entry = ZipEntry { + name: "payload".into(), + method: 8, + flags: 0, + crc32: zip::crc32(body), + compressed_size: compressed.len() as u64, + uncompressed_size: body.len() as u64, + local_header_offset: 0, + is_dir: false, + encrypted: false, + unix_mode: None, + }; + (data, entry) +} + +#[test] +fn decoded_length_must_match_the_directory_even_when_the_crc_matches() { + let (data, good) = deflated(b"complete body"); + for declared in [1, good.uncompressed_size + 1] { + let entry = ZipEntry { uncompressed_size: declared, ..good.clone() }; + assert!(matches!(zip::read_entry(&data, &entry, 100), Err(ZipError::Corrupt(_)))); + assert!(matches!(zip::read_entry_unverified(&data, &entry, 100), Err(ZipError::Corrupt(_)))); + } + assert_eq!(zip::read_entry(&data, &good, 100).unwrap(), b"complete body"); +} + +#[test] +fn stored_length_must_also_match_the_directory() { + let data = common::zip_stored(&[("payload", b"hello")]); + let mut entry = zip::parse(&data, 10).unwrap().entries.remove(0); + entry.uncompressed_size = 4; + assert!(matches!(zip::read_entry(&data, &entry, 100), Err(ZipError::Corrupt(_)))); +} diff --git a/crates/zip/tests/lenient_zip_directory_bounds.rs b/crates/zip/tests/lenient_zip_directory_bounds.rs new file mode 100644 index 000000000..8e912fc3b --- /dev/null +++ b/crates/zip/tests/lenient_zip_directory_bounds.rs @@ -0,0 +1,76 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip_directory_bounds.rs. +use craft_zip::lenient::{self as zip, ZipError}; +#[path = "lenient_common/mod.rs"] +mod common; +use common::zip_stored; + +#[test] +fn a_short_declared_entry_is_rejected_before_its_signature_is_checked() { + for remaining in 0..46 { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + data[end + 8..end + 12].copy_from_slice(&[2, 0, 2, 0]); + data[end + 12..end + 16].copy_from_slice(&(size + remaining).to_le_bytes()); + data.splice(end..end, vec![0; remaining as usize]); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Truncated("directory entry"))), "remaining {remaining} bytes"); + } +} + +#[test] +fn directory_names_and_comments_cannot_borrow_end_record_bytes() { + for field in [28, 32] { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let length = if field == 28 { 2u16 } else { 1 }; + data[central + field..central + field + 2].copy_from_slice(&length.to_le_bytes()); + assert!(zip::parse(&data, 10).is_err(), "field {field}"); + } +} + +#[test] +fn directory_extra_fields_cannot_borrow_end_record_bytes() { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + data[central + 30..central + 32].copy_from_slice(&2u16.to_le_bytes()); + assert_eq!(zip::parse(&data, 10), Err(ZipError::Truncated("directory entry fields"))); +} + +#[test] +fn a_directory_cannot_extend_into_its_end_record() { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + data[end + 12..end + 16].copy_from_slice(&(size + 1).to_le_bytes()); + assert!(zip::parse(&data, 10).is_err()); +} + +#[test] +fn self_extracting_archives_keep_both_offset_conventions() { + for adjusted in [false, true] { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + let stub = b"executable stub"; + if adjusted { + data[central + 42..central + 46].copy_from_slice(&(stub.len() as u32).to_le_bytes()); + data[end + 16..end + 20].copy_from_slice(&((central + stub.len()) as u32).to_le_bytes()); + } + let mut file = stub.to_vec(); + file.extend(data); + let archive = zip::parse(&file, 10).unwrap(); + assert_eq!(zip::read_entry(&file, &archive.entries[0], 10).unwrap(), b"x"); + } +} + +#[test] +fn a_directory_signature_after_the_entries_is_not_an_extra_file() { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let signature = b"PK\x05\x05\x01\x00x"; + data[end + 12..end + 16].copy_from_slice(&(size + signature.len() as u32).to_le_bytes()); + data.splice(end..end, signature.iter().copied()); + assert_eq!(zip::parse(&data, 10).unwrap().entries.len(), 1); +} diff --git a/crates/zip/tests/lenient_zip_entry_detection.rs b/crates/zip/tests/lenient_zip_entry_detection.rs new file mode 100644 index 000000000..8708b6400 --- /dev/null +++ b/crates/zip/tests/lenient_zip_entry_detection.rs @@ -0,0 +1,45 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip_entry_detection.rs. +use craft_zip::lenient::has_entry as zip_has_entry; +#[path = "lenient_common/mod.rs"] +mod common; +use common::zip_stored; + +/// A directory-shaped payload is ordinary file content, not an archive member. +#[test] +fn embedded_archive_does_not_supply_outer_entries() { + let inner = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + let outer = zip_stored(&[("backup.zip", &inner)]); + assert!(!zip_has_entry(&outer, "AndroidManifest.xml")); + assert!(zip_has_entry(&outer, "backup.zip")); +} + +/// A ZIP comment can contain another directory without changing the archive's entries. +#[test] +fn comment_does_not_supply_entries() { + let mut outer = zip_stored(&[("readme.txt", b"notes")]); + let inner = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + let len = outer.len(); + outer[len - 2..].copy_from_slice(&(inner.len() as u16).to_le_bytes()); + outer.extend(inner); + assert!(!zip_has_entry(&outer, "AndroidManifest.xml")); + assert!(zip_has_entry(&outer, "readme.txt")); +} + +/// Actual manifests are found regardless of member order, including prefixed archives. +#[test] +fn actual_entries_are_found() { + let data = zip_stored(&[("classes.dex", b"dex"), ("AndroidManifest.xml", b"manifest")]); + assert!(zip_has_entry(&data, "AndroidManifest.xml")); + let mut prefixed = b"stub".to_vec(); + prefixed.extend(data); + assert!(zip_has_entry(&prefixed, "AndroidManifest.xml")); + assert!(!zip_has_entry(&prefixed, "missing")); +} + +/// A stray central-directory signature without an archive is not an entry. +#[test] +fn incomplete_archive_is_not_a_manifest() { + let data = zip_stored(&[("AndroidManifest.xml", b"manifest")]); + assert!(!zip_has_entry(&data[..data.len() - 22], "AndroidManifest.xml")); +} diff --git a/crates/zip/tests/lenient_zip_split_volumes.rs b/crates/zip/tests/lenient_zip_split_volumes.rs new file mode 100644 index 000000000..9675177ce --- /dev/null +++ b/crates/zip/tests/lenient_zip_split_volumes.rs @@ -0,0 +1,167 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip_split_volumes.rs. +use craft_zip::lenient::{self as zip, ZipError}; +#[path = "lenient_common/mod.rs"] +mod common; +use common::zip_stored; + +fn assert_split(data: &[u8]) { + assert!(matches!(zip::parse(data, 10), Err(ZipError::Unsupported(_)))); +} + +#[test] +fn classic_end_records_must_describe_one_complete_volume() { + for field in [4, 6, 8] { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let value = if field == 8 { 0u16 } else { 1 }; + data[end + field..end + field + 2].copy_from_slice(&value.to_le_bytes()); + assert_split(&data); + } +} + +#[test] +fn central_entries_cannot_point_to_another_volume() { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + data[central + 34..central + 36].copy_from_slice(&1u16.to_le_bytes()); + assert_split(&data); +} + +fn zip64_entry_disk(disk: u32, with_sizes_and_offset: bool) -> Vec { + let mut data = zip_stored(&[("a", b"x")]); + let central = data.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + let end = data.len() - 22; + data[central + 34..central + 36].copy_from_slice(&u16::MAX.to_le_bytes()); + let mut extra = vec![1, 0, 4, 0]; + if with_sizes_and_offset { + data[central + 20..central + 28].fill(0xff); + data[central + 42..central + 46].fill(0xff); + extra[2..4].copy_from_slice(&28u16.to_le_bytes()); + // Uncompressed size, compressed size, and local header offset precede the disk. + for value in [1u64, 1, 0] { + extra.extend(value.to_le_bytes()); + } + } + extra.extend(disk.to_le_bytes()); + data[central + 30..central + 32].copy_from_slice(&(extra.len() as u16).to_le_bytes()); + let directory_size = (end - central + extra.len()) as u32; + data.splice(end..end, extra); + let end = data.len() - 22; + data[end + 12..end + 16].copy_from_slice(&directory_size.to_le_bytes()); + data +} + +#[test] +fn a_zip64_disk_sentinel_is_resolved_before_rejecting_the_entry() { + let data = zip64_entry_disk(0, false); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + assert_split(&zip64_entry_disk(1, false)); +} + +#[test] +fn a_zip64_disk_sentinel_follows_sizes_and_the_local_offset() { + let data = zip64_entry_disk(0, true); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.entries[0].uncompressed_size, 1); + assert_eq!(archive.entries[0].compressed_size, 1); + assert_eq!(archive.entries[0].local_header_offset, 0); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + assert_split(&zip64_entry_disk(1, true)); +} + +fn zip64_archive(record_disk: u32, central_disk: u32, locator_disk: u32, disks: u32, entries_on_disk: u64) -> Vec { + let mut data = zip_stored(&[("a", b"x")]); + let end = data.len() - 22; + let central = u32::from_le_bytes(data[end + 16..end + 20].try_into().unwrap()); + let size = u32::from_le_bytes(data[end + 12..end + 16].try_into().unwrap()); + let mut record = b"PK\x06\x06".to_vec(); + record.extend(44u64.to_le_bytes()); + record.extend([45, 0, 45, 0]); + record.extend(record_disk.to_le_bytes()); + record.extend(central_disk.to_le_bytes()); + record.extend(entries_on_disk.to_le_bytes()); + record.extend(1u64.to_le_bytes()); + record.extend(u64::from(size).to_le_bytes()); + record.extend(u64::from(central).to_le_bytes()); + record.extend(b"PK\x06\x07"); + record.extend(locator_disk.to_le_bytes()); + record.extend((end as u64).to_le_bytes()); + record.extend(disks.to_le_bytes()); + data[end + 8..end + 12].fill(0xff); + data.splice(end..end, record); + data +} + +#[test] +fn zip64_end_records_and_locators_must_also_describe_one_volume() { + for fields in [(1, 0, 0, 1, 1), (0, 1, 0, 1, 1), (0, 0, 1, 1, 1), (0, 0, 0, 2, 1), (0, 0, 0, 1, 0)] { + assert_split(&zip64_archive(fields.0, fields.1, fields.2, fields.3, fields.4)); + } + assert_eq!(zip::parse(&zip64_archive(0, 0, 0, 1, 1), 10).unwrap().entries.len(), 1); +} + +#[test] +fn zip64_does_not_erase_explicit_classic_volume_markers() { + for field in [4, 6] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&1u16.to_le_bytes()); + assert_split(&data); + data[end + field..end + field + 2].fill(0xff); + assert_eq!(zip::parse(&data, 10).unwrap().entries.len(), 1); + } +} + +#[test] +fn zip64_does_not_erase_explicit_classic_entry_counts() { + for field in [8, 10] { + for count in [0u16, 2] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&count.to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Unsupported(_))), "field {field}, count {count}"); + } + } +} + +#[test] +fn mixed_classic_and_zip64_entry_counts_can_describe_one_volume() { + for field in [8, 10] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + field..end + field + 2].copy_from_slice(&1u16.to_le_bytes()); + let archive = zip::parse(&data, 10).unwrap(); + assert_eq!(archive.entries.len(), 1, "field {field}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x"); + } +} + +#[test] +fn zip64_cannot_replace_explicit_classic_directory_bounds() { + for field in [12, 16] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + let value = u32::from_le_bytes(data[end + field..end + field + 4].try_into().unwrap()); + data[end + field..end + field + 4].copy_from_slice(&(value + 1).to_le_bytes()); + assert!(matches!(zip::parse(&data, 10), Err(ZipError::Corrupt(_))), "field {field}"); + } +} + +#[test] +fn each_zip64_field_can_resolve_its_own_classic_sentinel() { + for (field, width) in [(4, 2), (6, 2), (8, 2), (10, 2), (12, 4), (16, 4)] { + let mut data = zip64_archive(0, 0, 0, 1, 1); + let end = data.len() - 22; + data[end + 8..end + 10].copy_from_slice(&1u16.to_le_bytes()); + data[end + 10..end + 12].copy_from_slice(&1u16.to_le_bytes()); + data[end + field..end + field + width].fill(0xff); + let archive = zip::parse(&data, 10).unwrap(); + assert!(archive.zip64, "field {field}"); + assert_eq!(archive.declared_entries, 1, "field {field}"); + assert_eq!(archive.entries.len(), 1, "field {field}"); + assert_eq!(archive.prefix_bytes, 0, "field {field}"); + assert_eq!(zip::read_entry(&data, &archive.entries[0], 10).unwrap(), b"x", "field {field}"); + } +} diff --git a/crates/zip/tests/lenient_zip_stored_cap.rs b/crates/zip/tests/lenient_zip_stored_cap.rs new file mode 100644 index 000000000..a5424605f --- /dev/null +++ b/crates/zip/tests/lenient_zip_stored_cap.rs @@ -0,0 +1,23 @@ +#![cfg(feature = "lenient")] +// From Craft Anything crates/formats/tests/zip_stored_cap.rs. + +#[path = "lenient_common/mod.rs"] +mod common; +use craft_zip::lenient::{self as zip, ZipError}; + +#[test] +fn a_stored_entry_cannot_understate_its_size_to_bypass_the_output_limit() { + let data = common::zip_stored(&[("payload", &[0x5a; 64])]); + let mut entry = zip::parse(&data, 10).unwrap().entries.remove(0); + entry.uncompressed_size = 1; + assert!(matches!(zip::read_entry(&data, &entry, 8), Err(ZipError::TooLarge(8)))); + assert!(matches!(zip::read_entry_unverified(&data, &entry, 8), Err(ZipError::TooLarge(8)))); +} + +#[test] +fn stored_entries_at_the_limit_and_empty_entries_still_read() { + let data = common::zip_stored(&[("payload", b"hello"), ("empty", b"")]); + let entries = zip::parse(&data, 10).unwrap().entries; + assert_eq!(zip::read_entry(&data, &entries[0], 5).unwrap(), b"hello"); + assert_eq!(zip::read_entry(&data, &entries[1], 0).unwrap(), b""); +}