diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index 94d8add68b..92ec32a617 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -874,6 +874,10 @@ pub use buildcmds::live_paint_gaps; /// Control bar enable it when it changes one). pub use brushsym::reset_transformation_changes; +/// The byte order of UTF-16 text without a byte-order mark, by Import Data Sets' rule (the Graph +/// Data window reads files by it too). +pub use variablecmds::utf16_order_without_mark; + /// Command ids kept only so older scripts keep working: `(alias, the command it duplicates)`. /// They run as before but the command palette leaves them out (it lists the command once). pub const ALIASES: &[(&str, &str)] = &[("object.convertDocumentColorMode", "file.documentColorMode")]; diff --git a/crates/engine/src/cmd/variablecmds.rs b/crates/engine/src/cmd/variablecmds.rs index 9acd31e664..5f8d6605b3 100644 --- a/crates/engine/src/cmd/variablecmds.rs +++ b/crates/engine/src/cmd/variablecmds.rs @@ -26,6 +26,7 @@ use super::{CommandSpec, EngineError, Result, Session, bad, bool_or, checked_ids mod csv; mod library; +pub use csv::utf16_order_without_mark; use csv::{MAX_GRAPH_CSV, dataset_export, dataset_import}; pub(crate) use library::{give_svg_library, svg_library}; use library::{load_library, save_library}; diff --git a/crates/engine/src/cmd/variablecmds/csv.rs b/crates/engine/src/cmd/variablecmds/csv.rs index 38dd42d922..be9db3b841 100644 --- a/crates/engine/src/cmd/variablecmds/csv.rs +++ b/crates/engine/src/cmd/variablecmds/csv.rs @@ -59,25 +59,35 @@ fn utf16(bytes: &[u8], le: bool) -> Option { char::decode_utf16(units).collect::>().ok().filter(|_| odd.is_empty()) } -/// `bytes`, which hold NUL bytes and no byte-order mark, as UTF-16, or `None`. At least half of -/// the two-byte units must hold a NUL byte, as characters from U+0001 to U+00FF do; a file with -/// fewer, such as UTF-8 or Windows-1252 text with stray NUL bytes, is refused. The byte order is -/// the one that reads a line break (`\n` or `\r`). The other order reads one only from U+0A00 or -/// U+0D00, so a file that reads one in both orders is refused. A file that reads none in either -/// order has no data rows, and is read little-endian so that the import reports that. The text -/// must be valid UTF-16 in that order and hold no NUL character. -fn utf16_without_mark(bytes: &[u8]) -> Option { - let (units, odd) = bytes.as_chunks::<2>(); - if !odd.is_empty() || units.iter().filter(|c| c.contains(&0)).count() * 2 < units.len() { +/// The byte order of `bytes`, which hold NUL bytes and no byte-order mark, read as UTF-16: +/// `Some(true)` for little-endian, or `None` when they don't read as UTF-16. At least half of the +/// two-byte units must hold a NUL byte, as characters from U+0001 to U+00FF do; a file with fewer, +/// such as UTF-8 or Windows-1252 text with stray NUL bytes, is refused. The byte order is the one +/// that reads a line break (`\n` or `\r`). The other order reads one only from U+0A00 or U+0D00, +/// so a file that reads one in both orders is refused. A file that reads none in either order (one +/// line) is big-endian when more of its units start with a NUL byte than end with one, as +/// characters from U+0001 to U+00FF do in that order, and little-endian otherwise. An odd last +/// byte is left out of the count. +pub fn utf16_order_without_mark(bytes: &[u8]) -> Option { + let units = bytes.as_chunks::<2>().0; + if units.iter().filter(|c| c.contains(&0)).count() * 2 < units.len() { return None; } let is_line_break = |u: u16| matches!(u, 0x0A | 0x0D); let lines_le = units.iter().any(|&c| is_line_break(u16::from_le_bytes(c))); let lines_be = units.iter().any(|&c| is_line_break(u16::from_be_bytes(c))); - if lines_le && lines_be { - return None; + match (lines_le, lines_be) { + (true, true) => None, + (false, false) => Some(units.iter().filter(|c| c[0] == 0).count() <= units.iter().filter(|c| c[1] == 0).count()), + (_, lines_be) => Some(!lines_be), } - utf16(bytes, !lines_be).filter(|text| !text.contains('\0')) +} + +/// `bytes`, which hold NUL bytes and no byte-order mark, as UTF-16 in the byte order +/// [`utf16_order_without_mark`] gives, or `None`. The file must have an even length, and the text +/// must be valid UTF-16 in that order and hold no NUL character. +fn utf16_without_mark(bytes: &[u8]) -> Option { + utf16(bytes, utf16_order_without_mark(bytes)?).filter(|text| !text.contains('\0')) } /// A row of a datasets CSV: the file line it starts on, its cells, and for each cell whether it was quoted. diff --git a/crates/engine/src/tests/variables.rs b/crates/engine/src/tests/variables.rs index eea8ae4986..6256088842 100644 --- a/crates/engine/src/tests/variables.rs +++ b/crates/engine/src/tests/variables.rs @@ -2201,6 +2201,16 @@ fn datasets_csv_reads_utf16_files_without_a_byte_order_mark() { let e = import(&mut s, &bytes).unwrap_err().to_string(); assert!(e.contains("CSV has a header but no data rows"), "{e}"); } + // With no line break, the side of each unit that holds the NUL byte gives the byte order, and a + // tie is little-endian. + let order = crate::cmd::utf16_order_without_mark; + assert_eq!(order(&utf16("City", false)), Some(false)); + assert_eq!(order(&utf16("City", true)), Some(true)); + assert_eq!(order(b"\0aa\0"), Some(true)); + assert_eq!(order(&[utf16("City\n", false), vec![0x0A]].concat()), Some(false), "an odd last byte is left out"); + // Read big-endian, this one-line file ends in an unpaired surrogate. + let e = import(&mut s, b"\0a\0b\xd8\0").unwrap_err().to_string(); + assert!(e.contains("CSV holds NUL bytes"), "{e}"); import(&mut s, &utf16("City\tName\n一二三丁目\tKai\n", true)).unwrap(); assert_eq!(value(&s, "Data Set 1"), text_value("一二三丁目")); // Most units hold their NUL byte where a big-endian file would (U+4E00); the line breaks diff --git a/crates/ui-egui/src/dialogs/graph_data.rs b/crates/ui-egui/src/dialogs/graph_data.rs index 16d22d4430..2dbcfad8fe 100644 --- a/crates/ui-egui/src/dialogs/graph_data.rs +++ b/crates/ui-egui/src/dialogs/graph_data.rs @@ -147,7 +147,10 @@ pub(crate) fn grid_from_text(text: &str) -> Grid { } /// A text file's bytes as text: UTF-16 with its byte-order mark (spreadsheets' "Unicode Text"), -/// else UTF-8 without one. +/// else UTF-16 without one when the bytes hold NUL bytes and Import Data Sets' rule gives them a +/// byte order ([`vectorcraft_engine::cmd::utf16_order_without_mark`]), else UTF-8 with or without +/// one. UTF-16 is read as it is after a byte-order mark: an odd last byte is left out, and an +/// unpaired surrogate reads as U+FFFD. pub(crate) fn decode(bytes: &[u8]) -> String { let utf16 = |rest: &[u8], le: bool| { let units: Vec = rest.as_chunks::<2>().0.iter().map(|p| if le { u16::from_le_bytes(*p) } else { u16::from_be_bytes(*p) }).collect(); @@ -157,7 +160,10 @@ pub(crate) fn decode(bytes: &[u8]) -> String { [0xFF, 0xFE, rest @ ..] => utf16(rest, true), [0xFE, 0xFF, rest @ ..] => utf16(rest, false), [0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest).into_owned(), - _ => String::from_utf8_lossy(bytes).into_owned(), + _ => match bytes.contains(&0).then(|| vectorcraft_engine::cmd::utf16_order_without_mark(bytes)).flatten() { + Some(le) => utf16(bytes, le), + None => String::from_utf8_lossy(bytes).into_owned(), + }, } } @@ -517,6 +523,28 @@ mod tests { assert_eq!(grid_from_text(&wide)[0].len(), MAX_COLS); } + #[test] + fn utf16_without_a_byte_order_mark_reads_by_its_line_breaks() { + let text = "\tSales é\r\nQ1\t1\n"; + let le: Vec = text.encode_utf16().flat_map(u16::to_le_bytes).collect(); + let be: Vec = text.encode_utf16().flat_map(u16::to_be_bytes).collect(); + assert_eq!(decode(&le), text, "little-endian"); + assert_eq!(decode(&be), text, "big-endian"); + assert_eq!(grid_from_text(&decode(&le)), g(&[&["", "Sales é"], &["Q1", "1"]])); + // One row has no line break: the side of each unit that holds the NUL byte gives the order. + for (bytes, order) in [(b"\0Q\x001\0\t\x002".to_vec(), "big-endian"), (b"Q\x001\0\t\x002\0".to_vec(), "little-endian")] { + assert_eq!(decode(&bytes), "Q1\t2", "one line, {order}"); + } + // A file cut short by one byte, or with an unpaired surrogate, reads as it would after a + // byte-order mark. + assert_eq!(decode(&[le.as_slice(), &[0x0A]].concat()), text, "odd length"); + assert_eq!(decode(&[le.as_slice(), &[0x00, 0xD8]].concat()), format!("{text}\u{FFFD}"), "unpaired surrogate"); + // A line break in both byte orders leaves the order unknown, and UTF-8 with a stray NUL + // byte isn't UTF-16: both read as UTF-8, as before. + assert_eq!(decode(&[0x0A, 0, 0, 0x0A]), "\n\0\0\n"); + assert_eq!(decode(b"a,b\n1,2\0"), "a,b\n1,2\0"); + } + #[test] fn a_sheet_sent_wider_and_taller_than_a_graph_is_cut_to_one() { // One long row and one long column: their product would be ten billion cells. diff --git a/docs/mcp.md b/docs/mcp.md index f756bc8c97..a68645348e 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -3175,8 +3175,9 @@ already walks Image Trace. Short agent entry points: byte order that reads a line break (`\n` or `\r`). It is refused when fewer of its units hold a NUL byte (UTF-8 or Windows-1252 text with stray NUL bytes, or UTF-16 text made mostly of other characters, which needs a byte-order mark), when both byte orders read a line break (the text holds U+0A00 or U+0D00), when it has an odd length, and - when it isn't valid UTF-16 in that order or holds a NUL character. A file with no line break has no data rows and is - refused for that. `csv` and `data` are read as the text they hold. + when it isn't valid UTF-16 in that order or holds a NUL character. A file with no line break is read big-endian when + more of its two-byte units start with a NUL byte than end with one, and little-endian otherwise; it has no data rows, + so the import is refused. `csv` and `data` are read as the text they hold. - Variable libraries: `variable.loadLibrary {path|xml|dataBase64}` (the Variables panel menu's Load Variable Library…) replaces the document's variables and data sets with a variable library's in one undo step. A load, like a `dataset.import`, adds no undo step when it leaves the variables and data sets as they were. The library diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 296bd84b2b..8fe37830a2 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -793,6 +793,7 @@ Moved from the ROADMAP's "Shipped so far" on 2026-10-10. Update it when a featur - Export for Screens numbers distinct row files whose names collide, compared without case, as `-2`, `-3`… instead of leaving them out; identical rows still write one file. It puts row folders that differ only in case into one folder (`document.exportForScreens`). - Variables: Import Data Sets… (`dataset.import`) reads tab-separated files as well as comma-separated ones. Every header cell is a variable, and the rows replace the document's data sets as Data Set 1, Data Set 2 and so on. An import error about a row's values gives the file line the row starts on. - Variables: Import Data Sets… (`dataset.import`) reads a UTF-16 file without a byte-order mark, instead of refusing it, when at least half of its two-byte units hold a NUL byte; it uses the byte order that reads its line breaks. + - Graphs: the Graph Data window's Import Data reads a UTF-16 text file without a byte-order mark by the same rule as Import Data Sets…, instead of filling the cells with NUL characters. It reads a file cut short by one byte, or with an unpaired surrogate, as it reads one after a byte-order mark. - Variables: Import Data Sets… (`dataset.import`) defines the variables a file's header names that the document lacks, unbound. `#Shown` defines a visibility variable, `@Photo` defines an image variable, and a header without a prefix defines a text variable. A header that is a variable's exact name keeps naming it, and any other `%` header (graph data files) is refused. - Variables: Load Variable Library… in the Variables panel menu (`variable.loadLibrary`) replaces the document's variables and data sets with a variable library's, read from its XML file, in one undo step: every data set is deleted, and every variable is deleted with its bindings and values, except a text variable the library also declares, which stays bound so the library's data sets change its objects (#4136); then the library's other text variables are defined, unbound, and its data sets take the text of each value's `p` elements, one paragraph each. Variables of other kinds are left out with a warning that names their kinds. - Variables: Save Variable Library… in the Variables panel menu (`variable.saveLibrary`) writes the document's text variables and data sets as a variable library's XML file, which Load Variable Library reads back with the same text variables, data set names and text values. Variables of other kinds are left out with a warning that names their kinds.