From c1adf9b239900a03e5d04a434691590324c5eb04 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 12 Nov 2026 01:17:21 +0000 Subject: [PATCH] xlsx, engine: CSV decoding, parsing and quoting, and SYLK, through craft-csv Text decoding (byte order marks, UTF-8, Windows-1256/1252), delimiter sniffing, the record parser and field quoting moved to storytold/craft-libs as craft-csv (#889), with the SYLK reader as craft_csv::sylk. GridCraft keeps typed values, date-order inference, the used-range limit and the sheet writer in csv.rs, and in io/sylk.rs the workbook a SYLK file becomes: formulas parsed (an unparsable one is skipped and counted, as before), error values, number-format styles and column widths. --- Cargo.lock | 10 + Cargo.toml | 3 + crates/engine/Cargo.toml | 1 + crates/engine/src/io/sylk.rs | 338 ++++------------------------- crates/engine/src/io/tests.rs | 2 +- crates/xlsx/Cargo.toml | 1 + crates/xlsx/src/csv.rs | 261 +--------------------- crates/xlsx/src/tests/csv_tests.rs | 33 +-- 8 files changed, 66 insertions(+), 583 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba1bf3641..74e877cde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -912,6 +912,14 @@ name = "craft-cfb" version = "0.1.0" source = "git+https://github.com/storytold/craft-libs?rev=119415f46f601e20b9bc6b1b4d6979ceda86d88e#119415f46f601e20b9bc6b1b4d6979ceda86d88e" +[[package]] +name = "craft-csv" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=119415f46f601e20b9bc6b1b4d6979ceda86d88e#119415f46f601e20b9bc6b1b4d6979ceda86d88e" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-fonts-build" version = "0.1.0" @@ -2120,6 +2128,7 @@ dependencies = [ name = "gridcraft-engine" version = "0.19.1" dependencies = [ + "craft-csv", "gridcraft-calc", "gridcraft-chart", "gridcraft-core", @@ -2261,6 +2270,7 @@ name = "gridcraft-xlsx" version = "0.19.1" dependencies = [ "aes", + "craft-csv", "craft-xls", "gridcraft-core", "gridcraft-formula", diff --git a/Cargo.toml b/Cargo.toml index ad9ae0f1a..071fca480 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,9 @@ gridcraft-math = { path = "crates/math" } # craft-libs revision. # craft-update: app-specific pretend versions apply to startup and manual checks. craft-update = { git = "https://github.com/storytold/craft-libs", rev = "119415f46f601e20b9bc6b1b4d6979ceda86d88e", version = "=0.1.0", features = ["egui"] } +# Delimited text (CSV, TSV) and SYLK; `crates/xlsx/src/csv.rs` and `crates/engine/src/io/sylk.rs` +# turn them into workbooks. +craft-csv = { git = "https://github.com/storytold/craft-libs", rev = "119415f46f601e20b9bc6b1b4d6979ceda86d88e", version = "0.1.0" } # Excel 97-2003 (.xls, BIFF8) workbooks; `crates/xlsx/src/xls.rs` maps them onto the model. craft-xls = { git = "https://github.com/storytold/craft-libs", rev = "119415f46f601e20b9bc6b1b4d6979ceda86d88e", version = "0.1.0" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 341d1b35c..9481b0476 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,6 +21,7 @@ web-time = { workspace = true } thiserror = { workspace = true } log = { workspace = true } gridcraft-xlsx = { workspace = true } +craft-csv = { workspace = true } gridcraft-chart = { workspace = true } gridcraft-pdf = { workspace = true } gridcraft-math = { workspace = true } diff --git a/crates/engine/src/io/sylk.rs b/crates/engine/src/io/sylk.rs index 8eded1042..114574e9f 100644 --- a/crates/engine/src/io/sylk.rs +++ b/crates/engine/src/io/sylk.rs @@ -1,156 +1,57 @@ -//! SYLK (`.slk`) import: the symbolic link text format older spreadsheets and some legacy systems -//! export. One record per line, fields separated by `;` (`;;` is a literal semicolon): -//! -//! - `ID` starts the file; `E` ends it. -//! - `C;Y;X;K;E` is a cell. `Y` and `X` may be left out to stay on the -//! current row or column. `K` is a number, `"text"`, `TRUE`/`FALSE` or an error; `E` is a -//! formula in R1C1 notation; `S;R;C` shares the formula of another cell. -//! - `P;P` adds a number format to the file's list; `F;P;Y;X` gives a cell the -//! n-th format (0-based), and `F;W ` sets column widths in characters. -//! -//! Other records (fonts, options, window settings) are skipped. Formulas are converted to A1 with -//! the same relative and absolute references, as Excel does when it opens the file. +//! SYLK (`.slk`) import. `craft_csv::sylk` reads the file (values, formulas rewritten from R1C1 to +//! A1, number formats, column widths); this module makes it a one-sheet workbook: formulas parsed +//! (a formula GridCraft can't parse is skipped and counted), error values recognised, formats +//! interned as styles and widths converted from characters. use std::collections::HashMap; use std::sync::Arc; -use gridcraft_core::{CellError, CellRef, MAX_COLS, MAX_ROWS, Value, col_to_letters}; +use craft_csv::sylk; +use gridcraft_core::{CellError, CellRef, Value}; use gridcraft_model::{Cell, Formula, NumFmt, Sheet, Style, Workbook}; use crate::{EngineError, Result}; -/// The largest SYLK file read, as for other text imports: past it the file is refused. -const MAX_BYTES: usize = 512 << 20; -/// Column-width records kept from one file; each can cover every column. -const MAX_WIDTH_RECORDS: usize = 1024; - /// Reads a SYLK file into a one-sheet workbook. Returns the workbook and its warnings. pub fn read(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec)> { - if bytes.len() > MAX_BYTES { - return Err(EngineError::Other(format!("We can't open '{name}': the SYLK file is larger than {} MB.", MAX_BYTES >> 20))); - } - let text = match std::str::from_utf8(bytes) { - Ok(s) => s.to_string(), - // Older exports are in a single-byte code page: keep each byte as its Latin-1 character. - Err(_) => bytes.iter().map(|&b| char::from(b)).collect(), - }; - let text = text.strip_prefix('\u{feff}').unwrap_or(&text); - if !text.starts_with("ID") { - return Err(EngineError::Other(format!("We can't open '{name}': it isn't a SYLK file (it doesn't start with an ID record)."))); - } + let file = sylk::read(bytes).map_err(|e| match e { + sylk::Error::TooLarge => EngineError::Other(format!("We can't open '{name}': the SYLK file is larger than {} MB.", sylk::MAX_BYTES >> 20)), + _ => EngineError::Other(format!("We can't open '{name}': it isn't a SYLK file (it doesn't start with an ID record).")), + })?; + let mut wb = Workbook::new(); let mut sheet = Sheet::new("Sheet1"); - let mut formats: Vec = Vec::new(); - // Cells' number formats by index into `formats`, applied once every format is known. - let mut cell_formats: Vec<(CellRef, usize)> = Vec::new(); - let mut formulas: HashMap = HashMap::new(); - let mut widths: Vec<(u32, u32, f64)> = Vec::new(); - let (mut row, mut col) = (0u32, 0u32); - let mut skipped = 0usize; - for line in text.lines() { - let fields = split_fields(line); - let Some((kind, rest)) = fields.split_first() else { continue }; - match kind.as_str() { - "C" => { - let (mut value, mut formula, mut shared) = (None, None, None::<(Option, Option)>); - let mut bad = false; - for f in rest { - let (tag, arg) = f.split_at(f.chars().next().map_or(0, char::len_utf8)); - match tag { - "Y" => match position(arg, MAX_ROWS) { - Some(r) => row = r, - None => bad = true, - }, - "X" => match position(arg, MAX_COLS) { - Some(c) => col = c, - None => bad = true, - }, - "K" => value = Some(parse_value(arg)), - "E" => formula = Some(arg.to_string()), - "S" => shared = Some((None, None)), - // In a shared-formula cell, R and C name the cell whose formula it shares. - "R" if shared.is_some() => shared = shared.map(|(_, c)| (position(arg, MAX_ROWS), c)), - "C" if shared.is_some() => shared = shared.map(|(r, _)| (r, position(arg, MAX_COLS))), - _ => {} - } - } - if bad { - skipped += 1; - continue; - } - let at = CellRef::new(row, col); - if let Some((Some(r), Some(c))) = shared { - formula = formulas.get(&CellRef::new(r, c)).cloned(); - } - let mut cell = sheet.cell(at).cloned().unwrap_or_else(|| Cell::value(Value::Empty)); - if let Some(v) = value { - cell.value = v; - } - if let Some(r1c1) = formula { - let a1 = r1c1_formula_to_a1(&r1c1, at); - let f = Formula::new(&a1); - if f.expr.is_some() { - cell.formula = Some(Arc::new(f)); - formulas.insert(at, r1c1); - } else { - skipped += 1; - } - } - sheet.set_cell(at, cell); + let mut skipped = file.skipped; + let mut styles = HashMap::new(); + for (&(row, col), c) in &file.cells { + let value = match &c.value { + sylk::Value::Number(n) => Value::Number(*n), + sylk::Value::Text(s) => Value::Text(Arc::from(s.as_str())), + sylk::Value::Bool(b) => Value::Bool(*b), + sylk::Value::Error(s) => CellError::parse(s).map_or_else(|| Value::Text(Arc::from(s.as_str())), Value::Error), + _ => Value::Empty, + }; + let mut cell = Cell::value(value); + if let Some(a1) = &c.formula { + let f = Formula::new(a1); + if f.expr.is_some() { + cell.formula = Some(Arc::new(f)); + } else { + skipped += 1; } - "F" => { - let mut format = None; - for f in rest { - let (tag, arg) = f.split_at(f.chars().next().map_or(0, char::len_utf8)); - match tag { - "Y" => row = position(arg, MAX_ROWS).unwrap_or(row), - "X" => col = position(arg, MAX_COLS).unwrap_or(col), - "P" => format = arg.parse::().ok(), - "W" => { - let mut parts = arg.split_whitespace(); - if let (Some(first), Some(last), Some(width)) = (parts.next(), parts.next(), parts.next()) - && let (Some(first), Some(last), Ok(width)) = - (position(first, MAX_COLS), position(last, MAX_COLS), width.parse::()) - && first <= last - && (0.0..=255.0).contains(&width) - // A file of width records can't make the import loop over every column - // again and again. - && widths.len() < MAX_WIDTH_RECORDS - { - widths.push((first, last, width)); - } - } - _ => {} - } - } - if let Some(format) = format { - cell_formats.push((CellRef::new(row, col), format)); - } - } - // Number formats only: `P;F…` and `P;E…` are fonts. - "P" => { - if let Some(code) = rest.first().and_then(|f| f.strip_prefix('P')) { - formats.push(code.to_string()); - } - } - "E" => break, - _ => {} } + if let Some(code) = &c.format { + cell.style = *styles + .entry(code.as_str()) + .or_insert_with(|| wb.styles.intern(Style { num_fmt: NumFmt(Arc::from(code.as_str())), ..Style::default() })); + } + sheet.set_cell(CellRef::new(row, col), cell); } - for (first, last, width) in widths { - let size = crate::cmd::format::chars_to_points(width) as f32; - for c in first..=last { + for w in &file.column_widths { + let size = crate::cmd::format::chars_to_points(w.width) as f32; + for c in w.first..=w.last { sheet.cols.entry(c).or_default().size = Some(size); } } - let mut wb = Workbook::new(); - let mut styles = HashMap::new(); - for (at, index) in cell_formats { - let Some(code) = formats.get(index).filter(|code| !code.is_empty() && *code != "General") else { continue }; - let style = *styles.entry(index).or_insert_with(|| wb.styles.intern(Style { num_fmt: NumFmt(Arc::from(code.as_str())), ..Style::default() })); - let mut cell = sheet.cell(at).cloned().unwrap_or_else(|| Cell::value(Value::Empty)); - cell.style = style; - sheet.set_cell(at, cell); - } if let Some(first) = wb.sheet_mut(0) { sheet.name = std::mem::take(&mut first.name); *first = sheet; @@ -161,168 +62,3 @@ pub fn read(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec)> { } Ok((wb, warnings)) } - -/// A record's fields: `;` separates them and `;;` is a literal semicolon. -fn split_fields(line: &str) -> Vec { - let mut fields = vec![String::new()]; - let mut chars = line.trim_end_matches('\r').chars().peekable(); - while let Some(c) = chars.next() { - if c == ';' { - if chars.peek() == Some(&';') { - chars.next(); - if let Some(f) = fields.last_mut() { - f.push(';'); - } - } else { - fields.push(String::new()); - } - } else if let Some(f) = fields.last_mut() { - f.push(c); - } - } - fields -} - -/// A 1-based row or column number → 0-based, within `max`. -fn position(arg: &str, max: u32) -> Option { - arg.trim().parse::().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1) -} - -/// A `K` value: `"text"` (everything between the first and the last quote, as written), a -/// boolean, an error or a number. -fn parse_value(arg: &str) -> Value { - if let Some(text) = arg.strip_prefix('"') { - let text = text.strip_suffix('"').unwrap_or(text); - return Value::Text(Arc::from(text)); - } - if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") { - return Value::Bool(arg.eq_ignore_ascii_case("TRUE")); - } - if let Some(e) = CellError::parse(arg) { - return Value::Error(e); - } - match arg.trim().parse::() { - Ok(n) if n.is_finite() => Value::Number(n), - _ => Value::Text(Arc::from(arg)), - } -} - -/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or -/// nothing (the cell's own row or column). -#[derive(Clone, Copy, Debug, PartialEq)] -enum Part { - Absolute(u32), - Relative(i64), -} - -impl Part { - /// The 0-based index and whether it is absolute, or `None` when it falls off the sheet. - fn resolve(self, base: u32, max: u32) -> Option<(u32, bool)> { - match self { - Part::Absolute(n) => (1..=max).contains(&n).then(|| (n - 1, true)), - Part::Relative(k) => i64::from(base).checked_add(k).and_then(|v| u32::try_from(v).ok()).filter(|v| *v < max).map(|v| (v, false)), - } - } -} - -/// Parses the number after `R` or `C` at `i`: digits, `[offset]` or nothing. Returns the part and -/// the index after it, or `None` when the text isn't a reference part. -fn parse_part(chars: &[char], i: usize) -> Option<(Part, usize)> { - match chars.get(i) { - Some('[') => { - let close = chars[i..].iter().position(|&c| c == ']')? + i; - let inner: String = chars[i + 1..close].iter().collect(); - Some((Part::Relative(inner.parse().ok()?), close + 1)) - } - Some(c) if c.is_ascii_digit() => { - let end = chars[i..].iter().position(|c| !c.is_ascii_digit()).map_or(chars.len(), |p| p + i); - let n: String = chars[i..end].iter().collect(); - Some((Part::Absolute(n.parse().unwrap_or(u32::MAX)), end)) - } - _ => Some((Part::Relative(0), i)), - } -} - -fn is_word_char(c: char) -> bool { - c.is_alphanumeric() || matches!(c, '_' | '.' | '\\' | '?' | '$') -} - -/// Rewrites the R1C1 references in a SYLK formula as A1 references for the cell at `at`: -/// `RC[-1]` → `B2`, `R1C1` → `$A$1`, `R[-2]C:R[-1]C` → `D2:D3`, a whole column `C4` → `$D:$D` and -/// a whole row `R3` → `$3:$3`. Text in quotes and quoted sheet names are left alone; a reference -/// off the sheet becomes `#REF!`. -pub(crate) fn r1c1_formula_to_a1(formula: &str, at: CellRef) -> String { - let chars: Vec = formula.chars().collect(); - let mut out = String::with_capacity(formula.len()); - let mut i = 0; - while i < chars.len() { - let c = chars[i]; - if c == '"' || c == '\'' { - // Copy the quoted run as it is; a doubled quote stays inside it. - let mut j = i + 1; - while j < chars.len() && !(chars[j] == c && chars.get(j + 1) != Some(&c)) { - j += if chars[j] == c { 2 } else { 1 }; - } - let stop = (j + 1).min(chars.len()); - out.extend(&chars[i..stop]); - i = stop; - continue; - } - let starts_word = i == 0 || !is_word_char(chars[i - 1]); - if starts_word - && matches!(c, 'R' | 'r' | 'C' | 'c') - && let Some((text, next)) = reference_at(&chars, i, at) - { - out.push_str(&text); - i = next; - continue; - } - // Copy the rest of a word as it is, so `ROUND` or `Sheet2` isn't read as a reference. - if is_word_char(c) { - let end = (i..chars.len()).find(|&j| !is_word_char(chars[j])).unwrap_or(chars.len()); - out.extend(&chars[i..end]); - i = end; - } else { - out.push(c); - i += 1; - } - } - out -} - -/// The A1 text of the R1C1 reference starting at `i`, and the index after it, or `None` when the -/// word there isn't a reference (a function or a name). -fn reference_at(chars: &[char], i: usize, at: CellRef) -> Option<(String, usize)> { - let mut j = i; - let mut row = None; - let mut col = None; - if matches!(chars.get(j), Some('R' | 'r')) { - let (part, next) = parse_part(chars, j + 1)?; - row = Some(part); - j = next; - } - if matches!(chars.get(j), Some('C' | 'c')) { - let (part, next) = parse_part(chars, j + 1)?; - col = Some(part); - j = next; - } - if row.is_none() && col.is_none() { - return None; - } - // A reference ends the word: `ROUND(`, `RATE` or `R2D2` are names. - if chars.get(j).is_some_and(|&c| is_word_char(c) || c == '(' || c == '[' || c == '!') { - return None; - } - let row_text = |p: Part| p.resolve(at.row, MAX_ROWS).map(|(r, abs)| format!("{}{}", if abs { "$" } else { "" }, r + 1)); - let col_text = |p: Part| p.resolve(at.col, MAX_COLS).map(|(c, abs)| format!("{}{}", if abs { "$" } else { "" }, col_to_letters(c))); - // One side of a range (`R2:R4`, `C1:C3`) is written alone; a lone row or column is the whole of it. - let prev = chars[..i].iter().rev().find(|c| !c.is_whitespace()); - let in_range = prev == Some(&':') || chars[j..].iter().find(|c| !c.is_whitespace()) == Some(&':'); - let text = match (row, col) { - (Some(r), Some(c)) => row_text(r).zip(col_text(c)).map(|(r, c)| format!("{c}{r}")), - (Some(r), None) => row_text(r).map(|r| if in_range { r } else { format!("{r}:{r}") }), - (None, Some(c)) => col_text(c).map(|c| if in_range { c } else { format!("{c}:{c}") }), - (None, None) => None, - }; - Some((text.unwrap_or_else(|| "#REF!".into()), j)) -} diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index ccdc361d1..ad8ca40f2 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -763,7 +763,7 @@ fn sylk_r1c1_formulas_become_a1() { ("R[-5]C", "#REF!"), ("\"RC\"&R1C1", "\"RC\"&$A$1"), ] { - assert_eq!(sylk::r1c1_formula_to_a1(r1c1, b2), a1, "{r1c1}"); + assert_eq!(craft_csv::sylk::r1c1_formula_to_a1(r1c1, (b2.row, b2.col)), a1, "{r1c1}"); } } diff --git a/crates/xlsx/Cargo.toml b/crates/xlsx/Cargo.toml index ab322412d..9d09548ce 100644 --- a/crates/xlsx/Cargo.toml +++ b/crates/xlsx/Cargo.toml @@ -18,6 +18,7 @@ gridcraft-core = { workspace = true } gridcraft-formula = { workspace = true } gridcraft-math = { workspace = true } gridcraft-model = { workspace = true } +craft-csv = { workspace = true } craft-xls = { workspace = true } thiserror = { workspace = true } quick-xml = { workspace = true } diff --git a/crates/xlsx/src/csv.rs b/crates/xlsx/src/csv.rs index 254e903f0..ce9ef5a65 100644 --- a/crates/xlsx/src/csv.rs +++ b/crates/xlsx/src/csv.rs @@ -1,4 +1,6 @@ -//! CSV import and export (RFC 4180 with the usual real-world leniency). +//! CSV import and export (RFC 4180 with the usual real-world leniency). Decoding, delimiter +//! sniffing, the record parser and field quoting are `craft-csv`'s; this module turns records into +//! typed cells (with each column's date order) and writes a sheet. use std::sync::Arc; @@ -12,241 +14,11 @@ use crate::{CsvOptions, IoError, ReadReport}; /// The largest CSV or tab-separated file written (bytes). pub const MAX_CSV_BYTES: usize = 512 << 20; -/// Windows-1252 code points for bytes 0x80–0x9F (others map to Latin-1). -const CP1252: [u32; 32] = [ - 0x20AC, 0x81, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x8D, 0x017D, 0x8F, 0x90, 0x2018, 0x2019, - 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178, -]; +pub use craft_csv::{decode_text, sniff_delimiter}; -/// Windows-1256 code points for bytes 0x80–0xFF (unicode.org `CP1256.TXT`). -pub(crate) const CP1256: [u32; 128] = [ - 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018, - 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3, - 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, - 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, - 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638, - 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, - 0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, - 0x200F, 0x06D2, -]; - -/// Evidence that bytes which aren't UTF-8 are Windows-1256 Arabic rather than Windows-1252. -#[derive(Default)] -struct Cp1256Evidence { - /// High bytes that decode to different characters in the two code pages, apart from invisible - /// marks and ordinals after a digit. - differing: usize, - /// Of those, Arabic letters and Arabic punctuation (، ؛ ؟) in Windows-1256. - arabic: usize, - /// Arabic bytes next to an ASCII letter: Western accents sit inside Latin words, Arabic doesn't. - arabic_touching_latin: usize, - /// Accented Latin bytes the two code pages share, next to another high byte: Hebrew - /// Windows-1255 letters land there, Arabic and Western text rarely do. - shared_letters_in_runs: usize, -} - -impl Cp1256Evidence { - fn of(bytes: &[u8]) -> Self { - let mut e = Self::default(); - for (i, &b) in bytes.iter().enumerate() { - if b < 0x80 { - continue; - } - let before = i.checked_sub(1).and_then(|j| bytes.get(j)).copied(); - let after = bytes.get(i + 1).copied(); - let next_to = |pred: fn(u8) -> bool| before.is_some_and(pred) || after.is_some_and(pred); - if same_in_cp1252_and_cp1256(b) { - if is_shared_latin_letter(b) && next_to(|x| x >= 0x80) { - e.shared_letters_in_runs += 1; - } - continue; - } - if is_invisible_mark(b) || is_ordinal_after_digit(b, before) { - continue; - } - e.differing += 1; - if is_cp1256_arabic(b) { - e.arabic += 1; - if next_to(|x| x.is_ascii_alphabetic()) { - e.arabic_touching_latin += 1; - } - } - } - e - } - - /// Thresholds chosen against month, day and language names from 95 glibc locales (Western in - /// Windows-1252, Arabic, Persian and Urdu in 1256, Hebrew and Yiddish in 1255) plus CSV samples. - fn is_arabic(&self) -> bool { - self.arabic >= 2 - && self.arabic * 10 >= self.differing * 9 - && self.arabic_touching_latin * 10 <= self.arabic * 3 - && self.shared_letters_in_runs * 20 <= self.arabic - } -} - -fn same_in_cp1252_and_cp1256(b: u8) -> bool { - matches!( - b, - 0x80 | 0x82..=0x89 | 0x8B..=0x8C | 0x91..=0x97 | 0x99 | 0x9B..=0x9C | 0xA0 | 0xA2..=0xA9 | 0xAB..=0xB9 | 0xBB..=0xBE | 0xD7 | 0xE0 | 0xE2 | 0xE7..=0xEB - | 0xEE..=0xEF | 0xF4 | 0xF7 | 0xF9 | 0xFB..=0xFC - ) -} - -/// ZWNJ, ZWJ, LRM and RLM: common inside Persian words and around Arabic cells, but no evidence -/// either way. -fn is_invisible_mark(b: u8) -> bool { - matches!(b, 0x9D | 0x9E | 0xFD | 0xFE) -} - -/// `º` and `ª` after a digit (`1º`, `2ª`) are Western ordinals, though Windows-1256 reads the bytes -/// as `؛` and `ھ`; either way they are no evidence. -fn is_ordinal_after_digit(b: u8, before: Option) -> bool { - matches!(b, 0xAA | 0xBA) && before.is_some_and(|x| x.is_ascii_digit()) -} - -fn is_shared_latin_letter(b: u8) -> bool { - matches!(b, 0xE0 | 0xE2 | 0xE7..=0xEB | 0xEE..=0xEF | 0xF4 | 0xF9 | 0xFB..=0xFC) -} - -fn is_cp1256_arabic(b: u8) -> bool { - matches!( - b, - 0x81 | 0x8A | 0x8D..=0x90 | 0x98 | 0x9A | 0x9F | 0xA1 | 0xAA | 0xBA | 0xBF..=0xD6 | 0xD8..=0xDB | 0xDD..=0xDF | 0xE1 | 0xE3..=0xE6 | 0xEC..=0xED | 0xFF - ) -} - -/// Decodes CSV bytes with BOM-aware UTF-8 and UTF-16 handling. Without a BOM, valid UTF-8 is kept. -/// Other bytes decode as Windows-1256 when they read as Arabic (see [`Cp1256Evidence`]), else as -/// Windows-1252. -pub fn decode_text(bytes: &[u8]) -> String { - if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) { - return String::from_utf8_lossy(rest).into_owned(); - } - let utf16 = |b: &[u8], le: bool| -> String { - let units = b.as_chunks::<2>().0.iter().map(|c| if le { u16::from_le_bytes(*c) } else { u16::from_be_bytes(*c) }); - char::decode_utf16(units).map(|r| r.unwrap_or('\u{FFFD}')).collect() - }; - if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) { - return utf16(rest, true); - } - if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) { - return utf16(rest, false); - } - match std::str::from_utf8(bytes) { - Ok(s) => s.to_string(), - Err(_) => { - let use_windows_1256 = Cp1256Evidence::of(bytes).is_arabic(); - bytes - .iter() - .map(|&b| match b { - 0x80..=0xFF if use_windows_1256 => { - char::from_u32(CP1256.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}') - } - 0x80..=0x9F => char::from_u32(CP1252.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}'), - _ => b as char, - }) - .collect() - } - } -} - -/// Picks comma, semicolon or tab: the candidate with the most consistent count per line -/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins. -pub fn sniff_delimiter(text: &str, quote: char) -> u8 { - let mut best = (b',', 0usize, 0usize); - for d in [',', ';', '\t'] { - let mut counts = vec![]; - let mut n = 0usize; - let mut in_q = false; - let mut blank = true; - for ch in text.chars().take(64 * 1024) { - if !in_q && ch == '\n' { - // Blank lines carry no delimiter evidence. - if !blank { - counts.push(n); - } - n = 0; - blank = true; - if counts.len() >= 20 { - break; - } - continue; - } - if ch != '\r' { - blank = false; - } - if ch == quote { - in_q = !in_q; - } else if !in_q && ch == d { - n += 1; - } - } - if !blank { - counts.push(n); - } - let min = counts.iter().copied().min().unwrap_or(0); - let total: usize = counts.iter().sum(); - if min > 0 && (min, total) > (best.1, best.2) { - best = (d as u8, min, total); - } - } - best.0 -} - -/// Splits CSV text into records of fields. A record keeps at most [`MAX_COLS`] fields; the rest are -/// dropped as they are read, so a row of endless delimiters cannot allocate a string per field. -pub(crate) fn parse_records(text: &str, delim: char, quote: char, mut on_record: impl FnMut(Vec) -> bool) { - let mut field = String::new(); - let mut record: Vec = vec![]; - let mut in_q = false; - let mut chars = text.chars().peekable(); - let mut field_started_quoted = false; - while let Some(c) = chars.next() { - if in_q { - if c == quote { - if chars.peek() == Some("e) { - field.push(quote); - chars.next(); - } else { - in_q = false; - } - } else { - field.push(c); - } - continue; - } - if c == quote && field.is_empty() && !field_started_quoted { - in_q = true; - field_started_quoted = true; - } else if c == delim { - let f = std::mem::take(&mut field); - if record.len() < MAX_COLS as usize { - record.push(f); - } - field_started_quoted = false; - } else if c == '\r' || c == '\n' { - if c == '\r' && chars.peek() == Some(&'\n') { - chars.next(); - } - let f = std::mem::take(&mut field); - if record.len() < MAX_COLS as usize { - record.push(f); - } - field_started_quoted = false; - if !on_record(std::mem::take(&mut record)) { - return; - } - } else { - field.push(c); - } - } - if !field.is_empty() || !record.is_empty() || field_started_quoted { - if record.len() < MAX_COLS as usize { - record.push(field); - } - on_record(record); - } +/// [`craft_csv::parse_records`] with a record kept to the grid's [`MAX_COLS`] fields. +fn parse_records(text: &str, delim: char, quote: char, on_record: impl FnMut(Vec) -> bool) { + craft_csv::parse_records(text, delim, quote, MAX_COLS as usize, on_record); } /// How many rows and bytes of a text file are read to decide each column's date order. @@ -587,25 +359,12 @@ pub fn write_csv_limited(sheet: &Sheet, delimiter: u8, limit: usize, mut cell_te out.push(delim); } let t = cell_text(CellRef::new(row, col)); - let quoted = t.contains(delim) || t.contains('"') || t.contains('\n') || t.contains('\r'); // The exact size of the field as written (the delimiter was pushed above), so a row - // never grows past the cap unseen and no doubled-quote copy of the text is built. - let field_len = if quoted { t.len() + t.matches('"').count() + 2 } else { t.len() }; - if out.len().saturating_add(field_len) > limit { + // never grows past the cap unseen. + if out.len().saturating_add(craft_csv::field_len(&t, delim)) > limit { return Err(too_large(format!("more than {} MiB (stopped at row {})", limit >> 20, row + 1))); } - if quoted { - out.push('"'); - for ch in t.chars() { - if ch == '"' { - out.push('"'); - } - out.push(ch); - } - out.push('"'); - } else { - out.push_str(&t); - } + craft_csv::push_field(&mut out, &t, delim); } out.push_str("\r\n"); if out.len() > limit { diff --git a/crates/xlsx/src/tests/csv_tests.rs b/crates/xlsx/src/tests/csv_tests.rs index ec4405f1b..c277febf4 100644 --- a/crates/xlsx/src/tests/csv_tests.rs +++ b/crates/xlsx/src/tests/csv_tests.rs @@ -1,8 +1,8 @@ use gridcraft_core::{CellRef, DateSystem, MAX_COLS, Value}; use gridcraft_model::{Sheet, Workbook}; -use crate::csv::{decode_text, parse_records, sniff_delimiter}; use crate::{CsvOptions, Format, IoError, read_csv, read_csv_with_report, sniff, write_csv, write_csv_limited}; +use craft_csv::{decode_text, parse_records, sniff_delimiter}; fn v(wb: &Workbook, r: u32, c: u32) -> Value { wb.sheet(0).unwrap().value(CellRef::new(r, c)) @@ -148,33 +148,6 @@ fn encodings() { assert_eq!(v(&wb, 0, 1), Value::Number(1.0)); } -#[test] -fn windows_1256_table_matches_the_unicode_mapping() { - let at = |b: u8| crate::csv::CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c)); - let expected = [ - (0x80, '€'), - (0x81, 'پ'), - (0x8D, 'چ'), - (0x98, 'ک'), - (0x9D, '\u{200C}'), - (0xA1, '،'), - (0xBA, '؛'), - (0xBF, '؟'), - (0xC7, 'ا'), - (0xC8, 'ب'), - (0xDC, 'ـ'), - (0xE1, 'ل'), - (0xE3, 'م'), - (0xE9, 'é'), - (0xFD, '\u{200E}'), - (0xFE, '\u{200F}'), - (0xFF, 'ے'), - ]; - for (b, c) in expected { - assert_eq!(at(b), Some(c), "0x{b:02X}"); - } -} - #[test] fn arabic_csv_in_windows_1256_reads_as_arabic() { let bytes = [ @@ -393,14 +366,14 @@ fn parser_stops_collecting_fields_at_max_cols() { let mut text = ",".repeat((MAX_COLS + extra) as usize - 1); text.push_str("end\nx,y\n"); let mut lens = vec![]; - parse_records(&text, ',', '"', |rec| { + parse_records(&text, ',', '"', MAX_COLS as usize, |rec| { lens.push(rec.len()); true }); assert_eq!(lens, vec![MAX_COLS as usize, 2]); // An unterminated last row is capped the same way. let mut lens = vec![]; - parse_records(&",".repeat(MAX_COLS as usize + extra as usize), ',', '"', |rec| { + parse_records(&",".repeat(MAX_COLS as usize + extra as usize), ',', '"', MAX_COLS as usize, |rec| { lens.push(rec.len()); true });