pauljd/gridcraft
1
0
Fork 0
mirror of https://github.com/storytold/gridcraft.git synced 2026-11-12 07:32:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

xlsx, engine: CSV decoding, parsing and quoting, and SYLK, through craft-csv

Browse Source 
Text decoding (byte order marks, UTF-8, Windows-1256/1252), delimiter sniffing,
the record parser and field quoting moved to storytold/craft-libs as craft-csv
(#889), with the SYLK reader as craft_csv::sylk. GridCraft keeps typed values,
date-order inference, the used-range limit and the sheet writer in csv.rs, and in
io/sylk.rs the workbook a SYLK file becomes: formulas parsed (an unparsable one
is skipped and counted, as before), error values, number-format styles and column
widths.
This commit is contained in:
Dylan M. Taylor
2026-11-12 01:17:21 +00:00
parent 21df49c636
commit c1adf9b239
8 changed files with 66 additions and 583 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+10
View File
@@ -912,6 +912,14 @@ name = "craft-cfb"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=119415f46f601e20b9bc6b1b4d6979ceda86d88e#119415f46f601e20b9bc6b1b4d6979ceda86d88e"
[[package]]
name = "craft-csv"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=119415f46f601e20b9bc6b1b4d6979ceda86d88e#119415f46f601e20b9bc6b1b4d6979ceda86d88e"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-fonts-build"
version = "0.1.0"
@@ -2120,6 +2128,7 @@ dependencies = [
name = "gridcraft-engine"
version = "0.19.1"
dependencies = [
"craft-csv",
"gridcraft-calc",
"gridcraft-chart",
"gridcraft-core",
@@ -2261,6 +2270,7 @@ name = "gridcraft-xlsx"
version = "0.19.1"
dependencies = [
"aes",
"craft-csv",
"craft-xls",
"gridcraft-core",
"gridcraft-formula",
Cargo.toml
+3
View File
@@ -31,6 +31,9 @@ gridcraft-math = { path = "crates/math" }
# craft-libs revision.
# craft-update: app-specific pretend versions apply to startup and manual checks.
craft-update = { git = "https://github.com/storytold/craft-libs", rev = "119415f46f601e20b9bc6b1b4d6979ceda86d88e", version = "=0.1.0", features = ["egui"] }
# Delimited text (CSV, TSV) and SYLK; `crates/xlsx/src/csv.rs` and `crates/engine/src/io/sylk.rs`
# turn them into workbooks.
craft-csv = { git = "https://github.com/storytold/craft-libs", rev = "119415f46f601e20b9bc6b1b4d6979ceda86d88e", version = "0.1.0" }
# Excel 97-2003 (.xls, BIFF8) workbooks; `crates/xlsx/src/xls.rs` maps them onto the model.
craft-xls = { git = "https://github.com/storytold/craft-libs", rev = "119415f46f601e20b9bc6b1b4d6979ceda86d88e", version = "0.1.0" }
crates/engine/Cargo.toml
+1
View File
@@ -21,6 +21,7 @@ web-time = { workspace = true }
thiserror = { workspace = true }
log = { workspace = true }
gridcraft-xlsx = { workspace = true }
craft-csv = { workspace = true }
gridcraft-chart = { workspace = true }
gridcraft-pdf = { workspace = true }
gridcraft-math = { workspace = true }
crates/engine/src/io/sylk.rs
+37 -301
View File
@@ -1,156 +1,57 @@
//! SYLK (`.slk`) import: the symbolic link text format older spreadsheets and some legacy systems
//! export. One record per line, fields separated by `;` (`;;` is a literal semicolon):
//!
//! - `ID` starts the file; `E` ends it.
//! - `C;Y<row>;X<col>;K<value>;E<formula>` is a cell. `Y` and `X` may be left out to stay on the
//! current row or column. `K` is a number, `"text"`, `TRUE`/`FALSE` or an error; `E` is a
//! formula in R1C1 notation; `S;R<row>;C<col>` shares the formula of another cell.
//! - `P;P<code>` adds a number format to the file's list; `F;P<n>;Y<row>;X<col>` gives a cell the
//! n-th format (0-based), and `F;W<first> <last> <width>` sets column widths in characters.
//!
//! Other records (fonts, options, window settings) are skipped. Formulas are converted to A1 with
//! the same relative and absolute references, as Excel does when it opens the file.
//! SYLK (`.slk`) import. `craft_csv::sylk` reads the file (values, formulas rewritten from R1C1 to
//! A1, number formats, column widths); this module makes it a one-sheet workbook: formulas parsed
//! (a formula GridCraft can't parse is skipped and counted), error values recognised, formats
//! interned as styles and widths converted from characters.
use std::collections::HashMap;
use std::sync::Arc;
use gridcraft_core::{CellError, CellRef, MAX_COLS, MAX_ROWS, Value, col_to_letters};
use craft_csv::sylk;
use gridcraft_core::{CellError, CellRef, Value};
use gridcraft_model::{Cell, Formula, NumFmt, Sheet, Style, Workbook};
use crate::{EngineError, Result};
/// The largest SYLK file read, as for other text imports: past it the file is refused.
const MAX_BYTES: usize = 512 << 20;
/// Column-width records kept from one file; each can cover every column.
const MAX_WIDTH_RECORDS: usize = 1024;
/// Reads a SYLK file into a one-sheet workbook. Returns the workbook and its warnings.
pub fn read(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec<String>)> {
if bytes.len() > MAX_BYTES {
return Err(EngineError::Other(format!("We can't open '{name}': the SYLK file is larger than {} MB.", MAX_BYTES >> 20)));
}
let text = match std::str::from_utf8(bytes) {
Ok(s) => s.to_string(),
// Older exports are in a single-byte code page: keep each byte as its Latin-1 character.
Err(_) => bytes.iter().map(|&b| char::from(b)).collect(),
};
let text = text.strip_prefix('\u{feff}').unwrap_or(&text);
if !text.starts_with("ID") {
return Err(EngineError::Other(format!("We can't open '{name}': it isn't a SYLK file (it doesn't start with an ID record).")));
}
let file = sylk::read(bytes).map_err(|e| match e {
sylk::Error::TooLarge => EngineError::Other(format!("We can't open '{name}': the SYLK file is larger than {} MB.", sylk::MAX_BYTES >> 20)),
_ => EngineError::Other(format!("We can't open '{name}': it isn't a SYLK file (it doesn't start with an ID record).")),
})?;
let mut wb = Workbook::new();
let mut sheet = Sheet::new("Sheet1");
let mut formats: Vec<String> = Vec::new();
// Cells' number formats by index into `formats`, applied once every format is known.
let mut cell_formats: Vec<(CellRef, usize)> = Vec::new();
let mut formulas: HashMap<CellRef, String> = HashMap::new();
let mut widths: Vec<(u32, u32, f64)> = Vec::new();
let (mut row, mut col) = (0u32, 0u32);
let mut skipped = 0usize;
for line in text.lines() {
let fields = split_fields(line);
let Some((kind, rest)) = fields.split_first() else { continue };
match kind.as_str() {
"C" => {
let (mut value, mut formula, mut shared) = (None, None, None::<(Option<u32>, Option<u32>)>);
let mut bad = false;
for f in rest {
let (tag, arg) = f.split_at(f.chars().next().map_or(0, char::len_utf8));
match tag {
"Y" => match position(arg, MAX_ROWS) {
Some(r) => row = r,
None => bad = true,
},
"X" => match position(arg, MAX_COLS) {
Some(c) => col = c,
None => bad = true,
},
"K" => value = Some(parse_value(arg)),
"E" => formula = Some(arg.to_string()),
"S" => shared = Some((None, None)),
// In a shared-formula cell, R and C name the cell whose formula it shares.
"R" if shared.is_some() => shared = shared.map(|(_, c)| (position(arg, MAX_ROWS), c)),
"C" if shared.is_some() => shared = shared.map(|(r, _)| (r, position(arg, MAX_COLS))),
_ => {}
}
}
if bad {
skipped += 1;
continue;
}
let at = CellRef::new(row, col);
if let Some((Some(r), Some(c))) = shared {
formula = formulas.get(&CellRef::new(r, c)).cloned();
}
let mut cell = sheet.cell(at).cloned().unwrap_or_else(|| Cell::value(Value::Empty));
if let Some(v) = value {
cell.value = v;
}
if let Some(r1c1) = formula {
let a1 = r1c1_formula_to_a1(&r1c1, at);
let f = Formula::new(&a1);
if f.expr.is_some() {
cell.formula = Some(Arc::new(f));
formulas.insert(at, r1c1);
} else {
skipped += 1;
}
}
sheet.set_cell(at, cell);
let mut skipped = file.skipped;
let mut styles = HashMap::new();
for (&(row, col), c) in &file.cells {
let value = match &c.value {
sylk::Value::Number(n) => Value::Number(*n),
sylk::Value::Text(s) => Value::Text(Arc::from(s.as_str())),
sylk::Value::Bool(b) => Value::Bool(*b),
sylk::Value::Error(s) => CellError::parse(s).map_or_else(|| Value::Text(Arc::from(s.as_str())), Value::Error),
_ => Value::Empty,
};
let mut cell = Cell::value(value);
if let Some(a1) = &c.formula {
let f = Formula::new(a1);
if f.expr.is_some() {
cell.formula = Some(Arc::new(f));
} else {
skipped += 1;
}
"F" => {
let mut format = None;
for f in rest {
let (tag, arg) = f.split_at(f.chars().next().map_or(0, char::len_utf8));
match tag {
"Y" => row = position(arg, MAX_ROWS).unwrap_or(row),
"X" => col = position(arg, MAX_COLS).unwrap_or(col),
"P" => format = arg.parse::<usize>().ok(),
"W" => {
let mut parts = arg.split_whitespace();
if let (Some(first), Some(last), Some(width)) = (parts.next(), parts.next(), parts.next())
&& let (Some(first), Some(last), Ok(width)) =
(position(first, MAX_COLS), position(last, MAX_COLS), width.parse::<f64>())
&& first <= last
&& (0.0..=255.0).contains(&width)
// A file of width records can't make the import loop over every column
// again and again.
&& widths.len() < MAX_WIDTH_RECORDS
{
widths.push((first, last, width));
}
}
_ => {}
}
}
if let Some(format) = format {
cell_formats.push((CellRef::new(row, col), format));
}
}
// Number formats only: `P;F…` and `P;E…` are fonts.
"P" => {
if let Some(code) = rest.first().and_then(|f| f.strip_prefix('P')) {
formats.push(code.to_string());
}
}
"E" => break,
_ => {}
}
if let Some(code) = &c.format {
cell.style = *styles
.entry(code.as_str())
.or_insert_with(|| wb.styles.intern(Style { num_fmt: NumFmt(Arc::from(code.as_str())), ..Style::default() }));
}
sheet.set_cell(CellRef::new(row, col), cell);
}
for (first, last, width) in widths {
let size = crate::cmd::format::chars_to_points(width) as f32;
for c in first..=last {
for w in &file.column_widths {
let size = crate::cmd::format::chars_to_points(w.width) as f32;
for c in w.first..=w.last {
sheet.cols.entry(c).or_default().size = Some(size);
}
}
let mut wb = Workbook::new();
let mut styles = HashMap::new();
for (at, index) in cell_formats {
let Some(code) = formats.get(index).filter(|code| !code.is_empty() && *code != "General") else { continue };
let style = *styles.entry(index).or_insert_with(|| wb.styles.intern(Style { num_fmt: NumFmt(Arc::from(code.as_str())), ..Style::default() }));
let mut cell = sheet.cell(at).cloned().unwrap_or_else(|| Cell::value(Value::Empty));
cell.style = style;
sheet.set_cell(at, cell);
}
if let Some(first) = wb.sheet_mut(0) {
sheet.name = std::mem::take(&mut first.name);
*first = sheet;
@@ -161,168 +62,3 @@ pub fn read(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec<String>)> {
}
Ok((wb, warnings))
}
/// A record's fields: `;` separates them and `;;` is a literal semicolon.
fn split_fields(line: &str) -> Vec<String> {
let mut fields = vec![String::new()];
let mut chars = line.trim_end_matches('\r').chars().peekable();
while let Some(c) = chars.next() {
if c == ';' {
if chars.peek() == Some(&';') {
chars.next();
if let Some(f) = fields.last_mut() {
f.push(';');
}
} else {
fields.push(String::new());
}
} else if let Some(f) = fields.last_mut() {
f.push(c);
}
}
fields
}
/// A 1-based row or column number → 0-based, within `max`.
fn position(arg: &str, max: u32) -> Option<u32> {
arg.trim().parse::<u32>().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1)
}
/// A `K` value: `"text"` (everything between the first and the last quote, as written), a
/// boolean, an error or a number.
fn parse_value(arg: &str) -> Value {
if let Some(text) = arg.strip_prefix('"') {
let text = text.strip_suffix('"').unwrap_or(text);
return Value::Text(Arc::from(text));
}
if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") {
return Value::Bool(arg.eq_ignore_ascii_case("TRUE"));
}
if let Some(e) = CellError::parse(arg) {
return Value::Error(e);
}
match arg.trim().parse::<f64>() {
Ok(n) if n.is_finite() => Value::Number(n),
_ => Value::Text(Arc::from(arg)),
}
}
/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or
/// nothing (the cell's own row or column).
#[derive(Clone, Copy, Debug, PartialEq)]
enum Part {
Absolute(u32),
Relative(i64),
}
impl Part {
/// The 0-based index and whether it is absolute, or `None` when it falls off the sheet.
fn resolve(self, base: u32, max: u32) -> Option<(u32, bool)> {
match self {
Part::Absolute(n) => (1..=max).contains(&n).then(|| (n - 1, true)),
Part::Relative(k) => i64::from(base).checked_add(k).and_then(|v| u32::try_from(v).ok()).filter(|v| *v < max).map(|v| (v, false)),
}
}
}
/// Parses the number after `R` or `C` at `i`: digits, `[offset]` or nothing. Returns the part and
/// the index after it, or `None` when the text isn't a reference part.
fn parse_part(chars: &[char], i: usize) -> Option<(Part, usize)> {
match chars.get(i) {
Some('[') => {
let close = chars[i..].iter().position(|&c| c == ']')? + i;
let inner: String = chars[i + 1..close].iter().collect();
Some((Part::Relative(inner.parse().ok()?), close + 1))
}
Some(c) if c.is_ascii_digit() => {
let end = chars[i..].iter().position(|c| !c.is_ascii_digit()).map_or(chars.len(), |p| p + i);
let n: String = chars[i..end].iter().collect();
Some((Part::Absolute(n.parse().unwrap_or(u32::MAX)), end))
}
_ => Some((Part::Relative(0), i)),
}
}
fn is_word_char(c: char) -> bool {
c.is_alphanumeric() || matches!(c, '_' | '.' | '\\' | '?' | '$')
}
/// Rewrites the R1C1 references in a SYLK formula as A1 references for the cell at `at`:
/// `RC[-1]` → `B2`, `R1C1` → `$A$1`, `R[-2]C:R[-1]C` → `D2:D3`, a whole column `C4` → `$D:$D` and
/// a whole row `R3` → `$3:$3`. Text in quotes and quoted sheet names are left alone; a reference
/// off the sheet becomes `#REF!`.
pub(crate) fn r1c1_formula_to_a1(formula: &str, at: CellRef) -> String {
let chars: Vec<char> = formula.chars().collect();
let mut out = String::with_capacity(formula.len());
let mut i = 0;
while i < chars.len() {
let c = chars[i];
if c == '"' || c == '\'' {
// Copy the quoted run as it is; a doubled quote stays inside it.
let mut j = i + 1;
while j < chars.len() && !(chars[j] == c && chars.get(j + 1) != Some(&c)) {
j += if chars[j] == c { 2 } else { 1 };
}
let stop = (j + 1).min(chars.len());
out.extend(&chars[i..stop]);
i = stop;
continue;
}
let starts_word = i == 0 || !is_word_char(chars[i - 1]);
if starts_word
&& matches!(c, 'R' | 'r' | 'C' | 'c')
&& let Some((text, next)) = reference_at(&chars, i, at)
{
out.push_str(&text);
i = next;
continue;
}
// Copy the rest of a word as it is, so `ROUND` or `Sheet2` isn't read as a reference.
if is_word_char(c) {
let end = (i..chars.len()).find(|&j| !is_word_char(chars[j])).unwrap_or(chars.len());
out.extend(&chars[i..end]);
i = end;
} else {
out.push(c);
i += 1;
}
}
out
}
/// The A1 text of the R1C1 reference starting at `i`, and the index after it, or `None` when the
/// word there isn't a reference (a function or a name).
fn reference_at(chars: &[char], i: usize, at: CellRef) -> Option<(String, usize)> {
let mut j = i;
let mut row = None;
let mut col = None;
if matches!(chars.get(j), Some('R' | 'r')) {
let (part, next) = parse_part(chars, j + 1)?;
row = Some(part);
j = next;
}
if matches!(chars.get(j), Some('C' | 'c')) {
let (part, next) = parse_part(chars, j + 1)?;
col = Some(part);
j = next;
}
if row.is_none() && col.is_none() {
return None;
}
// A reference ends the word: `ROUND(`, `RATE` or `R2D2` are names.
if chars.get(j).is_some_and(|&c| is_word_char(c) || c == '(' || c == '[' || c == '!') {
return None;
}
let row_text = |p: Part| p.resolve(at.row, MAX_ROWS).map(|(r, abs)| format!("{}{}", if abs { "$" } else { "" }, r + 1));
let col_text = |p: Part| p.resolve(at.col, MAX_COLS).map(|(c, abs)| format!("{}{}", if abs { "$" } else { "" }, col_to_letters(c)));
// One side of a range (`R2:R4`, `C1:C3`) is written alone; a lone row or column is the whole of it.
let prev = chars[..i].iter().rev().find(|c| !c.is_whitespace());
let in_range = prev == Some(&':') || chars[j..].iter().find(|c| !c.is_whitespace()) == Some(&':');
let text = match (row, col) {
(Some(r), Some(c)) => row_text(r).zip(col_text(c)).map(|(r, c)| format!("{c}{r}")),
(Some(r), None) => row_text(r).map(|r| if in_range { r } else { format!("{r}:{r}") }),
(None, Some(c)) => col_text(c).map(|c| if in_range { c } else { format!("{c}:{c}") }),
(None, None) => None,
};
Some((text.unwrap_or_else(|| "#REF!".into()), j))
}
crates/engine/src/io/tests.rs
+1 -1
View File
@@ -763,7 +763,7 @@ fn sylk_r1c1_formulas_become_a1() {
("R[-5]C", "#REF!"),
("\"RC\"&R1C1", "\"RC\"&$A$1"),
] {
assert_eq!(sylk::r1c1_formula_to_a1(r1c1, b2), a1, "{r1c1}");
assert_eq!(craft_csv::sylk::r1c1_formula_to_a1(r1c1, (b2.row, b2.col)), a1, "{r1c1}");
}
}
crates/xlsx/Cargo.toml
+1
View File
@@ -18,6 +18,7 @@ gridcraft-core = { workspace = true }
gridcraft-formula = { workspace = true }
gridcraft-math = { workspace = true }
gridcraft-model = { workspace = true }
craft-csv = { workspace = true }
craft-xls = { workspace = true }
thiserror = { workspace = true }
quick-xml = { workspace = true }
crates/xlsx/src/csv.rs
+10 -251
View File
@@ -1,4 +1,6 @@
//! CSV import and export (RFC 4180 with the usual real-world leniency).
//! CSV import and export (RFC 4180 with the usual real-world leniency). Decoding, delimiter
//! sniffing, the record parser and field quoting are `craft-csv`'s; this module turns records into
//! typed cells (with each column's date order) and writes a sheet.
use std::sync::Arc;
@@ -12,241 +14,11 @@ use crate::{CsvOptions, IoError, ReadReport};
/// The largest CSV or tab-separated file written (bytes).
pub const MAX_CSV_BYTES: usize = 512 << 20;
/// Windows-1252 code points for bytes 0x80–0x9F (others map to Latin-1).
const CP1252: [u32; 32] = [
0x20AC, 0x81, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x8D, 0x017D, 0x8F, 0x90, 0x2018, 0x2019,
0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178,
];
pub use craft_csv::{decode_text, sniff_delimiter};
/// Windows-1256 code points for bytes 0x80–0xFF (unicode.org `CP1256.TXT`).
pub(crate) const CP1256: [u32; 128] = [
0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018,
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3,
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638,
0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E,
0x200F, 0x06D2,
];
/// Evidence that bytes which aren't UTF-8 are Windows-1256 Arabic rather than Windows-1252.
#[derive(Default)]
struct Cp1256Evidence {
/// High bytes that decode to different characters in the two code pages, apart from invisible
/// marks and ordinals after a digit.
differing: usize,
/// Of those, Arabic letters and Arabic punctuation (، ؛ ؟) in Windows-1256.
arabic: usize,
/// Arabic bytes next to an ASCII letter: Western accents sit inside Latin words, Arabic doesn't.
arabic_touching_latin: usize,
/// Accented Latin bytes the two code pages share, next to another high byte: Hebrew
/// Windows-1255 letters land there, Arabic and Western text rarely do.
shared_letters_in_runs: usize,
}
impl Cp1256Evidence {
fn of(bytes: &[u8]) -> Self {
let mut e = Self::default();
for (i, &b) in bytes.iter().enumerate() {
if b < 0x80 {
continue;
}
let before = i.checked_sub(1).and_then(|j| bytes.get(j)).copied();
let after = bytes.get(i + 1).copied();
let next_to = |pred: fn(u8) -> bool| before.is_some_and(pred) || after.is_some_and(pred);
if same_in_cp1252_and_cp1256(b) {
if is_shared_latin_letter(b) && next_to(|x| x >= 0x80) {
e.shared_letters_in_runs += 1;
}
continue;
}
if is_invisible_mark(b) || is_ordinal_after_digit(b, before) {
continue;
}
e.differing += 1;
if is_cp1256_arabic(b) {
e.arabic += 1;
if next_to(|x| x.is_ascii_alphabetic()) {
e.arabic_touching_latin += 1;
}
}
}
e
}
/// Thresholds chosen against month, day and language names from 95 glibc locales (Western in
/// Windows-1252, Arabic, Persian and Urdu in 1256, Hebrew and Yiddish in 1255) plus CSV samples.
fn is_arabic(&self) -> bool {
self.arabic >= 2
&& self.arabic * 10 >= self.differing * 9
&& self.arabic_touching_latin * 10 <= self.arabic * 3
&& self.shared_letters_in_runs * 20 <= self.arabic
}
}
fn same_in_cp1252_and_cp1256(b: u8) -> bool {
matches!(
b,
0x80 | 0x82..=0x89 | 0x8B..=0x8C | 0x91..=0x97 | 0x99 | 0x9B..=0x9C | 0xA0 | 0xA2..=0xA9 | 0xAB..=0xB9 | 0xBB..=0xBE | 0xD7 | 0xE0 | 0xE2 | 0xE7..=0xEB
| 0xEE..=0xEF | 0xF4 | 0xF7 | 0xF9 | 0xFB..=0xFC
)
}
/// ZWNJ, ZWJ, LRM and RLM: common inside Persian words and around Arabic cells, but no evidence
/// either way.
fn is_invisible_mark(b: u8) -> bool {
matches!(b, 0x9D | 0x9E | 0xFD | 0xFE)
}
/// `º` and `ª` after a digit (`1º`, `2ª`) are Western ordinals, though Windows-1256 reads the bytes
/// as `؛` and `ھ`; either way they are no evidence.
fn is_ordinal_after_digit(b: u8, before: Option<u8>) -> bool {
matches!(b, 0xAA | 0xBA) && before.is_some_and(|x| x.is_ascii_digit())
}
fn is_shared_latin_letter(b: u8) -> bool {
matches!(b, 0xE0 | 0xE2 | 0xE7..=0xEB | 0xEE..=0xEF | 0xF4 | 0xF9 | 0xFB..=0xFC)
}
fn is_cp1256_arabic(b: u8) -> bool {
matches!(
b,
0x81 | 0x8A | 0x8D..=0x90 | 0x98 | 0x9A | 0x9F | 0xA1 | 0xAA | 0xBA | 0xBF..=0xD6 | 0xD8..=0xDB | 0xDD..=0xDF | 0xE1 | 0xE3..=0xE6 | 0xEC..=0xED | 0xFF
)
}
/// Decodes CSV bytes with BOM-aware UTF-8 and UTF-16 handling. Without a BOM, valid UTF-8 is kept.
/// Other bytes decode as Windows-1256 when they read as Arabic (see [`Cp1256Evidence`]), else as
/// Windows-1252.
pub fn decode_text(bytes: &[u8]) -> String {
if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) {
return String::from_utf8_lossy(rest).into_owned();
}
let utf16 = |b: &[u8], le: bool| -> String {
let units = b.as_chunks::<2>().0.iter().map(|c| if le { u16::from_le_bytes(*c) } else { u16::from_be_bytes(*c) });
char::decode_utf16(units).map(|r| r.unwrap_or('\u{FFFD}')).collect()
};
if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) {
return utf16(rest, true);
}
if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) {
return utf16(rest, false);
}
match std::str::from_utf8(bytes) {
Ok(s) => s.to_string(),
Err(_) => {
let use_windows_1256 = Cp1256Evidence::of(bytes).is_arabic();
bytes
.iter()
.map(|&b| match b {
0x80..=0xFF if use_windows_1256 => {
char::from_u32(CP1256.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}')
}
0x80..=0x9F => char::from_u32(CP1252.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}'),
_ => b as char,
})
.collect()
}
}
}
/// Picks comma, semicolon or tab: the candidate with the most consistent count per line
/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins.
pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
let mut best = (b',', 0usize, 0usize);
for d in [',', ';', '\t'] {
let mut counts = vec![];
let mut n = 0usize;
let mut in_q = false;
let mut blank = true;
for ch in text.chars().take(64 * 1024) {
if !in_q && ch == '\n' {
// Blank lines carry no delimiter evidence.
if !blank {
counts.push(n);
}
n = 0;
blank = true;
if counts.len() >= 20 {
break;
}
continue;
}
if ch != '\r' {
blank = false;
}
if ch == quote {
in_q = !in_q;
} else if !in_q && ch == d {
n += 1;
}
}
if !blank {
counts.push(n);
}
let min = counts.iter().copied().min().unwrap_or(0);
let total: usize = counts.iter().sum();
if min > 0 && (min, total) > (best.1, best.2) {
best = (d as u8, min, total);
}
}
best.0
}
/// Splits CSV text into records of fields. A record keeps at most [`MAX_COLS`] fields; the rest are
/// dropped as they are read, so a row of endless delimiters cannot allocate a string per field.
pub(crate) fn parse_records(text: &str, delim: char, quote: char, mut on_record: impl FnMut(Vec<String>) -> bool) {
let mut field = String::new();
let mut record: Vec<String> = vec![];
let mut in_q = false;
let mut chars = text.chars().peekable();
let mut field_started_quoted = false;
while let Some(c) = chars.next() {
if in_q {
if c == quote {
if chars.peek() == Some(&quote) {
field.push(quote);
chars.next();
} else {
in_q = false;
}
} else {
field.push(c);
}
continue;
}
if c == quote && field.is_empty() && !field_started_quoted {
in_q = true;
field_started_quoted = true;
} else if c == delim {
let f = std::mem::take(&mut field);
if record.len() < MAX_COLS as usize {
record.push(f);
}
field_started_quoted = false;
} else if c == '\r' || c == '\n' {
if c == '\r' && chars.peek() == Some(&'\n') {
chars.next();
}
let f = std::mem::take(&mut field);
if record.len() < MAX_COLS as usize {
record.push(f);
}
field_started_quoted = false;
if !on_record(std::mem::take(&mut record)) {
return;
}
} else {
field.push(c);
}
}
if !field.is_empty() || !record.is_empty() || field_started_quoted {
if record.len() < MAX_COLS as usize {
record.push(field);
}
on_record(record);
}
/// [`craft_csv::parse_records`] with a record kept to the grid's [`MAX_COLS`] fields.
fn parse_records(text: &str, delim: char, quote: char, on_record: impl FnMut(Vec<String>) -> bool) {
craft_csv::parse_records(text, delim, quote, MAX_COLS as usize, on_record);
}
/// How many rows and bytes of a text file are read to decide each column's date order.
@@ -587,25 +359,12 @@ pub fn write_csv_limited(sheet: &Sheet, delimiter: u8, limit: usize, mut cell_te
out.push(delim);
}
let t = cell_text(CellRef::new(row, col));
let quoted = t.contains(delim) || t.contains('"') || t.contains('\n') || t.contains('\r');
// The exact size of the field as written (the delimiter was pushed above), so a row
// never grows past the cap unseen and no doubled-quote copy of the text is built.
let field_len = if quoted { t.len() + t.matches('"').count() + 2 } else { t.len() };
if out.len().saturating_add(field_len) > limit {
// never grows past the cap unseen.
if out.len().saturating_add(craft_csv::field_len(&t, delim)) > limit {
return Err(too_large(format!("more than {} MiB (stopped at row {})", limit >> 20, row + 1)));
}
if quoted {
out.push('"');
for ch in t.chars() {
if ch == '"' {
out.push('"');
}
out.push(ch);
}
out.push('"');
} else {
out.push_str(&t);
}
craft_csv::push_field(&mut out, &t, delim);
}
out.push_str("\r\n");
if out.len() > limit {
crates/xlsx/src/tests/csv_tests.rs
+3 -30
View File
@@ -1,8 +1,8 @@
use gridcraft_core::{CellRef, DateSystem, MAX_COLS, Value};
use gridcraft_model::{Sheet, Workbook};
use crate::csv::{decode_text, parse_records, sniff_delimiter};
use crate::{CsvOptions, Format, IoError, read_csv, read_csv_with_report, sniff, write_csv, write_csv_limited};
use craft_csv::{decode_text, parse_records, sniff_delimiter};
fn v(wb: &Workbook, r: u32, c: u32) -> Value {
wb.sheet(0).unwrap().value(CellRef::new(r, c))
@@ -148,33 +148,6 @@ fn encodings() {
assert_eq!(v(&wb, 0, 1), Value::Number(1.0));
}
#[test]
fn windows_1256_table_matches_the_unicode_mapping() {
let at = |b: u8| crate::csv::CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c));
let expected = [
(0x80, '€'),
(0x81, 'پ'),
(0x8D, 'چ'),
(0x98, 'ک'),
(0x9D, '\u{200C}'),
(0xA1, '،'),
(0xBA, '؛'),
(0xBF, '؟'),
(0xC7, 'ا'),
(0xC8, 'ب'),
(0xDC, 'ـ'),
(0xE1, 'ل'),
(0xE3, 'م'),
(0xE9, 'é'),
(0xFD, '\u{200E}'),
(0xFE, '\u{200F}'),
(0xFF, 'ے'),
];
for (b, c) in expected {
assert_eq!(at(b), Some(c), "0x{b:02X}");
}
}
#[test]
fn arabic_csv_in_windows_1256_reads_as_arabic() {
let bytes = [
@@ -393,14 +366,14 @@ fn parser_stops_collecting_fields_at_max_cols() {
let mut text = ",".repeat((MAX_COLS + extra) as usize - 1);
text.push_str("end\nx,y\n");
let mut lens = vec![];
parse_records(&text, ',', '"', |rec| {
parse_records(&text, ',', '"', MAX_COLS as usize, |rec| {
lens.push(rec.len());
true
});
assert_eq!(lens, vec![MAX_COLS as usize, 2]);
// An unterminated last row is capped the same way.
let mut lens = vec![];
parse_records(&",".repeat(MAX_COLS as usize + extra as usize), ',', '"', |rec| {
parse_records(&",".repeat(MAX_COLS as usize + extra as usize), ',', '"', MAX_COLS as usize, |rec| {
lens.push(rec.len());
true
});