mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 21:52:32 +00:00
Merge storytold main into next (17 Nov sync)
storytold main 83cb1133dd (291 commits) into the integration head bab1a99b85; 47 conflicted files.
- Domain test files (metafile, prepress, pdf plates, settings store, heif, media-testkit, atomic): tests next cut stay cut; tests new on main are kept, as trials where the crate's test target runs trials.
- atomic: main's group-bits test and Faulty::set_group go to src/tests.rs. audio: main's GSM and SBC oracles join tests/main.rs. cms constructors, pdf knockout_backdrop and testing_pdf join their crates' one test binary; timeline's preflight tests become src/preflight_tests.rs trials.
- update: main's changes to detect, install (linux, macos) and the tests land in next's split modules. The app's own offer is inspected on a thread of its own, picked up by tick's polling; settle_for_tests settles offers too. Later on a ready update drops the close it asked for. updater_pin_tests.rs stays deleted; its new pin test is in tests/it/update.rs.
- craft-color: main added a different crate at crates/color (the colour tags of compositor and display frames); it is next's craft-color module `tags`, re-exported at the root.
- Old crates/ui (combo, docking, testing, README) and keymap's ui/view.rs stay deleted; main's new crates/ui/src/testing files are dropped with them.
- New crates in the craft facade: ai, control, dds, frame, ipc, media-source, palette, press-print, score, ssip. testkit and xtask are listed outside it.
- Cargo.lock: the union of both locks, resolved by cargo metadata. Workflow: main's nextest Windows leg, with next's --keep-going clippy.
This commit is contained in:
@@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work.
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 |
|
||||
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
|
||||
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; when no candidate finds coherent complete records with a delimiter in each, a candidate that finds no complete record counts the delimiters of the first record up to the end of the sample, or up to a quoted field still open there, and a quote inside a field that does not start with one makes that record incoherent; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
|
||||
| `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text |
|
||||
| `records(text, delim, quote, max_records, max_fields)` | the same, collected |
|
||||
| `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing |
|
||||
|
||||
+160
-66
@@ -34,8 +34,13 @@ pub use decode::{decode_text, decode_utf8};
|
||||
/// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields
|
||||
/// are discarded when another coherent candidate separates the records. Ties prefer tab,
|
||||
/// then comma, then semicolon. Comma is the fallback when no candidate separates the text.
|
||||
/// Only the first 64 KiB are read. A record cut off at the end of that sample is left out,
|
||||
/// except when no candidate finds coherent complete records with a delimiter in each. Then a candidate
|
||||
/// that finds no complete record counts the delimiters read so far in the first record, and that
|
||||
/// record is incoherent when it has a quote inside a field that does not start with one. When one
|
||||
/// of those candidates ends the sample inside a quoted field, each of them counts only up to that
|
||||
/// field's opening quote.
|
||||
pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
let mut candidates = [(b'\t', 0usize, 0usize, true), (b',', 0, 0, true), (b';', 0, 0, true)];
|
||||
const SAMPLE_BYTES: usize = 64 * 1024;
|
||||
let mut sample_end = text.len().min(SAMPLE_BYTES);
|
||||
while !text.is_char_boundary(sample_end) {
|
||||
@@ -43,78 +48,46 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
}
|
||||
let sample = &text[..sample_end];
|
||||
let truncated = sample_end < text.len();
|
||||
for (delimiter, min_count, total, coherent) in &mut candidates {
|
||||
let mut counts = Vec::new();
|
||||
let mut count = 0usize;
|
||||
let mut quoted = false;
|
||||
let mut after_quote = false;
|
||||
let mut field_start = true;
|
||||
let mut record_has_content = false;
|
||||
let mut record_coherent = true;
|
||||
let mut candidate_coherent = true;
|
||||
let mut reached_record_limit = false;
|
||||
let mut chars = sample.chars().peekable();
|
||||
while let Some(ch) = chars.next() {
|
||||
if quoted {
|
||||
if ch == quote {
|
||||
if chars.peek() == Some("e) {
|
||||
chars.next();
|
||||
record_has_content = true;
|
||||
} else {
|
||||
quoted = false;
|
||||
after_quote = true;
|
||||
}
|
||||
} else if !ch.is_whitespace() {
|
||||
record_has_content = true;
|
||||
}
|
||||
let mut scans = b"\t,;".map(|delimiter| scan_delimiter(sample, delimiter, quote));
|
||||
if truncated {
|
||||
if scans.iter().any(|scan| scan.coherent && scan.counts.iter().min().is_some_and(|&min| min > 0)) {
|
||||
return pick_delimiter(scans);
|
||||
}
|
||||
// Every quote in the scans is at a char boundary, so the cut is one too.
|
||||
let cut = scans.iter().filter(|scan| scan.counts.is_empty()).filter_map(|scan| scan.open_quote).min().unwrap_or(sample_end);
|
||||
for scan in &mut scans {
|
||||
if !scan.counts.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if ch == '\r' || ch == '\n' {
|
||||
if ch == '\r' && chars.peek() == Some(&'\n') {
|
||||
chars.next();
|
||||
}
|
||||
if record_has_content {
|
||||
counts.push(count);
|
||||
candidate_coherent &= record_coherent;
|
||||
if counts.len() >= 20 {
|
||||
reached_record_limit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
count = 0;
|
||||
quoted = false;
|
||||
after_quote = false;
|
||||
field_start = true;
|
||||
record_has_content = false;
|
||||
record_coherent = true;
|
||||
if cut < sample_end {
|
||||
let before_cut = scan_delimiter(&sample[..cut], scan.delimiter, quote);
|
||||
(scan.count, scan.record_has_content, scan.record_coherent, scan.record_bare_quote) =
|
||||
(before_cut.count, before_cut.record_has_content, before_cut.record_coherent, before_cut.record_bare_quote);
|
||||
}
|
||||
if scan.record_has_content {
|
||||
scan.coherent &= scan.record_coherent && !scan.record_bare_quote;
|
||||
scan.counts.push(scan.count);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for scan in &mut scans {
|
||||
if scan.reached_record_limit {
|
||||
continue;
|
||||
}
|
||||
if ch == quote && field_start {
|
||||
quoted = true;
|
||||
field_start = false;
|
||||
record_has_content = true;
|
||||
} else if ch == char::from(*delimiter) {
|
||||
count += 1;
|
||||
field_start = true;
|
||||
after_quote = false;
|
||||
} else {
|
||||
record_coherent &= !after_quote || ch.is_whitespace();
|
||||
record_has_content |= !ch.is_whitespace();
|
||||
field_start = false;
|
||||
if scan.open_quote.is_some() {
|
||||
scan.coherent = false;
|
||||
} else if scan.record_has_content {
|
||||
scan.coherent &= scan.record_coherent;
|
||||
scan.counts.push(scan.count);
|
||||
}
|
||||
}
|
||||
if !reached_record_limit && !truncated {
|
||||
if quoted {
|
||||
candidate_coherent = false;
|
||||
} else if record_has_content {
|
||||
candidate_coherent &= record_coherent;
|
||||
counts.push(count);
|
||||
}
|
||||
}
|
||||
*coherent &= candidate_coherent;
|
||||
*min_count = counts.iter().copied().min().unwrap_or(0);
|
||||
*total = counts.iter().sum();
|
||||
}
|
||||
pick_delimiter(scans)
|
||||
}
|
||||
|
||||
fn pick_delimiter(scans: [DelimiterScan; 3]) -> u8 {
|
||||
let candidates =
|
||||
scans.map(|scan| (scan.delimiter, scan.counts.iter().copied().min().unwrap_or(0), scan.counts.iter().sum::<usize>(), scan.coherent));
|
||||
let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent);
|
||||
let mut best = (b',', 0usize, 0usize);
|
||||
for (delimiter, min, total, coherent) in candidates {
|
||||
@@ -125,6 +98,93 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
best.0
|
||||
}
|
||||
|
||||
/// One candidate delimiter's pass over the sample in [`sniff_delimiter`].
|
||||
struct DelimiterScan {
|
||||
delimiter: u8,
|
||||
/// Delimiter counts of the complete non-blank records, at most 20.
|
||||
counts: Vec<usize>,
|
||||
/// Whether every counted record had coherent quoted fields.
|
||||
coherent: bool,
|
||||
reached_record_limit: bool,
|
||||
/// The record still open at the end of the sample: its delimiter count so far, whether it
|
||||
/// has content, and whether its quoted fields so far are coherent.
|
||||
count: usize,
|
||||
record_has_content: bool,
|
||||
record_coherent: bool,
|
||||
/// Whether the open record has a quote inside a field that does not start with one.
|
||||
record_bare_quote: bool,
|
||||
/// The byte offset of the quote that opens a field still unterminated at the end.
|
||||
open_quote: Option<usize>,
|
||||
}
|
||||
|
||||
fn scan_delimiter(sample: &str, delimiter: u8, quote: char) -> DelimiterScan {
|
||||
let mut scan = DelimiterScan {
|
||||
delimiter,
|
||||
counts: Vec::new(),
|
||||
coherent: true,
|
||||
reached_record_limit: false,
|
||||
count: 0,
|
||||
record_has_content: false,
|
||||
record_coherent: true,
|
||||
record_bare_quote: false,
|
||||
open_quote: None,
|
||||
};
|
||||
let mut after_quote = false;
|
||||
let mut field_start = true;
|
||||
let mut chars = sample.char_indices().peekable();
|
||||
while let Some((at, ch)) = chars.next() {
|
||||
if scan.open_quote.is_some() {
|
||||
if ch == quote {
|
||||
if chars.peek().map(|&(_, next)| next) == Some(quote) {
|
||||
chars.next();
|
||||
scan.record_has_content = true;
|
||||
} else {
|
||||
scan.open_quote = None;
|
||||
after_quote = true;
|
||||
}
|
||||
} else if !ch.is_whitespace() {
|
||||
scan.record_has_content = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ch == '\r' || ch == '\n' {
|
||||
if ch == '\r' && chars.peek().map(|&(_, next)| next) == Some('\n') {
|
||||
chars.next();
|
||||
}
|
||||
if scan.record_has_content {
|
||||
scan.counts.push(scan.count);
|
||||
scan.coherent &= scan.record_coherent;
|
||||
if scan.counts.len() >= 20 {
|
||||
scan.reached_record_limit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
scan.count = 0;
|
||||
after_quote = false;
|
||||
field_start = true;
|
||||
scan.record_has_content = false;
|
||||
scan.record_coherent = true;
|
||||
scan.record_bare_quote = false;
|
||||
continue;
|
||||
}
|
||||
if ch == quote && field_start {
|
||||
scan.open_quote = Some(at);
|
||||
field_start = false;
|
||||
scan.record_has_content = true;
|
||||
} else if ch == char::from(delimiter) {
|
||||
scan.count += 1;
|
||||
field_start = true;
|
||||
after_quote = false;
|
||||
} else {
|
||||
scan.record_coherent &= !after_quote || ch.is_whitespace();
|
||||
scan.record_bare_quote |= ch == quote;
|
||||
scan.record_has_content |= !ch.is_whitespace();
|
||||
field_start = false;
|
||||
}
|
||||
}
|
||||
scan
|
||||
}
|
||||
|
||||
/// Splits delimited text into records of fields and hands each to `on_record`, which returns
|
||||
/// whether to go on. A record ends at CR, LF or CRLF; a blank line is a record with one empty
|
||||
/// field. A field that starts with `quote` runs to the closing quote (a doubled quote is one
|
||||
@@ -309,6 +369,40 @@ mod tests {
|
||||
assert_eq!(sniff_delimiter(&text, '"'), b'\t');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimiter_sniffing_counts_a_first_record_longer_than_the_sample() {
|
||||
let wide = |field: &str, delimiter: char| {
|
||||
let mut record = String::new();
|
||||
while record.len() <= 64 * 1024 {
|
||||
record.push_str(field);
|
||||
record.push(delimiter);
|
||||
}
|
||||
record
|
||||
};
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n1;2\n", wide("cell", ';')), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n1\t2\n", wide("cell", '\t')), '"'), b'\t');
|
||||
// Delimiters inside quoted fields of the long record are text.
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a,b\tc\"", ';')), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a;b\tc\"", ',')), '"'), b',');
|
||||
// A candidate that reads a quoted field as text inside an unquoted one is incoherent.
|
||||
assert_eq!(sniff_delimiter(&format!("a,\"{}\",{}", "x;y".repeat(20_000), "z".repeat(10_000)), '"'), b',');
|
||||
// The sample ends inside a quoted field: the delimiters before that field count.
|
||||
let text = format!("a;b;c;\"{}\"\n1;2;3;4\n", "long text ".repeat(8_000));
|
||||
assert_eq!(sniff_delimiter(&text, '"'), b';');
|
||||
// Candidates without a complete record do not count delimiters after that quote, also
|
||||
// when they read it as text.
|
||||
assert_eq!(sniff_delimiter(&format!("a,\"{}\"\n1,2\n", "x;y".repeat(30_000)), '"'), b',');
|
||||
assert_eq!(sniff_delimiter(&format!("a;b;\"{}\"\n1;2;3\n", "x,y\t".repeat(20_000)), '"'), b';');
|
||||
// A line break inside the open quoted field gives the other candidates a record without
|
||||
// a delimiter, which does not stop the first record from counting.
|
||||
assert_eq!(sniff_delimiter(&format!("a;b;\"hello\n{}\"\n", "x".repeat(70_000)), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("\"a\";b;\"hello,\n{}\"\n", "x".repeat(70_000)), '"'), b';');
|
||||
// When a candidate finds coherent complete records with a delimiter in each, a record cut
|
||||
// off at the end is left out.
|
||||
assert_eq!(sniff_delimiter(&format!("a;\"hello,\";b\n{}", "x;y;z\n".repeat(12_000)), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("a,b,c;\"hello,\";d\n{}", "x;y;z\n".repeat(12_000)), '"'), b';');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_stops_collecting_fields_at_the_cap() {
|
||||
let (max, extra) = (16_384usize, 50);
|
||||
|
||||
+147
-3
@@ -219,12 +219,13 @@ fn position(arg: &str, max: u32) -> Option<u32> {
|
||||
arg.trim().parse::<u32>().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1)
|
||||
}
|
||||
|
||||
/// A `K` value: `"text"` (everything between the first and the last quote, as written), a
|
||||
/// boolean, a number, or other text (an error value when it starts with `#`).
|
||||
/// A `K` value: `"text"` (everything between the first and the last quote, with Excel's
|
||||
/// character escapes decoded), a boolean, a number, or other text (an error value when it starts
|
||||
/// with `#`).
|
||||
fn parse_value(arg: &str) -> Value {
|
||||
if let Some(text) = arg.strip_prefix('"') {
|
||||
let text = text.strip_suffix('"').unwrap_or(text);
|
||||
return Value::Text(text.to_string());
|
||||
return Value::Text(decode_escapes(text));
|
||||
}
|
||||
if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") {
|
||||
return Value::Bool(arg.eq_ignore_ascii_case("TRUE"));
|
||||
@@ -238,6 +239,149 @@ fn parse_value(arg: &str) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// ISO 6937's diacritics (codes 0xC1–0xCF): each code, its combining mark, and the letters it
|
||||
/// composes with, as pairs of letter and precomposed letter (from Unicode's canonical composition).
|
||||
/// A diacritic on a SPACE is the spacing accent (`´`, `¨`, `ˇ`, ...), as in glibc's ISO_6937 map; the
|
||||
/// grave, circumflex and tilde have none, since `` ` ``, `^` and `~` are plain ASCII.
|
||||
const ISO6937_DIACRITICS: [(u8, char, &str); 13] = [
|
||||
(0xC1, '\u{0300}', "AÀaàEÈeèIÌiìNǸnǹOÒoòUÙuùWẀwẁYỲyỳ"), // grave
|
||||
(0xC2, '\u{0301}', " ´AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute
|
||||
(0xC3, '\u{0302}', "AÂaâCĈcĉEÊeêGĜgĝHĤhĥIÎiîJĴjĵOÔoôSŜsŝUÛuûWŴwŵYŶyŷZẐzẑ"), // circumflex
|
||||
(0xC4, '\u{0303}', "AÃaãEẼeẽIĨiĩNÑnñOÕoõUŨuũVṼvṽYỸyỹ"), // tilde
|
||||
(0xC5, '\u{0304}', " ¯AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron
|
||||
(0xC6, '\u{0306}', " ˘AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve
|
||||
(0xC7, '\u{0307}', " ˙AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above
|
||||
(0xC8, '\u{0308}', " ¨AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis
|
||||
(0xCA, '\u{030a}', " ˚AÅaåUŮuůwẘyẙ"), // ring
|
||||
(0xCB, '\u{0327}', " ¸CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla
|
||||
(0xCD, '\u{030b}', " ˝OŐoőUŰuű"), // double acute
|
||||
(0xCE, '\u{0328}', " ˛AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek
|
||||
(0xCF, '\u{030c}', " ˇAǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron
|
||||
];
|
||||
|
||||
/// ISO 6937's spacing characters above ASCII (codes 0xA1–0xFE that aren't diacritics).
|
||||
const ISO6937_SPACING: [(u8, char); 71] = [
|
||||
(0xA1, '¡'),
|
||||
(0xA2, '¢'),
|
||||
(0xA3, '£'),
|
||||
(0xA5, '¥'),
|
||||
(0xA7, '§'),
|
||||
(0xA8, '¤'),
|
||||
(0xA9, '‘'),
|
||||
(0xAA, '“'),
|
||||
(0xAB, '«'),
|
||||
(0xAC, '←'),
|
||||
(0xAD, '↑'),
|
||||
(0xAE, '→'),
|
||||
(0xAF, '↓'),
|
||||
(0xB0, '°'),
|
||||
(0xB1, '±'),
|
||||
(0xB2, '²'),
|
||||
(0xB3, '³'),
|
||||
(0xB4, '×'),
|
||||
(0xB5, 'µ'),
|
||||
(0xB6, '¶'),
|
||||
(0xB7, '·'),
|
||||
(0xB8, '÷'),
|
||||
(0xB9, '’'),
|
||||
(0xBA, '”'),
|
||||
(0xBB, '»'),
|
||||
(0xBC, '¼'),
|
||||
(0xBD, '½'),
|
||||
(0xBE, '¾'),
|
||||
(0xBF, '¿'),
|
||||
(0xD0, '―'),
|
||||
(0xD1, '¹'),
|
||||
(0xD2, '®'),
|
||||
(0xD3, '©'),
|
||||
(0xD4, '™'),
|
||||
(0xD5, '♪'),
|
||||
(0xD6, '¬'),
|
||||
(0xD7, '¦'),
|
||||
(0xDC, '⅛'),
|
||||
(0xDD, '⅜'),
|
||||
(0xDE, '⅝'),
|
||||
(0xDF, '⅞'),
|
||||
(0xE0, 'Ω'),
|
||||
(0xE1, 'Æ'),
|
||||
(0xE2, 'Đ'),
|
||||
(0xE3, 'ª'),
|
||||
(0xE4, 'Ħ'),
|
||||
(0xE6, 'IJ'),
|
||||
(0xE7, 'Ŀ'),
|
||||
(0xE8, 'Ł'),
|
||||
(0xE9, 'Ø'),
|
||||
(0xEA, 'Œ'),
|
||||
(0xEB, 'º'),
|
||||
(0xEC, 'Þ'),
|
||||
(0xED, 'Ŧ'),
|
||||
(0xEE, 'Ŋ'),
|
||||
(0xEF, 'ʼn'),
|
||||
(0xF0, 'ĸ'),
|
||||
(0xF1, 'æ'),
|
||||
(0xF2, 'đ'),
|
||||
(0xF3, 'ð'),
|
||||
(0xF4, 'ħ'),
|
||||
(0xF5, 'ı'),
|
||||
(0xF6, 'ij'),
|
||||
(0xF7, 'ŀ'),
|
||||
(0xF8, 'ł'),
|
||||
(0xF9, 'ø'),
|
||||
(0xFA, 'œ'),
|
||||
(0xFB, 'ß'),
|
||||
(0xFC, 'þ'),
|
||||
(0xFD, 'ŧ'),
|
||||
(0xFE, 'ŋ'),
|
||||
];
|
||||
|
||||
/// Text with Excel's SYLK character escapes decoded. Excel writes a character outside ASCII as
|
||||
/// `ESC N` and one byte, the character's ISO 6937 code less 0x80. A diacritic's code is followed
|
||||
/// by the letter it goes on (`ESC N H o` is `ö`, `ESC N {` is `ß`). A code ISO 6937 doesn't define,
|
||||
/// or an `ESC` that isn't followed by `N` and a code, is kept as written.
|
||||
fn decode_escapes(text: &str) -> String {
|
||||
if !text.contains('\u{1b}') {
|
||||
return text.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut chars = text.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
let mut ahead = chars.clone();
|
||||
let code = (c == '\u{1b}' && ahead.next() == Some('N'))
|
||||
.then(|| ahead.next())
|
||||
.flatten()
|
||||
.and_then(|b| u8::try_from(b).ok())
|
||||
.filter(|b| (0x20..0x80).contains(b));
|
||||
let Some(code) = code.map(|b| b + 0x80) else {
|
||||
out.push(c);
|
||||
continue;
|
||||
};
|
||||
if let Some(&(_, mark, pairs)) = ISO6937_DIACRITICS.iter().find(|(d, _, _)| *d == code) {
|
||||
// The letter it goes on: precomposed where Unicode has the pair, else letter and mark.
|
||||
// The base is an ordinary character: another escape (or any control character) after a
|
||||
// diacritic leaves this escape as written, and the next one is decoded on its own.
|
||||
let Some(base) = ahead.next().filter(|b| !b.is_control()) else {
|
||||
out.push(c);
|
||||
continue;
|
||||
};
|
||||
let letters: Vec<char> = pairs.chars().collect();
|
||||
match letters.chunks(2).find(|p| p.first() == Some(&base)).and_then(|p| p.get(1)) {
|
||||
Some(&composed) => out.push(composed),
|
||||
None => {
|
||||
out.push(base);
|
||||
out.push(mark);
|
||||
}
|
||||
}
|
||||
} else if let Some(&(_, ch)) = ISO6937_SPACING.iter().find(|(s, _)| *s == code) {
|
||||
out.push(ch);
|
||||
} else {
|
||||
out.push(c);
|
||||
continue;
|
||||
}
|
||||
chars = ahead;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or
|
||||
/// nothing (the cell's own row or column).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
|
||||
@@ -115,3 +115,36 @@ fn survives_huge_relative_offsets_and_width_floods() {
|
||||
let _ = read(&b);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_excels_character_escapes_in_text() {
|
||||
// storytold/gridcraft#4321: Excel writes `Größe für Köln` with `ESC N` escapes (ISO 6937 codes
|
||||
// less 0x80: `H` is the diaeresis, put on the next letter; `{` is ß).
|
||||
let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\
|
||||
C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\
|
||||
C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\
|
||||
C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nC;Y8;X1;K\"\u{1b}NH\u{1b}NHo \u{1b}NB\tx\"\r\n\
|
||||
C;Y9;X1;K\"\u{1b}NB |\u{1b}NE |\u{1b}NF |\u{1b}NG |\u{1b}NH |\u{1b}NJ |\u{1b}NK |\u{1b}NM |\u{1b}NN |\u{1b}NO |\u{1b}NA |\u{1b}NC \"\r\nE\r\n";
|
||||
let sh = read(file.as_bytes()).unwrap();
|
||||
let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default();
|
||||
let text = |s: &str| Value::Text(s.into());
|
||||
assert_eq!(value("A1"), text("Größe für Köln"));
|
||||
assert_eq!(value("A2"), text("Plain ASCII 123"), "plain ASCII is unchanged");
|
||||
// Acute, grave, circumflex, cedilla, caron, double acute, ring, tilde.
|
||||
assert_eq!(value("A3"), text("é à â ç š ő å ñ"));
|
||||
// A letter the mark has no precomposed form with keeps the combining mark.
|
||||
assert_eq!(value("A4"), text("à Z\u{300}"));
|
||||
// Spacing characters, one code each: `q` æ, `y` ø, `{` ß, `a` Æ, `i` Ø.
|
||||
assert_eq!(value("A5"), text("æ øStraße ÆØ"));
|
||||
// A diacritic on a letter with no precomposed form, and a stray `ESC N` before an escape.
|
||||
assert_eq!(value("A6"), text("q\u{308} \u{1b}Nö end\u{1b}N"));
|
||||
// Codes ISO 6937 doesn't define (0xFF, 0xCC) and an `ESC` without `N` stay as written.
|
||||
assert_eq!(value("A7"), text("\u{1b}N\u{7f} \u{1b}NL \u{1b}X"));
|
||||
// A diacritic followed by another escape (or a control character) is kept as written, and the
|
||||
// next escape still decodes.
|
||||
assert_eq!(value("A8"), text("\u{1b}NHö \u{1b}NB\tx"));
|
||||
// A diacritic on a SPACE is the spacing accent: acute, macron, breve, dot above, diaeresis, ring,
|
||||
// cedilla, double acute, ogonek, caron. The grave and circumflex have none (`` ` `` and `^` are
|
||||
// ASCII), so they stay a space with the combining mark.
|
||||
assert_eq!(value("A9"), text("´|¯|˘|˙|¨|˚|¸|˝|˛|ˇ| \u{300}| \u{302}"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user