Merge pull request #1920 from dotCooCoo/fix/csv-sniff-wide-first-record

fix(csv): count a first record longer than the delimiter sample
This commit is contained in:
Dylan M. Taylor
2026-11-16 19:12:46 -05:00
committed by GitHub
2 changed files with 161 additions and 67 deletions
+1 -1
View File
@@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work.
| Function | What it does |
|---|---|
| `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 |
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; when no candidate finds coherent complete records with a delimiter in each, a candidate that finds no complete record counts the delimiters of the first record up to the end of the sample, or up to a quoted field still open there, and a quote inside a field that does not start with one makes that record incoherent; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
| `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text |
| `records(text, delim, quote, max_records, max_fields)` | the same, collected |
| `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing |
+160 -66
View File
@@ -34,8 +34,13 @@ pub use decode::{decode_text, decode_utf8};
/// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields
/// are discarded when another coherent candidate separates the records. Ties prefer tab,
/// then comma, then semicolon. Comma is the fallback when no candidate separates the text.
/// Only the first 64 KiB are read. A record cut off at the end of that sample is left out,
/// except when no candidate finds coherent complete records with a delimiter in each. Then a candidate
/// that finds no complete record counts the delimiters read so far in the first record, and that
/// record is incoherent when it has a quote inside a field that does not start with one. When one
/// of those candidates ends the sample inside a quoted field, each of them counts only up to that
/// field's opening quote.
pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
let mut candidates = [(b'\t', 0usize, 0usize, true), (b',', 0, 0, true), (b';', 0, 0, true)];
const SAMPLE_BYTES: usize = 64 * 1024;
let mut sample_end = text.len().min(SAMPLE_BYTES);
while !text.is_char_boundary(sample_end) {
@@ -43,78 +48,46 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
}
let sample = &text[..sample_end];
let truncated = sample_end < text.len();
for (delimiter, min_count, total, coherent) in &mut candidates {
let mut counts = Vec::new();
let mut count = 0usize;
let mut quoted = false;
let mut after_quote = false;
let mut field_start = true;
let mut record_has_content = false;
let mut record_coherent = true;
let mut candidate_coherent = true;
let mut reached_record_limit = false;
let mut chars = sample.chars().peekable();
while let Some(ch) = chars.next() {
if quoted {
if ch == quote {
if chars.peek() == Some(&quote) {
chars.next();
record_has_content = true;
} else {
quoted = false;
after_quote = true;
}
} else if !ch.is_whitespace() {
record_has_content = true;
}
let mut scans = b"\t,;".map(|delimiter| scan_delimiter(sample, delimiter, quote));
if truncated {
if scans.iter().any(|scan| scan.coherent && scan.counts.iter().min().is_some_and(|&min| min > 0)) {
return pick_delimiter(scans);
}
// Every quote in the scans is at a char boundary, so the cut is one too.
let cut = scans.iter().filter(|scan| scan.counts.is_empty()).filter_map(|scan| scan.open_quote).min().unwrap_or(sample_end);
for scan in &mut scans {
if !scan.counts.is_empty() {
continue;
}
if ch == '\r' || ch == '\n' {
if ch == '\r' && chars.peek() == Some(&'\n') {
chars.next();
}
if record_has_content {
counts.push(count);
candidate_coherent &= record_coherent;
if counts.len() >= 20 {
reached_record_limit = true;
break;
}
}
count = 0;
quoted = false;
after_quote = false;
field_start = true;
record_has_content = false;
record_coherent = true;
if cut < sample_end {
let before_cut = scan_delimiter(&sample[..cut], scan.delimiter, quote);
(scan.count, scan.record_has_content, scan.record_coherent, scan.record_bare_quote) =
(before_cut.count, before_cut.record_has_content, before_cut.record_coherent, before_cut.record_bare_quote);
}
if scan.record_has_content {
scan.coherent &= scan.record_coherent && !scan.record_bare_quote;
scan.counts.push(scan.count);
}
}
} else {
for scan in &mut scans {
if scan.reached_record_limit {
continue;
}
if ch == quote && field_start {
quoted = true;
field_start = false;
record_has_content = true;
} else if ch == char::from(*delimiter) {
count += 1;
field_start = true;
after_quote = false;
} else {
record_coherent &= !after_quote || ch.is_whitespace();
record_has_content |= !ch.is_whitespace();
field_start = false;
if scan.open_quote.is_some() {
scan.coherent = false;
} else if scan.record_has_content {
scan.coherent &= scan.record_coherent;
scan.counts.push(scan.count);
}
}
if !reached_record_limit && !truncated {
if quoted {
candidate_coherent = false;
} else if record_has_content {
candidate_coherent &= record_coherent;
counts.push(count);
}
}
*coherent &= candidate_coherent;
*min_count = counts.iter().copied().min().unwrap_or(0);
*total = counts.iter().sum();
}
pick_delimiter(scans)
}
fn pick_delimiter(scans: [DelimiterScan; 3]) -> u8 {
let candidates =
scans.map(|scan| (scan.delimiter, scan.counts.iter().copied().min().unwrap_or(0), scan.counts.iter().sum::<usize>(), scan.coherent));
let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent);
let mut best = (b',', 0usize, 0usize);
for (delimiter, min, total, coherent) in candidates {
@@ -125,6 +98,93 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
best.0
}
/// One candidate delimiter's pass over the sample in [`sniff_delimiter`].
struct DelimiterScan {
delimiter: u8,
/// Delimiter counts of the complete non-blank records, at most 20.
counts: Vec<usize>,
/// Whether every counted record had coherent quoted fields.
coherent: bool,
reached_record_limit: bool,
/// The record still open at the end of the sample: its delimiter count so far, whether it
/// has content, and whether its quoted fields so far are coherent.
count: usize,
record_has_content: bool,
record_coherent: bool,
/// Whether the open record has a quote inside a field that does not start with one.
record_bare_quote: bool,
/// The byte offset of the quote that opens a field still unterminated at the end.
open_quote: Option<usize>,
}
fn scan_delimiter(sample: &str, delimiter: u8, quote: char) -> DelimiterScan {
let mut scan = DelimiterScan {
delimiter,
counts: Vec::new(),
coherent: true,
reached_record_limit: false,
count: 0,
record_has_content: false,
record_coherent: true,
record_bare_quote: false,
open_quote: None,
};
let mut after_quote = false;
let mut field_start = true;
let mut chars = sample.char_indices().peekable();
while let Some((at, ch)) = chars.next() {
if scan.open_quote.is_some() {
if ch == quote {
if chars.peek().map(|&(_, next)| next) == Some(quote) {
chars.next();
scan.record_has_content = true;
} else {
scan.open_quote = None;
after_quote = true;
}
} else if !ch.is_whitespace() {
scan.record_has_content = true;
}
continue;
}
if ch == '\r' || ch == '\n' {
if ch == '\r' && chars.peek().map(|&(_, next)| next) == Some('\n') {
chars.next();
}
if scan.record_has_content {
scan.counts.push(scan.count);
scan.coherent &= scan.record_coherent;
if scan.counts.len() >= 20 {
scan.reached_record_limit = true;
break;
}
}
scan.count = 0;
after_quote = false;
field_start = true;
scan.record_has_content = false;
scan.record_coherent = true;
scan.record_bare_quote = false;
continue;
}
if ch == quote && field_start {
scan.open_quote = Some(at);
field_start = false;
scan.record_has_content = true;
} else if ch == char::from(delimiter) {
scan.count += 1;
field_start = true;
after_quote = false;
} else {
scan.record_coherent &= !after_quote || ch.is_whitespace();
scan.record_bare_quote |= ch == quote;
scan.record_has_content |= !ch.is_whitespace();
field_start = false;
}
}
scan
}
/// Splits delimited text into records of fields and hands each to `on_record`, which returns
/// whether to go on. A record ends at CR, LF or CRLF; a blank line is a record with one empty
/// field. A field that starts with `quote` runs to the closing quote (a doubled quote is one
@@ -309,6 +369,40 @@ mod tests {
assert_eq!(sniff_delimiter(&text, '"'), b'\t');
}
#[test]
fn delimiter_sniffing_counts_a_first_record_longer_than_the_sample() {
let wide = |field: &str, delimiter: char| {
let mut record = String::new();
while record.len() <= 64 * 1024 {
record.push_str(field);
record.push(delimiter);
}
record
};
assert_eq!(sniff_delimiter(&format!("{}end\n1;2\n", wide("cell", ';')), '"'), b';');
assert_eq!(sniff_delimiter(&format!("{}end\n1\t2\n", wide("cell", '\t')), '"'), b'\t');
// Delimiters inside quoted fields of the long record are text.
assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a,b\tc\"", ';')), '"'), b';');
assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a;b\tc\"", ',')), '"'), b',');
// A candidate that reads a quoted field as text inside an unquoted one is incoherent.
assert_eq!(sniff_delimiter(&format!("a,\"{}\",{}", "x;y".repeat(20_000), "z".repeat(10_000)), '"'), b',');
// The sample ends inside a quoted field: the delimiters before that field count.
let text = format!("a;b;c;\"{}\"\n1;2;3;4\n", "long text ".repeat(8_000));
assert_eq!(sniff_delimiter(&text, '"'), b';');
// Candidates without a complete record do not count delimiters after that quote, also
// when they read it as text.
assert_eq!(sniff_delimiter(&format!("a,\"{}\"\n1,2\n", "x;y".repeat(30_000)), '"'), b',');
assert_eq!(sniff_delimiter(&format!("a;b;\"{}\"\n1;2;3\n", "x,y\t".repeat(20_000)), '"'), b';');
// A line break inside the open quoted field gives the other candidates a record without
// a delimiter, which does not stop the first record from counting.
assert_eq!(sniff_delimiter(&format!("a;b;\"hello\n{}\"\n", "x".repeat(70_000)), '"'), b';');
assert_eq!(sniff_delimiter(&format!("\"a\";b;\"hello,\n{}\"\n", "x".repeat(70_000)), '"'), b';');
// When a candidate finds coherent complete records with a delimiter in each, a record cut
// off at the end is left out.
assert_eq!(sniff_delimiter(&format!("a;\"hello,\";b\n{}", "x;y;z\n".repeat(12_000)), '"'), b';');
assert_eq!(sniff_delimiter(&format!("a,b,c;\"hello,\";d\n{}", "x;y;z\n".repeat(12_000)), '"'), b';');
}
#[test]
fn parser_stops_collecting_fields_at_the_cap() {
let (max, extra) = (16_384usize, 50);