Variables: Import Data Sets reads UTF-16 CSV files without a byte-order mark

A CSV file that holds NUL bytes and has no byte-order mark was refused.
It is now read as UTF-16 when at least half of its two-byte units hold
a NUL byte, in the byte order that reads a line break. A file with fewer
NUL bytes (UTF-8 or Windows-1252 text with stray NUL bytes, or UTF-16
text made mostly of characters above U+00FF) is still refused, and so is
a file that reads a line break in both byte orders, has an odd length,
isn't valid UTF-16 in that order or holds a NUL character. Refused files
get the same message as before.

crates/engine/src/cmd/variablecmds/csv.rs

@@ -32,20 +32,17 @@ fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result<(Vec<u8>, bool)> {
 }
 
 /// The text of a datasets CSV given as a file or as `dataBase64`: UTF-16 after a UTF-16
-/// byte-order mark, else UTF-8, with or without a byte-order mark, else Windows-1252. A file
-/// that starts with a byte-order mark must be valid in its encoding, and a file without one must
-/// not hold NUL bytes (UTF-16 without a byte-order mark).
+/// byte-order mark, else UTF-16 without one when the file holds NUL bytes (see
+/// [`utf16_without_mark`]), else UTF-8, with or without a byte-order mark, else Windows-1252. A
+/// file that starts with a byte-order mark must be valid in its encoding, and a file with NUL
+/// bytes and no byte-order mark must read as UTF-16 by [`utf16_without_mark`]'s rule.
 fn datasets_csv_text(bytes: &[u8], cmd: &str) -> Result<String> {
-    let utf16 = |rest: &[u8], le: bool| {
-        let (units, odd) = rest.as_chunks::<2>();
-        let units = units.iter().map(|&c| if le { u16::from_le_bytes(c) } else { u16::from_be_bytes(c) });
-        let text = char::decode_utf16(units).collect::<std::result::Result<String, _>>();
-        text.ok().filter(|_| odd.is_empty()).ok_or_else(|| bad(cmd, "CSV is not UTF-16"))
-    };
     match bytes {
-        [0xFF, 0xFE, rest @ ..] => return utf16(rest, true),
-        [0xFE, 0xFF, rest @ ..] => return utf16(rest, false),
-        _ if bytes.contains(&0) => return Err(bad(cmd, "CSV holds NUL bytes: save it as UTF-8, or as UTF-16 with a byte-order mark")),
+        [0xFF, 0xFE, rest @ ..] => return utf16(rest, true).ok_or_else(|| bad(cmd, "CSV is not UTF-16")),
+        [0xFE, 0xFF, rest @ ..] => return utf16(rest, false).ok_or_else(|| bad(cmd, "CSV is not UTF-16")),
+        _ if bytes.contains(&0) => {
+            return utf16_without_mark(bytes).ok_or_else(|| bad(cmd, "CSV holds NUL bytes: save it as UTF-8, or as UTF-16 with a byte-order mark"));
+        }
         _ => {}
     }
     match std::str::from_utf8(bytes) {
@@ -55,6 +52,34 @@ fn datasets_csv_text(bytes: &[u8], cmd: &str) -> Result<String> {
     }
 }
 
+/// `bytes` as UTF-16, little-endian when `le` is set, or `None` when they aren't valid UTF-16.
+fn utf16(bytes: &[u8], le: bool) -> Option<String> {
+    let (units, odd) = bytes.as_chunks::<2>();
+    let units = units.iter().map(|&c| if le { u16::from_le_bytes(c) } else { u16::from_be_bytes(c) });
+    char::decode_utf16(units).collect::<std::result::Result<String, _>>().ok().filter(|_| odd.is_empty())
+}
+
+/// `bytes`, which hold NUL bytes and no byte-order mark, as UTF-16, or `None`. At least half of
+/// the two-byte units must hold a NUL byte, as characters from U+0001 to U+00FF do; a file with
+/// fewer, such as UTF-8 or Windows-1252 text with stray NUL bytes, is refused. The byte order is
+/// the one that reads a line break (`\n` or `\r`). The other order reads one only from U+0A00 or
+/// U+0D00, so a file that reads one in both orders is refused. A file that reads none in either
+/// order has no data rows, and is read little-endian so that the import reports that. The text
+/// must be valid UTF-16 in that order and hold no NUL character.
+fn utf16_without_mark(bytes: &[u8]) -> Option<String> {
+    let (units, odd) = bytes.as_chunks::<2>();
+    if !odd.is_empty() || units.iter().filter(|c| c.contains(&0)).count() * 2 < units.len() {
+        return None;
+    }
+    let is_line_break = |u: u16| matches!(u, 0x0A | 0x0D);
+    let lines_le = units.iter().any(|&c| is_line_break(u16::from_le_bytes(c)));
+    let lines_be = units.iter().any(|&c| is_line_break(u16::from_be_bytes(c)));
+    if lines_le && lines_be {
+        return None;
+    }
+    utf16(bytes, !lines_be).filter(|text| !text.contains('\0'))
+}
+
 /// A row of a datasets CSV: the file line it starts on, its cells, and for each cell whether it was quoted.
 type CsvRow = (usize, Vec<String>, Vec<bool>);