diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs index 11abc766..06c6adfb 100644 --- a/crates/dxf/src/codepage.rs +++ b/crates/dxf/src/codepage.rs @@ -6,8 +6,9 @@ //! Asian code page) escapes, which [`crate::parse`] decodes after the bytes. //! //! The Windows, ISO and East Asian code pages are decoded with `encoding_rs`; DOS 437, 850 and -//! 852 with the tables below (the upper halves of the IBM PC code pages; the lower halves are -//! ASCII). Decoding never fails: a byte sequence that is invalid in the code page (a truncated +//! 852 with the tables below, Korean Johab (`ANSI_1361`) by its definition in KS X 1001 annex 3, +//! and the other DOS pages with those of `dos_tables.rs` (the upper +//! halves of the IBM PC code pages; the lower halves are ASCII). Decoding never fails: a byte sequence that is invalid in the code page (a truncated //! double-byte character, an unassigned byte) becomes U+FFFD and the bytes after it decode //! normally, so line ends and the group structure survive. @@ -15,6 +16,9 @@ use std::borrow::Cow; use encoding_rs::Encoding; +use crate::Version; +use crate::dos_tables as dos; + /// How the bytes of a code page become text. #[derive(Clone, Copy, Debug)] pub(crate) enum Decoder { @@ -22,6 +26,21 @@ pub(crate) enum Decoder { Encoding(&'static Encoding), /// Single-byte DOS code page: ASCII, then these 128 characters for bytes 0x80–0xFF. Table(&'static [char; 128]), + /// DOS 864: like [`Decoder::Table`], but byte 0x25 is U+066A ARABIC PERCENT SIGN, not `%`. + Dos864, + /// Korean Johab (code page 1361). + Johab, +} + +/// What byte 0x25 of DOS 864 stands for. +const DOS864_PERCENT: char = '\u{066A}'; + +impl Decoder { + /// True when the ASCII character `c` is not what its byte means in this code page, so a + /// writer has to escape it. + pub(crate) fn shadows_ascii(self, c: char) -> bool { + matches!(self, Decoder::Dos864) && c == '%' + } } /// The decoder for a `$DWGCODEPAGE` value (case-insensitive), or `None` when CADCraft doesn't @@ -52,11 +71,24 @@ pub(crate) fn decoder(code_page: &str) -> Option { "ISO8859_6" | "ISO_8859_6" => e::ISO_8859_6, "ISO8859_7" | "ISO_8859_7" => e::ISO_8859_7, "ISO8859_8" | "ISO_8859_8" => e::ISO_8859_8, + "ISO8859_15" | "ISO_8859_15" => e::ISO_8859_15, "DOS866" => e::IBM866, "MACINTOSH" | "MAC_ROMAN" => e::MACINTOSH, "DOS437" => return Some(Decoder::Table(&DOS437)), "DOS850" => return Some(Decoder::Table(&DOS850)), "DOS852" => return Some(Decoder::Table(&DOS852)), + "DOS720" => return Some(Decoder::Table(&dos::DOS720)), + "DOS737" => return Some(Decoder::Table(&dos::DOS737)), + "DOS775" => return Some(Decoder::Table(&dos::DOS775)), + "DOS855" => return Some(Decoder::Table(&dos::DOS855)), + "DOS857" => return Some(Decoder::Table(&dos::DOS857)), + "DOS860" => return Some(Decoder::Table(&dos::DOS860)), + "DOS861" => return Some(Decoder::Table(&dos::DOS861)), + "DOS863" => return Some(Decoder::Table(&dos::DOS863)), + "DOS864" => return Some(Decoder::Dos864), + "DOS865" => return Some(Decoder::Table(&dos::DOS865)), + "DOS869" => return Some(Decoder::Table(&dos::DOS869)), + "ANSI_1361" | "JOHAB" => return Some(Decoder::Johab), _ => return None, }; Some(Decoder::Encoding(enc)) @@ -86,21 +118,32 @@ pub(crate) fn encode_char(c: char, d: Decoder, out: &mut Vec) -> bool { let mut buf = [0; 4]; match d { // Not a pre-2007 code page ([`encoder`] never gives it): escaped. - Decoder::Utf8 => return false, + // Johab is read only: a writer escapes every character. + Decoder::Utf8 | Decoder::Johab => return false, Decoder::Encoding(e) => { let (bytes, _, unmappable) = e.encode(c.encode_utf8(&mut buf)); // A code page byte that is ASCII would read back as another character, and a NUL, // CR or LF would break the group, so only all-high-bit sequences or proper double-byte // characters are kept (their trail byte may be ASCII, as in Shift-JIS). - if unmappable || bytes.is_empty() || !bytes.first().is_some_and(|b| *b >= 0x80) || bytes.iter().any(|b| matches!(b, 0 | b'\r' | b'\n')) { + if unmappable || bytes.is_empty() || bytes.first().is_none_or(|b| *b < 0x80) || bytes.iter().any(|b| matches!(b, 0 | b'\r' | b'\n')) { return false; } out.extend_from_slice(&bytes); } + // U+FFFD marks unassigned bytes in the tables; it must be escaped, not written as one. + Decoder::Table(_) | Decoder::Dos864 if c == '\u{FFFD}' => return false, Decoder::Table(t) => { let Some(i) = t.iter().position(|&x| x == c).and_then(|i| u8::try_from(i).ok()) else { return false }; out.push(0x80 | i); } + Decoder::Dos864 => { + if c == DOS864_PERCENT { + out.push(b'%'); + } else { + let Some(i) = dos::DOS864.iter().position(|&x| x == c).and_then(|i| u8::try_from(i).ok()) else { return false }; + out.push(0x80 | i); + } + } } true } @@ -113,6 +156,25 @@ pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { Err(_) => encoding_rs::WINDOWS_1252.decode_without_bom_handling(bytes).0, }, Decoder::Encoding(e) => e.decode_without_bom_handling(bytes).0, + Decoder::Johab => match std::str::from_utf8(bytes) { + Ok(s) if s.is_ascii() => Cow::Borrowed(s), + _ => Cow::Owned(decode_johab(bytes)), + }, + Decoder::Dos864 => { + if bytes.is_ascii() && !bytes.contains(&b'%') { + return Cow::Borrowed(std::str::from_utf8(bytes).unwrap_or_default()); + } + Cow::Owned( + bytes + .iter() + .map(|&b| match b { + b'%' => DOS864_PERCENT, + _ if b.is_ascii() => char::from(b), + _ => dos::DOS864.get(usize::from(b & 0x7F)).copied().unwrap_or('\u{FFFD}'), + }) + .collect(), + ) + } Decoder::Table(t) => match std::str::from_utf8(bytes) { Ok(s) if s.is_ascii() => Cow::Borrowed(s), _ => Cow::Owned( @@ -125,18 +187,18 @@ pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { } } -/// The `$DWGCODEPAGE` value of an ASCII DXF file: the line two lines after a `$DWGCODEPAGE` -/// line (the group code `3` between them), looked for in the HEADER section only (the scan stops -/// at the first `ENDSEC`). Works on the raw bytes, before they are decoded. -fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { +/// The value of the header variable `name` (group `code` after it) of an ASCII DXF file: looked +/// for in the HEADER section only (the scan stops at the first `ENDSEC`). Works on the raw bytes, +/// before they are decoded. +fn ascii_header_value<'a>(bytes: &'a [u8], name: &[u8], code: &[u8]) -> Option<&'a [u8]> { let mut lines = bytes.split(|&b| b == b'\n' || b == b'\r').map(<[u8]>::trim_ascii).filter(|l| !l.is_empty()); while let Some(line) = lines.next() { if line == b"ENDSEC" { return None; } - if line.eq_ignore_ascii_case(b"$DWGCODEPAGE") { + if line.eq_ignore_ascii_case(name) { return match (lines.next(), lines.next()) { - (Some(b"3"), Some(value)) => Some(value), + (Some(c), Some(value)) if c == code => Some(value), _ => None, }; } @@ -144,13 +206,118 @@ fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { None } +/// Decode Johab text: ASCII, double-byte Hangul (lead bytes 0x84–0xD3), symbols (0xD9–0xDE) and +/// Hanja (0xE0–0xF9). An unassigned code in that area is one U+FFFD for its two bytes; any other +/// byte that can't start a character is one U+FFFD and the next byte starts afresh. +fn decode_johab(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if b.is_ascii() { + out.push(char::from(b)); + i += 1; + continue; + } + let trail = bytes.get(i + 1).copied(); + match trail.and_then(|t| johab_char(b, t)) { + Some(c) => { + out.push(c); + i += 2; + } + None => { + out.push('\u{FFFD}'); + i += if trail.is_some_and(|t| johab_lead(b) && johab_trail(b, t)) { 2 } else { 1 }; + } + } + } + out +} + +fn johab_lead(b: u8) -> bool { + matches!(b, 0x84..=0xD3 | 0xD8..=0xDE | 0xE0..=0xF9) +} + +fn johab_trail(lead: u8, t: u8) -> bool { + if lead <= 0xD3 { matches!(t, 0x41..=0x7E | 0x81..=0xFE) } else { matches!(t, 0x31..=0x7E | 0x91..=0xFE) } +} + +/// The character of the Johab code `lead trail`, if it has one. +pub(crate) fn johab_char(lead: u8, trail: u8) -> Option { + if !johab_lead(lead) || !johab_trail(lead, trail) { + return None; + } + if lead <= 0xD3 { + return johab_hangul((u16::from(lead) << 8) | u16::from(trail)); + } + // Symbols and Hanja: each lead byte holds two KS X 1001 rows of 94 cells, the first in + // trail bytes 0x31–0x7E and 0x91–0xA0, the second in 0xA1–0xFE (0xD8 is user-defined). + let first = match lead { + 0xD9..=0xDE => (lead - 0xD9) * 2 + 0x21, + 0xE0..=0xF9 => (lead - 0xE0) * 2 + 0x4A, + _ => return None, + }; + let (row, col) = match trail { + 0x31..=0x7E => (first, trail - 0x31 + 0x21), + 0x91..=0xA0 => (first, trail - 0x91 + 0x6F), + _ => (first + 1, trail.saturating_sub(0xA1) + 0x21), + }; + let euc = [row | 0x80, col | 0x80]; + let text = encoding_rs::EUC_KR.decode_without_bom_handling_and_without_replacement(&euc)?; + let mut it = text.chars(); + it.next().filter(|_| it.next().is_none()) +} + +/// A Johab Hangul code (bit 15 set, then 5 bits each for the initial, medial and final jamo): +/// a precomposed syllable, or a lone jamo (as a compatibility jamo) when the others are fillers. +fn johab_hangul(code: u16) -> Option { + let (i, m, f) = ((code >> 10) & 0x1F, (code >> 5) & 0x1F, code & 0x1F); + // Initial: 1 filler, 2–20 the 19 consonants. Medial: 2 filler, then the 21 vowels in four + // runs (3–7, 10–15, 18–23, 26–29). Final: 1 none, 2–17 and 19–29 the 27 final consonants. + let lead = match i { + 1 => None, + 2..=20 => Some(i - 2), + _ => return None, + }; + let vowel = match m { + 2 => None, + 3..=7 => Some(m - 3), + 10..=15 => Some(m - 5), + 18..=23 => Some(m - 7), + 26..=29 => Some(m - 9), + _ => return None, + }; + let tail = match f { + 1 => 0, + 2..=17 => f - 1, + 19..=29 => f - 2, + _ => return None, + }; + // The compatibility jamo (offsets from U+3131) of the initial and of the final consonants. + const LEADS: [u16; 19] = [0, 1, 3, 6, 7, 8, 16, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]; + const TAILS: [u16; 27] = [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29]; + let cp = match (lead, vowel, tail) { + (Some(l), Some(v), t) => 0xAC00 + (u32::from(l) * 21 + u32::from(v)) * 28 + u32::from(t), + (None, None, 0) => 0x3164, + (Some(l), None, 0) => 0x3131 + u32::from(*LEADS.get(usize::from(l))?), + (None, Some(v), 0) => 0x314F + u32::from(v), + (None, None, t) => 0x3131 + u32::from(*TAILS.get(usize::from(t.saturating_sub(1)))?), + _ => return None, + }; + char::from_u32(cp) +} + +/// The `$DWGCODEPAGE` value of an ASCII DXF file. +fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { + ascii_header_value(bytes, b"$DWGCODEPAGE", b"3") +} + impl Decoder { - /// True for the multi-byte East Asian code pages (Shift-JIS, GBK, EUC-KR/Wansung, Big5). + /// True for the multi-byte East Asian code pages (Shift-JIS, GBK, EUC-KR/Wansung, Big5, Johab). /// Their short double-byte strings are often also valid UTF-8, so UTF-8 is only trusted for /// a whole file there (see [`decode_ascii_file`] and [`BinaryText`]). fn is_east_asian(self) -> bool { use encoding_rs as e; - matches!(self, Decoder::Encoding(x) if x == e::SHIFT_JIS || x == e::GBK || x == e::EUC_KR || x == e::BIG5) + matches!(self, Decoder::Johab) || matches!(self, Decoder::Encoding(x) if x == e::SHIFT_JIS || x == e::GBK || x == e::EUC_KR || x == e::BIG5) } } @@ -175,7 +342,10 @@ pub(crate) fn decode_text(bytes: &[u8], d: Decoder) -> Cow<'_, str> { /// pairs) can be valid UTF-8 by accident and must not be misread line by line. pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { if let Ok(s) = std::str::from_utf8(bytes) { - return Cow::Borrowed(s); + if !s.contains('%') || !is_pre_2007_dos864(bytes) { + return Cow::Borrowed(s); + } + return decode(bytes, Decoder::Dos864); } let d = match ascii_code_page(bytes) { Some(name) => { @@ -199,6 +369,13 @@ pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { Cow::Owned(out) } +/// True for an ASCII-only file whose header names `DOS864` and is not R2007+ (which is UTF-8). +fn is_pre_2007_dos864(bytes: &[u8]) -> bool { + bytes.is_ascii() + && ascii_code_page(bytes).is_some_and(|n| n.eq_ignore_ascii_case(b"DOS864")) + && !ascii_header_value(bytes, b"$ACADVER", b"1").is_some_and(|v| Version::for_acadver(&String::from_utf8_lossy(v)).is_some_and(Version::utf8)) +} + /// The text state of a binary DXF file, learnt from the header as the tags go by: `$ACADVER` /// (R2007+ strings are UTF-8) and `$DWGCODEPAGE` (the code page of older files). Both come /// first in the HEADER section and are ASCII, so every later string is decoded right. @@ -265,7 +442,7 @@ impl BinaryText { _ => None, } } - (1, Some(HeaderVar::AcadVer)) => self.utf8 = is_utf8_version(&s), + (1, Some(HeaderVar::AcadVer)) => self.utf8 = Version::for_acadver(&s).is_some_and(Version::utf8), (3, Some(HeaderVar::CodePage)) => { self.code_page = decoder(&s); if self.code_page.is_none() { @@ -278,22 +455,21 @@ impl BinaryText { } } -/// True for an `$ACADVER` of R2007 (AC1021) or later, whose text is UTF-8. -fn is_utf8_version(acadver: &str) -> bool { - acadver.trim().strip_prefix("AC").and_then(|n| n.parse::().ok()).is_some_and(|n| n >= 1021) -} - -/// The double-byte code page of a `\M+n` escape: 1 Japanese (Shift-JIS), 2 Traditional Chinese -/// (Big5), 3 Korean (Wansung), 5 Simplified Chinese (GB 2312). 4 (Korean Johab) is not -/// supported, so its escapes stay literal. -pub(crate) fn mif_encoding(n: char) -> Option<&'static Encoding> { - match n { - '1' => Some(encoding_rs::SHIFT_JIS), - '2' => Some(encoding_rs::BIG5), - '3' => Some(encoding_rs::EUC_KR), - '5' => Some(encoding_rs::GBK), - _ => None, - } +/// The character of a `\M+n` escape's two bytes in its double-byte code page `n`: 1 Japanese +/// (Shift-JIS), 2 Traditional Chinese (Big5), 3 Korean (Wansung), 4 Korean (Johab), 5 Simplified +/// Chinese (GB 2312). Other code pages and unassigned codes give `None`. +pub(crate) fn mif_char(n: char, bytes: [u8; 2]) -> Option { + let enc: &'static Encoding = match n { + '1' => encoding_rs::SHIFT_JIS, + '2' => encoding_rs::BIG5, + '3' => encoding_rs::EUC_KR, + '4' => return johab_char(bytes[0], bytes[1]), + '5' => encoding_rs::GBK, + _ => return None, + }; + let text = enc.decode_without_bom_handling_and_without_replacement(&bytes)?; + let mut it = text.chars(); + it.next().filter(|_| it.next().is_none()) } // Generated from the IBM PC code page definitions (bytes 0x80–0xFF in order). @@ -359,11 +535,75 @@ mod tests { assert_eq!(d(b"\xBD\xA2\x88w", "DOS852").as_deref(), Some("Żółw")); assert_eq!(d(b"\x91\xAB\xAE\xA9", "DOS866").as_deref(), Some("Слой")); assert_eq!(d("é".as_bytes(), "UTF8").as_deref(), Some("é")); - assert_eq!(d(b"x", "ANSI_1361"), None); assert_eq!(d(b"x", "EBCDIC"), None); assert_eq!(d(b"x", ""), None); } + #[test] + fn replacement_char_is_never_encoded_as_an_unassigned_byte() { + for cp in ["DOS857", "DOS864", "DOS437"] { + let d = encoder(cp).unwrap(); + let mut out = Vec::new(); + assert!(!encode_char('\u{FFFD}', d, &mut out), "{cp}"); + assert!(out.is_empty(), "{cp}"); + } + } + + #[test] + fn dos864_percent_is_the_arabic_percent_sign() { + // Byte 0x25 is U+066A in CP864 (Python's cp864 agrees); the rest of the low half is ASCII. + assert_eq!(decode_code_page(b"50%", "DOS864").as_deref(), Some("50\u{066A}")); + assert_eq!(decode_code_page(b"a%\x80%", "dos864").as_deref(), Some("a\u{066A}\u{00B0}\u{066A}")); + assert_eq!(decode_code_page(b"50&", "DOS864").as_deref(), Some("50&")); + // The other DOS pages keep `%`. + assert_eq!(decode_code_page(b"50%", "DOS437").as_deref(), Some("50%")); + // Every other low byte of every table is ASCII. + for cp in [ + "DOS437", "DOS850", "DOS852", "DOS720", "DOS737", "DOS775", "DOS855", "DOS857", "DOS860", "DOS861", "DOS863", "DOS864", "DOS865", + "DOS869", + ] { + for b in 0u8..0x80 { + if b == 0x25 && cp == "DOS864" { + continue; + } + assert_eq!(decode_code_page(&[b], cp).as_deref(), Some(char::from(b).to_string().as_str()), "{cp} {b:#x}"); + } + } + } + + #[test] + fn ascii_only_dos864_file_keeps_the_arabic_percent_sign() { + let file = |ver: &str, cp: &str| format!("0\nSECTION\n2\nHEADER\n9\n$ACADVER\n1\n{ver}\n9\n$DWGCODEPAGE\n3\n{cp}\n0\nENDSEC\n1\n50%\n"); + assert!(decode_ascii_file(file("AC1015", "DOS864").as_bytes()).ends_with("50\u{066A}\n")); + // Other code pages, and R2007+ files (UTF-8), keep `%`. + assert!(decode_ascii_file(file("AC1015", "DOS437").as_bytes()).ends_with("50%\n")); + assert!(decode_ascii_file(file("AC1021", "DOS864").as_bytes()).ends_with("50%\n")); + } + + /// Korean Johab (`ANSI_1361`): Hangul syllables composed from their jamo, lone jamo, + /// symbols and Hanja through KS X 1001. + #[test] + fn johab() { + let d = |bytes: &[u8]| decode_code_page(bytes, "ANSI_1361"); + // 한글 (U+D55C U+AE00) and 가 (U+AC00). + assert_eq!(d(b"\xD0\x65\x8B\x69 \x88\x61").as_deref(), Some("한글 가")); + // Lone initial ㄱ, lone vowel ㅏ, the Hangul filler. + assert_eq!(d(b"\x88\x41\x84\x61\x84\x41").as_deref(), Some("ㄱㅏ\u{3164}")); + // The ideographic space (KS X 1001 0x2121) and 伽 (0x4A21, the first Hanja). + assert_eq!(d(b"\xD9\x31\xE0\x31").as_deref(), Some("\u{3000}伽")); + assert_eq!(decode_code_page(b"\x88\x61", "johab").as_deref(), Some("가")); + // A truncated lead byte, a lead byte without a valid trail byte, a user-defined code. + assert_eq!(d(b"A\x88").as_deref(), Some("A\u{FFFD}")); + assert_eq!(d(b"\x88\x20B").as_deref(), Some("\u{FFFD} B")); + assert_eq!(d(b"\xD8\x31C").as_deref(), Some("\u{FFFD}C")); + // Every pair of bytes decodes to something without panicking. + for a in 0x80..=0xFFu8 { + for b in 0..=0xFFu8 { + assert!(!d(&[a, b, b'\n']).unwrap_or_default().is_empty()); + } + } + } + #[test] fn invalid_sequences_become_replacement_characters() { // A Shift-JIS lead byte cut off by the line end keeps the line end. @@ -381,8 +621,9 @@ mod tests { // Wrong group code, or the file ends early. assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n1\nANSI_1251\n"), None); assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n3"), None); - assert!(is_utf8_version("AC1021") && is_utf8_version(" AC1032 ")); - assert!(!is_utf8_version("AC1018") && !is_utf8_version("AC1009") && !is_utf8_version("garbage")); + let utf8 = |v: &str| Version::for_acadver(v).is_some_and(Version::utf8); + assert!(utf8("AC1021") && utf8(" AC1032 ")); + assert!(!utf8("AC1018") && !utf8("AC1009") && !utf8("garbage")); } #[test] @@ -390,8 +631,10 @@ mod tests { let one = |s: &str| crate::parse(format!("1\n{s}\n0\nEOF\n").as_bytes()).unwrap()[0].str(); assert_eq!(one("\\M+5BCFE\\M+1907D\\M+2B9CF\\M+3B5B5!"), "件図圖도!"); assert_eq!(one("\\M+5BCFE\\U+0021"), "件!"); - // Johab, unknown code pages, bad hex, short escapes and unassigned codes stay literal. - for s in ["\\M+4ABCD", "\\M+9BCFE", "\\M+5GGGG", "\\M+5BC", "\\M+", "\\M+50041", "\\M+5FFFF"] { + // Johab: 한. + assert_eq!(one("\\M+4D065"), "한"); + // Unknown code pages, bad hex, short escapes and unassigned codes stay literal. + for s in ["\\M+4D820", "\\M+9BCFE", "\\M+5GGGG", "\\M+5BC", "\\M+", "\\M+50041", "\\M+5FFFF"] { assert_eq!(one(s), s); } } diff --git a/crates/dxf/src/dos_tables.rs b/crates/dxf/src/dos_tables.rs new file mode 100644 index 00000000..b1826eae --- /dev/null +++ b/crates/dxf/src/dos_tables.rs @@ -0,0 +1,168 @@ +//! More DOS code pages for `$DWGCODEPAGE` (`DOS855` …): the upper halves (bytes 0x80–0xFF) of the IBM PC +//! code page definitions, as Unicode characters; the lower halves are ASCII. Bytes the code page leaves +//! unassigned are U+FFFD. +//! +//! Source check (the tables were checked against these files, not generated from them): +//! - DOS737, 775, 855, 857, 860, 861, 863, 864, 865 and 869 were checked byte for byte (all 128 bytes +//! 0x80–0xFF of each) against the Unicode Consortium mapping files CP737.TXT, CP775.TXT, CP855.TXT, +//! CP857.TXT, CP860.TXT, CP861.TXT, CP863.TXT, CP864.TXT, CP865.TXT and CP869.TXT in +//! . Bytes those files list as `#UNDEFINED` +//! are U+FFFD here. +//! - DOS720 has no Unicode Consortium mapping file, so it was checked only against Python's `cp720` +//! codec (not a published mapping file). 120 of the 128 bytes agree. The 8 bytes 0x80, 0x81, 0x84, 0x86, +//! 0x8D, 0x8E, 0x8F and 0x90 decode to U+FFFD here (unassigned), while that codec maps them to the C1 +//! controls U+0080..U+0090. + +pub(crate) const DOS720: [char; 128] = [ + '\u{FFFD}', '\u{FFFD}', '\u{00E9}', '\u{00E2}', '\u{FFFD}', '\u{00E0}', '\u{FFFD}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{0651}', '\u{0652}', '\u{00F4}', '\u{00A4}', '\u{0640}', '\u{00FB}', '\u{00F9}', + '\u{0621}', '\u{0622}', '\u{0623}', '\u{0624}', '\u{00A3}', '\u{0625}', '\u{0626}', '\u{0627}', '\u{0628}', '\u{0629}', '\u{062A}', '\u{062B}', + '\u{062C}', '\u{062D}', '\u{062E}', '\u{062F}', '\u{0630}', '\u{0631}', '\u{0632}', '\u{0633}', '\u{0634}', '\u{0635}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{0636}', '\u{0637}', '\u{0638}', '\u{0639}', '\u{063A}', '\u{0641}', '\u{00B5}', '\u{0642}', '\u{0643}', '\u{0644}', '\u{0645}', '\u{0646}', + '\u{0647}', '\u{0648}', '\u{0649}', '\u{064A}', '\u{2261}', '\u{064B}', '\u{064C}', '\u{064D}', '\u{064E}', '\u{064F}', '\u{0650}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS737: [char; 128] = [ + '\u{0391}', '\u{0392}', '\u{0393}', '\u{0394}', '\u{0395}', '\u{0396}', '\u{0397}', '\u{0398}', '\u{0399}', '\u{039A}', '\u{039B}', '\u{039C}', + '\u{039D}', '\u{039E}', '\u{039F}', '\u{03A0}', '\u{03A1}', '\u{03A3}', '\u{03A4}', '\u{03A5}', '\u{03A6}', '\u{03A7}', '\u{03A8}', '\u{03A9}', + '\u{03B1}', '\u{03B2}', '\u{03B3}', '\u{03B4}', '\u{03B5}', '\u{03B6}', '\u{03B7}', '\u{03B8}', '\u{03B9}', '\u{03BA}', '\u{03BB}', '\u{03BC}', + '\u{03BD}', '\u{03BE}', '\u{03BF}', '\u{03C0}', '\u{03C1}', '\u{03C3}', '\u{03C2}', '\u{03C4}', '\u{03C5}', '\u{03C6}', '\u{03C7}', '\u{03C8}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03C9}', '\u{03AC}', '\u{03AD}', '\u{03AE}', '\u{03CA}', '\u{03AF}', '\u{03CC}', '\u{03CD}', '\u{03CB}', '\u{03CE}', '\u{0386}', '\u{0388}', + '\u{0389}', '\u{038A}', '\u{038C}', '\u{038E}', '\u{038F}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{03AA}', '\u{03AB}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS775: [char; 128] = [ + '\u{0106}', '\u{00FC}', '\u{00E9}', '\u{0101}', '\u{00E4}', '\u{0123}', '\u{00E5}', '\u{0107}', '\u{0142}', '\u{0113}', '\u{0156}', '\u{0157}', + '\u{012B}', '\u{0179}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{014D}', '\u{00F6}', '\u{0122}', '\u{00A2}', '\u{015A}', + '\u{015B}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{00D7}', '\u{00A4}', '\u{0100}', '\u{012A}', '\u{00F3}', '\u{017B}', + '\u{017C}', '\u{017A}', '\u{201D}', '\u{00A6}', '\u{00A9}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{0141}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{0104}', '\u{010C}', '\u{0118}', '\u{0116}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{012E}', '\u{0160}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{0172}', '\u{016A}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{017D}', '\u{0105}', '\u{010D}', '\u{0119}', '\u{0117}', + '\u{012F}', '\u{0161}', '\u{0173}', '\u{016B}', '\u{017E}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{014C}', '\u{0143}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{0144}', '\u{0136}', '\u{0137}', '\u{013B}', '\u{013C}', + '\u{0146}', '\u{0112}', '\u{0145}', '\u{2019}', '\u{00AD}', '\u{00B1}', '\u{201C}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{201E}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS855: [char; 128] = [ + '\u{0452}', '\u{0402}', '\u{0453}', '\u{0403}', '\u{0451}', '\u{0401}', '\u{0454}', '\u{0404}', '\u{0455}', '\u{0405}', '\u{0456}', '\u{0406}', + '\u{0457}', '\u{0407}', '\u{0458}', '\u{0408}', '\u{0459}', '\u{0409}', '\u{045A}', '\u{040A}', '\u{045B}', '\u{040B}', '\u{045C}', '\u{040C}', + '\u{045E}', '\u{040E}', '\u{045F}', '\u{040F}', '\u{044E}', '\u{042E}', '\u{044A}', '\u{042A}', '\u{0430}', '\u{0410}', '\u{0431}', '\u{0411}', + '\u{0446}', '\u{0426}', '\u{0434}', '\u{0414}', '\u{0435}', '\u{0415}', '\u{0444}', '\u{0424}', '\u{0433}', '\u{0413}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{0445}', '\u{0425}', '\u{0438}', '\u{0418}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{0439}', '\u{0419}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{043A}', '\u{041A}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{043B}', '\u{041B}', '\u{043C}', '\u{041C}', + '\u{043D}', '\u{041D}', '\u{043E}', '\u{041E}', '\u{043F}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{041F}', '\u{044F}', '\u{2580}', + '\u{042F}', '\u{0440}', '\u{0420}', '\u{0441}', '\u{0421}', '\u{0442}', '\u{0422}', '\u{0443}', '\u{0423}', '\u{0436}', '\u{0416}', '\u{0432}', + '\u{0412}', '\u{044C}', '\u{042C}', '\u{2116}', '\u{00AD}', '\u{044B}', '\u{042B}', '\u{0437}', '\u{0417}', '\u{0448}', '\u{0428}', '\u{044D}', + '\u{042D}', '\u{0449}', '\u{0429}', '\u{0447}', '\u{0427}', '\u{00A7}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS857: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{0131}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{0130}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{015E}', '\u{015F}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{011E}', '\u{011F}', '\u{00BF}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{00C0}', '\u{00A9}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{00A2}', '\u{00A5}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{00E3}', '\u{00C3}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{00BA}', '\u{00AA}', '\u{00CA}', '\u{00CB}', + '\u{00C8}', '\u{FFFD}', '\u{00CD}', '\u{00CE}', '\u{00CF}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{00A6}', '\u{00CC}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{00D2}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{FFFD}', '\u{00D7}', '\u{00DA}', '\u{00DB}', '\u{00D9}', + '\u{00EC}', '\u{00FF}', '\u{00AF}', '\u{00B4}', '\u{00AD}', '\u{00B1}', '\u{FFFD}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{00B8}', + '\u{00B0}', '\u{00A8}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS860: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E3}', '\u{00E0}', '\u{00C1}', '\u{00E7}', '\u{00EA}', '\u{00CA}', '\u{00E8}', '\u{00CD}', + '\u{00D4}', '\u{00EC}', '\u{00C3}', '\u{00C2}', '\u{00C9}', '\u{00C0}', '\u{00C8}', '\u{00F4}', '\u{00F5}', '\u{00F2}', '\u{00DA}', '\u{00F9}', + '\u{00CC}', '\u{00D5}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00D9}', '\u{20A7}', '\u{00D3}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{00D2}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS861: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00D0}', + '\u{00F0}', '\u{00DE}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00FE}', '\u{00FB}', '\u{00DD}', + '\u{00FD}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{20A7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00C1}', '\u{00CD}', '\u{00D3}', '\u{00DA}', '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS863: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00C2}', '\u{00E0}', '\u{00B6}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{2017}', '\u{00C0}', '\u{00A7}', '\u{00C9}', '\u{00C8}', '\u{00CA}', '\u{00F4}', '\u{00CB}', '\u{00CF}', '\u{00FB}', '\u{00F9}', + '\u{00A4}', '\u{00D4}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00D9}', '\u{00DB}', '\u{0192}', '\u{00A6}', '\u{00B4}', '\u{00F3}', '\u{00FA}', + '\u{00A8}', '\u{00B8}', '\u{00B3}', '\u{00AF}', '\u{00CE}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00BE}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS864: [char; 128] = [ + '\u{00B0}', '\u{00B7}', '\u{2219}', '\u{221A}', '\u{2592}', '\u{2500}', '\u{2502}', '\u{253C}', '\u{2524}', '\u{252C}', '\u{251C}', '\u{2534}', + '\u{2510}', '\u{250C}', '\u{2514}', '\u{2518}', '\u{03B2}', '\u{221E}', '\u{03C6}', '\u{00B1}', '\u{00BD}', '\u{00BC}', '\u{2248}', '\u{00AB}', + '\u{00BB}', '\u{FEF7}', '\u{FEF8}', '\u{FFFD}', '\u{FFFD}', '\u{FEFB}', '\u{FEFC}', '\u{FFFD}', '\u{00A0}', '\u{00AD}', '\u{FE82}', '\u{00A3}', + '\u{00A4}', '\u{FE84}', '\u{FFFD}', '\u{FFFD}', '\u{FE8E}', '\u{FE8F}', '\u{FE95}', '\u{FE99}', '\u{060C}', '\u{FE9D}', '\u{FEA1}', '\u{FEA5}', + '\u{0660}', '\u{0661}', '\u{0662}', '\u{0663}', '\u{0664}', '\u{0665}', '\u{0666}', '\u{0667}', '\u{0668}', '\u{0669}', '\u{FED1}', '\u{061B}', + '\u{FEB1}', '\u{FEB5}', '\u{FEB9}', '\u{061F}', '\u{00A2}', '\u{FE80}', '\u{FE81}', '\u{FE83}', '\u{FE85}', '\u{FECA}', '\u{FE8B}', '\u{FE8D}', + '\u{FE91}', '\u{FE93}', '\u{FE97}', '\u{FE9B}', '\u{FE9F}', '\u{FEA3}', '\u{FEA7}', '\u{FEA9}', '\u{FEAB}', '\u{FEAD}', '\u{FEAF}', '\u{FEB3}', + '\u{FEB7}', '\u{FEBB}', '\u{FEBF}', '\u{FEC1}', '\u{FEC5}', '\u{FECB}', '\u{FECF}', '\u{00A6}', '\u{00AC}', '\u{00F7}', '\u{00D7}', '\u{FEC9}', + '\u{0640}', '\u{FED3}', '\u{FED7}', '\u{FEDB}', '\u{FEDF}', '\u{FEE3}', '\u{FEE7}', '\u{FEEB}', '\u{FEED}', '\u{FEEF}', '\u{FEF3}', '\u{FEBD}', + '\u{FECC}', '\u{FECE}', '\u{FECD}', '\u{FEE1}', '\u{FE7D}', '\u{0651}', '\u{FEE5}', '\u{FEE9}', '\u{FEEC}', '\u{FEF0}', '\u{FEF2}', '\u{FED0}', + '\u{FED5}', '\u{FEF5}', '\u{FEF6}', '\u{FEDD}', '\u{FED9}', '\u{FEF1}', '\u{25A0}', '\u{FFFD}', +]; + +pub(crate) const DOS865: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{20A7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00A4}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS869: [char; 128] = [ + '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{0386}', '\u{FFFD}', '\u{00B7}', '\u{00AC}', '\u{00A6}', '\u{2018}', + '\u{2019}', '\u{0388}', '\u{2015}', '\u{0389}', '\u{038A}', '\u{03AA}', '\u{038C}', '\u{FFFD}', '\u{FFFD}', '\u{038E}', '\u{03AB}', '\u{00A9}', + '\u{038F}', '\u{00B2}', '\u{00B3}', '\u{03AC}', '\u{00A3}', '\u{03AD}', '\u{03AE}', '\u{03AF}', '\u{03CA}', '\u{0390}', '\u{03CC}', '\u{03CD}', + '\u{0391}', '\u{0392}', '\u{0393}', '\u{0394}', '\u{0395}', '\u{0396}', '\u{0397}', '\u{00BD}', '\u{0398}', '\u{0399}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{039A}', '\u{039B}', '\u{039C}', '\u{039D}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{039E}', '\u{039F}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{03A0}', '\u{03A1}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{03A3}', '\u{03A4}', '\u{03A5}', '\u{03A6}', '\u{03A7}', + '\u{03A8}', '\u{03A9}', '\u{03B1}', '\u{03B2}', '\u{03B3}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{03B4}', '\u{03B5}', '\u{2580}', + '\u{03B6}', '\u{03B7}', '\u{03B8}', '\u{03B9}', '\u{03BA}', '\u{03BB}', '\u{03BC}', '\u{03BD}', '\u{03BE}', '\u{03BF}', '\u{03C0}', '\u{03C1}', + '\u{03C3}', '\u{03C2}', '\u{03C4}', '\u{0384}', '\u{00AD}', '\u{00B1}', '\u{03C5}', '\u{03C6}', '\u{03C7}', '\u{00A7}', '\u{03C8}', '\u{0385}', + '\u{00B0}', '\u{00A8}', '\u{03C9}', '\u{03CB}', '\u{03B0}', '\u{03CE}', '\u{25A0}', '\u{00A0}', +]; diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 286219fc..7694eb01 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -7,7 +7,6 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] -use std::fmt::Write as _; use std::io::Write as _; mod version; @@ -52,6 +51,10 @@ impl Value { Value::Bool(b) => i64::from(*b).to_string(), } } + #[expect( + clippy::cast_precision_loss, + reason = "DXF numeric accessors intentionally convert integer values to f64, accepting precision loss beyond 2^53." + )] pub fn as_f64(&self) -> f64 { match self { Value::Real(r) if r.is_finite() => *r, @@ -61,6 +64,7 @@ impl Value { _ => 0.0, } } + #[expect(clippy::cast_possible_truncation, reason = "DXF numeric accessors intentionally truncate finite real values to i64.")] pub fn as_i64(&self) -> i64 { match self { Value::Int(i) => *i, @@ -96,6 +100,13 @@ impl Tag { pub fn str(&self) -> String { self.value.as_str() } + /// The value when it is text, without a copy. + pub fn str_ref(&self) -> Option<&str> { + match &self.value { + Value::Str(s) | Value::Hex(s) => Some(s), + _ => None, + } + } pub fn f64(&self) -> f64 { self.value.as_f64() } @@ -118,46 +129,21 @@ pub enum Kind { pub fn kind_of(code: i32) -> Kind { match code { - 0..=9 => Kind::Str, - 10..=59 => Kind::Real, - 60..=79 => Kind::Int16, - 90..=99 => Kind::Int32, - 100..=109 => Kind::Str, - 110..=149 => Kind::Real, + 10..=59 | 110..=149 | 210..=239 | 460..=469 | 1010..=1059 => Kind::Real, + 60..=79 | 170..=179 | 270..=289 | 370..=389 | 400..=409 | 1060..=1070 => Kind::Int16, + 90..=99 | 420..=429 | 440..=449 | 450..=459 | 1071 => Kind::Int32, 160..=169 => Kind::Int64, - 170..=179 => Kind::Int16, - 210..=239 => Kind::Real, - 270..=289 => Kind::Int16, 290..=299 => Kind::Bool, - 300..=309 => Kind::Str, - 310..=319 => Kind::Hex, - 320..=369 => Kind::Str, - 370..=389 => Kind::Int16, - 390..=399 => Kind::Str, - 400..=409 => Kind::Int16, - 410..=419 => Kind::Str, - 420..=429 => Kind::Int32, - 430..=439 => Kind::Str, - 440..=449 => Kind::Int32, - 450..=459 => Kind::Int32, - 460..=469 => Kind::Real, - 470..=479 => Kind::Str, - 480..=481 => Kind::Str, - 999 => Kind::Str, - 1000..=1003 => Kind::Str, - 1004 => Kind::Hex, - 1005..=1009 => Kind::Str, - 1010..=1059 => Kind::Real, - 1060..=1070 => Kind::Int16, - 1071 => Kind::Int32, + 310..=319 | 1004 => Kind::Hex, _ => Kind::Str, } } +#[expect(clippy::cast_possible_truncation, reason = "DXF numeric parsing intentionally truncates finite real values to i64.")] fn typed(code: i32, text: &str) -> Value { let t = text.trim(); match kind_of(code) { - Kind::Real => t.parse::().ok().filter(|v| v.is_finite()).map(Value::Real).unwrap_or(Value::Real(0.0)), + Kind::Real => t.parse::().ok().filter(|v| v.is_finite()).map_or(Value::Real(0.0), Value::Real), Kind::Int16 | Kind::Int32 | Kind::Int64 => { // A float spelling ("1e3", "12.0") is accepted when finite, clamped to what the // group's width holds, signed or unsigned (files write 16-bit flags up to 65535, as @@ -168,8 +154,7 @@ fn typed(code: i32, text: &str) -> Value { _ => (i64::MIN, i64::MAX), }; t.parse::() - .map(Value::Int) - .unwrap_or_else(|_| Value::Int(t.parse::().ok().filter(|f| f.is_finite()).map_or(0, |f| (f as i64).clamp(lo, hi)))) + .map_or_else(|_| Value::Int(t.parse::().ok().filter(|f| f.is_finite()).map_or(0, |f| (f as i64).clamp(lo, hi))), Value::Int) } Kind::Bool => Value::Bool(t.parse::().unwrap_or(0) != 0), Kind::Hex => Value::Hex(normalize_hex(t)), @@ -201,8 +186,15 @@ fn max_tags_from(v: Option<&str>) -> usize { mod codepage; pub use codepage::decode_code_page; +mod dos_tables; /// Parse a DXF file (ASCII or binary). +/// +/// # Errors +/// +/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair, +/// truncated binary data, or more than [`MAX_TAGS`] group codes. For ASCII input, invalid numeric +/// text becomes `0.0` for real values or `0` for integer values, and an EOF tag is not required. pub fn parse(bytes: &[u8]) -> Result> { parse_with_limit(bytes, max_tags()) } @@ -210,6 +202,12 @@ pub fn parse(bytes: &[u8]) -> Result> { /// [`parse`] with an explicit group-code limit. For ASCII DXF the limit is checked up front from /// the line count, before the text is decoded or any tag allocated, so an oversized file fails in /// milliseconds instead of after it has filled memory. +/// +/// # Errors +/// +/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair, +/// truncated binary data, or more than `max_tags` group codes. For ASCII input, invalid numeric +/// text becomes `0.0` for real values or `0` for integer values, and an EOF tag is not required. pub fn parse_with_limit(bytes: &[u8], max_tags: usize) -> Result> { if let Some(body) = bytes.strip_prefix(BINARY_SENTINEL) { return parse_binary_with_limit(body, max_tags); @@ -254,16 +252,13 @@ pub fn mif_escape_at(chars: &[char], i: usize) -> Option { if chars.get(i) != Some(&'\\') || chars.get(i + 1) != Some(&'M') || chars.get(i + 2) != Some(&'+') { return None; } - let enc = codepage::mif_encoding(*chars.get(i + 3)?)?; + let n = *chars.get(i + 3)?; let hex = chars.get(i + 4..i + 8)?; if !hex.iter().all(char::is_ascii_hexdigit) { return None; } let code = hex.iter().try_fold(0u16, |acc, c| Some(acc * 16 + u16::try_from(c.to_digit(16)?).ok()?))?; - let bytes = code.to_be_bytes(); - let text = enc.decode_without_bom_handling_and_without_replacement(&bytes)?; - let mut it = text.chars(); - it.next().filter(|_| it.next().is_none()) + codepage::mif_char(n, code.to_be_bytes()) } /// Unescape the `\U+XXXX` sequences pre-R2007 DXF uses for characters outside the file's code @@ -271,36 +266,68 @@ pub fn mif_escape_at(chars: &[char], i: usize) -> Option { /// escapes), and the `\M+nXXXX` sequences for double-byte East Asian characters. Malformed /// escapes, unpaired surrogates and unknown `\M+` code pages are kept as literal text. fn unescape_unicode(s: &str) -> String { - if !s.contains("\\U+") && !s.contains("\\M+") { + if !has_escape(s) { return s.to_string(); } + unescape_found(s) +} + +/// [`unescape_unicode`] for a string the caller owns: without escapes (nearly always) it is +/// returned as it is, with no copy. +fn unescape_owned(s: String) -> String { + if has_escape(&s) { unescape_found(&s) } else { s } +} + +fn has_escape(s: &str) -> bool { + s.contains("\\U+") || s.contains("\\M+") +} + +/// The longest escape: a surrogate pair of two `\U+XXXX` (14 characters). +const MAX_ESCAPE_CHARS: usize = 14; + +fn unescape_found(s: &str) -> String { let mut out = String::with_capacity(s.len()); - let chars: Vec = s.chars().collect(); - let mut i = 0; - while let Some(&c) = chars.get(i) { - if let Some(ch) = mif_escape_at(&chars, i) { - out.push(ch); - i += 8; - continue; + let mut rest = s; + // Everything up to the next backslash is copied as it is; an escape is looked for in a window + // of the next few characters only, so no copy of the whole string (4 bytes a character) is made. + while let Some(at) = rest.find('\\') { + let (plain, tail) = rest.split_at(at); + out.push_str(plain); + let mut buf = ['\0'; MAX_ESCAPE_CHARS]; + let mut n = 0; + for (slot, c) in buf.iter_mut().zip(tail.chars()) { + *slot = c; + n += 1; } - if let Some(unit) = escape_at(&chars, i) { + let window = buf.get(..n).unwrap_or_default(); + let (ch, used) = if let Some(ch) = mif_escape_at(window, 0) { + (Some(ch), 8) + } else if let Some(unit) = escape_at(window, 0) { if let Some(ch) = char::from_u32(unit) { - out.push(ch); - i += 7; - continue; - } - if (0xD800..0xDC00).contains(&unit) - && let Some(low) = escape_at(&chars, i + 7).filter(|u| (0xDC00..0xE000).contains(u)) - && let Some(ch) = char::from_u32(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)) + (Some(ch), 7) + } else if (0xD800..0xDC00).contains(&unit) + && let Some(low) = escape_at(window, 7).filter(|u| (0xDC00..0xE000).contains(u)) { + (char::from_u32(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)), 14) + } else { + (None, 0) + } + } else { + (None, 0) + }; + match ch { + Some(ch) => { out.push(ch); - i += 14; - continue; + // Every escape character is ASCII, so `used` characters are `used` bytes. + rest = tail.get(used..).unwrap_or_default(); + } + None => { + out.push('\\'); + rest = tail.get(1..).unwrap_or_default(); } } - out.push(c); - i += 1; } + out.push_str(rest); out } @@ -337,6 +364,13 @@ fn byte_lines(mut rest: &[u8]) -> impl Iterator { }) } +/// Parse ASCII DXF text into typed tags. +/// +/// # Errors +/// +/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair, +/// or too many tags. Invalid numeric text becomes `0.0` for real values or `0` for integer values. +/// A complete tag stream does not require an EOF tag. pub fn parse_ascii(text: &str) -> Result> { parse_ascii_counted(text, max_tags(), 0) } @@ -358,16 +392,17 @@ fn parse_ascii_counted(text: &str, max_tags: usize, expected: usize) -> Result Value::Str(unescape_unicode(&s)), + let v = match typed(code, val) { + // Only text with a backslash can hold an escape; the rest moves in as it is. + Value::Str(s) => Value::Str(unescape_owned(s)), o => o, }; + let eof = code == 0 && matches!(&v, Value::Str(s) if s.trim_end() == "EOF"); tags.push(Tag { code, value: v }); if tags.len() > max_tags { return Err(DxfError::TooLarge { tags: tags.len(), limit: max_tags }); } - if code == 0 && tags.last().is_some_and(|t| t.str().trim_end() == "EOF") { + if eof { break; } } @@ -414,24 +449,38 @@ impl<'a> Cur<'a> { } /// The bytes of a zero-terminated string (without the zero), decoded by the caller. fn cstr(&mut self) -> Result<&'a [u8]> { - let start = self.p; - while self.p < self.b.len() && self.b.get(self.p) != Some(&0) { - self.p += 1; - } - let s = self.b.get(start..self.p).unwrap_or(&[]); - if self.p >= self.b.len() { - return Err(DxfError::Eof); - } - self.p += 1; - Ok(s) + let rest = self.b.get(self.p..).unwrap_or(&[]); + let len = rest.iter().position(|&b| b == 0).ok_or(DxfError::Eof)?; + self.p += len + 1; + Ok(rest.get(..len).unwrap_or(&[])) } } -/// Binary DXF (R13+: 2-byte group codes). +const HEX_DIGITS: &[u8; 16] = b"0123456789ABCDEF"; + +/// Binary DXF, the body after the sentinel: R13+ (2-byte group codes) or R12 and earlier (1-byte +/// group codes, where 255 is followed by a 2-byte code). +/// +/// # Errors +/// +/// Returns an error for an empty or malformed file, an unexpected end of file, or too many tags. pub fn parse_binary(b: &[u8]) -> Result> { parse_binary_with_limit(b, max_tags()) } +/// Whether a binary DXF body uses the pre-R13 layout with 1-byte group codes, told from its first +/// tag: R13+ starts with a 2-byte code (`00 00` for the usual `0 SECTION`, `E7 03` for a 999 +/// comment), the old layout with a 1-byte `00` followed by the string's first character, or with +/// the 255 escape and a code that needs it (255 to 1071; an R13+ file whose first group is code 255 +/// has the string's first character there instead). +fn legacy_binary_layout(b: &[u8]) -> bool { + match b { + [0, next, ..] => *next != 0, + [255, lo, hi, ..] => (255..=1071).contains(&u16::from_le_bytes([*lo, *hi])), + _ => false, + } +} + fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { let mut text = codepage::BinaryText::default(); let tags = parse_binary_pass(b, max_tags, &mut text)?; @@ -447,11 +496,17 @@ fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { fn parse_binary_pass(b: &[u8], max_tags: usize, text: &mut codepage::BinaryText) -> Result> { let mut c = Cur { b, p: 0 }; let mut tags = Vec::new(); + let legacy = legacy_binary_layout(b); while c.p < b.len() { - let mut code = i32::from(c.i16()?); - if code == 255 { - code = i32::from(c.i16()?); - } + // 255 escapes a 2-byte code only in the 1-byte layout; in R13+ it is an ordinary code. + let code = if legacy { + match c.u8()? { + 255 => i32::from(c.i16()?), + x => i32::from(x), + } + } else { + i32::from(c.i16()?) + }; let v = match kind_of(code) { Kind::Str => Value::Str(unescape_unicode(&text.string(code, c.cstr()?))), Kind::Real => Value::Real(Some(c.f64()?).filter(|v| v.is_finite()).unwrap_or(0.0)), @@ -463,8 +518,9 @@ fn parse_binary_pass(b: &[u8], max_tags: usize, text: &mut codepage::BinaryText) let n = usize::from(c.u8()?); let bytes = c.take(n)?; let mut s = String::with_capacity(n * 2); - for x in bytes { - let _ = write!(s, "{x:02X}"); + for &x in bytes { + s.push(char::from(HEX_DIGITS[usize::from(x >> 4)])); + s.push(char::from(HEX_DIGITS[usize::from(x & 15)])); } Value::Hex(s) } @@ -489,10 +545,44 @@ pub fn format_real(v: f64) -> String { if !v.is_finite() { return "0.0".into(); } + // Very large and very small values in exponent form, as AutoCAD writes them (`1.0E+20`): + // plain digits would run to hundreds of characters. + let a = v.abs(); + if a != 0.0 && !(1e-10..1e16).contains(&a) { + let e = format!("{v:E}"); + let (m, x) = e.split_once('E').unwrap_or((&e, "0")); + let x: i32 = x.parse().unwrap_or(0); + let m = if m.contains('.') { m.to_string() } else { format!("{m}.0") }; + return format!("{m}E{}{:02}", if x < 0 { '-' } else { '+' }, x.abs()); + } let s = format!("{v}"); if s.contains('.') || s.contains('e') || s.contains("inf") || s.contains("NaN") { s } else { format!("{s}.0") } } +/// Most characters one string group holds: longer text is split over several groups (MTEXT +/// contents, field codes). +pub const MAX_STRING_CHARS: usize = 250; + +/// `s` split into strings of at most [`MAX_STRING_CHARS`] characters, in order; none for an +/// empty string. +pub fn string_groups(s: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut n = 0; + for c in s.chars() { + if n == MAX_STRING_CHARS { + out.push(std::mem::take(&mut cur)); + n = 0; + } + cur.push(c); + n += 1; + } + if !cur.is_empty() { + out.push(cur); + } + out +} + /// Write tags as ASCII DXF (CRLF line ends, codes right-aligned to 3 as is conventional). /// /// The output is pure ASCII: non-ASCII characters in strings are written as `\U+XXXX` escapes, @@ -513,18 +603,26 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// Write tags as pre-R2007 ASCII DXF with text in the code page `code_page` names (a /// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: /// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A -/// code page CADCraft doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) +/// code page this crate doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) /// give [`write_ascii`]'s pure ASCII output. So does a file whose code page bytes happen to be /// valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, U+0472), since readers, ours included, take an ASCII /// DXF that is valid UTF-8 as UTF-8. pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { - let bytes = ascii(tags, Strings::Escaped(codepage::encoder(code_page))); + let bytes = write_ascii_in_page(tags, code_page); if !bytes.is_ascii() && std::str::from_utf8(&bytes).is_ok() { return ascii(tags, Strings::Escaped(None)); } bytes } +/// [`write_ascii_in`] without its fallback to pure ASCII: the text is in the code page even when +/// its bytes happen to be valid UTF-8. For a reader that decodes a pre-2007 DXF by the +/// `$DWGCODEPAGE` it declares and never guesses UTF-8, as the DWG bridge's acadrust does. A code +/// page this crate doesn't know, `ASCII`, and `UTF8`/`UNICODE` still give pure ASCII. +pub fn write_ascii_in_page(tags: &[Tag], code_page: &str) -> Vec { + ascii(tags, Strings::Escaped(codepage::encoder(code_page))) +} + /// How a writer stores string values. #[derive(Clone, Copy)] enum Strings { @@ -694,6 +792,9 @@ fn push_string(s: &str, strings: Strings, out: &mut Vec) { out.extend_from_slice(b"\\U+005C"); } (c, Strings::Utf8) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), + (c, Strings::Escaped(Some(d))) if d.shadows_ascii(c) => { + let _ = write!(out, "\\U+{:04X}", u32::from(c)); + } (c, _) if c.is_ascii() => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), (c, Strings::Escaped(Some(d))) if code_page_bytes(c, d, escape_at(i + 1), out) => {} (c, Strings::Escaped(_)) => { @@ -729,18 +830,32 @@ pub struct Section { /// Split a tag stream into sections. pub fn sections(tags: &[Tag]) -> Vec
{ + section_slices(tags).into_iter().map(|s| Section { name: s.name.to_string(), tags: s.tags.to_vec() }).collect() +} + +/// A section borrowed from the tag stream it was split from: [`Section`] without the copy. +#[derive(Clone, Copy, Debug)] +pub struct SectionRef<'a> { + pub name: &'a str, + pub tags: &'a [Tag], +} + +/// [`sections`] without copying the tags: each section is a slice of `tags`. A large file's tags +/// take gigabytes (40 bytes a tag, 25 million tags in a 0.5 GB DXF), so a reader that holds the +/// stream while it reads the sections should take them this way. +pub fn section_slices(tags: &[Tag]) -> Vec> { + let is = |t: &Tag, word: &str| t.code == 0 && t.str_ref().is_some_and(|s| s.trim_end() == word); let mut out = Vec::new(); let mut i = 0; while i < tags.len() { - let is_sec = tags.get(i).is_some_and(|t| t.code == 0 && t.str().trim_end() == "SECTION"); - if is_sec { - let name = tags.get(i + 1).filter(|t| t.code == 2).map(|t| t.str().trim_end().to_string()).unwrap_or_default(); + if tags.get(i).is_some_and(|t| is(t, "SECTION")) { + let name = tags.get(i + 1).filter(|t| t.code == 2).and_then(Tag::str_ref).unwrap_or_default().trim_end(); let start = i + 2; let mut j = start; - while j < tags.len() && !(tags.get(j).is_some_and(|t| t.code == 0 && t.str().trim_end() == "ENDSEC")) { + while j < tags.len() && !tags.get(j).is_some_and(|t| is(t, "ENDSEC")) { j += 1; } - out.push(Section { name, tags: tags.get(start..j).unwrap_or(&[]).to_vec() }); + out.push(SectionRef { name, tags: tags.get(start..j).unwrap_or(&[]) }); i = j + 1; } else { i += 1; @@ -751,15 +866,20 @@ pub fn sections(tags: &[Tag]) -> Vec
{ /// Split a run of tags into records starting at each code-0 tag: (type, tags after the 0). pub fn records(tags: &[Tag]) -> Vec<(String, Vec)> { - let mut out: Vec<(String, Vec)> = Vec::new(); - for t in tags { - if t.code == 0 { - out.push((t.str(), Vec::new())); - } else if let Some(last) = out.last_mut() { - last.1.push(t.clone()); - } - } - out + record_slices(tags).into_iter().map(|(kind, tags)| (kind, tags.to_vec())).collect() +} + +/// [`records`] without copying the tags: each record's tags are a slice of `tags`. +pub fn record_slices(tags: &[Tag]) -> Vec<(String, &[Tag])> { + let starts: Vec = tags.iter().enumerate().filter(|(_, t)| t.code == 0).map(|(i, _)| i).collect(); + starts + .iter() + .enumerate() + .map(|(n, &i)| { + let end = starts.get(n + 1).copied().unwrap_or(tags.len()); + (tags.get(i).map(Tag::str).unwrap_or_default(), tags.get(i + 1..end).unwrap_or(&[])) + }) + .collect() } #[cfg(test)] @@ -828,6 +948,128 @@ mod tests { assert_eq!(tags[2], Tag::i(62, 5)); } + /// A pre-R13 binary body: 1-byte group codes, 255 escaping a 2-byte code (1001 here). + fn r12_binary() -> Vec { + let mut b = BINARY_SENTINEL.to_vec(); + let push_str = |b: &mut Vec, code: u8, s: &str| { + b.push(code); + b.extend_from_slice(s.as_bytes()); + b.push(0); + }; + push_str(&mut b, 0, "SECTION"); + push_str(&mut b, 2, "ENTITIES"); + push_str(&mut b, 0, "LINE"); + push_str(&mut b, 8, "RAILROAD"); + b.push(10); + b.extend_from_slice(&3.25f64.to_le_bytes()); + b.push(62); + b.extend_from_slice(&5i16.to_le_bytes()); + b.push(255); + b.extend_from_slice(&1001i16.to_le_bytes()); + b.extend_from_slice(b"APP\0"); + push_str(&mut b, 0, "ENDSEC"); + push_str(&mut b, 0, "EOF"); + b + } + + #[test] + fn r12_binary_parse() { + let tags = parse(&r12_binary()).unwrap(); + assert_eq!(tags[0], Tag::s(0, "SECTION")); + assert_eq!(tags[3], Tag::s(8, "RAILROAD")); + assert_eq!(tags[4], Tag::f(10, 3.25)); + assert_eq!(tags[5], Tag::i(62, 5)); + assert_eq!(tags[6], Tag::s(1001, "APP")); + assert_eq!(tags.last(), Some(&Tag::s(0, "EOF"))); + } + + #[test] + fn r13_binary_code_255_is_ordinary() { + let mut b = BINARY_SENTINEL.to_vec(); + for (code, s) in [(0i16, "LINE"), (255, "x"), (0, "EOF")] { + b.extend_from_slice(&code.to_le_bytes()); + b.extend_from_slice(s.as_bytes()); + b.push(0); + } + let tags = parse(&b).unwrap(); + assert_eq!(tags.get(1).map(|t| t.code), Some(255)); + assert_eq!(tags.len(), 3); + } + + #[test] + fn binary_layout_detection() { + assert!(legacy_binary_layout(b"\0SECTION\0")); + assert!(legacy_binary_layout(&[255, 0xE7, 0x03])); + assert!(!legacy_binary_layout(b"\0\0SECTION\0")); + assert!(!legacy_binary_layout(&[0xE7, 0x03, b'x', 0])); + assert!(!legacy_binary_layout(b"")); + assert!(!legacy_binary_layout(&[0])); + // R13+ code 255 with the string "x": not the escape + assert!(!legacy_binary_layout(&[255, 0, b'x', 0])); + assert!(!legacy_binary_layout(&[255, 0xE7])); + } + + #[test] + fn truncated_r12_binary_never_panics() { + let b = r12_binary(); + for n in BINARY_SENTINEL.len()..b.len() { + let _ = parse(b.get(..n).unwrap_or(&[])); + } + let mut cut = BINARY_SENTINEL.to_vec(); + cut.extend_from_slice(&[0, b'S', 0, 255, 0x10]); + assert!(parse(&cut).is_err()); + } + + #[test] + fn borrowed_sections_and_records_match_the_copies() { + let text = "999\nstray\n 0\nSECTION\n 2\nHEADER\n 9\n$ACADVER\n 1\nAC1015\n 0\nENDSEC\n 0\nSECTION\n 2\nENTITIES\n 5\n1A\n 0\nLINE\n 8\n0\n 10\n1.5\n 0\nCIRCLE\n 40\n2.0\n 0\nENDSEC\n 0\nEOF\n"; + let tags = parse(text.as_bytes()).unwrap(); + let (copies, slices) = (sections(&tags), section_slices(&tags)); + assert_eq!(slices.iter().map(|s| s.name).collect::>(), ["HEADER", "ENTITIES"]); + assert_eq!(copies.len(), slices.len()); + for (c, s) in copies.iter().zip(&slices) { + assert_eq!(c.name, s.name); + assert_eq!(c.tags.as_slice(), s.tags); + let (rc, rs) = (records(&c.tags), record_slices(s.tags)); + assert_eq!(rc.len(), rs.len()); + for ((kc, tc), (ks, ts)) in rc.iter().zip(&rs) { + assert_eq!((kc, tc.as_slice()), (ks, *ts)); + } + } + // Tags before the first record start none, as in `records`. + assert_eq!(record_slices(slices[1].tags).iter().map(|(k, _)| k.as_str()).collect::>(), ["LINE", "CIRCLE"]); + assert_eq!(slices[1].tags[0].str_ref(), Some("1A")); + assert_eq!(Tag::f(10, 1.5).str_ref(), None); + } + + #[test] + fn huge_and_tiny_reals_in_exponent_form() { + // Plain digits would run to hundreds of characters; the value reads back exactly. + assert_eq!(format_real(1e20), "1.0E+20"); + assert_eq!(format_real(-2.5e-12), "-2.5E-12"); + assert_eq!(format_real(0.5), "0.5"); + assert_eq!(format_real(12.0), "12.0"); + for v in [f64::MAX, -f64::MAX, 1e308, f64::MIN_POSITIVE, 5e-324, 1.2345678901234567e16] { + let s = format_real(v); + assert!(s.len() < 32, "{s}"); + assert_eq!(s.parse::().ok(), Some(v), "{s}"); + // and through the writer and the reader + let back = parse(write_ascii(&[Tag::f(10, v), Tag::s(0, "EOF")]).as_bytes()).unwrap(); + assert_eq!(back[0], Tag::f(10, v)); + } + } + + #[test] + fn long_strings_split_into_250_character_groups() { + assert!(string_groups("").is_empty()); + assert_eq!(string_groups("abc"), ["abc"]); + let s = "é".repeat(251); + let g = string_groups(&s); + assert_eq!(g.iter().map(|x| x.chars().count()).collect::>(), [250, 1]); + assert_eq!(g.concat(), s); + assert_eq!(string_groups(&"x".repeat(500)).len(), 2); + } + #[test] fn malformed_inputs() { assert!(parse(b"").is_err()); @@ -839,7 +1081,7 @@ mod tests { assert!(parse(&b).is_err()); // Bad numbers become 0 rather than failing. let t = parse(b"10\nnotanumber\n0\nEOF\n").unwrap(); - assert_eq!(t[0].f64(), 0.0); + assert!(t[0].f64().abs() < f64::EPSILON); } #[test] diff --git a/crates/dxf/src/version.rs b/crates/dxf/src/version.rs index 3d0b96c1..9f0fff08 100644 --- a/crates/dxf/src/version.rs +++ b/crates/dxf/src/version.rs @@ -1,20 +1,20 @@ //! The DXF/DWG file versions CADCraft writes (DXF Reference, HEADER `$ACADVER`). -/// A file version CADCraft can save: the AutoCAD release that introduced the format. Each format -/// also serves the releases after it (2018 is still AutoCAD's current format). +/// A file version CADCraft can save: the `AutoCAD` release that introduced the format. Each format +/// also serves the releases after it (2018 is still `AutoCAD`'s current format). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Version { - /// AutoCAD 2000/2002 (AC1015). + /// `AutoCAD` 2000/2002 (AC1015). R2000, - /// AutoCAD 2004–2006 (AC1018). + /// `AutoCAD` 2004–2006 (AC1018). R2004, - /// AutoCAD 2007–2009 (AC1021): the first version whose DXF text is UTF-8. + /// `AutoCAD` 2007–2009 (AC1021): the first version whose DXF text is UTF-8. R2007, - /// AutoCAD 2010–2012 (AC1024). + /// `AutoCAD` 2010–2012 (AC1024). R2010, - /// AutoCAD 2013–2017 (AC1027). + /// `AutoCAD` 2013–2017 (AC1027). R2013, - /// AutoCAD 2018 and later (AC1032), the default. + /// `AutoCAD` 2018 and later (AC1032), the default. #[default] R2018, } diff --git a/crates/dxf/tests/ascii_bom.rs b/crates/dxf/tests/ascii_bom.rs index 9d80aff1..8d08bb4d 100644 --- a/crates/dxf/tests/ascii_bom.rs +++ b/crates/dxf/tests/ascii_bom.rs @@ -9,7 +9,7 @@ fn utf8_bom_bytes_are_skipped() { bytes.extend_from_slice(BODY.as_bytes()); let with_bom = craft_dxf::parse(&bytes); assert!(with_bom.is_ok(), "{with_bom:?}"); - assert_eq!(with_bom.map(|t| t.len()).unwrap_or(0), plain.len()); + assert_eq!(with_bom.map_or(0, |t| t.len()), plain.len()); } #[test] diff --git a/crates/dxf/tests/audit_followups.rs b/crates/dxf/tests/audit_followups.rs new file mode 100644 index 00000000..de61bf17 --- /dev/null +++ b/crates/dxf/tests/audit_followups.rs @@ -0,0 +1,139 @@ +//! Follow-ups of the crates/dxf audit (#1269): group code 255, the extra DOS and ISO code pages, +//! `format_real`, and the escape and integer writers that were rewritten without a change of +//! behaviour. + +use craft_dxf::{Tag, Value, decode_code_page, format_real, parse, write_ascii, write_binary}; + +#[test] +fn group_code_255_is_an_ordinary_code_in_binary_dxf() { + // Code 255 used to be read as an R12 escape (`FF 00 `), so a file with that code could + // not be read back. + let tags = vec![Tag::s(255, "x"), Tag::s(0, "EOF")]; + let back = parse(&write_binary(&tags, true)).unwrap(); + assert_eq!(back.len(), 2); + assert_eq!((back[0].code, back[0].str()), (255, "x".to_string())); +} + +#[test] +fn more_dos_and_iso_code_pages_decode() { + // (code page, byte, the character the code page defines there) + let cases: [(&str, u8, char); 12] = [ + ("DOS720", 0x82, 'é'), + ("DOS737", 0x80, 'Α'), + ("DOS775", 0x80, 'Ć'), + ("DOS855", 0x80, 'ђ'), + ("DOS857", 0x8D, 'ı'), + ("DOS860", 0x84, 'ã'), + ("DOS861", 0x8B, 'Ð'), + ("DOS863", 0x84, 'Â'), + ("DOS864", 0x80, '°'), + ("DOS865", 0x9B, 'ø'), + ("DOS869", 0x86, 'Ά'), + ("ISO8859_15", 0xA4, '€'), + ]; + for (name, byte, ch) in cases { + assert_eq!(decode_code_page(&[byte], name), Some(ch.to_string()), "{name}"); + assert_eq!(decode_code_page(b"A", &name.to_lowercase()), Some("A".into()), "{name}"); + } + // An unassigned byte is the replacement character, the rest decodes. + assert_eq!(decode_code_page(b"\xD5A", "DOS857").as_deref(), Some("\u{FFFD}A")); +} + +#[test] +fn dos_code_pages_decode_known_characters() { + // Four bytes per table with the character the reference mapping gives there (code point in the + // comment); U+FFFD marks a byte the reference leaves undefined. DOS737, 775, 855, 857, 860, 861, 863, + // 864, 865 and 869 come from the Unicode Consortium files in + // https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/ (CP.TXT). DOS720 has no such file: + // its reference is only Python's `cp720` codec, which maps the bytes below that we decode to U+FFFD + // (0x80, 0x81, 0x84, 0x86, 0x8D, 0x8E, 0x8F, 0x90) to the C1 controls U+0080..U+0090. + let tables: [(&str, [(u8, char); 4]); 11] = [ + ("DOS720", [(0x82, 'é'), (0x9B, 'ؤ'), (0xB3, '│'), (0xE3, 'ع')]), // U+00E9, U+0624, U+2502, U+0639 + ("DOS737", [(0x80, 'Α'), (0xAB, 'τ'), (0xB3, '│'), (0xF3, '≤')]), // U+0391, U+03C4, U+2502, U+2264 + ("DOS775", [(0x80, 'Ć'), (0xAB, '½'), (0xB3, '│'), (0xE3, 'Ń')]), // U+0106, U+00BD, U+2502, U+0143 + ("DOS855", [(0x80, 'ђ'), (0xA3, 'Б'), (0xB3, '│'), (0xFB, 'ч')]), // U+0452, U+0411, U+2502, U+0447 + ("DOS857", [(0x8D, 'ı'), (0xB3, '│'), (0xE3, 'Ò'), (0xD5, '\u{FFFD}')]), // U+0131, U+2502, U+00D2, undefined + ("DOS860", [(0x84, 'ã'), (0x9B, '¢'), (0xB3, '│'), (0xE3, 'π')]), // U+00E3, U+00A2, U+2502, U+03C0 + ("DOS861", [(0x8B, 'Ð'), (0xAB, '½'), (0xB3, '│'), (0xFB, '√')]), // U+00D0, U+00BD, U+2502, U+221A + ("DOS863", [(0x84, 'Â'), (0x9B, '¢'), (0xB3, '│'), (0xE3, 'π')]), // U+00C2, U+00A2, U+2502, U+03C0 + ("DOS864", [(0x80, '°'), (0xAB, 'ﺙ'), (0xB3, '٣'), (0xFF, '\u{FFFD}')]), // U+00B0, U+FE99, U+0663, undefined + ("DOS865", [(0x9B, 'ø'), (0xAB, '½'), (0xB3, '│'), (0xE3, 'π')]), // U+00F8, U+00BD, U+2502, U+03C0 + ("DOS869", [(0x86, 'Ά'), (0x9B, 'ά'), (0xB3, '│'), (0x80, '\u{FFFD}')]), // U+0386, U+03AC, U+2502, undefined + ]; + for (name, cases) in tables { + for (byte, ch) in cases { + assert_eq!(decode_code_page(&[byte], name), Some(ch.to_string()), "{name} 0x{byte:02X}"); + } + } + // DOS720: the 8 bytes the Python codec maps to C1 controls are unassigned here. + for byte in [0x80u8, 0x81, 0x84, 0x86, 0x8D, 0x8E, 0x8F, 0x90] { + assert_eq!(decode_code_page(&[byte], "DOS720").as_deref(), Some("\u{FFFD}"), "DOS720 0x{byte:02X}"); + } +} + +#[test] +fn ascii_file_in_a_dos_code_page_reads_its_text() { + let mut dxf = b"0\nSECTION\n2\nHEADER\n9\n$DWGCODEPAGE\n3\nDOS855\n0\nENDSEC\n0\nSECTION\n2\nTABLES\n8\n".to_vec(); + dxf.extend_from_slice(b"\xA1\xA2\n0\nENDSEC\n0\nEOF\n"); + let tags = parse(&dxf).unwrap(); + let layer = tags.iter().find(|t| t.code == 8).unwrap(); + assert_eq!(layer.str(), decode_code_page(b"\xA1\xA2", "DOS855").unwrap()); + assert_eq!(layer.str(), "Аб"); +} + +#[test] +fn format_real_always_has_a_point() { + assert_eq!(format_real(5.0), "5.0"); + assert_eq!(format_real(-0.0), "-0.0"); + assert_eq!(format_real(0.1), "0.1"); + assert_eq!(format_real(1e15), "1000000000000000.0"); + assert_eq!(format_real(1e21), "1.0E+21"); + assert_eq!(format_real(1e-7), "0.0000001"); + assert_eq!(format_real(f64::MAX).parse::().unwrap(), f64::MAX); + assert_eq!(format_real(f64::NAN), "0.0"); + assert_eq!(format_real(f64::INFINITY), "0.0"); + assert_eq!(format_real(f64::NEG_INFINITY), "0.0"); +} + +#[test] +fn escapes_next_to_other_text_and_backslashes() { + let one = |s: &str| parse(format!("1\n{s}\n0\nEOF\n").as_bytes()).unwrap()[0].str(); + assert_eq!(one("a\\U+00E9b"), "aéb"); + assert_eq!(one("é\\U+0041\\ ü\\M+5BCFE\\U+D83D\\U+DE00 x\\"), "éA\\ ü件😀 x\\"); + assert_eq!(one("\\\\U+0041"), "\\A"); + assert_eq!(one("\\U+D83D\\U+0041 \\U+DE00"), "\\U+D83DA \\U+DE00"); + assert_eq!(one("日本\\U+00"), "日本\\U+00"); + // A long run of backslashes (hostile) is read in linear time and stays literal. + let slashes = "\\".repeat(100_000); + assert_eq!(one(&slashes), slashes); + // The same strings through the binary reader. + // Built by hand: the writer escapes a literal `\U+` (#2050), so it would not produce an escape. + let mut file = b"AutoCAD Binary DXF\r\n\x1a\0".to_vec(); + file.extend_from_slice(&1i16.to_le_bytes()); + file.extend_from_slice("é\\U+0041\\ ü".as_bytes()); + file.push(0); + file.extend_from_slice(&0i16.to_le_bytes()); + file.extend_from_slice(b"EOF\0"); + assert_eq!(parse(&file).unwrap()[0].str(), "éA\\ ü"); + // And the writer keeps the literal text literal. + let tags = [Tag::s(1, "é\\U+0041\\ ü"), Tag::s(0, "EOF")]; + assert_eq!(parse(&write_binary(&tags, true)).unwrap()[0].str(), "é\\U+0041\\ ü"); +} + +#[test] +fn integers_are_written_in_full() { + let text = write_ascii(&[Tag::i(70, i64::MIN), Tag::i(90, -7), Tag::i(160, 0), Tag::new(290, Value::Bool(true))]); + assert_eq!(text, " 70\r\n-9223372036854775808\r\n 90\r\n-7\r\n160\r\n0\r\n290\r\n1\r\n"); +} + +#[test] +fn binary_hex_reads_every_byte_in_upper_case() { + let all: Vec = (0..=255).collect(); + let mut file = b"AutoCAD Binary DXF\r\n\x1a\0".to_vec(); + file.extend_from_slice(&[0x36, 0x01, 0xFF]); + file.extend_from_slice(&all[..255]); + file.extend_from_slice(&[0x00, 0x00, b'E', b'O', b'F', 0]); + let tags = parse(&file).unwrap(); + let want: String = all[..255].iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(tags[0].value, Value::Hex(want)); +} diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 647d8115..02e8a75d 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -1,7 +1,7 @@ //! Pre-R2007 DXF text is written in the `$DWGCODEPAGE` code page: characters it has as their //! bytes, the others as `\U+XXXX` escapes; every form reads back to the same text. -use craft_dxf::{Tag, parse, write_ascii, write_ascii_in, write_binary_in}; +use craft_dxf::{Tag, parse, write_ascii, write_ascii_in, write_ascii_in_page, write_binary_in}; fn file(code_page: &str, text: &str) -> Vec { vec![ @@ -28,11 +28,15 @@ fn contains(bytes: &[u8], part: &[u8]) -> bool { #[test] fn text_is_written_in_the_code_page() { // (code page, text, the bytes written for it) - let cases: [(&str, &str, &[u8]); 6] = [ + let cases: [(&str, &str, &[u8]); 8] = [ ("ANSI_1252", "Ação € 图", b"A\xE7\xE3o \x80 \\U+56FE"), ("ANSI_1251", "Слой é", b"\xD1\xEB\xEE\xE9 \\U+00E9"), ("ANSI_932", "表図面 é", b"\x95\x5C\x90\x7D\x96\xCA \\U+00E9"), ("DOS850", "Größe €", b"Gr\x94\xE1e \\U+20AC"), + // DOS 864: byte 0x25 is the Arabic percent sign, so a literal `%` is escaped. + ("DOS864", "50% \u{066A}\u{00B0}", b"50\\U+0025 %\x80"), + // Only the percent sign: every byte is ASCII, and it still reads back as U+066A. + ("DOS864", "50\u{066A}", b"50%"), ("ANSI_1252", "😀", b"\\U+D83D\\U+DE00"), // A code page CADCraft doesn't know: everything non-ASCII is escaped. ("ANSI_1361", "Ação", b"A\\U+00E7\\U+00E3o"), @@ -60,7 +64,7 @@ fn unknown_ascii_and_utf8_code_pages_are_pure_ascii() { } } -/// Code page bytes that happen to form valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, which is U+0472 +/// Code page bytes that happen to form valid UTF-8 (`СІ` in `ANSI_1251` is D1 B2, which is U+0472 /// in UTF-8) would be read back as UTF-8 from an ASCII file, so that file is written with /// escapes. Binary DXF reads strings by the header's code page and keeps the bytes. #[test] @@ -77,6 +81,27 @@ fn code_page_bytes_that_are_valid_utf8_are_escaped_in_ascii() { assert!(contains(&write_ascii_in(&file("ANSI_1251", "Слой"), "ANSI_1251"), b"\xD1\xEB\xEE\xE9")); } +/// `write_ascii_in_page`, for a reader that decodes by `$DWGCODEPAGE` and never guesses UTF-8 (the +/// DWG bridge's acadrust), keeps the code page bytes even when they form valid UTF-8 (`é` in +/// ANSI_1252 is C3 A9, `é` in UTF-8), escapes only what the code page lacks, and escapes a +/// backslash that starts an escape. Otherwise it writes what `write_ascii_in` writes. +#[test] +fn page_writer_keeps_code_page_bytes_that_are_valid_utf8() { + let tags = file("ANSI_1252", "é Média Ω \\U+00E9"); + let page = write_ascii_in_page(&tags, "ANSI_1252"); + assert!(contains(&page, b"\xC3\xA9 M\xE9dia \\U+03A9 \\U+005CU+00E9"), "{:?}", String::from_utf8_lossy(&page)); + // Text whose bytes are all valid UTF-8: `write_ascii_in` still falls back to escapes, the + // page writer keeps them. + let utf8_like = file("ANSI_1252", "é"); + assert!(write_ascii_in(&utf8_like, "ANSI_1252").is_ascii()); + assert!(contains(&write_ascii_in_page(&utf8_like, "ANSI_1252"), b"\r\n\xC3\xA9\r\n")); + let plain = file("ANSI_1252", "Ação"); + assert_eq!(write_ascii_in_page(&plain, "ANSI_1252"), write_ascii_in(&plain, "ANSI_1252")); + for cp in ["EBCDIC", "ASCII", "UTF8"] { + assert_eq!(write_ascii_in_page(&plain, cp), write_ascii(&plain).into_bytes(), "{cp}"); + } +} + /// A literal `\U+` or `\M+` in the text (#2050) reads back as written alongside code page bytes, /// in ASCII and binary, including an ASCII file that falls back to escapes because its bytes /// would read as UTF-8. diff --git a/crates/dxf/tests/conformance.rs b/crates/dxf/tests/conformance.rs index 1eaec8fb..2c09e28e 100644 --- a/crates/dxf/tests/conformance.rs +++ b/crates/dxf/tests/conformance.rs @@ -1,8 +1,8 @@ //! Small independent byte fixtures for DXF tag types, text encoding, framing and failures. //! Handwritten examples, not files from a CAD application or generated by our writer. //! Background on the tag model and file structure: -//! https://ezdxf.readthedocs.io/en/stable/dxfinternals/dxftags.html -//! https://ezdxf.readthedocs.io/en/stable/dxfinternals/filestructure.html +//! +//! //! ASCII spacing is our writer's documented convention; binary fixtures specify actual bytes. use craft_dxf::{DxfError, Tag, Value, parse, parse_with_limit, write_ascii_as, write_binary}; @@ -16,8 +16,8 @@ fn fixtures() -> Vec<(Tag, &'static [u8], &'static [u8])> { (Tag::s(105, "AB12"), b"105\r\nAB12\r\n", b"\x69\0AB12\0"), (Tag::new(10, Value::Real(-3.25)), b" 10\r\n-3.25\r\n", b"\x0a\0\0\0\0\0\0\0\x0a\xc0"), (Tag::i(70, -32768), b" 70\r\n-32768\r\n", b"\x46\0\0\x80"), - (Tag::i(90, -2147483648), b" 90\r\n-2147483648\r\n", b"\x5a\0\0\0\0\x80"), - (Tag::i(160, 9223372036854775807), b"160\r\n9223372036854775807\r\n", b"\xa0\0\xff\xff\xff\xff\xff\xff\xff\x7f"), + (Tag::i(90, -2_147_483_648), b" 90\r\n-2147483648\r\n", b"\x5a\0\0\0\0\x80"), + (Tag::i(160, 9_223_372_036_854_775_807), b"160\r\n9223372036854775807\r\n", b"\xa0\0\xff\xff\xff\xff\xff\xff\xff\x7f"), (Tag::new(290, Value::Bool(false)), b"290\r\n0\r\n", b"\x22\x01\0"), (Tag::new(299, Value::Bool(true)), b"299\r\n1\r\n", b"\x2b\x01\x01"), (Tag::new(310, Value::Hex("00FF80".into())), b"310\r\n00FF80\r\n", b"\x36\x01\x03\0\xff\x80"), diff --git a/crates/dxf/tests/hex_chunks.rs b/crates/dxf/tests/hex_chunks.rs index 0be58f1a..61100b08 100644 --- a/crates/dxf/tests/hex_chunks.rs +++ b/crates/dxf/tests/hex_chunks.rs @@ -4,8 +4,10 @@ //! code and following-record framing. Oversized binary read fixtures test tolerance only. //! Pending writer regressions extend the chunk-size finding in issue #1025. //! Public DXF Reference, Group Codes in Numerical Order and Binary DXF Files: -//! https://help.autodesk.com/cloudhelp/2025/ENU/AutoCAD-DXF/files/GUID-3F0380A5-1C15-464D-BC66-2C5F094BCFB9.htm -//! https://help.autodesk.com/cloudhelp/2025/ENU/AutoCAD-DXF/files/GUID-FC1C3C69-DBC2-49E4-893A-000D6538C0FE.htm +//! +//! + +use std::fmt::Write as _; use craft_dxf::{Tag, Value, parse, write_ascii_as, write_binary}; @@ -17,7 +19,10 @@ fn payload(length: usize) -> Vec { } fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02X}")).collect() + bytes.iter().fold(String::new(), |mut out, byte| { + let _ = write!(out, "{byte:02X}"); + out + }) } fn input(code: i32, bytes: &[u8]) -> [Tag; 2] { diff --git a/crates/dxf/tests/roundtrip.rs b/crates/dxf/tests/roundtrip.rs index 1d9dbb91..87f1e812 100644 --- a/crates/dxf/tests/roundtrip.rs +++ b/crates/dxf/tests/roundtrip.rs @@ -122,7 +122,11 @@ fn hex_chunks_keep_bytes_and_order() { // Larger payloads need chunk-size/reassembly checks rather than exact tag equality; // see hex_chunks.rs for the independent wire checks and pending writer regressions. for length in [0, 1, 2, 126, 127] { - let hex: String = (0..length).map(|i| format!("{:02X}", (i * 37) % 256)).collect(); + let hex: String = (0..length).fold(String::new(), |mut out, i| { + use std::fmt::Write as _; + let _ = write!(out, "{:02X}", (i * 37) % 256); + out + }); let tags: Vec<_> = (310..=319).chain([1004]).map(|code| Tag::new(code, Value::Hex(hex.clone()))).collect(); check(&format!("hex/bytes={length}"), &tags); }