From ebb40fa0c9b6eae718a2c0bd9da594f782782d80 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:58:05 -0500 Subject: [PATCH] craft-dxf: R12 binary, more code pages, borrowing sections, exponent reals (ports of open CadCraft PRs) (#847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * craft-dxf: the ASCII reader unescapes only text that holds a backslash Only a string with a backslash can hold a \U+ or \M+ escape; every other string moves into the tag as it is instead of being copied. The EOF check reads the value before it moves. Ported from storytold/cadcraft#1833. * craft-dxf: write_ascii_in_page writes the code page without the UTF-8 fallback write_ascii_in falls back to pure ASCII when the code page bytes happen to be valid UTF-8, for readers that guess UTF-8. A reader that decodes a pre-2007 DXF strictly by its $DWGCODEPAGE needs the code page bytes. Ported from storytold/cadcraft#2851. * craft-dxf: behaviour-preserving cleanups Exact group-code mappings in one place, documented errors on the fallible public parsers, and simpler option, result and formatting idioms. No change to parsing, writing, numeric conversion, encoding or the public API. Ported from storytold/cadcraft#2571. * craft-dxf: read R12 and earlier binary DXF (1-byte group codes) Binary DXF written before R13 stores each group code in one byte, with 255 escaping a 2-byte code. The reader only knew the R13+ layout, where codes are two bytes, and also treated 255 as an escape there. The layout is now told from the first tag; 255 is an escape only in the 1-byte layout and an ordinary code in R13+. Ported from storytold/cadcraft#1499. * craft-dxf: section_slices, record_slices and Tag::str_ref borrow instead of copying sections() and records() copy every tag out of the parsed list, so a reader that keeps the list holds a large file's tags two or three times. The new functions return slices of the list. sections() and records() now build their copies from them, so there is one implementation of each split. Ported from storytold/cadcraft#1708. storytold/cadcraft#1588 solves the same problem by changing Section itself; this keeps the existing API. * craft-dxf: reals beyond 1e16 or below 1e-10 are written in exponent form; string_groups format_real wrote f64::MAX as 309 digits. Values outside 1e-10..1e16 are now written as AutoCAD writes them (1.0E+20) and read back exactly. string_groups splits text into the 250-character groups long strings are stored in (MAX_STRING_CHARS). Ported from storytold/cadcraft#1790. * craft-dxf: DOS 720/737/775/855/857/860/861/863/864/865/869 and ISO 8859-15 code pages; cheaper unescape and hex $DWGCODEPAGE values DOS720, 737, 775, 855, 857, 860, 861, 863, 864, 865, 869 and ISO8859_15 had no decoder and fell back to Windows-1252. ISO 8859-15 decodes through encoding_rs, the DOS pages through tables checked against the Unicode Consortium mapping files (dos_tables.rs). DOS 864's byte 0x25 is U+066A, so a writer escapes '%' there. U+FFFD is never written as a table byte. A string without an escape moves into its tag without a copy, cstr finds its terminator with one scan, and binary chunks are hex-encoded from a table. Ported from storytold/cadcraft#1271. Its group-code-255 change is carried by the R12 binary port: the 1-byte layout is detected only when the escaped code is one that needs the escape, so an R13+ file starting with code 255 still reads. * craft-dxf: Korean Johab (ANSI_1361) text and \M+4 escapes decode Johab had no decoder: files declaring ANSI_1361 were read as Windows-1252 and \M+4 escapes stayed literal. Hangul syllables are composed from the three 5-bit jamo indices (KS X 1001 annex 3); symbols and Hanja map onto KS X 1001 rows decoded as EUC-KR. Read only: a writer escapes instead. Ported from storytold/cadcraft#1911 (its DOS code pages are covered by the tables from storytold/cadcraft#1271). --------- Co-authored-by: mo <131036780+mbolling@users.noreply.github.com> Co-authored-by: ffishman <20211677+ffishman@users.noreply.github.com> Co-authored-by: Cédric Legrand Co-authored-by: valentinevg812-eng Co-authored-by: Mads Holten Rasmussen Co-authored-by: LastSkywalker Co-authored-by: vangel76 Co-authored-by: Francesco BV --- crates/dxf/src/codepage.rs | 313 +++++++++++++++++--- crates/dxf/src/dos_tables.rs | 168 +++++++++++ crates/dxf/src/lib.rs | 436 +++++++++++++++++++++------- crates/dxf/src/version.rs | 16 +- crates/dxf/tests/ascii_bom.rs | 2 +- crates/dxf/tests/audit_followups.rs | 139 +++++++++ crates/dxf/tests/code_page_write.rs | 31 +- crates/dxf/tests/conformance.rs | 8 +- crates/dxf/tests/hex_chunks.rs | 11 +- crates/dxf/tests/roundtrip.rs | 6 +- 10 files changed, 978 insertions(+), 152 deletions(-) create mode 100644 crates/dxf/src/dos_tables.rs create mode 100644 crates/dxf/tests/audit_followups.rs diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs index 11abc766..06c6adfb 100644 --- a/crates/dxf/src/codepage.rs +++ b/crates/dxf/src/codepage.rs @@ -6,8 +6,9 @@ //! Asian code page) escapes, which [`crate::parse`] decodes after the bytes. //! //! The Windows, ISO and East Asian code pages are decoded with `encoding_rs`; DOS 437, 850 and -//! 852 with the tables below (the upper halves of the IBM PC code pages; the lower halves are -//! ASCII). Decoding never fails: a byte sequence that is invalid in the code page (a truncated +//! 852 with the tables below, Korean Johab (`ANSI_1361`) by its definition in KS X 1001 annex 3, +//! and the other DOS pages with those of `dos_tables.rs` (the upper +//! halves of the IBM PC code pages; the lower halves are ASCII). Decoding never fails: a byte sequence that is invalid in the code page (a truncated //! double-byte character, an unassigned byte) becomes U+FFFD and the bytes after it decode //! normally, so line ends and the group structure survive. @@ -15,6 +16,9 @@ use std::borrow::Cow; use encoding_rs::Encoding; +use crate::Version; +use crate::dos_tables as dos; + /// How the bytes of a code page become text. #[derive(Clone, Copy, Debug)] pub(crate) enum Decoder { @@ -22,6 +26,21 @@ pub(crate) enum Decoder { Encoding(&'static Encoding), /// Single-byte DOS code page: ASCII, then these 128 characters for bytes 0x80–0xFF. Table(&'static [char; 128]), + /// DOS 864: like [`Decoder::Table`], but byte 0x25 is U+066A ARABIC PERCENT SIGN, not `%`. + Dos864, + /// Korean Johab (code page 1361). + Johab, +} + +/// What byte 0x25 of DOS 864 stands for. +const DOS864_PERCENT: char = '\u{066A}'; + +impl Decoder { + /// True when the ASCII character `c` is not what its byte means in this code page, so a + /// writer has to escape it. + pub(crate) fn shadows_ascii(self, c: char) -> bool { + matches!(self, Decoder::Dos864) && c == '%' + } } /// The decoder for a `$DWGCODEPAGE` value (case-insensitive), or `None` when CADCraft doesn't @@ -52,11 +71,24 @@ pub(crate) fn decoder(code_page: &str) -> Option { "ISO8859_6" | "ISO_8859_6" => e::ISO_8859_6, "ISO8859_7" | "ISO_8859_7" => e::ISO_8859_7, "ISO8859_8" | "ISO_8859_8" => e::ISO_8859_8, + "ISO8859_15" | "ISO_8859_15" => e::ISO_8859_15, "DOS866" => e::IBM866, "MACINTOSH" | "MAC_ROMAN" => e::MACINTOSH, "DOS437" => return Some(Decoder::Table(&DOS437)), "DOS850" => return Some(Decoder::Table(&DOS850)), "DOS852" => return Some(Decoder::Table(&DOS852)), + "DOS720" => return Some(Decoder::Table(&dos::DOS720)), + "DOS737" => return Some(Decoder::Table(&dos::DOS737)), + "DOS775" => return Some(Decoder::Table(&dos::DOS775)), + "DOS855" => return Some(Decoder::Table(&dos::DOS855)), + "DOS857" => return Some(Decoder::Table(&dos::DOS857)), + "DOS860" => return Some(Decoder::Table(&dos::DOS860)), + "DOS861" => return Some(Decoder::Table(&dos::DOS861)), + "DOS863" => return Some(Decoder::Table(&dos::DOS863)), + "DOS864" => return Some(Decoder::Dos864), + "DOS865" => return Some(Decoder::Table(&dos::DOS865)), + "DOS869" => return Some(Decoder::Table(&dos::DOS869)), + "ANSI_1361" | "JOHAB" => return Some(Decoder::Johab), _ => return None, }; Some(Decoder::Encoding(enc)) @@ -86,21 +118,32 @@ pub(crate) fn encode_char(c: char, d: Decoder, out: &mut Vec) -> bool { let mut buf = [0; 4]; match d { // Not a pre-2007 code page ([`encoder`] never gives it): escaped. - Decoder::Utf8 => return false, + // Johab is read only: a writer escapes every character. + Decoder::Utf8 | Decoder::Johab => return false, Decoder::Encoding(e) => { let (bytes, _, unmappable) = e.encode(c.encode_utf8(&mut buf)); // A code page byte that is ASCII would read back as another character, and a NUL, // CR or LF would break the group, so only all-high-bit sequences or proper double-byte // characters are kept (their trail byte may be ASCII, as in Shift-JIS). - if unmappable || bytes.is_empty() || !bytes.first().is_some_and(|b| *b >= 0x80) || bytes.iter().any(|b| matches!(b, 0 | b'\r' | b'\n')) { + if unmappable || bytes.is_empty() || bytes.first().is_none_or(|b| *b < 0x80) || bytes.iter().any(|b| matches!(b, 0 | b'\r' | b'\n')) { return false; } out.extend_from_slice(&bytes); } + // U+FFFD marks unassigned bytes in the tables; it must be escaped, not written as one. + Decoder::Table(_) | Decoder::Dos864 if c == '\u{FFFD}' => return false, Decoder::Table(t) => { let Some(i) = t.iter().position(|&x| x == c).and_then(|i| u8::try_from(i).ok()) else { return false }; out.push(0x80 | i); } + Decoder::Dos864 => { + if c == DOS864_PERCENT { + out.push(b'%'); + } else { + let Some(i) = dos::DOS864.iter().position(|&x| x == c).and_then(|i| u8::try_from(i).ok()) else { return false }; + out.push(0x80 | i); + } + } } true } @@ -113,6 +156,25 @@ pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { Err(_) => encoding_rs::WINDOWS_1252.decode_without_bom_handling(bytes).0, }, Decoder::Encoding(e) => e.decode_without_bom_handling(bytes).0, + Decoder::Johab => match std::str::from_utf8(bytes) { + Ok(s) if s.is_ascii() => Cow::Borrowed(s), + _ => Cow::Owned(decode_johab(bytes)), + }, + Decoder::Dos864 => { + if bytes.is_ascii() && !bytes.contains(&b'%') { + return Cow::Borrowed(std::str::from_utf8(bytes).unwrap_or_default()); + } + Cow::Owned( + bytes + .iter() + .map(|&b| match b { + b'%' => DOS864_PERCENT, + _ if b.is_ascii() => char::from(b), + _ => dos::DOS864.get(usize::from(b & 0x7F)).copied().unwrap_or('\u{FFFD}'), + }) + .collect(), + ) + } Decoder::Table(t) => match std::str::from_utf8(bytes) { Ok(s) if s.is_ascii() => Cow::Borrowed(s), _ => Cow::Owned( @@ -125,18 +187,18 @@ pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { } } -/// The `$DWGCODEPAGE` value of an ASCII DXF file: the line two lines after a `$DWGCODEPAGE` -/// line (the group code `3` between them), looked for in the HEADER section only (the scan stops -/// at the first `ENDSEC`). Works on the raw bytes, before they are decoded. -fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { +/// The value of the header variable `name` (group `code` after it) of an ASCII DXF file: looked +/// for in the HEADER section only (the scan stops at the first `ENDSEC`). Works on the raw bytes, +/// before they are decoded. +fn ascii_header_value<'a>(bytes: &'a [u8], name: &[u8], code: &[u8]) -> Option<&'a [u8]> { let mut lines = bytes.split(|&b| b == b'\n' || b == b'\r').map(<[u8]>::trim_ascii).filter(|l| !l.is_empty()); while let Some(line) = lines.next() { if line == b"ENDSEC" { return None; } - if line.eq_ignore_ascii_case(b"$DWGCODEPAGE") { + if line.eq_ignore_ascii_case(name) { return match (lines.next(), lines.next()) { - (Some(b"3"), Some(value)) => Some(value), + (Some(c), Some(value)) if c == code => Some(value), _ => None, }; } @@ -144,13 +206,118 @@ fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { None } +/// Decode Johab text: ASCII, double-byte Hangul (lead bytes 0x84–0xD3), symbols (0xD9–0xDE) and +/// Hanja (0xE0–0xF9). An unassigned code in that area is one U+FFFD for its two bytes; any other +/// byte that can't start a character is one U+FFFD and the next byte starts afresh. +fn decode_johab(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if b.is_ascii() { + out.push(char::from(b)); + i += 1; + continue; + } + let trail = bytes.get(i + 1).copied(); + match trail.and_then(|t| johab_char(b, t)) { + Some(c) => { + out.push(c); + i += 2; + } + None => { + out.push('\u{FFFD}'); + i += if trail.is_some_and(|t| johab_lead(b) && johab_trail(b, t)) { 2 } else { 1 }; + } + } + } + out +} + +fn johab_lead(b: u8) -> bool { + matches!(b, 0x84..=0xD3 | 0xD8..=0xDE | 0xE0..=0xF9) +} + +fn johab_trail(lead: u8, t: u8) -> bool { + if lead <= 0xD3 { matches!(t, 0x41..=0x7E | 0x81..=0xFE) } else { matches!(t, 0x31..=0x7E | 0x91..=0xFE) } +} + +/// The character of the Johab code `lead trail`, if it has one. +pub(crate) fn johab_char(lead: u8, trail: u8) -> Option { + if !johab_lead(lead) || !johab_trail(lead, trail) { + return None; + } + if lead <= 0xD3 { + return johab_hangul((u16::from(lead) << 8) | u16::from(trail)); + } + // Symbols and Hanja: each lead byte holds two KS X 1001 rows of 94 cells, the first in + // trail bytes 0x31–0x7E and 0x91–0xA0, the second in 0xA1–0xFE (0xD8 is user-defined). + let first = match lead { + 0xD9..=0xDE => (lead - 0xD9) * 2 + 0x21, + 0xE0..=0xF9 => (lead - 0xE0) * 2 + 0x4A, + _ => return None, + }; + let (row, col) = match trail { + 0x31..=0x7E => (first, trail - 0x31 + 0x21), + 0x91..=0xA0 => (first, trail - 0x91 + 0x6F), + _ => (first + 1, trail.saturating_sub(0xA1) + 0x21), + }; + let euc = [row | 0x80, col | 0x80]; + let text = encoding_rs::EUC_KR.decode_without_bom_handling_and_without_replacement(&euc)?; + let mut it = text.chars(); + it.next().filter(|_| it.next().is_none()) +} + +/// A Johab Hangul code (bit 15 set, then 5 bits each for the initial, medial and final jamo): +/// a precomposed syllable, or a lone jamo (as a compatibility jamo) when the others are fillers. +fn johab_hangul(code: u16) -> Option { + let (i, m, f) = ((code >> 10) & 0x1F, (code >> 5) & 0x1F, code & 0x1F); + // Initial: 1 filler, 2–20 the 19 consonants. Medial: 2 filler, then the 21 vowels in four + // runs (3–7, 10–15, 18–23, 26–29). Final: 1 none, 2–17 and 19–29 the 27 final consonants. + let lead = match i { + 1 => None, + 2..=20 => Some(i - 2), + _ => return None, + }; + let vowel = match m { + 2 => None, + 3..=7 => Some(m - 3), + 10..=15 => Some(m - 5), + 18..=23 => Some(m - 7), + 26..=29 => Some(m - 9), + _ => return None, + }; + let tail = match f { + 1 => 0, + 2..=17 => f - 1, + 19..=29 => f - 2, + _ => return None, + }; + // The compatibility jamo (offsets from U+3131) of the initial and of the final consonants. + const LEADS: [u16; 19] = [0, 1, 3, 6, 7, 8, 16, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]; + const TAILS: [u16; 27] = [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29]; + let cp = match (lead, vowel, tail) { + (Some(l), Some(v), t) => 0xAC00 + (u32::from(l) * 21 + u32::from(v)) * 28 + u32::from(t), + (None, None, 0) => 0x3164, + (Some(l), None, 0) => 0x3131 + u32::from(*LEADS.get(usize::from(l))?), + (None, Some(v), 0) => 0x314F + u32::from(v), + (None, None, t) => 0x3131 + u32::from(*TAILS.get(usize::from(t.saturating_sub(1)))?), + _ => return None, + }; + char::from_u32(cp) +} + +/// The `$DWGCODEPAGE` value of an ASCII DXF file. +fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { + ascii_header_value(bytes, b"$DWGCODEPAGE", b"3") +} + impl Decoder { - /// True for the multi-byte East Asian code pages (Shift-JIS, GBK, EUC-KR/Wansung, Big5). + /// True for the multi-byte East Asian code pages (Shift-JIS, GBK, EUC-KR/Wansung, Big5, Johab). /// Their short double-byte strings are often also valid UTF-8, so UTF-8 is only trusted for /// a whole file there (see [`decode_ascii_file`] and [`BinaryText`]). fn is_east_asian(self) -> bool { use encoding_rs as e; - matches!(self, Decoder::Encoding(x) if x == e::SHIFT_JIS || x == e::GBK || x == e::EUC_KR || x == e::BIG5) + matches!(self, Decoder::Johab) || matches!(self, Decoder::Encoding(x) if x == e::SHIFT_JIS || x == e::GBK || x == e::EUC_KR || x == e::BIG5) } } @@ -175,7 +342,10 @@ pub(crate) fn decode_text(bytes: &[u8], d: Decoder) -> Cow<'_, str> { /// pairs) can be valid UTF-8 by accident and must not be misread line by line. pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { if let Ok(s) = std::str::from_utf8(bytes) { - return Cow::Borrowed(s); + if !s.contains('%') || !is_pre_2007_dos864(bytes) { + return Cow::Borrowed(s); + } + return decode(bytes, Decoder::Dos864); } let d = match ascii_code_page(bytes) { Some(name) => { @@ -199,6 +369,13 @@ pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { Cow::Owned(out) } +/// True for an ASCII-only file whose header names `DOS864` and is not R2007+ (which is UTF-8). +fn is_pre_2007_dos864(bytes: &[u8]) -> bool { + bytes.is_ascii() + && ascii_code_page(bytes).is_some_and(|n| n.eq_ignore_ascii_case(b"DOS864")) + && !ascii_header_value(bytes, b"$ACADVER", b"1").is_some_and(|v| Version::for_acadver(&String::from_utf8_lossy(v)).is_some_and(Version::utf8)) +} + /// The text state of a binary DXF file, learnt from the header as the tags go by: `$ACADVER` /// (R2007+ strings are UTF-8) and `$DWGCODEPAGE` (the code page of older files). Both come /// first in the HEADER section and are ASCII, so every later string is decoded right. @@ -265,7 +442,7 @@ impl BinaryText { _ => None, } } - (1, Some(HeaderVar::AcadVer)) => self.utf8 = is_utf8_version(&s), + (1, Some(HeaderVar::AcadVer)) => self.utf8 = Version::for_acadver(&s).is_some_and(Version::utf8), (3, Some(HeaderVar::CodePage)) => { self.code_page = decoder(&s); if self.code_page.is_none() { @@ -278,22 +455,21 @@ impl BinaryText { } } -/// True for an `$ACADVER` of R2007 (AC1021) or later, whose text is UTF-8. -fn is_utf8_version(acadver: &str) -> bool { - acadver.trim().strip_prefix("AC").and_then(|n| n.parse::().ok()).is_some_and(|n| n >= 1021) -} - -/// The double-byte code page of a `\M+n` escape: 1 Japanese (Shift-JIS), 2 Traditional Chinese -/// (Big5), 3 Korean (Wansung), 5 Simplified Chinese (GB 2312). 4 (Korean Johab) is not -/// supported, so its escapes stay literal. -pub(crate) fn mif_encoding(n: char) -> Option<&'static Encoding> { - match n { - '1' => Some(encoding_rs::SHIFT_JIS), - '2' => Some(encoding_rs::BIG5), - '3' => Some(encoding_rs::EUC_KR), - '5' => Some(encoding_rs::GBK), - _ => None, - } +/// The character of a `\M+n` escape's two bytes in its double-byte code page `n`: 1 Japanese +/// (Shift-JIS), 2 Traditional Chinese (Big5), 3 Korean (Wansung), 4 Korean (Johab), 5 Simplified +/// Chinese (GB 2312). Other code pages and unassigned codes give `None`. +pub(crate) fn mif_char(n: char, bytes: [u8; 2]) -> Option { + let enc: &'static Encoding = match n { + '1' => encoding_rs::SHIFT_JIS, + '2' => encoding_rs::BIG5, + '3' => encoding_rs::EUC_KR, + '4' => return johab_char(bytes[0], bytes[1]), + '5' => encoding_rs::GBK, + _ => return None, + }; + let text = enc.decode_without_bom_handling_and_without_replacement(&bytes)?; + let mut it = text.chars(); + it.next().filter(|_| it.next().is_none()) } // Generated from the IBM PC code page definitions (bytes 0x80–0xFF in order). @@ -359,11 +535,75 @@ mod tests { assert_eq!(d(b"\xBD\xA2\x88w", "DOS852").as_deref(), Some("Żółw")); assert_eq!(d(b"\x91\xAB\xAE\xA9", "DOS866").as_deref(), Some("Слой")); assert_eq!(d("é".as_bytes(), "UTF8").as_deref(), Some("é")); - assert_eq!(d(b"x", "ANSI_1361"), None); assert_eq!(d(b"x", "EBCDIC"), None); assert_eq!(d(b"x", ""), None); } + #[test] + fn replacement_char_is_never_encoded_as_an_unassigned_byte() { + for cp in ["DOS857", "DOS864", "DOS437"] { + let d = encoder(cp).unwrap(); + let mut out = Vec::new(); + assert!(!encode_char('\u{FFFD}', d, &mut out), "{cp}"); + assert!(out.is_empty(), "{cp}"); + } + } + + #[test] + fn dos864_percent_is_the_arabic_percent_sign() { + // Byte 0x25 is U+066A in CP864 (Python's cp864 agrees); the rest of the low half is ASCII. + assert_eq!(decode_code_page(b"50%", "DOS864").as_deref(), Some("50\u{066A}")); + assert_eq!(decode_code_page(b"a%\x80%", "dos864").as_deref(), Some("a\u{066A}\u{00B0}\u{066A}")); + assert_eq!(decode_code_page(b"50&", "DOS864").as_deref(), Some("50&")); + // The other DOS pages keep `%`. + assert_eq!(decode_code_page(b"50%", "DOS437").as_deref(), Some("50%")); + // Every other low byte of every table is ASCII. + for cp in [ + "DOS437", "DOS850", "DOS852", "DOS720", "DOS737", "DOS775", "DOS855", "DOS857", "DOS860", "DOS861", "DOS863", "DOS864", "DOS865", + "DOS869", + ] { + for b in 0u8..0x80 { + if b == 0x25 && cp == "DOS864" { + continue; + } + assert_eq!(decode_code_page(&[b], cp).as_deref(), Some(char::from(b).to_string().as_str()), "{cp} {b:#x}"); + } + } + } + + #[test] + fn ascii_only_dos864_file_keeps_the_arabic_percent_sign() { + let file = |ver: &str, cp: &str| format!("0\nSECTION\n2\nHEADER\n9\n$ACADVER\n1\n{ver}\n9\n$DWGCODEPAGE\n3\n{cp}\n0\nENDSEC\n1\n50%\n"); + assert!(decode_ascii_file(file("AC1015", "DOS864").as_bytes()).ends_with("50\u{066A}\n")); + // Other code pages, and R2007+ files (UTF-8), keep `%`. + assert!(decode_ascii_file(file("AC1015", "DOS437").as_bytes()).ends_with("50%\n")); + assert!(decode_ascii_file(file("AC1021", "DOS864").as_bytes()).ends_with("50%\n")); + } + + /// Korean Johab (`ANSI_1361`): Hangul syllables composed from their jamo, lone jamo, + /// symbols and Hanja through KS X 1001. + #[test] + fn johab() { + let d = |bytes: &[u8]| decode_code_page(bytes, "ANSI_1361"); + // 한글 (U+D55C U+AE00) and 가 (U+AC00). + assert_eq!(d(b"\xD0\x65\x8B\x69 \x88\x61").as_deref(), Some("한글 가")); + // Lone initial ㄱ, lone vowel ㅏ, the Hangul filler. + assert_eq!(d(b"\x88\x41\x84\x61\x84\x41").as_deref(), Some("ㄱㅏ\u{3164}")); + // The ideographic space (KS X 1001 0x2121) and 伽 (0x4A21, the first Hanja). + assert_eq!(d(b"\xD9\x31\xE0\x31").as_deref(), Some("\u{3000}伽")); + assert_eq!(decode_code_page(b"\x88\x61", "johab").as_deref(), Some("가")); + // A truncated lead byte, a lead byte without a valid trail byte, a user-defined code. + assert_eq!(d(b"A\x88").as_deref(), Some("A\u{FFFD}")); + assert_eq!(d(b"\x88\x20B").as_deref(), Some("\u{FFFD} B")); + assert_eq!(d(b"\xD8\x31C").as_deref(), Some("\u{FFFD}C")); + // Every pair of bytes decodes to something without panicking. + for a in 0x80..=0xFFu8 { + for b in 0..=0xFFu8 { + assert!(!d(&[a, b, b'\n']).unwrap_or_default().is_empty()); + } + } + } + #[test] fn invalid_sequences_become_replacement_characters() { // A Shift-JIS lead byte cut off by the line end keeps the line end. @@ -381,8 +621,9 @@ mod tests { // Wrong group code, or the file ends early. assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n1\nANSI_1251\n"), None); assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n3"), None); - assert!(is_utf8_version("AC1021") && is_utf8_version(" AC1032 ")); - assert!(!is_utf8_version("AC1018") && !is_utf8_version("AC1009") && !is_utf8_version("garbage")); + let utf8 = |v: &str| Version::for_acadver(v).is_some_and(Version::utf8); + assert!(utf8("AC1021") && utf8(" AC1032 ")); + assert!(!utf8("AC1018") && !utf8("AC1009") && !utf8("garbage")); } #[test] @@ -390,8 +631,10 @@ mod tests { let one = |s: &str| crate::parse(format!("1\n{s}\n0\nEOF\n").as_bytes()).unwrap()[0].str(); assert_eq!(one("\\M+5BCFE\\M+1907D\\M+2B9CF\\M+3B5B5!"), "件図圖도!"); assert_eq!(one("\\M+5BCFE\\U+0021"), "件!"); - // Johab, unknown code pages, bad hex, short escapes and unassigned codes stay literal. - for s in ["\\M+4ABCD", "\\M+9BCFE", "\\M+5GGGG", "\\M+5BC", "\\M+", "\\M+50041", "\\M+5FFFF"] { + // Johab: 한. + assert_eq!(one("\\M+4D065"), "한"); + // Unknown code pages, bad hex, short escapes and unassigned codes stay literal. + for s in ["\\M+4D820", "\\M+9BCFE", "\\M+5GGGG", "\\M+5BC", "\\M+", "\\M+50041", "\\M+5FFFF"] { assert_eq!(one(s), s); } } diff --git a/crates/dxf/src/dos_tables.rs b/crates/dxf/src/dos_tables.rs new file mode 100644 index 00000000..b1826eae --- /dev/null +++ b/crates/dxf/src/dos_tables.rs @@ -0,0 +1,168 @@ +//! More DOS code pages for `$DWGCODEPAGE` (`DOS855` …): the upper halves (bytes 0x80–0xFF) of the IBM PC +//! code page definitions, as Unicode characters; the lower halves are ASCII. Bytes the code page leaves +//! unassigned are U+FFFD. +//! +//! Source check (the tables were checked against these files, not generated from them): +//! - DOS737, 775, 855, 857, 860, 861, 863, 864, 865 and 869 were checked byte for byte (all 128 bytes +//! 0x80–0xFF of each) against the Unicode Consortium mapping files CP737.TXT, CP775.TXT, CP855.TXT, +//! CP857.TXT, CP860.TXT, CP861.TXT, CP863.TXT, CP864.TXT, CP865.TXT and CP869.TXT in +//! . Bytes those files list as `#UNDEFINED` +//! are U+FFFD here. +//! - DOS720 has no Unicode Consortium mapping file, so it was checked only against Python's `cp720` +//! codec (not a published mapping file). 120 of the 128 bytes agree. The 8 bytes 0x80, 0x81, 0x84, 0x86, +//! 0x8D, 0x8E, 0x8F and 0x90 decode to U+FFFD here (unassigned), while that codec maps them to the C1 +//! controls U+0080..U+0090. + +pub(crate) const DOS720: [char; 128] = [ + '\u{FFFD}', '\u{FFFD}', '\u{00E9}', '\u{00E2}', '\u{FFFD}', '\u{00E0}', '\u{FFFD}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{0651}', '\u{0652}', '\u{00F4}', '\u{00A4}', '\u{0640}', '\u{00FB}', '\u{00F9}', + '\u{0621}', '\u{0622}', '\u{0623}', '\u{0624}', '\u{00A3}', '\u{0625}', '\u{0626}', '\u{0627}', '\u{0628}', '\u{0629}', '\u{062A}', '\u{062B}', + '\u{062C}', '\u{062D}', '\u{062E}', '\u{062F}', '\u{0630}', '\u{0631}', '\u{0632}', '\u{0633}', '\u{0634}', '\u{0635}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{0636}', '\u{0637}', '\u{0638}', '\u{0639}', '\u{063A}', '\u{0641}', '\u{00B5}', '\u{0642}', '\u{0643}', '\u{0644}', '\u{0645}', '\u{0646}', + '\u{0647}', '\u{0648}', '\u{0649}', '\u{064A}', '\u{2261}', '\u{064B}', '\u{064C}', '\u{064D}', '\u{064E}', '\u{064F}', '\u{0650}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS737: [char; 128] = [ + '\u{0391}', '\u{0392}', '\u{0393}', '\u{0394}', '\u{0395}', '\u{0396}', '\u{0397}', '\u{0398}', '\u{0399}', '\u{039A}', '\u{039B}', '\u{039C}', + '\u{039D}', '\u{039E}', '\u{039F}', '\u{03A0}', '\u{03A1}', '\u{03A3}', '\u{03A4}', '\u{03A5}', '\u{03A6}', '\u{03A7}', '\u{03A8}', '\u{03A9}', + '\u{03B1}', '\u{03B2}', '\u{03B3}', '\u{03B4}', '\u{03B5}', '\u{03B6}', '\u{03B7}', '\u{03B8}', '\u{03B9}', '\u{03BA}', '\u{03BB}', '\u{03BC}', + '\u{03BD}', '\u{03BE}', '\u{03BF}', '\u{03C0}', '\u{03C1}', '\u{03C3}', '\u{03C2}', '\u{03C4}', '\u{03C5}', '\u{03C6}', '\u{03C7}', '\u{03C8}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03C9}', '\u{03AC}', '\u{03AD}', '\u{03AE}', '\u{03CA}', '\u{03AF}', '\u{03CC}', '\u{03CD}', '\u{03CB}', '\u{03CE}', '\u{0386}', '\u{0388}', + '\u{0389}', '\u{038A}', '\u{038C}', '\u{038E}', '\u{038F}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{03AA}', '\u{03AB}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS775: [char; 128] = [ + '\u{0106}', '\u{00FC}', '\u{00E9}', '\u{0101}', '\u{00E4}', '\u{0123}', '\u{00E5}', '\u{0107}', '\u{0142}', '\u{0113}', '\u{0156}', '\u{0157}', + '\u{012B}', '\u{0179}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{014D}', '\u{00F6}', '\u{0122}', '\u{00A2}', '\u{015A}', + '\u{015B}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{00D7}', '\u{00A4}', '\u{0100}', '\u{012A}', '\u{00F3}', '\u{017B}', + '\u{017C}', '\u{017A}', '\u{201D}', '\u{00A6}', '\u{00A9}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{0141}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{0104}', '\u{010C}', '\u{0118}', '\u{0116}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{012E}', '\u{0160}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{0172}', '\u{016A}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{017D}', '\u{0105}', '\u{010D}', '\u{0119}', '\u{0117}', + '\u{012F}', '\u{0161}', '\u{0173}', '\u{016B}', '\u{017E}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{014C}', '\u{0143}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{0144}', '\u{0136}', '\u{0137}', '\u{013B}', '\u{013C}', + '\u{0146}', '\u{0112}', '\u{0145}', '\u{2019}', '\u{00AD}', '\u{00B1}', '\u{201C}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{201E}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS855: [char; 128] = [ + '\u{0452}', '\u{0402}', '\u{0453}', '\u{0403}', '\u{0451}', '\u{0401}', '\u{0454}', '\u{0404}', '\u{0455}', '\u{0405}', '\u{0456}', '\u{0406}', + '\u{0457}', '\u{0407}', '\u{0458}', '\u{0408}', '\u{0459}', '\u{0409}', '\u{045A}', '\u{040A}', '\u{045B}', '\u{040B}', '\u{045C}', '\u{040C}', + '\u{045E}', '\u{040E}', '\u{045F}', '\u{040F}', '\u{044E}', '\u{042E}', '\u{044A}', '\u{042A}', '\u{0430}', '\u{0410}', '\u{0431}', '\u{0411}', + '\u{0446}', '\u{0426}', '\u{0434}', '\u{0414}', '\u{0435}', '\u{0415}', '\u{0444}', '\u{0424}', '\u{0433}', '\u{0413}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{0445}', '\u{0425}', '\u{0438}', '\u{0418}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{0439}', '\u{0419}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{043A}', '\u{041A}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{043B}', '\u{041B}', '\u{043C}', '\u{041C}', + '\u{043D}', '\u{041D}', '\u{043E}', '\u{041E}', '\u{043F}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{041F}', '\u{044F}', '\u{2580}', + '\u{042F}', '\u{0440}', '\u{0420}', '\u{0441}', '\u{0421}', '\u{0442}', '\u{0422}', '\u{0443}', '\u{0423}', '\u{0436}', '\u{0416}', '\u{0432}', + '\u{0412}', '\u{044C}', '\u{042C}', '\u{2116}', '\u{00AD}', '\u{044B}', '\u{042B}', '\u{0437}', '\u{0417}', '\u{0448}', '\u{0428}', '\u{044D}', + '\u{042D}', '\u{0449}', '\u{0429}', '\u{0447}', '\u{0427}', '\u{00A7}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS857: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{0131}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{0130}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{015E}', '\u{015F}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{011E}', '\u{011F}', '\u{00BF}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{00C0}', '\u{00A9}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{00A2}', '\u{00A5}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{00E3}', '\u{00C3}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{00BA}', '\u{00AA}', '\u{00CA}', '\u{00CB}', + '\u{00C8}', '\u{FFFD}', '\u{00CD}', '\u{00CE}', '\u{00CF}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{00A6}', '\u{00CC}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{00D2}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{FFFD}', '\u{00D7}', '\u{00DA}', '\u{00DB}', '\u{00D9}', + '\u{00EC}', '\u{00FF}', '\u{00AF}', '\u{00B4}', '\u{00AD}', '\u{00B1}', '\u{FFFD}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{00B8}', + '\u{00B0}', '\u{00A8}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS860: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E3}', '\u{00E0}', '\u{00C1}', '\u{00E7}', '\u{00EA}', '\u{00CA}', '\u{00E8}', '\u{00CD}', + '\u{00D4}', '\u{00EC}', '\u{00C3}', '\u{00C2}', '\u{00C9}', '\u{00C0}', '\u{00C8}', '\u{00F4}', '\u{00F5}', '\u{00F2}', '\u{00DA}', '\u{00F9}', + '\u{00CC}', '\u{00D5}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00D9}', '\u{20A7}', '\u{00D3}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{00D2}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS861: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00D0}', + '\u{00F0}', '\u{00DE}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00FE}', '\u{00FB}', '\u{00DD}', + '\u{00FD}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{20A7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00C1}', '\u{00CD}', '\u{00D3}', '\u{00DA}', '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS863: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00C2}', '\u{00E0}', '\u{00B6}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{2017}', '\u{00C0}', '\u{00A7}', '\u{00C9}', '\u{00C8}', '\u{00CA}', '\u{00F4}', '\u{00CB}', '\u{00CF}', '\u{00FB}', '\u{00F9}', + '\u{00A4}', '\u{00D4}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00D9}', '\u{00DB}', '\u{0192}', '\u{00A6}', '\u{00B4}', '\u{00F3}', '\u{00FA}', + '\u{00A8}', '\u{00B8}', '\u{00B3}', '\u{00AF}', '\u{00CE}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00BE}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS864: [char; 128] = [ + '\u{00B0}', '\u{00B7}', '\u{2219}', '\u{221A}', '\u{2592}', '\u{2500}', '\u{2502}', '\u{253C}', '\u{2524}', '\u{252C}', '\u{251C}', '\u{2534}', + '\u{2510}', '\u{250C}', '\u{2514}', '\u{2518}', '\u{03B2}', '\u{221E}', '\u{03C6}', '\u{00B1}', '\u{00BD}', '\u{00BC}', '\u{2248}', '\u{00AB}', + '\u{00BB}', '\u{FEF7}', '\u{FEF8}', '\u{FFFD}', '\u{FFFD}', '\u{FEFB}', '\u{FEFC}', '\u{FFFD}', '\u{00A0}', '\u{00AD}', '\u{FE82}', '\u{00A3}', + '\u{00A4}', '\u{FE84}', '\u{FFFD}', '\u{FFFD}', '\u{FE8E}', '\u{FE8F}', '\u{FE95}', '\u{FE99}', '\u{060C}', '\u{FE9D}', '\u{FEA1}', '\u{FEA5}', + '\u{0660}', '\u{0661}', '\u{0662}', '\u{0663}', '\u{0664}', '\u{0665}', '\u{0666}', '\u{0667}', '\u{0668}', '\u{0669}', '\u{FED1}', '\u{061B}', + '\u{FEB1}', '\u{FEB5}', '\u{FEB9}', '\u{061F}', '\u{00A2}', '\u{FE80}', '\u{FE81}', '\u{FE83}', '\u{FE85}', '\u{FECA}', '\u{FE8B}', '\u{FE8D}', + '\u{FE91}', '\u{FE93}', '\u{FE97}', '\u{FE9B}', '\u{FE9F}', '\u{FEA3}', '\u{FEA7}', '\u{FEA9}', '\u{FEAB}', '\u{FEAD}', '\u{FEAF}', '\u{FEB3}', + '\u{FEB7}', '\u{FEBB}', '\u{FEBF}', '\u{FEC1}', '\u{FEC5}', '\u{FECB}', '\u{FECF}', '\u{00A6}', '\u{00AC}', '\u{00F7}', '\u{00D7}', '\u{FEC9}', + '\u{0640}', '\u{FED3}', '\u{FED7}', '\u{FEDB}', '\u{FEDF}', '\u{FEE3}', '\u{FEE7}', '\u{FEEB}', '\u{FEED}', '\u{FEEF}', '\u{FEF3}', '\u{FEBD}', + '\u{FECC}', '\u{FECE}', '\u{FECD}', '\u{FEE1}', '\u{FE7D}', '\u{0651}', '\u{FEE5}', '\u{FEE9}', '\u{FEEC}', '\u{FEF0}', '\u{FEF2}', '\u{FED0}', + '\u{FED5}', '\u{FEF5}', '\u{FEF6}', '\u{FEDD}', '\u{FED9}', '\u{FEF1}', '\u{25A0}', '\u{FFFD}', +]; + +pub(crate) const DOS865: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{20A7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00A4}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +pub(crate) const DOS869: [char; 128] = [ + '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{0386}', '\u{FFFD}', '\u{00B7}', '\u{00AC}', '\u{00A6}', '\u{2018}', + '\u{2019}', '\u{0388}', '\u{2015}', '\u{0389}', '\u{038A}', '\u{03AA}', '\u{038C}', '\u{FFFD}', '\u{FFFD}', '\u{038E}', '\u{03AB}', '\u{00A9}', + '\u{038F}', '\u{00B2}', '\u{00B3}', '\u{03AC}', '\u{00A3}', '\u{03AD}', '\u{03AE}', '\u{03AF}', '\u{03CA}', '\u{0390}', '\u{03CC}', '\u{03CD}', + '\u{0391}', '\u{0392}', '\u{0393}', '\u{0394}', '\u{0395}', '\u{0396}', '\u{0397}', '\u{00BD}', '\u{0398}', '\u{0399}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{039A}', '\u{039B}', '\u{039C}', '\u{039D}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{039E}', '\u{039F}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{03A0}', '\u{03A1}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{03A3}', '\u{03A4}', '\u{03A5}', '\u{03A6}', '\u{03A7}', + '\u{03A8}', '\u{03A9}', '\u{03B1}', '\u{03B2}', '\u{03B3}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{03B4}', '\u{03B5}', '\u{2580}', + '\u{03B6}', '\u{03B7}', '\u{03B8}', '\u{03B9}', '\u{03BA}', '\u{03BB}', '\u{03BC}', '\u{03BD}', '\u{03BE}', '\u{03BF}', '\u{03C0}', '\u{03C1}', + '\u{03C3}', '\u{03C2}', '\u{03C4}', '\u{0384}', '\u{00AD}', '\u{00B1}', '\u{03C5}', '\u{03C6}', '\u{03C7}', '\u{00A7}', '\u{03C8}', '\u{0385}', + '\u{00B0}', '\u{00A8}', '\u{03C9}', '\u{03CB}', '\u{03B0}', '\u{03CE}', '\u{25A0}', '\u{00A0}', +]; diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 286219fc..7694eb01 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -7,7 +7,6 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] -use std::fmt::Write as _; use std::io::Write as _; mod version; @@ -52,6 +51,10 @@ impl Value { Value::Bool(b) => i64::from(*b).to_string(), } } + #[expect( + clippy::cast_precision_loss, + reason = "DXF numeric accessors intentionally convert integer values to f64, accepting precision loss beyond 2^53." + )] pub fn as_f64(&self) -> f64 { match self { Value::Real(r) if r.is_finite() => *r, @@ -61,6 +64,7 @@ impl Value { _ => 0.0, } } + #[expect(clippy::cast_possible_truncation, reason = "DXF numeric accessors intentionally truncate finite real values to i64.")] pub fn as_i64(&self) -> i64 { match self { Value::Int(i) => *i, @@ -96,6 +100,13 @@ impl Tag { pub fn str(&self) -> String { self.value.as_str() } + /// The value when it is text, without a copy. + pub fn str_ref(&self) -> Option<&str> { + match &self.value { + Value::Str(s) | Value::Hex(s) => Some(s), + _ => None, + } + } pub fn f64(&self) -> f64 { self.value.as_f64() } @@ -118,46 +129,21 @@ pub enum Kind { pub fn kind_of(code: i32) -> Kind { match code { - 0..=9 => Kind::Str, - 10..=59 => Kind::Real, - 60..=79 => Kind::Int16, - 90..=99 => Kind::Int32, - 100..=109 => Kind::Str, - 110..=149 => Kind::Real, + 10..=59 | 110..=149 | 210..=239 | 460..=469 | 1010..=1059 => Kind::Real, + 60..=79 | 170..=179 | 270..=289 | 370..=389 | 400..=409 | 1060..=1070 => Kind::Int16, + 90..=99 | 420..=429 | 440..=449 | 450..=459 | 1071 => Kind::Int32, 160..=169 => Kind::Int64, - 170..=179 => Kind::Int16, - 210..=239 => Kind::Real, - 270..=289 => Kind::Int16, 290..=299 => Kind::Bool, - 300..=309 => Kind::Str, - 310..=319 => Kind::Hex, - 320..=369 => Kind::Str, - 370..=389 => Kind::Int16, - 390..=399 => Kind::Str, - 400..=409 => Kind::Int16, - 410..=419 => Kind::Str, - 420..=429 => Kind::Int32, - 430..=439 => Kind::Str, - 440..=449 => Kind::Int32, - 450..=459 => Kind::Int32, - 460..=469 => Kind::Real, - 470..=479 => Kind::Str, - 480..=481 => Kind::Str, - 999 => Kind::Str, - 1000..=1003 => Kind::Str, - 1004 => Kind::Hex, - 1005..=1009 => Kind::Str, - 1010..=1059 => Kind::Real, - 1060..=1070 => Kind::Int16, - 1071 => Kind::Int32, + 310..=319 | 1004 => Kind::Hex, _ => Kind::Str, } } +#[expect(clippy::cast_possible_truncation, reason = "DXF numeric parsing intentionally truncates finite real values to i64.")] fn typed(code: i32, text: &str) -> Value { let t = text.trim(); match kind_of(code) { - Kind::Real => t.parse::().ok().filter(|v| v.is_finite()).map(Value::Real).unwrap_or(Value::Real(0.0)), + Kind::Real => t.parse::().ok().filter(|v| v.is_finite()).map_or(Value::Real(0.0), Value::Real), Kind::Int16 | Kind::Int32 | Kind::Int64 => { // A float spelling ("1e3", "12.0") is accepted when finite, clamped to what the // group's width holds, signed or unsigned (files write 16-bit flags up to 65535, as @@ -168,8 +154,7 @@ fn typed(code: i32, text: &str) -> Value { _ => (i64::MIN, i64::MAX), }; t.parse::() - .map(Value::Int) - .unwrap_or_else(|_| Value::Int(t.parse::().ok().filter(|f| f.is_finite()).map_or(0, |f| (f as i64).clamp(lo, hi)))) + .map_or_else(|_| Value::Int(t.parse::().ok().filter(|f| f.is_finite()).map_or(0, |f| (f as i64).clamp(lo, hi))), Value::Int) } Kind::Bool => Value::Bool(t.parse::().unwrap_or(0) != 0), Kind::Hex => Value::Hex(normalize_hex(t)), @@ -201,8 +186,15 @@ fn max_tags_from(v: Option<&str>) -> usize { mod codepage; pub use codepage::decode_code_page; +mod dos_tables; /// Parse a DXF file (ASCII or binary). +/// +/// # Errors +/// +/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair, +/// truncated binary data, or more than [`MAX_TAGS`] group codes. For ASCII input, invalid numeric +/// text becomes `0.0` for real values or `0` for integer values, and an EOF tag is not required. pub fn parse(bytes: &[u8]) -> Result> { parse_with_limit(bytes, max_tags()) } @@ -210,6 +202,12 @@ pub fn parse(bytes: &[u8]) -> Result> { /// [`parse`] with an explicit group-code limit. For ASCII DXF the limit is checked up front from /// the line count, before the text is decoded or any tag allocated, so an oversized file fails in /// milliseconds instead of after it has filled memory. +/// +/// # Errors +/// +/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair, +/// truncated binary data, or more than `max_tags` group codes. For ASCII input, invalid numeric +/// text becomes `0.0` for real values or `0` for integer values, and an EOF tag is not required. pub fn parse_with_limit(bytes: &[u8], max_tags: usize) -> Result> { if let Some(body) = bytes.strip_prefix(BINARY_SENTINEL) { return parse_binary_with_limit(body, max_tags); @@ -254,16 +252,13 @@ pub fn mif_escape_at(chars: &[char], i: usize) -> Option { if chars.get(i) != Some(&'\\') || chars.get(i + 1) != Some(&'M') || chars.get(i + 2) != Some(&'+') { return None; } - let enc = codepage::mif_encoding(*chars.get(i + 3)?)?; + let n = *chars.get(i + 3)?; let hex = chars.get(i + 4..i + 8)?; if !hex.iter().all(char::is_ascii_hexdigit) { return None; } let code = hex.iter().try_fold(0u16, |acc, c| Some(acc * 16 + u16::try_from(c.to_digit(16)?).ok()?))?; - let bytes = code.to_be_bytes(); - let text = enc.decode_without_bom_handling_and_without_replacement(&bytes)?; - let mut it = text.chars(); - it.next().filter(|_| it.next().is_none()) + codepage::mif_char(n, code.to_be_bytes()) } /// Unescape the `\U+XXXX` sequences pre-R2007 DXF uses for characters outside the file's code @@ -271,36 +266,68 @@ pub fn mif_escape_at(chars: &[char], i: usize) -> Option { /// escapes), and the `\M+nXXXX` sequences for double-byte East Asian characters. Malformed /// escapes, unpaired surrogates and unknown `\M+` code pages are kept as literal text. fn unescape_unicode(s: &str) -> String { - if !s.contains("\\U+") && !s.contains("\\M+") { + if !has_escape(s) { return s.to_string(); } + unescape_found(s) +} + +/// [`unescape_unicode`] for a string the caller owns: without escapes (nearly always) it is +/// returned as it is, with no copy. +fn unescape_owned(s: String) -> String { + if has_escape(&s) { unescape_found(&s) } else { s } +} + +fn has_escape(s: &str) -> bool { + s.contains("\\U+") || s.contains("\\M+") +} + +/// The longest escape: a surrogate pair of two `\U+XXXX` (14 characters). +const MAX_ESCAPE_CHARS: usize = 14; + +fn unescape_found(s: &str) -> String { let mut out = String::with_capacity(s.len()); - let chars: Vec = s.chars().collect(); - let mut i = 0; - while let Some(&c) = chars.get(i) { - if let Some(ch) = mif_escape_at(&chars, i) { - out.push(ch); - i += 8; - continue; + let mut rest = s; + // Everything up to the next backslash is copied as it is; an escape is looked for in a window + // of the next few characters only, so no copy of the whole string (4 bytes a character) is made. + while let Some(at) = rest.find('\\') { + let (plain, tail) = rest.split_at(at); + out.push_str(plain); + let mut buf = ['\0'; MAX_ESCAPE_CHARS]; + let mut n = 0; + for (slot, c) in buf.iter_mut().zip(tail.chars()) { + *slot = c; + n += 1; } - if let Some(unit) = escape_at(&chars, i) { + let window = buf.get(..n).unwrap_or_default(); + let (ch, used) = if let Some(ch) = mif_escape_at(window, 0) { + (Some(ch), 8) + } else if let Some(unit) = escape_at(window, 0) { if let Some(ch) = char::from_u32(unit) { - out.push(ch); - i += 7; - continue; - } - if (0xD800..0xDC00).contains(&unit) - && let Some(low) = escape_at(&chars, i + 7).filter(|u| (0xDC00..0xE000).contains(u)) - && let Some(ch) = char::from_u32(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)) + (Some(ch), 7) + } else if (0xD800..0xDC00).contains(&unit) + && let Some(low) = escape_at(window, 7).filter(|u| (0xDC00..0xE000).contains(u)) { + (char::from_u32(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)), 14) + } else { + (None, 0) + } + } else { + (None, 0) + }; + match ch { + Some(ch) => { out.push(ch); - i += 14; - continue; + // Every escape character is ASCII, so `used` characters are `used` bytes. + rest = tail.get(used..).unwrap_or_default(); + } + None => { + out.push('\\'); + rest = tail.get(1..).unwrap_or_default(); } } - out.push(c); - i += 1; } + out.push_str(rest); out } @@ -337,6 +364,13 @@ fn byte_lines(mut rest: &[u8]) -> impl Iterator { }) } +/// Parse ASCII DXF text into typed tags. +/// +/// # Errors +/// +/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair, +/// or too many tags. Invalid numeric text becomes `0.0` for real values or `0` for integer values. +/// A complete tag stream does not require an EOF tag. pub fn parse_ascii(text: &str) -> Result> { parse_ascii_counted(text, max_tags(), 0) } @@ -358,16 +392,17 @@ fn parse_ascii_counted(text: &str, max_tags: usize, expected: usize) -> Result Value::Str(unescape_unicode(&s)), + let v = match typed(code, val) { + // Only text with a backslash can hold an escape; the rest moves in as it is. + Value::Str(s) => Value::Str(unescape_owned(s)), o => o, }; + let eof = code == 0 && matches!(&v, Value::Str(s) if s.trim_end() == "EOF"); tags.push(Tag { code, value: v }); if tags.len() > max_tags { return Err(DxfError::TooLarge { tags: tags.len(), limit: max_tags }); } - if code == 0 && tags.last().is_some_and(|t| t.str().trim_end() == "EOF") { + if eof { break; } } @@ -414,24 +449,38 @@ impl<'a> Cur<'a> { } /// The bytes of a zero-terminated string (without the zero), decoded by the caller. fn cstr(&mut self) -> Result<&'a [u8]> { - let start = self.p; - while self.p < self.b.len() && self.b.get(self.p) != Some(&0) { - self.p += 1; - } - let s = self.b.get(start..self.p).unwrap_or(&[]); - if self.p >= self.b.len() { - return Err(DxfError::Eof); - } - self.p += 1; - Ok(s) + let rest = self.b.get(self.p..).unwrap_or(&[]); + let len = rest.iter().position(|&b| b == 0).ok_or(DxfError::Eof)?; + self.p += len + 1; + Ok(rest.get(..len).unwrap_or(&[])) } } -/// Binary DXF (R13+: 2-byte group codes). +const HEX_DIGITS: &[u8; 16] = b"0123456789ABCDEF"; + +/// Binary DXF, the body after the sentinel: R13+ (2-byte group codes) or R12 and earlier (1-byte +/// group codes, where 255 is followed by a 2-byte code). +/// +/// # Errors +/// +/// Returns an error for an empty or malformed file, an unexpected end of file, or too many tags. pub fn parse_binary(b: &[u8]) -> Result> { parse_binary_with_limit(b, max_tags()) } +/// Whether a binary DXF body uses the pre-R13 layout with 1-byte group codes, told from its first +/// tag: R13+ starts with a 2-byte code (`00 00` for the usual `0 SECTION`, `E7 03` for a 999 +/// comment), the old layout with a 1-byte `00` followed by the string's first character, or with +/// the 255 escape and a code that needs it (255 to 1071; an R13+ file whose first group is code 255 +/// has the string's first character there instead). +fn legacy_binary_layout(b: &[u8]) -> bool { + match b { + [0, next, ..] => *next != 0, + [255, lo, hi, ..] => (255..=1071).contains(&u16::from_le_bytes([*lo, *hi])), + _ => false, + } +} + fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { let mut text = codepage::BinaryText::default(); let tags = parse_binary_pass(b, max_tags, &mut text)?; @@ -447,11 +496,17 @@ fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { fn parse_binary_pass(b: &[u8], max_tags: usize, text: &mut codepage::BinaryText) -> Result> { let mut c = Cur { b, p: 0 }; let mut tags = Vec::new(); + let legacy = legacy_binary_layout(b); while c.p < b.len() { - let mut code = i32::from(c.i16()?); - if code == 255 { - code = i32::from(c.i16()?); - } + // 255 escapes a 2-byte code only in the 1-byte layout; in R13+ it is an ordinary code. + let code = if legacy { + match c.u8()? { + 255 => i32::from(c.i16()?), + x => i32::from(x), + } + } else { + i32::from(c.i16()?) + }; let v = match kind_of(code) { Kind::Str => Value::Str(unescape_unicode(&text.string(code, c.cstr()?))), Kind::Real => Value::Real(Some(c.f64()?).filter(|v| v.is_finite()).unwrap_or(0.0)), @@ -463,8 +518,9 @@ fn parse_binary_pass(b: &[u8], max_tags: usize, text: &mut codepage::BinaryText) let n = usize::from(c.u8()?); let bytes = c.take(n)?; let mut s = String::with_capacity(n * 2); - for x in bytes { - let _ = write!(s, "{x:02X}"); + for &x in bytes { + s.push(char::from(HEX_DIGITS[usize::from(x >> 4)])); + s.push(char::from(HEX_DIGITS[usize::from(x & 15)])); } Value::Hex(s) } @@ -489,10 +545,44 @@ pub fn format_real(v: f64) -> String { if !v.is_finite() { return "0.0".into(); } + // Very large and very small values in exponent form, as AutoCAD writes them (`1.0E+20`): + // plain digits would run to hundreds of characters. + let a = v.abs(); + if a != 0.0 && !(1e-10..1e16).contains(&a) { + let e = format!("{v:E}"); + let (m, x) = e.split_once('E').unwrap_or((&e, "0")); + let x: i32 = x.parse().unwrap_or(0); + let m = if m.contains('.') { m.to_string() } else { format!("{m}.0") }; + return format!("{m}E{}{:02}", if x < 0 { '-' } else { '+' }, x.abs()); + } let s = format!("{v}"); if s.contains('.') || s.contains('e') || s.contains("inf") || s.contains("NaN") { s } else { format!("{s}.0") } } +/// Most characters one string group holds: longer text is split over several groups (MTEXT +/// contents, field codes). +pub const MAX_STRING_CHARS: usize = 250; + +/// `s` split into strings of at most [`MAX_STRING_CHARS`] characters, in order; none for an +/// empty string. +pub fn string_groups(s: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut n = 0; + for c in s.chars() { + if n == MAX_STRING_CHARS { + out.push(std::mem::take(&mut cur)); + n = 0; + } + cur.push(c); + n += 1; + } + if !cur.is_empty() { + out.push(cur); + } + out +} + /// Write tags as ASCII DXF (CRLF line ends, codes right-aligned to 3 as is conventional). /// /// The output is pure ASCII: non-ASCII characters in strings are written as `\U+XXXX` escapes, @@ -513,18 +603,26 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// Write tags as pre-R2007 ASCII DXF with text in the code page `code_page` names (a /// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: /// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A -/// code page CADCraft doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) +/// code page this crate doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) /// give [`write_ascii`]'s pure ASCII output. So does a file whose code page bytes happen to be /// valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, U+0472), since readers, ours included, take an ASCII /// DXF that is valid UTF-8 as UTF-8. pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { - let bytes = ascii(tags, Strings::Escaped(codepage::encoder(code_page))); + let bytes = write_ascii_in_page(tags, code_page); if !bytes.is_ascii() && std::str::from_utf8(&bytes).is_ok() { return ascii(tags, Strings::Escaped(None)); } bytes } +/// [`write_ascii_in`] without its fallback to pure ASCII: the text is in the code page even when +/// its bytes happen to be valid UTF-8. For a reader that decodes a pre-2007 DXF by the +/// `$DWGCODEPAGE` it declares and never guesses UTF-8, as the DWG bridge's acadrust does. A code +/// page this crate doesn't know, `ASCII`, and `UTF8`/`UNICODE` still give pure ASCII. +pub fn write_ascii_in_page(tags: &[Tag], code_page: &str) -> Vec { + ascii(tags, Strings::Escaped(codepage::encoder(code_page))) +} + /// How a writer stores string values. #[derive(Clone, Copy)] enum Strings { @@ -694,6 +792,9 @@ fn push_string(s: &str, strings: Strings, out: &mut Vec) { out.extend_from_slice(b"\\U+005C"); } (c, Strings::Utf8) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), + (c, Strings::Escaped(Some(d))) if d.shadows_ascii(c) => { + let _ = write!(out, "\\U+{:04X}", u32::from(c)); + } (c, _) if c.is_ascii() => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), (c, Strings::Escaped(Some(d))) if code_page_bytes(c, d, escape_at(i + 1), out) => {} (c, Strings::Escaped(_)) => { @@ -729,18 +830,32 @@ pub struct Section { /// Split a tag stream into sections. pub fn sections(tags: &[Tag]) -> Vec
{ + section_slices(tags).into_iter().map(|s| Section { name: s.name.to_string(), tags: s.tags.to_vec() }).collect() +} + +/// A section borrowed from the tag stream it was split from: [`Section`] without the copy. +#[derive(Clone, Copy, Debug)] +pub struct SectionRef<'a> { + pub name: &'a str, + pub tags: &'a [Tag], +} + +/// [`sections`] without copying the tags: each section is a slice of `tags`. A large file's tags +/// take gigabytes (40 bytes a tag, 25 million tags in a 0.5 GB DXF), so a reader that holds the +/// stream while it reads the sections should take them this way. +pub fn section_slices(tags: &[Tag]) -> Vec> { + let is = |t: &Tag, word: &str| t.code == 0 && t.str_ref().is_some_and(|s| s.trim_end() == word); let mut out = Vec::new(); let mut i = 0; while i < tags.len() { - let is_sec = tags.get(i).is_some_and(|t| t.code == 0 && t.str().trim_end() == "SECTION"); - if is_sec { - let name = tags.get(i + 1).filter(|t| t.code == 2).map(|t| t.str().trim_end().to_string()).unwrap_or_default(); + if tags.get(i).is_some_and(|t| is(t, "SECTION")) { + let name = tags.get(i + 1).filter(|t| t.code == 2).and_then(Tag::str_ref).unwrap_or_default().trim_end(); let start = i + 2; let mut j = start; - while j < tags.len() && !(tags.get(j).is_some_and(|t| t.code == 0 && t.str().trim_end() == "ENDSEC")) { + while j < tags.len() && !tags.get(j).is_some_and(|t| is(t, "ENDSEC")) { j += 1; } - out.push(Section { name, tags: tags.get(start..j).unwrap_or(&[]).to_vec() }); + out.push(SectionRef { name, tags: tags.get(start..j).unwrap_or(&[]) }); i = j + 1; } else { i += 1; @@ -751,15 +866,20 @@ pub fn sections(tags: &[Tag]) -> Vec
{ /// Split a run of tags into records starting at each code-0 tag: (type, tags after the 0). pub fn records(tags: &[Tag]) -> Vec<(String, Vec)> { - let mut out: Vec<(String, Vec)> = Vec::new(); - for t in tags { - if t.code == 0 { - out.push((t.str(), Vec::new())); - } else if let Some(last) = out.last_mut() { - last.1.push(t.clone()); - } - } - out + record_slices(tags).into_iter().map(|(kind, tags)| (kind, tags.to_vec())).collect() +} + +/// [`records`] without copying the tags: each record's tags are a slice of `tags`. +pub fn record_slices(tags: &[Tag]) -> Vec<(String, &[Tag])> { + let starts: Vec = tags.iter().enumerate().filter(|(_, t)| t.code == 0).map(|(i, _)| i).collect(); + starts + .iter() + .enumerate() + .map(|(n, &i)| { + let end = starts.get(n + 1).copied().unwrap_or(tags.len()); + (tags.get(i).map(Tag::str).unwrap_or_default(), tags.get(i + 1..end).unwrap_or(&[])) + }) + .collect() } #[cfg(test)] @@ -828,6 +948,128 @@ mod tests { assert_eq!(tags[2], Tag::i(62, 5)); } + /// A pre-R13 binary body: 1-byte group codes, 255 escaping a 2-byte code (1001 here). + fn r12_binary() -> Vec { + let mut b = BINARY_SENTINEL.to_vec(); + let push_str = |b: &mut Vec, code: u8, s: &str| { + b.push(code); + b.extend_from_slice(s.as_bytes()); + b.push(0); + }; + push_str(&mut b, 0, "SECTION"); + push_str(&mut b, 2, "ENTITIES"); + push_str(&mut b, 0, "LINE"); + push_str(&mut b, 8, "RAILROAD"); + b.push(10); + b.extend_from_slice(&3.25f64.to_le_bytes()); + b.push(62); + b.extend_from_slice(&5i16.to_le_bytes()); + b.push(255); + b.extend_from_slice(&1001i16.to_le_bytes()); + b.extend_from_slice(b"APP\0"); + push_str(&mut b, 0, "ENDSEC"); + push_str(&mut b, 0, "EOF"); + b + } + + #[test] + fn r12_binary_parse() { + let tags = parse(&r12_binary()).unwrap(); + assert_eq!(tags[0], Tag::s(0, "SECTION")); + assert_eq!(tags[3], Tag::s(8, "RAILROAD")); + assert_eq!(tags[4], Tag::f(10, 3.25)); + assert_eq!(tags[5], Tag::i(62, 5)); + assert_eq!(tags[6], Tag::s(1001, "APP")); + assert_eq!(tags.last(), Some(&Tag::s(0, "EOF"))); + } + + #[test] + fn r13_binary_code_255_is_ordinary() { + let mut b = BINARY_SENTINEL.to_vec(); + for (code, s) in [(0i16, "LINE"), (255, "x"), (0, "EOF")] { + b.extend_from_slice(&code.to_le_bytes()); + b.extend_from_slice(s.as_bytes()); + b.push(0); + } + let tags = parse(&b).unwrap(); + assert_eq!(tags.get(1).map(|t| t.code), Some(255)); + assert_eq!(tags.len(), 3); + } + + #[test] + fn binary_layout_detection() { + assert!(legacy_binary_layout(b"\0SECTION\0")); + assert!(legacy_binary_layout(&[255, 0xE7, 0x03])); + assert!(!legacy_binary_layout(b"\0\0SECTION\0")); + assert!(!legacy_binary_layout(&[0xE7, 0x03, b'x', 0])); + assert!(!legacy_binary_layout(b"")); + assert!(!legacy_binary_layout(&[0])); + // R13+ code 255 with the string "x": not the escape + assert!(!legacy_binary_layout(&[255, 0, b'x', 0])); + assert!(!legacy_binary_layout(&[255, 0xE7])); + } + + #[test] + fn truncated_r12_binary_never_panics() { + let b = r12_binary(); + for n in BINARY_SENTINEL.len()..b.len() { + let _ = parse(b.get(..n).unwrap_or(&[])); + } + let mut cut = BINARY_SENTINEL.to_vec(); + cut.extend_from_slice(&[0, b'S', 0, 255, 0x10]); + assert!(parse(&cut).is_err()); + } + + #[test] + fn borrowed_sections_and_records_match_the_copies() { + let text = "999\nstray\n 0\nSECTION\n 2\nHEADER\n 9\n$ACADVER\n 1\nAC1015\n 0\nENDSEC\n 0\nSECTION\n 2\nENTITIES\n 5\n1A\n 0\nLINE\n 8\n0\n 10\n1.5\n 0\nCIRCLE\n 40\n2.0\n 0\nENDSEC\n 0\nEOF\n"; + let tags = parse(text.as_bytes()).unwrap(); + let (copies, slices) = (sections(&tags), section_slices(&tags)); + assert_eq!(slices.iter().map(|s| s.name).collect::>(), ["HEADER", "ENTITIES"]); + assert_eq!(copies.len(), slices.len()); + for (c, s) in copies.iter().zip(&slices) { + assert_eq!(c.name, s.name); + assert_eq!(c.tags.as_slice(), s.tags); + let (rc, rs) = (records(&c.tags), record_slices(s.tags)); + assert_eq!(rc.len(), rs.len()); + for ((kc, tc), (ks, ts)) in rc.iter().zip(&rs) { + assert_eq!((kc, tc.as_slice()), (ks, *ts)); + } + } + // Tags before the first record start none, as in `records`. + assert_eq!(record_slices(slices[1].tags).iter().map(|(k, _)| k.as_str()).collect::>(), ["LINE", "CIRCLE"]); + assert_eq!(slices[1].tags[0].str_ref(), Some("1A")); + assert_eq!(Tag::f(10, 1.5).str_ref(), None); + } + + #[test] + fn huge_and_tiny_reals_in_exponent_form() { + // Plain digits would run to hundreds of characters; the value reads back exactly. + assert_eq!(format_real(1e20), "1.0E+20"); + assert_eq!(format_real(-2.5e-12), "-2.5E-12"); + assert_eq!(format_real(0.5), "0.5"); + assert_eq!(format_real(12.0), "12.0"); + for v in [f64::MAX, -f64::MAX, 1e308, f64::MIN_POSITIVE, 5e-324, 1.2345678901234567e16] { + let s = format_real(v); + assert!(s.len() < 32, "{s}"); + assert_eq!(s.parse::().ok(), Some(v), "{s}"); + // and through the writer and the reader + let back = parse(write_ascii(&[Tag::f(10, v), Tag::s(0, "EOF")]).as_bytes()).unwrap(); + assert_eq!(back[0], Tag::f(10, v)); + } + } + + #[test] + fn long_strings_split_into_250_character_groups() { + assert!(string_groups("").is_empty()); + assert_eq!(string_groups("abc"), ["abc"]); + let s = "é".repeat(251); + let g = string_groups(&s); + assert_eq!(g.iter().map(|x| x.chars().count()).collect::>(), [250, 1]); + assert_eq!(g.concat(), s); + assert_eq!(string_groups(&"x".repeat(500)).len(), 2); + } + #[test] fn malformed_inputs() { assert!(parse(b"").is_err()); @@ -839,7 +1081,7 @@ mod tests { assert!(parse(&b).is_err()); // Bad numbers become 0 rather than failing. let t = parse(b"10\nnotanumber\n0\nEOF\n").unwrap(); - assert_eq!(t[0].f64(), 0.0); + assert!(t[0].f64().abs() < f64::EPSILON); } #[test] diff --git a/crates/dxf/src/version.rs b/crates/dxf/src/version.rs index 3d0b96c1..9f0fff08 100644 --- a/crates/dxf/src/version.rs +++ b/crates/dxf/src/version.rs @@ -1,20 +1,20 @@ //! The DXF/DWG file versions CADCraft writes (DXF Reference, HEADER `$ACADVER`). -/// A file version CADCraft can save: the AutoCAD release that introduced the format. Each format -/// also serves the releases after it (2018 is still AutoCAD's current format). +/// A file version CADCraft can save: the `AutoCAD` release that introduced the format. Each format +/// also serves the releases after it (2018 is still `AutoCAD`'s current format). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Version { - /// AutoCAD 2000/2002 (AC1015). + /// `AutoCAD` 2000/2002 (AC1015). R2000, - /// AutoCAD 2004–2006 (AC1018). + /// `AutoCAD` 2004–2006 (AC1018). R2004, - /// AutoCAD 2007–2009 (AC1021): the first version whose DXF text is UTF-8. + /// `AutoCAD` 2007–2009 (AC1021): the first version whose DXF text is UTF-8. R2007, - /// AutoCAD 2010–2012 (AC1024). + /// `AutoCAD` 2010–2012 (AC1024). R2010, - /// AutoCAD 2013–2017 (AC1027). + /// `AutoCAD` 2013–2017 (AC1027). R2013, - /// AutoCAD 2018 and later (AC1032), the default. + /// `AutoCAD` 2018 and later (AC1032), the default. #[default] R2018, } diff --git a/crates/dxf/tests/ascii_bom.rs b/crates/dxf/tests/ascii_bom.rs index 9d80aff1..8d08bb4d 100644 --- a/crates/dxf/tests/ascii_bom.rs +++ b/crates/dxf/tests/ascii_bom.rs @@ -9,7 +9,7 @@ fn utf8_bom_bytes_are_skipped() { bytes.extend_from_slice(BODY.as_bytes()); let with_bom = craft_dxf::parse(&bytes); assert!(with_bom.is_ok(), "{with_bom:?}"); - assert_eq!(with_bom.map(|t| t.len()).unwrap_or(0), plain.len()); + assert_eq!(with_bom.map_or(0, |t| t.len()), plain.len()); } #[test] diff --git a/crates/dxf/tests/audit_followups.rs b/crates/dxf/tests/audit_followups.rs new file mode 100644 index 00000000..de61bf17 --- /dev/null +++ b/crates/dxf/tests/audit_followups.rs @@ -0,0 +1,139 @@ +//! Follow-ups of the crates/dxf audit (#1269): group code 255, the extra DOS and ISO code pages, +//! `format_real`, and the escape and integer writers that were rewritten without a change of +//! behaviour. + +use craft_dxf::{Tag, Value, decode_code_page, format_real, parse, write_ascii, write_binary}; + +#[test] +fn group_code_255_is_an_ordinary_code_in_binary_dxf() { + // Code 255 used to be read as an R12 escape (`FF 00 `), so a file with that code could + // not be read back. + let tags = vec![Tag::s(255, "x"), Tag::s(0, "EOF")]; + let back = parse(&write_binary(&tags, true)).unwrap(); + assert_eq!(back.len(), 2); + assert_eq!((back[0].code, back[0].str()), (255, "x".to_string())); +} + +#[test] +fn more_dos_and_iso_code_pages_decode() { + // (code page, byte, the character the code page defines there) + let cases: [(&str, u8, char); 12] = [ + ("DOS720", 0x82, 'é'), + ("DOS737", 0x80, 'Α'), + ("DOS775", 0x80, 'Ć'), + ("DOS855", 0x80, 'ђ'), + ("DOS857", 0x8D, 'ı'), + ("DOS860", 0x84, 'ã'), + ("DOS861", 0x8B, 'Ð'), + ("DOS863", 0x84, 'Â'), + ("DOS864", 0x80, '°'), + ("DOS865", 0x9B, 'ø'), + ("DOS869", 0x86, 'Ά'), + ("ISO8859_15", 0xA4, '€'), + ]; + for (name, byte, ch) in cases { + assert_eq!(decode_code_page(&[byte], name), Some(ch.to_string()), "{name}"); + assert_eq!(decode_code_page(b"A", &name.to_lowercase()), Some("A".into()), "{name}"); + } + // An unassigned byte is the replacement character, the rest decodes. + assert_eq!(decode_code_page(b"\xD5A", "DOS857").as_deref(), Some("\u{FFFD}A")); +} + +#[test] +fn dos_code_pages_decode_known_characters() { + // Four bytes per table with the character the reference mapping gives there (code point in the + // comment); U+FFFD marks a byte the reference leaves undefined. DOS737, 775, 855, 857, 860, 861, 863, + // 864, 865 and 869 come from the Unicode Consortium files in + // https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/ (CP.TXT). DOS720 has no such file: + // its reference is only Python's `cp720` codec, which maps the bytes below that we decode to U+FFFD + // (0x80, 0x81, 0x84, 0x86, 0x8D, 0x8E, 0x8F, 0x90) to the C1 controls U+0080..U+0090. + let tables: [(&str, [(u8, char); 4]); 11] = [ + ("DOS720", [(0x82, 'é'), (0x9B, 'ؤ'), (0xB3, '│'), (0xE3, 'ع')]), // U+00E9, U+0624, U+2502, U+0639 + ("DOS737", [(0x80, 'Α'), (0xAB, 'τ'), (0xB3, '│'), (0xF3, '≤')]), // U+0391, U+03C4, U+2502, U+2264 + ("DOS775", [(0x80, 'Ć'), (0xAB, '½'), (0xB3, '│'), (0xE3, 'Ń')]), // U+0106, U+00BD, U+2502, U+0143 + ("DOS855", [(0x80, 'ђ'), (0xA3, 'Б'), (0xB3, '│'), (0xFB, 'ч')]), // U+0452, U+0411, U+2502, U+0447 + ("DOS857", [(0x8D, 'ı'), (0xB3, '│'), (0xE3, 'Ò'), (0xD5, '\u{FFFD}')]), // U+0131, U+2502, U+00D2, undefined + ("DOS860", [(0x84, 'ã'), (0x9B, '¢'), (0xB3, '│'), (0xE3, 'π')]), // U+00E3, U+00A2, U+2502, U+03C0 + ("DOS861", [(0x8B, 'Ð'), (0xAB, '½'), (0xB3, '│'), (0xFB, '√')]), // U+00D0, U+00BD, U+2502, U+221A + ("DOS863", [(0x84, 'Â'), (0x9B, '¢'), (0xB3, '│'), (0xE3, 'π')]), // U+00C2, U+00A2, U+2502, U+03C0 + ("DOS864", [(0x80, '°'), (0xAB, 'ﺙ'), (0xB3, '٣'), (0xFF, '\u{FFFD}')]), // U+00B0, U+FE99, U+0663, undefined + ("DOS865", [(0x9B, 'ø'), (0xAB, '½'), (0xB3, '│'), (0xE3, 'π')]), // U+00F8, U+00BD, U+2502, U+03C0 + ("DOS869", [(0x86, 'Ά'), (0x9B, 'ά'), (0xB3, '│'), (0x80, '\u{FFFD}')]), // U+0386, U+03AC, U+2502, undefined + ]; + for (name, cases) in tables { + for (byte, ch) in cases { + assert_eq!(decode_code_page(&[byte], name), Some(ch.to_string()), "{name} 0x{byte:02X}"); + } + } + // DOS720: the 8 bytes the Python codec maps to C1 controls are unassigned here. + for byte in [0x80u8, 0x81, 0x84, 0x86, 0x8D, 0x8E, 0x8F, 0x90] { + assert_eq!(decode_code_page(&[byte], "DOS720").as_deref(), Some("\u{FFFD}"), "DOS720 0x{byte:02X}"); + } +} + +#[test] +fn ascii_file_in_a_dos_code_page_reads_its_text() { + let mut dxf = b"0\nSECTION\n2\nHEADER\n9\n$DWGCODEPAGE\n3\nDOS855\n0\nENDSEC\n0\nSECTION\n2\nTABLES\n8\n".to_vec(); + dxf.extend_from_slice(b"\xA1\xA2\n0\nENDSEC\n0\nEOF\n"); + let tags = parse(&dxf).unwrap(); + let layer = tags.iter().find(|t| t.code == 8).unwrap(); + assert_eq!(layer.str(), decode_code_page(b"\xA1\xA2", "DOS855").unwrap()); + assert_eq!(layer.str(), "Аб"); +} + +#[test] +fn format_real_always_has_a_point() { + assert_eq!(format_real(5.0), "5.0"); + assert_eq!(format_real(-0.0), "-0.0"); + assert_eq!(format_real(0.1), "0.1"); + assert_eq!(format_real(1e15), "1000000000000000.0"); + assert_eq!(format_real(1e21), "1.0E+21"); + assert_eq!(format_real(1e-7), "0.0000001"); + assert_eq!(format_real(f64::MAX).parse::().unwrap(), f64::MAX); + assert_eq!(format_real(f64::NAN), "0.0"); + assert_eq!(format_real(f64::INFINITY), "0.0"); + assert_eq!(format_real(f64::NEG_INFINITY), "0.0"); +} + +#[test] +fn escapes_next_to_other_text_and_backslashes() { + let one = |s: &str| parse(format!("1\n{s}\n0\nEOF\n").as_bytes()).unwrap()[0].str(); + assert_eq!(one("a\\U+00E9b"), "aéb"); + assert_eq!(one("é\\U+0041\\ ü\\M+5BCFE\\U+D83D\\U+DE00 x\\"), "éA\\ ü件😀 x\\"); + assert_eq!(one("\\\\U+0041"), "\\A"); + assert_eq!(one("\\U+D83D\\U+0041 \\U+DE00"), "\\U+D83DA \\U+DE00"); + assert_eq!(one("日本\\U+00"), "日本\\U+00"); + // A long run of backslashes (hostile) is read in linear time and stays literal. + let slashes = "\\".repeat(100_000); + assert_eq!(one(&slashes), slashes); + // The same strings through the binary reader. + // Built by hand: the writer escapes a literal `\U+` (#2050), so it would not produce an escape. + let mut file = b"AutoCAD Binary DXF\r\n\x1a\0".to_vec(); + file.extend_from_slice(&1i16.to_le_bytes()); + file.extend_from_slice("é\\U+0041\\ ü".as_bytes()); + file.push(0); + file.extend_from_slice(&0i16.to_le_bytes()); + file.extend_from_slice(b"EOF\0"); + assert_eq!(parse(&file).unwrap()[0].str(), "éA\\ ü"); + // And the writer keeps the literal text literal. + let tags = [Tag::s(1, "é\\U+0041\\ ü"), Tag::s(0, "EOF")]; + assert_eq!(parse(&write_binary(&tags, true)).unwrap()[0].str(), "é\\U+0041\\ ü"); +} + +#[test] +fn integers_are_written_in_full() { + let text = write_ascii(&[Tag::i(70, i64::MIN), Tag::i(90, -7), Tag::i(160, 0), Tag::new(290, Value::Bool(true))]); + assert_eq!(text, " 70\r\n-9223372036854775808\r\n 90\r\n-7\r\n160\r\n0\r\n290\r\n1\r\n"); +} + +#[test] +fn binary_hex_reads_every_byte_in_upper_case() { + let all: Vec = (0..=255).collect(); + let mut file = b"AutoCAD Binary DXF\r\n\x1a\0".to_vec(); + file.extend_from_slice(&[0x36, 0x01, 0xFF]); + file.extend_from_slice(&all[..255]); + file.extend_from_slice(&[0x00, 0x00, b'E', b'O', b'F', 0]); + let tags = parse(&file).unwrap(); + let want: String = all[..255].iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(tags[0].value, Value::Hex(want)); +} diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 647d8115..02e8a75d 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -1,7 +1,7 @@ //! Pre-R2007 DXF text is written in the `$DWGCODEPAGE` code page: characters it has as their //! bytes, the others as `\U+XXXX` escapes; every form reads back to the same text. -use craft_dxf::{Tag, parse, write_ascii, write_ascii_in, write_binary_in}; +use craft_dxf::{Tag, parse, write_ascii, write_ascii_in, write_ascii_in_page, write_binary_in}; fn file(code_page: &str, text: &str) -> Vec { vec![ @@ -28,11 +28,15 @@ fn contains(bytes: &[u8], part: &[u8]) -> bool { #[test] fn text_is_written_in_the_code_page() { // (code page, text, the bytes written for it) - let cases: [(&str, &str, &[u8]); 6] = [ + let cases: [(&str, &str, &[u8]); 8] = [ ("ANSI_1252", "Ação € 图", b"A\xE7\xE3o \x80 \\U+56FE"), ("ANSI_1251", "Слой é", b"\xD1\xEB\xEE\xE9 \\U+00E9"), ("ANSI_932", "表図面 é", b"\x95\x5C\x90\x7D\x96\xCA \\U+00E9"), ("DOS850", "Größe €", b"Gr\x94\xE1e \\U+20AC"), + // DOS 864: byte 0x25 is the Arabic percent sign, so a literal `%` is escaped. + ("DOS864", "50% \u{066A}\u{00B0}", b"50\\U+0025 %\x80"), + // Only the percent sign: every byte is ASCII, and it still reads back as U+066A. + ("DOS864", "50\u{066A}", b"50%"), ("ANSI_1252", "😀", b"\\U+D83D\\U+DE00"), // A code page CADCraft doesn't know: everything non-ASCII is escaped. ("ANSI_1361", "Ação", b"A\\U+00E7\\U+00E3o"), @@ -60,7 +64,7 @@ fn unknown_ascii_and_utf8_code_pages_are_pure_ascii() { } } -/// Code page bytes that happen to form valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, which is U+0472 +/// Code page bytes that happen to form valid UTF-8 (`СІ` in `ANSI_1251` is D1 B2, which is U+0472 /// in UTF-8) would be read back as UTF-8 from an ASCII file, so that file is written with /// escapes. Binary DXF reads strings by the header's code page and keeps the bytes. #[test] @@ -77,6 +81,27 @@ fn code_page_bytes_that_are_valid_utf8_are_escaped_in_ascii() { assert!(contains(&write_ascii_in(&file("ANSI_1251", "Слой"), "ANSI_1251"), b"\xD1\xEB\xEE\xE9")); } +/// `write_ascii_in_page`, for a reader that decodes by `$DWGCODEPAGE` and never guesses UTF-8 (the +/// DWG bridge's acadrust), keeps the code page bytes even when they form valid UTF-8 (`é` in +/// ANSI_1252 is C3 A9, `é` in UTF-8), escapes only what the code page lacks, and escapes a +/// backslash that starts an escape. Otherwise it writes what `write_ascii_in` writes. +#[test] +fn page_writer_keeps_code_page_bytes_that_are_valid_utf8() { + let tags = file("ANSI_1252", "é Média Ω \\U+00E9"); + let page = write_ascii_in_page(&tags, "ANSI_1252"); + assert!(contains(&page, b"\xC3\xA9 M\xE9dia \\U+03A9 \\U+005CU+00E9"), "{:?}", String::from_utf8_lossy(&page)); + // Text whose bytes are all valid UTF-8: `write_ascii_in` still falls back to escapes, the + // page writer keeps them. + let utf8_like = file("ANSI_1252", "é"); + assert!(write_ascii_in(&utf8_like, "ANSI_1252").is_ascii()); + assert!(contains(&write_ascii_in_page(&utf8_like, "ANSI_1252"), b"\r\n\xC3\xA9\r\n")); + let plain = file("ANSI_1252", "Ação"); + assert_eq!(write_ascii_in_page(&plain, "ANSI_1252"), write_ascii_in(&plain, "ANSI_1252")); + for cp in ["EBCDIC", "ASCII", "UTF8"] { + assert_eq!(write_ascii_in_page(&plain, cp), write_ascii(&plain).into_bytes(), "{cp}"); + } +} + /// A literal `\U+` or `\M+` in the text (#2050) reads back as written alongside code page bytes, /// in ASCII and binary, including an ASCII file that falls back to escapes because its bytes /// would read as UTF-8. diff --git a/crates/dxf/tests/conformance.rs b/crates/dxf/tests/conformance.rs index 1eaec8fb..2c09e28e 100644 --- a/crates/dxf/tests/conformance.rs +++ b/crates/dxf/tests/conformance.rs @@ -1,8 +1,8 @@ //! Small independent byte fixtures for DXF tag types, text encoding, framing and failures. //! Handwritten examples, not files from a CAD application or generated by our writer. //! Background on the tag model and file structure: -//! https://ezdxf.readthedocs.io/en/stable/dxfinternals/dxftags.html -//! https://ezdxf.readthedocs.io/en/stable/dxfinternals/filestructure.html +//! +//! //! ASCII spacing is our writer's documented convention; binary fixtures specify actual bytes. use craft_dxf::{DxfError, Tag, Value, parse, parse_with_limit, write_ascii_as, write_binary}; @@ -16,8 +16,8 @@ fn fixtures() -> Vec<(Tag, &'static [u8], &'static [u8])> { (Tag::s(105, "AB12"), b"105\r\nAB12\r\n", b"\x69\0AB12\0"), (Tag::new(10, Value::Real(-3.25)), b" 10\r\n-3.25\r\n", b"\x0a\0\0\0\0\0\0\0\x0a\xc0"), (Tag::i(70, -32768), b" 70\r\n-32768\r\n", b"\x46\0\0\x80"), - (Tag::i(90, -2147483648), b" 90\r\n-2147483648\r\n", b"\x5a\0\0\0\0\x80"), - (Tag::i(160, 9223372036854775807), b"160\r\n9223372036854775807\r\n", b"\xa0\0\xff\xff\xff\xff\xff\xff\xff\x7f"), + (Tag::i(90, -2_147_483_648), b" 90\r\n-2147483648\r\n", b"\x5a\0\0\0\0\x80"), + (Tag::i(160, 9_223_372_036_854_775_807), b"160\r\n9223372036854775807\r\n", b"\xa0\0\xff\xff\xff\xff\xff\xff\xff\x7f"), (Tag::new(290, Value::Bool(false)), b"290\r\n0\r\n", b"\x22\x01\0"), (Tag::new(299, Value::Bool(true)), b"299\r\n1\r\n", b"\x2b\x01\x01"), (Tag::new(310, Value::Hex("00FF80".into())), b"310\r\n00FF80\r\n", b"\x36\x01\x03\0\xff\x80"), diff --git a/crates/dxf/tests/hex_chunks.rs b/crates/dxf/tests/hex_chunks.rs index 0be58f1a..61100b08 100644 --- a/crates/dxf/tests/hex_chunks.rs +++ b/crates/dxf/tests/hex_chunks.rs @@ -4,8 +4,10 @@ //! code and following-record framing. Oversized binary read fixtures test tolerance only. //! Pending writer regressions extend the chunk-size finding in issue #1025. //! Public DXF Reference, Group Codes in Numerical Order and Binary DXF Files: -//! https://help.autodesk.com/cloudhelp/2025/ENU/AutoCAD-DXF/files/GUID-3F0380A5-1C15-464D-BC66-2C5F094BCFB9.htm -//! https://help.autodesk.com/cloudhelp/2025/ENU/AutoCAD-DXF/files/GUID-FC1C3C69-DBC2-49E4-893A-000D6538C0FE.htm +//! +//! + +use std::fmt::Write as _; use craft_dxf::{Tag, Value, parse, write_ascii_as, write_binary}; @@ -17,7 +19,10 @@ fn payload(length: usize) -> Vec { } fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02X}")).collect() + bytes.iter().fold(String::new(), |mut out, byte| { + let _ = write!(out, "{byte:02X}"); + out + }) } fn input(code: i32, bytes: &[u8]) -> [Tag; 2] { diff --git a/crates/dxf/tests/roundtrip.rs b/crates/dxf/tests/roundtrip.rs index 1d9dbb91..87f1e812 100644 --- a/crates/dxf/tests/roundtrip.rs +++ b/crates/dxf/tests/roundtrip.rs @@ -122,7 +122,11 @@ fn hex_chunks_keep_bytes_and_order() { // Larger payloads need chunk-size/reassembly checks rather than exact tag equality; // see hex_chunks.rs for the independent wire checks and pending writer regressions. for length in [0, 1, 2, 126, 127] { - let hex: String = (0..length).map(|i| format!("{:02X}", (i * 37) % 256)).collect(); + let hex: String = (0..length).fold(String::new(), |mut out, i| { + use std::fmt::Write as _; + let _ = write!(out, "{:02X}", (i * 37) % 256); + out + }); let tags: Vec<_> = (310..=319).chain([1004]).map(|code| Tag::new(code, Value::Hex(hex.clone()))).collect(); check(&format!("hex/bytes={length}"), &tags); }