From 81a7a6838892486ec86837957bbf69313a316769 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sat, 7 Nov 2026 04:27:08 -0800 Subject: [PATCH] .ai import: legacy type in CJK fonts decodes by the font's CMap Legacy type in a composite font such as HeiseiMin-W3-83pv-RKSJ-H was decoded one byte to a character, so Japanese, Chinese and Korean type opened as unrelated single-byte characters, and the font name kept the CMap. Tf now finds a predefined CMap at the end of the font name, unless a TZ gives the font encoding pairs, and the font is the name without it. Strings in that font are read by the CMap's code space, mapped to CIDs with the predefined CMaps craft-pdf carries, then to Unicode by the character collection's CID-to-Unicode CMap. Codes the CMap doesn't map are dropped. A -V CMap is read with its -H form's codes. Codes in the Unicode CMaps (UCS2, UTF8, UTF16, UTF32) are read as Unicode; those helpers follow mockydevs' #6067. A 0x0D byte ends a paragraph and a 0x09 byte is a tab in these fonts too. The predefined CMaps craft-pdf doesn't carry (RKSJ, 78-RKSJ, 78ms-RKSJ, 78-EUC, 78, Add, Ext, NWP, GB, CNS1, KSC, B5 and the Hong Kong B5 sets other than HKscs-B5) are read with the nearest carried CMap, which can give a different character for some codes. GBT, GBT-EUC, GBTpc-EUC, CNS2 and KSC-Johab type still decodes byte by byte, with the CMap removed from the font name. Docs: docs/file-format-parity.md. --- Cargo.lock | 1 + crates/engine/tests/import_fuzz.rs | 20 ++ crates/eps/Cargo.toml | 2 + crates/eps/src/import/ai/legacy_text.rs | 317 +++++++++++++++++++++++- crates/eps/src/import/ai/tests.rs | 90 +++++++ docs/file-format-parity.md | 12 +- 6 files changed, 438 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f92cb310..f133fb5cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6283,6 +6283,7 @@ dependencies = [ name = "vectorcraft-eps" version = "0.21.0" dependencies = [ + "craft-pdf", "fax", "flate2", "hayro-ccitt", diff --git a/crates/engine/tests/import_fuzz.rs b/crates/engine/tests/import_fuzz.rs index 6c0f3167c..0de4f73f7 100644 --- a/crates/engine/tests/import_fuzz.rs +++ b/crates/engine/tests/import_fuzz.rs @@ -3390,6 +3390,26 @@ proptest! { survive("hostile .ai line widths", || Some(vectorcraft_eps::layered_ai(&private, Document::new(100.0, 100.0), vec![], false, false, 1.0).0))?; } + /// A `.ai` whose legacy type is set in a CJK composite font (a predefined CMap the reader + /// carries, one it reads through another, one it decodes byte by byte, a Unicode CMap, or a + /// name that only looks like one), with any bytes in its strings. + #[test] + fn ai_legacy_type_in_cjk_fonts_never_panics( + cmap in prop::sample::select(vec![ + "83pv-RKSJ-H", "90ms-RKSJ-V", "90pv-RKSJ-V", "H", "V", "EUC-H", "GBK2K-H", "CNS-EUC-V", "KSCpc-EUC-H", "KSCms-UHC-HW-V", + "78-RKSJ-H", "GB-H", "KSC-V", "HKdlb-B5-H", "GBT-H", "UniJIS-UCS2-H", "UniGB-UTF8-V", "UniKS-UTF32-H", "UniAKR-UTF16-H", + "Identity-H", "UniFoo-UTF8-H", "-H", "RKSJ", + ]), + strings in prop::collection::vec(prop::collection::vec(any::(), 0..24), 1..4), + ) { + use vectorcraft_testkit::ai; + let tx: String = strings.iter().map(|s| format!("({}) Tx\n", s.iter().map(|b| format!("\\{b:03o}")).collect::())).collect(); + let art = format!("0 To\n1 0 0 1 10 50 0 Tp\nTP\n0 Tr\n/_Test-W3-{cmap} 9 Tf\n{tx}(\\r) TX\nTO\n"); + let data = ai::editing_data(100.0, 100.0, &ai::layer("L", &art)); + let private = [b"%AI24_ZStandard_Data".as_slice(), &zstd(&data)].concat(); + survive("legacy type in CJK fonts", || Some(vectorcraft_eps::layered_ai(&private, Document::new(100.0, 100.0), vec![], false, false, 1.0).0))?; + } + /// A `.ai` whose pattern definition (a marked tile box, a gradient placed by `Bg`'s matrix) and /// pattern fill are damaged or hostile. #[test] diff --git a/crates/eps/Cargo.toml b/crates/eps/Cargo.toml index 0004a65d3..eac7b16b2 100644 --- a/crates/eps/Cargo.toml +++ b/crates/eps/Cargo.toml @@ -21,6 +21,8 @@ vectorcraft-text = { workspace = true } vectorcraft-metafile = { workspace = true } # Non-native art in Illustrator editing data (a placed PDF's content, kept as a PDF). vectorcraft-pdf = { workspace = true } +# The predefined CMaps that strings of legacy type in CJK fonts are decoded by. +craft-pdf = { workspace = true } # The TIFF writer (previews) and PackBits, shared with raster export. vectorcraft-render = { workspace = true } kurbo = { workspace = true } diff --git a/crates/eps/src/import/ai/legacy_text.rs b/crates/eps/src/import/ai/legacy_text.rs index 1fb369053..3141220aa 100644 --- a/crates/eps/src/import/ai/legacy_text.rs +++ b/crates/eps/src/import/ai/legacy_text.rs @@ -9,8 +9,12 @@ //! computed positions, §15.1) are skipped: the type is laid out again. Strings are decoded by the //! encoding pairs of the current font's `TZ`, then by the file's `TE` encoding unless that `TZ`'s //! `useDefault` is 0, else as Windows-1252; a re-encoded font `/_name` is the font its `%AI3_BeginEncoding` or `TZ` names -//! (else `name`). +//! (else `name`). Strings in a composite font named `base-CMap` (CJK type) whose `TZ` gives no +//! encoding pairs are decoded by that predefined CMap instead, and the font is `base`. +use std::sync::Arc; + +use craft_pdf::interpret::hayro_cmap::{BfString, CMap, CMapName, load_embedded}; use kurbo::{Affine, BezPath, ParamCurve, ParamCurveArclen}; use vectorcraft_color::Paint; use vectorcraft_doc::{CharStyle, Justify, NodeKind, ParaStyle, TabAlign, TabStop, TextKind, TextObject, TextRun}; @@ -206,8 +210,14 @@ pub(super) struct TypeState { /// Re-encoded fonts' own encodings (`TZ`), by the new font's name: what its encoding pairs /// change, by code (None where they say nothing), and whether `TE`'s encoding applies to it. font_encodings: Vec<(String, Vec>, bool)>, + /// The new fonts' names whose `TZ` gives encoding pairs, which no composite font has. + paired_fonts: Vec, /// The font `Tf` set last, as the file names it. font: Option, + /// How strings in that font decode, when it is a composite font ([`composite_font`]). + cmap: Option>, + /// The predefined CMaps read so far, by name (`None` when one couldn't be read). + cmaps: Vec<(String, Option>)>, /// Between `Xb` and `XB`: the tab stops of the tab definition read so far. tabs: Option>, chars: usize, @@ -225,7 +235,10 @@ impl Default for TypeState { encoding: vec![None; 256], fonts: vec![], font_encodings: vec![], + paired_fonts: vec![], font: None, + cmap: None, + cmaps: vec![], tabs: None, chars: 0, } @@ -251,6 +264,9 @@ impl TypeState { /// A string's bytes, in the current font. fn decode(&self, bytes: &[u8]) -> String { + if let Some(cmap) = &self.cmap { + return cmap.decode(bytes); + } let own = self.font.as_ref().and_then(|f| self.font_encodings.iter().find(|(n, _, _)| n == f)); let (own, te) = own.map_or((None, true), |(_, e, te)| (Some(e.as_slice()), *te)); bytes @@ -265,6 +281,260 @@ impl TypeState { }) .collect() } + + /// How strings in a font with predefined CMap `cmap` decode; `None` when the CMap can't be read. + fn cmap_codes(&mut self, cmap: &str) -> Option> { + if let Some(form) = unicode_cmap(cmap) { + return Some(Arc::new(CMapCodes::Unicode(form))); + } + if let Some((_, codes)) = self.cmaps.iter().find(|(n, _)| n == cmap) { + return codes.clone(); + } + let codes = cid_codes(cmap).map(Arc::new); + self.cmaps.push((cmap.to_string(), codes.clone())); + codes + } +} + +/// How a composite font's strings decode. +#[derive(Debug)] +enum CMapCodes { + /// By a predefined CMap's codes to CIDs, then by its character collection's CID → Unicode CMap; + /// with `high_bit`, each byte 0x21 to 0x7E gets its high bit set first. `space` is the CMap's + /// code space ([`code_space`]). + Cids { codes: Box, unicode: Box, high_bit: bool, space: Option }, + /// A predefined Unicode CMap: the codes are the characters. + Unicode(Unicode), +} + +/// A CMap's code space: ranges of codes, each given by its lowest and highest code, both with the +/// code's length in bytes. +type CodeSpace = &'static [(&'static [u8], &'static [u8])]; + +/// What control byte `b` is in a composite font's string: `\r` a paragraph break, `\t` a tab. +fn control(b: u8) -> Option { + match b { + b'\r' => Some('\n'), + b'\t' => Some('\t'), + _ => None, + } +} + +impl CMapCodes { + /// The characters of string `bytes`, leaving out codes the CMaps don't map; a `\r` is a + /// paragraph break and a `\t` a tab (also as a byte left over after the last whole code of a + /// UTF-16 or UTF-32 string). + fn decode(&self, bytes: &[u8]) -> String { + let (codes, unicode, high_bit, space) = match self { + Self::Unicode(form) => { + let unit = match form { + Unicode::Utf16 => 2, + Unicode::Utf8 => 1, + Unicode::Utf32 => 4, + }; + let (whole, rest) = bytes.split_at_checked(bytes.len() - bytes.len() % unit).unwrap_or((bytes, &[])); + let mut text = unicode_text(whole, *form).replace('\r', "\n"); + text.extend(rest.last().copied().and_then(control)); + return text; + } + Self::Cids { codes, unicode, high_bit, space } => (codes, unicode, *high_bit, space), + }; + let euc: Vec; + let bytes = if high_bit { + euc = bytes.iter().map(|&b| if (0x21..0x7f).contains(&b) { b | 0x80 } else { b }).collect(); + euc.as_slice() + } else { + bytes + }; + let mut out = String::new(); + let mut at = 0; + while let Some(&first) = bytes.get(at) { + // VectorCraft reads a 0x0D byte as a paragraph break and a 0x09 byte as a tab in every + // CMap, as in other fonts' strings, including the row-and-cell CMaps, whose code space + // has no one-byte codes. + if let Some(control) = control(first) { + out.push(control); + at += 1; + continue; + } + let rest = bytes.get(at..).unwrap_or_default(); + let code = |len: u8| rest.get(..usize::from(len)).map(|b| b.iter().fold(0u32, |c, &b| c << 8 | u32::from(b))); + let found = match space { + // The code is as long as the code space range it falls in; a byte that starts no + // code in the code space is skipped. + Some(space) => { + let Some(len) = code_len(space, rest) else { + at += 1; + continue; + }; + code(len).map(|c| (len, codes.lookup_cid_code(c, len))) + } + None => (1..=4u8).find_map(|len| Some((len, Some(codes.lookup_cid_code(code(len)?, len)?)))), + }; + let Some((len, cid)) = found else { + at += 1; + continue; + }; + at += usize::from(len); + // A code the CMap leaves out is dropped. + let Some(cid) = cid else { continue }; + match unicode.lookup_bf_string(cid) { + // A one-byte Roman code whose CID the table gives as a combining mark alone (0x7E in + // the Mac Shift-JIS CMaps, U+0303): the ASCII character, as Adobe's cid2code.txt lists first. + Some(BfString::Char('\u{300}'..='\u{36f}')) if len == 1 && (0x20..0x7f).contains(&first) => out.push(char::from(first)), + Some(BfString::Char(c)) if c != char::REPLACEMENT_CHARACTER => out.push(c), + Some(BfString::String(s)) => out.extend(s.chars().filter(|c| *c != char::REPLACEMENT_CHARACTER)), + _ => {} + } + } + out + } +} + +/// How a predefined Unicode CMap's codes encode their characters. +#[derive(Clone, Copy, PartialEq, Debug)] +enum Unicode { + /// Two bytes a character (`UCS2`) or a UTF-16 surrogate pair, the high byte first. + Utf16, + Utf8, + /// Four bytes, the high byte first. + Utf32, +} + +/// The encoding of predefined Unicode CMap `name`: `Uni` and a character collection, then `-UCS2`, +/// `-UTF8`, `-UTF16` or `-UTF32` (and `-HW` for half-width forms), then `-H` or `-V` +/// (`UniJIS-UCS2-H`, `UniGB-UTF16-V`: ). +fn unicode_cmap(name: &str) -> Option { + let rest = name.strip_suffix("-H").or_else(|| name.strip_suffix("-V"))?; + let rest = rest.strip_suffix("-HW").unwrap_or(rest); + let (collection, form) = rest.rsplit_once('-')?; + let collections = ["UniJIS", "UniJISPro", "UniJIS2004", "UniJISX0213", "UniJISX02132004", "UniHojo", "UniGB", "UniCNS", "UniKS", "UniAKR"]; + if !collections.contains(&collection) { + return None; + } + match form { + "UCS2" | "UTF16" => Some(Unicode::Utf16), + "UTF8" => Some(Unicode::Utf8), + "UTF32" => Some(Unicode::Utf32), + _ => None, + } +} + +/// The characters string `s` encodes in Unicode form `form` (invalid codes left out). +fn unicode_text(s: &[u8], form: Unicode) -> String { + match form { + Unicode::Utf16 => { + let units = s.as_chunks::<2>().0.iter().map(|b| u16::from_be_bytes(*b)); + char::decode_utf16(units).filter_map(Result::ok).collect() + } + Unicode::Utf8 => s.utf8_chunks().map(|chunk| chunk.valid()).collect(), + Unicode::Utf32 => s.as_chunks::<4>().0.iter().filter_map(|b| char::from_u32(u32::from_be_bytes(*b))).collect(), + } +} + +/// Predefined CMap `name` among those VectorCraft carries, leaving out the Identity CMaps, whose +/// codes are CIDs of no known character collection. +fn predefined(name: &str) -> Option> { + match CMapName::from_bytes(name.as_bytes()) { + CMapName::Custom(_) | CMapName::IdentityH | CMapName::IdentityV => None, + name => Some(name), + } +} + +/// Adobe's predefined CMaps that VectorCraft doesn't carry, by name without `-H` or `-V` +/// (): the nearest carried CMap with the same +/// code space, and whether each byte of a row-and-cell code gets its high bit set first (the EUC +/// form of the same code; `GB-H`, `CNS1-H` and `KSC-H` have no one-byte codes, their code space +/// is `<2121>` to `<7E7E>`); `None` for those no carried CMap reads. +fn stand_in(stem: &str) -> Option> { + Some(match stem { + "RKSJ" | "78-RKSJ" | "78ms-RKSJ" => Some(("90ms-RKSJ-H", false)), + "78-EUC" => Some(("EUC-H", false)), + "78" | "Add" | "Ext" | "NWP" => Some(("H", false)), + "GB" => Some(("GB-EUC-H", true)), + "CNS1" => Some(("CNS-EUC-H", true)), + "KSC" => Some(("KSC-EUC-H", true)), + "B5" => Some(("ETen-B5-H", false)), + "ETHK-B5" | "HKdla-B5" | "HKdlb-B5" | "HKgccs-B5" | "HKm314-B5" | "HKm471-B5" => Some(("HKscs-B5-H", false)), + "GBT" | "GBT-EUC" | "GBTpc-EUC" | "CNS2" | "KSC-Johab" => None, + _ => return None, + }) +} + +/// Predefined CMap `cmap`'s codes to CIDs, and its character collection's CID → Unicode CMap. A +/// vertical CMap's codes are read with its horizontal CMap when VectorCraft carries that: the two +/// differ only in the CIDs of vertical glyph forms. A CMap VectorCraft doesn't carry is read with +/// its [`stand_in`]. +fn cid_codes(cmap: &str) -> Option { + let read = |name: &str| CMap::parse(load_embedded(predefined(name)?)?, load_embedded); + let horizontal = cmap.strip_suffix('V').map(|stem| format!("{stem}H")); + let stem = cmap.strip_suffix("-H").or_else(|| cmap.strip_suffix("-V")); + let (codes, high_bit) = horizontal.iter().map(String::as_str).chain([cmap]).find_map(|n| Some((read(n)?, false))).or_else(|| { + let (name, high_bit) = stand_in(stem?).flatten()?; + Some((read(name)?, high_bit)) + })?; + let table = codes.metadata().character_collection.as_ref()?.family.ucs2_cmap()?; + let unicode = CMap::parse(load_embedded(table)?, load_embedded)?; + // A row-and-cell code read in its EUC form falls in the EUC CMap's code space. + let space = if high_bit { stand_in(stem?).flatten().and_then(|(name, _)| code_space(name)) } else { code_space(cmap) }; + Some(CMapCodes::Cids { codes: Box::new(codes), unicode: Box::new(unicode), high_bit, space }) +} + +/// The code space of predefined CMap `cmap`, as Adobe's CMap files give it +/// (); a CMap's vertical form has its +/// horizontal form's. `None` for a CMap not listed here. +fn code_space(cmap: &str) -> Option { + const RKSJ: CodeSpace = &[(b"\x00", b"\x80"), (b"\xa0", b"\xdf"), (b"\x81\x40", b"\x9f\xfc"), (b"\xe0\x40", b"\xfc\xfc")]; + const RKSJ_MAC: CodeSpace = &[(b"\x00", b"\x80"), (b"\xa0", b"\xdf"), (b"\xfd", b"\xff"), (b"\x81\x40", b"\x9f\xfc"), (b"\xe0\x40", b"\xfc\xfc")]; + const EUC_JP: CodeSpace = &[(b"\x00", b"\x80"), (b"\x8e\xa0", b"\x8e\xdf"), (b"\xa1\xa1", b"\xfe\xfe")]; + const ROW_CELL: CodeSpace = &[(b"\x21\x21", b"\x7e\x7e")]; + const EUC: CodeSpace = &[(b"\x00", b"\x80"), (b"\xa1\xa1", b"\xfe\xfe")]; + const GB_PC: CodeSpace = &[(b"\x00", b"\x80"), (b"\xfd", b"\xff"), (b"\xa1\xa1", b"\xfc\xfe")]; + const GBK: CodeSpace = &[(b"\x00", b"\x80"), (b"\x81\x40", b"\xfe\xfe")]; + const GBK2K: CodeSpace = &[(b"\x00", b"\x7f"), (b"\x81\x40", b"\xfe\xfe"), (b"\x81\x30\x81\x30", b"\xfe\x39\xfe\x39")]; + const B5_PC: CodeSpace = &[(b"\x00", b"\x80"), (b"\xfd", b"\xff"), (b"\xa1\x40", b"\xfc\xfe")]; + const B5: CodeSpace = &[(b"\x00", b"\x80"), (b"\xa1\x40", b"\xfe\xfe")]; + const HKSCS: CodeSpace = &[(b"\x00", b"\x80"), (b"\x87\x40", b"\xfe\xfe")]; + const HKDLB: CodeSpace = &[(b"\x00", b"\x80"), (b"\x8e\x40", b"\xfe\xfe")]; + const HKGCCS: CodeSpace = &[(b"\x00", b"\x80"), (b"\x8a\x40", b"\xfe\xfe")]; + const CNS_EUC: CodeSpace = &[(b"\x00", b"\x80"), (b"\xa1\xa1", b"\xfe\xfe"), (b"\x8e\xa1\xa1\xa1", b"\x8e\xa3\xfe\xfe")]; + const KSC_PC: CodeSpace = &[(b"\x00", b"\x84"), (b"\xfe", b"\xff"), (b"\xa1\x41", b"\xfd\xfe")]; + const UHC: CodeSpace = &[(b"\x00", b"\x80"), (b"\x81\x41", b"\xfe\xfe")]; + let stem = match cmap { + "H" | "V" => "", + _ => cmap.strip_suffix("-H").or_else(|| cmap.strip_suffix("-V"))?, + }; + let stem = stem.strip_suffix("-HW").unwrap_or(stem); + Some(match stem { + "83pv-RKSJ" | "90pv-RKSJ" => RKSJ_MAC, + _ if stem.ends_with("RKSJ") => RKSJ, + "EUC" | "78-EUC" => EUC_JP, + "" | "78" | "Add" | "Ext" | "NWP" => ROW_CELL, + "GB-EUC" | "GBT-EUC" | "KSC-EUC" => EUC, + "GBpc-EUC" | "GBTpc-EUC" => GB_PC, + "GBK-EUC" | "GBKp-EUC" => GBK, + "GBK2K" => GBK2K, + "B5pc" => B5_PC, + "B5" | "ETen-B5" | "ETenms-B5" | "HKdla-B5" | "HKm314-B5" | "HKm471-B5" => B5, + "HKscs-B5" | "ETHK-B5" => HKSCS, + "HKdlb-B5" => HKDLB, + "HKgccs-B5" => HKGCCS, + "CNS-EUC" => CNS_EUC, + "KSCpc-EUC" => KSC_PC, + "KSCms-UHC" => UHC, + _ => return None, + }) +} + +/// The length of the code string `bytes` starts with: the length of the code space range whose +/// lowest and highest codes hold each of its bytes between them. `None` when no range does. +fn code_len(space: CodeSpace, bytes: &[u8]) -> Option { + space + .iter() + .filter(|(lo, hi)| bytes.get(..lo.len()).is_some_and(|b| b.iter().zip(lo.iter().zip(hi.iter())).all(|(b, (lo, hi))| (lo..=hi).contains(&b)))) + .map(|(lo, _)| lo.len()) + .min() + .and_then(|len| u8::try_from(len).ok()) } /// Encoding pairs (§3.3.1): codes, each followed by the glyph names from it on, into `table`. @@ -286,6 +556,36 @@ fn encoding_pairs(table: &mut [Option], vals: &[V]) { } } +/// A composite font's name `base-CMap`: the base font's name and its CMap's name, when that is a +/// predefined Unicode CMap ([`unicode_cmap`]), one of the predefined CMaps VectorCraft carries +/// ([`predefined`]) or one of the others ([`stand_in`]), in its horizontal (`-H`) or vertical +/// (`-V`) form. A name ending in an Identity CMap is not read as one, nor is one that ends in an +/// unknown CMap name with an encoding part (`UniFoo-UTF8-H`), which would otherwise end in `H`. +fn composite_font(name: &str) -> Option<(&str, &str)> { + if !(name.ends_with("-H") || name.ends_with("-V")) { + return None; + } + // The longest CMap name first: `83pv-RKSJ-H` before `H`. + // No predefined CMap name is longer than 32 bytes. + for (at, _) in name.match_indices('-').filter(|(at, _)| name.len() - at <= 33) { + let (Some(base), Some(cmap)) = (name.get(..at).filter(|b| !b.is_empty()), name.get(at + 1..)) else { continue }; + if matches!(cmap, "Identity-H" | "Identity-V") { + return None; + } + let encoding_part = + base.rsplit('-').next().is_some_and(|part| ["UCS2", "UTF8", "UTF16", "UTF32", "HW", "EUC", "RKSJ", "B5", "UHC", "Johab"].contains(&part)); + if matches!(cmap, "H" | "V") && encoding_part { + return None; + } + let vertical_of_predefined = || cmap.strip_suffix('V').is_some_and(|stem| predefined(&format!("{stem}H")).is_some()); + let other = || cmap.strip_suffix("-H").or_else(|| cmap.strip_suffix("-V")).is_some_and(|stem| stand_in(stem).is_some()); + if unicode_cmap(cmap).is_some() || predefined(cmap).is_some() || vertical_of_predefined() || other() { + return Some((base, cmap)); + } + } + None +} + /// An operator's `N` operands, the last `N` values before it, when each is a finite number. fn nums(vals: &[V]) -> Option<[f64; N]> { let operands = vals.get(vals.len().checked_sub(N)?..)?; @@ -341,6 +641,9 @@ impl Reader<'_> { let pairs = pairs.iter().rposition(|v| matches!(v, V::Mark)).map_or(pairs, |m| pairs.get(m + 1..).unwrap_or_default()); let mut table = vec![None; 256]; encoding_pairs(&mut table, pairs); + if pairs.iter().any(|v| matches!(v, V::Name(_))) { + st.paired_fonts.push(new.clone()); + } st.font_encodings.push((new.clone(), table, te)); } @@ -469,9 +772,17 @@ impl Reader<'_> { let Some(at) = vals.iter().position(|v| matches!(v, V::Name(_))) else { return }; if let Some(V::Name(name)) = vals.get(at) { st.font = Some(name.clone()); - let name = + let full = st.fonts.iter().find(|(new, _)| new == name).map_or_else(|| name.trim_start_matches('_').to_string(), |(_, old)| old.clone()); - let (family, style) = super::super::super::text::family_style(&name); + // A font whose `TZ` gives encoding pairs is re-encoded, which a composite font + // can't be; a `TZ` without pairs only names the font. + let reencoded = st.paired_fonts.contains(name); + let (name, cmap) = match composite_font(&full).filter(|_| !reencoded) { + Some((base, cmap)) => (base, st.cmap_codes(cmap)), + None => (full.as_str(), None), + }; + st.cmap = cmap; + let (family, style) = super::super::super::text::family_style(name); st.style.font_family = family; st.style.font_style = style; st.style.font_version = None; diff --git a/crates/eps/src/import/ai/tests.rs b/crates/eps/src/import/ai/tests.rs index 23b529c3d..5cd25216d 100644 --- a/crates/eps/src/import/ai/tests.rs +++ b/crates/eps/src/import/ai/tests.rs @@ -3474,6 +3474,96 @@ fn a_tz_with_use_default_0_leaves_te_out() { assert_eq!(legacy_chars(setup, &["_F", "_S", "_M", "_G"], "\\047\\140\\200"), ["'`€", "'`€", "'`€", "’`Ä"]); } +/// Strings in a composite font `base-CMap` decode by that predefined CMap (codes to CIDs, then the +/// character collection's CIDs to Unicode), and the font is `base`; `TE`'s encoding does not apply +/// to them. Codes the CMap leaves out are dropped. A font whose name ends in no predefined CMap +/// decodes byte by byte. +#[test] +fn legacy_type_in_a_cjk_font_decodes_by_its_cmap() { + let typed = |font: &str, chars: &str| point_type("", chars).replace("/_Helvetica", &format!("/{font}")); + let cases: &[(&str, &str, &str)] = &[ + // 日 is 0x93FA and 本 0x967B in Shift-JIS; `\r` ends a paragraph. + ("83pv-RKSJ-H", "\\223\\372\\226{A\\r\\223\\372", "日本A\n日"), + // A tab stays a tab, also after a UTF-16 string's last whole code. + ("90ms-RKSJ-H", "\\223\\372\\tA", "日\tA"), + ("H", "F|\\tF|", "日\t日"), + ("UniJIS-UTF16-H", "e\\345\\t", "日\t"), + // One-byte codes in the Mac Shift-JIS CMap: 0x5C yen, 0x7E tilde, 0x80 backslash, 0xFD + // copyright, and the underscore and grave accent. + ("83pv-RKSJ-H", "\\\\~\\200\\375_`", "¥~\\©_`"), + ("90ms-RKSJ-H", "\\\\~_`", "¥~_`"), + ("90ms-RKSJ-V", "\\223\\372", "日"), + ("90pv-RKSJ-V", "\\223\\372", "日"), + // JIS row and cell: 日 is 0x4652. + ("H", "F|", "日"), + ("V", "F|", "日"), + ("EUC-H", "\\306\\374", "日"), + ("GBK-EUC-H", "\\326\\320", "中"), + ("GBpc-EUC-H", "\\326\\320", "中"), + ("B5pc-H", "\\244\\244", "中"), + ("KSCms-UHC-H", "\\307\\321", "한"), + ("KSCpc-EUC-H", "\\307\\321", "한"), + ("UniJIS-UCS2-H", "e\\345", "日"), + ("UniGB-UTF8-H", "\\344\\270\\255", "中"), + // An encoded U+FFFD is a character; the invalid byte 0xFF is dropped. + ("UniGB-UTF8-H", "\\357\\277\\275\\377\\344\\270\\255", "\u{fffd}中"), + ("UniJIS-UTF32-H", "\\000\\000e\\345", "日"), + ("UniAKR-UTF16-H", "\\254\\000", "가"), + // CMaps VectorCraft doesn't carry: read with the nearest carried one, row-and-cell codes in + // their EUC form (中 is 0x5650 in GB 2312, 한 0x4751 in KS X 1001), or byte by byte. + ("78-RKSJ-H", "\\223\\372", "日"), + ("RKSJ-V", "\\223\\372", "日"), + ("GB-H", "VP", "中"), + ("KSC-V", "GQ", "한"), + ("GBT-H", "AB", "AB"), + // Codes are as long as the code space range they fall in, and unmapped ones are dropped: + // 0x8540 as two bytes, and 0xFD and EUC's 0x8F (in no range) as one. + ("90ms-RKSJ-H", "\\205\\100\\223\\372A\\375B", "日AB"), + ("EUC-H", "\\217A\\306\\374", "A日"), + ]; + let art: String = cases.iter().map(|(cmap, chars, _)| typed(&format!("_Test-W3-{cmap}"), chars)).collect(); + let art = [art, typed("_Test-W3-Identity-H", "\\223"), typed("_Test-W3-UniFoo-UTF8-H", "\\223"), typed("_Helvetica", "\\223")].concat(); + let data = type_stream7(&art).replace("%%BeginSetup\n%%EndSetup", "%%BeginSetup\n[147/Adieresis\nTE\n%%EndSetup"); + let texts = art_texts(&read(data.as_bytes()).unwrap().doc); + let chars: Vec = texts.iter().map(|t| t.plain_text()).collect(); + let mut want: Vec<&str> = cases.iter().map(|(_, _, want)| *want).collect(); + want.extend(["Ä", "Ä", "Ä"]); + assert_eq!(chars, want); + let test = crate::import::family_style("Test-W3"); + for (t, (cmap, _, _)) in texts.iter().zip(cases) { + assert_eq!((&t.runs[0].style.font_family, &t.runs[0].style.font_style), (&test.0, &test.1), "{cmap}"); + } + assert_eq!(texts[0].paragraph_count(), 2); + assert_ne!(texts[cases.len()].runs[0].style.font_style, test.1, "an Identity CMap is not read"); + assert_ne!(texts[cases.len() + 1].runs[0].style.font_style, test.1, "`UniFoo` is not a character collection"); +} + +/// A font whose `TZ` gives encoding pairs is not a composite font, even when its name ends in a +/// CMap name: its strings decode byte by byte, and the font keeps its whole name. A `TZ` without +/// encoding pairs only names a composite font. +#[test] +fn a_re_encoded_font_ending_in_a_cmap_name_is_not_composite() { + let setup = "%AI3_BeginEncoding: _F-Bold-H F-Bold-H\n[ 39/quotesingle\n/_F-Bold-H/F-Bold-H 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n"; + assert_eq!(legacy_chars(setup, &["_F-Bold-H"], "F|'"), ["F|'"]); + let setup = "%AI3_BeginEncoding: _G-Bold-H G-Bold-H\n[ 65/A\n/_G-Bold-H/G-Bold-H 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n"; + assert_eq!(legacy_chars(setup, &["_G-Bold-H"], "F|"), ["F|"]); + let setup = "%AI3_BeginEncoding: _R-W3-83pv-RKSJ-H R-W3-83pv-RKSJ-H\n[/_R-W3-83pv-RKSJ-H/R-W3-83pv-RKSJ-H 0 1 1 TZ\n%AI3_EndEncoding AdobeType\n"; + assert_eq!(legacy_chars(setup, &["_R-W3-83pv-RKSJ-H"], "\\223\\372"), ["日"]); +} + +/// In a font with a Unicode CMap, a one-byte `(\r)` string, or a 0x0D byte after a string's last +/// whole UTF-16 code, still ends a paragraph, as in other fonts, and an overflow `(\r) TX` still +/// ends the story. +#[test] +fn legacy_type_in_a_utf16_font_keeps_its_paragraph_breaks() { + let art = point_type("", "e\\345") + .replace("/_Helvetica", "/_Test-W3-UniJIS-UTF16-H") + .replace("TO\n", "(\\r) Tx\n(e\\345\\r) Tx\n(e\\345) Tx\n(\\r) TX\nTO\n"); + let texts = legacy_texts(&art); + assert_eq!(texts[0].plain_text(), "日\n日\n日"); + assert_eq!(texts[0].paragraph_count(), 3); +} + fn art_texts(doc: &Document) -> Vec { art(doc).iter().filter_map(|n| if let NodeKind::Text(t) = &n.kind { Some((**t).clone()) } else { None }).collect() } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 3c7fb5910..7fa181db5 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -141,7 +141,17 @@ private editing data. VectorCraft: stops. Paragraphs and frames past either limit take the default stops, with a warning. Its special characters (`TS`) join the story, and end render (`Te`) and pop text matrix (`TM`) are read and ignored. Its type in a font re-encoded with encoding pairs of its own (`TZ`) is decoded with those pairs, and with the file's encoding - (`TE`) for the codes they leave out unless the `TZ` sets `useDefault` to 0. + (`TE`) for the codes they leave out unless the `TZ` sets `useDefault` to 0. Its type in a CJK composite font, + whose name ends in a predefined CMap (`HeiseiMin-W3-83pv-RKSJ-H`) and which no `TZ` gives encoding pairs, opens + with the font named without the CMap. Its codes are read by the CMap's code space, mapped to CIDs and then to + Unicode by the character collection's CID-to-Unicode CMap, and codes the CMap does not map are dropped. A `-V` + CMap is read with its `-H` form's codes, and a 0x0D byte ends a paragraph. The Unicode CMaps `Uni*-UCS2`, `-UTF8`, + `-UTF16` and `-UTF32` are read as Unicode. VectorCraft reads the predefined CMaps that craft-pdf does not carry + (`RKSJ`, `78-RKSJ`, `78ms-RKSJ`, `78-EUC`, `78`, `Add`, `Ext`, `NWP`, `GB`, `CNS1`, `KSC`, `B5` and the Hong Kong + `-B5` sets other than `HKscs-B5`) with its own choice of the nearest carried CMap, which can give a different + character for some codes, and decodes `GBT`, `GBT-EUC`, `GBTpc-EUC`, `CNS2` and `KSC-Johab` type byte by byte with + the CMap removed from the font name. A font whose name ends in `-H` or `-V` without a `TZ` that gives encoding pairs + is read with the `H` or `V` CMap. Its area type with several linked areas opens as threaded frames, one for each area with its own matrix, and the story flows through them. Its writing direction (`TV`) opens as vertical type. VectorCraft keeps one direction for each text object, the