Merge pull request #5793 from dotCooCoo/feat/ai-tz-encodings

.ai import: a re-encoded font's own encoding pairs decode its type

crates/engine/tests/import_fuzz.rs

@@ -4078,6 +4078,11 @@ fn arb_ai_token() -> impl Strategy<Value = String> {
             "[1e308/a -1/b TE",
             "\n%AI3_BeginEncoding: _Helvetica Helvetica\n[/_Helvetica/Helvetica 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n",
             "\n%AI3_BeginEncoding:\n",
+            "[ 39/quotesingle 1e308/a -1/b 200/eacute\n/_Helvetica/Helvetica 0 0 1 [0.5 1e308] TZ",
+            "/_ 0 TZ",
+            "[ ] ] [ /a /b (c) TZ",
+            "\n%AI3_EndEncoding AdobeType\n",
+            "\n%%BeginEncoding: _x x\n[ 39/grave\n/_x/x 0 1 TZ\n%%EndEncoding\n",
             "800 Ar",
             "frobnicate",
             "\n%AI5_BeginLayer\n",

crates/eps/src/import/ai/legacy_text.rs

@@ -5,8 +5,9 @@
 //! punctuation `Tq` and hyphenation `Th`), the paint style, and the characters (`Tx`, `Tj`,
 //! overflow `TX`; `\r` ends a paragraph). Final-form operators (the
 //! computed positions, §15.1) are skipped: the type is laid out again. Strings are decoded by the
-//! file's `TE` encoding, else as Windows-1252; a re-encoded font `/_name` is the font its
-//! `%AI3_BeginEncoding` names (else `name`).
+//! encoding pairs of the current font's `TZ`, then by the file's `TE` encoding unless that `TZ`'s
+//! `useDefault` is 0, else as Windows-1252; a re-encoded font `/_name` is the font its `%AI3_BeginEncoding` or `TZ` names
+//! (else `name`).
 
 use kurbo::{Affine, BezPath, ParamCurve, ParamCurveArclen};
 use vectorcraft_color::Paint;
@@ -22,6 +23,10 @@ use crate::latin;
 const MAX_CHARS: usize = 1 << 22;
 /// Largest type size and indent read (points).
 const MAX_SIZE: f64 = 1e4;
+/// Most re-encoded fonts read in one file, for their names and for their encodings each.
+const MAX_FONTS: usize = 4096;
+/// Most values read before one operator in a re-encoding section.
+const MAX_ENCODING_VALUES: usize = 1024;
 /// Most text paths read in one text object (its linked areas); the frames of others are left out,
 /// with a warning.
 const MAX_TEXT_PATHS: usize = 4096;
@@ -76,14 +81,28 @@ pub(super) struct TypeState {
     render: i64,
     /// The encoding strings are in, by code (`TE`); None where it says nothing.
     encoding: Vec<Option<char>>,
-    /// Re-encoded fonts' original names (`%AI3_BeginEncoding: _new old`).
+    /// Re-encoded fonts' original names (`%AI3_BeginEncoding: _new old`, or `TZ`'s).
     fonts: Vec<(String, String)>,
+    /// Re-encoded fonts' own encodings (`TZ`), by the new font's name: what its encoding pairs
+    /// change, by code (None where they say nothing), and whether `TE`'s encoding applies to it.
+    font_encodings: Vec<(String, Vec<Option<char>>, bool)>,
+    /// The font `Tf` set last, as the file names it.
+    font: Option<String>,
     chars: usize,
 }
 
 impl Default for TypeState {
     fn default() -> Self {
-        Self { style: CharStyle::default(), para: ParaStyle::default(), render: 0, encoding: vec![None; 256], fonts: vec![], chars: 0 }
+        Self {
+            style: CharStyle::default(),
+            para: ParaStyle::default(),
+            render: 0,
+            encoding: vec![None; 256],
+            fonts: vec![],
+            font_encodings: vec![],
+            font: None,
+            chars: 0,
+        }
     }
 }
 
@@ -92,10 +111,51 @@ impl TypeState {
     pub(super) fn encoding_comment(&mut self, line: &str) {
         let Some((_, names)) = line.split_once(':') else { return };
         let mut words = names.split_whitespace();
-        if let (Some(new), Some(old)) = (words.next(), words.next())
-            && self.fonts.len() < 4096
-        {
-            self.fonts.push((new.trim_start_matches('/').to_string(), old.trim_start_matches('/').to_string()));
+        if let (Some(new), Some(old)) = (words.next(), words.next()) {
+            self.font_names(new.trim_start_matches('/'), old.trim_start_matches('/'));
+        }
+    }
+
+    /// Re-encoded font `new` is font `old`. The first statement for a name is used.
+    fn font_names(&mut self, new: &str, old: &str) {
+        if self.fonts.len() < MAX_FONTS && !self.fonts.iter().any(|(n, _)| n == new) {
+            self.fonts.push((new.to_string(), old.to_string()));
+        }
+    }
+
+    /// A string's bytes, in the current font.
+    fn decode(&self, bytes: &[u8]) -> String {
+        let own = self.font.as_ref().and_then(|f| self.font_encodings.iter().find(|(n, _, _)| n == f));
+        let (own, te) = own.map_or((None, true), |(_, e, te)| (Some(e.as_slice()), *te));
+        bytes
+            .iter()
+            .map(|&b| {
+                if b == b'\r' {
+                    return '\n';
+                }
+                own.and_then(|e| e.get(usize::from(b)).copied().flatten())
+                    .or_else(|| if te { self.encoding.get(usize::from(b)).copied().flatten() } else { None })
+                    .unwrap_or_else(|| latin::decode(b))
+            })
+            .collect()
+    }
+}
+
+/// Encoding pairs (§3.3.1): codes, each followed by the glyph names from it on, into `table`.
+fn encoding_pairs(table: &mut [Option<char>], vals: &[V]) {
+    let mut code: Option<usize> = None;
+    for v in vals {
+        match v {
+            V::Num(n) if n.is_finite() && (0.0..256.0).contains(n) => code = Some(*n as usize),
+            V::Name(name) => {
+                if let Some(c) = code {
+                    if let (Some(slot), Ok(b)) = (table.get_mut(c), u8::try_from(c)) {
+                        *slot = latin::glyph(b, name);
+                    }
+                    code = Some(c + 1);
+                }
+            }
+            _ => {}
         }
     }
 }
@@ -122,19 +182,86 @@ fn flag(v: f64) -> Option<bool> {
 impl Reader<'_> {
     /// `[encodingPairs TE`: codes, each followed by the glyph names from it on (§3.3.1).
     pub(super) fn te(&mut self, vals: &[V]) {
-        let mut code: Option<usize> = None;
-        for v in vals {
-            match v {
-                V::Num(n) if n.is_finite() && (0.0..256.0).contains(n) => code = Some(*n as usize),
-                V::Name(name) => {
-                    if let Some(c) = code {
-                        if let (Some(slot), Ok(b)) = (self.type_state.encoding.get_mut(c), u8::try_from(c)) {
-                            *slot = latin::glyph(b, name);
-                        }
-                        code = Some(c + 1);
+        encoding_pairs(&mut self.type_state.encoding, vals);
+    }
+
+    /// `[encodingPairs newFontName oldFontName direction fontScript useDefault TZ`, with a Multiple
+    /// Master weight vector `[w0 … wn]` after it, and without the encoding pairs or the direction
+    /// in some versions (§3.3.1): the new font's own encoding pairs, which strings in it are
+    /// decoded by first. `useDefault` 1 (or no number after the names) decodes the codes the pairs
+    /// leave out by `TE`'s encoding, and 0 leaves `TE`'s encoding out for the font. The font names
+    /// are the last two names; only numbers and the weight vector follow them. The first `TZ` for
+    /// a font is used.
+    pub(super) fn tz(&mut self, vals: &[V]) {
+        let mut names = vals.iter().enumerate().filter_map(|(i, v)| if let V::Name(n) = v { Some((i, n)) } else { None }).rev();
+        let (Some((old_at, old)), Some((new_at, new))) = (names.next(), names.next()) else { return };
+        if new_at + 1 != old_at {
+            return;
+        }
+        let tail = vals.get(old_at + 1..).unwrap_or_default();
+        let weights = matches!(tail.last(), Some(V::Arr(_)));
+        let numbers = tail.get(..tail.len() - usize::from(weights)).unwrap_or_default();
+        if !numbers.iter().all(|v| matches!(v, V::Num(_))) {
+            return;
+        }
+        let te = !matches!(numbers.last(), Some(V::Num(n)) if *n == 0.0);
+        let st = &mut self.type_state;
+        st.font_names(new, old);
+        if st.font_encodings.iter().any(|(n, _, _)| n == new) || st.font_encodings.len() >= MAX_FONTS {
+            return;
+        }
+        // The pairs start after the `[` that opens them.
+        let pairs = vals.get(..new_at).unwrap_or_default();
+        let pairs = pairs.iter().rposition(|v| matches!(v, V::Mark)).map_or(pairs, |m| pairs.get(m + 1..).unwrap_or_default());
+        let mut table = vec![None; 256];
+        encoding_pairs(&mut table, pairs);
+        st.font_encodings.push((new.clone(), table, te));
+    }
+
+    /// What is between `%AI3_BeginEncoding` and `%AI3_EndEncoding` (§3.3): its `TZ` is read, and
+    /// every other operator is skipped with its operands.
+    pub(super) fn encoding_section(&mut self, section: &[u8]) {
+        let mut lex = super::Lexer::new(section);
+        let mut vals: Vec<V> = Vec::new();
+        let mut marks: Vec<usize> = Vec::new();
+        // `[`s past the value limit, which are not kept: as many `]`s after them are skipped.
+        let mut dropped: usize = 0;
+        while let Some(t) = lex.next_token() {
+            let v = match t.tok {
+                super::Tok::Num(n) => V::Num(n),
+                super::Tok::Name(n) => V::Name(super::obj::text(n)),
+                super::Tok::Str(s) | super::Tok::Hex(s) => V::Str(s),
+                super::Tok::Word(b"[") => {
+                    // Each `[` kept is a value, so the open arrays are no more than the values.
+                    if vals.len() >= MAX_ENCODING_VALUES {
+                        dropped = dropped.saturating_add(1);
+                        continue;
                     }
+                    marks.push(vals.len());
+                    V::Mark
                 }
-                _ => {}
+                super::Tok::Word(b"]") => {
+                    if dropped > 0 {
+                        dropped -= 1;
+                        continue;
+                    }
+                    let Some(m) = marks.pop() else { continue };
+                    let items: Vec<V> = vals.split_off(m.min(vals.len())).into_iter().skip(1).collect();
+                    V::Arr(items)
+                }
+                super::Tok::Word(w) => {
+                    if w == b"TZ" {
+                        self.tz(&vals);
+                    }
+                    vals.clear();
+                    marks.clear();
+                    dropped = 0;
+                    continue;
+                }
+                _ => continue,
+            };
+            if vals.len() < MAX_ENCODING_VALUES {
+                vals.push(v);
             }
         }
     }
@@ -212,6 +339,7 @@ impl Reader<'_> {
                 // `/name size Tf`, or `/name size ascent descent Tf`.
                 let Some(at) = vals.iter().position(|v| matches!(v, V::Name(_))) else { return };
                 if let Some(V::Name(name)) = vals.get(at) {
+                    st.font = Some(name.clone());
                     let name =
                         st.fonts.iter().find(|(new, _)| new == name).map_or_else(|| name.trim_start_matches('_').to_string(), |(_, old)| old.clone());
                     let (family, style) = super::super::super::text::family_style(&name);
@@ -323,10 +451,7 @@ impl Reader<'_> {
         let Some(bytes) = vals.iter().rev().find_map(|v| if let V::Str(s) = v { Some(s.clone()) } else { None }) else { return };
         let st = &mut self.type_state;
         let Some(t) = self.text.as_mut() else { return };
-        let decoded: String = bytes
-            .iter()
-            .map(|&b| if b == b'\r' { '\n' } else { st.encoding.get(usize::from(b)).copied().flatten().unwrap_or_else(|| latin::decode(b)) })
-            .collect();
+        let decoded = st.decode(&bytes);
         if decoded.is_empty() {
             return;
         }

crates/eps/src/import/ai/read/directives.rs

@@ -127,6 +127,9 @@ impl<'a> Reader<'a> {
         }
         if m == "BeginEncoding" {
             self.type_state.encoding_comment(line);
+            let section = self.section("EndEncoding")?;
+            self.encoding_section(section);
+            return Ok(());
         }
         // Versioned content holds alternatives of the same art, the newest first: the first is
         // read, and the next one (`%AI17_Alternate_Content`, or `…_if_version_gt:` for a version

crates/eps/src/import/ai/read/ops.rs

@@ -369,6 +369,7 @@ impl<'a> Reader<'a> {
             // `Xb`/`XB` a tab definition (and the `Xe` tab descriptions between them).
             "XL" | "Xb" | "XB" | "Xe" => {}
             "TE" => self.te(&vals),
+            "TZ" => self.tz(&vals),
             // An image whose samples follow in comment lines (read at `%AI5_EndRaster`).
             "XI" if self.raster.is_some() => {
                 if let Some(r) = self.raster.as_mut() {

crates/eps/src/import/ai/read/types.rs

@@ -292,7 +292,6 @@ pub(super) const IGNORED: &[&str] = &[
     "i",
     // Path resolution (`Ar`, §12): how finely curves print.
     "Ar",
-    "TZ",
     "Xs",
     "Xt",
     // A brushed path's width at each anchor (`(…) Xt n XD`), on its hidden source path.

crates/eps/src/import/ai/tests.rs

@@ -2937,6 +2937,60 @@ fn legacy_texts(art: &str) -> Vec<vectorcraft_doc::TextObject> {
     art_texts(&doc)
 }
 
+/// The characters of each text object of a legacy-format file whose setup is `setup` and whose
+/// layer holds point type with the string `(chars)` in each font of `fonts`.
+fn legacy_chars(setup: &str, fonts: &[&str], chars: &str) -> Vec<String> {
+    let art: String = fonts.iter().map(|f| point_type("", chars).replace("/_Helvetica 12 Tf", &format!("/{f} 12 Tf"))).collect();
+    let data = type_stream7(&art).replace("%%BeginSetup\n%%EndSetup", &format!("%%BeginSetup\n{setup}%%EndSetup"));
+    let doc = match read(data.as_bytes()) {
+        Ok(r) => r.doc,
+        Err(e) => panic!("{e}"),
+    };
+    art_texts(&doc).iter().map(|t| t.runs.iter().map(|r| r.text.as_str()).collect()).collect()
+}
+
+/// `TZ` (§3.3.1) gives a re-encoded font encoding pairs of its own: its strings decode by them
+/// first, then by `TE`'s encoding when `useDefault` is 1, and other fonts' strings keep `TE`'s. `TZ` is read in an
+/// `%AI3_BeginEncoding` section and in an `%%BeginEncoding` one, with a Multiple Master weight
+/// vector after it, and without the direction operand.
+#[test]
+fn a_re_encoded_fonts_own_encoding_pairs_decode_only_its_strings() {
+    let setup = "[39/quoteright 96/grave 128/Adieresis\nTE\n\
+                 %AI3_BeginEncoding: _F F\n[ 39/quotesingle 200/eacute\n/_F/F 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %%BeginEncoding: _M M\n[ 39/quotesingle\n/_M/M 0 0 1 [0.5 0.25] TZ\n%%EndEncoding\n\
+                 %AI3_BeginEncoding: _N N\n[ 39/quotesingle\n/_N/N 0 1 TZ\n%AI3_EndEncoding AdobeType\n";
+    let got = legacy_chars(setup, &["_F", "_M", "_N", "_G"], "\\047\\140\\200\\310");
+    assert_eq!(got, ["'`Äé", "'`ÄÈ", "'`ÄÈ", "’`ÄÈ"]);
+    // The re-encoded fonts are the fonts they were made from.
+    let data = type_stream7(&point_type("", "a").replace("/_Helvetica", "/_M"))
+        .replace("%%BeginSetup\n%%EndSetup", "%%BeginSetup\n%%BeginEncoding: _M Courier\n[/_M/Courier 0 0 1 TZ\n%%EndEncoding\n%%EndSetup");
+    let texts = art_texts(&read(data.as_bytes()).unwrap().doc);
+    assert_eq!(texts[0].runs[0].style.font_family, "Courier");
+}
+
+/// The reader does not use the pairs of a `TZ` whose font names aren't the last two operands, or
+/// are followed by other values. The first `TZ` for a font is used, also when it has no pairs.
+#[test]
+fn a_tz_of_another_shape_changes_no_encoding() {
+    let setup = "[39/quoteright\nTE\n\
+                 %AI3_BeginEncoding: _F F\n[ 39/quotesingle\n/_F/F (x) 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _G G\n[ 39/quotesingle\n/_G 0 /G 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _H H\n[ 39/grave\n/_H/H 0 0 1 TZ\n[ 39/quotesingle\n/_H/H 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _K K\n[/_K/K 0 0 1 TZ\n[ 39/quotesingle\n/_K/K 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n";
+    assert_eq!(legacy_chars(setup, &["_F", "_G", "_H", "_K"], "\\047"), ["’", "’", "`", "’"]);
+}
+
+/// A `TZ` whose `useDefault` is 0 leaves `TE`'s encoding out for its font (§3.3.1): the codes its
+/// pairs leave out decode as Windows-1252.
+#[test]
+fn a_tz_with_use_default_0_leaves_te_out() {
+    let setup = "[39/quoteright 128/Adieresis\nTE\n\
+                 %AI3_BeginEncoding: _F F\n[ 96/grave\n/_F/F 0 0 0 TZ\n%AI3_EndEncoding TrueType\n\
+                 %AI3_BeginEncoding: _S S\n[/_S/S 0 0 0 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _M M\n[ 96/grave\n/_M/M 0 0 [0.5] TZ\n%AI3_EndEncoding AdobeType\n";
+    assert_eq!(legacy_chars(setup, &["_F", "_S", "_M", "_G"], "\\047\\140\\200"), ["'`€", "'`€", "'`€", "’`Ä"]);
+}
+
 fn art_texts(doc: &Document) -> Vec<vectorcraft_doc::TextObject> {
     art(doc).iter().filter_map(|n| if let NodeKind::Text(t) = &n.kind { Some((**t).clone()) } else { None }).collect()
 }

docs/file-format-parity.md

@@ -122,6 +122,9 @@ private editing data. VectorCraft:
   without the prolog opens since #1027, its legacy type (`To` … `TO`) since #868. Graphs (`%_Gs` … `%_GS`)
   come in as their drawn art, as plain groups, paths and type, without their data or settings, with a note.
   without the prolog opens since #1027, its legacy type (`To` … `TO`) since #868, with its paragraphs' letter
+  and word spacing (`TC`, `TW`), hanging punctuation (`Tq`) and hyphenation (`Th`). Its type in a font
+  re-encoded with encoding pairs of its own (`TZ`) is decoded with those pairs, and with the file's encoding
+  (`TE`) for the codes they leave out unless the `TZ` sets `useDefault` to 0. Its patterns in
   and word spacing (`TC`, `TW`), hanging punctuation (`Tq`) and hyphenation (`Th`); its special characters
   (`TS`) join the story, and end render (`Te`) and pop text matrix (`TM`) are read and ignored. Its patterns in
   the version 7 form (a list of `@` and `&` layers, `_ &` for the tile's background) open as pattern swatches,