.ai import: a re-encoded font's own encoding pairs decode its type

In the legacy format, `TZ` makes a re-encoded font from an existing one
and can give it encoding pairs of its own, which change only that font's
encoding. Its `useDefault` operand says whether the file's `TE` encoding
also applies to the font (Adobe Illustrator File Format Specification
7.0, section 3.3.1,
https://13thmonkey.org/documentation/fonts/AI7FileFormat.pdf). The
reader skipped the body of `%AI3_BeginEncoding` sections and ignored
`TZ`, so every string was decoded with `TE`'s encoding.

Strings in a re-encoded font are now decoded with its own pairs, then
with `TE`'s encoding when `useDefault` is 1, and as Windows-1252 for the
codes left. Strings in other fonts keep `TE`'s encoding. The reader
reads `TZ` in the body of an `%AI3_BeginEncoding` section and wherever
else it appears in the script, with or without the direction operand
and the Multiple Master weight vector. The font names are the last two
names, and only numbers and the weight vector may follow them; the
reader does not use a `TZ` of another shape. The first `TZ` for a font
is used.

`TZ`'s font names also map the re-encoded font to the font it was made
from, as the `%AI3_BeginEncoding` comment does. Type in a re-encoded
font that no such comment names now opens in the font `TZ` names; when
both name the font, the first one read is used.

crates/engine/tests/import_fuzz.rs

@@ -4078,6 +4078,11 @@ fn arb_ai_token() -> impl Strategy<Value = String> {
             "[1e308/a -1/b TE",
             "\n%AI3_BeginEncoding: _Helvetica Helvetica\n[/_Helvetica/Helvetica 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n",
             "\n%AI3_BeginEncoding:\n",
+            "[ 39/quotesingle 1e308/a -1/b 200/eacute\n/_Helvetica/Helvetica 0 0 1 [0.5 1e308] TZ",
+            "/_ 0 TZ",
+            "[ ] ] [ /a /b (c) TZ",
+            "\n%AI3_EndEncoding AdobeType\n",
+            "\n%%BeginEncoding: _x x\n[ 39/grave\n/_x/x 0 1 TZ\n%%EndEncoding\n",
             "800 Ar",
             "frobnicate",
             "\n%AI5_BeginLayer\n",

crates/eps/src/import/ai/legacy_text.rs

@@ -5,8 +5,9 @@
 //! punctuation `Tq` and hyphenation `Th`), the paint style, and the characters (`Tx`, `Tj`,
 //! overflow `TX`; `\r` ends a paragraph). Final-form operators (the
 //! computed positions, §15.1) are skipped: the type is laid out again. Strings are decoded by the
-//! file's `TE` encoding, else as Windows-1252; a re-encoded font `/_name` is the font its
-//! `%AI3_BeginEncoding` names (else `name`).
+//! encoding pairs of the current font's `TZ`, then by the file's `TE` encoding unless that `TZ`'s
+//! `useDefault` is 0, else as Windows-1252; a re-encoded font `/_name` is the font its `%AI3_BeginEncoding` or `TZ` names
+//! (else `name`).
 
 use kurbo::{Affine, BezPath, ParamCurve, ParamCurveArclen};
 use vectorcraft_color::Paint;
@@ -22,6 +23,10 @@ use crate::latin;
 const MAX_CHARS: usize = 1 << 22;
 /// Largest type size and indent read (points).
 const MAX_SIZE: f64 = 1e4;
+/// Most re-encoded fonts read in one file, for their names and for their encodings each.
+const MAX_FONTS: usize = 4096;
+/// Most values read before one operator in a re-encoding section.
+const MAX_ENCODING_VALUES: usize = 1024;
 /// Most text paths read in one text object (its linked areas); the frames of others are left out,
 /// with a warning.
 const MAX_TEXT_PATHS: usize = 4096;
@@ -76,14 +81,28 @@ pub(super) struct TypeState {
     render: i64,
     /// The encoding strings are in, by code (`TE`); None where it says nothing.
     encoding: Vec<Option<char>>,
-    /// Re-encoded fonts' original names (`%AI3_BeginEncoding: _new old`).
+    /// Re-encoded fonts' original names (`%AI3_BeginEncoding: _new old`, or `TZ`'s).
     fonts: Vec<(String, String)>,
+    /// Re-encoded fonts' own encodings (`TZ`), by the new font's name: what its encoding pairs
+    /// change, by code (None where they say nothing), and whether `TE`'s encoding applies to it.
+    font_encodings: Vec<(String, Vec<Option<char>>, bool)>,
+    /// The font `Tf` set last, as the file names it.
+    font: Option<String>,
     chars: usize,
 }
 
 impl Default for TypeState {
     fn default() -> Self {
-        Self { style: CharStyle::default(), para: ParaStyle::default(), render: 0, encoding: vec![None; 256], fonts: vec![], chars: 0 }
+        Self {
+            style: CharStyle::default(),
+            para: ParaStyle::default(),
+            render: 0,
+            encoding: vec![None; 256],
+            fonts: vec![],
+            font_encodings: vec![],
+            font: None,
+            chars: 0,
+        }
     }
 }
 
@@ -92,10 +111,51 @@ impl TypeState {
     pub(super) fn encoding_comment(&mut self, line: &str) {
         let Some((_, names)) = line.split_once(':') else { return };
         let mut words = names.split_whitespace();
-        if let (Some(new), Some(old)) = (words.next(), words.next())
-            && self.fonts.len() < 4096
-        {
-            self.fonts.push((new.trim_start_matches('/').to_string(), old.trim_start_matches('/').to_string()));
+        if let (Some(new), Some(old)) = (words.next(), words.next()) {
+            self.font_names(new.trim_start_matches('/'), old.trim_start_matches('/'));
+        }
+    }
+
+    /// Re-encoded font `new` is font `old`. The first statement for a name is used.
+    fn font_names(&mut self, new: &str, old: &str) {
+        if self.fonts.len() < MAX_FONTS && !self.fonts.iter().any(|(n, _)| n == new) {
+            self.fonts.push((new.to_string(), old.to_string()));
+        }
+    }
+
+    /// A string's bytes, in the current font.
+    fn decode(&self, bytes: &[u8]) -> String {
+        let own = self.font.as_ref().and_then(|f| self.font_encodings.iter().find(|(n, _, _)| n == f));
+        let (own, te) = own.map_or((None, true), |(_, e, te)| (Some(e.as_slice()), *te));
+        bytes
+            .iter()
+            .map(|&b| {
+                if b == b'\r' {
+                    return '\n';
+                }
+                own.and_then(|e| e.get(usize::from(b)).copied().flatten())
+                    .or_else(|| if te { self.encoding.get(usize::from(b)).copied().flatten() } else { None })
+                    .unwrap_or_else(|| latin::decode(b))
+            })
+            .collect()
+    }
+}
+
+/// Encoding pairs (§3.3.1): codes, each followed by the glyph names from it on, into `table`.
+fn encoding_pairs(table: &mut [Option<char>], vals: &[V]) {
+    let mut code: Option<usize> = None;
+    for v in vals {
+        match v {
+            V::Num(n) if n.is_finite() && (0.0..256.0).contains(n) => code = Some(*n as usize),
+            V::Name(name) => {
+                if let Some(c) = code {
+                    if let (Some(slot), Ok(b)) = (table.get_mut(c), u8::try_from(c)) {
+                        *slot = latin::glyph(b, name);
+                    }
+                    code = Some(c + 1);
+                }
+            }
+            _ => {}
         }
     }
 }
@@ -122,19 +182,86 @@ fn flag(v: f64) -> Option<bool> {
 impl Reader<'_> {
     /// `[encodingPairs TE`: codes, each followed by the glyph names from it on (§3.3.1).
     pub(super) fn te(&mut self, vals: &[V]) {
-        let mut code: Option<usize> = None;
-        for v in vals {
-            match v {
-                V::Num(n) if n.is_finite() && (0.0..256.0).contains(n) => code = Some(*n as usize),
-                V::Name(name) => {
-                    if let Some(c) = code {
-                        if let (Some(slot), Ok(b)) = (self.type_state.encoding.get_mut(c), u8::try_from(c)) {
-                            *slot = latin::glyph(b, name);
-                        }
-                        code = Some(c + 1);
+        encoding_pairs(&mut self.type_state.encoding, vals);
+    }
+
+    /// `[encodingPairs newFontName oldFontName direction fontScript useDefault TZ`, with a Multiple
+    /// Master weight vector `[w0 … wn]` after it, and without the encoding pairs or the direction
+    /// in some versions (§3.3.1): the new font's own encoding pairs, which strings in it are
+    /// decoded by first. `useDefault` 1 (or no number after the names) decodes the codes the pairs
+    /// leave out by `TE`'s encoding, and 0 leaves `TE`'s encoding out for the font. The font names
+    /// are the last two names; only numbers and the weight vector follow them. The first `TZ` for
+    /// a font is used.
+    pub(super) fn tz(&mut self, vals: &[V]) {
+        let mut names = vals.iter().enumerate().filter_map(|(i, v)| if let V::Name(n) = v { Some((i, n)) } else { None }).rev();
+        let (Some((old_at, old)), Some((new_at, new))) = (names.next(), names.next()) else { return };
+        if new_at + 1 != old_at {
+            return;
+        }
+        let tail = vals.get(old_at + 1..).unwrap_or_default();
+        let weights = matches!(tail.last(), Some(V::Arr(_)));
+        let numbers = tail.get(..tail.len() - usize::from(weights)).unwrap_or_default();
+        if !numbers.iter().all(|v| matches!(v, V::Num(_))) {
+            return;
+        }
+        let te = !matches!(numbers.last(), Some(V::Num(n)) if *n == 0.0);
+        let st = &mut self.type_state;
+        st.font_names(new, old);
+        if st.font_encodings.iter().any(|(n, _, _)| n == new) || st.font_encodings.len() >= MAX_FONTS {
+            return;
+        }
+        // The pairs start after the `[` that opens them.
+        let pairs = vals.get(..new_at).unwrap_or_default();
+        let pairs = pairs.iter().rposition(|v| matches!(v, V::Mark)).map_or(pairs, |m| pairs.get(m + 1..).unwrap_or_default());
+        let mut table = vec![None; 256];
+        encoding_pairs(&mut table, pairs);
+        st.font_encodings.push((new.clone(), table, te));
+    }
+
+    /// What is between `%AI3_BeginEncoding` and `%AI3_EndEncoding` (§3.3): its `TZ` is read, and
+    /// every other operator is skipped with its operands.
+    pub(super) fn encoding_section(&mut self, section: &[u8]) {
+        let mut lex = super::Lexer::new(section);
+        let mut vals: Vec<V> = Vec::new();
+        let mut marks: Vec<usize> = Vec::new();
+        // `[`s past the value limit, which are not kept: as many `]`s after them are skipped.
+        let mut dropped: usize = 0;
+        while let Some(t) = lex.next_token() {
+            let v = match t.tok {
+                super::Tok::Num(n) => V::Num(n),
+                super::Tok::Name(n) => V::Name(super::obj::text(n)),
+                super::Tok::Str(s) | super::Tok::Hex(s) => V::Str(s),
+                super::Tok::Word(b"[") => {
+                    // Each `[` kept is a value, so the open arrays are no more than the values.
+                    if vals.len() >= MAX_ENCODING_VALUES {
+                        dropped = dropped.saturating_add(1);
+                        continue;
                     }
+                    marks.push(vals.len());
+                    V::Mark
                 }
-                _ => {}
+                super::Tok::Word(b"]") => {
+                    if dropped > 0 {
+                        dropped -= 1;
+                        continue;
+                    }
+                    let Some(m) = marks.pop() else { continue };
+                    let items: Vec<V> = vals.split_off(m.min(vals.len())).into_iter().skip(1).collect();
+                    V::Arr(items)
+                }
+                super::Tok::Word(w) => {
+                    if w == b"TZ" {
+                        self.tz(&vals);
+                    }
+                    vals.clear();
+                    marks.clear();
+                    dropped = 0;
+                    continue;
+                }
+                _ => continue,
+            };
+            if vals.len() < MAX_ENCODING_VALUES {
+                vals.push(v);
             }
         }
     }
@@ -212,6 +339,7 @@ impl Reader<'_> {
                 // `/name size Tf`, or `/name size ascent descent Tf`.
                 let Some(at) = vals.iter().position(|v| matches!(v, V::Name(_))) else { return };
                 if let Some(V::Name(name)) = vals.get(at) {
+                    st.font = Some(name.clone());
                     let name =
                         st.fonts.iter().find(|(new, _)| new == name).map_or_else(|| name.trim_start_matches('_').to_string(), |(_, old)| old.clone());
                     let (family, style) = super::super::super::text::family_style(&name);
@@ -323,10 +451,7 @@ impl Reader<'_> {
         let Some(bytes) = vals.iter().rev().find_map(|v| if let V::Str(s) = v { Some(s.clone()) } else { None }) else { return };
         let st = &mut self.type_state;
         let Some(t) = self.text.as_mut() else { return };
-        let decoded: String = bytes
-            .iter()
-            .map(|&b| if b == b'\r' { '\n' } else { st.encoding.get(usize::from(b)).copied().flatten().unwrap_or_else(|| latin::decode(b)) })
-            .collect();
+        let decoded = st.decode(&bytes);
         if decoded.is_empty() {
             return;
         }

crates/eps/src/import/ai/read/directives.rs

@@ -127,6 +127,9 @@ impl<'a> Reader<'a> {
         }
         if m == "BeginEncoding" {
             self.type_state.encoding_comment(line);
+            let section = self.section("EndEncoding")?;
+            self.encoding_section(section);
+            return Ok(());
         }
         // Versioned content holds alternatives of the same art, the newest first: the first is
         // read, and the next one (`%AI17_Alternate_Content`, or `…_if_version_gt:` for a version

crates/eps/src/import/ai/read/ops.rs

@@ -369,6 +369,7 @@ impl<'a> Reader<'a> {
             // `Xb`/`XB` a tab definition (and the `Xe` tab descriptions between them).
             "XL" | "Xb" | "XB" | "Xe" => {}
             "TE" => self.te(&vals),
+            "TZ" => self.tz(&vals),
             // An image whose samples follow in comment lines (read at `%AI5_EndRaster`).
             "XI" if self.raster.is_some() => {
                 if let Some(r) = self.raster.as_mut() {

crates/eps/src/import/ai/read/types.rs

@@ -292,7 +292,6 @@ pub(super) const IGNORED: &[&str] = &[
     "i",
     // Path resolution (`Ar`, §12): how finely curves print.
     "Ar",
-    "TZ",
     "Xs",
     "Xt",
     // A brushed path's width at each anchor (`(…) Xt n XD`), on its hidden source path.

crates/eps/src/import/ai/tests.rs

@@ -2937,6 +2937,60 @@ fn legacy_texts(art: &str) -> Vec<vectorcraft_doc::TextObject> {
     art_texts(&doc)
 }
 
+/// The characters of each text object of a legacy-format file whose setup is `setup` and whose
+/// layer holds point type with the string `(chars)` in each font of `fonts`.
+fn legacy_chars(setup: &str, fonts: &[&str], chars: &str) -> Vec<String> {
+    let art: String = fonts.iter().map(|f| point_type("", chars).replace("/_Helvetica 12 Tf", &format!("/{f} 12 Tf"))).collect();
+    let data = type_stream7(&art).replace("%%BeginSetup\n%%EndSetup", &format!("%%BeginSetup\n{setup}%%EndSetup"));
+    let doc = match read(data.as_bytes()) {
+        Ok(r) => r.doc,
+        Err(e) => panic!("{e}"),
+    };
+    art_texts(&doc).iter().map(|t| t.runs.iter().map(|r| r.text.as_str()).collect()).collect()
+}
+
+/// `TZ` (§3.3.1) gives a re-encoded font encoding pairs of its own: its strings decode by them
+/// first, then by `TE`'s encoding when `useDefault` is 1, and other fonts' strings keep `TE`'s. `TZ` is read in an
+/// `%AI3_BeginEncoding` section and in an `%%BeginEncoding` one, with a Multiple Master weight
+/// vector after it, and without the direction operand.
+#[test]
+fn a_re_encoded_fonts_own_encoding_pairs_decode_only_its_strings() {
+    let setup = "[39/quoteright 96/grave 128/Adieresis\nTE\n\
+                 %AI3_BeginEncoding: _F F\n[ 39/quotesingle 200/eacute\n/_F/F 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %%BeginEncoding: _M M\n[ 39/quotesingle\n/_M/M 0 0 1 [0.5 0.25] TZ\n%%EndEncoding\n\
+                 %AI3_BeginEncoding: _N N\n[ 39/quotesingle\n/_N/N 0 1 TZ\n%AI3_EndEncoding AdobeType\n";
+    let got = legacy_chars(setup, &["_F", "_M", "_N", "_G"], "\\047\\140\\200\\310");
+    assert_eq!(got, ["'`Äé", "'`ÄÈ", "'`ÄÈ", "’`ÄÈ"]);
+    // The re-encoded fonts are the fonts they were made from.
+    let data = type_stream7(&point_type("", "a").replace("/_Helvetica", "/_M"))
+        .replace("%%BeginSetup\n%%EndSetup", "%%BeginSetup\n%%BeginEncoding: _M Courier\n[/_M/Courier 0 0 1 TZ\n%%EndEncoding\n%%EndSetup");
+    let texts = art_texts(&read(data.as_bytes()).unwrap().doc);
+    assert_eq!(texts[0].runs[0].style.font_family, "Courier");
+}
+
+/// The reader does not use the pairs of a `TZ` whose font names aren't the last two operands, or
+/// are followed by other values. The first `TZ` for a font is used, also when it has no pairs.
+#[test]
+fn a_tz_of_another_shape_changes_no_encoding() {
+    let setup = "[39/quoteright\nTE\n\
+                 %AI3_BeginEncoding: _F F\n[ 39/quotesingle\n/_F/F (x) 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _G G\n[ 39/quotesingle\n/_G 0 /G 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _H H\n[ 39/grave\n/_H/H 0 0 1 TZ\n[ 39/quotesingle\n/_H/H 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _K K\n[/_K/K 0 0 1 TZ\n[ 39/quotesingle\n/_K/K 0 0 1 TZ\n%AI3_EndEncoding AdobeType\n";
+    assert_eq!(legacy_chars(setup, &["_F", "_G", "_H", "_K"], "\\047"), ["’", "’", "`", "’"]);
+}
+
+/// A `TZ` whose `useDefault` is 0 leaves `TE`'s encoding out for its font (§3.3.1): the codes its
+/// pairs leave out decode as Windows-1252.
+#[test]
+fn a_tz_with_use_default_0_leaves_te_out() {
+    let setup = "[39/quoteright 128/Adieresis\nTE\n\
+                 %AI3_BeginEncoding: _F F\n[ 96/grave\n/_F/F 0 0 0 TZ\n%AI3_EndEncoding TrueType\n\
+                 %AI3_BeginEncoding: _S S\n[/_S/S 0 0 0 TZ\n%AI3_EndEncoding AdobeType\n\
+                 %AI3_BeginEncoding: _M M\n[ 96/grave\n/_M/M 0 0 [0.5] TZ\n%AI3_EndEncoding AdobeType\n";
+    assert_eq!(legacy_chars(setup, &["_F", "_S", "_M", "_G"], "\\047\\140\\200"), ["'`€", "'`€", "'`€", "’`Ä"]);
+}
+
 fn art_texts(doc: &Document) -> Vec<vectorcraft_doc::TextObject> {
     art(doc).iter().filter_map(|n| if let NodeKind::Text(t) = &n.kind { Some((**t).clone()) } else { None }).collect()
 }

docs/file-format-parity.md

@@ -122,6 +122,9 @@ private editing data. VectorCraft:
   without the prolog opens since #1027, its legacy type (`To` … `TO`) since #868. Graphs (`%_Gs` … `%_GS`)
   come in as their drawn art, as plain groups, paths and type, without their data or settings, with a note.
   without the prolog opens since #1027, its legacy type (`To` … `TO`) since #868, with its paragraphs' letter
+  and word spacing (`TC`, `TW`), hanging punctuation (`Tq`) and hyphenation (`Th`). Its type in a font
+  re-encoded with encoding pairs of its own (`TZ`) is decoded with those pairs, and with the file's encoding
+  (`TE`) for the codes they leave out unless the `TZ` sets `useDefault` to 0. Its patterns in
   and word spacing (`TC`, `TW`), hanging punctuation (`Tq`) and hyphenation (`Th`); its special characters
   (`TS`) join the story, and end render (`Te`) and pop text matrix (`TM`) are read and ignored. Its patterns in
   the version 7 form (a list of `@` and `&` layers, `_ &` for the tile's background) open as pattern swatches,