.ai import: ASCII85 data in a section does not hide the section's end

A section scan lexed a /ForeignObject's /Data, a /Binary dictionary's data and a text document's data as tokens. A `%_` line of that data holding a `(` opened a string, so the section ran past its end marker: the editing data failed to read, or the art after it was misread. Reader::section now skips the data up to its `~>`, as the reader does for art outside a section. Data without its `~>` before the section's end marker is not skipped. The scan for swatch names does the same.

crates/engine/tests/import_fuzz.rs

@@ -3633,8 +3633,8 @@ proptest! {
     }
 }
 
-/// Editing data with two symbols (S2's art holds an instance of S1), the symbol list and a layer of
-/// instances, as the format writes them.
+/// Editing data with symbols (S2's art holds an instance of S1, F's holds non-native art), the
+/// symbol list and a layer of instances, as the format writes them.
 fn symbols_text() -> String {
     use vectorcraft_testkit::ai;
     let def = |name: &str, art: &str, reg: u32| {
@@ -3643,7 +3643,12 @@ fn symbols_text() -> String {
         )
     };
     let inst = |name: &str, m: &str| format!("/SymbolInstance :\n({name}) /SymbolRef ,\n{m} /RTransformMatrix\n;\n9 () XW\n");
-    let defs = def("S1", "0 1 1 0 1 0 0 Xa\n-2 -2 m\n2 -2 L\n2 2 L\n-2 2 L\nf\n", 5) + &def("S2", &inst("S1", "1 0 0 -1 -8081 8237"), 1);
+    // Non-native art keeps ASCII85 data in comment lines, some of them `%_` lines holding a `(`.
+    let foreign = "/ForeignObject :\n1 /Version ,\n2 0 0 2 10 20  /RTransform ,\n10 20  /Origin ,\n1 2 5 8  /Bounds ,\n/Data ,\n\
+                   %,u@!!/MSk8%41#ocdN=10d&\n%_O%*mGn(RduU.]4`[u[.f\n%_;(]<[%\n%1H@<P2`V<S,pbuU7L]\\~>\n;\n";
+    let defs = def("F", &format!("{foreign}-2 -2 m\n2 -2 L\n2 2 L\nf\n"), 5)
+        + &def("S1", "0 1 1 0 1 0 0 Xa\n-2 -2 m\n2 -2 L\n2 2 L\n-2 2 L\nf\n", 5)
+        + &def("S2", &inst("S1", "1 0 0 -1 -8081 8237"), 1);
     let list = "%AI24_BeginSymbolList\n(S2)\n(S1)\n%AI24_EndSymbolList\n";
     let page = ai::layer(
         "L",
@@ -3668,7 +3673,7 @@ fn the_symbols_fuzz_data_opens_live() {
     let private = [b"%AI24_ZStandard_Data".as_slice(), &zstd(&symbols_text())].concat();
     let (doc, warnings, layered) = vectorcraft_eps::layered_ai(&private, Document::new(200.0, 100.0), vec![], false, true, 1.0);
     assert!(layered, "{warnings:?}");
-    let names: Vec<&str> = doc.symbols.iter().map(|s| s.name.as_str()).collect();
+    let names: Vec<&str> = doc.symbols.iter().map(|s| s.name.as_str()).filter(|n| *n != "F").collect();
     assert_eq!(names, ["S2", "S1"]);
     let mut instances = 0;
     doc.walk(|n| instances += usize::from(matches!(n.kind, vectorcraft_doc::NodeKind::SymbolInstance { .. })));
@@ -4165,6 +4170,7 @@ fn arb_ai_token() -> impl Strategy<Value = String> {
             "\n%_/AI11Text :\n%_0 /StoryIndex ,\n%_;\n",
             "/SymbolInstance :",
             "/Binary : /ASCII85Decode ,",
+            "/AI11TextDocument : /ASCII85Decode ,",
             "~>",
             "p",
             "P",

crates/eps/src/import/ai/lex.rs

@@ -133,7 +133,8 @@ impl<'a> Lexer<'a> {
         self.src.get(self.pos).copied()
     }
 
-    fn rest(&self) -> &'a [u8] {
+    /// The data from where the next token is looked for.
+    pub fn rest(&self) -> &'a [u8] {
         self.src.get(self.pos..).unwrap_or_default()
     }
 

crates/eps/src/import/ai/read.rs

@@ -279,6 +279,8 @@ fn swatch_names(data: &[u8]) -> ShownNames {
     let (mut palette_at, mut palette) = (None, None);
     // A pattern's name is the first string of its definition.
     let mut head = false;
+    // ASCII85 data in comment lines is skipped past, as the reader skips it.
+    let mut dictionaries = ops::Dictionaries::default();
     loop {
         let before = lex.offset();
         let Some(t) = lex.next_token() else {
@@ -291,6 +293,8 @@ fn swatch_names(data: &[u8]) -> ShownNames {
                 _ => break,
             }
         };
+        let end = skip.map(|s| s.1).or(palette_at.map(|_| "EndPalette")).or(section.map(|s| s.1));
+        dictionaries.pass(&t.tok, &mut lex, end);
         match &t.tok {
             Tok::Comment(c) => {
                 let line = String::from_utf8_lossy(c);

crates/eps/src/import/ai/read/directives.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 
 use super::super::brush;
 use super::art::{four, sane, two};
+use super::ops::Dictionaries;
 use super::types::{
     FLIP, GState, Kind, MAX_DEPTH, MAX_HEX, MAX_LISTED, MAX_NODES, MAX_NOTE, MAX_PATTERNS, MAX_PLACED_FOLDER, MAX_STACK, MAX_SUBREADERS, MAX_SYMBOLS,
     PatternHead, Place, Raster, SKIPPED, SymbolHead, VERSIONED_UNENDED,
@@ -330,15 +331,18 @@ impl<'a> Reader<'a> {
         self.section(end).map(|_| ())
     }
 
-    /// [`Self::skip_to`] → what was skipped, up to the comment that ends it.
+    /// [`Self::skip_to`] → what was skipped, up to the comment that ends it. ASCII85 data in comment
+    /// lines is skipped past as [`Self::add_object_entry`] skips it.
     fn section(&mut self, end: &str) -> Result<&'a [u8], String> {
         let begin = SKIPPED.iter().find(|(_, e)| *e == end).map_or("", |(b, _)| *b);
         let begin = if end == "End_Versioned_Content" { "Begin_Content_if_version_gt" } else { begin };
         let start = self.lex.offset();
         let mut depth = 0usize;
+        let mut dictionaries = Dictionaries::default();
         loop {
             let before = self.lex.offset();
             let Some(t) = self.lex.next_token() else { break };
+            dictionaries.pass(&t.tok, &mut self.lex, Some(end));
             if let Tok::Comment(c) = t.tok {
                 let line = String::from_utf8_lossy(c);
                 let m = marker(&line);

crates/eps/src/import/ai/read/ops.rs

@@ -34,10 +34,11 @@ impl<'a> Reader<'a> {
             // of it may start with `%_` and read as hidden tokens: a `(` there would open
             // a string that swallows the rest of the layer.
             let key = o.entries.last().and_then(|(k, _)| k.as_deref());
-            if o.ty == "Binary" && key == Some("ASCII85Decode") {
-                self.lex.skip_past(b"~>");
-            } else if o.ty == "ForeignObject" && key == Some("Data") {
-                self.foreign_data = Some(self.lex.skip_past(b"~>").to_vec());
+            if ascii85_follows(&o.ty, key) {
+                let data = self.lex.skip_past(b"~>");
+                if o.ty == "ForeignObject" {
+                    self.foreign_data = Some(data.to_vec());
+                }
             }
         }
     }
@@ -1122,6 +1123,94 @@ fn strip(n: &mut Node, hidden: &HashSet<NodeId>) {
     }
 }
 
+/// Whether ASCII85 data in comment lines, up to its `~>`, follows the entry keyed `key` of a
+/// dictionary of type `ty`: the `/ASCII85Decode` of a `/Binary` dictionary or a text document, or a
+/// non-native art object's (`/ForeignObject`'s) `/Data`.
+pub(super) fn ascii85_follows(ty: &str, key: Option<&str>) -> bool {
+    matches!((ty, key), ("Binary" | "AI11TextDocument" | "AI11UndoFreeTextDocument", Some("ASCII85Decode")) | ("ForeignObject", Some("Data")))
+}
+
+/// The dictionaries (`/Type :` … `;`) a scan of tokens passes that the reader doesn't read, kept to
+/// tell where an entry's ASCII85 data follows ([`ascii85_follows`]), so the scan can skip past it
+/// as the reader does: a `(` on a `%_` line of the data would otherwise open a string.
+#[derive(Default)]
+pub(super) struct Dictionaries<'s> {
+    /// The open dictionaries' types, innermost last, up to [`MAX_DEPTH`].
+    types: Vec<&'s [u8]>,
+    /// How many more are open inside the last of `types`.
+    deeper: usize,
+    /// The name just before the token taken.
+    name: Option<&'s [u8]>,
+    /// Whether the string just before the token taken is a key that ASCII85 data follows.
+    string_key: bool,
+    /// Data that starts before this offset has no `~>`, so it isn't searched for again.
+    no_end_before: usize,
+}
+
+/// The offset just past the `~>` of ASCII85 data at the start of `rest`. `Err` holds the offset of
+/// the comment with marker `end` that comes first, or the end of `rest`. A line of the data that
+/// reads as that marker counts as the marker.
+fn data_end(rest: &[u8], end: Option<&str>) -> Result<usize, usize> {
+    let mut at = 0;
+    for line in rest.split(|b| matches!(b, b'\r' | b'\n')) {
+        let comment = line.strip_prefix(b"%").and_then(|c| std::str::from_utf8(c).ok());
+        if comment.is_some_and(|c| Some(super::marker(c)) == end) {
+            return Err(at);
+        }
+        if let Some(i) = line.windows(2).position(|w| w == b"~>") {
+            return Ok(at + i + 2);
+        }
+        at += line.len() + 1;
+    }
+    Err(rest.len())
+}
+
+impl<'s> Dictionaries<'s> {
+    /// Take `tok`, the token `lex` just gave, and skip past the ASCII85 data that follows it. Data
+    /// without its `~>` before `end`, the end marker of the section being scanned ([`data_end`]),
+    /// isn't skipped, so the scan still finds that marker.
+    pub(super) fn pass(&mut self, tok: &super::Tok<'s>, lex: &mut super::Lexer<'s>, end: Option<&str>) {
+        let at = lex.offset();
+        if self.data_follows(tok) && at >= self.no_end_before {
+            match data_end(lex.rest(), end) {
+                Ok(end) => lex.seek(at + end),
+                Err(stop) => self.no_end_before = at + stop,
+            }
+        }
+    }
+
+    /// Take the next token → whether ASCII85 data follows it.
+    fn data_follows(&mut self, tok: &super::Tok<'s>) -> bool {
+        // Comments between a name and what follows it are passed over, as the reader passes them.
+        if matches!(tok, super::Tok::Comment(_)) {
+            return false;
+        }
+        let name = self.name.take();
+        let string_key = std::mem::take(&mut self.string_key);
+        match tok {
+            super::Tok::Name(n) => self.name = Some(n),
+            // A string keys an entry as a name does, but gives a dictionary no type.
+            super::Tok::Str(s) | super::Tok::Hex(s) => self.string_key = self.keys_data(s),
+            super::Tok::Word(b":") if self.types.len() < MAX_DEPTH => self.types.push(name.unwrap_or_default()),
+            super::Tok::Word(b":") => self.deeper = self.deeper.saturating_add(1),
+            super::Tok::Word(b";") if self.deeper > 0 => self.deeper -= 1,
+            super::Tok::Word(b";") => {
+                self.types.pop();
+            }
+            super::Tok::Word(b",") => return string_key || name.is_some_and(|key| self.keys_data(key)),
+            _ => {}
+        }
+        false
+    }
+
+    /// Whether ASCII85 data follows an entry keyed `key` of the innermost dictionary.
+    fn keys_data(&self, key: &[u8]) -> bool {
+        let Some(ty) = self.types.last().filter(|_| self.deeper == 0) else { return false };
+        let (Ok(ty), Ok(key)) = (std::str::from_utf8(ty), std::str::from_utf8(key)) else { return false };
+        ascii85_follows(ty, Some(key))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/eps/src/import/ai/tests.rs

@@ -4503,6 +4503,98 @@ fn non_native_art_that_is_not_a_pdf_is_left_out_with_a_note() {
     assert!(s.warnings.iter().any(|w| w.starts_with(read::NON_NATIVE_ART_UNREAD)), "{:?}", s.warnings);
 }
 
+/// The same data in a symbol's definition: its `%_` line with a `(` opens no string, so the
+/// definition ends at its own end marker and the symbol after it comes in.
+#[test]
+fn non_native_art_in_a_symbol_ends_at_the_definitions_end() {
+    let foreign = "/ForeignObject :\n1 /Version ,\n2 0 0 2 10 20  /RTransform ,\n10 20  /Origin ,\n1 2 5 8  /Bounds ,\n/Data ,\n%,u@!!/MSk8%41#ocdN=10d&\n%_O%*mGn(RduU.]4`[u[.f\n%_;(]<[%\n%1H@<P2`V<S,pbuU7L]\\~>\n;\n";
+    let defs = symbols(&(symbol("Pdf", &format!("{foreign}{}", dot()), 5) + &symbol("Dot", &dot(), 5)));
+    let s = read_ok(&(defs + &layer(&instance("Dot", [1.0, 0.0, 0.0, -1.0, 20.0, 50.0]))));
+    let names: Vec<&str> = s.doc.symbols.iter().map(|s| s.name.as_str()).collect();
+    assert!(names.contains(&"Dot"), "{names:?}");
+    let a = art(&s.doc);
+    assert_eq!(a.len(), 1);
+    assert!(matches!(&a[0].kind, NodeKind::SymbolInstance { symbol, .. } if symbol == "Dot"), "{:?}", a[0].kind);
+}
+
+/// Comment lines between a dictionary's name and its `:`, or an entry's key and its `,`, don't hide
+/// the data from the scan of a symbol's definition.
+#[test]
+fn non_native_art_with_comments_in_a_symbol_ends_at_the_definitions_end() {
+    let foreign = "/ForeignObject\n%note\n:\n1 /Version ,\n2 0 0 2 10 20  /RTransform ,\n10 20  /Origin ,\n1 2 5 8  /Bounds ,\n/Data\n%note\n,\n%,u@!!/MSk8%41#ocdN=10d&\n%_O%*mGn(RduU.]4`[u[.f\n%_;(]<[%\n%1H@<P2`V<S,pbuU7L]\\~>\n;\n";
+    let defs = symbols(&(symbol("Pdf", &format!("{foreign}{}", dot()), 5) + &symbol("Dot", &dot(), 5)));
+    let s = read_ok(&(defs + &layer(&instance("Dot", [1.0, 0.0, 0.0, -1.0, 20.0, 50.0]))));
+    let names: Vec<&str> = s.doc.symbols.iter().map(|s| s.name.as_str()).collect();
+    assert!(names.contains(&"Dot"), "{names:?}");
+}
+
+/// A string keys an entry as a name does: `(Data) ,` doesn't hide the data from the scan of a
+/// symbol's definition either.
+#[test]
+fn non_native_art_keyed_by_a_string_in_a_symbol_ends_at_the_definitions_end() {
+    for key in ["(Data)", "<44617461>"] {
+        let foreign = format!(
+            "/ForeignObject :\n1 /Version ,\n2 0 0 2 10 20  /RTransform ,\n10 20  /Origin ,\n1 2 5 8  /Bounds ,\n{key} ,\n%,u@!!/MSk8%41#ocdN=10d&\n%_O%*mGn(RduU.]4`[u[.f\n%_;(]<[%\n%1H@<P2`V<S,pbuU7L]\\~>\n;\n"
+        );
+        let defs = symbols(&(symbol("Pdf", &format!("{foreign}{}", dot()), 5) + &symbol("Dot", &dot(), 5)));
+        let s = read_ok(&(defs + &layer(&instance("Dot", [1.0, 0.0, 0.0, -1.0, 20.0, 50.0]))));
+        let names: Vec<&str> = s.doc.symbols.iter().map(|s| s.name.as_str()).collect();
+        assert!(names.contains(&"Dot"), "{key}: {names:?}");
+    }
+}
+
+/// A `/Binary` dictionary's data in a skipped alternate: its `%_` line with a `(` opens no string,
+/// so the block ends at its own end marker.
+#[test]
+fn binary_data_in_a_skipped_alternate_ends_at_the_blocks_end() {
+    let binary = "%_/Document :\n%_/Binary : /ASCII85Decode ,\n%_(]<[%\n%;;;~>\n%_; (Profile) ,\n%_;\n";
+    let body = format!(
+        "%AI17_Begin_Content_if_version_gt:24 4\n%AI17_Alternate_Content\n{binary}frobnicate\n%AI17_End_Versioned_Content\n{}",
+        layer(&(rect_path(0.0, 0.0, 1.0, 1.0) + "f\n"))
+    );
+    let s = read_ok(&body);
+    assert_eq!(art(&s.doc).len(), 1);
+    assert!(!s.warnings.iter().any(|w| w == read::VERSIONED_UNENDED), "{:?}", s.warnings);
+}
+
+/// A text document's data, the free type's undo copy included: its `%_` line with a `(` opens no
+/// string, even after a line that reads as another section's end marker, so the section ends at
+/// its own end marker and the art after it comes in.
+#[test]
+fn text_document_data_ends_at_the_sections_end() {
+    for ty in ["AI11TextDocument", "AI11UndoFreeTextDocument"] {
+        let document =
+            format!("%AI11_BeginTextDocument\n/{ty} : /ASCII85Decode ,\n%,u@!!/MSk8\n%AI3_EndPattern\n%_(]<[%\n%;;;~>\n%_;\n%AI11_EndTextDocument\n");
+        let s = read_ok(&format!("{document}{}", layer(&(rect_path(0.0, 0.0, 1.0, 1.0) + "f\n"))));
+        assert_eq!(art(&s.doc).len(), 1, "{ty}");
+    }
+}
+
+/// Data without its `~>` isn't skipped: the section still ends at its own end marker, with no `~>`
+/// after it, with one in a later section, and with later data that holds a `(` on a `%_` line.
+#[test]
+fn data_without_its_end_does_not_hide_the_sections_end() {
+    let unended = "%AI11_BeginTextDocument\n/AI11TextDocument : /ASCII85Decode ,\n%,u@!!/MSk8\n%AI11_EndTextDocument\n";
+    let ended = "%AI11_BeginTextDocument\n/AI11TextDocument : /ASCII85Decode ,\n%,u@!!/MSk8\n%_(]<[%\n%;;;~>\n%_;\n%AI11_EndTextDocument\n";
+    let art_layer = layer(&(rect_path(0.0, 0.0, 1.0, 1.0) + "f\n"));
+    for body in [format!("{unended}{art_layer}"), format!("{unended}{art_layer}{ended}"), format!("{unended}{ended}{art_layer}")] {
+        let s = read_ok(&body);
+        assert_eq!(art(&s.doc).len(), 1, "{body}");
+    }
+}
+
+/// The names the swatches take are found past the same data: a named color after it still takes
+/// its name first, and the pattern named after it is numbered.
+#[test]
+fn swatch_names_are_found_past_non_native_art_data() {
+    let foreign = "/ForeignObject :\n1 /Version ,\n2 0 0 2 10 20  /RTransform ,\n10 20  /Origin ,\n1 2 5 8  /Bounds ,\n/Data ,\n%,u@!!/MSk8%41#ocdN=10d&\n%_O%*mGn(RduU.]4`[u[.f\n%_;(]<[%\n%1H@<P2`V<S,pbuU7L]\\~>\n;\n0 0 Xd\n6 () XW\n0 Ae\n";
+    let sea = format!("0 1 1 0 (Sea) 0 x\n{}f\n", rect_path(0.0, 0.0, 5.0, 5.0));
+    let body = format!("{}{}", DOTS.replace("(Dots)", "(Sea\\n1)"), layer(&format!("{foreign}{sea}")));
+    let d = read(stream(&body).as_bytes()).unwrap().doc;
+    let names: Vec<&str> = d.patterns.iter().map(|p| p.name.as_str()).collect();
+    assert_eq!(names, ["Sea 2"]);
+}
+
 /// A PDF with one page of `size` points holding `content`.
 fn tiny_pdf(size: f64, content: &str) -> Vec<u8> {
     let objects = [