Merge pull request #5293 from storytold/fix/sec-brush-mask-root-symbol

Brushes: leave out an opacity mask whose root symbol draws a brushed stroke

crates/brush/src/lib.rs

@@ -884,7 +884,16 @@ fn leave_out_brushed(doc: &Document, nodes: &mut Vec<Node>, known: &mut Known) {
                 walk(doc, Arc::make_mut(c), known);
             }
         }
-        if n.extras().mask.is_some()
+        // A mask's art is drawn too. Its root is filtered like any other node: a symbol instance
+        // there that draws a brushed stroke would run the brush again from inside the mask, so the
+        // mask goes, as such an instance goes from the pieces and their children.
+        let rejected_root = matches!(
+            n.extras().mask.as_ref().map(|m| &m.art.kind),
+            Some(NodeKind::SymbolInstance { symbol, .. }) if draws_brushed(doc, (false, symbol.as_str()), known)
+        );
+        if rejected_root {
+            n.extras_mut().mask = None;
+        } else if n.extras().mask.is_some()
             && let Some(mask) = n.extras_mut().mask.as_mut()
         {
             walk(doc, Arc::make_mut(&mut mask.art), known);

crates/brush/src/tests.rs

@@ -1316,6 +1316,42 @@ fn pieces_leave_out_what_draws_a_brushed_stroke_by_name() {
     }
 }
 
+/// A symbol instance at the root of an opacity mask is filtered like one among the children: when
+/// its symbol draws a brushed stroke the mask goes (it would run the brush again from inside the
+/// mask, without end), and a mask whose root symbol is plain stays.
+#[test]
+fn pieces_leave_out_a_mask_whose_root_draws_a_brushed_stroke() {
+    use vectorcraft_doc::{OpacityMask, Symbol};
+    let mut doc = Document::new(100.0, 100.0);
+    let mut brushed = rect_art(10.0, 4.0);
+    brushed.appearance.items.push(vectorcraft_doc::AppearanceItem::Stroke(StrokeLayer { brush: Some("s".into()), ..stroke(1.0) }));
+    for (name, art) in [("plain", rect_art(10.0, 4.0)), ("brushed", brushed)] {
+        doc.symbols.push(Symbol { dynamic: false, name: name.into(), art: std::sync::Arc::new(art) });
+    }
+    let instance = |symbol: &str| Node::new(NodeId(0), NodeKind::SymbolInstance { symbol: symbol.into(), xf: vectorcraft_geom::Affine::IDENTITY });
+    let masked = |symbol: &str| {
+        let mut n = rect_art(10.0, 4.0);
+        n.extras_mut().mask = Some(Box::new(OpacityMask::new(instance(symbol), true)));
+        n
+    };
+    let art = Node::group(NodeId(0), [masked("plain"), masked("brushed")].map(std::sync::Arc::new).into());
+    let b = Brush { name: "s".into(), kind: BrushKind::Scatter(Scatter { art, colorization: Colorization::None, ..Default::default() }) };
+    let out = stroke_pieces(&doc, &b, &line(0.0, 0.0, 100.0, 0.0), &stroke(1.0));
+    assert!(!out.is_empty());
+    for piece in &out {
+        let roots: Vec<Option<&str>> = piece
+            .children()
+            .unwrap()
+            .iter()
+            .map(|c| match c.extras().mask.as_ref().map(|m| &m.art.kind) {
+                Some(NodeKind::SymbolInstance { symbol, .. }) => Some(symbol.as_str()),
+                _ => None,
+            })
+            .collect();
+        assert_eq!(roots, [Some("plain"), None], "the plain mask stays and the brushed one goes");
+    }
+}
+
 /// Anchors a brushed stroke emitted, which is what the output budget counts.
 fn anchors(nodes: &[Node]) -> usize {
     nodes