Selection: symbol parts count the Preview of the instance's layers

Direct Selection on a dynamic symbol's instance below a layer whose
Preview is off tested and outlined the art in Preview, where the canvas
draws it in outline with the plain clipping paths. When a clipping
path's effects left no region, a selected object in its clip group kept
an outline cut to the plain clipping path.

- `hit::symbol_part_at` and `hit::symbol_part_outline_in_view` count the
  layers around the instance, the isolated container and the layers
  around it apart, as clicks outside symbols do.
  `symbol_part_outline_in_view` takes the click options in place of the
  outline flag.
- The outline of an object is empty when the effects of a clipping path
  around it leave no region.

crates/doc/src/hit.rs

@@ -1202,8 +1202,8 @@ fn moved(n: &Node, xf: vectorcraft_geom::Affine) -> Node {
 /// group's original shapes, which the canvas does not draw, and guide paths with
 /// [`HitOptions::skip_guides`]. Nothing is picked in art that placing through the symbol's 9-slice
 /// guides could make more than [`crate::MAX_SYMBOL_NODES`] objects of, which the canvas does not
-/// draw. Art in a layer of the symbol whose Preview is off is tested in outline
-/// ([`HitOptions::outline`]), as the canvas draws it. `None` when `inst` is not a symbol instance or
+/// draw. Art in a layer of the symbol whose Preview is off, and the art of an instance below such a
+/// layer, is tested in outline ([`HitOptions::outline`]), as the canvas draws it. `None` when `inst` is not a symbol instance or
 /// no editable object of its art is under `p`.
 pub fn symbol_part_at(doc: &Document, inst: &Node, p: Point, opt: HitOptions) -> Option<Vec<u32>> {
     struct Find<'a> {
@@ -1263,7 +1263,17 @@ pub fn symbol_part_at(doc: &Document, inst: &Node, p: Point, opt: HitOptions) ->
     let art = InstanceArt::of(doc, inst)?;
     let symbols = doc.symbol_boxes();
     let mut path = vec![];
-    Find { doc, art: &art, p, opt, symbols: &symbols }.find(&art.art, &mut path, None, opt.outline).then_some(path)
+    let outline = opt.outline || below_preview_off(doc, inst, opt.scope);
+    Find { doc, art: &art, p, opt, symbols: &symbols }.find(&art.art, &mut path, None, outline).then_some(path)
+}
+
+/// Is `n` below a layer whose Preview is off, so that the canvas draws it in outline? As for clicks
+/// ([`path_options`]), the Preview of the isolated container `scope` and of the layers around it
+/// does not count.
+fn below_preview_off(doc: &Document, n: &Node, scope: Option<NodeId>) -> bool {
+    let Some(chain) = doc.ancestry(n.id) else { return false };
+    let below = scope.and_then(|s| chain.iter().position(|a| *a == s)).map_or(0, |i| i + 1);
+    chain.iter().skip(below).any(|a| doc.node(*a).is_some_and(previews_off))
 }
 
 /// The outlines of the object at `path` in symbol instance `inst`'s art (see [`symbol_part_at`]),
@@ -1276,15 +1286,19 @@ pub fn symbol_part_at(doc: &Document, inst: &Node, p: Point, opt: HitOptions) ->
 /// has any.
 /// `None` for art past the 9-slice object limit that [`symbol_part_at`] applies.
 pub fn symbol_part_outline(doc: &Document, inst: &Node, path: &[u32]) -> Option<BezPath> {
-    symbol_part_outline_in_view(doc, inst, path, false)
+    symbol_part_outline_in_view(doc, inst, path, HitOptions::default())
 }
 
-/// [`symbol_part_outline`] in the view the canvas shows: with `outline` (Outline view), and below a
-/// layer of the art whose Preview is off, the outlines are cut to the bounds of the plain clipping
-/// paths, without their live effects, as the clicks of [`HitOptions::outline`] count them.
-pub fn symbol_part_outline_in_view(doc: &Document, inst: &Node, path: &[u32], mut outline: bool) -> Option<BezPath> {
+/// [`symbol_part_outline`] in the view the canvas shows, for the clicks of `opt`: in Outline view
+/// ([`HitOptions::outline`]) and below a layer whose Preview is off, in the art or around the
+/// instance (the isolated container [`HitOptions::scope`] and the layers around it apart), the
+/// outlines are cut to the bounds of the plain clipping paths, without their live effects, as
+/// [`symbol_part_at`] counts them. The outline is empty when the effects of a clipping path around
+/// the object leave no region.
+pub fn symbol_part_outline_in_view(doc: &Document, inst: &Node, path: &[u32], opt: HitOptions) -> Option<BezPath> {
     let art = InstanceArt::of(doc, inst)?;
     let part = crate::symbol_overrides::part(&art.art, path)?;
+    let mut outline = opt.outline || below_preview_off(doc, inst, opt.scope);
     let mut out = BezPath::new();
     let mut limit = None;
     let mut whole = None;
@@ -1296,12 +1310,9 @@ pub fn symbol_part_outline_in_view(doc: &Document, inst: &Node, path: &[u32], mu
         if i != 0
             && n.clips()
             && let Some(first) = children.first()
-            && let Some(clip) = clip_limit(&art.place_unclipped(first, doc, whole)?, outline)
         {
-            match meet(Some(clip), limit) {
-                Some(r) => limit = Some(r),
-                None => return Some(out),
-            }
+            let Some(within) = limit_within(&art.place_unclipped(first, doc, whole)?, limit, outline) else { return Some(out) };
+            limit = within;
         }
         n = children.get(usize::try_from(i).ok()?)?;
     }
@@ -1309,15 +1320,21 @@ pub fn symbol_part_outline_in_view(doc: &Document, inst: &Node, path: &[u32], mu
     Some(out)
 }
 
-/// The box that the art clipped by `clip` is drawn in: the bounds of the region the live geometry
-/// effects of `clip` give it ([`clip_region`]), or the bounds of `clip` when it has no such effects
-/// and with `outline`, where the canvas draws no effects.
-fn clip_limit(clip: &Node, outline: bool) -> Option<Rect> {
+/// `limit` cut to the box that the art clipped by `clip` is drawn in: the bounds of the region the
+/// live geometry effects of `clip` give it ([`clip_region`]), or the bounds of `clip` when it has no
+/// such effects and with `outline`, where the canvas draws no effects. `None` when none of that art
+/// is drawn: the effects leave no region, or the box is outside `limit`.
+fn limit_within(clip: &Node, limit: Option<Rect>, outline: bool) -> Option<Option<Rect>> {
     use vectorcraft_geom::kurbo::Shape;
     let region = if outline { None } else { clip_region(clip) };
-    match region.as_deref() {
+    let bounds = match region.as_deref() {
         Some(Some((bp, _))) if !bp.elements().is_empty() => Some(bp.bounding_box()),
-        _ => clip.geometric_bounds(),
+        Some(_) => return None,
+        None => clip.geometric_bounds(),
+    };
+    match bounds {
+        Some(b) => meet(Some(b), limit).map(Some),
+        None => Some(limit),
     }
 }
 
@@ -1340,13 +1357,7 @@ fn outline_within(n: &Node, limit: Option<Rect>, outline: bool, out: &mut BezPat
     if n.clips()
         && let Some((first, rest)) = children.split_first()
     {
-        let inner = match clip_limit(first, outline) {
-            Some(c) => match meet(Some(c), limit) {
-                Some(r) => Some(r),
-                None => return,
-            },
-            None => limit,
-        };
+        let Some(inner) = limit_within(first, limit, outline) else { return };
         rest.iter().for_each(|c| outline_within(c, inner, outline, out));
         return;
     }

crates/doc/src/hit/tests.rs

@@ -111,10 +111,17 @@ fn half_clipped(d: &mut Document) -> (Node, Node) {
 }
 
 /// A stand-in for the engine's [`ClipRegion`]: with an effect named `test.move` in the clipping
-/// path, its first shape moved 100 pt right; without one, no geometry effect.
+/// path, its first shape moved 100 pt right; with one named `test.empty`, no region left to clip
+/// by; without either, no geometry effect.
 fn moved_clip_region(clip: &Node) -> Option<Option<(BezPath, FillRule)>> {
-    let mut moves = false;
-    clip.walk(&mut |n| moves |= n.appearance.effects.iter().any(|e| e.visible && e.id == "test.move"));
+    let (mut moves, mut empty) = (false, false);
+    clip.walk(&mut |n| {
+        moves |= n.appearance.effects.iter().any(|e| e.visible && e.id == "test.move");
+        empty |= n.appearance.effects.iter().any(|e| e.visible && e.id == "test.empty");
+    });
+    if empty {
+        return Some(None);
+    }
     if !moves {
         return None;
     }
@@ -1790,18 +1797,24 @@ fn an_object_a_clipping_layer_in_symbol_art_hides_is_not_picked_there() {
 
 /// In a dynamic symbol's art, an object in a clipping layer whose clipping path has a live effect
 /// is picked and outlined in the region the effect gives the clipping path. In Outline view, and
-/// when the layer's Preview is off, the plain clipping path counts, as the canvas draws it.
+/// when Preview is off for the layer in the art or for the layer the instance is on, the plain
+/// clipping path counts, as the canvas draws it. An effect that leaves no region leaves no outline.
 #[test]
 fn a_symbol_part_follows_the_region_its_clipping_layers_effects_give_it() {
     set_clip_region(moved_clip_region);
-    let instance = |preview: bool| {
+    // The instance of a symbol whose art is a clipping layer; `preview` is the Preview of that
+    // layer and of the document layer the instance is on.
+    let instance = |effect: &str, preview: [bool; 2]| {
         let mut d = Document::new(300.0, 200.0);
+        if let NodeKind::Layer { preview: p, .. } = &mut Arc::make_mut(&mut d.layers[0]).kind {
+            *p = preview[1];
+        }
         let l = d.layers[0].id;
         let (mut mask, art) = half_clipped(&mut d);
-        mask.appearance.effects.push(test_move());
+        mask.appearance.effects.push(crate::Effect { id: effect.into(), ..test_move() });
         let mut layer = Node::layer(d.alloc_id(), "Masked", crate::LayerColor::Preset(2));
         if let NodeKind::Layer { children, preview: p, .. } = &mut layer.kind {
-            (*children, *p) = (vec![Arc::new(mask), Arc::new(art)], preview);
+            (*children, *p) = (vec![Arc::new(mask), Arc::new(art)], preview[0]);
         }
         layer.set_clips(true);
         let root = Node::group(d.alloc_id(), vec![Arc::new(layer)]);
@@ -1810,26 +1823,33 @@ fn a_symbol_part_follows_the_region_its_clipping_layers_effects_give_it() {
         d.insert(Some(l), 0, Node::new(inst, NodeKind::SymbolInstance { symbol: "Clipped".into(), xf: vectorcraft_geom::Affine::IDENTITY })).unwrap();
         (d, inst)
     };
-    let outline_box = |d: &Document, inst: NodeId, outline: bool| {
-        let o = symbol_part_outline_in_view(d, d.node(inst).unwrap(), &[0, 1], outline).unwrap();
+    let outline_box = |d: &Document, inst: NodeId, opt: HitOptions| {
+        let o = symbol_part_outline_in_view(d, d.node(inst).unwrap(), &[0, 1], opt).unwrap();
         vectorcraft_geom::kurbo::Shape::bounding_box(&o)
     };
     let opt = HitOptions::default();
-    let (d, inst) = instance(true);
+    let (d, inst) = instance("test.move", [true, true]);
     let n = d.node(inst).unwrap();
     assert_eq!(symbol_part_at(&d, n, Point::new(150.0, 50.0), opt), Some(vec![0, 1]), "in the moved region");
     assert_eq!(symbol_part_at(&d, n, Point::new(50.0, 50.0), opt), None, "where the clipping path was");
-    let b = outline_box(&d, inst, false);
+    let b = outline_box(&d, inst, opt);
     assert!((b.x0 - 100.0).abs() < 1e-6 && (b.x1 - 200.0).abs() < 1e-6, "the outline is cut to the moved region: {b:?}");
-    let b = outline_box(&d, inst, true);
+    let b = outline_box(&d, inst, HitOptions { outline: true, ..opt });
     assert!(b.x0.abs() < 1e-6 && (b.x1 - 100.0).abs() < 1e-6, "in Outline view, to the plain clipping path: {b:?}");
-    // With the layer's Preview off, the art's top edge counts within the plain clipping path.
-    let (d, inst) = instance(false);
+    // With Preview off, the art's top edge counts within the plain clipping path.
+    for (preview, what) in [([false, true], "the layer in the art"), ([true, false], "the layer the instance is on")] {
+        let (d, inst) = instance("test.move", preview);
+        let n = d.node(inst).unwrap();
+        assert_eq!(symbol_part_at(&d, n, Point::new(50.0, 0.5), opt), Some(vec![0, 1]), "on the edge the canvas draws, Preview off for {what}");
+        assert_eq!(symbol_part_at(&d, n, Point::new(150.0, 0.5), opt), None, "past the plain clipping path, Preview off for {what}");
+        let b = outline_box(&d, inst, opt);
+        assert!(b.x0.abs() < 1e-6 && (b.x1 - 100.0).abs() < 1e-6, "Preview off for {what}: the plain clipping path: {b:?}");
+    }
+    // An effect that leaves the clipping path no region hides the art: no pick and no outline.
+    let (d, inst) = instance("test.empty", [true, true]);
     let n = d.node(inst).unwrap();
-    assert_eq!(symbol_part_at(&d, n, Point::new(50.0, 0.5), opt), Some(vec![0, 1]), "on the edge the canvas draws");
-    assert_eq!(symbol_part_at(&d, n, Point::new(150.0, 0.5), opt), None, "past the plain clipping path");
-    let b = outline_box(&d, inst, false);
-    assert!(b.x0.abs() < 1e-6 && (b.x1 - 100.0).abs() < 1e-6, "Preview off: the plain clipping path: {b:?}");
+    assert_eq!(symbol_part_at(&d, n, Point::new(50.0, 50.0), opt), None);
+    assert!(symbol_part_outline(&d, n, &[0, 1]).unwrap().elements().is_empty(), "nothing is drawn, so nothing is outlined");
 }
 
 /// In a dynamic symbol's art that placing through its 9-slice guides could make more than

crates/tools/src/direct.rs

@@ -1029,7 +1029,7 @@ impl Tool for DirectSelectionTool {
                 out.extend(self.mesh.overlays(cx, &|id| cx.selection.contains(id)));
                 if !self.group {
                     if let Some(sp) = &cx.selection.symbol_part
-                        && let Some(path) = cx.doc.node(sp.instance).and_then(|n| symbol_part_outline_in_view(cx.doc, n, &sp.path, cx.outline))
+                        && let Some(path) = cx.doc.node(sp.instance).and_then(|n| symbol_part_outline_in_view(cx.doc, n, &sp.path, cx.hit_options()))
                     {
                         out.push(Overlay::Path { path, color: cx.doc.layer_color(sp.instance), width: 1.0, dashed: false });
                     }