Selection: a nested symbol instance's box past the outer box is not a hit

A click or marquee outside a symbol instance's box walks the instance's
art. An instance nested in that art was hit or touched anywhere in its
own box, so empty space inside the nested box, where a moved clipping
path reaches past the outer box, selected the outer instance. In that
walk a nested instance now counts only where its own art is, by the
same rule at every level. Inside the outer box nothing changes.

crates/doc/src/hit.rs

@@ -1066,9 +1066,10 @@ fn hit_leaf(n: &Node, p: Point, opt: HitOptions, symbols: &SymbolBoxes) -> Optio
             n.geometric_bounds_with(symbols).filter(|b| b.inflate(opt.tol, opt.tol).contains(p)).map(|_| HitKind::Bounds)
         }
         // A symbol instance is hit within its bounds, and outside them where its art paints
-        // ([`instance_art_hit`]).
+        // ([`instance_art_hit`]). In the art of another instance walked outside that instance's
+        // bounds, it is hit only where its own art paints ([`ArtOnly`]).
         NodeKind::SymbolInstance { .. } => {
-            if n.geometric_bounds_with(symbols).is_some_and(|b| b.inflate(opt.tol, opt.tol).contains(p)) {
+            if !ArtOnly::on() && n.geometric_bounds_with(symbols).is_some_and(|b| b.inflate(opt.tol, opt.tol).contains(p)) {
                 return Some(HitKind::Bounds);
             }
             instance_art_hit(n, p, opt, symbols)
@@ -1331,9 +1332,40 @@ fn instance_art_hit(n: &Node, p: Point, opt: HitOptions, symbols: &SymbolBoxes)
     #[cfg(test)]
     ARTS_WALKED.with(|c| c.set(c.get() + 1));
     let opt = HitOptions { clip_outlines: false, ..opt };
+    let _art_only = ArtOnly::set();
     FindPart { art: &art, p, opt, symbols, whole_instance: true }.find(&art.art, &mut vec![], None, false)
 }
 
+thread_local! {
+    /// Whether a symbol instance's art is being walked outside the instance's box ([`ArtOnly`]).
+    static ART_ONLY: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+}
+
+/// While a guard lives, a symbol instance is hit ([`hit_leaf`]) and touched by a marquee
+/// ([`marquee_with`]) only where its own art is, by the same rule at every level, and not anywhere
+/// in its box. [`instance_art_hit`] and the marquee's walk of an instance's art hold one: they walk
+/// the art outside the instance's box, where the box of an instance nested in the art can reach
+/// past it over empty space.
+struct ArtOnly {
+    was: bool,
+}
+
+impl ArtOnly {
+    fn set() -> Self {
+        Self { was: ART_ONLY.replace(true) }
+    }
+
+    fn on() -> bool {
+        ART_ONLY.get()
+    }
+}
+
+impl Drop for ArtOnly {
+    fn drop(&mut self) {
+        ART_ONLY.set(self.was);
+    }
+}
+
 thread_local! {
     /// How many more objects of symbol art the click or marquee under way may visit ([`ArtWalk`]),
     /// `None` outside one.
@@ -1703,8 +1735,9 @@ fn marquee_with(doc: &Document, r: Rect, scope: Option<NodeId>, leaves: bool, wh
             NodeKind::Compound { children, .. } => children.iter().any(|c| c.visible && touches(c, s, r, scope, whole_compounds, outline)),
             NodeKind::Repeat(spec) => spec.expand(Some(s.document())).iter().any(|copy| touches(copy, s, r, scope, whole_compounds, outline)),
             // A symbol instance is touched by its box, and outside it where the live geometry
-            // effects of a clipping path in its art show the art.
-            NodeKind::SymbolInstance { .. } => box_touches(n, s, r) || !outline && instance_art_touches(n, s, r, whole_compounds),
+            // effects of a clipping path in its art show the art. In the art of another instance
+            // walked outside that instance's box, it is touched only by its own art ([`ArtOnly`]).
+            NodeKind::SymbolInstance { .. } => !ArtOnly::on() && box_touches(n, s, r) || !outline && instance_art_touches(n, s, r, whole_compounds),
             _ => box_touches(n, s, r),
         }
     }
@@ -1718,13 +1751,13 @@ fn marquee_with(doc: &Document, r: Rect, scope: Option<NodeId>, leaves: bool, wh
     }
     /// Does the marquee `r` touch the art of symbol instance `n` placed as the canvas draws it, as
     /// it touches a group of that art? Only for art whose clipping paths have live geometry effects
-    /// ([`SymbolBoxes::art`]), and only when `r` meets the instance's reach
-    /// ([`Node::reach_bounds_with`]): elsewhere the art lies within the instance's box. Not once the
-    /// marquee has placed as many objects of symbol art as [`ArtWalk`] allows.
+    /// ([`SymbolBoxes::art`]), or for any art while [`ArtOnly`] is on, and only when `r` meets the
+    /// instance's reach ([`Node::reach_bounds_with`]): elsewhere the art lies within the instance's
+    /// box. Not once the marquee has placed as many objects of symbol art as [`ArtWalk`] allows.
     fn instance_art_touches(n: &Node, s: &SymbolBoxes, r: Rect, whole_compounds: bool) -> bool {
         let NodeKind::SymbolInstance { symbol, .. } = &n.kind else { return false };
         let meets = |b: Rect| b.x0 <= r.x1 && r.x0 <= b.x1 && b.y0 <= r.y1 && r.y0 <= b.y1;
-        if s.art(symbol).is_none_or(|a| a.reshaped.is_none()) || !n.reach_bounds_with(s).is_some_and(meets) {
+        if s.art(symbol).is_none_or(|a| a.reshaped.is_none() && !ArtOnly::on()) || !n.reach_bounds_with(s).is_some_and(meets) {
             return false;
         }
         let _walk = ArtWalk::open();
@@ -1737,6 +1770,7 @@ fn marquee_with(doc: &Document, r: Rect, scope: Option<NodeId>, leaves: bool, wh
         }
         #[cfg(test)]
         ARTS_WALKED.with(|c| c.set(c.get() + 1));
+        let _art_only = ArtOnly::set();
         art.place(&art.art, None).is_some_and(|placed| placed.visible && touches(&placed, s, r, None, whole_compounds, false))
     }
     fn collect_leaves(n: &Node, s: &SymbolBoxes, r: Rect, scope: Option<NodeId>, whole_compounds: bool, outline: bool, out: &mut Vec<NodeId>) {

crates/doc/src/hit/tests.rs

@@ -1961,6 +1961,48 @@ fn a_symbol_instance_is_hit_on_the_stroke_of_its_art_past_its_box() {
     assert_eq!(at(108.0, HitOptions { outline: true, ..HitOptions::default() }), None, "Outline view");
 }
 
+/// Outside a symbol instance's box, an instance nested in its art counts where its own art is, not
+/// anywhere in its box: a click or a marquee on empty space in the nested box, where a moved
+/// clipping path shows the art past the outer box, selects nothing, and on art drawn there it
+/// selects the outer instance.
+#[test]
+fn a_nested_symbol_instances_box_past_the_outer_box_is_not_a_hit() {
+    set_clip_region(moved_clip_region);
+    // `Two` draws squares at the corners of 0..200 × 0..100, and with `middle` one at 140..160 ×
+    // 40..60. The outer symbol clips an instance of it to 0..100 × 0..100, which `test.move` moves
+    // to 100..200: the outer box is 0..100, the nested box 0..200.
+    let build = |middle: bool| {
+        let mut d = Document::new(600.0, 400.0);
+        let l = d.layers[0].id;
+        let square = |d: &mut Document, r: Rect| Arc::new(Node::path(d.alloc_id(), shapes::rectangle(r), Appearance::default_art()));
+        let mut squares = vec![square(&mut d, Rect::new(0.0, 0.0, 10.0, 10.0)), square(&mut d, Rect::new(190.0, 90.0, 200.0, 100.0))];
+        if middle {
+            squares.push(square(&mut d, Rect::new(140.0, 40.0, 160.0, 60.0)));
+        }
+        group_symbol(&mut d, "Two", squares);
+        let (mut mask, _) = half_clipped(&mut d);
+        mask.appearance.effects.push(test_move());
+        let nested = instance_of(&mut d, "Two");
+        let mut clip = Node::group(d.alloc_id(), vec![Arc::new(mask), nested]);
+        clip.set_clips(true);
+        group_symbol(&mut d, "Outer", vec![Arc::new(clip)]);
+        let inst = d.alloc_id();
+        d.insert(Some(l), 0, Node::new(inst, NodeKind::SymbolInstance { symbol: "Outer".into(), xf: vectorcraft_geom::Affine::IDENTITY })).unwrap();
+        (d, inst)
+    };
+    let at = |d: &Document, x: f64, y: f64| hit_test(d, Point::new(x, y), HitOptions::default()).map(|h| h.leaf);
+    let inside = Rect::new(145.0, 45.0, 155.0, 55.0);
+    let (d, inst) = build(false);
+    assert_eq!(at(&d, 150.0, 50.0), None, "empty space in the nested box");
+    assert!(marquee(&d, inside, None, false).is_empty(), "marquee on empty space in the nested box");
+    assert_eq!(at(&d, 195.0, 95.0), Some(inst), "on the nested art past the outer box");
+    assert_eq!(at(&d, 50.0, 50.0), Some(inst), "in the outer box");
+    let (d, inst) = build(true);
+    assert_eq!(at(&d, 150.0, 50.0), Some(inst), "on the nested art");
+    // Across the middle square's left edge.
+    assert_eq!(marquee(&d, Rect::new(135.0, 45.0, 145.0, 55.0), None, false), [inst], "marquee on the nested art");
+}
+
 /// Adds the symbol `Clipped`: a clip group whose clipping path's region `test.move` moves to
 /// 100..200 × 0..100, past the 0..100 box, over a 0..200 × 0..100 rectangle shown when `shown`.
 /// A click at (150, 50) hits an instance of it on the rectangle, or, with the rectangle hidden,