diff --git a/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs b/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs index ba029dd3b..db8696a10 100644 --- a/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs +++ b/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs @@ -115,15 +115,27 @@ fn both_lexers_take_the_same_characters_into_names() { assert!(checker_member("\u{88F}\u{1ACF}", "\u{88F}\u{1ACF}") && one_identifier("\u{88F}\u{1ACF}", "\u{88F}\u{1ACF}")); for &c in NAME_CHARS { let escape = format!("\\u{:04X}", u32::from(c)); + let braced = format!("\\u{{{:X}}}", u32::from(c)); for (src, name) in [ (format!("a{c}b"), format!("a{c}b")), (format!("{c}b"), format!("{c}b")), (format!("a{escape}b"), format!("a{c}b")), (format!("{escape}b"), format!("{c}b")), + (format!("a{braced}b"), format!("a{c}b")), + (format!("{braced}b"), format!("{c}b")), ] { assert_eq!(checker_member(&src, &name), one_identifier(&src, &name), "{src:?} (U+{:04X})", u32::from(c)); } } + // Braced escapes past U+FFFF and with leading zeros are read by both. + for (src, name) in [("\\u{1D49C}b", "\u{1D49C}b"), ("a\\u{0000000062}", "ab")] { + assert!(checker_member(src, name) && one_identifier(src, name), "{src:?}"); + } + // Out of range, a surrogate and an empty one: the parser reads no identifier, and the checker's + // name ends before the escape. + for src in ["a\\u{110000}", "a\\u{D800}", "a\\u{}"] { + assert!(checker_member(src, "a") && !one_identifier(src, "a"), "{src:?}"); + } } #[test] diff --git a/crates/scriptcheck/src/analyze.rs b/crates/scriptcheck/src/analyze.rs index 9a637274d..b8f887ff3 100644 --- a/crates/scriptcheck/src/analyze.rs +++ b/crates/scriptcheck/src/analyze.rs @@ -88,12 +88,25 @@ pub(crate) struct Scan { /// Scan `src` (already through [`unescaped`]): the script itself (`file` none) or a file it includes /// (`file`: its path inside the script's folder), with the names it declares (`declared`, from /// [`declarations`] on the same `src`), the `#includepath` folders `paths` in force at its start and -/// its variables holding what `known` says. -pub(crate) fn scan<'s>(src: &'s str, declared: &Declared<'s>, file: Option<&str>, paths: Rc<[String]>, known: &Known, acc: &mut Acc) -> Scan { +/// its variables holding what `known` says. Each of the `\u{…}` escapes `braced` that [`unescaped`] +/// read is syntax later than ECMAScript 3, reported when the scan reaches the escape's line. +pub(crate) fn scan<'s>( + src: &'s str, + declared: &Declared<'s>, + braced: &[Braced], + file: Option<&str>, + paths: Rc<[String]>, + known: &Known, + acc: &mut Acc, +) -> Scan { let mut lexer = Lexer::new(src, acc.limits.nesting); let mut a = Analyzer::new(acc, file, paths, known, declared); let mut breaks = LineBreaks::default(); + let mut braced = braced.iter().peekable(); for t in lexer.by_ref() { + while let Some((line, escape)) = braced.next_if(|(line, _)| *line <= t.line) { + a.acc.problem(report::NEWER_SYNTAX, Some(*line), file, Some(escape)); + } if let Some(code) = t.problem { a.problem(code, t.line); } @@ -103,6 +116,9 @@ pub(crate) fn scan<'s>(src: &'s str, declared: &Declared<'s>, file: Option<&str> Read::Token(sig) => a.token(sig), } } + for (line, escape) in braced { + a.acc.problem(report::NEWER_SYNTAX, Some(*line), file, Some(escape)); + } if let Some(line) = lexer.unclosed_xml { a.problem(report::UNTERMINATED_XML, line); } @@ -115,29 +131,61 @@ pub(crate) fn scan<'s>(src: &'s str, declared: &Declared<'s>, file: Option<&str> Scan { includes, module, lines: lexer.lines(), known } } -/// `src` with the `\uXXXX` escapes in its names written as the characters they stand for (ECMA-262 -/// 3rd edition, section 7.6), and `src` itself when no name has one. An escape for a character a -/// name may not hold where the escape stands stays as written. Line breaks don't change, so every -/// line keeps its number. -pub(crate) fn unescaped<'a>(src: Cow<'a, str>, acc: &mut Acc) -> Cow<'a, str> { - match unescape_names(&src, acc) { - Some(out) => Cow::Owned(out), - None => src, +/// A `\u{…}` escape in a name, which ECMAScript 3 does not have: its line and its text. +pub(crate) type Braced = (usize, String); + +/// `src` with the `\uXXXX` and `\u{…}` escapes in its names and private names written as the +/// characters they stand for (ECMA-262, Names and Keywords), and `src` itself when no name has one, +/// with the `\u{…}` escapes read, each listed once per line in a row. An escape for a character a +/// name may not hold where the escape stands stays as written, and so does a private name that +/// starts a line where the name it spells would start a preprocessor line. Line breaks don't +/// change, so every line keeps its number. +pub(crate) fn unescaped<'a>(src: Cow<'a, str>, acc: &mut Acc) -> (Cow<'a, str>, Vec) { + let mut braced = vec![]; + match unescape_names(&src, acc, &mut braced) { + Some(out) => (Cow::Owned(out), braced), + None => (src, braced), } } -fn unescape_names(src: &str, acc: &mut Acc) -> Option { +fn unescape_names(src: &str, acc: &mut Acc, braced: &mut Vec) -> Option { if !src.contains("\\u") { return None; } let mut out = String::with_capacity(src.len()); let mut changed = false; let mut lexer = Lexer::new(src, acc.limits.nesting); - for t in lexer.by_ref() { - if t.kind == T::Ident && t.text.contains("\\u") { - let name = unescape_name(t.text); - changed |= name != t.text; - out.push_str(&name); + // One more than the report keeps, so that the report is marked truncated. + let room = acc.limits.entries.saturating_add(1); + // Only blanks since the last line break (or the start). + let mut line_start = true; + let mut found = vec![]; + while let Some(t) = lexer.next() { + // A private name is `#` and a name. + let (mark, name) = match t.kind { + T::Ident => ("", t.text), + T::Other => t.text.strip_prefix('#').map_or(("", ""), |name| ("#", name)), + _ => ("", ""), + }; + let starts_line = line_start; + line_start = t.kind == T::Newline || (line_start && t.kind == T::Space); + if name.contains("\\u") { + found.clear(); + let written = unescape_name(name, t.line, &mut found, room); + for escape in found.drain(..) { + if braced.last() != Some(&escape) && braced.len() < room { + braced.push(escape); + } + } + // A private name that starts a line stays as written where the name it spells would + // start a preprocessor line. + if mark == "#" && starts_line && !lexer.private_line(&written) { + out.push_str(t.text); + continue; + } + changed |= written != name; + out.push_str(mark); + out.push_str(&written); } else { out.push_str(t.text); } @@ -146,25 +194,31 @@ fn unescape_names(src: &str, acc: &mut Acc) -> Option { changed.then_some(out) } -/// The name `text` with each `\uXXXX` escape for a character a name may hold where it stands written -/// as that character: a character a name may start with first, and one a name may hold after that. -fn unescape_name(text: &str) -> String { +/// The name `text`, on `line`, with each escape for a character a name may hold where it stands +/// written as that character: a character a name may start with first, and one a name may hold +/// after that. Each `\u{…}` escape read goes into `braced`, unless the last one there is the same +/// escape on the same line or `braced` holds `room` of them. +fn unescape_name(text: &str, line: usize, braced: &mut Vec, room: usize) -> String { let mut out = String::with_capacity(text.len()); let mut rest = text; while let Some(i) = rest.find("\\u") { out.push_str(rest.get(..i).unwrap_or_default()); - let after = rest.get(i + 2..).unwrap_or_default(); + let at = rest.get(i..).unwrap_or_default(); let first = out.is_empty(); - let c = crate::lex::escaped(rest.get(i..).unwrap_or_default()) - .filter(|&c| if first { crate::lex::ident_start(c) } else { crate::lex::ident_part(c) }); + let c = crate::lex::escape(at).filter(|&(c, _)| if first { crate::lex::ident_start(c) } else { crate::lex::ident_part(c) }); match c { - Some(c) => { + Some((c, len)) => { out.push(c); - rest = after.get(4..).unwrap_or_default(); + let source = at.get(..len).unwrap_or_default(); + let repeated = braced.last().is_some_and(|(l, s)| *l == line && s == source); + if source.starts_with("\\u{") && !repeated && braced.len() < room { + braced.push((line, source.to_string())); + } + rest = at.get(len..).unwrap_or_default(); } None => { out.push_str("\\u"); - rest = after; + rest = at.get(2..).unwrap_or_default(); } } } diff --git a/crates/scriptcheck/src/includes.rs b/crates/scriptcheck/src/includes.rs index f6e9c82b0..b4a7be721 100644 --- a/crates/scriptcheck/src/includes.rs +++ b/crates/scriptcheck/src/includes.rs @@ -141,9 +141,9 @@ pub(crate) fn follow( read.insert(path.clone(), (status, Some(file))); let (text, _) = crate::decode(&b); let mut tallies = Acc::new(limits); - let text = analyze::unescaped(text, &mut tallies); + let (text, _) = analyze::unescaped(text, &mut tallies); let declared = analyze::declarations(&text, &mut tallies); - let scan = analyze::scan(&text, &declared, Some(&path), Rc::clone(&inc.paths), known, &mut tallies); + let scan = analyze::scan(&text, &declared, &[], Some(&path), Rc::clone(&inc.paths), known, &mut tallies); acc.steps = acc.steps.saturating_add(tallies.steps); known.add(scan.known, limits.entries); checked.push(ReadFile { path: path.clone(), bytes: b, paths: Rc::clone(&inc.paths) }); diff --git a/crates/scriptcheck/src/lex.rs b/crates/scriptcheck/src/lex.rs index 715a0582b..a9a805d87 100644 --- a/crates/scriptcheck/src/lex.rs +++ b/crates/scriptcheck/src/lex.rs @@ -41,7 +41,9 @@ pub(crate) enum T { /// end or the next `{`. XmlMore, Punct, - /// A character no token starts with, or `#` and a name away from the start of a line. + /// A character no token starts with, or a private name: `#` and a name away from the start of a + /// line, or at its start where the name starts with an escape, or inside braces where the name + /// is not a directive's. Other, } @@ -166,14 +168,33 @@ pub(crate) fn ident_part(c: char) -> bool { } } -/// The character the `\uXXXX` escape at the start of `s` stands for: `None` when `s` doesn't start -/// with `\u` and four hexadecimal digits, or when they give a lone surrogate. -pub(crate) fn escaped(s: &str) -> Option { - s.strip_prefix("\\u")? - .get(..4) - .filter(|hex| hex.bytes().all(|b| b.is_ascii_hexdigit())) - .and_then(|hex| u32::from_str_radix(hex, 16).ok()) - .and_then(char::from_u32) +/// The character the escape at the start of `s` stands for and the escape's length in bytes: `\u` +/// and four hexadecimal digits, or `\u{…}` with one or more hexadecimal digits of a value up to +/// U+10FFFF, leading zeros allowed (ECMA-262, UnicodeEscapeSequence and CodePoint). `None` when `s` +/// starts with neither, or when the escape gives a lone surrogate. +pub(crate) fn escape(s: &str) -> Option<(char, usize)> { + let rest = s.strip_prefix("\\u")?; + let Some(braced) = rest.strip_prefix('{') else { + let c = rest + .get(..4) + .filter(|hex| hex.bytes().all(|b| b.is_ascii_hexdigit())) + .and_then(|hex| u32::from_str_radix(hex, 16).ok()) + .and_then(char::from_u32)?; + return Some((c, 6)); + }; + // The value stops growing past U+10FFFF, so any number of digits is read once and never + // overflows. + let mut value: u32 = 0; + for (i, b) in braced.bytes().enumerate() { + match b { + b'}' if i > 0 => return char::from_u32(value).map(|c| (c, i + 4)), + _ => { + let digit = char::from(b).to_digit(16)?; + value = (value * 16 + digit).min(0x11_0000); + } + } + } + None } /// `Other_ID_Start` (Unicode PropList 17.0): characters kept in `ID_Start` though their category is @@ -394,18 +415,66 @@ impl<'s> Lexer<'s> { } /// The rest of a name (its first character included): the characters [`ident_part`] allows, - /// written out or as `\uXXXX` escapes. An escape for any other character, or one without four - /// hexadecimal digits, ends the name. + /// written out or as `\uXXXX` or `\u{…}` escapes ([`escape`]). An escape for any other + /// character, or a malformed one, ends the name. Each character of an escape is a step. fn name(&mut self) { loop { match self.peek() { - Some('\\') if escaped(self.rest()).is_some_and(ident_part) => self.skip(6), + Some('\\') => match escape(self.rest()) { + Some((c, len)) if ident_part(c) => { + self.pos += len; + self.steps += len; + } + _ => break, + }, Some(c) if ident_part(c) => self.bump(c), _ => break, } } } + /// A name starts right after the `#` here: its first character written out, or an escape for a + /// character a name may start with. + fn private_name_follows(&self) -> bool { + let after = self.rest().get(1..).unwrap_or_default(); + match after.chars().next() { + Some('\\') => escape(after).is_some_and(|(c, _)| ident_start(c)), + Some(c) => ident_start(c), + None => false, + } + } + + /// The `#` at the start of this line starts a private name, not a preprocessor line: the name + /// starts with an escape, which no directive's name holds, or [`Lexer::private_line`] holds for + /// it. + fn private_at_line_start(&self) -> bool { + if !self.private_name_follows() { + return false; + } + let after = self.rest().get(1..).unwrap_or_default(); + after.starts_with('\\') || self.private_line(after) + } + + /// A line that starts with `#` and then `rest` is a private name here, not a preprocessor line: + /// the line is inside braces or in the code of an XML literal, and the name `rest` starts with + /// is not a directive's. The name runs over every character a name may hold, written out or as + /// an escape, so `#include$size` is not `#include`. + pub fn private_line(&self, rest: &str) -> bool { + let in_code = !self.braces.is_empty() || self.deeper_braces > 0 || !self.xml.is_empty(); + let mut end = 0; + while let Some(more) = rest.get(end..) { + let len = match more.chars().next() { + Some('\\') => escape(more).filter(|&(c, _)| ident_part(c)).map(|(_, len)| len), + Some(c) if ident_part(c) => Some(c.len_utf8()), + _ => None, + }; + let Some(len) = len else { break }; + end += len; + } + let name = rest.get(..end).unwrap_or_default(); + in_code && !catalog::DIRECTIVES.iter().any(|d| d.eq_ignore_ascii_case(name)) + } + fn digits(&mut self, ok: fn(char) -> bool) { while let Some(c) = self.peek().filter(|&c| ok(c) || c == '_') { self.bump(c); @@ -871,10 +940,10 @@ impl<'s> Iterator for Lexer<'s> { } else if self.starts("/*") { problem = self.block_comment(); T::Comment - } else if c == '#' && self.line_start { + } else if c == '#' && self.line_start && !self.private_at_line_start() { self.until_line_break(); T::Directive - } else if c == '#' && self.peek_at(1).is_some_and(ident_start) { + } else if c == '#' && self.private_name_follows() { self.bump('#'); self.name(); T::Other @@ -906,7 +975,7 @@ impl<'s> Iterator for Lexer<'s> { self.close_level(); self.bump('}'); return Some(self.markup_token(start, line, T::XmlMore, stmt, expr)); - } else if ident_start(c) || (c == '\\' && escaped(self.rest()).is_some_and(ident_start)) { + } else if ident_start(c) || (c == '\\' && escape(self.rest()).is_some_and(|(c, _)| ident_start(c))) { self.name(); T::Ident } else if c.is_ascii_digit() || (c == '.' && self.peek_at(1).is_some_and(|d| d.is_ascii_digit())) { diff --git a/crates/scriptcheck/src/lib.rs b/crates/scriptcheck/src/lib.rs index 0c62419f1..2bd8c60fb 100644 --- a/crates/scriptcheck/src/lib.rs +++ b/crates/scriptcheck/src/lib.rs @@ -166,14 +166,14 @@ pub fn check(name: &str, bytes: &[u8], loader: Option<&mut dyn Loader>, limits: if invalid { acc.problem(report::NOT_UTF8, None, None, None); } - let text = analyze::unescaped(text, &mut acc); + let (text, braced) = analyze::unescaped(text, &mut acc); let declared = analyze::declarations(&text, &mut acc); let none = analyze::Known::default(); // With a loader, this first scan finds the includes and what the script's variables hold; its // tallies are left out. let mut tallies = report::Acc::new(limits); let pre = acc.clone(); - let scan = analyze::scan(&text, &declared, None, std::rc::Rc::from([]), &none, if loader.is_some() { &mut tallies } else { &mut acc }); + let scan = analyze::scan(&text, &declared, &braced, None, std::rc::Rc::from([]), &none, if loader.is_some() { &mut tallies } else { &mut acc }); acc.steps = acc.steps.saturating_add(tallies.steps); if scan.module { // A module's report holds its kind and its lines, and none of what the scan found. @@ -183,7 +183,7 @@ pub fn check(name: &str, bytes: &[u8], loader: Option<&mut dyn Loader>, limits: return acc.finish(Kind::Module, scan.lines); } let Some(loader) = loader else { - let mut acc = settle(acc, pre, &text, &declared, scan.known); + let mut acc = settle(acc, pre, &text, &declared, &braced, scan.known); includes::not_read(&mut acc, scan.includes); return acc.finish(Kind::Script, scan.lines); }; @@ -191,17 +191,23 @@ pub fn check(name: &str, bytes: &[u8], loader: Option<&mut dyn Loader>, limits: let read = includes::follow(&mut acc, scan.includes, loader, name, bytes.len(), &mut known); // Each file's text and the names it declares are worked out once for all the scans below. let decoded: Vec<_> = read.iter().map(|file| decode(&file.bytes)).collect(); - let texts: Vec<_> = decoded.into_iter().map(|(text, invalid)| (analyze::unescaped(text, &mut acc), invalid)).collect(); - let declarations: Vec<_> = texts.iter().map(|(text, _)| analyze::declarations(text, &mut acc)).collect(); + let texts: Vec<_> = decoded + .into_iter() + .map(|(text, invalid)| { + let (text, braced) = analyze::unescaped(text, &mut acc); + (text, braced, invalid) + }) + .collect(); + let declarations: Vec<_> = texts.iter().map(|(text, ..)| analyze::declarations(text, &mut acc)).collect(); // A variable may get what it holds from another file's variable: the files are scanned again, // each with what is known so far, while that grows, at most three more times. for _ in 0..3 { let before = known.len(); let mut tallies = report::Acc::new(limits); - let more = analyze::scan(&text, &declared, None, std::rc::Rc::from([]), &known, &mut tallies).known; + let more = analyze::scan(&text, &declared, &[], None, std::rc::Rc::from([]), &known, &mut tallies).known; known.add(more, limits.entries); - for ((file, (text, _)), declared) in read.iter().zip(&texts).zip(&declarations) { - let more = analyze::scan(text, declared, Some(&file.path), std::rc::Rc::clone(&file.paths), &known, &mut tallies).known; + for ((file, (text, ..)), declared) in read.iter().zip(&texts).zip(&declarations) { + let more = analyze::scan(text, declared, &[], Some(&file.path), std::rc::Rc::clone(&file.paths), &known, &mut tallies).known; known.add(more, limits.entries); } acc.steps = acc.steps.saturating_add(tallies.steps); @@ -210,12 +216,12 @@ pub fn check(name: &str, bytes: &[u8], loader: Option<&mut dyn Loader>, limits: } } // The scans for the report: every file starts with what any file's variables hold. - let lines = analyze::scan(&text, &declared, None, std::rc::Rc::from([]), &known, &mut acc).lines; - for ((file, (text, invalid)), declared) in read.iter().zip(&texts).zip(&declarations) { + let lines = analyze::scan(&text, &declared, &braced, None, std::rc::Rc::from([]), &known, &mut acc).lines; + for ((file, (text, braced, invalid)), declared) in read.iter().zip(&texts).zip(&declarations) { if *invalid { acc.problem(report::NOT_UTF8, None, Some(&file.path), None); } - analyze::scan(text, declared, Some(&file.path), std::rc::Rc::clone(&file.paths), &known, &mut acc); + analyze::scan(text, declared, braced, Some(&file.path), std::rc::Rc::clone(&file.paths), &known, &mut acc); } acc.finish(Kind::Script, lines) } @@ -230,6 +236,7 @@ fn settle<'l>( pre: report::Acc<'l>, text: &str, declared: &analyze::Declared<'_>, + braced: &[analyze::Braced], mut known: analyze::Known, ) -> report::Acc<'l> { if known.len() == 0 { @@ -240,7 +247,7 @@ fn settle<'l>( for _ in 0..3 { let before = known.len(); let mut acc = pre.clone(); - let more = analyze::scan(text, declared, None, std::rc::Rc::from([]), &known, &mut acc).known; + let more = analyze::scan(text, declared, braced, None, std::rc::Rc::from([]), &known, &mut acc).known; known.add(more, entries); if known.len() == before { acc.steps = acc.steps.saturating_add(spent); @@ -249,7 +256,7 @@ fn settle<'l>( spent = spent.saturating_add(acc.steps.saturating_sub(pre.steps)); } let mut acc = pre; - analyze::scan(text, declared, None, std::rc::Rc::from([]), &known, &mut acc); + analyze::scan(text, declared, braced, None, std::rc::Rc::from([]), &known, &mut acc); acc.steps = acc.steps.saturating_add(spent); acc } diff --git a/crates/scriptcheck/src/tests.rs b/crates/scriptcheck/src/tests.rs index 31cbef823..853f212c1 100644 --- a/crates/scriptcheck/src/tests.rs +++ b/crates/scriptcheck/src/tests.rs @@ -271,7 +271,7 @@ fn names_written_with_escapes_are_read_as_the_names_they_spell() { /// `src` with the escapes in its names written as the characters they stand for. fn unescaped(src: &str) -> String { let limits = Limits::default(); - crate::analyze::unescaped(std::borrow::Cow::Borrowed(src), &mut crate::report::Acc::new(&limits)).into_owned() + crate::analyze::unescaped(std::borrow::Cow::Borrowed(src), &mut crate::report::Acc::new(&limits)).0.into_owned() } #[test] @@ -364,6 +364,155 @@ fn malformed_escapes_in_bulk_are_read_once_through() { assert_eq!(r.lines, 3); assert_eq!(feature(&r, "notify").map(|f| f.lines.as_slice()), Some(&[3][..])); assert!(r.steps <= 5 * src.len() + 16, "{} steps for {} bytes", r.steps, src.len()); + // Braced escapes: long runs of digits, closed and not, and ones that end at once. + let zeros = "0".repeat(100_000); + let src = format!( + "a\\u{{{zeros}62}};\n\\u{{{}\n{}\n{}\nalert(1);\n", + "f".repeat(100_000), + "\\u{".repeat(20_000), + "\\u{0000000061}\\u{110000}\\u{}\\u{D800}#\\u{61}".repeat(10_000) + ); + let r = report(&src); + assert_eq!(r.lines, 5); + assert_eq!(feature(&r, "notify").map(|f| f.lines.as_slice()), Some(&[5][..])); + assert!(r.steps <= 5 * src.len() + 16, "{} steps for {} bytes", r.steps, src.len()); +} + +#[test] +fn braced_escapes_in_names_are_read_by_their_value() { + for (src, want) in [ + ("\\u{61}pp", "app"), + ("a\\u{62}c", "abc"), + ("\\u{0000000061}", "a"), + ("\\u{2118}\\u{203F}", "\u{2118}\u{203F}"), + ("a\\u{1D7CE}", "a\u{1D7CE}"), + ("\\u{1D49C}", "\u{1D49C}"), + // Not at the start of a name, in no name, out of range, empty, a surrogate and not closed. + ("\\u{30}a", "\\u{30}a"), + ("a\\u{1F600}", "a\\u{1F600}"), + ("a\\u{110000}", "a\\u{110000}"), + ("a\\u{FFFFFFFFFFFFFFFFFFFF}b", "a\\u{FFFFFFFFFFFFFFFFFFFF}b"), + ("a\\u{}b", "a\\u{}b"), + ("a\\u{D800}", "a\\u{D800}"), + ("a\\u{61", "a\\u{61"), + ("a\\u{6 1}", "a\\u{6 1}"), + ] { + assert_eq!(unescaped(src), want, "{src:?}"); + } + assert_eq!(tokens("\\u{61}pp.x"), [(T::Ident, "\\u{61}pp"), (T::Punct, "."), (T::Ident, "x")]); + assert_eq!(tokens("a\\u{0000000062}"), [(T::Ident, "a\\u{0000000062}")]); + assert_eq!(tokens("a\\u{1F600}").get(..2), Some(&[(T::Ident, "a"), (T::Other, "\\")][..])); +} + +#[test] +fn braced_name_escape_reports_later_syntax_and_member() { + let r = report("\\u{61}pp.activeDocument.acmeName;\nvar a\\u{62}c = 1, \\u{0000000061}pp2 = 2;\n\\u{61}pp.x + \\u{61}pp.y;\n"); + let members: Vec<&str> = r.members.iter().map(|m| m.name.as_str()).collect(); + assert_eq!(members, ["app.activeDocument.acmeName", "app.x", "app.y"]); + assert_eq!( + problems(&r), + [ + ("newerSyntax", Some(1), Some("\\u{61}")), + ("newerSyntax", Some(2), Some("\\u{62}")), + ("newerSyntax", Some(2), Some("\\u{0000000061}")), + ("newerSyntax", Some(3), Some("\\u{61}")), + ] + ); + // An escape that isn't a name's character, or isn't closed, makes no name and no report of one. + for src in [ + "\\u{1F600}pp.activeDocument;", + "\\u{110000}pp.activeDocument;", + "\\u{}pp.activeDocument;", + "\\u{D800}pp.activeDocument;", + "\\u{61pp.activeDocument;", + ] { + let r = report(src); + assert!(r.members.iter().all(|m| !m.name.starts_with("app")), "{src:?}: {:?}", r.members); + assert!(r.problems.iter().all(|p| p.detail.as_deref().is_none_or(|d| !d.starts_with("\\u{"))), "{src:?}: {:?}", r.problems); + } + // In an included file, the report names the file. + let mut m = mem(&[("lib.jsxinc", "\\u{61}pp.name;\n")]); + let r = check("main.jsx", b"#include 'lib.jsxinc'\n", Some(&mut m), &Limits::default()); + let found: Vec<_> = r.problems.iter().map(|p| (p.code, p.line, p.file.as_deref(), p.detail.as_deref())).collect(); + assert_eq!(found, [("newerSyntax", Some(1), Some("lib.jsxinc"), Some("\\u{61}"))]); +} + +#[test] +fn private_names_with_escaped_starts_are_single_tokens() { + assert_eq!(tokens("a #\\u0061b"), [(T::Ident, "a"), (T::Other, "#\\u0061b")]); + assert_eq!(tokens("this.#\\u0061pp"), [(T::Ident, "this"), (T::Punct, "."), (T::Other, "#\\u0061pp")]); + assert_eq!(tokens("o.#\\u{61}b"), [(T::Ident, "o"), (T::Punct, "."), (T::Other, "#\\u{61}b")]); + // An escape for a character no name starts with leaves `#` on its own. + assert_eq!(tokens("a #\\u0030b").get(..3), Some(&[(T::Ident, "a"), (T::Other, "#"), (T::Other, "\\")][..])); + let r = report("var q = this.#\\u0061pp;\nvar r = o.#\\u{61}b;\n"); + assert!(r.members.is_empty(), "{:?}", r.members); + assert_eq!( + problems(&r), + [("newerSyntax", Some(1), Some("#app")), ("newerSyntax", Some(2), Some("\\u{61}")), ("newerSyntax", Some(2), Some("#ab"))] + ); +} + +#[test] +fn private_name_reports_decoded_construct() { + let r = report("var q = this.#hid\\u0064en;\nvar r = this.#\\u0061\\u0062;\n"); + assert_eq!(problems(&r), [("newerSyntax", Some(1), Some("#hidden")), ("newerSyntax", Some(2), Some("#ab"))]); +} + +#[test] +fn private_name_at_line_start_is_not_a_directive() { + let r = report(concat!( + "class Box {\n", + " #size = 1;\n", + " #\\u0061rea = 2;\n", + " #include 'a.jsxinc'\n", + " #\\u0069nclude = 4;\n", + " #\\u{69}nclude = 5;\n", + "}\n", + "#\\u0062ox = 3;\n", + "#pragma once\n", + "#target acme\n", + )); + let directives: Vec<_> = r.directives.iter().map(|d| (d.name, d.line)).collect(); + assert_eq!(directives, [("include", 4), ("target", 10)]); + // Where the name a private name's escapes spell would start a preprocessor line, the report + // gives the name as written. + assert_eq!( + problems(&r), + [ + ("newerSyntax", Some(1), Some("class")), + ("newerSyntax", Some(2), Some("#size")), + ("newerSyntax", Some(3), Some("#area")), + ("newerSyntax", Some(5), Some("#\\u0069nclude")), + ("newerSyntax", Some(6), Some("\\u{69}")), + ("newerSyntax", Some(6), Some("#\\u{69}nclude")), + ("newerSyntax", Some(8), Some("#\\u0062ox")), + ("unknownDirective", Some(9), Some("#pragma")), + ] + ); +} + +#[test] +fn private_name_that_starts_with_a_directive_name_is_not_a_directive() { + let r = report(concat!( + "class Box {\n", + " #include$size = 1;\n", + " #include\u{e9} = 2;\n", + " #include\\u0041 = 3;\n", + " #include\\u{41} = 4;\n", + " #INCLUDE 'a.jsxinc'\n", + "}\n", + )); + let directives: Vec<_> = r.directives.iter().map(|d| (d.name, d.line)).collect(); + assert_eq!(directives, [("include", 6)]); +} + +#[test] +fn xml_code_uses_escaped_names() { + let r = report("var x = {\\u{61}pp.activeDocument};\nvar y = \\u0061pp.z \\u{61};\n"); + let members: Vec<&str> = r.members.iter().map(|m| m.name.as_str()).collect(); + assert_eq!(members, ["app.activeDocument"]); + assert_eq!(problems(&r), [("newerSyntax", Some(1), Some("\\u{61}"))]); + assert!(tokens("var x = {this.#\\u0061b};").contains(&(T::Other, "#\\u0061b"))); } #[test] @@ -1590,7 +1739,7 @@ fn include_lines_share_the_folders_of_the_include_path_before_them() { let limits = Limits::default(); let mut acc = report::Acc::new(&limits); let declared = analyze::declarations(&src, &mut acc); - let scan = analyze::scan(&src, &declared, None, std::rc::Rc::from([]), &analyze::Known::default(), &mut acc); + let scan = analyze::scan(&src, &declared, &[], None, std::rc::Rc::from([]), &analyze::Known::default(), &mut acc); let (first, rest) = scan.includes.split_first().unwrap(); assert_eq!((first.paths.len(), rest.len()), (64, 999)); assert!(rest.iter().all(|i| std::rc::Rc::ptr_eq(&i.paths, &first.paths)), "one list of folders, not a copy for each line"); diff --git a/docs/mcp.md b/docs/mcp.md index 335445a3f..47b4fbac5 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -1648,9 +1648,9 @@ control character, in the first 8 KiB) or `tooLarge` (over 8 MiB). For a script, `menuCommand` or `e4x`), each with a count and its lines. - `dom.members` lists the document-model member chains as the script writes them (`app.activeDocument.layers[].name`, with `[]` for an index and `()` for a call); a variable assigned from a chain - is replaced by that chain, and a name written with `\u` escapes is read as the name it spells. `dom.types` lists - the type names the script compares with `.typename` or creates with `new`, apart from the language's, the platform's - and the script's own functions. + is replaced by that chain, and a name written with `\uXXXX` or `\u{…}` escapes is read as the name it spells. + `dom.types` lists the type names the script compares with `.typename` or creates with `new`, apart from the + language's, the platform's and the script's own functions. - `menuCommands.strings` lists the string literals the script passes to `executeMenuCommand` (`app.executeMenuCommand('…')`), each with a count and its lines, and `menuCommands.dynamic` counts the calls whose argument isn't a string literal. @@ -1662,7 +1662,10 @@ control character, in the first 8 KiB) or `tooLarge` (over 8 MiB). For a script, template literals and XML literals that aren't closed, brackets that don't match, syntax newer than ES3 and unknown directives) and notes on its text (bytes that aren't valid UTF-8, mixed line endings, and nesting deeper than the check follows: more than 4,096 open brackets, or XML literals and the code in their braces nested more - than 4,096 levels deep). + than 4,096 levels deep). Newer syntax includes a private name (`#name`, given with its escapes read, unless the + name they spell would make the line a directive) and a `\u{…}` escape in a name (given as written). A `#` and a + name at the start of a line is a private name, not a directive, when the name starts with an escape, or when the + line is inside braces and the name is not a directive's. Each entry of `targets`, `directives`, `includes` and `problems` has the `line` it is on (a problem with the whole text has none) and, when it is in an included file, that file's path as `file`. `features`, `menuCommands.strings` diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 7b06a3804..f6b154639 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -769,7 +769,7 @@ Moved from the ROADMAP's "Shipped so far" on 2026-10-10. Update it when a featur - MCP protocol: five prompts (`prompts/list` / `prompts/get`: poster, icon set, recolour, trace-and-style, export set), `completion/complete` for prompt arguments and template variables read from the live catalogues (formats, effect ids, command ids, trace presets, swatch and object ids), four resource templates (`vectorcraft://object/{id}`, `//command/{id}`, `//effect/{id}`, `//swatch/{name}`) so an agent reads one thing instead of the whole document, and `logging/setLevel` with `notifications/message`. `window.panel` ids are case-insensitive (display labels work too), `vectorcraft://object/{id}` reads through `document.node {summary: true}` (the same reply bytes from one lookup instead of summarizing the whole document: a leaf read on a 2,000-object document drops from ~20 ms to ~0.06 ms), large containers slice with `document.node {depth?, childLimit?}` (truncated levels report `childCount`), `document.inspect` and the `inspect_document` tool slice the same way so a depth-0 read is the layer skeleton (~9.7 KB instead of ~2 MB on a 2,000-object document), `document.find` locates nodes by name/kind/text with ancestor paths, and the CLI sweeps the protocol surface over stdio against the compiled binary. `docs/mcp.md` has a Protocol section covering the capability table and what is deliberately absent (subscriptions, progress, cancellation, sampling, elicitation, Streamable HTTP — all of which need the transport to interleave reads). - Headless CLI (`vectorcraft-cli run`, `convert`, `info`, `bench`, `perf`, `mcp`). A `run` step, like a `command_batch` step over MCP, can use an earlier step's result (`"$1.id"`, #785); `run --commands FILE` (or `-` for standard input) reads the steps from a JSON array or JSON Lines, past the Windows command-line limit (#3147, `apps/vectorcraft-cli/tests/it/cli.rs` `run_reads_*`, `run_refuses_malformed_commands_naming_the_step`). - File › Scripts › Check Script… reads a script written for Illustrator (`.jsx`, `.jsxinc` or a compiled `.jsxbin`) without running any of it and reports what it needs (the document model, files, dialogs, menu commands, features only Illustrator has), what it would do outside the document, the menu commands and document-model members it uses, its directives and included files, and the problems that would stop it loading (`script.check`, which also reads `.js` files); File › Open and drag-and-drop show the report for `.jsx`, `.jsxinc` and `.jsxbin` files. - - Check Script reads names by the same identifier rule as `vectorcraft-js`: Unicode 17.0 `ID_Start` characters, `$` and `_` start a name, and `ID_Continue` characters and `$` continue it (https://www.unicode.org/reports/tr31/, https://tc39.es/ecma262/#sec-names-and-keywords). U+2118, U+203F, U+00B7 and the zero width non-joiner and joiner are name characters, and U+00B2 and U+2E2F are not. Of the characters Unicode 17.0 assigned, U+088F starts a name and U+1ACF may follow its first character. A `\uXXXX` escape is read as its character only where a name may hold it, so `\u0030abc` does not become a name that starts with a digit. Tests: `crates/scriptcheck/src/tests.rs` (`names_are_made_of_the_unicode_identifier_characters`, `an_escape_stands_for_a_character_only_where_a_name_may_hold_it`), `crates/engine/tests/it/js_lexer_vs_scriptcheck.rs` (`both_lexers_take_the_same_characters_into_names`). + - Check Script reads names by the same identifier rule as `vectorcraft-js`: Unicode 17.0 `ID_Start` characters, `$` and `_` start a name, and `ID_Continue` characters and `$` continue it (https://www.unicode.org/reports/tr31/, https://tc39.es/ecma262/#sec-names-and-keywords). U+2118, U+203F, U+00B7 and the zero width non-joiner and joiner are name characters, and U+00B2 and U+2E2F are not. Of the characters Unicode 17.0 assigned, U+088F starts a name and U+1ACF may follow its first character. A `\uXXXX` escape is read as its character only where a name may hold it, so `\u0030abc` does not become a name that starts with a digit; a `\u{…}` escape takes any number of hexadecimal digits up to U+10FFFF, and one that is empty, unclosed, out of range or a surrogate is not part of a name. Each `\u{…}` escape read in a name is reported as syntax newer than ES3, which has only the four-digit form (https://ecma-international.org/wp-content/uploads/ECMA-262_3rd_edition_december_1999.pdf, section 7.6). A private name may start with an escape (`this.#\u{61}pp` is the private name `#app`), and a private name at the start of a line inside braces is not read as a directive unless its name is a directive's. Tests: `crates/scriptcheck/src/tests.rs` (`names_are_made_of_the_unicode_identifier_characters`, `an_escape_stands_for_a_character_only_where_a_name_may_hold_it`, `braced_escapes_in_names_are_read_by_their_value`, `braced_name_escape_reports_later_syntax_and_member`, `private_names_with_escaped_starts_are_single_tokens`, `private_name_at_line_start_is_not_a_directive`), `crates/engine/tests/it/js_lexer_vs_scriptcheck.rs` (`both_lexers_take_the_same_characters_into_names`). - `vectorcraft-js` parses JavaScript as ECMAScript 5.1 defines it, with the legacy octal forms, block functions and lenient regular expressions of Annex B, into a syntax tree with source spans; it reads the `#target`, `#include` and other preprocessor lines as directives without running them, and a script that breaks the grammar or an early-error rule gives a `SyntaxError` with its line and column. Nesting past the parser's depth limit (`Options::max_depth`, 200 by default) gives a `SyntaxError` instead of exhausting the stack. `cargo xtask test262` fetches the conformance suite at a pinned commit and parses its ES5 tests: 5,411 of 5,418 runs pass (the 7 others use `let` and `const`). This entry covers parsing; the next entry covers running scripts. Tests: `crates/js/src/tests.rs`, `crates/engine/tests/it/js_lexer_vs_scriptcheck.rs`. - `vectorcraft-js` identifiers, written out or as `\uXXXX` and `\u{...}` escapes, take the characters Unicode's `ID_Start` and `ID_Continue` properties allow (https://www.unicode.org/reports/tr31/), plus `$` and, at the start, `_`, as later ECMAScript editions define them (https://tc39.es/ecma262/#sec-names-and-keywords). `Other_ID_Start` characters such as U+2118 and `Other_ID_Continue` characters such as U+00B7 and the zero width non-joiner and joiner are accepted, and no `Pattern_Syntax` or `Pattern_White_Space` character is. Letter categories come from Unicode 17.0, so U+088F starts an identifier and U+1ACF may follow its first character; characters assigned after Unicode 17.0 are not yet identifier characters. Tests: `crates/js/src/tests.rs` (`identifier_characters_are_unicode_17_id_start_and_id_continue`, which checks every code point, `identifiers_take_the_characters_unicode_17_assigned`, `identifiers_take_the_other_id_start_and_continue_characters`, `identifiers_refuse_pattern_syntax_surrogates_and_bad_escapes`, `no_pattern_syntax_or_white_space_character_is_an_identifier_character`). - `vectorcraft-js`'s `Realm` runs ECMAScript 5.1 scripts: the parsed script compiles to verified bytecode that a non-recursive interpreter runs, with closures, `this`, `arguments`, `with`, direct and indirect `eval`, `new Function`, `try`/`catch`/`finally`, labeled `break` and `continue`, and `for`-`in` in insertion order, over a mark-and-sweep heap, with the built-ins the next two entries list. Every run is bounded by fuel, heap bytes, frames, nested host calls and an interrupt flag (smaller defaults in the wasm build), and a run that reaches a limit stops with a limit error that `try` cannot catch. `cargo xtask test262` also runs the suite's ES5 language tests that need only the built-ins present: 4,280 of 4,286 runs pass (4 of the others expect the completion values of later editions, and 2 expect a configurable function `length`). The app does not call it yet. Tests: `crates/js/src/interp_tests.rs`, `crates/js/tests/test262.rs`.