pauljd/vectorcraft
1
0
Fork 0
mirror of https://github.com/storytold/vectorcraft.git synced 2026-11-12 23:05:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into feat/script-target-engine

Browse Source
This commit is contained in:
dotCooCoo
2026-11-11 12:09:30 -08:00
parent f1c17603bb 541f8beae7
commit 7fbac74885
297 changed files with 16961 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scriptcheck/src/analyze.rs
+8 -10
View File
@@ -116,8 +116,9 @@ pub(crate) fn scan<'s>(src: &'s str, declared: &Declared<'s>, file: Option<&str>
}
/// `src` with the `\uXXXX` escapes in its names written as the characters they stand for (ECMA-262
/// 3rd edition, section 7.6), and `src` itself when no name has one. An escape for a character no
/// name may hold stays as written. Line breaks don't change, so every line keeps its number.
/// 3rd edition, section 7.6), and `src` itself when no name has one. An escape for a character a
/// name may not hold where the escape stands stays as written. Line breaks don't change, so every
/// line keeps its number.
pub(crate) fn unescaped<'a>(src: Cow<'a, str>, acc: &mut Acc) -> Cow<'a, str> {
match unescape_names(&src, acc) {
Some(out) => Cow::Owned(out),
@@ -145,20 +146,17 @@ fn unescape_names(src: &str, acc: &mut Acc) -> Option<String> {
changed.then_some(out)
}
/// The name `text` with each `\uXXXX` escape for a character a name may hold written as that
/// character.
/// The name `text` with each `\uXXXX` escape for a character a name may hold where it stands written
/// as that character: a character a name may start with first, and one a name may hold after that.
fn unescape_name(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(i) = rest.find("\\u") {
out.push_str(rest.get(..i).unwrap_or_default());
let after = rest.get(i + 2..).unwrap_or_default();
let c = after
.get(..4)
.filter(|hex| hex.bytes().all(|b| b.is_ascii_hexdigit()))
.and_then(|hex| u32::from_str_radix(hex, 16).ok())
.and_then(char::from_u32)
.filter(|&c| crate::lex::ident_part(c));
let first = out.is_empty();
let c = crate::lex::escaped(rest.get(i..).unwrap_or_default())
.filter(|&c| if first { crate::lex::ident_start(c) } else { crate::lex::ident_part(c) });
match c {
Some(c) => {
out.push(c);
crates/scriptcheck/src/lex.rs
+105 -16
View File
@@ -8,6 +8,10 @@
//! level and the levels of XML literals and their code are kept on stacks capped at the nesting
//! limit.
use std::cmp::Ordering;
use unicode_general_category::{GeneralCategory as G, get_general_category};
use crate::catalog;
use crate::report::{TOO_DEEP, UNTERMINATED_COMMENT, UNTERMINATED_REGEXP, UNTERMINATED_STRING, UNTERMINATED_TEMPLATE};
@@ -130,13 +134,105 @@ fn is_blank(c: char) -> bool {
!is_break(c) && (c.is_whitespace() || c == '\u{feff}')
}
fn ident_start(c: char) -> bool {
c == '$' || c == '_' || c.is_ascii_alphabetic() || (!c.is_ascii() && c.is_alphabetic())
/// A character a name may start with: a Unicode `ID_Start` character, `$` or `_` (the Names and
/// Keywords clause of current ECMA-262 editions). `ID_Start` is the letters (categories Lu, Ll, Lt,
/// Lm, Lo and Nl) and `Other_ID_Start`, less `Pattern_Syntax` and `Pattern_White_Space` (Unicode
/// Standard Annex #31). The categories come from Unicode 16.0.
pub(crate) fn ident_start(c: char) -> bool {
match c {
'$' | '_' => true,
_ if c.is_ascii() => c.is_ascii_alphabetic(),
_ if is_other_id_start(c) => true,
_ => {
matches!(
get_general_category(c),
G::UppercaseLetter | G::LowercaseLetter | G::TitlecaseLetter | G::ModifierLetter | G::OtherLetter | G::LetterNumber
) && !is_pattern_syntax(c)
}
}
}
/// A character a name may hold after its first.
/// A character a name may hold after its first: a Unicode `ID_Continue` character or `$`.
/// `ID_Continue` adds to `ID_Start` the decimal digits (Nd), combining marks (Mn, Mc), connector
/// punctuation (Pc, which holds `_`) and `Other_ID_Continue` (which holds the zero width non-joiner
/// and joiner), less `Pattern_Syntax` and `Pattern_White_Space`.
pub(crate) fn ident_part(c: char) -> bool {
ident_start(c) || c.is_ascii_digit() || (!c.is_ascii() && c.is_alphanumeric())
match c {
'0'..='9' => true,
_ if ident_start(c) => true,
_ if c.is_ascii() => false,
_ if is_other_id_continue(c) => true,
_ => {
matches!(get_general_category(c), G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation)
&& !is_pattern_syntax(c)
}
}
}
/// The character the `\uXXXX` escape at the start of `s` stands for: `None` when `s` doesn't start
/// with `\u` and four hexadecimal digits, or when they give a lone surrogate.
pub(crate) fn escaped(s: &str) -> Option<char> {
s.strip_prefix("\\u")?
.get(..4)
.filter(|hex| hex.bytes().all(|b| b.is_ascii_hexdigit()))
.and_then(|hex| u32::from_str_radix(hex, 16).ok())
.and_then(char::from_u32)
}
/// `Other_ID_Start` (Unicode PropList 16.0): characters kept in `ID_Start` though their category is
/// not a letter.
fn is_other_id_start(c: char) -> bool {
matches!(c, '\u{1885}' | '\u{1886}' | '\u{2118}' | '\u{212E}' | '\u{309B}' | '\u{309C}')
}
/// `Other_ID_Continue` (Unicode PropList 16.0): characters kept in `ID_Continue` though their
/// category is not a letter, mark, digit or connector.
fn is_other_id_continue(c: char) -> bool {
matches!(c, '\u{B7}' | '\u{387}' | '\u{1369}'..='\u{1371}' | '\u{19DA}' | '\u{200C}' | '\u{200D}' | '\u{30FB}' | '\u{FF65}')
}
/// Non-ASCII `Pattern_Syntax` ranges (Unicode PropList). Unicode does not change this set, and no
/// identifier character belongs to it. `Pattern_White_Space` outside ASCII (U+0085, U+200E, U+200F,
/// U+2028 and U+2029) holds no letter, mark, digit or connector, so the category checks leave it out.
const PATTERN_SYNTAX: [(char, char); 23] = [
('\u{A1}', '\u{A7}'),
('\u{A9}', '\u{A9}'),
('\u{AB}', '\u{AC}'),
('\u{AE}', '\u{AE}'),
('\u{B0}', '\u{B1}'),
('\u{B6}', '\u{B6}'),
('\u{BB}', '\u{BB}'),
('\u{BF}', '\u{BF}'),
('\u{D7}', '\u{D7}'),
('\u{F7}', '\u{F7}'),
('\u{2010}', '\u{2027}'),
('\u{2030}', '\u{203E}'),
('\u{2041}', '\u{2053}'),
('\u{2055}', '\u{205E}'),
('\u{2190}', '\u{245F}'),
('\u{2500}', '\u{2775}'),
('\u{2794}', '\u{2BFF}'),
('\u{2E00}', '\u{2E7F}'),
('\u{3001}', '\u{3003}'),
('\u{3008}', '\u{3020}'),
('\u{3030}', '\u{3030}'),
('\u{FD3E}', '\u{FD3F}'),
('\u{FE45}', '\u{FE46}'),
];
/// Whether a non-ASCII character is `Pattern_Syntax`.
fn is_pattern_syntax(c: char) -> bool {
PATTERN_SYNTAX
.binary_search_by(|&(lo, hi)| {
if hi < c {
Ordering::Less
} else if lo > c {
Ordering::Greater
} else {
Ordering::Equal
}
})
.is_ok()
}
/// After `<`, the start of an XML literal: a name, `>` (a list), `!` (a comment or CDATA), `?` (a
@@ -300,20 +396,13 @@ impl<'s> Lexer<'s> {
}
}
/// The rest of a name (its first character included): letters, digits, `$`, `_` and `\u`
/// escapes.
/// The rest of a name (its first character included): the characters [`ident_part`] allows,
/// written out or as `\uXXXX` escapes. An escape for any other character, or one without four
/// hexadecimal digits, ends the name.
fn name(&mut self) {
loop {
match self.peek() {
Some('\\') if self.peek_at(1) == Some('u') => {
self.skip(2);
for _ in 0..4 {
match self.peek() {
Some(h) if h.is_ascii_hexdigit() => self.bump(h),
_ => break,
}
}
}
Some('\\') if escaped(self.rest()).is_some_and(ident_part) => self.skip(6),
Some(c) if ident_part(c) => self.bump(c),
_ => break,
}
@@ -820,7 +909,7 @@ impl<'s> Iterator for Lexer<'s> {
self.close_level();
self.bump('}');
return Some(self.markup_token(start, line, T::XmlMore, stmt, expr));
} else if ident_start(c) || (c == '\\' && self.peek_at(1) == Some('u')) {
} else if ident_start(c) || (c == '\\' && escaped(self.rest()).is_some_and(ident_start)) {
self.name();
T::Ident
} else if c.is_ascii_digit() || (c == '.' && self.peek_at(1).is_some_and(|d| d.is_ascii_digit())) {
crates/scriptcheck/src/tests.rs
+93
View File
@@ -268,6 +268,99 @@ fn names_written_with_escapes_are_read_as_the_names_they_spell() {
assert_eq!(r.lines, 3);
}
/// `src` with the escapes in its names written as the characters they stand for.
fn unescaped(src: &str) -> String {
let limits = Limits::default();
crate::analyze::unescaped(std::borrow::Cow::Borrowed(src), &mut crate::report::Acc::new(&limits)).into_owned()
}
#[test]
fn names_are_made_of_the_unicode_identifier_characters() {
// U+2118 (Other_ID_Start) starts a name, and U+203F (connector punctuation), U+00B7
// (Other_ID_Continue) and the zero width non-joiner and joiner continue one.
for name in ["\u{2118}a", "a\u{203F}b", "a\u{B7}b", "a\u{200C}b", "a\u{200D}b", "_\u{2118}", "$_a1", "\u{E9}t\u{E9}"] {
assert_eq!(tokens(name), [(T::Ident, name)], "{name:?}");
}
for (src, want) in [
// Characters that only continue a name, and digits, don't start one.
("\u{203F}a", [(T::Other, "\u{203F}"), (T::Ident, "a")]),
("\u{B7}a", [(T::Other, "\u{B7}"), (T::Ident, "a")]),
("\u{200C}a", [(T::Other, "\u{200C}"), (T::Ident, "a")]),
("0a", [(T::Number, "0"), (T::Ident, "a")]),
// A superscript digit (category No) and a letter that is Pattern_Syntax are in no name.
("a\u{B2}", [(T::Ident, "a"), (T::Other, "\u{B2}")]),
("a\u{2E2F}", [(T::Ident, "a"), (T::Other, "\u{2E2F}")]),
("\u{2E2F}a", [(T::Other, "\u{2E2F}"), (T::Ident, "a")]),
// `#` and a name away from the start of a line, and a regular expression's flags.
("a #\u{2118}b", [(T::Ident, "a"), (T::Other, "#\u{2118}b")]),
("=/a/g\u{B2}", [(T::RegExp, "/a/g"), (T::Other, "\u{B2}")]),
] {
let mut got = tokens(src);
if src.starts_with('=') {
got.remove(0);
}
assert_eq!(got, want, "{src:?}");
}
assert_eq!(tokens("a #\u{B2}"), [(T::Ident, "a"), (T::Other, "#"), (T::Other, "\u{B2}")]);
assert_eq!(tokens("=/a/g\u{200D}"), [(T::Punct, "="), (T::RegExp, "/a/g\u{200D}")]);
// The code in an XML literal's braces reads names by the same rule.
assert!(tokens("var x = <a>{\u{2118}b + c\u{203F}d}</a>;").iter().filter(|t| t.0 == T::Ident).any(|t| t.1 == "c\u{203F}d"));
assert!(tokens("var x = <a>{\u{2118}b}</a>;").contains(&(T::Ident, "\u{2118}b")));
}
#[test]
fn an_escape_stands_for_a_character_only_where_a_name_may_hold_it() {
for (src, want) in [
("\\u2118a", "\u{2118}a"),
("a\\u203Fb", "a\u{203F}b"),
("a\\u00B7b", "a\u{B7}b"),
("a\\u200Cb", "a\u{200C}b"),
("a\\u200Db", "a\u{200D}b"),
("a\\u0030", "a0"),
("\\u0024\\u005F", "$_"),
// Not at the start of a name.
("\\u0030abc", "\\u0030abc"),
("\\u203Fa", "\\u203Fa"),
("\\u00B7a", "\\u00B7a"),
// In no name.
("a\\u00B2", "a\\u00B2"),
("a\\u2E2F", "a\\u2E2F"),
("\\u2E2F", "\\u2E2F"),
// Malformed escapes and a lone surrogate.
("\\u", "\\u"),
("a\\u12;", "a\\u12;"),
("a\\uZZZZ", "a\\uZZZZ"),
("a\\uD800", "a\\uD800"),
("", ""),
] {
assert_eq!(unescaped(src), want, "{src:?}");
}
let r = report("\\u0061pp.a\\u203Fb;\napp.\\u2118c;\n");
let members: Vec<&str> = r.members.iter().map(|m| m.name.as_str()).collect();
assert_eq!(members, ["app.a\u{203F}b", "app.\u{2118}c"]);
// An escape a name may not hold where it stands, or a malformed one, is not part of a name.
for (src, want) in [
("\\u0030a", [(T::Other, "\\"), (T::Ident, "u0030a")]),
("a\\u00B2", [(T::Ident, "a"), (T::Other, "\\")]),
("a\\u12", [(T::Ident, "a"), (T::Other, "\\")]),
("a\\uD800", [(T::Ident, "a"), (T::Other, "\\")]),
] {
assert_eq!(tokens(src).get(..2), Some(&want[..]), "{src:?}");
}
assert_eq!(tokens("a\\u0030\\u200C"), [(T::Ident, "a\\u0030\\u200C")]);
let r = report("app.\\u0030abc;\napp.a\\u00B2;\n");
assert!(r.members.iter().all(|m| !m.name.contains('\\') && !m.name.contains("0abc")), "{:?}", r.members);
}
#[test]
fn malformed_escapes_in_bulk_are_read_once_through() {
let src = format!("{}\n{}\nalert(1);\n", "\\u\\uD800\\u12\\u0030\\u2118".repeat(20_000), "a\u{203F}".repeat(20_000));
let r = report(&src);
assert_eq!(r.lines, 3);
assert_eq!(feature(&r, "notify").map(|f| f.lines.as_slice()), Some(&[3][..]));
assert!(r.steps <= 5 * src.len() + 16, "{} steps for {} bytes", r.steps, src.len());
}
#[test]
fn the_scripts_own_platform_names_are_not_the_platforms() {
let r = report(concat!(