pauljd/vectorcraft
1
0
Fork 0
mirror of https://github.com/storytold/vectorcraft.git synced 2026-11-12 07:45:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

JavaScript: characters assigned in Unicode 17 are identifier characters

Browse Source 
vectorcraft-js and Check Script took identifier categories from
unicode-general-category 1.1.0, which carries Unicode 16.0, so letters
and marks assigned in Unicode 17.0 (U+088F at the start of a name,
U+1ACF after its first character) were refused. Both lexers now read
general categories from unicode-properties 0.1.4 (Unicode 17.0), which
Cargo.lock already holds. The version is exact because 0.1.5 adds the
Unicode-3.0 license, which is not on the license allowlist. The
Other_ID_Start, Other_ID_Continue and Pattern_Syntax lists are the same
in PropList 17.0.

The exhaustive identifier test now expects the DerivedCoreProperties
17.0 totals (145,916 ID_Start, 149,240 ID_Continue), and new tests read
U+088F and U+1ACF written out and as escapes in both lexers.
This commit is contained in:
dotCooCoo
2026-11-11 13:57:54 -08:00
parent 566f720868
commit 283d127e2a
9 changed files with 59 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scriptcheck/src/lex.rs
+6 -9
View File
@@ -10,7 +10,7 @@
use std::cmp::Ordering;
use unicode_general_category::{GeneralCategory as G, get_general_category};
use unicode_properties::{GeneralCategory as G, UnicodeGeneralCategory};
use crate::catalog;
use crate::report::{TOO_DEEP, UNTERMINATED_COMMENT, UNTERMINATED_REGEXP, UNTERMINATED_STRING, UNTERMINATED_TEMPLATE};
@@ -137,7 +137,7 @@ fn is_blank(c: char) -> bool {
/// A character a name may start with: a Unicode `ID_Start` character, `$` or `_` (the Names and
/// Keywords clause of current ECMA-262 editions). `ID_Start` is the letters (categories Lu, Ll, Lt,
/// Lm, Lo and Nl) and `Other_ID_Start`, less `Pattern_Syntax` and `Pattern_White_Space` (Unicode
/// Standard Annex #31). The categories come from Unicode 16.0.
/// Standard Annex #31). The categories come from Unicode 17.0.
pub(crate) fn ident_start(c: char) -> bool {
match c {
'$' | '_' => true,
@@ -145,7 +145,7 @@ pub(crate) fn ident_start(c: char) -> bool {
_ if is_other_id_start(c) => true,
_ => {
matches!(
get_general_category(c),
c.general_category(),
G::UppercaseLetter | G::LowercaseLetter | G::TitlecaseLetter | G::ModifierLetter | G::OtherLetter | G::LetterNumber
) && !is_pattern_syntax(c)
}
@@ -162,10 +162,7 @@ pub(crate) fn ident_part(c: char) -> bool {
_ if ident_start(c) => true,
_ if c.is_ascii() => false,
_ if is_other_id_continue(c) => true,
_ => {
matches!(get_general_category(c), G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation)
&& !is_pattern_syntax(c)
}
_ => matches!(c.general_category(), G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation) && !is_pattern_syntax(c),
}
}
@@ -179,13 +176,13 @@ pub(crate) fn escaped(s: &str) -> Option<char> {
.and_then(char::from_u32)
}
/// `Other_ID_Start` (Unicode PropList 16.0): characters kept in `ID_Start` though their category is
/// `Other_ID_Start` (Unicode PropList 17.0): characters kept in `ID_Start` though their category is
/// not a letter.
fn is_other_id_start(c: char) -> bool {
matches!(c, '\u{1885}' | '\u{1886}' | '\u{2118}' | '\u{212E}' | '\u{309B}' | '\u{309C}')
}
/// `Other_ID_Continue` (Unicode PropList 16.0): characters kept in `ID_Continue` though their
/// `Other_ID_Continue` (Unicode PropList 17.0): characters kept in `ID_Continue` though their
/// category is not a letter, mark, digit or connector.
fn is_other_id_continue(c: char) -> bool {
matches!(c, '\u{B7}' | '\u{387}' | '\u{1369}'..='\u{1371}' | '\u{19DA}' | '\u{200C}' | '\u{200D}' | '\u{30FB}' | '\u{FF65}')
crates/scriptcheck/src/tests.rs
+6 -1
View File
@@ -278,12 +278,14 @@ fn unescaped(src: &str) -> String {
fn names_are_made_of_the_unicode_identifier_characters() {
// U+2118 (Other_ID_Start) starts a name, and U+203F (connector punctuation), U+00B7
// (Other_ID_Continue) and the zero width non-joiner and joiner continue one.
for name in ["\u{2118}a", "a\u{203F}b", "a\u{B7}b", "a\u{200C}b", "a\u{200D}b", "_\u{2118}", "$_a1", "\u{E9}t\u{E9}"] {
// Unicode 17.0 assigned the letter U+088F and the combining mark U+1ACF.
for name in ["\u{2118}a", "a\u{203F}b", "a\u{B7}b", "a\u{200C}b", "a\u{200D}b", "_\u{2118}", "$_a1", "\u{E9}t\u{E9}", "\u{88F}", "_\u{1ACF}"] {
assert_eq!(tokens(name), [(T::Ident, name)], "{name:?}");
}
for (src, want) in [
// Characters that only continue a name, and digits, don't start one.
("\u{203F}a", [(T::Other, "\u{203F}"), (T::Ident, "a")]),
("\u{1ACF}a", [(T::Other, "\u{1ACF}"), (T::Ident, "a")]),
("\u{B7}a", [(T::Other, "\u{B7}"), (T::Ident, "a")]),
("\u{200C}a", [(T::Other, "\u{200C}"), (T::Ident, "a")]),
("0a", [(T::Number, "0"), (T::Ident, "a")]),
@@ -318,8 +320,11 @@ fn an_escape_stands_for_a_character_only_where_a_name_may_hold_it() {
("a\\u200Db", "a\u{200D}b"),
("a\\u0030", "a0"),
("\\u0024\\u005F", "$_"),
("\\u088F", "\u{88F}"),
("_\\u1ACF", "_\u{1ACF}"),
// Not at the start of a name.
("\\u0030abc", "\\u0030abc"),
("\\u1ACFa", "\\u1ACFa"),
("\\u203Fa", "\\u203Fa"),
("\\u00B7a", "\\u00B7a"),
// In no name.