JavaScript: characters assigned in Unicode 17 are identifier characters

vectorcraft-js and Check Script took identifier categories from
unicode-general-category 1.1.0, which carries Unicode 16.0, so letters
and marks assigned in Unicode 17.0 (U+088F at the start of a name,
U+1ACF after its first character) were refused. Both lexers now read
general categories from unicode-properties 0.1.4 (Unicode 17.0), which
Cargo.lock already holds. The version is exact because 0.1.5 adds the
Unicode-3.0 license, which is not on the license allowlist. The
Other_ID_Start, Other_ID_Continue and Pattern_Syntax lists are the same
in PropList 17.0.

The exhaustive identifier test now expects the DerivedCoreProperties
17.0 totals (145,916 ID_Start, 149,240 ID_Continue), and new tests read
U+088F and U+1ACF written out and as escapes in both lexers.

crates/scriptcheck/Cargo.toml

@@ -10,7 +10,9 @@ rust-version.workspace = true
 workspace = true
 
 [dependencies]
-unicode-general-category = "1"
+# Unicode 17.0 general categories for names (lex.rs). 0.1.5 adds the Unicode-3.0 license, which is
+# not on the license allowlist, so the version is exact.
+unicode-properties = { version = "=0.1.4", default-features = false, features = ["general-category"] }
 
 [dev-dependencies]
 proptest = { workspace = true }

crates/scriptcheck/src/lex.rs

@@ -10,7 +10,7 @@
 
 use std::cmp::Ordering;
 
-use unicode_general_category::{GeneralCategory as G, get_general_category};
+use unicode_properties::{GeneralCategory as G, UnicodeGeneralCategory};
 
 use crate::catalog;
 use crate::report::{TOO_DEEP, UNTERMINATED_COMMENT, UNTERMINATED_REGEXP, UNTERMINATED_STRING, UNTERMINATED_TEMPLATE};
@@ -137,7 +137,7 @@ fn is_blank(c: char) -> bool {
 /// A character a name may start with: a Unicode `ID_Start` character, `$` or `_` (the Names and
 /// Keywords clause of current ECMA-262 editions). `ID_Start` is the letters (categories Lu, Ll, Lt,
 /// Lm, Lo and Nl) and `Other_ID_Start`, less `Pattern_Syntax` and `Pattern_White_Space` (Unicode
-/// Standard Annex #31). The categories come from Unicode 16.0.
+/// Standard Annex #31). The categories come from Unicode 17.0.
 pub(crate) fn ident_start(c: char) -> bool {
     match c {
         '$' | '_' => true,
@@ -145,7 +145,7 @@ pub(crate) fn ident_start(c: char) -> bool {
         _ if is_other_id_start(c) => true,
         _ => {
             matches!(
-                get_general_category(c),
+                c.general_category(),
                 G::UppercaseLetter | G::LowercaseLetter | G::TitlecaseLetter | G::ModifierLetter | G::OtherLetter | G::LetterNumber
             ) && !is_pattern_syntax(c)
         }
@@ -162,10 +162,7 @@ pub(crate) fn ident_part(c: char) -> bool {
         _ if ident_start(c) => true,
         _ if c.is_ascii() => false,
         _ if is_other_id_continue(c) => true,
-        _ => {
-            matches!(get_general_category(c), G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation)
-                && !is_pattern_syntax(c)
-        }
+        _ => matches!(c.general_category(), G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation) && !is_pattern_syntax(c),
     }
 }
 
@@ -179,13 +176,13 @@ pub(crate) fn escaped(s: &str) -> Option<char> {
         .and_then(char::from_u32)
 }
 
-/// `Other_ID_Start` (Unicode PropList 16.0): characters kept in `ID_Start` though their category is
+/// `Other_ID_Start` (Unicode PropList 17.0): characters kept in `ID_Start` though their category is
 /// not a letter.
 fn is_other_id_start(c: char) -> bool {
     matches!(c, '\u{1885}' | '\u{1886}' | '\u{2118}' | '\u{212E}' | '\u{309B}' | '\u{309C}')
 }
 
-/// `Other_ID_Continue` (Unicode PropList 16.0): characters kept in `ID_Continue` though their
+/// `Other_ID_Continue` (Unicode PropList 17.0): characters kept in `ID_Continue` though their
 /// category is not a letter, mark, digit or connector.
 fn is_other_id_continue(c: char) -> bool {
     matches!(c, '\u{B7}' | '\u{387}' | '\u{1369}'..='\u{1371}' | '\u{19DA}' | '\u{200C}' | '\u{200D}' | '\u{30FB}' | '\u{FF65}')

crates/scriptcheck/src/tests.rs

@@ -278,12 +278,14 @@ fn unescaped(src: &str) -> String {
 fn names_are_made_of_the_unicode_identifier_characters() {
     // U+2118 (Other_ID_Start) starts a name, and U+203F (connector punctuation), U+00B7
     // (Other_ID_Continue) and the zero width non-joiner and joiner continue one.
-    for name in ["\u{2118}a", "a\u{203F}b", "a\u{B7}b", "a\u{200C}b", "a\u{200D}b", "_\u{2118}", "$_a1", "\u{E9}t\u{E9}"] {
+    // Unicode 17.0 assigned the letter U+088F and the combining mark U+1ACF.
+    for name in ["\u{2118}a", "a\u{203F}b", "a\u{B7}b", "a\u{200C}b", "a\u{200D}b", "_\u{2118}", "$_a1", "\u{E9}t\u{E9}", "\u{88F}", "_\u{1ACF}"] {
         assert_eq!(tokens(name), [(T::Ident, name)], "{name:?}");
     }
     for (src, want) in [
         // Characters that only continue a name, and digits, don't start one.
         ("\u{203F}a", [(T::Other, "\u{203F}"), (T::Ident, "a")]),
+        ("\u{1ACF}a", [(T::Other, "\u{1ACF}"), (T::Ident, "a")]),
         ("\u{B7}a", [(T::Other, "\u{B7}"), (T::Ident, "a")]),
         ("\u{200C}a", [(T::Other, "\u{200C}"), (T::Ident, "a")]),
         ("0a", [(T::Number, "0"), (T::Ident, "a")]),
@@ -318,8 +320,11 @@ fn an_escape_stands_for_a_character_only_where_a_name_may_hold_it() {
         ("a\\u200Db", "a\u{200D}b"),
         ("a\\u0030", "a0"),
         ("\\u0024\\u005F", "$_"),
+        ("\\u088F", "\u{88F}"),
+        ("_\\u1ACF", "_\u{1ACF}"),
         // Not at the start of a name.
         ("\\u0030abc", "\\u0030abc"),
+        ("\\u1ACFa", "\\u1ACFa"),
         ("\\u203Fa", "\\u203Fa"),
         ("\\u00B7a", "\\u00B7a"),
         // In no name.