pauljd/vectorcraft
1
0
Fork 0
mirror of https://github.com/storytold/vectorcraft.git synced 2026-11-09 04:54:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

JavaScript: identifiers follow the Unicode categories of section 7.6, and label and parameter checks are linear

Browse Source 
is_id_start accepts the letter categories (Lu, Ll, Lt, Lm, Lo, Nl) and
is_id_part adds Mn, Mc, Nd and Pc, read with unicode-general-category.
A superscript digit (No) inside an identifier and a mark such as U+09BE
at its start are now SyntaxErrors, and identifiers with marks outside
the old list, such as the Bengali and Tamil viramas, now parse.

Labels in scope are kept in a map, and duplicate labels in a chain and
duplicate parameters of a strict function are found with a hash set, so
long chains, many labeled statements and long parameter lists are
checked in linear time.
This commit is contained in:
dotCooCoo
2026-11-07 14:09:25 -08:00
parent 6bb3d3dfbe
commit ea319ebcda
5 changed files with 70 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -6338,6 +6338,7 @@ name = "vectorcraft-js"
version = "0.21.0"
dependencies = [
"proptest",
"unicode-general-category",
]
[[package]]
crates/js/Cargo.toml
+3
View File
@@ -9,5 +9,8 @@ rust-version.workspace = true
[lints]
workspace = true
[dependencies]
unicode-general-category = "1"
[dev-dependencies]
proptest = { workspace = true }
crates/js/src/chars.rs
+9 -41
View File
@@ -1,6 +1,8 @@
//! Character classes of ECMA-262 5.1 section 7: white space, line terminators and the characters
//! identifiers are made of.
use unicode_general_category::{GeneralCategory as G, get_general_category};
/// A line terminator: LF, CR, U+2028 or U+2029 (section 7.3).
pub fn is_line_terminator(c: u32) -> bool {
matches!(c, 0x0A | 0x0D | 0x2028 | 0x2029)
@@ -21,7 +23,9 @@ pub fn is_id_start(c: u32) -> bool {
0..=0x7F => false,
// U+2E2F VERTICAL TILDE is a letter (Lm) that Unicode leaves out of identifiers.
0x2E2F => false,
_ => char::from_u32(c).is_some_and(|ch| ch.is_alphabetic() && !is_combining_mark(c)),
_ => category(c).is_some_and(|g| {
matches!(g, G::UppercaseLetter | G::LowercaseLetter | G::TitlecaseLetter | G::ModifierLetter | G::OtherLetter | G::LetterNumber)
}),
}
}
@@ -35,49 +39,13 @@ pub fn is_id_part(c: u32) -> bool {
0x2E2F => false,
_ if is_id_start(c) => true,
0..=0x7F => false,
_ => char::from_u32(c).is_some_and(|ch| ch.is_alphanumeric()) || is_combining_mark(c) || is_connector(c),
_ => category(c).is_some_and(|g| matches!(g, G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation)),
}
}
/// The blocks of combining marks (categories Mn and Mc) that scripts are most likely to use.
fn is_combining_mark(c: u32) -> bool {
matches!(
c,
0x0300..=0x036F
| 0x0483..=0x0487
| 0x0591..=0x05BD
| 0x05BF
| 0x05C1..=0x05C2
| 0x05C4..=0x05C5
| 0x05C7
| 0x0610..=0x061A
| 0x064B..=0x065F
| 0x0670
| 0x06D6..=0x06DC
| 0x06DF..=0x06E4
| 0x06E7..=0x06E8
| 0x06EA..=0x06ED
| 0x0900..=0x0903
| 0x093A..=0x093C
| 0x093E..=0x094F
| 0x0951..=0x0957
| 0x0962..=0x0963
| 0x0E31
| 0x0E34..=0x0E3A
| 0x0E47..=0x0E4E
| 0x1AB0..=0x1AFF
| 0x1DC0..=0x1DFF
| 0x20D0..=0x20FF
| 0x302A..=0x302F
| 0x3099..=0x309A
| 0xFE00..=0xFE0F
| 0xFE20..=0xFE2F
)
}
/// Connector punctuation (category Pc) other than `_`.
fn is_connector(c: u32) -> bool {
matches!(c, 0x203F | 0x2040 | 0x2054 | 0xFE33 | 0xFE34 | 0xFE4D..=0xFE4F | 0xFF3F)
/// The Unicode general category of a code point; `None` for a lone surrogate.
fn category(c: u32) -> Option<G> {
char::from_u32(c).map(get_general_category)
}
/// The value of a hexadecimal digit.
crates/js/src/parser.rs
+17 -19
View File
@@ -7,6 +7,8 @@
//! stack, and prefix operators, assignment chains, `else if` chains and labels with loops, so long
//! runs of them do not nest at all.
use std::collections::{HashMap, HashSet};
use crate::LineTable;
use crate::ast::{
AssignOp, Ast, BinaryOp, CatchClause, Expr, ExprId, ForInTarget, ForInit, Function, FunctionKind, LogicalOp, Name, Program, Property,
@@ -18,20 +20,14 @@ use crate::token::{Directive, Keyword, Punct, Span, Token, TokenKind, strict_res
type Result<T> = std::result::Result<T, SyntaxError>;
/// A label in scope, and whether it labels an iteration statement (`continue` may name only
/// those).
#[derive(Clone, Debug)]
struct Label {
name: String,
is_loop: bool,
}
/// What the code being parsed allows, saved and replaced at each function body.
#[derive(Clone, Debug, Default)]
struct Context {
strict: bool,
in_function: bool,
labels: Vec<Label>,
/// The labels in scope, each with whether it labels an iteration statement (`continue` may
/// name only those). Labels in scope have distinct names.
labels: HashMap<String, bool>,
/// Iteration statements open in the current function.
loops: u32,
/// Iteration and `switch` statements open in the current function.
@@ -543,10 +539,9 @@ impl<'s> Parser<'s> {
let name = name.clone();
self.check_identifier(&self.tok, &name)?;
let token = self.bump()?;
let found = self.ctx.labels.iter().rev().find(|l| l.name == name);
match found {
match self.ctx.labels.get(&name) {
None => return Err(self.error_at_token(&token, format!("undefined label '{name}'"))),
Some(l) if !is_break && !l.is_loop => {
Some(&is_loop) if !is_break && !is_loop => {
return Err(self.error_at_token(&token, format!("label '{name}' does not name a loop, so continue cannot use it")));
}
Some(_) => {}
@@ -674,14 +669,15 @@ impl<'s> Parser<'s> {
/// `in_list`: the labeled statement stands in a statement list, so outside strict mode code
/// its body may be a function declaration.
fn parse_labeled(&mut self, in_list: bool) -> Result<StmtId> {
let mut labels = Vec::new();
let mut labels: Vec<(Name, Span)> = Vec::new();
let mut chain = HashSet::new();
while let TokenKind::Ident(name) = &self.tok.kind {
if !self.peek().is_some_and(|t| t.is_punct(Punct::Colon)) {
break;
}
let name = name.clone();
self.check_identifier(&self.tok, &name)?;
if self.ctx.labels.iter().any(|l| l.name == name) || labels.iter().any(|(l, _): &(Name, Span)| l.name == name) {
if self.ctx.labels.contains_key(&name) || !chain.insert(name.clone()) {
return Err(self.error_at_token(&self.tok, format!("duplicate label '{name}'")));
}
let token = self.bump()?;
@@ -689,11 +685,12 @@ impl<'s> Parser<'s> {
labels.push((Name { name, span: token.span }, token.span));
}
let is_loop = self.at_keyword(Keyword::For) || self.at_keyword(Keyword::While) || self.at_keyword(Keyword::Do);
let outer = self.ctx.labels.len();
self.ctx.labels.extend(labels.iter().map(|(n, _)| Label { name: n.name.clone(), is_loop }));
self.ctx.labels.extend(chain.into_iter().map(|name| (name, is_loop)));
self.function_allowed = in_list && !self.ctx.strict;
let body = self.parse_statement();
self.ctx.labels.truncate(outer);
for (label, _) in &labels {
self.ctx.labels.remove(&label.name);
}
let mut body = body?;
for (label, label_start) in labels.into_iter().rev() {
let span = Span::new(label_start.start, self.ast.stmt_span(body).end);
@@ -792,8 +789,9 @@ impl<'s> Parser<'s> {
return Err(self.error_at_token(token, format!("'{}' is a reserved word in strict mode", n.name)));
}
}
for (i, (p, token)) in params.iter().enumerate() {
if params.iter().take(i).any(|(q, _)| q.name == p.name) {
let mut seen = HashSet::new();
for (p, token) in params {
if !seen.insert(p.name.as_str()) {
return Err(self.error_at_token(token, format!("duplicate parameter name '{}' is not allowed in strict mode", p.name)));
}
}
crates/js/src/tests.rs
+40
View File
@@ -373,6 +373,31 @@ fn identifiers_with_unicode_and_escapes() {
assert!(err("var a\u{2e2f};").message.contains("unexpected character"));
}
#[test]
fn identifier_characters_follow_their_unicode_categories() {
// Letters (Lu, Ll, Lt, Lm, Lo, Nl) start an identifier; marks (Mn, Mc), decimal digits (Nd)
// and connector punctuation (Pc) may follow.
for good in [
"var \u{995}\u{9cd}\u{9b7} = 1;",
"var \u{b95}\u{bcd}\u{b95} = 1;",
"var a\u{9be} = 1;",
"var x\u{966} = 1;",
"var a\u{203f}b = 1;",
"var \u{2b0} = 1;",
"var \u{2160} = 1;",
"var \u{1c5} = 1;",
"var \u{5d0}\u{5b7} = 1;",
] {
ok(good);
}
// Other numbers (No) are not identifier characters, and a mark cannot start an identifier.
for bad in ["var a\u{b2} = 1;", "var x\u{bd} = 1;", "var \u{9be}a = 1;", "var \u{300}a = 1;", "var \u{966} = 1;"] {
assert!(err(bad).message.contains("unexpected character"), "{bad:?}");
}
assert!(parse("var \\u09BEa = 1;", &Options::default()).is_err());
assert!(parse("var a\\u00B2 = 1;", &Options::default()).is_err());
}
#[test]
fn comments_and_line_terminators() {
let p = ok("a /* one\ntwo */ b");
@@ -627,6 +652,21 @@ fn long_flat_runs_do_not_nest() {
ok(&"a;\n".repeat(N));
}
#[test]
fn long_label_chains_and_parameter_lists_are_checked_in_linear_time() {
// Each check looks a name up in a hash set or map. Comparing every name with every earlier
// one would take about 2 * 10^10 steps at this size.
const N: usize = 200_000;
let labels: String = (0..N).map(|i| format!("l{i}: ")).collect();
let body: String = (0..N).map(|i| format!("m{i}: break l{i}; ")).collect();
ok(&format!("{labels}for (;;) {{ {body}continue l0; }}"));
assert!(err(&format!("{labels}l0: x;")).message.contains("duplicate label"));
assert!(err(&format!("{labels}for (;;) {{ m: break l{N}; }}")).message.contains("undefined label"));
let params = (0..N).map(|i| format!("p{i}")).collect::<Vec<_>>().join(", ");
ok(&format!("function f({params}) {{ 'use strict'; }}"));
assert!(err(&format!("function f({params}, p0) {{ 'use strict'; }}")).message.contains("duplicate parameter"));
}
#[test]
fn nesting_to_the_default_limit_fits_a_small_stack() {
// The deepest nesting each production allows, parsed on a thread with a 1 MiB stack (the