From e592e8277b4826e4105b9fc2befc4919ba1726e0 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sat, 7 Nov 2026 10:29:51 -0800 Subject: [PATCH 01/27] JavaScript: parser for ES5 scripts, with cargo xtask test262 New crate vectorcraft-js reads ECMAScript 5.1 source into a syntax tree with source positions: a UTF-16 lexer (line and column tracking, automatic semicolon insertion, the regexp and division goals), preprocessor lines read as directive tokens, a recursive-descent parser to an arena tree, and the ES5 early errors, strict mode included. Nesting deeper than Options::max_depth (200 by default) gives a SyntaxError with its line and column rather than overflowing the stack. Nothing runs scripts yet. cargo xtask test262 downloads test262 at a pinned commit, checks its SHA-256, unpacks it into the gitignored corpus/ folder and runs the ES5 syntax tests. A differential test in the engine compares the lexer with scriptcheck on line counts, directives and unterminated literals. --- Cargo.lock | 8 + Cargo.toml | 1 + crates/engine/Cargo.toml | 2 + .../tests/it/js_lexer_vs_scriptcheck.rs | 102 ++ crates/engine/tests/it/main.rs | 1 + crates/js/Cargo.toml | 13 + crates/js/src/ast.rs | 334 +++++ crates/js/src/chars.rs | 99 ++ crates/js/src/error.rs | 29 + crates/js/src/lexer.rs | 637 +++++++++ crates/js/src/lib.rs | 149 ++ crates/js/src/number.rs | 69 + crates/js/src/parser.rs | 1233 +++++++++++++++++ crates/js/src/regexp.rs | 191 +++ crates/js/src/tests.rs | 760 ++++++++++ crates/js/src/token.rs | 391 ++++++ crates/js/tests/test262.rs | 178 +++ docs/target-app-parity.md | 1 + xtask/src/layers.rs | 1 + xtask/src/main.rs | 6 + xtask/src/test262.rs | 73 + 21 files changed, 4278 insertions(+) create mode 100644 crates/engine/tests/it/js_lexer_vs_scriptcheck.rs create mode 100644 crates/js/Cargo.toml create mode 100644 crates/js/src/ast.rs create mode 100644 crates/js/src/chars.rs create mode 100644 crates/js/src/error.rs create mode 100644 crates/js/src/lexer.rs create mode 100644 crates/js/src/lib.rs create mode 100644 crates/js/src/number.rs create mode 100644 crates/js/src/parser.rs create mode 100644 crates/js/src/regexp.rs create mode 100644 crates/js/src/tests.rs create mode 100644 crates/js/src/token.rs create mode 100644 crates/js/tests/test262.rs create mode 100644 xtask/src/test262.rs diff --git a/Cargo.lock b/Cargo.lock index d06f43371..b6b88bcfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6264,6 +6264,7 @@ dependencies = [ "vectorcraft-eps", "vectorcraft-format", "vectorcraft-geom", + "vectorcraft-js", "vectorcraft-metafile", "vectorcraft-pathops", "vectorcraft-pdf", @@ -6332,6 +6333,13 @@ dependencies = [ "vectorcraft-testkit", ] +[[package]] +name = "vectorcraft-js" +version = "0.21.0" +dependencies = [ + "proptest", +] + [[package]] name = "vectorcraft-mcp" version = "0.21.0" diff --git a/Cargo.toml b/Cargo.toml index 1f81e7569..a3e3e9890 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ vectorcraft-metafile = { path = "crates/metafile" } vectorcraft-plugins = { path = "crates/plugins" } vectorcraft-affinity = { path = "crates/affinity" } vectorcraft-scriptcheck = { path = "crates/scriptcheck" } +vectorcraft-js = { path = "crates/js" } # foundation kurbo = { version = "0.13", features = ["serde"] } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 6789a2116..eddf966db 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -66,3 +66,5 @@ vectorcraft-pdf = { workspace = true, features = ["read-count"] } wat = { workspace = true } # Zstandard streams for the Illustrator editing data fuzz tests. ruzstd = { workspace = true } +# The JavaScript parser's lexer, compared with the script checker's on the same scripts. +vectorcraft-js = { workspace = true } diff --git a/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs b/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs new file mode 100644 index 000000000..6c84c66b3 --- /dev/null +++ b/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs @@ -0,0 +1,102 @@ +//! The JavaScript parser's lexer (`vectorcraft-js`) and the script checker's (`vectorcraft-scriptcheck`) +//! read the same scripts the same way: both crates stand alone, so each has its own lexer, and these +//! tests run both over one set of scripts and fail on any disagreement about the lines a script has, +//! its preprocessor lines, and where a string, comment or regular expression that is never closed +//! starts. The scripts stay inside ECMAScript 3, the language both read (no XML literals, which +//! only the checker reads). + +use vectorcraft_js::{Options, TokenKind, parse, tokenize}; +use vectorcraft_scriptcheck::{Limits, check}; + +/// The problems the checker reports from its lexer, which the parser reports as a `SyntaxError`. +const LEXICAL: [(&str, &str); 3] = [ + ("unterminatedString", "unterminated string"), + ("unterminatedComment", "unterminated comment"), + ("unterminatedRegExp", "unterminated regular expression"), +]; + +/// Scripts both lexers read without a problem, each with a `/` or a quote in a place where reading +/// it the other way would change the tokens. +const VALID: &[&str] = &[ + "var a = b / c / d;\nvar r = /ab+c/gi;\n", + "if (x) /re/.test(y);\nwhile (z) /q/g.exec(w);\n", + "function f() { return /x/; }\nvar g = function () {} / 2;\n", + "{}\n/re/.test(s);\n", + "x = {a: 1} / 2;\n", + "a = b\n/c/g\n", + "s = 'it\\'s \"q\" // not a comment';\nt = \"/* nor this */\";\n", + "s = 'line\\\ncontinued';\nu = 1;\n", + "x = a++ / 2;\ny = (b) / 2;\nz = c[0] / 2;\n", + "r = /[/]/;\nq = /\\//;\n", + "#target vectorcraft\n#include \"lib.jsxinc\"\n//@includepath \"~/x\"\nvar a = 1;\n#targetengine session\n", + "a = 1;\r\nb = 2;\r\nc = 3;", + "a = 1;\u{2028}b = 2;\u{2029}c = 3;\n", + "typeof /x/ === 'object';\nvoid /y/;\n", + "switch (x) { case /a/.test(y): break; }\n", + "x = y ? /a/ : /b/;\n", + "label: for (;;) { break label; }\n/* multi\nline */ z = 1 / 2;\n", + "var o = { re: /x/, d: 4 / 2 };\n", + "x = [/a/, 1 / 2];\n", + "if (a) {\n} else /x/.test(b);\n", + "a = 'q' + \"r\";\n// a comment with a 'quote\n", + "x = a\n++b\n", + "for (var i = 0; i < n / 2; i++) { s += /\\d/.exec(t[i]) ? 1 : 0; }\n", + "", + "\n\n", +]; + +/// Scripts with a string, comment or regular expression that is never closed: the problem the +/// checker gives and the line it starts on. +const INVALID: &[(&str, &str, usize)] = &[ + ("var a = 1;\nvar s = 'abc;\n", "unterminatedString", 2), + ("a = 1;\n/* never closed\nb = 2;\n", "unterminatedComment", 2), + ("x = 1;\ny = /abc\n", "unterminatedRegExp", 2), + ("if (x)\n /abc\n", "unterminatedRegExp", 2), + ("a = 1;\r\nb = \"x\r\n", "unterminatedString", 2), + ("q = 2 / 3;\nr = /[/]\n", "unterminatedRegExp", 2), +]; + +fn js_options() -> Options { + Options { preprocessor_lines: true, ..Options::default() } +} + +#[test] +fn both_lexers_count_the_same_lines_and_read_the_same_directives() { + for src in VALID { + let program = parse(src, &js_options()).unwrap_or_else(|e| panic!("{src:?}: {e}")); + let report = check("script.jsx", src.as_bytes(), None, &Limits::default()); + assert_eq!(report.lines, program.line_count() as usize, "lines of {src:?}"); + let lexical: Vec<_> = report.problems.iter().filter(|p| LEXICAL.iter().any(|(code, _)| *code == p.code)).collect(); + assert!(lexical.is_empty(), "{src:?}: the checker reports {lexical:?}"); + let theirs: Vec<(String, usize)> = report.directives.iter().map(|d| (d.name.to_string(), d.line)).collect(); + let ours: Vec<(String, usize)> = program.directives.iter().map(|d| (d.name.to_lowercase(), d.line as usize)).collect(); + assert_eq!(ours, theirs, "directives of {src:?}"); + } +} + +#[test] +fn both_lexers_find_the_same_unclosed_token_on_the_same_line() { + for (src, code, line) in INVALID { + let report = check("script.jsx", src.as_bytes(), None, &Limits::default()); + let theirs = report.problems.iter().find(|p| LEXICAL.iter().any(|(c, _)| *c == p.code)); + let theirs = theirs.unwrap_or_else(|| panic!("{src:?}: the checker reports no lexical problem")); + assert_eq!((theirs.code, theirs.line), (*code, Some(*line)), "{src:?}"); + let ours = parse(src, &js_options()).expect_err(src); + let message = LEXICAL.iter().find(|(c, _)| c == code).map(|(_, m)| *m).unwrap(); + assert!(ours.message.contains(message), "{src:?}: {}", ours.message); + assert_eq!(ours.line as usize, *line, "{src:?}"); + } +} + +#[test] +fn the_parser_reads_a_regexp_wherever_the_checker_does() { + // Each `/` that divides has no second `/` after it on its line, so a lexer that read one as a + // regular expression would report it as never closed. The checker reports nothing, and the + // parser reads regular expressions on lines 2 and 3 only. + let src = "a = b / c;\nr = /x/;\nif (a) /y/.test(b);\nz = (1) / 2;\n"; + let tokens = tokenize(src, &js_options()).unwrap(); + let lines: Vec = tokens.iter().filter(|t| matches!(t.kind, TokenKind::RegExp { .. })).map(|t| t.line).collect(); + assert_eq!(lines, [2, 3]); + let report = check("script.jsx", src.as_bytes(), None, &Limits::default()); + assert!(report.problems.iter().all(|p| !LEXICAL.iter().any(|(c, _)| *c == p.code)), "{:?}", report.problems); +} diff --git a/crates/engine/tests/it/main.rs b/crates/engine/tests/it/main.rs index a6f0b02c7..3689d9cae 100644 --- a/crates/engine/tests/it/main.rs +++ b/crates/engine/tests/it/main.rs @@ -18,6 +18,7 @@ mod dimensions; mod eyedropper_sources; mod find_replace_paras; mod inline_text; +mod js_lexer_vs_scriptcheck; mod known_bugs; mod lossy_overwrite; mod model_based; diff --git a/crates/js/Cargo.toml b/crates/js/Cargo.toml new file mode 100644 index 000000000..f39ab33b6 --- /dev/null +++ b/crates/js/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "vectorcraft-js" +description = "JavaScript for scripts: an ECMAScript 5.1 lexer and parser to a syntax tree with source positions" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/js/src/ast.rs b/crates/js/src/ast.rs new file mode 100644 index 000000000..f0dd6b44b --- /dev/null +++ b/crates/js/src/ast.rs @@ -0,0 +1,334 @@ +//! The syntax tree: expressions, statements and functions kept in arenas ([`Ast`]) and referred to +//! by index, each with its source span. Nodes hold indices rather than boxes, so dropping a deeply +//! nested tree does not recurse. + +use crate::token::Span; + +/// An expression's index in [`Ast::exprs`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ExprId(pub u32); + +/// A statement's index in [`Ast::stmts`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct StmtId(pub u32); + +/// A function's index in [`Ast::functions`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FunctionId(pub u32); + +/// An identifier as written, with its span. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Name { + pub name: String, + pub span: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UnaryOp { + Delete, + Void, + Typeof, + Plus, + Minus, + BitNot, + Not, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UpdateOp { + Increment, + Decrement, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BinaryOp { + Mul, + Div, + Mod, + Add, + Sub, + Shl, + Shr, + UShr, + Lt, + Gt, + LtEq, + GtEq, + Instanceof, + In, + Eq, + NotEq, + StrictEq, + StrictNotEq, + BitAnd, + BitXor, + BitOr, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LogicalOp { + And, + Or, +} + +/// `=` (`None`) or a compound assignment's operator (`+=` is `Some(BinaryOp::Add)`). +pub type AssignOp = Option; + +/// A property name in an object literal. +#[derive(Clone, Debug, PartialEq)] +pub enum PropertyKey { + Ident(String), + String(Vec), + Number(f64), +} + +impl PropertyKey { + /// The key as the string a property is stored under (a number through ToString). + pub fn to_utf16(&self) -> Vec { + match self { + PropertyKey::Ident(name) => name.encode_utf16().collect(), + PropertyKey::String(s) => s.clone(), + PropertyKey::Number(n) => crate::number::to_string(*n).encode_utf16().collect(), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum PropertyValue { + Init(ExprId), + Get(FunctionId), + Set(FunctionId), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Property { + pub key: PropertyKey, + pub value: PropertyValue, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Expr { + This, + Ident(String), + Null, + Bool(bool), + Number(f64), + String(Vec), + RegExp { + pattern: Vec, + flags: String, + }, + /// An array literal; `None` is a hole (`[1, , 3]`). + Array(Vec>), + Object(Vec), + Function(FunctionId), + /// `object.property`. + Member { + object: ExprId, + property: Name, + }, + /// `object[index]`. + Index { + object: ExprId, + index: ExprId, + }, + Call { + callee: ExprId, + args: Vec, + }, + New { + callee: ExprId, + args: Vec, + }, + Update { + op: UpdateOp, + prefix: bool, + target: ExprId, + }, + Unary { + op: UnaryOp, + arg: ExprId, + }, + Binary { + op: BinaryOp, + left: ExprId, + right: ExprId, + }, + Logical { + op: LogicalOp, + left: ExprId, + right: ExprId, + }, + Conditional { + test: ExprId, + consequent: ExprId, + alternate: ExprId, + }, + Assign { + op: AssignOp, + target: ExprId, + value: ExprId, + }, + /// Comma-separated expressions. + Sequence(Vec), + /// A parenthesized expression, kept so a rule about the bare form (a label, a directive) can + /// tell them apart. + Paren(ExprId), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct VarDecl { + pub name: Name, + pub init: Option, +} + +/// What a `for (… in …)` statement assigns each property name to. +#[derive(Clone, Debug, PartialEq)] +pub enum ForInTarget { + /// `for (var x in o)`, with the declaration's initializer if it has one. + Var(VarDecl), + Expr(ExprId), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ForInit { + Var(Vec), + Expr(ExprId), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SwitchCase { + /// `None` for `default`. + pub test: Option, + pub body: Vec, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CatchClause { + pub param: Name, + pub body: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Stmt { + Var(Vec), + FunctionDecl(FunctionId), + Expr(ExprId), + Block(Vec), + Empty, + If { test: ExprId, consequent: StmtId, alternate: Option }, + DoWhile { body: StmtId, test: ExprId }, + While { test: ExprId, body: StmtId }, + For { init: Option, test: Option, update: Option, body: StmtId }, + ForIn { target: ForInTarget, object: ExprId, body: StmtId }, + Continue(Option), + Break(Option), + Return(Option), + With { object: ExprId, body: StmtId }, + Switch { discriminant: ExprId, cases: Vec }, + Labeled { label: Name, body: StmtId }, + Throw(ExprId), + Try { block: Vec, handler: Option, finalizer: Option> }, + Debugger, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FunctionKind { + Declaration, + Expression, + Getter, + Setter, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Function { + pub kind: FunctionKind, + pub name: Option, + pub params: Vec, + pub body: Vec, + /// The function's code is strict mode code: its own directive prologue holds `"use strict"`, + /// or the code around it is strict. + pub strict: bool, + pub span: Span, +} + +/// The arenas of a parsed script. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Ast { + pub exprs: Vec<(Expr, Span)>, + pub stmts: Vec<(Stmt, Span)>, + pub functions: Vec, +} + +impl Ast { + pub fn expr(&self, id: ExprId) -> Option<&Expr> { + self.exprs.get(id.0 as usize).map(|(e, _)| e) + } + + pub fn expr_span(&self, id: ExprId) -> Span { + self.exprs.get(id.0 as usize).map(|(_, s)| *s).unwrap_or_default() + } + + pub fn stmt(&self, id: StmtId) -> Option<&Stmt> { + self.stmts.get(id.0 as usize).map(|(s, _)| s) + } + + pub fn stmt_span(&self, id: StmtId) -> Span { + self.stmts.get(id.0 as usize).map(|(_, s)| *s).unwrap_or_default() + } + + pub fn function(&self, id: FunctionId) -> Option<&Function> { + self.functions.get(id.0 as usize) + } + + /// The expression with any parentheses around it removed. + pub fn unparenthesized(&self, mut id: ExprId) -> ExprId { + while let Some(Expr::Paren(inner)) = self.expr(id) { + id = *inner; + } + id + } + + pub(crate) fn push_expr(&mut self, expr: Expr, span: Span) -> ExprId { + self.exprs.push((expr, span)); + ExprId(u32::try_from(self.exprs.len() - 1).unwrap_or(u32::MAX)) + } + + pub(crate) fn push_stmt(&mut self, stmt: Stmt, span: Span) -> StmtId { + self.stmts.push((stmt, span)); + StmtId(u32::try_from(self.stmts.len() - 1).unwrap_or(u32::MAX)) + } + + pub(crate) fn push_function(&mut self, function: Function) -> FunctionId { + self.functions.push(function); + FunctionId(u32::try_from(self.functions.len() - 1).unwrap_or(u32::MAX)) + } +} + +/// A parsed script. +#[derive(Clone, Debug, PartialEq)] +pub struct Program { + pub ast: Ast, + pub body: Vec, + /// The script is strict mode code (its directive prologue holds `"use strict"`, or it was + /// parsed with [`crate::Options::strict`]). + pub strict: bool, + /// Its preprocessor lines, in source order. + pub directives: Vec, + pub(crate) lines: crate::LineTable, +} + +impl Program { + /// The line and column (both from 1, the column in UTF-16 code units) of a source offset. + pub fn line_column(&self, offset: u32) -> (u32, u32) { + self.lines.line_column(offset) + } + + /// The number of lines in the source: its line terminators plus one for a last line without + /// one. + pub fn line_count(&self) -> u32 { + self.lines.line_count() + } +} diff --git a/crates/js/src/chars.rs b/crates/js/src/chars.rs new file mode 100644 index 000000000..fb6da4f36 --- /dev/null +++ b/crates/js/src/chars.rs @@ -0,0 +1,99 @@ +//! Character classes of ECMA-262 5.1 section 7: white space, line terminators and the characters +//! identifiers are made of. + +/// A line terminator: LF, CR, U+2028 or U+2029 (section 7.3). +pub fn is_line_terminator(c: u32) -> bool { + matches!(c, 0x0A | 0x0D | 0x2028 | 0x2029) +} + +/// White space (section 7.2): tab, vertical tab, form feed, space, no-break space, the byte order +/// mark and the other space separators (Unicode category Zs). +pub fn is_whitespace(c: u32) -> bool { + matches!(c, 0x09 | 0x0B | 0x0C | 0x20 | 0xA0 | 0xFEFF | 0x1680 | 0x2000..=0x200A | 0x202F | 0x205F | 0x3000) +} + +/// A character an identifier may start with: a letter (categories Lu, Ll, Lt, Lm, Lo and Nl), +/// `$` or `_` (section 7.6). `\u` escapes are decoded before this check. +pub fn is_id_start(c: u32) -> bool { + match c { + 0x24 | 0x5F => true, + 0x41..=0x5A | 0x61..=0x7A => true, + 0..=0x7F => false, + // U+2E2F VERTICAL TILDE is a letter (Lm) that Unicode leaves out of identifiers. + 0x2E2F => false, + _ => char::from_u32(c).is_some_and(|ch| ch.is_alphabetic() && !is_combining_mark(c)), + } +} + +/// A character an identifier may hold after its first: those it may start with, decimal digits +/// (Nd), combining marks (Mn, Mc), connector punctuation (Pc), and the zero width non-joiner and +/// joiner. +pub fn is_id_part(c: u32) -> bool { + match c { + 0x30..=0x39 => true, + 0x200C | 0x200D => true, + 0x2E2F => false, + _ if is_id_start(c) => true, + 0..=0x7F => false, + _ => char::from_u32(c).is_some_and(|ch| ch.is_alphanumeric()) || is_combining_mark(c) || is_connector(c), + } +} + +/// The blocks of combining marks (categories Mn and Mc) that scripts are most likely to use. +fn is_combining_mark(c: u32) -> bool { + matches!( + c, + 0x0300..=0x036F + | 0x0483..=0x0487 + | 0x0591..=0x05BD + | 0x05BF + | 0x05C1..=0x05C2 + | 0x05C4..=0x05C5 + | 0x05C7 + | 0x0610..=0x061A + | 0x064B..=0x065F + | 0x0670 + | 0x06D6..=0x06DC + | 0x06DF..=0x06E4 + | 0x06E7..=0x06E8 + | 0x06EA..=0x06ED + | 0x0900..=0x0903 + | 0x093A..=0x093C + | 0x093E..=0x094F + | 0x0951..=0x0957 + | 0x0962..=0x0963 + | 0x0E31 + | 0x0E34..=0x0E3A + | 0x0E47..=0x0E4E + | 0x1AB0..=0x1AFF + | 0x1DC0..=0x1DFF + | 0x20D0..=0x20FF + | 0x302A..=0x302F + | 0x3099..=0x309A + | 0xFE00..=0xFE0F + | 0xFE20..=0xFE2F + ) +} + +/// Connector punctuation (category Pc) other than `_`. +fn is_connector(c: u32) -> bool { + matches!(c, 0x203F | 0x2040 | 0x2054 | 0xFE33 | 0xFE34 | 0xFE4D..=0xFE4F | 0xFF3F) +} + +/// The value of a hexadecimal digit. +pub fn hex_value(c: u32) -> Option { + match c { + 0x30..=0x39 => Some(c - 0x30), + 0x41..=0x46 => Some(c - 0x41 + 10), + 0x61..=0x66 => Some(c - 0x61 + 10), + _ => None, + } +} + +pub fn is_decimal_digit(c: u32) -> bool { + (0x30..=0x39).contains(&c) +} + +pub fn is_octal_digit(c: u32) -> bool { + (0x30..=0x37).contains(&c) +} diff --git a/crates/js/src/error.rs b/crates/js/src/error.rs new file mode 100644 index 000000000..9c17bf8be --- /dev/null +++ b/crates/js/src/error.rs @@ -0,0 +1,29 @@ +//! The error a script that cannot be read gives: a `SyntaxError` with its message and where in the +//! source it is. + +use std::fmt; + +/// A script that does not follow the grammar, or breaks one of its early-error rules. `line` and +/// `column` count from 1; the column counts UTF-16 code units from the start of the line, as +/// JavaScript source positions do. `offset` is the UTF-16 offset from the start of the source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SyntaxError { + pub message: String, + pub line: u32, + pub column: u32, + pub offset: u32, +} + +impl fmt::Display for SyntaxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SyntaxError: {} (line {}, column {})", self.message, self.line, self.column) + } +} + +impl std::error::Error for SyntaxError {} + +/// The message of the error nesting deeper than [`crate::Options::max_depth`] gives. +pub const TOO_DEEP: &str = "nesting too deep"; + +/// The message of the error a source longer than [`crate::Options::max_source_units`] gives. +pub const TOO_LONG: &str = "source too long"; diff --git a/crates/js/src/lexer.rs b/crates/js/src/lexer.rs new file mode 100644 index 000000000..9b16b16c7 --- /dev/null +++ b/crates/js/src/lexer.rs @@ -0,0 +1,637 @@ +//! The lexer (ECMA-262 5.1 section 7): source text in UTF-16 code units into tokens, with each +//! token's line and column and whether a line terminator comes before it. +//! +//! A `/` starts a division punctuator or a regular expression literal depending on where the +//! grammar is (section 7, the InputElementDiv and InputElementRegExp goals). The lexer reads a +//! token for the goal it is given; the parser reads tokens for the division goal and asks for a +//! token to be read again as a regular expression where an expression starts +//! ([`Lexer::rescan_regexp`]). +//! +//! With [`crate::Options::preprocessor_lines`], a `#name value` line, or a `//@name value` comment +//! that names a directive, at the start of a line (only blanks before it) gives a +//! [`TokenKind::Directive`] token. +//! +//! The lexer moves forward through the source one character at a time and never recurses. + +use crate::chars::{hex_value, is_decimal_digit, is_id_part, is_id_start, is_line_terminator, is_octal_digit, is_whitespace}; +use crate::error::SyntaxError; +use crate::regexp; +use crate::token::{Directive, DirectiveName, Keyword, Punct, Span, Token, TokenKind}; + +/// Which goal symbol the next token is read for: whether a `/` there divides or starts a regular +/// expression literal. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Goal { + Div, + RegExp, +} + +const BACKSLASH: u32 = b'\\' as u32; +const SLASH: u32 = b'/' as u32; +const STAR: u32 = b'*' as u32; +const DOT: u32 = b'.' as u32; +const CR: u32 = 0x0D; +const LF: u32 = 0x0A; + +/// The lexer's state. It is small and cheap to clone, which is how the parser looks one token +/// ahead. +#[derive(Clone, Debug)] +pub struct Lexer<'s> { + src: &'s [u16], + pos: usize, + /// The current line (from 1) and the offset where it starts. + line: u32, + line_start: usize, + /// Only blanks since the start of the line: a preprocessor line may start here. + blank_line: bool, + preprocessor_lines: bool, + /// The deepest nesting of groups a regular expression literal may have. + max_depth: usize, +} + +impl<'s> Lexer<'s> { + /// A lexer of `src`. `preprocessor_lines` turns on the reading of preprocessor lines; + /// `max_depth` limits how deeply a regular expression literal's groups may nest. + pub fn new(src: &'s [u16], preprocessor_lines: bool, max_depth: usize) -> Self { + Self { src, pos: 0, line: 1, line_start: 0, blank_line: true, preprocessor_lines, max_depth } + } + + /// The code unit at `pos`, as a code point value. + fn unit(&self, pos: usize) -> Option { + self.src.get(pos).map(|&u| u32::from(u)) + } + + fn peek(&self) -> Option { + self.unit(self.pos) + } + + fn peek_at(&self, n: usize) -> Option { + self.unit(self.pos + n) + } + + /// The code point at `pos` (a surrogate pair read as one) and how many units it takes. + fn code_point(&self, pos: usize) -> Option<(u32, usize)> { + let hi = self.unit(pos)?; + if (0xD800..0xDC00).contains(&hi) + && let Some(lo) = self.unit(pos + 1).filter(|lo| (0xDC00..0xE000).contains(lo)) + { + return Some((0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00), 2)); + } + Some((hi, 1)) + } + + fn column_of(&self, pos: usize) -> u32 { + u32::try_from(pos.saturating_sub(self.line_start) + 1).unwrap_or(u32::MAX) + } + + fn offset(pos: usize) -> u32 { + u32::try_from(pos).unwrap_or(u32::MAX) + } + + /// An error at `pos` on the current line. + fn error_at(&self, pos: usize, message: impl Into) -> SyntaxError { + SyntaxError { message: message.into(), line: self.line, column: self.column_of(pos), offset: Self::offset(pos) } + } + + /// An error at a position recorded earlier. + fn error_from(start: (usize, u32, u32), message: impl Into) -> SyntaxError { + SyntaxError { message: message.into(), line: start.1, column: start.2, offset: Self::offset(start.0) } + } + + /// Reads a line terminator at `pos` (CR LF as one), counting the line. False when there is + /// none. + fn line_terminator(&mut self) -> bool { + match self.peek() { + Some(CR) if self.peek_at(1) == Some(LF) => self.pos += 2, + Some(c) if is_line_terminator(c) => self.pos += 1, + _ => return false, + } + self.line = self.line.saturating_add(1); + self.line_start = self.pos; + self.blank_line = true; + true + } + + /// The next token, read for `goal`. + pub fn next_token(&mut self, goal: Goal) -> Result { + let newline_before = match self.skip_trivia()? { + Trivia::Directive(token) => return Ok(*token), + Trivia::Skipped { newline } => newline, + }; + let start = self.pos; + let (line, column) = (self.line, self.column_of(start)); + let Some(c) = self.peek() else { + return Ok(self.token(TokenKind::Eof, start, line, column, newline_before)); + }; + self.blank_line = false; + let mut token = if c == BACKSLASH || is_id_start(c) || self.code_point(start).is_some_and(|(cp, n)| n == 2 && is_id_start(cp)) { + self.identifier(start, line, column)? + } else if is_decimal_digit(c) || (c == DOT && self.peek_at(1).is_some_and(is_decimal_digit)) { + self.number(start, line, column)? + } else if c == u32::from(b'"') || c == u32::from(b'\'') { + self.string(start, line, column)? + } else if c == SLASH && goal == Goal::RegExp { + self.regexp(start, line, column)? + } else { + let punct = self.punct().ok_or_else(|| self.error_at(start, unexpected_character(c)))?; + self.token(TokenKind::Punct(punct), start, line, column, false) + }; + // A string continued over a line ends on a new line, but not at its start. + self.blank_line = false; + token.newline_before = newline_before; + Ok(token) + } + + /// Reads the token `token` again as a regular expression literal: the parser read a `/` or + /// `/=` where an expression starts. + pub fn rescan_regexp(&mut self, token: &Token) -> Result { + let start = usize::try_from(token.span.start).unwrap_or(usize::MAX); + self.pos = start; + self.line = token.line; + self.line_start = start.saturating_sub(usize::try_from(token.column.saturating_sub(1)).unwrap_or(0)); + self.blank_line = false; + let mut regexp = self.regexp(start, token.line, token.column)?; + regexp.newline_before = token.newline_before; + Ok(regexp) + } + + fn token(&self, kind: TokenKind, start: usize, line: u32, column: u32, newline_before: bool) -> Token { + Token { + kind, + span: Span::new(Self::offset(start), Self::offset(self.pos)), + line, + column, + newline_before, + escaped: false, + legacy_octal: false, + } + } + + /// Skips white space, line terminators and comments, and reads a preprocessor line when one + /// starts here. + fn skip_trivia(&mut self) -> Result { + let mut newline = false; + while let Some(c) = self.peek() { + if is_whitespace(c) { + self.pos += 1; + } else if self.line_terminator() { + newline = true; + } else if c == SLASH && self.peek_at(1) == Some(SLASH) { + if self.blank_line && self.preprocessor_lines && self.names_directive(self.pos + 2) { + return Ok(Trivia::Directive(Box::new(self.directive("//@", newline)))); + } + while self.peek().is_some_and(|c| !is_line_terminator(c)) { + self.pos += 1; + } + } else if c == SLASH && self.peek_at(1) == Some(STAR) { + newline |= self.block_comment()?; + // The comment's end is on this line, so a `#` after it is not at the line's start. + self.blank_line = false; + } else if c == u32::from(b'#') && self.blank_line && self.preprocessor_lines && self.peek_at(1).is_some_and(is_name_char) { + return Ok(Trivia::Directive(Box::new(self.directive("#", newline)))); + } else { + break; + } + } + Ok(Trivia::Skipped { newline }) + } + + /// Skips a `/* */` comment; true when it holds a line terminator. + fn block_comment(&mut self) -> Result { + let start = (self.pos, self.line, self.column_of(self.pos)); + self.pos += 2; + let mut newline = false; + loop { + match self.peek() { + None => return Err(Self::error_from(start, "unterminated comment")), + Some(STAR) if self.peek_at(1) == Some(SLASH) => { + self.pos += 2; + return Ok(newline); + } + Some(_) if self.line_terminator() => newline = true, + Some(_) => self.pos += 1, + } + } + } + + /// `//@` at `pos - 2` is followed by a directive's name. + fn names_directive(&self, pos: usize) -> bool { + if self.unit(pos) != Some(u32::from(b'@')) { + return false; + } + let name: String = self.src[pos + 1..].iter().take_while(|&&u| is_name_char(u32::from(u))).map(|&u| char::from(u as u8)).collect(); + DirectiveName::from_name(&name).is_some() + } + + /// Reads a preprocessor line written with `form` (`#` or `//@`), up to the end of the line. + fn directive(&mut self, form: &'static str, newline_before: bool) -> Token { + let start = self.pos; + let (line, column) = (self.line, self.column_of(start)); + self.pos += form.encode_utf16().count(); + let name_start = self.pos; + while self.peek().is_some_and(is_name_char) { + self.pos += 1; + } + let name = String::from_utf16_lossy(&self.src[name_start..self.pos]); + let value_start = self.pos; + while self.peek().is_some_and(|c| !is_line_terminator(c)) { + self.pos += 1; + } + let value = String::from_utf16_lossy(&self.src[value_start..self.pos]).trim().to_string(); + let directive = Directive { + known: DirectiveName::from_name(&name), + name, + value, + form, + span: Span::new(Self::offset(start), Self::offset(self.pos)), + line, + }; + self.blank_line = false; + self.token(TokenKind::Directive(directive), start, line, column, newline_before) + } + + /// Reads an identifier or a keyword (section 7.6), `\u` escapes included. + fn identifier(&mut self, start: usize, line: u32, column: u32) -> Result { + let mut name = String::new(); + let mut escaped = false; + let mut first = true; + loop { + let at = self.pos; + let (c, len, is_escape) = match self.code_point(at) { + Some((BACKSLASH, _)) => { + let c = self.identifier_escape()?; + (c, 0, true) + } + Some((c, len)) => (c, len, false), + None => break, + }; + let ok = if first { is_id_start(c) } else { is_id_part(c) }; + if !ok { + if is_escape { + return Err(self.error_at(at, "invalid character in identifier escape")); + } + break; + } + self.pos += len; + escaped |= is_escape; + first = false; + match char::from_u32(c) { + Some(ch) => name.push(ch), + None => return Err(self.error_at(at, "invalid character in identifier")), + } + } + let kind = match Keyword::from_word(&name) { + Some(k) if !escaped => TokenKind::Keyword(k), + _ => TokenKind::Ident(name), + }; + let mut token = self.token(kind, start, line, column, false); + token.escaped = escaped; + Ok(token) + } + + /// Reads a `\uXXXX` escape in an identifier and returns its code point. + fn identifier_escape(&mut self) -> Result { + let at = self.pos; + if self.peek_at(1) != Some(u32::from(b'u')) { + return Err(self.error_at(at, "invalid escape in identifier")); + } + self.pos += 2; + self.unicode_escape().ok_or_else(|| self.error_at(at, "invalid Unicode escape sequence")) + } + + /// After `\u`: four hexadecimal digits, or a code point in braces (`\u{1F600}`, ECMA-262 2015 + /// section 11.8.4) up to U+10FFFF. + fn unicode_escape(&mut self) -> Option { + if self.peek() != Some(u32::from(b'{')) { + return self.hex_digits(4); + } + let mut n = 1; + let mut value: u32 = 0; + while let Some(d) = self.peek_at(n).and_then(hex_value) { + value = value.checked_mul(16)?.checked_add(d).filter(|&v| v <= 0x10FFFF)?; + n += 1; + } + if n == 1 || self.peek_at(n) != Some(u32::from(b'}')) { + return None; + } + self.pos += n + 1; + Some(value) + } + + /// Reads `n` hexadecimal digits and returns their value, or `None` (leaving the position) when + /// fewer follow. + fn hex_digits(&mut self, n: usize) -> Option { + let mut value = 0; + for i in 0..n { + value = value * 16 + hex_value(self.peek_at(i)?)?; + } + self.pos += n; + Some(value) + } + + /// Reads a numeric literal (section 7.8.3, with the legacy octal literals of Annex B). + fn number(&mut self, start: usize, line: u32, column: u32) -> Result { + let mut legacy_octal = false; + let value = if self.peek() == Some(u32::from(b'0')) && self.peek_at(1).is_some_and(|c| c == u32::from(b'x') || c == u32::from(b'X')) { + self.pos += 2; + self.radix_digits(16).ok_or_else(|| self.error_at(start, "hexadecimal literal without digits"))? + } else if self.peek() == Some(u32::from(b'0')) && self.peek_at(1).is_some_and(is_decimal_digit) { + legacy_octal = true; + self.legacy_octal_or_decimal() + } else { + self.decimal() + }; + if self.peek().is_some_and(|c| c == BACKSLASH || is_id_start(c) || is_decimal_digit(c)) { + return Err(self.error_at(self.pos, "identifier starts immediately after numeric literal")); + } + let mut token = self.token(TokenKind::Number(value), start, line, column, false); + token.legacy_octal = legacy_octal; + Ok(token) + } + + /// Reads digits in `radix` (8 or 16) and returns their value, or `None` when there are none. + fn radix_digits(&mut self, radix: u32) -> Option { + let begin = self.pos; + let mut exact: u128 = 0; + let mut value = 0.0_f64; + let mut overflow = false; + while let Some(d) = self.peek().and_then(hex_value).filter(|&d| d < radix) { + match exact.checked_mul(u128::from(radix)).and_then(|v| v.checked_add(u128::from(d))) { + Some(v) if !overflow => exact = v, + _ => { + if !overflow { + value = exact as f64; + overflow = true; + } + value = value * f64::from(radix) + f64::from(d); + } + } + self.pos += 1; + } + if self.pos == begin { + return None; + } + Some(if overflow { value } else { exact as f64 }) + } + + /// After `0` and a digit: an octal literal when every digit is octal, else a decimal literal + /// whose integer part starts with `0` (`08`, `09.5`). + fn legacy_octal_or_decimal(&mut self) -> f64 { + let begin = self.pos; + let mut end = begin; + while self.unit(end).is_some_and(is_decimal_digit) { + end += 1; + } + if self.src[begin..end].iter().all(|&u| is_octal_digit(u32::from(u))) { + self.pos += 1; + return self.radix_digits(8).unwrap_or(0.0); + } + self.decimal() + } + + /// Reads a decimal literal: digits, an optional fraction and an optional exponent. + fn decimal(&mut self) -> f64 { + let mut text = String::new(); + self.push_digits(&mut text); + if text.is_empty() { + text.push('0'); + } + if self.peek() == Some(DOT) { + self.pos += 1; + text.push('.'); + let before = text.len(); + self.push_digits(&mut text); + if text.len() == before { + text.push('0'); + } + } + if self.peek().is_some_and(|c| c == u32::from(b'e') || c == u32::from(b'E')) { + let sign = self.peek_at(1).filter(|&c| c == u32::from(b'+') || c == u32::from(b'-')); + let digit_at = if sign.is_some() { 2 } else { 1 }; + if self.peek_at(digit_at).is_some_and(is_decimal_digit) { + self.pos += digit_at; + text.push('e'); + if sign == Some(u32::from(b'-')) { + text.push('-'); + } + self.push_digits(&mut text); + } + } + text.parse().unwrap_or(f64::NAN) + } + + fn push_digits(&mut self, text: &mut String) { + while let Some(c) = self.peek().filter(|&c| is_decimal_digit(c)) { + text.push(char::from(c as u8)); + self.pos += 1; + } + } + + /// Reads a string literal (section 7.8.4, with the octal escapes of Annex B) and returns its + /// value. + fn string(&mut self, start: usize, line: u32, column: u32) -> Result { + let quote = self.peek().unwrap_or_default(); + self.pos += 1; + let mut value = Vec::new(); + let mut legacy_octal = false; + loop { + match self.peek() { + None => return Err(Self::error_from((start, line, column), "unterminated string literal")), + // U+2028 and U+2029 may stand in a string, as ECMA-262 2019 allows; they still + // start a new line for positions. + Some(c @ (0x2028 | 0x2029)) => { + value.push(c as u16); + self.line_terminator(); + } + Some(c) if is_line_terminator(c) => { + return Err(Self::error_from((start, line, column), "unterminated string literal")); + } + Some(c) if c == quote => { + self.pos += 1; + break; + } + Some(BACKSLASH) => legacy_octal |= self.string_escape(&mut value)?, + Some(c) => { + value.push(c as u16); + self.pos += 1; + } + } + } + let mut token = self.token(TokenKind::String(value), start, line, column, false); + token.legacy_octal = legacy_octal; + Ok(token) + } + + /// Reads an escape in a string literal into `value`; true for an octal escape (`\1`, `\08`) + /// or `\8` and `\9`. + fn string_escape(&mut self, value: &mut Vec) -> Result { + let at = self.pos; + self.pos += 1; + let Some(c) = self.peek() else { + return Err(self.error_at(at, "unterminated string literal")); + }; + if self.line_terminator() { + return Ok(false); + } + let single = match char::from_u32(c) { + Some('b') => Some(0x08), + Some('t') => Some(0x09), + Some('n') => Some(0x0A), + Some('v') => Some(0x0B), + Some('f') => Some(0x0C), + Some('r') => Some(0x0D), + _ => None, + }; + if let Some(unit) = single { + value.push(unit); + self.pos += 1; + return Ok(false); + } + match char::from_u32(c) { + Some('x') => { + self.pos += 1; + let unit = self.hex_digits(2).ok_or_else(|| self.error_at(at, "invalid hexadecimal escape sequence"))?; + value.push(unit as u16); + Ok(false) + } + Some('u') => { + self.pos += 1; + let code_point = self.unicode_escape().ok_or_else(|| self.error_at(at, "invalid Unicode escape sequence"))?; + match char::from_u32(code_point) { + Some(ch) if code_point > 0xFFFF => value.extend(ch.encode_utf16(&mut [0; 2]).iter()), + _ => value.push(code_point as u16), + } + Ok(false) + } + Some('0') if !self.peek_at(1).is_some_and(is_decimal_digit) => { + self.pos += 1; + value.push(0); + Ok(false) + } + Some('0'..='7') => { + value.push(self.octal_escape(c)); + Ok(true) + } + Some('8' | '9') => { + self.pos += 1; + value.push(c as u16); + Ok(true) + } + _ => { + self.pos += 1; + value.push(c as u16); + Ok(false) + } + } + } + + /// Reads an octal escape whose first digit is `first`: up to three digits when it is 0 to 3, + /// up to two when it is 4 to 7. + fn octal_escape(&mut self, first: u32) -> u16 { + let max = if first <= u32::from(b'3') { 3 } else { 2 }; + let mut value = 0; + let mut n = 0; + while n < max + && let Some(d) = self.peek().filter(|&d| is_octal_digit(d)) + { + value = value * 8 + (d - u32::from(b'0')); + self.pos += 1; + n += 1; + } + value as u16 + } + + /// Reads a regular expression literal (section 7.8.5) and checks its flags and pattern. + fn regexp(&mut self, start: usize, line: u32, column: u32) -> Result { + let at = (start, line, column); + self.pos = start + 1; + let mut in_class = false; + loop { + match self.peek() { + None => return Err(Self::error_from(at, "unterminated regular expression literal")), + Some(c) if is_line_terminator(c) => return Err(Self::error_from(at, "unterminated regular expression literal")), + Some(BACKSLASH) => { + if self.peek_at(1).is_none_or(is_line_terminator) { + return Err(Self::error_from(at, "unterminated regular expression literal")); + } + self.pos += 2; + } + Some(c) if c == u32::from(b'[') => { + in_class = true; + self.pos += 1; + } + Some(c) if c == u32::from(b']') => { + in_class = false; + self.pos += 1; + } + Some(SLASH) if !in_class => break, + Some(_) => self.pos += 1, + } + } + let pattern = self.src[start + 1..self.pos].to_vec(); + self.pos += 1; + let mut flags = String::new(); + while let Some((c, len)) = self.code_point(self.pos) { + if c == BACKSLASH { + return Err(Self::error_from(at, "invalid regular expression flags")); + } + if !is_id_part(c) { + break; + } + flags.push(char::from_u32(c).unwrap_or('\u{fffd}')); + self.pos += len; + } + if !valid_flags(&flags) { + return Err(Self::error_from(at, "invalid regular expression flags")); + } + if let Err(message) = regexp::check(&pattern, self.max_depth) { + return Err(Self::error_from(at, format!("invalid regular expression: {message}"))); + } + Ok(self.token(TokenKind::RegExp { pattern, flags }, start, line, column, false)) + } + + /// Reads the longest punctuator that starts here. + fn punct(&mut self) -> Option { + let rest = &self.src[self.pos..]; + for (spelling, punct) in Punct::ALL { + let bytes = spelling.as_bytes(); + if rest.len() >= bytes.len() && rest.iter().zip(bytes).all(|(&u, &b)| u == u16::from(b)) { + self.pos += bytes.len(); + return Some(*punct); + } + } + None + } +} + +/// What skipping the blanks and comments before a token found. +enum Trivia { + Skipped { newline: bool }, + Directive(Box), +} + +/// A character of a directive's name. +fn is_name_char(c: u32) -> bool { + c < 0x80 && (char::from(c as u8).is_ascii_alphanumeric() || c == u32::from(b'_')) +} + +/// The flags `g`, `i` and `m`, each at most once (section 15.10.4.1). +fn valid_flags(flags: &str) -> bool { + let mut seen = [false; 3]; + flags.chars().all(|c| { + let i = match c { + 'g' => 0, + 'i' => 1, + 'm' => 2, + _ => return false, + }; + !std::mem::replace(&mut seen[i], true) + }) +} + +fn unexpected_character(c: u32) -> String { + match char::from_u32(c) { + Some(ch) if !ch.is_control() => format!("unexpected character '{ch}'"), + _ => format!("unexpected character U+{c:04X}"), + } +} diff --git a/crates/js/src/lib.rs b/crates/js/src/lib.rs new file mode 100644 index 000000000..4d8d32271 --- /dev/null +++ b/crates/js/src/lib.rs @@ -0,0 +1,149 @@ +//! JavaScript for VectorCraft's scripts: an ECMAScript 5.1 lexer and parser +//! (), with the legacy forms of Annex B that older scripts +//! use (octal literals and escapes outside strict mode, function declarations in blocks, the +//! lenient regular expression patterns). +//! +//! [`parse`] reads a script into a [`Program`]: a syntax tree whose expressions, statements and +//! functions live in arenas and carry their source spans. A script that breaks the grammar or one +//! of its early-error rules (section 16: strict mode's restrictions, duplicate parameters in strict +//! code, invalid assignment targets, `break` and `continue` without a target, regular expression +//! literals that are not valid) gives a [`SyntaxError`] with its line and column. +//! +//! With [`Options::preprocessor_lines`], the lexer also reads the preprocessor lines scripts begin +//! with (`#include`, `#includepath`, `#script`, `#strict`, `#target`, `#targetengine`) into +//! [`Directive`]s. They are kept in [`Program::directives`] and never run. +//! +//! The lexer reads the source as UTF-16 code units, as JavaScript does: columns and spans count +//! code units. Nesting is counted: input nested deeper than [`Options::max_depth`] gives a +//! `SyntaxError` ("nesting too deep") instead of exhausting the stack, binary operators, `if` … +//! `else if` chains, labels and prefix operators are read with loops, and the syntax tree is +//! dropped without recursion. +#![forbid(unsafe_code)] + +pub mod ast; +mod chars; +mod error; +mod lexer; +mod number; +mod parser; +mod regexp; +mod token; + +pub use ast::{ + AssignOp, Ast, BinaryOp, CatchClause, Expr, ExprId, ForInTarget, ForInit, Function, FunctionId, FunctionKind, LogicalOp, Name, Program, Property, + PropertyKey, PropertyValue, Stmt, StmtId, SwitchCase, UnaryOp, UpdateOp, VarDecl, +}; +pub use error::{SyntaxError, TOO_DEEP, TOO_LONG}; +pub use lexer::{Goal, Lexer}; +pub use number::to_string as number_to_string; +pub use token::{Directive, DirectiveName, Keyword, Punct, Span, Token, TokenKind}; + +/// How a script is read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Options { + /// The script is strict mode code from its start, as code passed to a strict `eval` is. + pub strict: bool, + /// Read preprocessor lines (`#target …` and the other directives) into [`Directive`]s. Off, + /// a `#` is an unexpected character, as ECMA-262 has it. + pub preprocessor_lines: bool, + /// How deeply statements, expressions, functions and regular expression groups may nest. + pub max_depth: usize, + /// The most UTF-16 code units a source may hold. + pub max_source_units: usize, +} + +impl Default for Options { + fn default() -> Self { + Self { strict: false, preprocessor_lines: false, max_depth: DEFAULT_MAX_DEPTH, max_source_units: 64 << 20 } + } +} + +/// The default [`Options::max_depth`]: deeper than scripts nest, and shallow enough for the stack +/// of a wasm build or a secondary thread. +pub const DEFAULT_MAX_DEPTH: usize = 200; + +/// Parses `source` as a script. +pub fn parse(source: &str, options: &Options) -> Result { + let units = to_utf16(source, options)?; + parse_utf16(&units, options) +} + +/// Parses `source`, in UTF-16 code units, as a script. +pub fn parse_utf16(source: &[u16], options: &Options) -> Result { + check_length(source.len(), options)?; + parser::Parser::new(source, options, false)?.parse_program().map(|(program, _)| program) +} + +/// The tokens of `source`, in order, as the parser reads them: preprocessor lines included, and +/// each `/` read as division or as the start of a regular expression literal as the grammar +/// decides. The source must be a valid script. +pub fn tokenize(source: &str, options: &Options) -> Result, SyntaxError> { + let units = to_utf16(source, options)?; + check_length(units.len(), options)?; + parser::Parser::new(&units, options, true)?.parse_program().map(|(_, tokens)| tokens) +} + +fn to_utf16(source: &str, options: &Options) -> Result, SyntaxError> { + // A source's UTF-16 length is at most its UTF-8 length; count only a source that may be too + // long. + if source.len() > options.max_source_units { + check_length(source.encode_utf16().count(), options)?; + } + Ok(source.encode_utf16().collect()) +} + +fn check_length(units: usize, options: &Options) -> Result<(), SyntaxError> { + if units > options.max_source_units || u32::try_from(units).is_err() { + return Err(SyntaxError { message: TOO_LONG.into(), line: 1, column: 1, offset: 0 }); + } + Ok(()) +} + +/// Where each line of a source starts, for turning offsets into lines and columns. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct LineTable { + starts: Vec, + /// The source ends with a line terminator. + ends_with_break: bool, + empty: bool, +} + +impl LineTable { + pub(crate) fn new(src: &[u16]) -> Self { + let mut starts = vec![0]; + let mut i = 0; + while let Some(&u) = src.get(i) { + i += 1; + let terminator = match u { + 0x0D if src.get(i) == Some(&0x0A) => { + i += 1; + true + } + 0x0A | 0x0D | 0x2028 | 0x2029 => true, + _ => false, + }; + if terminator { + starts.push(u32::try_from(i).unwrap_or(u32::MAX)); + } + } + let ends_with_break = src.last().is_some_and(|&u| matches!(u, 0x0A | 0x0D | 0x2028 | 0x2029)); + Self { starts, ends_with_break, empty: src.is_empty() } + } + + pub(crate) fn line_column(&self, offset: u32) -> (u32, u32) { + let line = self.starts.partition_point(|&s| s <= offset).max(1); + let start = self.starts.get(line - 1).copied().unwrap_or(0); + (u32::try_from(line).unwrap_or(u32::MAX), offset.saturating_sub(start) + 1) + } + + pub(crate) fn line_count(&self) -> u32 { + if self.empty { + return 0; + } + let breaks = self.starts.len() - 1; + u32::try_from(breaks + usize::from(!self.ends_with_break)).unwrap_or(u32::MAX) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/js/src/number.rs b/crates/js/src/number.rs new file mode 100644 index 000000000..4afd48076 --- /dev/null +++ b/crates/js/src/number.rs @@ -0,0 +1,69 @@ +//! Number to string conversion (ECMA-262 5.1 section 9.8.1), for numeric property names. + +/// The string ToString gives for the number `x`: the shortest decimal digits that read back as +/// `x`, laid out as section 9.8.1 says. +pub fn to_string(x: f64) -> String { + if x.is_nan() { + return "NaN".into(); + } + if x == 0.0 { + return "0".into(); + } + if x.is_infinite() { + return if x > 0.0 { "Infinity".into() } else { "-Infinity".into() }; + } + if x < 0.0 { + return format!("-{}", to_string(-x)); + } + // Rust's `{:e}` writes the shortest digits that round-trip: `d.ddde±x`. + let sci = format!("{x:e}"); + let (mantissa, exp) = sci.split_once('e').unwrap_or((&sci, "0")); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + let k = digits.len() as i64; + let n = exp.parse::().unwrap_or(0) + 1; + if k <= n && n <= 21 { + let zeros = usize::try_from(n - k).unwrap_or(0); + format!("{digits}{}", "0".repeat(zeros)) + } else if 0 < n && n <= 21 { + let (int, frac) = digits.split_at_checked(usize::try_from(n).unwrap_or(0)).unwrap_or((&digits, "")); + format!("{int}.{frac}") + } else if -6 < n && n <= 0 { + let zeros = usize::try_from(-n).unwrap_or(0); + format!("0.{}{digits}", "0".repeat(zeros)) + } else { + let e = n - 1; + let sign = if e < 0 { '-' } else { '+' }; + let (first, rest) = digits.split_at_checked(1).unwrap_or((&digits, "")); + if rest.is_empty() { format!("{first}e{sign}{}", e.abs()) } else { format!("{first}.{rest}e{sign}{}", e.abs()) } + } +} + +#[cfg(test)] +mod tests { + use super::to_string; + + #[test] + fn numbers_print_as_section_9_8_1_lays_them_out() { + let cases: &[(f64, &str)] = &[ + (0.0, "0"), + (-0.0, "0"), + (1.0, "1"), + (-1.5, "-1.5"), + (123.456, "123.456"), + (1e21, "1e+21"), + (1e20, "100000000000000000000"), + (0.000001, "0.000001"), + (1e-7, "1e-7"), + (1.5e-7, "1.5e-7"), + (0.1 + 0.2, "0.30000000000000004"), + (f64::NAN, "NaN"), + (f64::INFINITY, "Infinity"), + (f64::NEG_INFINITY, "-Infinity"), + (5e-324, "5e-324"), + (1.7976931348623157e308, "1.7976931348623157e+308"), + ]; + for (x, want) in cases { + assert_eq!(to_string(*x), *want, "{x:?}"); + } + } +} diff --git a/crates/js/src/parser.rs b/crates/js/src/parser.rs new file mode 100644 index 000000000..ebecab4ca --- /dev/null +++ b/crates/js/src/parser.rs @@ -0,0 +1,1233 @@ +//! The parser: tokens into a [`Program`] (ECMA-262 5.1 sections 11 to 14), with the early errors +//! of section 16 and of strict mode (Annex C). +//! +//! It is a recursive-descent parser whose recursion is counted: each statement, assignment +//! expression, primary expression and function body it enters adds one to a depth, and going past +//! [`crate::Options::max_depth`] gives a `SyntaxError`. Binary operators are read with an operator +//! stack, and prefix operators, assignment chains, `else if` chains and labels with loops, so long +//! runs of them do not nest at all. + +use crate::LineTable; +use crate::ast::{ + AssignOp, Ast, BinaryOp, CatchClause, Expr, ExprId, ForInTarget, ForInit, Function, FunctionKind, LogicalOp, Name, Program, Property, + PropertyKey, PropertyValue, Stmt, StmtId, SwitchCase, UnaryOp, UpdateOp, VarDecl, +}; +use crate::error::{SyntaxError, TOO_DEEP}; +use crate::lexer::{Goal, Lexer}; +use crate::token::{Directive, Keyword, Punct, Span, Token, TokenKind, strict_reserved}; + +type Result = std::result::Result; + +/// A label in scope, and whether it labels an iteration statement (`continue` may name only +/// those). +#[derive(Clone, Debug)] +struct Label { + name: String, + is_loop: bool, +} + +/// What the code being parsed allows, saved and replaced at each function body. +#[derive(Clone, Debug, Default)] +struct Context { + strict: bool, + in_function: bool, + labels: Vec