//! Tokens: what the lexer gives the parser. /// A range of the source in UTF-16 code units, `start` included and `end` not. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Span { pub start: u32, pub end: u32, } impl Span { pub fn new(start: u32, end: u32) -> Self { Self { start, end } } /// The span from the start of `self` to the end of `other`. pub fn to(self, other: Span) -> Span { Span { start: self.start, end: other.end.max(self.start) } } } /// The reserved words of ECMA-262 5.1 section 7.6.1 that are reserved in all code, with the /// literals `null`, `true` and `false`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Keyword { Break, Case, Catch, Class, Const, Continue, Debugger, Default, Delete, Do, Else, Enum, Export, Extends, False, Finally, For, Function, If, Import, In, Instanceof, New, Null, Return, Super, Switch, This, Throw, True, Try, Typeof, Var, Void, While, With, } impl Keyword { /// The keyword spelled `s`, if `s` is one. pub fn from_word(s: &str) -> Option { use Keyword as K; Some(match s { "break" => K::Break, "case" => K::Case, "catch" => K::Catch, "class" => K::Class, "const" => K::Const, "continue" => K::Continue, "debugger" => K::Debugger, "default" => K::Default, "delete" => K::Delete, "do" => K::Do, "else" => K::Else, "enum" => K::Enum, "export" => K::Export, "extends" => K::Extends, "false" => K::False, "finally" => K::Finally, "for" => K::For, "function" => K::Function, "if" => K::If, "import" => K::Import, "in" => K::In, "instanceof" => K::Instanceof, "new" => K::New, "null" => K::Null, "return" => K::Return, "super" => K::Super, "switch" => K::Switch, "this" => K::This, "throw" => K::Throw, "true" => K::True, "try" => K::Try, "typeof" => K::Typeof, "var" => K::Var, "void" => K::Void, "while" => K::While, "with" => K::With, _ => return None, }) } pub fn as_str(self) -> &'static str { use Keyword as K; match self { K::Break => "break", K::Case => "case", K::Catch => "catch", K::Class => "class", K::Const => "const", K::Continue => "continue", K::Debugger => "debugger", K::Default => "default", K::Delete => "delete", K::Do => "do", K::Else => "else", K::Enum => "enum", K::Export => "export", K::Extends => "extends", K::False => "false", K::Finally => "finally", K::For => "for", K::Function => "function", K::If => "if", K::Import => "import", K::In => "in", K::Instanceof => "instanceof", K::New => "new", K::Null => "null", K::Return => "return", K::Super => "super", K::Switch => "switch", K::This => "this", K::Throw => "throw", K::True => "true", K::Try => "try", K::Typeof => "typeof", K::Var => "var", K::Void => "void", K::While => "while", K::With => "with", } } } /// The words reserved only in strict mode code (ECMA-262 5.1 section 7.6.1.2). pub fn strict_reserved(name: &str) -> bool { matches!(name, "implements" | "interface" | "let" | "package" | "private" | "protected" | "public" | "static" | "yield") } /// The punctuators of ECMA-262 5.1 sections 7.7 and 7.8.5 (division). #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Punct { LBrace, RBrace, LParen, RParen, LBracket, RBracket, Dot, Semicolon, Comma, Lt, Gt, LtEq, GtEq, EqEq, NotEq, EqEqEq, NotEqEq, Plus, Minus, Star, Percent, PlusPlus, MinusMinus, Shl, Shr, UShr, Amp, Pipe, Caret, Bang, Tilde, AmpAmp, PipePipe, Question, Colon, Eq, PlusEq, MinusEq, StarEq, PercentEq, ShlEq, ShrEq, UShrEq, AmpEq, PipeEq, CaretEq, Slash, SlashEq, } impl Punct { /// Punctuators by spelling, longest first, so the first that matches is the longest. pub(crate) const ALL: &'static [(&'static str, Punct)] = &[ (">>>=", Punct::UShrEq), ("===", Punct::EqEqEq), ("!==", Punct::NotEqEq), (">>>", Punct::UShr), ("<<=", Punct::ShlEq), (">>=", Punct::ShrEq), ("<=", Punct::LtEq), (">=", Punct::GtEq), ("==", Punct::EqEq), ("!=", Punct::NotEq), ("++", Punct::PlusPlus), ("--", Punct::MinusMinus), ("<<", Punct::Shl), (">>", Punct::Shr), ("&&", Punct::AmpAmp), ("||", Punct::PipePipe), ("+=", Punct::PlusEq), ("-=", Punct::MinusEq), ("*=", Punct::StarEq), ("%=", Punct::PercentEq), ("&=", Punct::AmpEq), ("|=", Punct::PipeEq), ("^=", Punct::CaretEq), ("/=", Punct::SlashEq), ("{", Punct::LBrace), ("}", Punct::RBrace), ("(", Punct::LParen), (")", Punct::RParen), ("[", Punct::LBracket), ("]", Punct::RBracket), (".", Punct::Dot), (";", Punct::Semicolon), (",", Punct::Comma), ("<", Punct::Lt), (">", Punct::Gt), ("+", Punct::Plus), ("-", Punct::Minus), ("*", Punct::Star), ("%", Punct::Percent), ("&", Punct::Amp), ("|", Punct::Pipe), ("^", Punct::Caret), ("!", Punct::Bang), ("~", Punct::Tilde), ("?", Punct::Question), (":", Punct::Colon), ("=", Punct::Eq), ("/", Punct::Slash), ]; pub fn as_str(self) -> &'static str { Punct::ALL.iter().find(|(_, p)| *p == self).map_or("", |(s, _)| s) } } /// A preprocessor line's directive: `#include`, `#includepath`, `#script`, `#strict`, `#target` /// and `#targetengine`, as the public JavaScript Tools Guide lists them /// (). /// Only the names are taken from it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum DirectiveName { Include, IncludePath, Script, Strict, Target, TargetEngine, } impl DirectiveName { /// The directive named `name`, compared without case. pub fn from_name(name: &str) -> Option { const NAMES: [(&str, DirectiveName); 6] = [ ("include", DirectiveName::Include), ("includepath", DirectiveName::IncludePath), ("script", DirectiveName::Script), ("strict", DirectiveName::Strict), ("target", DirectiveName::Target), ("targetengine", DirectiveName::TargetEngine), ]; NAMES.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, d)| *d) } pub fn as_str(self) -> &'static str { match self { DirectiveName::Include => "include", DirectiveName::IncludePath => "includepath", DirectiveName::Script => "script", DirectiveName::Strict => "strict", DirectiveName::Target => "target", DirectiveName::TargetEngine => "targetengine", } } } /// A preprocessor line: `#name value` at the start of a line, or a `//@name value` comment there /// that names a directive. It is read and kept, never run. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Directive { /// The directive, when the name is one of the six (`#name` lines with another name are kept /// with `None`). pub known: Option, /// The name as written. pub name: String, /// The text after the name, with blanks around it removed. pub value: String, /// `#` or `//@`. pub form: &'static str, pub span: Span, pub line: u32, } /// What a token is. #[derive(Clone, Debug, PartialEq)] pub enum TokenKind { /// The end of the source. Eof, /// An identifier, the strict-mode reserved words included. `escaped` tokens held a `\u` escape. Ident(String), Keyword(Keyword), Punct(Punct), /// A numeric literal's value. Number(f64), /// A string literal's value in UTF-16 code units. String(Vec), /// A regular expression literal: its pattern and flags as written. RegExp { pattern: Vec, flags: String, }, /// A preprocessor line. Directive(Directive), } /// A token, with where it is and the facts the grammar's rules need about it. #[derive(Clone, Debug, PartialEq)] pub struct Token { pub kind: TokenKind, pub span: Span, /// The line (from 1) and column (from 1, in UTF-16 code units) the token starts at. pub line: u32, pub column: u32, /// A line terminator comes between the previous token and this one (automatic semicolon /// insertion and the restricted productions use it). pub newline_before: bool, /// An identifier written with a `\u` escape. pub escaped: bool, /// A legacy octal numeric literal (`017`, `08`) or a string with an octal escape (`"\1"`), which /// strict mode code does not allow. pub legacy_octal: bool, } impl Token { pub fn is_punct(&self, p: Punct) -> bool { self.kind == TokenKind::Punct(p) } pub fn is_keyword(&self, k: Keyword) -> bool { self.kind == TokenKind::Keyword(k) } /// The identifier's name, for an identifier token. pub fn ident(&self) -> Option<&str> { match &self.kind { TokenKind::Ident(name) => Some(name), _ => None, } } /// The name this token gives when it stands where any identifier name may, as a property's /// name after `.` or in an object literal: identifiers, keywords and the literals `null`, /// `true` and `false`. pub fn identifier_name(&self) -> Option<&str> { match &self.kind { TokenKind::Ident(name) => Some(name), TokenKind::Keyword(k) => Some(k.as_str()), _ => None, } } }