//! A tolerant HTML tokenizer: text, start tags and end tags.
//!
//! It follows the general syntax of the HTML Living Standard closely enough for documents, mail
//! and clipboard fragments, and reads anything: a `<` that starts no tag is text, a tag or
//! comment that never ends runs to the end of the input. Comments, doctypes and processing
//! instructions never leave it.
use crate::entity::unescape_attr;
/// What a [`Lexer`] caps.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Limits {
/// Most attributes kept per tag; the rest are read and dropped. Default 64.
pub max_attrs: usize,
/// Longest tag name kept, in bytes; a longer one is no real element and its name is empty.
/// Default 64.
pub max_tag_name: usize,
}
impl Limits {
pub const DEFAULT: Limits = Limits { max_attrs: 64, max_tag_name: 64 };
}
impl Default for Limits {
fn default() -> Limits {
Limits::DEFAULT
}
}
/// How a [`Lexer`] reads. Element names are lower case.
#[derive(Clone, Copy, Debug, Default)]
pub struct Options<'o> {
pub limits: Limits,
/// Elements that never leave the lexer: the start tag, the content (raw text, not markup)
/// and the end tag are skipped. For `script`, `style`, `title` and the like, when the caller
/// has no use for them. A caller that wants a `style`'s text leaves it out of this list and
/// calls [`Lexer::raw_content`] on its start tag.
pub drop: &'o [&'o str],
/// Elements whose content is text, not markup, and is given: after the start tag comes one
/// [`Token::Text`] with everything up to the end tag. For `textarea`, `xmp`, `plaintext`.
pub raw_text: &'o [&'o str],
}
/// A start tag.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Tag<'a> {
/// Lower case; empty for a name longer than [`Limits::max_tag_name`].
pub name: String,
/// Names and values as written, in the order written.
pub attrs: Vec<(&'a str, &'a str)>,
/// The tag ends in `/>`.
pub self_close: bool,
}
impl Tag<'_> {
/// An attribute's value with character references decoded ([`unescape_attr`]); the first
/// of a name wins, as in browsers. Names match in any letter case.
pub fn attr(&self, name: &str) -> Option {
self.raw_attr(name).map(unescape_attr)
}
/// An attribute's value as written, character references and all.
pub fn raw_attr(&self, name: &str) -> Option<&str> {
self.attrs.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| *v)
}
/// Whether the tag has the attribute, with or without a value.
pub fn has(&self, name: &str) -> bool {
self.attrs.iter().any(|(n, _)| n.eq_ignore_ascii_case(name))
}
/// Every attribute in the order written: the name as written, the value decoded.
pub fn attributes(&self) -> impl Iterator- + '_ {
self.attrs.iter().map(|(n, v)| (*n, unescape_attr(v)))
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Token<'a> {
/// Character data as written: references are not decoded ([`crate::unescape`]).
Text(&'a str),
Start(Tag<'a>),
/// An end tag's name, lower case (empty when longer than [`Limits::max_tag_name`]).
End(String),
}
/// The tokenizer: an iterator of [`Token`]s over a string.
pub struct Lexer<'a, 'o> {
src: &'a str,
pos: usize,
opts: Options<'o>,
/// Set after the start tag of a raw-text element: its content comes next.
raw: Option,
}
impl<'a> Lexer<'a, 'static> {
/// A lexer with [`Options::default`]: nothing dropped, no raw-text elements.
pub fn new(src: &'a str) -> Lexer<'a, 'static> {
Lexer::with(src, Options::default())
}
}
impl<'a, 'o> Lexer<'a, 'o> {
pub fn with(src: &'a str, opts: Options<'o>) -> Lexer<'a, 'o> {
Lexer { src, pos: 0, opts, raw: None }
}
/// How far into the input the tokens given so far go, in bytes.
pub fn pos(&self) -> usize {
self.pos
}
/// Goes back (or forward) to `pos`, a byte offset an earlier [`Lexer::pos`] gave, to read
/// from there again: for a reader that looked at a token and hands it back. An offset past the
/// end, or inside a character, moves to the next character boundary.
pub fn set_pos(&mut self, pos: usize) {
let mut pos = pos.min(self.src.len());
while !self.src.is_char_boundary(pos) {
pos += 1;
}
self.pos = pos;
self.raw = None;
}
/// The raw content of element `name`, whose start tag was the last token: everything up to
/// its end tag (any letter case), or to the end of the input when it has none. The end tag
/// is consumed. For `style` and `script`, whose content is not markup.
pub fn raw_content(&mut self, name: &str) -> &'a str {
self.raw = None;
let b = self.bytes();
let end = find_end_tag(b, self.pos, name).unwrap_or(b.len());
let text = self.slice(self.pos, end);
self.pos = find_byte(b, end, b'>').map_or(b.len(), |i| i + 1);
text
}
fn bytes(&self) -> &'a [u8] {
self.src.as_bytes()
}
fn slice(&self, start: usize, end: usize) -> &'a str {
self.src.get(start..end).unwrap_or("")
}
/// A tag name in lower case; empty when it is too long to be a real element.
fn tag_name(&self, raw: &str) -> String {
if raw.len() > self.opts.limits.max_tag_name { String::new() } else { raw.to_ascii_lowercase() }
}
/// Whether the `<` at `i` starts a tag, a comment or a declaration (rather than being text).
fn markup_at(&self, i: usize) -> bool {
let b = self.bytes();
match b.get(i + 1) {
Some(c) if c.is_ascii_alphabetic() => true,
Some(b'!' | b'?') => true,
Some(b'/') => matches!(b.get(i + 2), Some(c) if c.is_ascii_alphabetic() || *c == b'>'),
_ => false,
}
}
/// The end of the text run that starts at `self.pos`: the next `<` that starts markup.
fn text_end(&self) -> usize {
let b = self.bytes();
let mut i = self.pos + 1;
loop {
match find_byte(b, i, b'<') {
Some(j) if self.markup_at(j) => return j,
Some(j) => i = j + 1,
None => return b.len(),
}
}
}
/// Skip a comment, doctype or processing instruction at `self.pos`.
fn skip_declaration(&mut self) {
let b = self.bytes();
let end = if b.get(self.pos..self.pos + 4) == Some(b"").map_or(b.len(), |i| i + 3)
}
} else {
find_byte(b, self.pos, b'>').map_or(b.len(), |i| i + 1)
};
self.pos = end.min(b.len());
}
/// Read the end tag at `self.pos` (``).
fn close_tag(&mut self) -> Option> {
let b = self.bytes();
let start = self.pos + 2;
let mut i = start;
while let Some(c) = b.get(i) {
if c.is_ascii_whitespace() || *c == b'/' || *c == b'>' {
break;
}
i += 1;
}
let name = self.slice(start, i);
self.pos = find_byte(b, i, b'>').map_or(b.len(), |j| j + 1);
// `>` is nothing at all.
(!name.is_empty()).then(|| Token::End(self.tag_name(name)))
}
/// Read the start tag at `self.pos` (``).
fn open_tag(&mut self) -> Tag<'a> {
let b = self.bytes();
let start = self.pos + 1;
let mut i = start;
while let Some(c) = b.get(i) {
if c.is_ascii_whitespace() || *c == b'/' || *c == b'>' {
break;
}
i += 1;
}
let name = self.tag_name(self.slice(start, i));
let mut attrs = Vec::new();
let mut self_close = false;
loop {
while matches!(b.get(i), Some(c) if c.is_ascii_whitespace() || *c == b'/') {
self_close = b.get(i) == Some(&b'/') && b.get(i + 1) == Some(&b'>');
i += 1;
}
match b.get(i) {
None => break,
Some(b'>') => {
i += 1;
break;
}
Some(_) => {}
}
// An attribute name runs to white space, `/`, `>` or `=`; a leading `=` is part of it.
let name_start = i;
i += 1;
while let Some(c) = b.get(i) {
if c.is_ascii_whitespace() || matches!(*c, b'/' | b'>' | b'=') {
break;
}
i += 1;
}
let attr = self.slice(name_start, i);
let (value, next) = self.attr_value(i);
i = next;
if attrs.len() < self.opts.limits.max_attrs {
attrs.push((attr, value));
}
}
self.pos = i.min(b.len());
Tag { name, attrs, self_close }
}
/// The value after an attribute name that ends at `i` (empty when it has none), and where
/// reading goes on.
fn attr_value(&self, i: usize) -> (&'a str, usize) {
let b = self.bytes();
let mut j = i;
while matches!(b.get(j), Some(c) if c.is_ascii_whitespace()) {
j += 1;
}
if b.get(j) != Some(&b'=') {
return ("", j);
}
j += 1;
while matches!(b.get(j), Some(c) if c.is_ascii_whitespace()) {
j += 1;
}
match b.get(j) {
Some(&q) if q == b'"' || q == b'\'' => {
let end = find_byte(b, j + 1, q).unwrap_or(b.len());
(self.slice(j + 1, end), (end + 1).min(b.len()))
}
_ => {
let mut end = j;
while let Some(c) = b.get(end) {
if c.is_ascii_whitespace() || *c == b'>' {
break;
}
end += 1;
}
(self.slice(j, end), end)
}
}
}
}
impl<'a> Iterator for Lexer<'a, '_> {
type Item = Token<'a>;
fn next(&mut self) -> Option> {
loop {
let b = self.bytes();
if self.pos >= b.len() {
return None;
}
if let Some(name) = self.raw.take() {
let end = find_end_tag(b, self.pos, &name).unwrap_or(b.len());
let text = self.slice(self.pos, end);
self.pos = end;
if !text.is_empty() {
return Some(Token::Text(text));
}
continue;
}
if b.get(self.pos) != Some(&b'<') || !self.markup_at(self.pos) {
let start = self.pos;
self.pos = self.text_end();
return Some(Token::Text(self.slice(start, self.pos)));
}
match b.get(self.pos + 1) {
Some(b'!' | b'?') => self.skip_declaration(),
Some(b'/') => {
if let Some(token) = self.close_tag() {
return Some(token);
}
}
_ => {
let tag = self.open_tag();
if self.opts.drop.contains(&tag.name.as_str()) {
self.raw_content(&tag.name);
continue;
}
if self.opts.raw_text.contains(&tag.name.as_str()) {
self.raw = Some(tag.name.clone());
}
return Some(Token::Start(tag));
}
}
}
}
}
fn find_byte(b: &[u8], from: usize, byte: u8) -> Option {
b.get(from..)?.iter().position(|c| *c == byte).map(|i| i + from)
}
fn find_sub(b: &[u8], from: usize, needle: &[u8]) -> Option {
b.get(from..)?.windows(needle.len()).position(|w| w == needle).map(|i| i + from)
}
/// Where `` or the end), at or after `from`.
fn find_end_tag(b: &[u8], from: usize, name: &str) -> Option {
let mut i = from;
while let Some(lt) = find_byte(b, i, b'<') {
let after = lt + 2 + name.len();
let named = b.get(lt + 1) == Some(&b'/') && b.get(lt + 2..after).is_some_and(|n| n.eq_ignore_ascii_case(name.as_bytes()));
if named && b.get(after).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')) {
return Some(lt);
}
i = lt + 1;
}
None
}
#[cfg(test)]
mod tests;