Files
craft-libs/crates/html/src/lib.rs
T
Dylan M. Taylor e5299a3be0 craft-html: a tolerant tokenizer, character references, escaping and the declared charset (#944)
The bottom layer every HTML reader in the Craft apps carried its own copy
of. The tokenizer is MailCraft's (crates/html/src/tokenizer.rs), with what
it dropped and capped turned into Options and Limits, a self-closing flag,
and raw_content for callers that read a style sheet's text. The character
references are WordCraft's (crates/formats/src/html/entity.rs at 06c2ae6f):
its name table and the 94 legacy names that decode without a semicolon,
with MailCraft's Latin-1 and Greek tables and its handling of numbers too
long to be numbers.

No tree, no CSS, no decoding of bytes.
2026-11-11 18:54:48 -05:00

79 lines
3.9 KiB
Rust

#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! The bottom layer of reading and writing HTML, shared by everything that reads a page, a
//! mail body or a clipboard fragment.
//!
//! - [`Lexer`] is a tolerant tokenizer: text, start tags and end tags, borrowed from the input.
//! It reads anything: a `<` that starts no tag is text, a tag or comment that never ends runs
//! to the end of the input. [`Options`] names the elements to drop whole (`script`), the
//! elements whose content is text (`textarea`), and the caps ([`Limits`]).
//! - [`unescape`], [`unescape_attr`] and [`decode_entity`] decode character references;
//! [`escape`] and [`escape_dq`] write them.
//! - [`declared_charset`] finds the character set a page names in its first bytes.
//!
//! There is no tree builder, no CSS and no rendering here: what the tags mean is the caller's.
//!
//! Input is hostile. Nothing here panics on any string, and the work is linear in the input.
//!
//! ```
//! use craft_html::{Lexer, Options, Token, unescape};
//!
//! let opts = Options { drop: &["script"], ..Options::default() };
//! let mut text = String::new();
//! let mut links = Vec::new();
//! for token in Lexer::with(r#"Tom &amp; <a href="?a=1&amp;b=2">Jerry</a><script>x<y</script>"#, opts) {
//! match token {
//! Token::Text(t) => text.push_str(&unescape(t)),
//! Token::Start(tag) if tag.name == "a" => links.extend(tag.attr("href")),
//! _ => {}
//! }
//! }
//! assert_eq!((text.as_str(), links.as_slice()), ("Tom & Jerry", &["?a=1&b=2".to_string()][..]));
//! ```
#[cfg(doctest)]
#[doc = include_str!("../README.md")]
struct ReadmeExamples;
mod entity;
mod lex;
pub use entity::{decode_entity, escape, escape_dq, reference, unescape, unescape_attr};
pub use lex::{Lexer, Limits, Options, Tag, Token};
/// How many bytes of a page [`declared_charset`] looks at.
pub const CHARSET_SNIFF_BYTES: usize = 2048;
/// The character set a page names in its first [`CHARSET_SNIFF_BYTES`] bytes: a
/// `<meta charset=…>`, a `content="text/html; charset=…"`, or an XML declaration's `encoding=…`.
/// Lower case, as written otherwise: the caller maps the label to a decoder. `None` when the
/// page names none. A byte order mark is the caller's to check first.
pub fn declared_charset(bytes: &[u8]) -> Option<String> {
let head = String::from_utf8_lossy(bytes.get(..bytes.len().min(CHARSET_SNIFF_BYTES)).unwrap_or(bytes)).to_ascii_lowercase();
let (i, n) = head.find("charset").map(|i| (i, 7)).or_else(|| head.find("encoding").map(|i| (i, 8)))?;
let after = head.get(i + n..)?.trim_start().strip_prefix('=')?;
let trimmed = after.trim_start().trim_start_matches(['"', '\'']);
let end = trimmed.find(|c: char| c.is_whitespace() || matches!(c, '"' | '\'' | ';' | '>' | '/')).unwrap_or(trimmed.len());
let label = trimmed.get(..end)?.trim();
(!label.is_empty()).then(|| label.to_string())
}
#[cfg(test)]
mod tests {
use super::declared_charset;
#[test]
fn the_declared_charset_is_found_in_each_form() {
assert_eq!(declared_charset(b"<meta charset=\"UTF-8\">").as_deref(), Some("utf-8"));
assert_eq!(declared_charset(b"<META http-equiv=Content-Type content='text/html; charset=windows-1251'>").as_deref(), Some("windows-1251"));
assert_eq!(declared_charset(b"<?xml version=\"1.0\" encoding = 'Shift_JIS'?>").as_deref(), Some("shift_jis"));
assert_eq!(declared_charset(b"<meta charset=koi8-r/>").as_deref(), Some("koi8-r"));
for none in [&b"<p>no charset here</p>"[..], b"charset", b"charset=", b"charset=''", b"", b"\xff\xfe\x00"] {
assert_eq!(declared_charset(none), None);
}
// Only the head is looked at.
let mut late = vec![b' '; super::CHARSET_SNIFF_BYTES];
late.extend_from_slice(b"<meta charset=utf-8>");
assert_eq!(declared_charset(&late), None);
}
}