diff --git a/Cargo.lock b/Cargo.lock index fc723721..c2c4d6cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1248,6 +1248,12 @@ dependencies = [ name = "craft-html" version = "0.1.0" dependencies = [ + "craft-codepage", + "craft-flow", + "craft-math", + "craft-mime", + "craft-wordmodel", + "encoding_rs", "proptest", ] diff --git a/README.md b/README.md index c689677f..0439d610 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " " '` as references; `escape_dq` leaves `'`, for content and attribute values in double quotes. +## HTML documents (feature `flow`) + +`craft_html::doc` reads a page or a pasted fragment into the [craft-flow](../flow/README.md) +model and a craft-wordmodel `Document`, and writes one as clean HTML. The reader is a tolerant +tag-soup builder on this crate's tokenizer (a stack of open elements with HTML's implicit closes), +with a CSS subset from `style` attributes and `

x

"), blue); assert_eq!(color("

x

"), red); assert_eq!(color("

x

"), blue); @@ -39,9 +39,9 @@ fn texts(f: &Flow) -> Vec { } fn table_style(html: &str) -> Option { - let doc = crate::model::to_doc(&parse(html)); + let doc = craft_flow::to_doc(&parse(html)); doc.body.iter().find_map(|b| match &**b { - wordcraft_doc::Block::Table(t) => Some(t.props.style.clone().unwrap_or_default()), + craft_wordmodel::Block::Table(t) => Some(t.props.style.clone().unwrap_or_default()), _ => None, }) } @@ -67,7 +67,7 @@ fn borderless_tables_export_without_cell_borders() { ("
AB
", false), ("
AB
", true), ] { - let out = export(&crate::model::to_doc(&parse(src))); + let out = export(&craft_flow::to_doc(&parse(src))); assert_eq!(out.contains("border:1px"), grid, "{src}: {out}"); assert_eq!(table_style(&out).as_deref(), Some(if grid { "TableGrid" } else { "" }), "{src}"); assert!(out.contains(">A") && out.contains(">B"), "{out}"); @@ -157,7 +157,7 @@ fn styles_and_links() { let f = parse("

red l
n

"); let FBlock::Para(p) = &f.blocks[0] else { panic!() }; assert_eq!(p.align, Some(Align::Center)); - assert!(matches!(&p.inlines[0], Inline::Text(t, f) if t == "red" && f.bold && f.color == Some(wordcraft_doc::Rgb(255,0,0)))); + assert!(matches!(&p.inlines[0], Inline::Text(t, f) if t == "red" && f.bold && f.color == Some(craft_wordmodel::Rgb(255,0,0)))); assert!(p.inlines.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "l" && f.link.as_deref() == Some("http://x")))); assert!(p.text().ends_with("l\nn")); } @@ -369,14 +369,14 @@ fn font_names_are_written_as_css_strings() { /// A document that reads "I read " then a phonetic guide over `base`, then " daily.". fn ruby_doc(base: &str, ruby: &str) -> Document { - let props = wordcraft_doc::CharProps::default(); - let mut p = wordcraft_doc::Paragraph::with_text("I read ", props.clone()); + let props = craft_wordmodel::CharProps::default(); + let mut p = craft_wordmodel::Paragraph::with_text("I read ", props.clone()); let n = p.len(); - p.insert_object(n, crate::model::ruby_object(base.into(), ruby.into(), 20.0), &props).unwrap(); + p.insert_object(n, craft_flow::ruby_object(base.into(), ruby.into(), 20.0), &props).unwrap(); let n = p.len(); p.insert_text(n, " daily.", &props).unwrap(); let mut d = Document::new(); - d.body = vec![wordcraft_doc::para_block(p)]; + d.body = vec![craft_wordmodel::para_block(p)]; d } @@ -394,8 +394,8 @@ fn ruby_of(f: &Flow) -> Option<(String, String)> { /// The first document ruby: its base text and reading. fn doc_ruby(d: &Document) -> Option<(String, String)> { d.body.iter().find_map(|b| match &**b { - wordcraft_doc::Block::Para(p) => p.objects.iter().find_map(|o| match o { - wordcraft_doc::InlineObject::Ruby { base, ruby, .. } => Some((base.clone(), ruby.clone())), + craft_wordmodel::Block::Para(p) => p.objects.iter().find_map(|o| match o { + craft_wordmodel::InlineObject::Ruby { base, ruby, .. } => Some((base.clone(), ruby.clone())), _ => None, }), _ => None, @@ -403,10 +403,10 @@ fn doc_ruby(d: &Document) -> Option<(String, String)> { } /// The first document ruby: base size, ruby size, raise and alignment. -fn doc_ruby_attrs(d: &Document) -> Option<(f32, f32, f32, wordcraft_doc::para::RubyAlign)> { +fn doc_ruby_attrs(d: &Document) -> Option<(f32, f32, f32, craft_wordmodel::para::RubyAlign)> { d.body.iter().find_map(|b| match &**b { - wordcraft_doc::Block::Para(p) => p.objects.iter().find_map(|o| match o { - wordcraft_doc::InlineObject::Ruby { base_size, size, raise, align, .. } => Some((*base_size, *size, *raise, *align)), + craft_wordmodel::Block::Para(p) => p.objects.iter().find_map(|o| match o { + craft_wordmodel::InlineObject::Ruby { base_size, size, raise, align, .. } => Some((*base_size, *size, *raise, *align)), _ => None, }), _ => None, @@ -432,7 +432,7 @@ fn ruby_imports_with_its_surrounding_text() { assert!(matches!(&p.inlines[0], Inline::Text(t, _) if t == "I read "), "{:?}", p.inlines); assert!(matches!(p.inlines.last(), Some(Inline::Text(t, _)) if t == " daily."), "{:?}", p.inlines); // One document object, holding the base text and the reading. - assert_eq!(doc_ruby(&crate::model::to_doc(&f)), Some(("漢字".into(), "かんじ".into()))); + assert_eq!(doc_ruby(&craft_flow::to_doc(&f)), Some(("漢字".into(), "かんじ".into()))); } #[test] @@ -442,19 +442,19 @@ fn ruby_rb_and_rp_fallbacks_import_as_one_guide() { let f = parse("

漢字(かんじ)

"); assert_eq!(ruby_of(&f), Some(("漢字".into(), "かんじ".into()))); assert_eq!(texts(&f), vec!["NormalNone:漢字"]); - assert_eq!(doc_ruby(&crate::model::to_doc(&f)), Some(("漢字".into(), "かんじ".into()))); + assert_eq!(doc_ruby(&craft_flow::to_doc(&f)), Some(("漢字".into(), "かんじ".into()))); } #[test] fn ruby_import_uses_the_phonetic_guide_defaults() { // HTML states no alignment, size or raise: the Phonetic Guide's own defaults over a // 20 pt run (half the base size, raised clear of the base text, centred)… - let d = crate::html::import("

漢字かんじ

"); + let d = crate::doc::import("

漢字かんじ

"); let Some((base_size, size, raise, align)) = doc_ruby_attrs(&d) else { panic!("no ruby") }; assert_eq!((base_size, size, align.ooxml()), (20.0, 10.0, "center")); assert!((raise - 18.0).abs() < 0.001, "{raise}"); // …and the guide's 10.5 pt fallback base size when the source states none. - let d = crate::html::import("

漢字かんじ

"); + let d = crate::doc::import("

漢字かんじ

"); let Some((base_size, size, raise, _)) = doc_ruby_attrs(&d) else { panic!("no ruby") }; assert_eq!((base_size, size), (10.5, 5.0)); assert!((raise - 9.4).abs() < 0.001, "{raise}"); @@ -464,19 +464,19 @@ fn ruby_import_uses_the_phonetic_guide_defaults() { fn ruby_survives_an_html_round_trip() { let html = export(&ruby_doc("漢字", "かんじ")); assert_eq!(ruby_of(&parse(&html)), Some(("漢字".into(), "かんじ".into()))); - let back = crate::html::import(&html); + let back = crate::doc::import(&html); assert_eq!(doc_ruby(&back), Some(("漢字".into(), "かんじ".into()))); assert_eq!(back.plain_text(Default::default()), "I read 漢字 daily."); } #[test] fn ruby_keeps_the_run_formatting() { - let props = wordcraft_doc::CharProps { bold: Some(true), ..Default::default() }; - let mut p = wordcraft_doc::Paragraph::with_text("I read ", props.clone()); + let props = craft_wordmodel::CharProps { bold: Some(true), ..Default::default() }; + let mut p = craft_wordmodel::Paragraph::with_text("I read ", props.clone()); let n = p.len(); - p.insert_object(n, crate::model::ruby_object("漢字".into(), "かんじ".into(), 20.0), &props).unwrap(); + p.insert_object(n, craft_flow::ruby_object("漢字".into(), "かんじ".into(), 20.0), &props).unwrap(); let mut d = Document::new(); - d.body = vec![wordcraft_doc::para_block(p)]; + d.body = vec![craft_wordmodel::para_block(p)]; assert!(export(&d).contains("漢字かんじ"), "{}", export(&d)); // Importing it back keeps the formatting for the base text and the reading. let f = parse("

漢字かんじ

"); @@ -533,7 +533,7 @@ fn css_important_casing_and_spacing() { assert_eq!(p.align, Some(Align::Center)); match &p.inlines[0] { Inline::Text(_, fmt) => { - assert_eq!(fmt.color, Some(wordcraft_doc::props::Rgb(0xFF, 0x00, 0x00))); + assert_eq!(fmt.color, Some(craft_wordmodel::props::Rgb(0xFF, 0x00, 0x00))); assert_eq!(fmt.size, Some(16.0)); assert!(fmt.bold); } @@ -548,8 +548,8 @@ fn table_cell_vertical_alignment() { assert_eq!(t.rows[0][0].valign, Some(VAlign::Top)); assert_eq!(t.rows[0][1].valign, Some(VAlign::Center)); assert_eq!(t.rows[0][2].valign, Some(VAlign::Bottom)); - let doc = crate::model::to_doc(&f); - let wordcraft_doc::Block::Table(dt) = &*doc.body[0] else { panic!() }; + let doc = craft_flow::to_doc(&f); + let craft_wordmodel::Block::Table(dt) = &*doc.body[0] else { panic!() }; assert_eq!(dt.rows[0].cells[0].props.valign, Some(VAlign::Top)); assert_eq!(dt.rows[0].cells[1].props.valign, Some(VAlign::Center)); assert_eq!(dt.rows[0].cells[2].props.valign, Some(VAlign::Bottom)); @@ -595,7 +595,7 @@ fn font_size_at_the_integer_limits_does_not_overflow() { #[test] fn empty_comments_do_not_swallow_the_rest() { for html in ["

ab

c

", "

ab

c

"] { - let doc = crate::html::import(html); + let doc = crate::doc::import(html); assert_eq!(doc.plain_text(Default::default()), "ab\nc", "{html}"); } } @@ -616,7 +616,7 @@ fn style_sheet_rules_apply_in_cascade_order() { ); let ps = paras(&f); let (plain, big, inline, digits) = (fmt_of(ps[0]), fmt_of(ps[1]), fmt_of(ps[2]), fmt_of(ps[3])); - assert_eq!((plain.color, plain.size, plain.bold, plain.italic), (Some(wordcraft_doc::Rgb(0, 255, 0)), Some(10.0), false, false)); + assert_eq!((plain.color, plain.size, plain.bold, plain.italic), (Some(craft_wordmodel::Rgb(0, 255, 0)), Some(10.0), false, false)); assert_eq!((big.size, big.bold), (Some(20.0), true)); assert_eq!(inline.size, Some(8.0)); assert_eq!(ps[3].align, Some(Align::Center)); @@ -662,7 +662,7 @@ fn a_page_opens_as_word_opens_one() { let FBlock::Para(cp) = &t.rows[0][0].blocks[0] else { panic!() }; assert_eq!(cp.spacing.line, Some(LineSpacing::Multiple(1.0))); let head = fmt_of(ps[1]); - assert_eq!((head.bold, head.size, head.color), (true, Some(24.0), Some(wordcraft_doc::Rgb(0, 0, 0)))); + assert_eq!((head.bold, head.size, head.color), (true, Some(24.0), Some(craft_wordmodel::Rgb(0, 0, 0)))); // A colour the heading inherits stays. // A generic family names a face on a page; pasted text keeps the document's font. let generic = parse_in("

a

b

", &|_| None, true); @@ -670,7 +670,7 @@ fn a_page_opens_as_word_opens_one() { assert_eq!(fonts, [Some("Arial".to_string()), Some(PAGE_FONT.to_string())]); assert_eq!(fmt_of(paras(&parse("

a

"))[0]).font, None); let inherited = parse_in("

Head

", &|_| None, true); - assert_eq!(fmt_of(paras(&inherited)[0]).color, Some(wordcraft_doc::Rgb(0, 0, 0x80))); + assert_eq!(fmt_of(paras(&inherited)[0]).color, Some(craft_wordmodel::Rgb(0, 0, 0x80))); // Automatic spacing where a p or heading leaves its margins out (layout drops it at the // top of the page and of a cell). let auto = |p: &Para| (p.spacing.before_auto, p.spacing.after_auto, p.spacing.before, p.spacing.after); @@ -684,7 +684,7 @@ fn a_page_opens_as_word_opens_one() { // The document says so, and the empty paragraph's mark keeps its size. let d = model::to_doc(&f); let props = |i: usize| match &*d.body[i] { - wordcraft_doc::Block::Para(p) => p.clone(), + craft_wordmodel::Block::Para(p) => p.clone(), b => panic!("{b:?}"), }; assert_eq!((props(0).props.space_before_auto, props(0).props.space_after_auto), (Some(true), Some(true))); @@ -719,16 +719,16 @@ fn cells_give_their_columns_widths() { #[test] fn cells_take_their_css_borders() { - use wordcraft_doc::props::{Border, BorderStyle}; + use craft_wordmodel::props::{Border, BorderStyle}; let f = parse( "
ab
", ); let FBlock::Table(t) = &f.blocks[0] else { panic!() }; // The style sheet's other cell properties apply too. - assert_eq!(t.rows[0][0].valign, Some(wordcraft_doc::props::VAlign::Bottom)); + assert_eq!(t.rows[0][0].valign, Some(craft_wordmodel::props::VAlign::Bottom)); let b = t.rows[0][0].borders.expect("borders"); assert_eq!(b.top, Some(Border::single(0.75))); - assert_eq!(b.bottom, Some(Border { style: BorderStyle::Double, width: 2.0, color: Some(wordcraft_doc::Rgb(255, 0, 0)), space: 0.0 })); + assert_eq!(b.bottom, Some(Border { style: BorderStyle::Double, width: 2.0, color: Some(craft_wordmodel::Rgb(255, 0, 0)), space: 0.0 })); assert_eq!(b.left.map(|l| l.style), Some(BorderStyle::None)); // A border of no width is none (OOXML has no zero-width line). assert_eq!(b.right.map(|r| r.style), Some(BorderStyle::None)); @@ -745,7 +745,7 @@ fn style_sheet_edge_cases() { ); let ps = paras(&f); // A rule right after statement at-rules still applies. - assert_eq!(fmt_of(ps[0]).color, Some(wordcraft_doc::Rgb(0, 0, 255))); + assert_eq!(fmt_of(ps[0]).color, Some(craft_wordmodel::Rgb(0, 0, 255))); // The body's margins aren't the paragraphs' indents. assert_eq!((ps[1].spacing.left, ps[1].spacing.before), (None, None)); // Bare numbers, percentages and keywords state no margin: automatic spacing stays. @@ -753,7 +753,7 @@ fn style_sheet_edge_cases() { // The `style` attribute wins even over a sheet with more declarations than are read. let many = (0..600).map(|i| format!("x-{i}: 1")).collect::>().join(";"); let f = parse(&format!("

c

")); - assert_eq!(fmt_of(paras(&f)[0]).color, Some(wordcraft_doc::Rgb(0, 255, 0))); + assert_eq!(fmt_of(paras(&f)[0]).color, Some(craft_wordmodel::Rgb(0, 255, 0))); } #[test] diff --git a/crates/html/src/lex.rs b/crates/html/src/lex.rs index bf6688ff..ef6b2063 100644 --- a/crates/html/src/lex.rs +++ b/crates/html/src/lex.rs @@ -110,6 +110,18 @@ impl<'a, 'o> Lexer<'a, 'o> { self.pos } + /// Goes back (or forward) to `pos`, a byte offset an earlier [`Lexer::pos`] gave, to read + /// from there again: for a reader that looked at a token and hands it back. An offset past the + /// end, or inside a character, moves to the next character boundary. + pub fn set_pos(&mut self, pos: usize) { + let mut pos = pos.min(self.src.len()); + while !self.src.is_char_boundary(pos) { + pos += 1; + } + self.pos = pos; + self.raw = None; + } + /// The raw content of element `name`, whose start tag was the last token: everything up to /// its end tag (any letter case), or to the end of the input when it has none. The end tag /// is consumed. For `style` and `script`, whose content is not markup. diff --git a/crates/html/src/lex/tests.rs b/crates/html/src/lex/tests.rs index 4f19e5b7..084caf70 100644 --- a/crates/html/src/lex/tests.rs +++ b/crates/html/src/lex/tests.rs @@ -175,3 +175,18 @@ proptest::proptest! { } } } + +#[test] +fn a_token_can_be_read_again() { + let mut l = Lexer::new("éx"); + assert_eq!(l.next(), Some(Token::Text("é"))); + let at = l.pos(); + assert!(matches!(l.next(), Some(Token::Start(t)) if t.name == "b")); + l.set_pos(at); + assert!(matches!(l.next(), Some(Token::Start(t)) if t.name == "b")); + // Inside a character or past the end: the next boundary. + l.set_pos(1); + assert!(matches!(l.next(), Some(Token::Start(t)) if t.name == "b")); + l.set_pos(1000); + assert_eq!(l.next(), None); +} diff --git a/crates/html/src/lib.rs b/crates/html/src/lib.rs index 45f29766..70f19746 100644 --- a/crates/html/src/lib.rs +++ b/crates/html/src/lib.rs @@ -34,6 +34,8 @@ #[doc = include_str!("../README.md")] struct ReadmeExamples; +#[cfg(feature = "flow")] +pub mod doc; mod entity; mod lex; diff --git a/crates/html/tests/doc.rs b/crates/html/tests/doc.rs new file mode 100644 index 00000000..fab55cc1 --- /dev/null +++ b/crates/html/tests/doc.rs @@ -0,0 +1,93 @@ +//! HTML documents (feature `flow`): the HTML parts of WordCraft's cross-format and MathML tests +//! (`crates/formats/src/tests.rs`, `mathml.rs`): equations through HTML, hostile input, entities. + +use craft_flow::{FBlock, Inline}; +use craft_html::doc::{import, inlines_html, parse}; +use std::time::{Duration, Instant}; + +fn equations(html: &str) -> Vec<(String, bool)> { + let mut out = Vec::new(); + for b in &parse(html).blocks { + if let FBlock::Para(p) = b { + for i in &p.inlines { + if let Inline::Equation { linear, display } = i { + out.push((linear.clone(), *display)); + } + } + } + } + out +} + +#[test] +fn html_export_round_trips_equations() { + for (linear, display) in [("y=2x_s", false), ("x=(-b±√(b^2-4ac))/2a", true), ("∑_(k=1)^n▒k", false), ("■(a&b@c&d)", false)] { + let html = inlines_html(&[Inline::Equation { linear: linear.into(), display }]); + assert!(html.starts_with("Before {html} after

")), vec![(linear.to_string(), display)], "{html}"); + } + // Text after the equation still arrives; an unclosed element ends with the input. + assert!(import("

x tail

").plain_text(Default::default()).contains("tail")); + assert!(!equations("

x").is_empty()); + let deep = format!("{}x{}", "".repeat(5000), "".repeat(5000)); + assert!(equations(&deep).len() <= 1); +} + +#[test] +fn math_over_the_token_cap_is_skipped_to_its_end() { + // Past the cap, the rest of the element is dropped, not read as body text. + let html = format!("

{}z tail

", "y".repeat(craft_math::mathml::MAX_NODES * 2)); + let mut text = String::new(); + for b in &parse(&html).blocks { + if let FBlock::Para(p) = b { + for i in &p.inlines { + if let Inline::Text(t, _) = i { + text.push_str(t); + } + } + } + } + assert_eq!(text.trim(), "tail"); +} + +#[test] +fn deep_nesting_is_bounded() { + let html = "
".repeat(5000) + "x"; + assert!(import(&html).plain_text(Default::default()).contains('x')); +} + +#[test] +fn entities_in_text() { + let d = import("

<&Aé &nosuchentity; A

"); + assert_eq!(d.plain_text(Default::default()).trim(), "<&Aé &nosuchentity; A"); + // An entity name longer than the scan window is left alone. + let long = format!("

&{};

", "a".repeat(40)); + assert!(import(&long).plain_text(Default::default()).contains(&"a".repeat(40))); +} + +/// A run of `&#` is read in linear time: four times as much takes well under the sixteen times +/// as long a quadratic read would (a ratio, so a busy machine does not fail it). +#[test] +fn a_run_of_references_takes_linear_time() { + let fastest = |n: usize| { + let text = "&#".repeat(n); + (0..3) + .map(|_| { + let t = Instant::now(); + let _ = import(&text).plain_text(Default::default()); + t.elapsed() + }) + .min() + .unwrap_or_default() + }; + let (small, large) = (fastest(125_000), fastest(500_000)); + assert!(large < small * 8 + Duration::from_millis(50), "{small:?} for 125,000, {large:?} for 500,000"); +} + +proptest::proptest! { + #[test] + fn fuzz_html(s in "(<(/)?(p|b|i|ul|ol|li|table|tr|td|th|pre|blockquote|a href=x|img src=data:,|br|h2|span style='color:red')>|[a-z &;#<>\"'=]){0,120}") { + let d = import(&s); + let _ = craft_html::doc::export(&d); + } +}