diff --git a/crates/export/src/flow.rs b/crates/export/src/flow.rs index 49ef73e38d..4349dd558e 100644 --- a/crates/export/src/flow.rs +++ b/crates/export/src/flow.rs @@ -140,7 +140,25 @@ pub(crate) fn list_item(text: &str) -> bool { /// paragraph spacing gives no other sign that a paragraph ends there, so such a line is left /// apart from the next rather than joined to it wrongly. fn ends_sentence(text: &str) -> bool { - text.trim_end().trim_end_matches(['"', '\'', '”', '’', '“', '»', ')', ']']).ends_with(['.', '!', '?', ':', ';', '…']) + text.trim_end() + .trim_end_matches(['"', '\'', '”', '’', '“', '»', ')', ']', '」', '』']) + .ends_with(['.', '!', '?', ':', ';', '…', '。', '!', '?']) +} + +/// Whether `c` is of the writing that puts no space between words: Chinese and Japanese +/// characters, their punctuation and fullwidth forms. A line that wraps between two of them +/// joins without a word space. +fn unspaced(c: char) -> bool { + matches!( + c, + '\u{3000}'..='\u{30FF}' // CJK punctuation, Hiragana, Katakana + | '\u{31F0}'..='\u{31FF}' // Katakana extension + | '\u{3400}'..='\u{4DBF}' // CJK extension A + | '\u{4E00}'..='\u{9FFF}' // CJK unified ideographs + | '\u{F900}'..='\u{FAFF}' // CJK compatibility ideographs + | '\u{FF01}'..='\u{FF9F}' // fullwidth forms, halfwidth Katakana + | '\u{20000}'..='\u{3FFFF}' // CJK extension B onwards + ) } /// `paras` (a page's paragraphs, in any order) top to bottom, with each paragraph that wraps onto @@ -169,10 +187,15 @@ pub(crate) fn join_wrapped<'a>(mut paras: Vec>>, level: impl Fn(&[Ru match out.last_mut() { Some((prev, 0)) if lvl == 0 && wraps_into(prev, ¶, [left, right]) => { // The line break was a word space, unless the line ends in a hyphen; the hyphen - // stays, as it may belong to the word ("well-known"). + // stays, as it may belong to the word ("well-known"). Chinese and Japanese text + // wraps between characters, with no space. + let unspaced_break = |end: &str| { + end.chars().next_back().is_some_and(unspaced) && para.first().and_then(|r| r.text.chars().next()).is_some_and(unspaced) + }; if let Some(end) = prev.last_mut() && !end.text.ends_with(char::is_whitespace) && !end.text.ends_with('-') + && !unspaced_break(&end.text) { end.text.push(' '); } diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index dc99894d1b..5af44d3c65 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -890,6 +890,9 @@ named in the schedule, Blue Finch Studio, who agree as follows to rent ); let r = rtf(std::slice::from_ref(&p)); assert_eq!(r.matches("{\\pard").count(), 2, "{r}"); + let md = markdown(std::slice::from_ref(&p)); + assert_eq!(md.trim().split("\n\n").count(), 2, "{md}"); + assert!(md.contains("of 4 Quay Street, and the tenant named in the schedule, "), "{md}"); // On a slide the last of the paragraphs set largest is the title; the sentence is one paragraph. let s = part(&pptx(std::slice::from_ref(&p), "Lease"), "ppt/slides/slide1.xml"); let slide: Vec = @@ -924,10 +927,21 @@ fn a_paragraph_breaks_where_its_line_does_not_run_on() { let second = |t: &str| line(t, 72.0, 240.0, 686.0); assert_eq!(word_texts(vec![first(full), second(next)]), [format!("{full} {next}")]); assert_eq!(word_texts(vec![first("Изпълнителят поддържа сървърите и мрежата на"), second("възложителя в работно състояние")]).len(), 1); + // Chinese and Japanese wrap between characters: their lines join without a space. Korean + // spaces its words, so its lines join with one. + let zh = ("本合同自双方签字之日起生效,有效期", "为一年,期满前一个月双方协商续签事宜"); + assert_eq!(word_texts(vec![first(zh.0), second(zh.1)]), [format!("{}{}", zh.0, zh.1)]); + assert_eq!( + word_texts(vec![first("この契約は双方が署名した日から"), second("一年間有効とする")]), + ["この契約は双方が署名した日から一年間有効とする"] + ); + assert_eq!(word_texts(vec![first("임차인은 방과 계단을 깨끗하게"), second("유지합니다")]), ["임차인은 방과 계단을 깨끗하게 유지합니다"]); let heading = |b: Block| Block { bold: true, ..b }; for (why, blocks) in [ ("a sentence ends", vec![first(&format!("{full}.")), second(next)]), ("a sentence ends in quotes", vec![first("Страните се договориха „за срок от една година.“"), second(next)]), + ("a Chinese sentence ends", vec![first("本合同自双方签字之日起生效。"), second(zh.1)]), + ("a Japanese sentence ends in quotes", vec![first("「この契約は一年間有効ですか?」"), second("双方が署名した日から")]), ("the line ends short", vec![line(next, 72.0, 240.0, 700.0), line(full, 72.0, 540.0, 686.0)]), ("paragraph space", vec![first(full), line(next, 72.0, 240.0, 676.0)]), ("an indent", vec![first(full), line(next, 90.0, 258.0, 686.0)]),