mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 16:54:45 +00:00
262 lines
10 KiB
Rust
262 lines
10 KiB
Rust
//! The text of an RTF file: `\uN` with its fallback characters, and 8-bit text in a code page.
|
|
|
|
use crate::codepage::{code_page_byte, multibyte_encoding_name};
|
|
|
|
fn multibyte(code_page: u16) -> Option<&'static encoding_rs::Encoding> {
|
|
encoding_rs::Encoding::for_label(multibyte_encoding_name(code_page)?.as_bytes())
|
|
}
|
|
|
|
/// Decodes the text tokens of one RTF file into `String`s the caller supplies.
|
|
///
|
|
/// The caller says which code page applies to each piece of 8-bit text (the current font's,
|
|
/// else the default font's, else the document's) and how many fallback characters follow a
|
|
/// `\uN` (`\ucN`, 1 when not set).
|
|
///
|
|
/// - `\uN` is one UTF-16 code unit in the signed/unsigned 16-bit range. A high surrogate pairs
|
|
/// only with a low surrogate in the very next `\u`; unmatched surrogates and out-of-range
|
|
/// values are dropped, matching the historical DesignCraft reader.
|
|
/// - After a `\uN`, the next `\ucN` characters are its fallback and are skipped: a text byte,
|
|
/// a `\'hh`, a control word and a control symbol each count as one ([`TextDecoder::skip`]).
|
|
/// The skipping ends with the group ([`TextDecoder::group`]).
|
|
/// - A single-byte code page maps each byte. A multibyte one (932, 936, 949, 950) streams
|
|
/// through one decoder, so a lead byte and its trail byte decode together even when a
|
|
/// `\'hh`, a text run or a formatting word falls between them, and a run that happens to
|
|
/// start `EF BB BF` is not taken for a UTF-8 byte order mark. A lead byte without its trail
|
|
/// byte reads as U+FFFD.
|
|
#[derive(Default)]
|
|
pub struct TextDecoder {
|
|
skip: usize,
|
|
high: Option<u16>,
|
|
mb: Option<encoding_rs::Decoder>,
|
|
mb_cp: u16,
|
|
}
|
|
|
|
impl TextDecoder {
|
|
pub fn new() -> TextDecoder {
|
|
TextDecoder::default()
|
|
}
|
|
|
|
/// A held high surrogate that nothing paired with.
|
|
fn flush_high(&mut self) {
|
|
self.high = None;
|
|
}
|
|
|
|
/// `\uN` with `uc` fallback characters to skip after it.
|
|
pub fn unicode(&mut self, param: i32, uc: usize, out: &mut String) {
|
|
self.finish_multibyte(out);
|
|
if !(-32_768..=65_535).contains(¶m) {
|
|
self.skip = uc;
|
|
return;
|
|
}
|
|
let u = i64::from(param).rem_euclid(65_536) as u16;
|
|
if (0xD800..0xDC00).contains(&u) {
|
|
self.flush_high();
|
|
self.high = Some(u);
|
|
} else {
|
|
match self.high.take() {
|
|
Some(h) if (0xDC00..0xE000).contains(&u) => {
|
|
if let Some(c) = char::decode_utf16([h, u]).next().and_then(Result::ok) {
|
|
out.push(c);
|
|
}
|
|
}
|
|
Some(_) | None if (0xDC00..0xE000).contains(&u) => {}
|
|
Some(_) | None => {
|
|
if let Some(c) = char::from_u32(u32::from(u)) {
|
|
out.push(c);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
self.skip = uc;
|
|
}
|
|
|
|
/// Is the control word or symbol just read a fallback character of the last `\uN`? Then it
|
|
/// is used up and the caller ignores it.
|
|
pub fn skip(&mut self) -> bool {
|
|
let skipped = self.skip > 0;
|
|
self.skip = self.skip.saturating_sub(1);
|
|
skipped
|
|
}
|
|
|
|
/// A character a control word or symbol stands for (`\emdash`, `\~`).
|
|
pub fn char(&mut self, c: char, out: &mut String) {
|
|
self.finish_multibyte(out);
|
|
self.flush_high();
|
|
out.push(c);
|
|
}
|
|
|
|
/// `\'hh`: one byte in `code_page`.
|
|
pub fn byte(&mut self, b: u8, code_page: u16, out: &mut String) {
|
|
if !self.skip() {
|
|
self.decode(&[b], code_page, out);
|
|
}
|
|
}
|
|
|
|
/// A run of text bytes in `code_page`.
|
|
pub fn bytes(&mut self, chunk: &[u8], code_page: u16, out: &mut String) {
|
|
let skip = self.skip.min(chunk.len());
|
|
self.skip -= skip;
|
|
let chunk = chunk.get(skip..).unwrap_or_default();
|
|
if !chunk.is_empty() {
|
|
self.decode(chunk, code_page, out);
|
|
}
|
|
}
|
|
|
|
fn decode(&mut self, chunk: &[u8], cp: u16, out: &mut String) {
|
|
self.flush_high();
|
|
if self.mb.is_none() && chunk.is_ascii() {
|
|
// ASCII is itself in every code page here; with no lead byte pending, as it is.
|
|
out.push_str(std::str::from_utf8(chunk).unwrap_or_default());
|
|
return;
|
|
}
|
|
let Some(encoding) = multibyte(cp) else {
|
|
self.finish_multibyte(out);
|
|
out.extend(chunk.iter().map(|b| code_page_byte(cp, *b)));
|
|
return;
|
|
};
|
|
// A code page switch ends the previous run before the new decoder starts.
|
|
if self.mb_cp != cp {
|
|
self.finish_multibyte(out);
|
|
}
|
|
self.mb_cp = cp;
|
|
let decoder = self.mb.get_or_insert_with(|| encoding.new_decoder_without_bom_handling());
|
|
let mut rest = chunk;
|
|
loop {
|
|
out.reserve(rest.len().saturating_mul(3).saturating_add(16));
|
|
let (result, read, _) = decoder.decode_to_string(rest, out, false);
|
|
rest = rest.get(read..).unwrap_or_default();
|
|
if !matches!(result, encoding_rs::CoderResult::OutputFull) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Ends the current multibyte run: a lead byte still waiting for its trail byte becomes
|
|
/// U+FFFD.
|
|
fn finish_multibyte(&mut self, out: &mut String) {
|
|
if let Some(mut decoder) = self.mb.take() {
|
|
out.reserve(16);
|
|
let _ = decoder.decode_to_string(&[], out, true);
|
|
}
|
|
}
|
|
|
|
/// A group opened or closed: the fallback of a `\uN` ends with its group, and so does a
|
|
/// multibyte run.
|
|
pub fn group(&mut self, out: &mut String) {
|
|
self.finish_multibyte(out);
|
|
self.skip = 0;
|
|
}
|
|
|
|
/// The text ends here (a paragraph end, a cell end, the end of the file): nothing waits
|
|
/// for a partner any longer.
|
|
pub fn finish(&mut self, out: &mut String) {
|
|
self.finish_multibyte(out);
|
|
self.flush_high();
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn units(ns: &[i32], uc: usize) -> String {
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
for n in ns {
|
|
d.unicode(*n, uc, &mut out);
|
|
}
|
|
d.finish(&mut out);
|
|
out
|
|
}
|
|
|
|
#[test]
|
|
fn surrogates_pair_only_with_the_next_unit() {
|
|
assert_eq!(units(&[-10179, -8704], 0), "\u{1F600}");
|
|
assert_eq!(units(&[0xD83D, 0xDE00], 0), "\u{1F600}", "written unsigned");
|
|
assert_eq!(units(&[8364], 0), "€");
|
|
assert_eq!(units(&[-10179], 0), "", "a lone high surrogate is dropped");
|
|
assert_eq!(units(&[-8704], 0), "", "a lone low surrogate is dropped");
|
|
assert_eq!(units(&[-8704, -10179], 0), "", "low then high are dropped");
|
|
assert_eq!(units(&[-10179, -10179, -8704], 0), "\u{1F600}", "an unpaired high is dropped before a pair");
|
|
assert_eq!(units(&[-10179, 65, -8704], 0), "A");
|
|
assert_eq!(units(&[65_536 + 65, -65_536 + 66], 0), "", "out-of-range units are dropped");
|
|
assert_eq!(units(&[i32::MAX, 65, i32::MIN], 0), "A", "extreme units are dropped");
|
|
}
|
|
|
|
#[test]
|
|
fn a_held_high_surrogate_does_not_pair_across_text() {
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.unicode(-10179, 0, &mut out);
|
|
d.bytes(b"x", 1252, &mut out);
|
|
d.unicode(-8704, 0, &mut out);
|
|
d.unicode(-10179, 0, &mut out);
|
|
d.char('\u{2014}', &mut out);
|
|
d.unicode(-10179, 1, &mut out);
|
|
// The fallback and a formatting group do not come between a pair.
|
|
d.bytes(b"?", 1252, &mut out);
|
|
d.group(&mut out);
|
|
d.unicode(-8704, 1, &mut out);
|
|
d.finish(&mut out);
|
|
assert_eq!(out, "x\u{2014}\u{1F600}");
|
|
}
|
|
|
|
#[test]
|
|
fn fallback_characters_are_skipped() {
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.unicode(8364, 1, &mut out);
|
|
d.bytes(b"? price", 1252, &mut out);
|
|
d.unicode(8220, 2, &mut out);
|
|
d.byte(0x93, 1252, &mut out);
|
|
assert!(d.skip(), "a control word is one fallback character");
|
|
assert!(!d.skip());
|
|
d.unicode(233, 3, &mut out);
|
|
d.bytes(b"e", 1252, &mut out);
|
|
d.group(&mut out);
|
|
d.bytes(b"AB", 1252, &mut out);
|
|
assert_eq!(out, "\u{20AC} price\u{201C}\u{E9}AB", "the skip ends with the group");
|
|
}
|
|
|
|
#[test]
|
|
fn single_byte_code_pages() {
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.bytes(&[b'C', b'a', b'f', 0xE9, b' '], 1252, &mut out);
|
|
d.byte(0xC4, 1251, &mut out);
|
|
d.byte(0x8E, 10000, &mut out);
|
|
d.byte(0x82, 437, &mut out);
|
|
d.byte(0xE9, 9999, &mut out);
|
|
assert_eq!(out, "Café Дééé");
|
|
}
|
|
|
|
#[test]
|
|
fn a_lead_byte_and_its_trail_byte_decode_together() {
|
|
// 日本 in Shift-JIS is 93 FA 96 7B: the first character as two `\'hh`, the second as a
|
|
// `\'hh` lead byte and a raw trail byte (`{`'s neighbour 0x7B is a text byte here).
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.byte(0x93, 932, &mut out);
|
|
d.byte(0xFA, 932, &mut out);
|
|
d.byte(0x96, 932, &mut out);
|
|
d.bytes(b"\x7bA", 932, &mut out);
|
|
assert_eq!(out, "日本A");
|
|
// A lead byte left at a group end, a code page switch or the end is U+FFFD.
|
|
for end in 0..3 {
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.byte(0x93, 932, &mut out);
|
|
match end {
|
|
0 => d.group(&mut out),
|
|
1 => d.bytes(&[0xE9], 1252, &mut out),
|
|
_ => d.finish(&mut out),
|
|
}
|
|
assert!(out.starts_with('\u{FFFD}'), "{end}: {out:?}");
|
|
}
|
|
// GBK text that starts EF BB BF is not a UTF-8 byte order mark.
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.bytes(&[0xEF, 0xBB, 0xBF, 0xC4, 0xE3], 936, &mut out);
|
|
d.finish(&mut out);
|
|
assert_eq!(out.chars().count(), 3, "{out:?}");
|
|
assert!(out.ends_with('\u{FFFD}') && !out.starts_with('\u{FEFF}'), "{out:?}");
|
|
// A long run decodes whole.
|
|
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
|
d.bytes(&[0x93, 0xFA].repeat(10_000), 932, &mut out);
|
|
assert_eq!(out.chars().count(), 10_000);
|
|
}
|
|
}
|