diff --git a/Cargo.lock b/Cargo.lock index ad48e08b..c7509a6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1050,6 +1050,15 @@ dependencies = [ "objc2-app-kit 0.3.2", ] +[[package]] +name = "craft-mime" +version = "0.1.0" +dependencies = [ + "chrono", + "encoding_rs", + "proptest", +] + [[package]] name = "craft-pcraft-format" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index dceee98a..82119032 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ craft-audio = { path = "crates/audio" } # No default features, so craft-audio's `encode` builds craft-media without `isobmff`. craft-media = { path = "crates/media", default-features = false } craft-media-testkit = { path = "crates/media-testkit" } +craft-mime = { path = "crates/mime" } craft-menu = { path = "crates/menu" } craft-pcraft-format = { path = "crates/pcraft-format" } craft-prepress = { path = "crates/prepress" } @@ -68,6 +69,8 @@ rayon = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" proptest = "1" +# Character sets of the WHATWG Encoding Standard (craft-mime). (Apache-2.0 OR MIT) AND BSD-3-Clause. +encoding_rs = "0.8" # All date/time conversion goes through chrono (no hand-rolled calendar code beyond the 400-year range extension; AGENTS.md). chrono = { version = "0.4.45", default-features = false, features = ["std"] } diff --git a/README.md b/README.md index 3f27d580..dd23658f 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` with the document as a type parameter, `TileRef`, the entry paths, the manifest decode limits and the migration runner. The document model stays in PhotoCraft. diff --git a/crates/mime/Cargo.toml b/crates/mime/Cargo.toml new file mode 100644 index 00000000..9d05e244 --- /dev/null +++ b/crates/mime/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "craft-mime" +description = "Internet messages and MIME: RFC 5322 headers, RFC 2045-2049 parts, encoded words, RFC 2231 parameters, transfer encodings, mbox and multipart/related lookups, read and written." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +encoding_rs = { workspace = true } +# Private: the weekday of a written date and whether a read date exists. Not in the public API. +chrono = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/mime/README.md b/crates/mime/README.md new file mode 100644 index 00000000..9fd1207b --- /dev/null +++ b/crates/mime/README.md @@ -0,0 +1,172 @@ +# craft-mime + +Internet messages and MIME, read and written: `.eml` files, what a mail server hands over, MHT / +MHTML web archives and mbox files. + +- **Reader.** `parse(bytes, &Limits)` gives a `Message`: a tree of `Part`s, each with its header + fields in order, its `Content-Type` and parameters, and its body with the transfer encoding + undone. Accessors give the media type, disposition, file name (RFC 2231 and encoded-words + decoded), `Content-ID`, `Content-Location` and the charset-decoded text. +- **Writer.** `Entity::text`, `Entity::file` and `Entity::multipart` build the parts, + `write_message` writes the message. Headers are folded at 78 columns, text is 7bit or + quoted-printable, files are base64 in 76-column lines, and boundaries are derived from the + content so the same input gives the same bytes. +- **The pieces**, for callers that work below the tree: `split_headers`, `write_field`, + `parse_params`, `decode_words` / `encode_words`, `transfer::{base64, base64_decode, qp_encode, + qp_decode}`, `charset::{decode, guess}`, `parse_address_list` / `format_address_list`, + `parse_date` / `format_date`, `split_multipart`, `mbox_messages` / `write_mbox_message`, and + `Resources` / `start_part` / `resolve_location` for `multipart/related`. + +Written from the public RFCs: 5322 (messages), 2045–2049 (MIME), 2047 (encoded-words), 2231 +(parameter values), 2387 and 2557 (`multipart/related`, MHTML), 4155 (mbox, `mboxrd` quoting). + +```rust +use craft_mime::{Limits, Resources, parse, start_part}; + +let message = parse(bytes, &Limits::default()); +let subject = message.root.headers.get("subject").map(craft_mime::decode_words); +for part in message.root.leaves() { + println!("{} {:?} {} bytes", part.media_type(), part.file_name(), part.data().len()); +} +// An MHT archive: the page and a picture it names. +let page = start_part(&message.root); +let picture = Resources::of(&message.root).find("images/logo.png", "https://example.test/page.html"); +``` + +## Hostile input + +The reader never fails and never panics. What can't be read is skipped; garbage gives a message +with no header fields and the bytes as its body. The work is linear in the input, and every +size, count and depth is capped by `Limits`. `Message::truncated` says that a limit left +something out. + +| Limit | Default | What happens past it | +|---|---|---| +| `max_input` | 1 GiB | the rest of the input is not read | +| `max_header_fields` | 2000 per header block | further fields are read past | +| `max_header_value` | 256 KiB | continuation lines are dropped | +| `max_header_block` | 4 MiB per header block | further fields are read past | +| `max_depth` | 20 (never more than 100) | a deeper multipart has no parts | +| `max_parts` | 1000, multiparts included | further parts are not read | +| `max_decoded` | 1 GiB of decoded bodies in all | a part that would pass it has an empty body | +| `reserved_fields` | none | named fields are kept past the field and block caps, `max_reserved_each` (4) of each, each cut at `max_reserved_value` (64) bytes with a `…` | +| `MAX_PARAMS` (constant) | 64 per header value | further parameters are not read | +| `MAX_ADDRESSES` (constant) | 10,000 per address list | further addresses are not read | + +The reader is tolerant of what real senders write: bare LF line ends, a missing blank line after +the headers, raw 8-bit header values, a multipart cut short, look-alike boundaries, missing +base64 padding and junk between base64 groups, stray `=` in quoted-printable, unquoted file +names with spaces, encoded-words where RFC 2231 belongs, comments in `Content-Type`, obsolete +date forms, addresses without commas between them. + +## Hostile values + +Nothing a caller gives the writer can add a header field, a part or a message: + +- `write_field` drops CR, LF and NUL from every value and writes nothing for a name that is not + a field name. `write_message` and `Entity::to_bytes` write every field through it. +- `format_address` strips from an address every character that could end it or start another; + `clean_id` does the same for message and content ids; `safe_media_type` turns anything that is + not `type/subtype`, and any `multipart/*`, into `application/octet-stream`. +- A multipart's subtype and parameter names must be tokens; parameter values are quoted. The + boundary is checked against the encoded parts and changed until none of them holds it. +- `write_mbox_message` strips the sender of white space and quotes `From ` lines. + +## Dates + +`parse_date` and `format_date` work on `DateTime`, a plain struct (year, month, day, hour, +minute, second, offset in minutes east of UTC). No date library is in the public API, so an app +converts it to whatever it keeps time in (`chrono`, `jiff`, a Unix timestamp). `chrono` is a +private dependency, used for the weekday and for whether a date exists. The raw `Date` text is +always there as `headers.get("date")`. + +## What stays in the apps + +- What a message means: which text part is the body, which parts are attachments or inline + pictures, how alternatives are chosen, HTML to text. +- Mail semantics: `Status` / `X-Status` flags, importance, keywords, kept-header policy, which + header fields have encoded-words decoded, message-id generation, recipient caps. +- S/MIME and PGP: detecting protected messages, canonical CRLF form, keeping the original + bytes. `split_headers` and `split_multipart` give the raw slices those need. +- File-name-to-media-type tables and default attachment names. +- `mailto:` links (RFC 6068), which are not MIME. +- For MHT: picking the page, reading its HTML, and the budget for pictures loaded. +- Strict base64 (RFC 4648 with refusals, URL-safe alphabet) for keys and tokens. This crate has + the mail kind only: an encoder, and a decoder that skips whatever is not base64. + +## Migration + +No app uses this crate yet. These tables say what replaces what. + +### MailCraft (`crates/mime`, `crates/charset`) + +| MailCraft | craft-mime | +|---|---| +| `header::split(bytes, top)` | `split_headers(bytes, top, &limits)`; the Exchange stamp fields go in `Limits::reserved_fields` | +| `header::Headers`, `Field`, `Params`, `parse_params` | the same names | +| `header::clean`, `write_field`, `is_token` | `clean`, `write_field` (now refuses a bad name), `is_field_name` | +| `words::decode_words`, `encode_words`, `encode_b`, `is_plain`, `WORD_BYTES` | the same names | +| `words::has_words` | stays (policy) | +| `transfer::base64_decode`, `base64`, `base64_lines`, `qp_decode`, `qp_encode` | `transfer::` the same names | +| `parse::decode(body, encoding)` | `transfer::decode` | +| `parse::split_multipart` | `split_multipart` | +| `parse::Walk` (the recursion over parts, depth and part caps) | `parse` / `parse_entity`, then a walk over `Part`s | +| `parse::is_media_type`, content id and file name extraction | `Part::declared_media_type`, `content_id`, `file_name`, `disposition` | +| `charset::decode`, `guess`, `unix_lines` (`mailcraft-charset`) | `charset::` the same names | +| `mailcraft_charset::base64` (strict) | stays | +| `address::parse_list`, `format_address`, `format_list` | `parse_address_list`, `format_address`, `format_address_list`, on `Mailbox { name, email }` | +| `mailcraft_time::parse_rfc5322`, `format_rfc5322` | `parse_date`, `format_date`, on `DateTime` | +| `mbox::mbox_messages`, `parse_mbox` | `mbox_messages` gives each message's bytes; MailCraft parses them | +| `mbox::write_mbox` | `write_mbox_message(out, sender, &date, bytes)` per message | +| `write::unstructured`, `kept` | `encode_unstructured`, `seven_bit_value` | +| `write::message_id` | `clean_id` | +| `write::Part`, `text_part`, `attachment_part`, `multipart` | `Entity`, `Entity::text`, `Entity::file(&File)`, `Entity::multipart` | +| `write::boundary`, `free_boundary` (`=_mc_` prefix, `counter`) | `Boundaries::new("=_mc_")` gives the same boundaries | +| `write::quoted`, `file_name_params`, `is_seven_bit_message` | `quoted`, `file_name_params`; the last is inside `Entity::file` | +| `media::safe_media_type` | `safe_media_type` | +| `media::media_type_for`, `default_name` | stay | +| `parse::read_headers`, `write::write_header`, `calendar_form`, `body` | stay (model mapping) | +| `protected`, `signed`, `rich`, `mailto` | stay | + +### WordCraft (`crates/formats/src/mht.rs`) + +| WordCraft | craft-mime | +|---|---| +| `split_headers`, `header` | `parse`, `Headers::get` | +| `param` | `parse_params` / `Part::content_type` (comments and quoted strings handled the same) | +| `collect`, `multipart_bodies`, `Part` | `parse`, `Part::leaves` | +| `quoted_printable`, `model::base64_decode` | done by `parse` (`transfer::qp_decode`, `transfer::base64_decode`) | +| the `start` lookup | `start_part` | +| the `by_location` / `by_id` maps and the loader's lookup | `Resources::of`, `Resources::find(src, base)` | +| `resolve`, `UriParts`, `remove_dots` | `resolve_location` | +| `html::decode_as(body, charset)` | stays, or `Part::text` | +| `is_mht`, `import`, `import_on`, the picture budget | stay | + +Differences a switch brings to WordCraft: the header block ends at the first line that is not a +header field (not only at a blank line); a delimiter line must hold nothing after the boundary +but white space; base64 with junk or missing padding decodes; boundaries longer than 70 +characters are accepted; the limits are the ones above (depth 20, 1000 parts by default, both +settable). + +## Origin + +| Part | Source | +|---|---| +| Headers, parameters, encoded-words, transfer encodings, address lists, multipart splitting, mbox, the writer's parts and boundaries, their tests | MailCraft `crates/mime` [a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5) | +| Character sets | MailCraft `crates/charset`, same commit | +| Date parsing and formatting | MailCraft `crates/time` (`text.rs`), same commit; the calendar arithmetic replaced by `chrono` | +| `multipart/related` lookups, location resolving, comment-aware parameters, the MHT tests | WordCraft `crates/formats/src/mht.rs` [639b813](https://github.com/storytold/wordcraft/commit/639b8133ea31efb63aac381a7ced8b8aed32aa30) | + +MailCraft's implementation is the base: it was the fuller one (the whole message grammar, a +writer, limits and property tests). From WordCraft come the parts MailCraft had no use for +(`Content-Location`, `start`, URI resolving) and one fix: a `;` or `"` inside a comment in +`Content-Type` no longer ends the value or supplies a false `boundary`. + +New here: the neutral `Part` tree, `Limits` as a caller's setting with `truncated` reporting, +the input and decoded-size caps, `write_field` refusing bad names, and the truncation, mutation +and injection tests in `tests/hostile.rs`. + +## Dependencies + +`encoding_rs` ((Apache-2.0 OR MIT) AND BSD-3-Clause) for the character sets of the WHATWG +Encoding Standard, and `chrono` (MIT OR Apache-2.0). No `unsafe`. diff --git a/crates/mime/src/address.rs b/crates/mime/src/address.rs new file mode 100644 index 00000000..f8a12bdd --- /dev/null +++ b/crates/mime/src/address.rs @@ -0,0 +1,413 @@ +//! Mailbox lists (RFC 5322 §3.4): `Ada , "Hopper, Grace" `. + +use crate::header::clean; +use crate::words::{decode_words, encode_words, is_plain}; + +/// Most addresses read from one header field. +pub const MAX_ADDRESSES: usize = 10_000; + +/// One mailbox of an address list: a display name and an address, either of which may be empty. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub struct Mailbox { + /// The display name, encoded-words decoded, quotes removed. + pub name: String, + /// The address (`local@domain`), as written. + pub email: String, +} + +/// What has been read of the mailbox being parsed. +#[derive(Default)] +struct Parts { + /// The words outside angle brackets, quoted strings unquoted: the display name. + text: String, + /// The same as written, quotes kept: the address when there are no angle brackets. + raw: String, + /// The last `(comment)`: the name, in the old `address (Name)` form. + comment: String, + /// What stood between `<` and `>`, and how long `text` was when it was read. + angle: Option<(String, usize)>, +} + +impl Parts { + /// White space between words: one space, none at the start. + fn space(&mut self) { + for s in [&mut self.text, &mut self.raw] { + if !s.is_empty() && !s.ends_with(' ') { + s.push(' '); + } + } + } + + /// End the mailbox: add it to `out` unless it is empty, and start afresh. + fn finish(&mut self, out: &mut Vec) { + let parts = std::mem::take(self); + let decoded = |s: &str| decode_words(s.trim()).trim().to_string(); + let comment = decoded(&parts.comment); + let address = match parts.angle { + Some((angle, _)) => { + let name = decoded(&parts.text); + Mailbox { name: if name.is_empty() { comment } else { name }, email: without_route(angle.trim()).to_string() } + } + None => { + let raw = parts.raw.trim(); + if raw.is_empty() { + // Nothing, or a comment on its own. + return; + } + let name = decoded(&parts.text); + if raw.contains('@') || (name == parts.text.trim() && !name.contains(' ')) { + // An address, or one word that can only be a local mailbox (`postmaster`). + Mailbox { name: comment, email: raw.to_string() } + } else { + // Words without an address: a name is all there is. + Mailbox { name, email: String::new() } + } + } + }; + if (!address.name.is_empty() || !address.email.is_empty()) && out.len() < MAX_ADDRESSES { + out.push(address); + } + } +} + +/// `user@host` from the obsolete routed form `@a,@b:user@host`. +fn without_route(angle: &str) -> &str { + if angle.starts_with('@') { angle.rsplit_once(':').map_or(angle, |(_, address)| address.trim()) } else { angle } +} + +/// Parse an address header (`To`, `Cc`, `From`…), leniently. +/// +/// Reads display names (quoted or not, with RFC 2047 encoded-words), `` addresses, bare +/// addresses, comments (`ada@example.org (Ada)` gives the name `Ada`), and groups +/// (`Team: a@example.org, b@example.org;` gives the two members). Mailboxes are separated by `,` +/// or `;`. Never fails: what makes no sense is kept as well as it can be, or dropped when empty. +pub fn parse_address_list(s: &str) -> Vec { + let mut out = Vec::new(); + let mut parts = Parts::default(); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + match c { + '"' => { + parts.raw.push('"'); + while let Some(q) = chars.next() { + match q { + '"' => break, + '\\' => { + if let Some(escaped) = chars.next() { + parts.text.push(escaped); + parts.raw.push('\\'); + parts.raw.push(escaped); + } + } + q => { + parts.text.push(q); + parts.raw.push(q); + } + } + } + parts.raw.push('"'); + } + '(' => { + let mut depth = 1usize; + let mut comment = String::new(); + while let Some(q) = chars.next() { + match q { + '\\' => comment.extend(chars.next()), + '(' => { + depth = depth.saturating_add(1); + comment.push(q); + } + ')' => { + depth = depth.saturating_sub(1); + if depth == 0 { + break; + } + comment.push(q); + } + q => comment.push(q), + } + } + parts.comment = comment; + parts.space(); + } + '<' => { + if let Some((_, at)) = parts.angle { + // A second address with no comma before it: the words since the first are its name. + let name = parts.text.get(at..).map(|tail| tail.trim_start().to_string()); + if name.is_some() { + parts.text.truncate(at); + } + parts.finish(&mut out); + parts.text = name.unwrap_or_default(); + } + let mut address = String::new(); + let mut quoted = false; + while let Some(q) = chars.next() { + match q { + '"' => { + quoted = !quoted; + address.push(q); + } + '\\' if quoted => { + address.push(q); + address.extend(chars.next()); + } + '>' if !quoted => break, + q => address.push(q), + } + } + parts.angle = Some((address, parts.text.len())); + } + ',' | ';' => parts.finish(&mut out), + // The name of a group ends here; its members follow. + ':' if parts.angle.is_none() => parts = Parts::default(), + ' ' | '\t' | '\r' | '\n' => parts.space(), + c => { + parts.text.push(c); + parts.raw.push(c); + } + } + if out.len() >= MAX_ADDRESSES { + return out; + } + } + parts.finish(&mut out); + out +} + +fn is_atext(b: u8) -> bool { + b.is_ascii_alphanumeric() || b"!#$%&'*+-/=?^_`{|}~".contains(&b) +} + +/// A display name as a header phrase: as it is when it is plain words, quoted when it holds +/// specials, encoded-words when it isn't printable ASCII. +fn phrase(name: &str) -> String { + if !is_plain(name) { + encode_words(name) + } else if name.split(' ').all(|word| !word.is_empty() && word.bytes().all(is_atext)) { + name.to_string() + } else { + format!("\"{}\"", name.replace('\\', "\\\\").replace('"', "\\\"")) + } +} + +/// Whether text can stand as an address as it is: nothing in it could end the address or start another. +fn is_addr_spec(s: &str) -> bool { + let mut quoted = false; + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c.is_control() { + return false; + } + if quoted { + match c { + '"' => quoted = false, + '\\' if chars.next().is_none_or(char::is_control) => return false, + _ => {} + } + } else if c == '"' { + quoted = true; + } else if c.is_whitespace() || "<>(),;:\\".contains(c) { + return false; + } + } + !quoted +} + +/// An address that is safe between angle brackets: kept when well formed, else stripped of every +/// character that could break out of it. +fn safe_email(email: &str) -> String { + let email = email.trim(); + if is_addr_spec(email) { + return email.to_string(); + } + email.chars().filter(|c| !c.is_control() && !c.is_whitespace() && !"<>(),;:\"\\".contains(*c)).collect() +} + +/// One mailbox for a header: `Name
`, or the address alone when there is no name (in +/// angle brackets unless it is a plain `local@domain`, so it can't be read as a name); `None` when +/// both are empty. +pub fn format_address(address: &Mailbox) -> Option { + let email = safe_email(&address.email); + let name = clean(&address.name); + let name = name.trim(); + if !name.is_empty() { + return Some(format!("{} <{email}>", phrase(name))); + } + if email.is_empty() { + return None; + } + let simple = email.contains('@') && !email.contains("=?") && email.bytes().all(|b| is_atext(b) || b == b'.' || b == b'@'); + Some(if simple { email } else { format!("<{email}>") }) +} + +/// An address list for a header, separated by `, `. +pub fn format_address_list(list: &[Mailbox]) -> String { + list.iter().filter_map(format_address).collect::>().join(", ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn a(name: &str, email: &str) -> Mailbox { + Mailbox { name: name.into(), email: email.into() } + } + + #[test] + fn the_usual_forms() { + assert_eq!(parse_address_list("ada@example.org"), [a("", "ada@example.org")]); + assert_eq!(parse_address_list(""), [a("", "ada@example.org")]); + assert_eq!(parse_address_list("Ada Lovelace "), [a("Ada Lovelace", "ada@example.org")]); + assert_eq!(parse_address_list(" Ada Lovelace\t< ada@example.org > "), [a("Ada Lovelace", "ada@example.org")]); + assert_eq!(parse_address_list(""), []); + assert_eq!(parse_address_list(" , ;, "), []); + } + + #[test] + fn lists() { + let got = parse_address_list("Ada , bob@example.org,\r\n \"Hopper, Grace\" ; carol@example.org"); + assert_eq!(got, [a("Ada", "ada@example.org"), a("", "bob@example.org"), a("Hopper, Grace", "grace@example.org"), a("", "carol@example.org")]); + } + + #[test] + fn quoted_names() { + assert_eq!(parse_address_list(r#""Lovelace, Ada" "#), [a("Lovelace, Ada", "ada@example.org")]); + assert_eq!(parse_address_list(r#""Ada \"The Countess\" L." "#), [a("Ada \"The Countess\" L.", "ada@example.org")]); + assert_eq!(parse_address_list(r#""a\\b" "#), [a("a\\b", "x@example.org")]); + assert_eq!(parse_address_list(r#""Ada L" "#), [a("Ada L", "ada@example.org")]); + assert_eq!(parse_address_list(r#""Dr." Ada "" "#), [a("Dr. Ada ", "ada@example.org")]); + assert_eq!(parse_address_list(r#""semi; colon: here" "#), [a("semi; colon: here", "s@example.org")]); + } + + #[test] + fn comments() { + assert_eq!(parse_address_list("ada@example.org (Ada Lovelace)"), [a("Ada Lovelace", "ada@example.org")]); + assert_eq!(parse_address_list("Ada (the first) Lovelace "), [a("Ada Lovelace", "ada@example.org")]); + assert_eq!(parse_address_list(" (Ada (nested \\) one))"), [a("Ada (nested ) one)", "ada@example.org")]); + assert_eq!(parse_address_list("(only a comment)"), []); + assert_eq!(parse_address_list("a@example.org (unterminated"), [a("unterminated", "a@example.org")]); + } + + #[test] + fn groups() { + let got = parse_address_list("Team: ada@example.org, Bob ;, carol@example.org"); + assert_eq!(got, [a("", "ada@example.org"), a("Bob", "bob@example.org"), a("", "carol@example.org")]); + assert_eq!(parse_address_list("undisclosed-recipients:;"), []); + assert_eq!(parse_address_list("Empty group: ; after@example.org"), [a("", "after@example.org")]); + } + + #[test] + fn encoded_words_in_names() { + assert_eq!(parse_address_list("=?UTF-8?B?QWTDoSBMw7N2ZWxhY2U=?= "), [a("Adá Lóvelace", "ada@example.org")]); + assert_eq!(parse_address_list("=?UTF-8?Q?Ad=C3=A1?= =?UTF-8?Q?_L?= "), [a("Adá L", "ada@example.org")]); + assert_eq!(parse_address_list("\"=?ISO-8859-1?Q?Andr=E9?=\" "), [a("André", "andre@example.org")]); + assert_eq!(parse_address_list("andre@example.org (=?ISO-8859-1?Q?Andr=E9?=)"), [a("André", "andre@example.org")]); + // A decoded name can't smuggle in an address. + assert_eq!( + parse_address_list("=?UTF-8?Q?Eve_=3Ceve@example.org=3E=2C?= "), + [a("Eve ,", "ada@example.org")] + ); + assert_eq!(parse_address_list("Renée "), [a("Renée", "renee@example.org")]); + } + + #[test] + fn names_without_addresses_and_local_mailboxes() { + assert_eq!(parse_address_list("undisclosed recipients"), [a("undisclosed recipients", "")]); + assert_eq!(parse_address_list("postmaster"), [a("", "postmaster")]); + assert_eq!(parse_address_list("=?UTF-8?Q?Ren=C3=A9e?="), [a("Renée", "")]); + assert_eq!(parse_address_list("Ada <>"), [a("Ada", "")]); + } + + #[test] + fn odd_addresses() { + assert_eq!(parse_address_list(r#""john doe"@example.org"#), [a("", r#""john doe"@example.org"#)]); + assert_eq!(parse_address_list(r#"J <"a>b"@example.org>"#), [a("J", r#""a>b"@example.org"#)]); + assert_eq!(parse_address_list("<@relay.example.org,@other.example.org:ada@example.org>"), [a("", "ada@example.org")]); + assert_eq!(parse_address_list("Ada Bob "), + [a("", "a@example.org"), a("Bob", "b@example.org"), a("", "c@example.org")] + ); + } + + #[test] + fn many_addresses_are_capped() { + let s = "a@example.org,".repeat(MAX_ADDRESSES + 100); + assert_eq!(parse_address_list(&s).len(), MAX_ADDRESSES); + let _ = parse_address_list(&"(".repeat(100_000)); + let _ = parse_address_list(&"<".repeat(100_000)); + let _ = parse_address_list(&"\"\\".repeat(100_000)); + } + + #[test] + fn writes_names_as_needed() { + assert_eq!(format_address_list(&[a("", "ada@example.org")]), "ada@example.org"); + assert_eq!(format_address_list(&[a("Ada Lovelace", "ada@example.org")]), "Ada Lovelace "); + assert_eq!(format_address_list(&[a("Lovelace, Ada", "ada@example.org")]), "\"Lovelace, Ada\" "); + assert_eq!(format_address_list(&[a("Ada \"C\" L\\", "ada@example.org")]), r#""Ada \"C\" L\\" "#); + assert_eq!(format_address_list(&[a("J. Doe", "j@example.org")]), "\"J. Doe\" "); + assert_eq!(format_address_list(&[a("Adá", "ada@example.org")]), "=?UTF-8?B?QWTDoQ==?= "); + assert_eq!(format_address_list(&[a("a", "a@example.org"), a("", ""), a("", "b@example.org")]), "a , b@example.org"); + assert_eq!(format_address_list(&[a("Name only", "")]), "Name only <>"); + assert_eq!(format_address_list(&[]), ""); + } + + #[test] + fn addresses_cannot_inject_recipients_or_headers() { + let evil = a("Eve\r\nBcc: x@example.org", "a@example.org>, "); + // A well-formed quoted local part is kept. + assert_eq!(format_address_list(&[a("", "\"john doe\"@example.org")]), "<\"john doe\"@example.org>"); + } + + #[test] + fn an_address_alone_reads_back_as_an_address() { + for email in ["ada@example.org", "postmaster", "=?UTF-8?Q?x?=", "\"a b\"", "\"john doe\"@example.org", "ünï@example.org", "a=?b@example.org"] + { + let list = [a("", email)]; + assert_eq!(parse_address_list(&format_address_list(&list)), list, "{email}"); + } + assert_eq!(format_address_list(&[a("", "postmaster")]), ""); + } + + fn name() -> impl proptest::strategy::Strategy { + use proptest::prelude::*; + prop_oneof![any::(), "[ -~]{0,30}", "[a-zA-Z .,\"\\\\()<>:;@=?]{0,20}"] + .prop_map(|s| s.replace(['\r', '\n', '\0'], "").trim().to_string()) + } + + proptest::proptest! { + #[test] + fn written_lists_read_back(names in proptest::collection::vec(name(), 0..5)) { + let list: Vec = names.iter().enumerate().map(|(i, n)| a(n, &format!("user{i}@example.org"))).collect(); + proptest::prop_assert_eq!(parse_address_list(&format_address_list(&list)), list); + } + + #[test] + fn any_text_parses(s in proptest::prelude::any::()) { + let _ = parse_address_list(&s); + } + + #[test] + fn address_soup_parses(s in "[a-c@<>\"(),;:\\\\ =?.]{0,40}") { + let _ = parse_address_list(&s); + } + + #[test] + fn any_address_writes_one_mailbox(n in proptest::prelude::any::(), e in proptest::prelude::any::()) { + let written = format_address_list(&[a(&n, &e)]); + proptest::prop_assert!(!written.contains(['\r', '\n', '\0'])); + proptest::prop_assert!(parse_address_list(&written).len() <= 1); + } + } +} diff --git a/crates/mime/src/charset.rs b/crates/mime/src/charset.rs new file mode 100644 index 00000000..59f45cd6 --- /dev/null +++ b/crates/mime/src/charset.rs @@ -0,0 +1,152 @@ +//! Bytes to text, for the character sets messages still arrive in. + +use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE, WINDOWS_1252}; + +/// Text from bytes in a named character set. +/// +/// Knows every character set of the WHATWG Encoding Standard by any of its names (UTF-8, the +/// Windows and ISO-8859 sets, KOI8, Shift_JIS, EUC-JP, ISO-2022-JP, GBK, Big5, EUC-KR, UTF-16…), +/// as a browser does. ISO-8859-1 is read as Windows-1252, its superset, because text that says +/// Latin-1 very often holds curly quotes and euro signs (the standard does the same). A missing +/// name, or US-ASCII with bytes above 0x7F, is guessed with [`guess`]. A name the standard doesn't +/// know, or one it reads as "replacement" (ISO-2022-KR, HZ…), is read as UTF-8, with U+FFFD for +/// what doesn't fit. +pub fn decode(bytes: &[u8], label: &str) -> String { + let label = label.trim().trim_matches(['"', '\'']).trim().to_ascii_lowercase(); + if matches!(label.as_str(), "" | "us-ascii" | "ascii" | "ansi_x3.4-1968" | "iso646-us") { + return guess(bytes); + } + let encoding = encoding(&label).unwrap_or(UTF_8); + // ASCII reads the same in every set that keeps it; UTF-16 and ISO-2022-JP don't. + if bytes.is_ascii() && encoding.is_ascii_compatible() { + return String::from_utf8_lossy(bytes).into_owned(); + } + if encoding == UTF_16LE || encoding == UTF_16BE { + // "UTF-16" names no byte order (it is read as little-endian): a byte order mark says which. + return encoding.decode(bytes).0.into_owned(); + } + // Elsewhere a byte order mark is text like any other (the reader drops one that opens a body). + encoding.decode_without_bom_handling(bytes).0.into_owned() +} + +/// The character set a name means: the standard's own names, and a few more that mail uses. +/// None for a name it doesn't know and for its "replacement" sets. +fn encoding(label: &str) -> Option<&'static Encoding> { + let label = match label { + "latin-1" => "latin1", + "latin9" | "latin-9" => "iso-8859-15", + "latin-2" => "latin2", + "cp-1250" => "windows-1250", + "cp-1252" => "windows-1252", + other => other, + }; + Encoding::for_label_no_replacement(label.as_bytes()) +} + +/// Text from bytes that don't say what they are: UTF-8 when they are valid UTF-8, else Windows-1252. +pub fn guess(bytes: &[u8]) -> String { + match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => WINDOWS_1252.decode_without_bom_handling(bytes).0.into_owned(), + } +} + +/// Line endings as `\n`: CRLF and a lone CR both become LF. +pub fn unix_lines(text: &str) -> String { + if !text.contains('\r') { + return text.to_string(); + } + text.replace("\r\n", "\n").replace('\r', "\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utf8_and_ascii() { + assert_eq!(decode("héllo €".as_bytes(), "UTF-8"), "héllo €"); + assert_eq!(decode("héllo".as_bytes(), "utf8"), "héllo"); + assert_eq!(decode(b"plain", "x-unknown"), "plain"); + assert_eq!(decode(b"a\xFFb", "utf-8"), "a\u{fffd}b"); + assert_eq!(decode(b"", "utf-8"), ""); + } + + #[test] + fn latin1_is_read_as_windows_1252() { + assert_eq!(decode(b"caf\xE9 \xA4", "ISO-8859-1"), "café ¤"); + assert_eq!(decode(b"\x93hi\x94 \x80 \x85 \x99", "iso-8859-1"), "“hi” € … ™"); + assert_eq!(decode(b"\x93hi\x94 \x80", "\"Windows-1252\""), "“hi” €"); + assert_eq!(decode(b"\x81\x8D\x8F\x90\x9D", "cp1252"), "\u{81}\u{8d}\u{8f}\u{90}\u{9d}"); + // Every byte gives a character. + let all: Vec = (0..=255).collect(); + assert_eq!(decode(&all, "windows-1252").chars().count(), 256); + } + + #[test] + fn latin9_has_the_euro() { + assert_eq!(decode(b"\xA4 \xBC\xBD \xE9 \xA6\xA8\xB4\xB8\xBE", "ISO-8859-15"), "€ Œœ é ŠšŽžŸ"); + assert_eq!(decode(b"\x80", "latin9"), "\u{80}"); + } + + #[test] + fn unlabelled_bytes_are_guessed() { + assert_eq!(decode("naïve".as_bytes(), ""), "naïve"); + assert_eq!(decode(b"na\xEFve", ""), "naïve"); + assert_eq!(decode(b"na\xEFve", "us-ascii"), "naïve"); + assert_eq!(guess(b"\xE2\x82\xAC"), "€"); + assert_eq!(guess(b"\xE2\x82"), "\u{e2}\u{201a}"); + } + + #[test] + fn line_endings() { + assert_eq!(unix_lines("a\r\nb\rc\nd\r\n"), "a\nb\nc\nd\n"); + assert_eq!(unix_lines("\r\r\n\r"), "\n\n\n"); + assert_eq!(unix_lines("none"), "none"); + } + + #[test] + fn unknown_charsets_fall_back_to_lossy_utf8() { + assert_eq!(decode("grüße".as_bytes(), "x-unknown"), "grüße"); + assert_eq!(decode(b"\xC0\xC1", "x-unknown"), "\u{fffd}\u{fffd}"); + // The standard's "replacement" sets are read the same way. + assert_eq!(decode("grüße".as_bytes(), "iso-2022-kr"), "grüße"); + assert_eq!(decode(b"a\xFFb", "hz-gb-2312"), "a\u{fffd}b"); + } + + #[test] + fn central_european_sets() { + // #178: windows-1250 and iso-8859-2 were read as UTF-8, every letter past ASCII a box. + assert_eq!(decode(b"Gvo\x9E\xF0e we\x92ve \x96 \x8E\x9A\xE8\xE6", "windows-1250"), "Gvožđe we’ve – Žščć"); + assert_eq!(decode(b"Gvo\x9E\xF0e", "cp1250"), "Gvožđe"); + assert_eq!(decode(b"Gvo\xBE\xF0e \xAE\xB9\xE8\xE6", "ISO-8859-2"), "Gvožđe Žščć"); + assert_eq!(decode(b"\xBE", "latin2"), "ž"); + let all: Vec = (0..=255).collect(); + for label in ["windows-1250", "iso-8859-2"] { + let text = decode(&all, label); + assert_eq!(text.chars().count(), 256, "{label}"); + assert!(!text.contains('\u{fffd}'), "{label}"); + } + } + + #[test] + fn other_sets_of_the_standard() { + assert_eq!(decode(b"\xD0\xD2\xC9\xD7\xC5\xD4", "koi8-r"), "привет"); + assert_eq!(decode(b"\x93\xFA\x96\x7B", "Shift_JIS"), "日本"); + // ISO-2022-JP is seven-bit, and still not ASCII. + assert_eq!(decode(b"\x1B$BF|K\\\x1B(B", "iso-2022-jp"), "日本"); + assert_eq!(decode(b"h\0i\0", "utf-16le"), "hi"); + assert_eq!(decode(b"\xFE\xFF\0h\0i", "utf-16"), "hi"); + assert_eq!(decode(b"\xA4", "latin9"), "€"); + } + + proptest::proptest! { + #[test] + fn any_bytes_in_any_set_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::(), 0..256), label in "\\PC{0,24}") { + let _ = (decode(&bytes, &label), guess(&bytes)); + for label in ["", "utf-16", "utf-16be", "iso-2022-jp", "shift_jis", "gb18030", "big5", "euc-kr", "windows-1250", "iso-8859-2"] { + let _ = decode(&bytes, label); + } + } + } +} diff --git a/crates/mime/src/date.rs b/crates/mime/src/date.rs new file mode 100644 index 00000000..9681d599 --- /dev/null +++ b/crates/mime/src/date.rs @@ -0,0 +1,236 @@ +//! The `Date` header (RFC 5322 §3.3) as plain numbers. +//! +//! No date type of any library is in the public API: a caller turns a [`DateTime`] into whatever +//! it keeps time in. `chrono` is used inside, for the weekday and for whether a date exists. + +use chrono::{Datelike, NaiveDate}; + +const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; +const WEEKDAYS: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; + +/// A wall-clock date and time and the zone it is said in. `2003-07-01 10:52:37 +0200` is +/// `offset_minutes: 120`; the instant is the wall clock minus the offset. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct DateTime { + pub year: i32, + /// 1 to 12. + pub month: u8, + /// 1 to 31. + pub day: u8, + pub hour: u8, + pub minute: u8, + pub second: u8, + /// Minutes east of UTC. + pub offset_minutes: i32, +} + +impl DateTime { + /// The date, when it exists and the clock is one. + fn date(&self) -> Option { + (self.hour < 24 && self.minute < 60 && self.second < 60) + .then(|| NaiveDate::from_ymd_opt(self.year, u32::from(self.month), u32::from(self.day))) + .flatten() + .filter(|_| (0..=9999).contains(&self.year)) + } + + /// The short names of the weekday and the month. + fn names(&self) -> Option<(&'static str, &'static str)> { + let weekday = WEEKDAYS.get(self.date()?.weekday().num_days_from_monday() as usize)?; + Some((weekday, MONTHS.get(usize::from(self.month).checked_sub(1)?)?)) + } + + /// `Tue Jul 1 08:52:37 2003`, as the separator line of an mbox file has it. `None` when + /// the date doesn't exist. + pub(crate) fn asctime(&self) -> Option { + let (weekday, month) = self.names()?; + Some(format!("{weekday} {month} {:>2} {:02}:{:02}:{:02} {:04}", self.day, self.hour, self.minute, self.second, self.year)) + } +} + +/// `Tue, 01 Jul 2003 10:52:37 +0200`. `None` when the date or the time doesn't exist (the year +/// must be 0 to 9999); the offset is kept within a day. +pub fn format_date(date: &DateTime) -> Option { + let (weekday, month) = date.names()?; + let offset = date.offset_minutes.clamp(-24 * 60, 24 * 60); + let sign = if offset < 0 { '-' } else { '+' }; + let a = offset.unsigned_abs(); + Some(format!( + "{weekday}, {:02} {month} {:04} {:02}:{:02}:{:02} {sign}{:02}{:02}", + date.day, + date.year, + date.hour, + date.minute, + date.second, + a / 60, + a % 60 + )) +} + +/// Parse an RFC 5322 date, leniently: a missing weekday or seconds, two-digit years, obsolete +/// zone names, comments. `None` when it is no date, or one that doesn't exist. A leap second +/// (60) is read as the last second of the minute; a zone that can't be read is UTC. +pub fn parse_date(s: &str) -> Option { + let s = strip_comments(s); + // The weekday, when present, ends at the first comma. + let s = match s.split_once(',') { + Some((day, rest)) if is_weekday(day) || day.trim().is_empty() => rest, + _ => s.as_str(), + }; + let mut it = s.split_whitespace(); + let day: u8 = it.next()?.parse().ok()?; + let month = it.next()?; + let month = MONTHS.iter().position(|m| month.get(..3).is_some_and(|p| p.eq_ignore_ascii_case(m)))?; + let year: i32 = match it.next()? { + y if y.len() <= 2 => { + // RFC 5322 §4.3: two-digit years 00–49 are 2000–2049, 50–99 are 1950–1999. + let y: i32 = y.parse::().ok()?.into(); + if y < 50 { 2000 + y } else { 1900 + y } + } + y if y.len() == 3 => 1900 + i32::from(y.parse::().ok()?), + y => y.parse::().ok()?.into(), + }; + let mut clock = it.next()?.split(':'); + let hour: u8 = clock.next()?.parse().ok()?; + let minute: u8 = clock.next()?.parse().ok()?; + let second: u8 = clock.next().map_or(Some(0), |x| x.parse().ok())?.min(59); + let offset_minutes = it.next().map_or(0, zone_offset); + let date = DateTime { year, month: u8::try_from(month).ok()?.checked_add(1)?, day, hour, minute, second, offset_minutes }; + date.date().map(|_| date) +} + +fn is_weekday(s: &str) -> bool { + let s = s.trim(); + WEEKDAYS.iter().any(|w| s.get(..3).is_some_and(|p| p.eq_ignore_ascii_case(w))) +} + +/// Minutes east of UTC for `+0200`, `-0500`, `Z`, `GMT`, `EST`…; unknown names are UTC. +fn zone_offset(z: &str) -> i32 { + let z = z.trim(); + if let Some(digits) = z.strip_prefix(['+', '-']) { + let digits: String = digits.chars().filter(char::is_ascii_digit).collect(); + let (h, m) = match digits.len() { + 4 => (digits.get(0..2), digits.get(2..4)), + 2 => (digits.get(0..2), Some("0")), + _ => (None, None), + }; + let (Some(h), Some(m)) = (h.and_then(|h| h.parse::().ok()), m.and_then(|m| m.parse::().ok())) else { return 0 }; + if h > 23 || m > 59 { + return 0; + } + let total = i32::from(h) * 60 + i32::from(m); + return if z.starts_with('-') { -total } else { total }; + } + match z.to_ascii_uppercase().as_str() { + "EDT" => -4 * 60, + "EST" | "CDT" => -5 * 60, + "CST" | "MDT" => -6 * 60, + "MST" | "PDT" => -7 * 60, + "PST" => -8 * 60, + _ => 0, + } +} + +/// Remove `(comments)`, which may nest. +fn strip_comments(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut depth = 0u32; + for c in s.chars() { + match c { + '(' => depth = depth.saturating_add(1), + ')' if depth > 0 => depth = depth.saturating_sub(1), + _ if depth == 0 => out.push(c), + _ => {} + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(year: i32, month: u8, day: u8, hour: u8, minute: u8, second: u8, offset_minutes: i32) -> DateTime { + DateTime { year, month, day, hour, minute, second, offset_minutes } + } + + #[test] + fn reads_the_usual_and_the_obsolete_forms() { + assert_eq!(parse_date("Tue, 1 Jul 2003 10:52:37 +0200"), Some(at(2003, 7, 1, 10, 52, 37, 120))); + assert_eq!(parse_date("1 Jul 2003 08:52:37 GMT"), Some(at(2003, 7, 1, 8, 52, 37, 0))); + assert_eq!(parse_date(" tuesday, 01 JULY 2003 08:52 -0500 (EST)"), Some(at(2003, 7, 1, 8, 52, 0, -300))); + assert_eq!(parse_date("(a (nested) comment) 1 Jul 03 08:52:60 PST"), Some(at(2003, 7, 1, 8, 52, 59, -480))); + assert_eq!(parse_date("1 Jul 99 00:00:00 Z").map(|d| d.year), Some(1999)); + assert_eq!(parse_date("1 Jul 103 00:00:00").map(|d| d.year), Some(2003)); + assert_eq!(parse_date("29 Feb 2024 12:00:00 +0530"), Some(at(2024, 2, 29, 12, 0, 0, 330))); + // A zone that can't be read is UTC. + for zone in ["+9999", "-12", "+123", "Mars", "+2460"] { + let want = if zone == "-12" { -720 } else { 0 }; + assert_eq!(parse_date(&format!("1 Jul 2003 08:52:37 {zone}")).map(|d| d.offset_minutes), Some(want), "{zone}"); + } + } + + #[test] + fn what_is_no_date_is_none() { + for bad in [ + "", + "next Tuesday", + "99 Abc 99999 99:99", + "31 Feb 2003 10:00:00", + "29 Feb 2023 10:00:00", + "1 Jul 2003", + "1 Jul 2003 24:00:00", + "1 Jul 2003 10:60", + "0 Jul 2003 10:00", + "1 Jul 99999 10:00", + "Foo, 1 Jul 2003 10:00", + "1 J 2003 10:00", + "1 Jül 2003 10:00", + ] { + assert_eq!(parse_date(bad), None, "{bad}"); + } + } + + #[test] + fn writes_what_it_reads() { + assert_eq!(format_date(&at(2003, 7, 1, 10, 52, 37, 120)).as_deref(), Some("Tue, 01 Jul 2003 10:52:37 +0200")); + assert_eq!(format_date(&at(1970, 1, 1, 0, 0, 0, 0)).as_deref(), Some("Thu, 01 Jan 1970 00:00:00 +0000")); + assert_eq!(format_date(&at(2024, 2, 29, 23, 59, 59, -570)).as_deref(), Some("Thu, 29 Feb 2024 23:59:59 -0930")); + assert_eq!(format_date(&at(2003, 7, 1, 0, 0, 0, i32::MIN)).as_deref(), Some("Tue, 01 Jul 2003 00:00:00 -2400")); + for bad in [ + at(2003, 2, 30, 0, 0, 0, 0), + at(2003, 0, 1, 0, 0, 0, 0), + at(2003, 13, 1, 0, 0, 0, 0), + at(2003, 7, 1, 24, 0, 0, 0), + at(10_000, 1, 1, 0, 0, 0, 0), + at(-1, 1, 1, 0, 0, 0, 0), + ] { + assert_eq!(format_date(&bad), None, "{bad:?}"); + } + assert_eq!(at(2003, 7, 1, 8, 52, 37, 0).asctime().as_deref(), Some("Tue Jul 1 08:52:37 2003")); + } + + proptest::proptest! { + #[test] + fn any_text_parses(s in proptest::prelude::any::()) { + let _ = parse_date(&s); + } + + #[test] + fn date_soup_parses(s in "[0-9a-zA-Z,:+() -]{0,40}") { + if let Some(date) = parse_date(&s) { + proptest::prop_assert!(format_date(&date).is_some()); + } + } + + #[test] + fn any_numbers_format_and_read_back(y in -5i32..10_005, mo in 0u8..14, d in 0u8..33, h in 0u8..26, mi in 0u8..62, s in 0u8..62, off in -1500i32..1500) { + let date = at(y, mo, d, h, mi, s, off); + if let Some(text) = format_date(&date) { + // Offsets are written whole within a day, and a day's worth of hours is no zone. + if off.abs() < 24 * 60 && y >= 1000 { + proptest::prop_assert_eq!(parse_date(&text), Some(date)); + } + } + } + } +} diff --git a/crates/mime/src/header.rs b/crates/mime/src/header.rs new file mode 100644 index 00000000..fdbf7d04 --- /dev/null +++ b/crates/mime/src/header.rs @@ -0,0 +1,708 @@ +//! Header fields: reading them (unfolding, parameters) and writing them (folding). + +use std::iter::Peekable; +use std::str::Chars; + +use crate::words::decode_words; +use crate::{Limits, charset}; + +/// Most parameters read from one `Content-Type` or `Content-Disposition`. +pub const MAX_PARAMS: usize = 64; + +/// Where a written header line is folded, when there is white space to fold at. +const FOLD_AT: usize = 78; + +/// A header field: its name as written, and its value unfolded and trimmed (encoded-words not decoded). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Field { + pub name: String, + pub value: String, +} + +/// The header fields of a message or of one MIME part, in order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Headers(pub Vec); + +impl Headers { + /// The value of the first field called `name` (any letter case). + pub fn get(&self, name: &str) -> Option<&str> { + self.0.iter().find(|f| f.name.eq_ignore_ascii_case(name)).map(|f| f.value.as_str()) + } + + /// The values of every field called `name` (any letter case), in order. + pub fn all<'a>(&'a self, name: &'a str) -> impl Iterator { + self.0.iter().filter(move |f| f.name.eq_ignore_ascii_case(name)).map(|f| f.value.as_str()) + } +} + +/// The line that starts at `pos`, without its line break, and where the next one starts. +pub(crate) fn line_at(bytes: &[u8], pos: usize) -> (&[u8], usize) { + let rest = bytes.get(pos..).unwrap_or_default(); + let (line, next) = match rest.iter().position(|b| *b == b'\n') { + Some(i) => (rest.get(..i).unwrap_or_default(), pos.saturating_add(i).saturating_add(1)), + None => (rest, bytes.len()), + }; + (line.strip_suffix(b"\r").unwrap_or(line), next) +} + +/// Split a message (or a MIME part) into its header fields and its body. +/// +/// Takes CRLF or bare LF line ends. Folded lines are joined. The header block ends at the first +/// empty line, or at the first line that can't be a header field (some senders forget the empty +/// line). At the `top` of a message a leading mbox `From ` line is skipped. A value in raw +/// 8-bit bytes is read as UTF-8 when it is valid UTF-8, else as Windows-1252. +/// +/// Bounded by `limits`: fields past [`Limits::max_header_fields`] or +/// [`Limits::max_header_block`] are read past, and a value stops growing at +/// [`Limits::max_header_value`]. Fields named in [`Limits::reserved_fields`] are kept past the +/// first two caps (see there). +pub fn split_headers<'a>(bytes: &'a [u8], top: bool, limits: &Limits) -> (Headers, &'a [u8]) { + let (headers, body, _) = split_counted(bytes, top, limits); + (headers, body) +} + +/// What [`split_counted`] keeps as it goes. +struct Kept<'l> { + limits: &'l Limits, + fields: Vec, + /// Bytes of names and values kept so far. + bytes: usize, + /// How many of each reserved field were kept. + reserved: Vec, + /// Whether a limit left something out. + cut: bool, +} + +impl Kept<'_> { + fn push(&mut self, current: Option<(String, Vec)>) { + let Some((name, value)) = current else { return }; + // Reserved fields get room past both caps, so that padding can't push them out; each is + // cut short, with a mark so that a cut value never reads as a shorter whole one. + let reserved = self.limits.reserved_fields.iter().position(|r| name.eq_ignore_ascii_case(r)); + if let Some(count) = reserved.and_then(|i| self.reserved.get_mut(i)) { + if *count < self.limits.max_reserved_each { + *count = count.saturating_add(1); + let value = value.trim_ascii(); + let mut text = charset::guess(value.get(..self.limits.max_reserved_value).unwrap_or(value)); + if value.len() > self.limits.max_reserved_value { + text.push('…'); + self.cut = true; + } + self.fields.push(Field { name, value: text }); + } else { + self.cut = true; + } + return; + } + let size = name.len().saturating_add(value.len()); + if self.fields.len() < self.limits.max_header_fields && self.bytes.saturating_add(size) <= self.limits.max_header_block { + self.bytes = self.bytes.saturating_add(size); + self.fields.push(Field { name, value: charset::guess(value.trim_ascii()) }); + } else { + self.cut = true; + } + } +} + +/// [`split_headers`], and whether a limit left something out. +pub(crate) fn split_counted<'a>(bytes: &'a [u8], top: bool, limits: &Limits) -> (Headers, &'a [u8], bool) { + let max_value = limits.max_header_value; + let mut kept = Kept { limits, fields: Vec::new(), bytes: 0, reserved: vec![0; limits.reserved_fields.len()], cut: false }; + let mut current: Option<(String, Vec)> = None; + let mut pos = 0; + let mut first = true; + let body = loop { + if pos >= bytes.len() { + break bytes.len(); + } + let (line, next) = line_at(bytes, pos); + if line.is_empty() { + break next; + } + if matches!(line.first(), Some(b' ' | b'\t')) { + match &mut current { + Some((_, value)) => { + if value.len().saturating_add(line.len()) <= max_value { + value.extend_from_slice(line); + } else { + kept.cut = true; + } + } + // A continuation of nothing: this is not a header block. + None => break pos, + } + } else if let Some((name, value)) = field(line) { + kept.push(current.take()); + kept.cut |= value.len() > max_value; + current = Some((name, value.get(..max_value).unwrap_or(value).to_vec())); + } else if !(top && first && (line.starts_with(b"From ") || line.starts_with(b">From "))) { + break pos; + } + first = false; + pos = next; + }; + kept.push(current.take()); + (Headers(kept.fields), bytes.get(body..).unwrap_or_default(), kept.cut) +} + +/// A header line as a name and a raw value, when it is one. +fn field(line: &[u8]) -> Option<(String, &[u8])> { + let colon = line.iter().position(|b| *b == b':')?; + let name = line.get(..colon)?.trim_ascii_end(); + if !is_field_name(name) { + return None; + } + Some((String::from_utf8_lossy(name).into_owned(), line.get(colon.saturating_add(1)..)?)) +} + +/// Whether bytes can be a header field name: printable ASCII without spaces or a colon. +pub fn is_field_name(name: &[u8]) -> bool { + !name.is_empty() && name.iter().all(|b| (33..=126).contains(b) && *b != b':') +} + +/// A structured header value: `text/plain; charset=utf-8`, `attachment; filename="a.txt"`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Params { + /// The value before the first `;`, lower case. + pub value: String, + params: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Param { + name: String, + value: String, + /// Came in the RFC 2231 form (`name*=`), which says its own character set. + extended: bool, +} + +impl Params { + /// A parameter as written (quotes removed). `name` in lower case. + pub fn get(&self, name: &str) -> Option<&str> { + self.params.iter().find(|p| p.name == name).map(|p| p.value.as_str()) + } + + /// A parameter that holds text for people (a file name): RFC 2231 decoded, or RFC 2047 + /// encoded-words decoded, which many senders put there although they shouldn't. + pub fn text(&self, name: &str) -> Option { + self.params.iter().find(|p| p.name == name).map(|p| if p.extended { p.value.clone() } else { decode_words(&p.value) }) + } + + /// The names of the parameters, lower case, in order. + pub fn names(&self) -> impl Iterator { + self.params.iter().map(|p| p.name.as_str()) + } +} + +/// One `name=value` piece of a parameter list, before RFC 2231 pieces are put together. +struct Piece { + name: String, + /// The position of a continuation (`name*1`). + index: Option, + /// Percent-encoded (`name*=` or `name*1*=`). + encoded: bool, + value: String, +} + +/// Step over the `(comment)` that `chars` stands at, nested ones and backslash escapes included. +/// Nothing is consumed, and `false` comes back, when it never closes: then the `(` is text, and +/// `open` (whether comments are still looked for) is cleared so that the scan stays linear. +fn skip_comment(chars: &mut Peekable>, open: &mut bool) -> bool { + if !*open || chars.peek() != Some(&'(') { + return false; + } + let mut ahead = chars.clone(); + let mut depth = 0usize; + while let Some(c) = ahead.next() { + match c { + '\\' => { + ahead.next(); + } + '(' => depth = depth.saturating_add(1), + ')' => { + depth = depth.saturating_sub(1); + if depth == 0 { + *chars = ahead; + return true; + } + } + _ => {} + } + } + *open = false; + false +} + +/// Parse a structured header value and its parameters, RFC 2231 forms included +/// (`filename*=utf-8''na%C3%AFve.txt`, and the continuations `filename*0=…; filename*1=…`). +/// +/// Comments (RFC 5322 §3.2.2) are stepped over where one can stand without being part of a +/// value: around the value before the first `;`, before a parameter's name, and after a quoted +/// value. So a `;` or a `"` inside a comment neither ends the value nor starts a parameter. +/// Inside an unquoted value a `(` is text (`filename=report (1).pdf`). +pub fn parse_params(s: &str) -> Params { + let mut chars = s.chars().peekable(); + let mut comments = true; + let mut main = String::new(); + while chars.peek().is_some() { + if skip_comment(&mut chars, &mut comments) { + main.push(' '); + } else if let Some(c) = chars.next() { + if c == ';' { + break; + } + main.push(c); + } + } + let value = main.split(|c: char| c.is_whitespace() || c == '(').find(|t| !t.is_empty()).unwrap_or("").to_ascii_lowercase(); + + let mut pieces: Vec = Vec::new(); + while pieces.len() < MAX_PARAMS { + loop { + while chars.next_if(|c| c.is_whitespace() || *c == ';').is_some() {} + if !skip_comment(&mut chars, &mut comments) { + break; + } + } + if chars.peek().is_none() { + break; + } + let mut name = String::new(); + while let Some(c) = chars.next_if(|c| *c != '=' && *c != ';') { + name.push(c); + } + let mut value = String::new(); + if chars.next_if_eq(&'=').is_some() { + while chars.next_if(|c| c.is_whitespace()).is_some() {} + if chars.next_if_eq(&'"').is_some() { + while let Some(c) = chars.next() { + match c { + '"' => break, + // Only `\"` and `\\` are escapes; a lone backslash (a Windows path) stays. + '\\' => value.push(chars.next_if(|n| *n == '"' || *n == '\\').unwrap_or('\\')), + c => value.push(c), + } + } + // Anything between the closing quote and the next `;` is noise. + while chars.peek().is_some_and(|c| *c != ';') { + if !skip_comment(&mut chars, &mut comments) { + chars.next(); + } + } + } else { + while let Some(c) = chars.next_if(|c| *c != ';') { + value.push(c); + } + value.truncate(value.trim_end().len()); + } + } + let name = name.trim().to_ascii_lowercase(); + if name.is_empty() { + continue; + } + let mut parts = name.split('*'); + let base = parts.next().unwrap_or("").to_string(); + let piece = match (parts.next(), parts.next()) { + (None, _) => Piece { name: base, index: None, encoded: false, value }, + (Some(""), _) => Piece { name: base, index: None, encoded: true, value }, + (Some(n), star) => match n.parse::() { + Ok(index) => Piece { name: base, index: Some(index), encoded: star == Some(""), value }, + Err(_) => continue, + }, + }; + pieces.push(piece); + } + + let mut params: Vec = Vec::new(); + for piece in &pieces { + if params.iter().any(|p| p.name == piece.name) { + continue; + } + let mut extended: Vec<&Piece> = pieces.iter().filter(|p| p.name == piece.name && (p.encoded || p.index.is_some())).collect(); + if extended.is_empty() { + params.push(Param { name: piece.name.clone(), value: piece.value.clone(), extended: false }); + continue; + } + // The RFC 2231 form wins over a plain parameter of the same name. + extended.sort_by_key(|p| p.index.unwrap_or(0)); + let mut charset = String::new(); + let mut bytes = Vec::new(); + for (i, p) in extended.iter().enumerate() { + if !p.encoded { + bytes.extend_from_slice(p.value.as_bytes()); + continue; + } + // The first piece starts with `charset'language'`. + let mut text = p.value.as_str(); + if i == 0 { + let mut it = text.splitn(3, '\''); + if let (Some(cs), Some(_language), Some(rest)) = (it.next(), it.next(), it.next()) { + charset = cs.to_string(); + text = rest; + } + } + bytes.extend(percent_decode(text)); + } + params.push(Param { name: piece.name.clone(), value: charset::decode(&bytes, &charset), extended: true }); + } + Params { value, params } +} + +fn percent_decode(s: &str) -> Vec { + let b = s.as_bytes(); + let hex = |i: usize| b.get(i).and_then(|c| char::from(*c).to_digit(16)).map(|d| d as u8); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0usize; + while let Some(&c) = b.get(i) { + match (c, hex(i.saturating_add(1)), hex(i.saturating_add(2))) { + (b'%', Some(h), Some(l)) => { + out.push((h << 4) | l); + i = i.saturating_add(3); + } + _ => { + out.push(c); + i = i.saturating_add(1); + } + } + } + out +} + +/// Remove what must never reach a header: CR, LF and NUL. +pub fn clean(value: &str) -> String { + value.chars().filter(|c| !matches!(c, '\r' | '\n' | '\0')).collect() +} + +/// Write one header field with its CRLF, folded at white space so lines stay within 78 +/// characters where the value allows. CR, LF and NUL in the value are dropped, so a value can +/// never start a new header field. A `name` that can't be a field name ([`is_field_name`]) +/// writes nothing and gives `false`. +/// +/// The value is written as it is otherwise: text that isn't printable ASCII goes through +/// [`crate::encode_unstructured`] (or [`crate::encode_words`]) first. +pub fn write_field(out: &mut Vec, name: &str, value: &str) -> bool { + if !is_field_name(name.as_bytes()) { + return false; + } + let value = clean(value); + let value = value.as_str(); + out.extend_from_slice(name.as_bytes()); + out.push(b':'); + if !value.is_empty() { + out.push(b' '); + } + let is_space = |c: char| c == ' ' || c == '\t'; + let mut col = name.len().saturating_add(2); + let mut pos = 0usize; + while pos < value.len() { + // A chunk is a run of white space and the word after it. + let rest = value.get(pos..).unwrap_or(""); + let word = rest.find(|c: char| !is_space(c)).unwrap_or(rest.len()); + let len = rest.get(word..).and_then(|w| w.find(is_space)).map_or(rest.len(), |i| word.saturating_add(i)); + // Fold before the white space, never before the first word and never to leave a blank line. + if pos > 0 && word > 0 && len > word && col.saturating_add(len) > FOLD_AT { + out.extend_from_slice(b"\r\n"); + col = 0; + } + out.extend_from_slice(rest.get(..len).unwrap_or("").as_bytes()); + col = col.saturating_add(len); + pos = pos.saturating_add(len.max(1)); + } + out.extend_from_slice(b"\r\n"); + true +} + +#[cfg(test)] +mod tests { + use super::*; + + const MAX_FIELDS: usize = Limits::DEFAULT.max_header_fields; + const MAX_VALUE: usize = Limits::DEFAULT.max_header_value; + + fn split(bytes: &[u8], top: bool) -> (Headers, &[u8]) { + split_headers(bytes, top, &Limits::default()) + } + + fn fields(bytes: &[u8]) -> Vec<(String, String)> { + split(bytes, true).0.0.into_iter().map(|f| (f.name, f.value)).collect() + } + + fn pair(n: &str, v: &str) -> (String, String) { + (n.to_string(), v.to_string()) + } + + fn written(name: &str, value: &str) -> String { + let mut out = Vec::new(); + assert!(write_field(&mut out, name, value)); + String::from_utf8(out).unwrap() + } + + #[test] + fn splits_headers_from_body() { + let (h, body) = split(b"Subject: Hi\r\nTo: bob@example.org\r\n\r\nBody\r\nmore", true); + assert_eq!(h.get("subject"), Some("Hi")); + assert_eq!(h.get("TO"), Some("bob@example.org")); + assert_eq!(h.get("cc"), None); + assert_eq!(body, b"Body\r\nmore"); + } + + #[test] + fn bare_lf_and_no_body() { + let (h, body) = split(b"A: 1\nB: 2\n\nBody\n", true); + assert_eq!(h.0.len(), 2); + assert_eq!(body, b"Body\n"); + assert_eq!(split(b"A: 1", true), (Headers(vec![Field { name: "A".into(), value: "1".into() }]), &b""[..])); + assert_eq!(split(b"A: 1\r\n", true).1, b""); + assert_eq!(split(b"", true), (Headers::default(), &b""[..])); + assert_eq!(split(b"\r\nBody", true), (Headers::default(), &b"Body"[..])); + } + + #[test] + fn folded_lines_are_joined() { + let got = fields(b"Subject: a\r\n b\r\n\tc\r\nX:\r\n folded value\r\n\r\n"); + assert_eq!(got, [pair("Subject", "a b\tc"), pair("X", "folded value")]); + } + + #[test] + fn odd_field_lines() { + assert_eq!(fields(b"Name : spaced\r\nEmpty:\r\nColons: a:b:c\r\n\r\n"), [pair("Name", "spaced"), pair("Empty", ""), pair("Colons", "a:b:c")]); + // Raw 8-bit values: UTF-8 when valid, else Windows-1252. + assert_eq!(fields("Subject: caf\u{e9}\n\n".as_bytes()), [pair("Subject", "café")]); + assert_eq!(fields(b"Subject: caf\xE9\n\n"), [pair("Subject", "café")]); + } + + #[test] + fn a_line_that_is_no_header_starts_the_body() { + let (h, body) = split(b"A: 1\r\nThis is text\r\nB: 2\r\n", true); + assert_eq!(h.0.len(), 1); + assert_eq!(body, b"This is text\r\nB: 2\r\n"); + assert_eq!(split(b"just text", true), (Headers::default(), &b"just text"[..])); + assert_eq!(split(b" leading space: x\r\n", true).1, b" leading space: x\r\n"); + assert_eq!(split(b"Bad Name: x\r\n", true).1, b"Bad Name: x\r\n"); + assert_eq!(split(b": no name\r\n", true).1, b": no name\r\n"); + } + + #[test] + fn an_mbox_from_line_is_skipped_at_the_top_only() { + let msg = b"From ada@example.org Sat Jan 3 01:05:34 1996\r\nSubject: x\r\n\r\nB"; + assert_eq!(fields(msg), [pair("Subject", "x")]); + assert_eq!(split(msg, false).0, Headers::default()); + assert_eq!(split(b"A: 1\r\nFrom here on\r\n", true).1, b"From here on\r\n"); + } + + #[test] + fn header_counts_and_sizes_are_capped() { + let many = "X: 1\r\n".repeat(MAX_FIELDS + 50) + "\r\nBody"; + let (h, body) = split(many.as_bytes(), true); + assert_eq!(h.0.len(), MAX_FIELDS); + assert_eq!(body, b"Body"); + let long = format!("X: start\r\n{}\r\nbody", " more\r\n".repeat(100_000)); + let (h, body) = split(long.as_bytes(), true); + assert!(h.get("x").is_some_and(|v| v.len() <= MAX_VALUE)); + assert_eq!(body, b"body"); + let one_line = format!("X: {}\r\n\r\n", "a".repeat(MAX_VALUE * 2)); + // The cut includes the space after the colon, which is then trimmed. + assert_eq!(split(one_line.as_bytes(), true).0.get("x").map(str::len), Some(MAX_VALUE - 1)); + } + + #[test] + fn each_header_limit_holds() { + let small = Limits { max_header_fields: 3, max_header_value: 10, max_header_block: 40, ..Limits::default() }; + let (h, body, cut) = split_counted(b"A: 1\r\nB: 2\r\nC: 3\r\nD: 4\r\n\r\nbody", true, &small); + assert_eq!((h.0.len(), body, cut), (3, &b"body"[..], true)); + let (h, _, cut) = split_counted(b"A: 0123456789abcdef\r\n more\r\n\r\n", true, &small); + assert_eq!((h.get("a"), cut), (Some("012345678"), true)); + let (h, _, cut) = split_counted(b"Aaaaaaaaaa: 123456789\r\nBbbbbbbbbb: 123456789\r\nC: 1\r\n\r\n", true, &small); + // The block cap counts names and values as they came: 20 + 20, and no room for `C`. + assert_eq!((h.0.len(), h.get("c"), cut), (2, None, true)); + assert!(!split_counted(b"A: 1\r\nB: 2\r\n\r\nbody", true, &small).2); + } + + #[test] + fn reserved_fields_survive_padding() { + let limits = Limits { max_header_fields: 2, reserved_fields: &["X-Stamp"], max_reserved_each: 2, max_reserved_value: 4, ..Limits::default() }; + let text = "P: 1\r\nP: 2\r\nP: 3\r\nx-stamp: Internal\r\nX-Stamp: ok\r\nX-Stamp: third\r\n\r\n"; + let (h, _, cut) = split_counted(text.as_bytes(), true, &limits); + let got: Vec<(&str, &str)> = h.0.iter().map(|f| (f.name.as_str(), f.value.as_str())).collect(); + // Cut values carry a mark, so that a cut one never reads as a shorter whole one. + assert_eq!(got, [("P", "1"), ("P", "2"), ("x-stamp", "Inte…"), ("X-Stamp", "ok")]); + assert!(cut); + } + + #[test] + fn a_name_that_is_no_field_name_writes_nothing() { + for name in ["", "Bad Name", "X:Y", "X\r\nBcc", "Sübject", "X\0"] { + let mut out = Vec::new(); + assert!(!write_field(&mut out, name, "v"), "{name:?}"); + assert!(out.is_empty()); + } + } + + #[test] + fn parameters() { + let p = parse_params("Text/Plain; charset=\"UTF-8\"; Format=flowed"); + assert_eq!(p.value, "text/plain"); + assert_eq!(p.get("charset"), Some("UTF-8")); + assert_eq!(p.get("format"), Some("flowed")); + assert_eq!(p.get("missing"), None); + assert_eq!(parse_params("attachment").value, "attachment"); + assert_eq!(parse_params("").value, ""); + assert_eq!(parse_params("text/html (a comment); charset=us-ascii").value, "text/html"); + } + + #[test] + fn quoted_parameters() { + let p = parse_params(r#"attachment; filename="a \"quoted\" name; v2.txt" ; size=12"#); + assert_eq!(p.get("filename"), Some("a \"quoted\" name; v2.txt")); + assert_eq!(p.get("size"), Some("12")); + assert_eq!(parse_params(r#"x; filename="C:\dir\file.txt""#).get("filename"), Some(r"C:\dir\file.txt")); + assert_eq!(parse_params(r#"x; a="back\\slash""#).get("a"), Some(r"back\slash")); + assert_eq!(parse_params(r#"x; a="unterminated"#).get("a"), Some("unterminated")); + assert_eq!(parse_params("x; filename=my file.pdf; b=2").get("filename"), Some("my file.pdf")); + assert_eq!(parse_params("x; flag; a=1;;; =novalue; b").get("a"), Some("1")); + } + + #[test] + fn rfc2231_extended_parameters() { + let p = parse_params("attachment; filename*=utf-8''na%C3%AFve%20r%C3%A9sum%C3%A9.pdf; filename=\"fallback.pdf\""); + assert_eq!(p.text("filename").as_deref(), Some("naïve résumé.pdf")); + let p = parse_params("attachment; filename=\"fallback.pdf\"; filename*=UTF-8'en'%E2%82%AC.txt"); + assert_eq!(p.text("filename").as_deref(), Some("€.txt")); + let p = parse_params("attachment; filename*=iso-8859-1''caf%E9.txt"); + assert_eq!(p.text("filename").as_deref(), Some("café.txt")); + // No character set: guessed. + assert_eq!(parse_params("x; filename*=''a%20b").text("filename").as_deref(), Some("a b")); + assert_eq!(parse_params("x; filename*=plain%41").text("filename").as_deref(), Some("plainA")); + assert_eq!(parse_params("x; filename*=utf-8''bad%ZZ%4").text("filename").as_deref(), Some("bad%ZZ%4")); + } + + #[test] + fn rfc2231_continuations() { + let p = parse_params("attachment; filename*1=\"second half.txt\"; filename*0=\"first half and \""); + assert_eq!(p.text("filename").as_deref(), Some("first half and second half.txt")); + let p = parse_params("attachment; filename*0*=utf-8''%E2%82%AC%20; filename*1*=r%C3%A9sum%C3%A9; filename*2=\" (1).pdf\""); + assert_eq!(p.text("filename").as_deref(), Some("€ résumé (1).pdf")); + // A character cut between two pieces. + let p = parse_params("x; n*0*=utf-8''caf%C3; n*1*=%A9"); + assert_eq!(p.text("n").as_deref(), Some("café")); + assert_eq!(parse_params("x; n*x=1; n*99999999999=2; m=3").get("m"), Some("3")); + } + + #[test] + fn encoded_words_in_parameters() { + let p = parse_params("application/pdf; name=\"=?UTF-8?B?csOpc3Vtw6kucGRm?=\""); + assert_eq!(p.text("name").as_deref(), Some("résumé.pdf")); + assert_eq!(p.get("name"), Some("=?UTF-8?B?csOpc3Vtw6kucGRm?=")); + // The extended form is not decoded twice. + let p = parse_params("x; name*=utf-8''%3D%3FUTF-8%3FQ%3Fa%3F%3D"); + assert_eq!(p.text("name").as_deref(), Some("=?UTF-8?Q?a?=")); + } + + #[test] + fn comments_neither_end_the_value_nor_start_a_parameter() { + // WordCraft's MHT reader: a quote or a `;` in a comment must not hide or fake a boundary. + for comment in [r#"(10" screen)"#, r#"(outer (10" screen); boundary=wrong)"#, r#"(10\)" screen; boundary=wrong)"#] { + let p = parse_params(&format!("multipart/related {comment}; boundary=\"----=_B\"")); + assert_eq!((p.value.as_str(), p.get("boundary")), ("multipart/related", Some("----=_B")), "{comment}"); + let p = parse_params(&format!("multipart/related; {comment} boundary=right; name=\"a\" {comment}; z=1")); + assert_eq!((p.get("boundary"), p.get("z")), (Some("right"), Some("1")), "{comment}"); + } + for name in [r#"name="notes; boundary=wrong""#, r#"name="notes\"; boundary=wrong""#, r#"name="notes (10\" screen; boundary=wrong""#] { + let p = parse_params(&format!("multipart/related; {name}; boundary=\"----=_B\"")); + assert_eq!(p.get("boundary"), Some("----=_B"), "{name}"); + } + assert_eq!(parse_params("(first) text/html (second); charset=x").value, "text/html"); + // In an unquoted value a parenthesis is text, and a comment that never closes is too. + assert_eq!(parse_params("x; filename=report (1).pdf").get("filename"), Some("report (1).pdf")); + let p = parse_params("text/html (oops; charset=utf-8"); + assert_eq!((p.value.as_str(), p.get("charset")), ("text/html", Some("utf-8"))); + // Linear on brackets that never close. + let _ = parse_params(&format!("x; a=\"b\"{}", "(".repeat(200_000))); + let _ = parse_params(&"(;".repeat(100_000)); + } + + #[test] + fn many_parameters_are_capped() { + let s = format!("x{}", (0..500).map(|i| format!("; p{i}=v")).collect::()); + let p = parse_params(&s); + assert_eq!(p.get("p0"), Some("v")); + assert_eq!(p.get("p499"), None); + } + + #[test] + fn short_fields_are_written_on_one_line() { + assert_eq!(written("Subject", "Hello there"), "Subject: Hello there\r\n"); + assert_eq!(written("X-Empty", ""), "X-Empty:\r\n"); + } + + #[test] + fn long_fields_fold_at_white_space() { + let value = "word ".repeat(40); + let value = value.trim_end(); + let text = written("Subject", value); + assert!(text.ends_with("\r\n")); + let lines: Vec<&str> = text.trim_end_matches("\r\n").split("\r\n").collect(); + assert!(lines.len() > 2); + for (i, line) in lines.iter().enumerate() { + assert!(line.len() <= 78, "{line}"); + assert_eq!(line.starts_with(' '), i > 0); + assert!(!line.trim().is_empty()); + } + // Unfolding gives the value back. + assert_eq!(fields(format!("{text}\r\n").as_bytes()), [pair("Subject", value)]); + } + + #[test] + fn folding_keeps_every_space() { + for value in [ + "a b", + "tab\tseparated\twords", + &format!("{} {} {}", "x".repeat(50), "y".repeat(50), "z".repeat(50)), + &format!("{} \t {}", "x".repeat(70), "y".repeat(70)), + ] { + let text = written("X-Test", value); + assert_eq!(fields(format!("{text}\r\n").as_bytes()), [pair("X-Test", value)], "{text}"); + } + } + + #[test] + fn a_word_that_cannot_fold_stays_whole() { + let long = "x".repeat(300); + assert_eq!(written("X", &long), format!("X: {long}\r\n")); + // No fold before the first word, and none that would leave a blank line. + assert_eq!(written("X", &format!("{long} ")), format!("X: {long} \r\n")); + assert_eq!(written("X", &format!(" {long}")), format!("X: {long}\r\n")); + } + + #[test] + fn line_breaks_cannot_be_injected() { + assert_eq!(written("Subject", "Hi\r\nBcc: eve@example.org"), "Subject: HiBcc: eve@example.org\r\n"); + assert_eq!(written("Subject", "a\nb\rc\0d"), "Subject: abcd\r\n"); + assert_eq!(clean("\r\n\0"), ""); + } + + proptest::proptest! { + #[test] + fn any_bytes_split(data in proptest::collection::vec(proptest::prelude::any::(), 0..400)) { + let (h, body) = split(&data, true); + proptest::prop_assert!(body.len() <= data.len()); + for f in &h.0 { + proptest::prop_assert!(is_field_name(f.name.as_bytes())); + } + } + + #[test] + fn any_parameters_parse(s in "[a-z*0-9=;\"'%\\\\ é]{0,60}") { + let p = parse_params(&s); + let _ = p.text("a"); + } + + #[test] + fn written_fields_read_back(value in "[ -~\t]{0,300}") { + let text = written("X-Any", &value); + proptest::prop_assert!(!text.trim_end_matches("\r\n").split("\r\n").any(|l| l.trim().is_empty())); + let (h, _) = split(format!("{text}\r\n").as_bytes(), true); + proptest::prop_assert_eq!(h.get("x-any"), Some(value.trim_matches([' ', '\t']))); + } + } +} diff --git a/crates/mime/src/lib.rs b/crates/mime/src/lib.rs new file mode 100644 index 00000000..c59d6e9c --- /dev/null +++ b/crates/mime/src/lib.rs @@ -0,0 +1,103 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! Internet messages and MIME, read and written. +//! +//! [`parse()`] turns the bytes of a message (an `.eml` file, an MHT web archive, what a server +//! hands over) into a [`Message`]: a tree of [`Part`]s, each with its header fields, its content +//! type and parameters, and its body with the transfer encoding undone. [`Entity`] and +//! [`write_message`] write one. Beside them are the pieces both are made of, for callers that +//! work below the tree: header fields ([`split_headers`], [`write_field`]), parameters +//! ([`parse_params`]), encoded-words ([`decode_words`], [`encode_words`]), transfer encodings +//! ([`transfer`]), character sets ([`charset`]), address lists ([`parse_address_list`], +//! [`format_address_list`]), dates ([`parse_date`], [`format_date`]), mbox files +//! ([`mbox_messages`], [`write_mbox_message`]) and the lookups of `multipart/related` +//! ([`Resources`]). +//! +//! Written from the public RFCs: 5322 (messages), 2045–2049 (MIME), 2047 (encoded-words), 2231 +//! (parameter values), 2387 and 2557 (`multipart/related`, MHTML) and 4155 (mbox). +//! +//! Input is hostile. Nothing here fails or panics on any bytes: what can't be read is skipped, +//! sizes, counts and nesting are capped ([`Limits`]), and the work is linear in the input. +//! Nothing written can be bent into extra header fields by the values it is given: every header +//! value written loses its CR, LF and NUL. +//! +//! What a message means (which part is the body, which parts are files, what a flag header +//! says) is the caller's: this crate has no notion of a mailbox or a document. + +pub mod charset; +pub mod transfer; + +mod address; +mod date; +mod header; +mod mbox; +mod part; +mod related; +mod words; +mod write; + +pub use address::{MAX_ADDRESSES, Mailbox, format_address, format_address_list, parse_address_list}; +pub use date::{DateTime, format_date, parse_date}; +pub use header::{Field, Headers, MAX_PARAMS, Params, clean, is_field_name, parse_params, split_headers, write_field}; +pub use mbox::{MboxMessages, mbox_messages, write_mbox_message}; +pub use part::{Body, Message, Part, parse, parse_entity, split_multipart}; +pub use related::{Resources, resolve_location, start_part}; +pub use words::{WORD_BYTES, decode_words, encode_b, encode_words, is_plain}; +pub use write::{Boundaries, Entity, File, clean_id, encode_unstructured, file_name_params, quoted, safe_media_type, seven_bit_value, write_message}; + +/// How much of a message is read. Whatever a limit leaves out is skipped, never an error, and +/// [`Message::truncated`] says that something was. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Most bytes of input looked at; what lies past them is not read. + pub max_input: usize, + /// Most header fields kept from one header block; the rest are read past. + pub max_header_fields: usize, + /// Longest header value kept, in bytes; continuation lines past it are dropped. + pub max_header_value: usize, + /// Most header bytes kept from one header block in all (names and values together). + pub max_header_block: usize, + /// Deepest nesting of multiparts that is opened; a multipart deeper than this has no parts. + /// Never more than [`Limits::HARD_MAX_DEPTH`], whatever is asked for. + pub max_depth: usize, + /// Most MIME parts read from one message, multiparts themselves included. + pub max_parts: usize, + /// Most bytes of decoded part bodies kept from one message in all; a part that would pass + /// it has an empty body. + pub max_decoded: usize, + /// Header fields (names, any letter case) that are kept past `max_header_fields` and + /// `max_header_block`, so that a sender's padding can't push them out: at most + /// `max_reserved_each` of each, each value cut at `max_reserved_value` bytes with a `…` + /// after it when it was longer. + pub reserved_fields: &'static [&'static str], + /// See `reserved_fields`. + pub max_reserved_each: usize, + /// See `reserved_fields`. + pub max_reserved_value: usize, +} + +impl Limits { + /// The deepest nesting any [`Limits`] can ask for: the reader recurses once per level. + pub const HARD_MAX_DEPTH: usize = 100; + + /// What [`Limits::default`] gives. + pub const DEFAULT: Limits = Limits { + max_input: 1 << 30, + max_header_fields: 2000, + max_header_value: 256 * 1024, + max_header_block: 4 * 1024 * 1024, + max_depth: 20, + max_parts: 1000, + max_decoded: 1 << 30, + reserved_fields: &[], + max_reserved_each: 4, + max_reserved_value: 64, + }; +} + +impl Default for Limits { + fn default() -> Self { + Self::DEFAULT + } +} diff --git a/crates/mime/src/mbox.rs b/crates/mime/src/mbox.rs new file mode 100644 index 00000000..44cd2c63 --- /dev/null +++ b/crates/mime/src/mbox.rs @@ -0,0 +1,237 @@ +//! mbox files (RFC 4155, in the `mboxrd` flavour): messages one after another, each after a +//! `From ` line. + +use crate::date::DateTime; +use crate::header; + +/// The messages of an mbox file, one at a time, each as the bytes of one message (for +/// [`crate::parse`]). +/// +/// A message starts at a line that begins with `From ` and either follows an empty line (or +/// starts the file) or holds a time (`12:08:34`), as the separator lines of every mbox writer +/// do. In the messages `>From ` lines lose one `>` (the `mboxrd` rule, which also reads `mboxo` +/// files), and the empty line that ended a message in the file is not part of it. Bytes before +/// the first separator are a message too, so a lone `.eml` gives one message. A message that is +/// nothing but white space is skipped. +/// +/// Linear in the input; a caller that takes only so many messages stops the iterator early. +pub fn mbox_messages(bytes: &[u8]) -> MboxMessages<'_> { + MboxMessages { bytes, pos: 0, current: None, previous_blank: true, done: false } +} + +/// The iterator behind [`mbox_messages`]. +#[derive(Clone, Debug)] +pub struct MboxMessages<'a> { + bytes: &'a [u8], + pos: usize, + current: Option>, + previous_blank: bool, + done: bool, +} + +impl Iterator for MboxMessages<'_> { + type Item = Vec; + + fn next(&mut self) -> Option> { + while !self.done { + if self.pos >= self.bytes.len() { + self.done = true; + if let Some(m) = finish(self.current.take()) { + return Some(m); + } + break; + } + let (line, next) = header::line_at(self.bytes, self.pos); + let mut out = None; + if line.starts_with(b"From ") && (self.previous_blank || has_time(line)) { + out = finish(self.current.take()); + self.current = Some(Vec::new()); + } else { + let whole = self.bytes.get(self.pos..next).unwrap_or_default(); + // `>From ` was `From ` before it was stored; `>>From ` was `>From `. + let quoted = + line.iter().position(|b| *b != b'>').is_some_and(|i| i > 0 && line.get(i..).is_some_and(|rest| rest.starts_with(b"From "))); + let unquoted = if quoted { whole.get(1..).unwrap_or_default() } else { whole }; + match &mut self.current { + Some(message) => message.extend_from_slice(unquoted), + // Before any separator: blank lines are nothing, anything else is a message. + None if line.is_empty() => {} + None => self.current = Some(unquoted.to_vec()), + } + } + self.previous_blank = line.is_empty(); + self.pos = next; + if out.is_some() { + return out; + } + } + None + } +} + +/// A message cut out of an mbox, without the empty line that ended it there. +fn finish(message: Option>) -> Option> { + let mut bytes = message?; + if bytes.ends_with(b"\n\r\n") { + bytes.truncate(bytes.len().saturating_sub(2)); + } else if bytes.ends_with(b"\n\n") { + bytes.truncate(bytes.len().saturating_sub(1)); + } + if bytes.iter().all(u8::is_ascii_whitespace) { None } else { Some(bytes) } +} + +/// Whether a line holds a clock time (`hh:mm`), as mbox separator lines do. +fn has_time(line: &[u8]) -> bool { + line.windows(5).any(|w| matches!(w, [a, b, b':', c, d] if a.is_ascii_digit() && b.is_ascii_digit() && c.is_ascii_digit() && d.is_ascii_digit())) +} + +/// Add one message to an mbox file. +/// +/// `message` (CRLF or LF line ends, kept as they are so that attachments survive byte for byte) +/// goes after a `From sender date` line and before an empty line. Lines of it that begin with +/// `From `, or with `>From `, get one more `>` (the `mboxrd` rule), which [`mbox_messages`] +/// takes off again. +/// +/// `sender` loses its white space and control characters, so that it can't end the separator +/// line, and is `MAILER-DAEMON` when nothing is left. `date` is written as it is given (mbox +/// files have it in UTC); one that doesn't exist is written as the start of 1970. +pub fn write_mbox_message(out: &mut Vec, sender: &str, date: &DateTime, message: &[u8]) { + let sender: String = sender.chars().filter(|c| !c.is_control() && !c.is_whitespace()).collect(); + let sender = if sender.is_empty() { "MAILER-DAEMON" } else { sender.as_str() }; + let stamp = date.asctime().unwrap_or_else(|| "Thu Jan 1 00:00:00 1970".to_string()); + out.extend_from_slice(format!("From {sender} {stamp}\r\n").as_bytes()); + let mut pos = 0; + while pos < message.len() { + let (line, next) = header::line_at(message, pos); + let text = line.iter().position(|b| *b != b'>').and_then(|i| line.get(i..)).unwrap_or_default(); + if text.starts_with(b"From ") { + out.push(b'>'); + } + out.extend_from_slice(message.get(pos..next).unwrap_or_default()); + pos = next; + } + if !out.ends_with(b"\n") { + out.extend_from_slice(b"\r\n"); + } + out.extend_from_slice(b"\r\n"); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(bytes: &[u8]) -> Vec { + mbox_messages(bytes).map(|m| String::from_utf8_lossy(&m).into_owned()).collect() + } + + const DATE: DateTime = DateTime { year: 2003, month: 7, day: 1, hour: 8, minute: 52, second: 37, offset_minutes: 0 }; + + #[test] + fn reads_a_unix_mbox() { + let mbox = b"From ada@example.org Tue Jul 1 08:52:37 2003\n\ +Subject: One\n\ +Status: RO\n\ +\n\ +First body\n\ +>From the start, quoted\n\ +>>From twice\n\ +> From not quoted\n\ +\n\ +From bob@example.org Wed Jul 2 09:00:00 2003\n\ +Subject: Two\n\ +\n\ +Second body\n"; + assert_eq!( + read(mbox), + ["Subject: One\nStatus: RO\n\nFirst body\nFrom the start, quoted\n>From twice\n> From not quoted\n", "Subject: Two\n\nSecond body\n"] + ); + } + + #[test] + fn separators_need_a_blank_line_or_a_time() { + // No blank line between the messages, but the separator is dated. + let tight = + b"From a@example.org Tue Jul 1 08:52:37 2003\nSubject: One\n\nx\nFrom b@example.org Wed Jul 2 09:00:00 2003\nSubject: Two\n\ny\n"; + assert_eq!(read(tight), ["Subject: One\n\nx\n", "Subject: Two\n\ny\n"]); + // A body line that starts with "From " in the middle of a paragraph is text. + let prose = b"From a@example.org Tue Jul 1 08:52:37 2003\nSubject: One\n\nHello\nFrom now on we meet on Fridays\n"; + assert_eq!(read(prose), ["Subject: One\n\nHello\nFrom now on we meet on Fridays\n"]); + } + + #[test] + fn empty_and_odd_files() { + assert!(read(b"").is_empty()); + assert!(read(b"\n\n\r\n").is_empty()); + assert!(read(b"From a@example.org Tue Jul 1 08:52:37 2003\n").is_empty()); + assert!(read(b"From \n\nFrom \n\nFrom ").is_empty()); + // A `From ` line in the middle of a paragraph, without a time, is content. + assert_eq!(read(b"From \nFrom \n\nFrom ").len(), 1); + // A lone message without a separator. + assert_eq!(read(b"\nSubject: Lone\n\nbody\n"), ["Subject: Lone\n\nbody\n"]); + // The iterator stays finished. + let mut messages = mbox_messages(b"Subject: x\n\ny\n"); + assert!(messages.next().is_some()); + assert!(messages.next().is_none() && messages.next().is_none()); + } + + #[test] + fn writes_separators_and_quotes_from_lines() { + let mut out = Vec::new(); + write_mbox_message(&mut out, "ada@example.org", &DATE, b"Subject: Hi\r\n\r\nFrom here\r\n>From there\r\nplain\r\n"); + assert_eq!( + String::from_utf8_lossy(&out), + "From ada@example.org Tue Jul 1 08:52:37 2003\r\nSubject: Hi\r\n\r\n>From here\r\n>>From there\r\nplain\r\n\r\n" + ); + let mut out = Vec::new(); + write_mbox_message(&mut out, " \r\n", &DateTime::default(), b"x"); + assert_eq!(String::from_utf8_lossy(&out), "From MAILER-DAEMON Thu Jan 1 00:00:00 1970\r\nx\r\n\r\n"); + } + + #[test] + fn a_written_mbox_reads_back() { + let messages: [&[u8]; 4] = [ + b"Subject: One\r\n\r\nFrom the top\r\n\r\nFrom after a blank line\r\n>From quoted already\r\n", + "Subject: Two\r\n\r\nGrüße\r\n\r\nFrom München\r\n".as_bytes(), + b"Subject: bare lf\n\nFrom here\n>>From there\n", + b"Subject: no body\r\n\r\n", + ]; + let mut out = Vec::new(); + for message in messages { + write_mbox_message(&mut out, "ada@example.org", &DATE, message); + } + let back: Vec> = mbox_messages(&out).collect(); + assert_eq!(back, messages); + } + + #[test] + fn hostile_senders_cannot_break_the_separator() { + let mut out = Vec::new(); + write_mbox_message(&mut out, "a@example.org\r\nFrom evil@example.org Tue Jul 1 08:52:37 2003", &DATE, b"Subject: x\r\n\r\ny\r\n"); + assert_eq!(mbox_messages(&out).count(), 1); + assert_eq!(out.iter().filter(|b| **b == b'\n').count(), 5); + } + + proptest::proptest! { + #[test] + fn any_bytes_read(data in proptest::collection::vec(proptest::prelude::any::(), 0..600)) { + let total: usize = mbox_messages(&data).map(|m| m.len()).sum(); + proptest::prop_assert!(total <= data.len()); + } + + #[test] + fn from_soup_reads(s in "(From |>From |>|\n|\r\n|a|12:34| |: ){0,60}") { + let _ = mbox_messages(s.as_bytes()).count(); + } + + #[test] + fn any_message_written_reads_back(s in "(From |>From |>|\r\n|a|12:34| |: ){0,40}") { + // A message is not white space alone, and ends its last line. + let message = format!("X: 1\r\n\r\n{s}\r\n"); + let mut out = Vec::new(); + write_mbox_message(&mut out, "a@example.org", &DATE, message.as_bytes()); + write_mbox_message(&mut out, "b@example.org", &DATE, message.as_bytes()); + let back: Vec> = mbox_messages(&out).collect(); + proptest::prop_assert_eq!(back, vec![message.clone().into_bytes(), message.into_bytes()]); + } + } +} diff --git a/crates/mime/src/part.rs b/crates/mime/src/part.rs new file mode 100644 index 00000000..d910b6aa --- /dev/null +++ b/crates/mime/src/part.rs @@ -0,0 +1,271 @@ +//! A message as a tree of MIME parts (RFC 2045, RFC 2046). + +use crate::header::{self, Headers, Params, parse_params}; +use crate::{Limits, charset, transfer}; + +/// A message read by [`parse`]. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Message { + /// The message itself: its header fields and its body, which is the tree of its parts. + pub root: Part, + /// Whether a [`Limits`] left something out: input past the size cap, header fields, parts + /// past the count or the depth, a body past the decoded size. + pub truncated: bool, +} + +/// One MIME entity: a message, or a part of a multipart. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Part { + /// Its header fields, in order, as written (unfolded; encoded-words not decoded). + pub headers: Headers, + /// Its `Content-Type`, parsed; empty when it has none. + pub content_type: Params, + pub body: Body, +} + +/// What a [`Part`] holds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Body { + /// The bytes of a leaf part, its `Content-Transfer-Encoding` undone. Also the raw body of a + /// `multipart/*` that names no boundary, which can't be split. + Data(Vec), + /// The parts of a `multipart/*`, in order. The text before the first boundary and after + /// the closing one is not kept. + Parts(Vec), +} + +impl Default for Body { + fn default() -> Self { + Body::Data(Vec::new()) + } +} + +/// `type/subtype`, both there. +fn is_media_type(t: &str) -> bool { + t.split_once('/').is_some_and(|(kind, sub)| !kind.is_empty() && !sub.is_empty() && !sub.contains('/')) +} + +/// A `Content-ID` or `Content-Location` value without the angle brackets and space around it. +fn bare(value: &str) -> &str { + value.trim().trim_start_matches('<').trim_end_matches('>').trim() +} + +impl Part { + /// The media type the part says it is (`text/html`, lower case), when it says one that has + /// both a type and a subtype. + pub fn declared_media_type(&self) -> Option<&str> { + Some(self.content_type.value.as_str()).filter(|t| is_media_type(t)) + } + + /// The media type of the part. RFC 2045: a part that doesn't say what it is, is plain text. + pub fn media_type(&self) -> &str { + self.declared_media_type().unwrap_or("text/plain") + } + + /// Whether the part is a `multipart/*`. + pub fn is_multipart(&self) -> bool { + self.media_type().starts_with("multipart/") + } + + /// The `Content-Disposition`, parsed: its value is `attachment`, `inline`, or empty. + pub fn disposition(&self) -> Params { + self.headers.get("content-disposition").map(parse_params).unwrap_or_default() + } + + /// The file name the part gives itself: the `filename` of its `Content-Disposition`, else + /// the `name` of its `Content-Type`, RFC 2231 and encoded-words decoded. As the sender wrote + /// it: whoever saves a file under it must not trust it as a path. + pub fn file_name(&self) -> Option { + self.disposition().text("filename").or_else(|| self.content_type.text("name")).filter(|n| !n.is_empty()) + } + + /// The `Content-ID`, without angle brackets; `None` when there is none or it is empty. + pub fn content_id(&self) -> Option<&str> { + self.headers.get("content-id").map(bare).filter(|v| !v.is_empty()) + } + + /// The `Content-Location` (RFC 2557); `None` when there is none or it is empty. + pub fn content_location(&self) -> Option<&str> { + self.headers.get("content-location").map(bare).filter(|v| !v.is_empty()) + } + + /// The `Content-Transfer-Encoding`, lower case; empty when there is none. + pub fn transfer_encoding(&self) -> String { + self.headers.get("content-transfer-encoding").map(|e| e.trim().to_ascii_lowercase()).unwrap_or_default() + } + + /// The decoded bytes of a leaf part; empty for a multipart that was split. + pub fn data(&self) -> &[u8] { + match &self.body { + Body::Data(data) => data, + Body::Parts(_) => &[], + } + } + + /// The parts of a multipart; none for a leaf. + pub fn parts(&self) -> &[Part] { + match &self.body { + Body::Data(_) => &[], + Body::Parts(parts) => parts, + } + } + + /// The bytes of a leaf part as text, in the `charset` its `Content-Type` names + /// ([`charset::decode`]; guessed when it names none), without a byte order mark at its + /// start. Line ends are as they came ([`charset::unix_lines`] makes them `\n`). + pub fn text(&self) -> String { + let text = charset::decode(self.data(), self.content_type.get("charset").unwrap_or("")); + match text.strip_prefix('\u{feff}') { + Some(rest) => rest.trim_start_matches('\u{feff}').to_string(), + None => text, + } + } + + /// Every part that isn't a split multipart, this one included, in the order of the message. + pub fn leaves(&self) -> Vec<&Part> { + let mut out = Vec::new(); + let mut stack = vec![self]; + while let Some(part) = stack.pop() { + match &part.body { + Body::Data(_) => out.push(part), + Body::Parts(parts) => stack.extend(parts.iter().rev()), + } + } + out + } +} + +/// Parse an Internet message: an `.eml` file, an MHT web archive, what a server hands over. +/// +/// Never fails: parts that can't be read are skipped, and garbage gives a message with no +/// header fields and the bytes as its body. A leading mbox `From ` line is skipped. +/// +/// A `multipart/*` with a boundary is split into its parts, down to [`Limits::max_depth`]; one +/// without a boundary keeps its raw body as [`Body::Data`]. Every other part is a leaf, its +/// transfer encoding undone; a `message/rfc822` part is a leaf too (its bytes are a message for +/// another call). +pub fn parse(bytes: &[u8], limits: &Limits) -> Message { + parse_at(bytes, limits, true) +} + +/// [`parse`] for an entity that isn't the top of a file (the content of a signed or wrapped +/// part): a first line that starts with `From ` is not skipped. +pub fn parse_entity(bytes: &[u8], limits: &Limits) -> Message { + parse_at(bytes, limits, false) +} + +fn parse_at(bytes: &[u8], limits: &Limits, top: bool) -> Message { + let mut walk = + Walk { limits, max_depth: limits.max_depth.min(Limits::HARD_MAX_DEPTH), parts: 0, decoded: 0, truncated: bytes.len() > limits.max_input }; + let root = walk.part(bytes.get(..limits.max_input).unwrap_or(bytes), top, 0); + Message { root, truncated: walk.truncated } +} + +struct Walk<'l> { + limits: &'l Limits, + max_depth: usize, + /// Parts read so far. + parts: usize, + /// Decoded bytes kept so far. + decoded: usize, + truncated: bool, +} + +impl Walk<'_> { + fn part(&mut self, bytes: &[u8], top: bool, depth: usize) -> Part { + self.parts = self.parts.saturating_add(1); + let (headers, body, cut) = header::split_counted(bytes, top, self.limits); + self.truncated |= cut; + let content_type = headers.get("content-type").map(parse_params).unwrap_or_default(); + let multipart = is_media_type(&content_type.value) && content_type.value.starts_with("multipart/"); + let boundary = content_type.get("boundary").filter(|b| multipart && !b.is_empty()); + let body = match boundary { + Some(_) if depth >= self.max_depth => { + self.truncated = true; + Body::Parts(Vec::new()) + } + Some(boundary) => { + let room = self.limits.max_parts.saturating_sub(self.parts); + // One more than there is room for, to know whether any were left out. + let pieces = split_multipart(body, boundary, room.saturating_add(1)); + self.truncated |= pieces.len() > room; + let mut parts = Vec::new(); + for piece in pieces { + if self.parts >= self.limits.max_parts { + self.truncated = true; + break; + } + parts.push(self.part(piece, false, depth.saturating_add(1))); + } + Body::Parts(parts) + } + // Nothing to split on: the raw body, so that a caller can show what is there. + None if multipart => Body::Data(self.keep(body.to_vec())), + None => { + let encoding = headers.get("content-transfer-encoding").unwrap_or(""); + Body::Data(self.keep(transfer::decode(body, encoding))) + } + }; + Part { headers, content_type, body } + } + + /// `data` when it fits in what is left of [`Limits::max_decoded`], else nothing. + fn keep(&mut self, data: Vec) -> Vec { + match self.decoded.checked_add(data.len()).filter(|total| *total <= self.limits.max_decoded) { + Some(total) => { + self.decoded = total; + data + } + None => { + self.truncated = true; + Vec::new() + } + } + } +} + +/// The parts of a multipart body: what lies between `--boundary` lines, without the line break +/// that belongs to the delimiter. The text before the first delimiter and after the closing +/// `--boundary--` is ignored. A body that is cut short still gives its last part. At most `max` +/// parts are given. +/// +/// For callers that need a part's bytes as they came (a signature is made over them); [`parse`] +/// gives the parts read. +pub fn split_multipart<'a>(body: &'a [u8], boundary: &str, max: usize) -> Vec<&'a [u8]> { + let delimiter = [b"--", boundary.as_bytes()].concat(); + let mut parts = Vec::new(); + let mut start: Option = None; + let mut pos = 0; + while pos < body.len() { + let (line, next) = header::line_at(body, pos); + if let Some(rest) = line.strip_prefix(delimiter.as_slice()) { + let (closing, tail) = match rest.strip_prefix(b"--") { + Some(tail) => (true, tail), + None => (false, rest), + }; + if tail.iter().all(|b| matches!(b, b' ' | b'\t')) { + if let Some(from) = start.take() + && parts.len() < max + { + let piece = body.get(from..pos).unwrap_or_default(); + parts.push(piece.strip_suffix(b"\r\n").or_else(|| piece.strip_suffix(b"\n")).unwrap_or(piece)); + } + if closing || parts.len() >= max { + return parts; + } + start = Some(next); + } + } + pos = next; + } + if let Some(from) = start + && parts.len() < max + { + parts.push(body.get(from..).unwrap_or_default()); + } + parts +} + +#[cfg(test)] +#[path = "part_tests.rs"] +mod tests; diff --git a/crates/mime/src/part_tests.rs b/crates/mime/src/part_tests.rs new file mode 100644 index 00000000..52539e07 --- /dev/null +++ b/crates/mime/src/part_tests.rs @@ -0,0 +1,495 @@ +//! The reader: MailCraft's message tests and WordCraft's MHT tests, at the level of the tree. + +use super::*; + +/// A message from lines, joined with CRLF. +fn eml(lines: &[&str]) -> Vec { + lines.join("\r\n").into_bytes() +} + +fn read(bytes: &[u8]) -> Message { + parse(bytes, &Limits::default()) +} + +/// The media type and text of every leaf. +fn leaves(m: &Message) -> Vec<(String, String)> { + m.root.leaves().iter().map(|p| (p.media_type().to_string(), p.text())).collect() +} + +fn pair(a: &str, b: &str) -> (String, String) { + (a.to_string(), b.to_string()) +} + +#[test] +fn a_plain_message() { + let m = read(&eml(&[ + "Return-Path: ", + "Received: from mail.example.org (mail.example.org [192.0.2.1])", + "\tby mx.example.org with ESMTP id 4F2A1; Tue, 1 Jul 2003 10:52:40 +0200", + "Date: Tue, 1 Jul 2003 10:52:37 +0200", + "From: Ada Lovelace ", + "Subject: =?UTF-8?Q?Caf=C3=A9_=E2=82=AC5?=", + " =?UTF-8?Q?_tomorrow?=", + "MIME-Version: 1.0", + "Content-Type: text/plain; charset=us-ascii", + "", + "Hello Bob,", + "", + "Ada", + "", + ])); + assert!(!m.truncated); + let names: Vec<&str> = m.root.headers.0.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, ["Return-Path", "Received", "Date", "From", "Subject", "MIME-Version", "Content-Type"]); + assert_eq!( + m.root.headers.get("received"), + Some("from mail.example.org (mail.example.org [192.0.2.1])\tby mx.example.org with ESMTP id 4F2A1; Tue, 1 Jul 2003 10:52:40 +0200") + ); + assert_eq!(m.root.headers.get("subject").map(crate::decode_words).as_deref(), Some("Café €5 tomorrow")); + assert_eq!(m.root.headers.get("date").and_then(crate::parse_date).map(|d| (d.day, d.offset_minutes)), Some((1, 120))); + assert_eq!(crate::parse_address_list(m.root.headers.get("FROM").unwrap_or("")).first().map(|a| a.email.as_str()), Some("ada@example.org")); + assert_eq!((m.root.media_type(), m.root.declared_media_type(), m.root.is_multipart()), ("text/plain", Some("text/plain"), false)); + assert_eq!(m.root.text(), "Hello Bob,\r\n\r\nAda\r\n"); + assert!(m.root.parts().is_empty() && m.root.file_name().is_none() && m.root.content_id().is_none()); + assert_eq!(m.root.leaves().len(), 1); +} + +#[test] +fn bare_lf_line_ends_and_no_content_type() { + let m = read(b"From: ada@example.org\nSubject: Unix\n\nLine one\nLine two\n"); + assert_eq!(m.root.headers.get("subject"), Some("Unix")); + assert_eq!((m.root.declared_media_type(), m.root.media_type()), (None, "text/plain")); + assert_eq!(m.root.data(), b"Line one\nLine two\n"); + // A type without a subtype is none. + assert_eq!(read(b"Content-Type: garbage\n\nx").root.declared_media_type(), None); +} + +#[test] +fn an_alternative_without_its_preamble_and_epilogue() { + let m = read(&eml(&[ + "Content-Type: multipart/alternative; boundary=\"alt-1\"", + "", + "This is the preamble, for readers that don't know MIME.", + "--alt-1", + "Content-Type: text/plain; charset=utf-8", + "", + "Hello *world*", + "--alt-1", + "Content-Type: text/html; charset=utf-8", + "", + "

Hello world

", + "--alt-1--", + "The epilogue.", + ])); + assert!(m.root.is_multipart() && m.root.data().is_empty()); + assert_eq!(leaves(&m), [pair("text/plain", "Hello *world*"), pair("text/html", "

Hello world

")]); +} + +#[test] +fn related_parts_with_ids_names_and_encodings() { + let m = read(&eml(&[ + "Content-Type: multipart/related; boundary=\"rel\"; type=\"text/html\"", + "", + "--rel", + "Content-Type: text/html; charset=utf-8", + "", + "

Logo: \"Our

", + "--rel", + "Content-Type: image/png; name=\"logo.png\"", + "Content-Transfer-Encoding: base64", + "Content-ID: ", + "", + "iVBORw0KGgo=", + "--rel", + "Content-Type: text/plain; charset=utf-8", + "Content-Disposition: attachment;", + " filename*0*=utf-8''r%C3%A9sum%C3%A9%20;", + " filename*1*=%E2%82%AC.txt", + "Content-Transfer-Encoding: quoted-printable", + "", + "Prix: 5 =E2=82=AC=", + "", + "--rel", + "Content-Type: application/octet-stream", + "Content-Disposition: inline; filename=\"=?UTF-8?B?csOpc3Vtw6kucGRm?=\"", + "Content-Location: ", + "", + "raw", + "--rel--", + ])); + let parts = m.root.parts(); + assert_eq!(parts.len(), 4); + assert_eq!(m.root.content_type.get("type"), Some("text/html")); + let logo = &parts[1]; + assert_eq!((logo.media_type(), logo.file_name().as_deref(), logo.content_id()), ("image/png", Some("logo.png"), Some("logo@example.org"))); + assert_eq!((logo.transfer_encoding().as_str(), logo.data()), ("base64", &b"\x89PNG\r\n\x1a\n"[..])); + assert_eq!(logo.disposition().value, ""); + let file = &parts[2]; + assert_eq!((file.disposition().value.as_str(), file.file_name().as_deref()), ("attachment", Some("résumé €.txt"))); + assert_eq!(file.text(), "Prix: 5 €"); + let raw = &parts[3]; + assert_eq!((raw.disposition().value.as_str(), raw.file_name().as_deref()), ("inline", Some("résumé.pdf"))); + assert_eq!((raw.content_location(), raw.data()), (Some("file:///C:/x/r.pdf"), &b"raw"[..])); + assert_eq!(parts[0].content_location(), None); +} + +#[test] +fn quoted_printable_latin1_and_base64_utf8() { + let m = read(&eml(&[ + "Content-Type: text/plain; charset=ISO-8859-1", + "Content-Transfer-Encoding: quoted-printable", + "", + "La r=E9union est =E0 10h. Co=FBt: 5=80 (=93environ=94).=20", + "Une ligne tr=E8s longue qui continue =", + "sur la suivante.", + ])); + assert_eq!(m.root.text(), "La réunion est à 10h. Coût: 5€ (“environ”). \r\nUne ligne très longue qui continue sur la suivante."); + let m = read(&eml(&[ + "Content-Type: text/plain; charset=\"UTF-8\"", + "Content-Transfer-Encoding: BASE64", + "", + "R3LDvMOfZSBhdXMg", + "TcO8bmNoZW4g8J+Ziw0KWndlaXRlIFplaWxl", + "", + ])); + assert_eq!(crate::charset::unix_lines(&m.root.text()), "Grüße aus München 🙋\nZweite Zeile"); +} + +#[test] +fn an_attached_message_is_not_opened() { + let m = read(&eml(&[ + "Subject: Fwd", + "Content-Type: multipart/mixed; boundary=fw", + "", + "--fw", + "", + "See below.", + "--fw", + "Content-Type: message/rfc822", + "", + "Subject: Inner", + "Content-Type: multipart/mixed; boundary=inner", + "", + "--inner", + "", + "Inner text", + "--inner--", + "--fw--", + ])); + let parts = m.root.parts(); + assert_eq!(parts.len(), 2); + assert_eq!((parts[0].media_type(), parts[0].data()), ("text/plain", &b"See below."[..])); + assert_eq!(parts[1].media_type(), "message/rfc822"); + assert!(parts[1].data().starts_with(b"Subject: Inner\r\n")); + let inner = parse_entity(parts[1].data(), &Limits::default()); + assert_eq!((inner.root.headers.get("subject"), leaves(&inner)), (Some("Inner"), vec![pair("text/plain", "Inner text")])); +} + +#[test] +fn nested_multiparts() { + let m = read(&eml(&[ + "Content-Type: multipart/mixed; boundary=outer", + "", + "--outer", + "Content-Type: multipart/alternative; boundary=alt", + "", + "--alt", + "Content-Type: text/plain", + "", + "Plain", + "--alt", + "Content-Type: multipart/related; boundary=rel", + "", + "--rel", + "Content-Type: text/html", + "", + "

Rich

", + "--rel", + "Content-Type: image/gif", + "Content-ID: ", + "Content-Transfer-Encoding: base64", + "", + "R0lGODlh", + "--rel--", + "--alt--", + "--outer", + "Content-Type: application/zip; name=files.zip", + "Content-Disposition: attachment", + "Content-Transfer-Encoding: base64", + "", + "UEsDBA==", + "--outer", + "Content-Type: text/plain", + "", + "-- ", + "Sent via the example list", + "--outer--", + ])); + let types: Vec<&str> = m.root.leaves().iter().map(|p| p.media_type()).collect(); + assert_eq!(types, ["text/plain", "text/html", "image/gif", "application/zip", "text/plain"]); + let outer = m.root.parts(); + assert_eq!(outer.iter().map(Part::media_type).collect::>(), ["multipart/alternative", "application/zip", "text/plain"]); + assert_eq!(outer[0].parts()[1].parts()[1].data(), b"GIF89a"); + assert_eq!(outer[2].text(), "-- \r\nSent via the example list"); + assert!(!m.truncated); +} + +#[test] +fn a_truncated_multipart_keeps_what_arrived() { + let m = read(&eml(&[ + "Content-Type: multipart/mixed; boundary=cut", + "", + "--cut", + "Content-Type: text/plain", + "", + "The text arrived.", + "--cut", + "Content-Type: application/pdf; name=half.pdf", + "Content-Transfer-Encoding: base64", + "", + "JVBERi0xLjQKJ", + ])); + assert_eq!(m.root.parts().len(), 2); + assert_eq!(m.root.parts()[0].data(), b"The text arrived."); + assert_eq!(m.root.parts()[1].data(), b"%PDF-1.4\n"); + // Cut inside the headers of a part. + let m = read(&eml(&["Content-Type: multipart/mixed; boundary=cut", "", "--cut", "Content-Type: text/pl"])); + assert_eq!(m.root.parts().len(), 1); + assert_eq!((m.root.parts()[0].media_type(), m.root.parts()[0].data()), ("text/pl", &b""[..])); +} + +#[test] +fn a_missing_or_wrong_boundary() { + // No boundary to split on: the raw body is kept, so that a caller can show it. + let m = read(&eml(&["Content-Type: multipart/mixed", "", "--who-knows", "", "Hidden text", "--who-knows--"])); + assert!(m.root.is_multipart() && m.root.parts().is_empty()); + assert_eq!(m.root.data(), b"--who-knows\r\n\r\nHidden text\r\n--who-knows--"); + let m = read(&eml(&["Content-Type: multipart/mixed; boundary=\"\"", "Content-Transfer-Encoding: base64", "", "text"])); + assert_eq!(m.root.data(), b"text"); + // The boundary never appears: there are no parts. + let m = read(&eml(&["Content-Type: multipart/mixed; boundary=absent", "", "--other", "", "text", "--other--"])); + assert_eq!(m.root.body, Body::Parts(Vec::new())); + // A boundary on something that is no multipart means nothing. + assert_eq!(read(b"Content-Type: text/plain; boundary=b\n\n--b\nx\n--b--").root.data(), b"--b\nx\n--b--"); +} + +#[test] +fn boundaries_that_look_alike() { + let m = read(&eml(&[ + "Content-Type: multipart/mixed; boundary=b", + "", + "--b", + "", + "--bb is not a delimiter", + "--b-- neither is this", + "--b ", + "", + "second", + "--b-- ", + "--b", + "", + "after the end", + ])); + assert_eq!(leaves(&m), [pair("text/plain", "--bb is not a delimiter\r\n--b-- neither is this"), pair("text/plain", "second")]); +} + +#[test] +fn garbage_gives_a_mostly_empty_message() { + assert_eq!(read(b""), Message::default()); + let m = read(b"\x00\x01\x02\xff\xfe binary \x80 junk"); + assert!(m.root.headers.0.is_empty()); + assert_eq!(m.root.data(), b"\x00\x01\x02\xff\xfe binary \x80 junk"); + assert_eq!(read(b"no headers at all, just text\r\nsecond line\r\n").root.data(), b"no headers at all, just text\r\nsecond line\r\n"); + assert_eq!(read(b"\r\n\r\nstarts with blank lines").root.data(), b"\r\nstarts with blank lines"); + let _ = read(&[b'\n'; 10_000]); + let _ = read(&[b':'; 10_000]); + let _ = read(b"Content-Type: multipart/mixed; boundary=\r\n\r\n--\r\n----\r\n"); + let _ = read(b"Content-Type: ;;;=;\"\r\nContent-Disposition: \"\r\nContent-Transfer-Encoding: base64\r\n\r\n===="); +} + +#[test] +fn a_byte_order_mark_and_raw_8bit_text() { + assert_eq!(read(b"Content-Type: text/plain; charset=utf-8\n\n\xEF\xBB\xBFHello").root.text(), "Hello"); + // No charset: UTF-8 when it is valid, else Windows-1252. + assert_eq!(read("Subject: x\n\ncafé".as_bytes()).root.text(), "café"); + assert_eq!(read(b"Subject: x\n\ncaf\xE9 \x80").root.text(), "café €"); + assert_eq!(read(b"Content-Type: text/plain; charset=x-unknown\n\ncaf\xE9").root.text(), "caf\u{fffd}"); + assert_eq!( + read(b"Content-Type: text/html; charset=windows-1250\nContent-Transfer-Encoding: quoted-printable\n\n

=E8lanak

").root.text(), + "

članak

" + ); +} + +#[test] +fn an_mbox_from_line_is_skipped_at_the_top_only() { + let bytes = b"From ada@example.org Sat Jan 3 01:05:34 1996\r\nSubject: x\r\n\r\nB"; + assert_eq!(read(bytes).root.headers.get("subject"), Some("x")); + let entity = parse_entity(bytes, &Limits::default()); + assert!(entity.root.headers.0.is_empty()); + assert_eq!(entity.root.data(), bytes); +} + +// WordCraft's MHT reader. + +const PNG_1X1: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + +fn mht(html_part: &str) -> String { + format!( + "MIME-Version: 1.0\r\nContent-Type: multipart/related; boundary=\"----=_B\"\r\n\r\n\ + ------=_B\r\n{html_part}\r\n\ + ------=_B\r\nContent-Location: file:///C:/page_files/dot.png\r\nContent-Transfer-Encoding: base64\r\nContent-Type: image/png\r\n\r\n{PNG_1X1}\r\n\ + ------=_B--\r\n" + ) +} + +#[test] +fn an_mht_page_and_its_pictures() { + let page = "Content-Location: file:///C:/page.htm\r\nContent-Transfer-Encoding: quoted-printable\r\nContent-Type: text/html; charset=\"utf-8\"\r\n\r\n\ +

Dobar dan, gospo=C4=91o, ovo je =\r\njedan red.

"; + let m = read(mht(page).as_bytes()); + let html = m.root.leaves().into_iter().find(|p| p.media_type() == "text/html").map(Part::text).unwrap_or_default(); + assert_eq!(html, "

Dobar dan, gospođo, ovo je jedan red.

"); + let resources = crate::Resources::of(&m.root); + let picture = resources.find("page_files/dot.png", "file:///C:/page.htm"); + assert_eq!(picture.map(|p| (p.media_type(), p.data().len())), Some(("image/png", 70))); +} + +#[test] +fn quotes_and_comments_do_not_supply_a_false_boundary() { + let page = "Content-Type: text/html\r\n\r\n

Kept content

"; + for name in [r#"name="notes; boundary=wrong""#, r#"name="notes\"; boundary=wrong""#, r#"name="notes (10\" screen; boundary=wrong""#] { + let m = read(mht(page).replace("multipart/related; boundary=", &format!("multipart/related; {name}; boundary=")).as_bytes()); + assert_eq!(m.root.parts().first().map(Part::text).as_deref(), Some("

Kept content

"), "{name}"); + } + for comment in [r#"(10" screen)"#, r#"(outer (10" screen); boundary=wrong)"#, r#"(10\)" screen; boundary=wrong)"#] { + let m = read(mht(page).replace("multipart/related; boundary=", &format!("multipart/related {comment}; boundary=")).as_bytes()); + assert_eq!(m.root.parts().first().map(Part::text).as_deref(), Some("

Kept content

"), "{comment}"); + assert_eq!(m.root.parts().len(), 2); + } +} + +#[test] +fn hostile_mht_never_panics() { + for m in [ + &b""[..], + b"MIME-Version: 1.0\r\nContent-Type: multipart/related; boundary=\"\"\r\n\r\n", + b"MIME-Version: 1.0\r\nContent-Type: multipart/related; boundary=x\r\n\r\n--x\r\n--x--", + b"MIME-Version: 1.0\nContent-Type: multipart/related; boundary=x\n\n--x\nContent-Transfer-Encoding: quoted-printable\nContent-Type: text/html\n\n=\n=Z=4", + b"Content-Type: text/html\r\nContent-Transfer-Encoding: base64\r\n\r\n!!!!", + ] { + let m = read(m); + let _ = (crate::Resources::of(&m.root), crate::start_part(&m.root)); + } +} + +// Each limit. + +#[test] +fn the_input_size_is_capped() { + let limits = Limits { max_input: 20, ..Limits::default() }; + let m = parse(b"Subject: x\r\n\r\n0123456789", &limits); + assert_eq!((m.root.data(), m.truncated), (&b"012345"[..], true)); + let m = parse(b"Subject: x\r\n\r\n012345", &limits); + assert_eq!((m.root.data(), m.truncated), (&b"012345"[..], false)); +} + +#[test] +fn deep_nesting_is_capped() { + let nested = |levels: usize| { + let mut deep = Vec::new(); + for i in 0..levels { + deep.extend_from_slice(format!("Content-Type: multipart/mixed; boundary=b{i}\r\n\r\n--b{i}\r\n").as_bytes()); + } + deep.extend_from_slice(b"Content-Type: text/plain\r\n\r\ndeep text\r\n"); + deep + }; + let depth = |m: &Message| { + let (mut part, mut depth) = (&m.root, 0); + while let Some(first) = part.parts().first() { + (part, depth) = (first, depth + 1); + } + (depth, part.data().to_vec()) + }; + let m = read(&nested(20)); + assert_eq!((depth(&m), m.truncated), ((20, b"deep text\r\n".to_vec()), false)); + // The multipart at the cap is not opened: it has no parts. + let m = read(&nested(21)); + assert_eq!((depth(&m), m.truncated), ((20, Vec::new()), true)); + assert!(m.root.leaves().is_empty()); + // Whatever is asked for, the hard cap holds (the reader recurses once per level). + let m = parse(&nested(5000), &Limits { max_depth: usize::MAX, max_parts: usize::MAX, ..Limits::default() }); + assert_eq!((depth(&m).0, m.truncated), (Limits::HARD_MAX_DEPTH, true)); + let m = parse(&nested(3), &Limits { max_depth: 0, ..Limits::default() }); + assert_eq!((m.root.body.clone(), m.truncated), (Body::Parts(Vec::new()), true)); +} + +#[test] +fn the_number_of_parts_is_capped() { + let wide = |n: usize| { + let mut wide = b"Content-Type: multipart/mixed; boundary=w\r\n\r\n".to_vec(); + for _ in 0..n { + wide.extend_from_slice(b"--w\r\nContent-Type: application/x-a\r\n\r\nx\r\n"); + } + wide.extend_from_slice(b"--w--\r\n"); + wide + }; + let max = Limits::DEFAULT.max_parts; + // The message itself is a part. + let m = read(&wide(max + 500)); + assert_eq!((m.root.parts().len(), m.truncated), (max - 1, true)); + let m = read(&wide(max - 1)); + assert_eq!((m.root.parts().len(), m.truncated), (max - 1, false)); + // Parts of nested multiparts count too. + let nested = b"Content-Type: multipart/mixed; boundary=a\n\n--a\nContent-Type: multipart/mixed; boundary=b\n\n--b\n\n1\n--b\n\n2\n--b\n\n3\n--b--\n--a\n\n4\n--a--\n"; + let m = parse(nested, &Limits { max_parts: 4, ..Limits::default() }); + assert_eq!((m.root.leaves().len(), m.truncated), (2, true)); + assert_eq!(parse(nested, &Limits { max_parts: 6, ..Limits::default() }).root.leaves().len(), 4); + assert_eq!(split_multipart(b"--b\n1\n--b\n2\n--b\n3\n--b--\n", "b", 2), [&b"1"[..], b"2"]); + assert!(split_multipart(b"--b\n1\n--b--\n", "b", 0).is_empty()); +} + +#[test] +fn the_decoded_size_is_capped() { + let bytes = b"Content-Type: multipart/mixed; boundary=b\n\n--b\n\n0123456789\n--b\nContent-Transfer-Encoding: base64\n\nMDEyMzQ1Njc4OQ==\n--b\n\nabc\n--b--\n"; + let m = parse(bytes, &Limits { max_decoded: 15, ..Limits::default() }); + let data: Vec<&[u8]> = m.root.parts().iter().map(Part::data).collect(); + // The second part would pass the cap and is empty; the third still fits. + assert_eq!((data, m.truncated), (vec![&b"0123456789"[..], b"", b"abc"], true)); + let m = parse(bytes, &Limits { max_decoded: 23, ..Limits::default() }); + assert_eq!((m.root.parts()[1].data(), m.truncated), (&b"0123456789"[..], false)); + assert!(parse(b"Content-Type: multipart/mixed\n\nraw", &Limits { max_decoded: 2, ..Limits::default() }).truncated); +} + +#[test] +fn header_limits_reach_the_message() { + let limits = Limits { max_header_fields: 2, ..Limits::default() }; + let m = parse(b"A: 1\nB: 2\nContent-Type: text/html\n\nx", &limits); + assert_eq!((m.root.headers.0.len(), m.root.media_type(), m.truncated), (2, "text/plain", true)); + // In a part's header block too. + let m = parse(b"Content-Type: multipart/mixed; boundary=b\n\n--b\nA: 1\nB: 2\nC: 3\n\nx\n--b--\n", &limits); + assert_eq!((m.root.parts()[0].headers.0.len(), m.truncated), (2, true)); +} + +proptest::proptest! { + #[test] + fn any_bytes_parse(data in proptest::collection::vec(proptest::prelude::any::(), 0..600)) { + let m = read(&data); + let total: usize = m.root.leaves().iter().map(|p| p.data().len()).sum(); + proptest::prop_assert!(total <= data.len()); + proptest::prop_assert!(!m.truncated); + } + + #[test] + fn mime_soup_parses(s in "(Content-Type: |multipart/mixed|; boundary=b|--b|--b--|\n|\r\n|text/html|Content-Transfer-Encoding: |base64|quoted-printable|=|a| |\\(|\\)|\"){0,60}") { + let small = Limits { max_parts: 5, max_depth: 2, max_decoded: 40, max_header_fields: 3, max_header_value: 30, max_header_block: 60, max_input: 300, ..Limits::default() }; + for limits in [Limits::default(), small] { + let m = parse(s.as_bytes(), &limits); + let leaves = m.root.leaves(); + let total: usize = leaves.iter().map(|p| p.data().len()).sum(); + proptest::prop_assert!(total <= limits.max_decoded.min(s.len())); + for part in leaves { + let _ = (part.text(), part.file_name(), part.content_id(), part.content_location(), part.disposition()); + } + } + } +} diff --git a/crates/mime/src/related.rs b/crates/mime/src/related.rs new file mode 100644 index 00000000..56ef6a1a --- /dev/null +++ b/crates/mime/src/related.rs @@ -0,0 +1,247 @@ +//! `multipart/related` (RFC 2387) and MHTML web archives (RFC 2557): which part is the root, +//! and which part a reference in it (`cid:…`, a location) names. + +use std::collections::HashMap; + +use crate::part::Part; + +/// The part that the `start` parameter of `root`'s `Content-Type` names by its `Content-ID` +/// (RFC 2387 §3.2), looked for among all the leaves of `root`. `None` when there is no `start` +/// or no part has that id; then the root of a `multipart/related` is its first part, or +/// whichever the caller's format says (MHTML: the first HTML part). +pub fn start_part(root: &Part) -> Option<&Part> { + let start = root.content_type.get("start")?.trim().trim_start_matches('<').trim_end_matches('>').trim(); + root.leaves().into_iter().find(|p| p.content_id() == Some(start)) +} + +/// The leaves of a message by `Content-ID` and by `Content-Location`, for the references of a +/// page to its pictures. The first part of each id and of each location wins. Locations are +/// compared without regard to ASCII letter case. +#[derive(Clone, Debug, Default)] +pub struct Resources<'a> { + by_id: HashMap<&'a str, &'a Part>, + by_location: HashMap, +} + +impl<'a> Resources<'a> { + /// The index of every leaf of `root`. + pub fn of(root: &'a Part) -> Self { + let mut resources = Resources::default(); + for part in root.leaves() { + if let Some(id) = part.content_id() { + resources.by_id.entry(id).or_insert(part); + } + if let Some(location) = part.content_location() { + resources.by_location.entry(location.to_ascii_lowercase()).or_insert(part); + } + } + resources + } + + /// The part with this `Content-ID` (without angle brackets and without `cid:`). + pub fn by_content_id(&self, id: &str) -> Option<&'a Part> { + self.by_id.get(id).copied() + } + + /// The part with this `Content-Location`, as written. + pub fn by_location(&self, location: &str) -> Option<&'a Part> { + self.by_location.get(&location.to_ascii_lowercase()).copied() + } + + /// The part a reference in a page names: `cid:id` by `Content-ID`; anything else by + /// `Content-Location`, as written, or resolved against `base`, the page's own location + /// ([`resolve_location`]; empty when it has none). Nothing is fetched. + pub fn find(&self, reference: &str, base: &str) -> Option<&'a Part> { + let reference = reference.trim(); + match reference.get(..4).filter(|s| s.eq_ignore_ascii_case("cid:")).and(reference.get(4..)) { + Some(id) => self.by_content_id(id), + None => self.by_location(reference).or_else(|| self.by_location(&resolve_location(base, reference))), + } + } +} + +/// `reference` resolved against the page's location `base`, as RFC 2557 §5 asks (RFC 3986 +/// §5.2.2): a reference with a scheme stays as it is, `//host/…` takes the base's scheme, `/…` +/// its scheme and host, an empty path its path and query, anything else is merged with its path +/// (§5.2.3); `.` and `..` segments go (§5.2.4) and fragments are dropped. +pub fn resolve_location(base: &str, reference: &str) -> String { + let (b, r) = (UriParts::of(base), UriParts::of(reference)); + let t = if r.scheme.is_some() { + UriParts { path: remove_dots(&r.path), ..r } + } else if r.authority.is_some() { + UriParts { scheme: b.scheme, path: remove_dots(&r.path), ..r } + } else if r.path.is_empty() { + UriParts { query: r.query.or(b.query), ..b } + } else if r.path.starts_with('/') { + UriParts { path: remove_dots(&r.path), query: r.query, ..b } + } else { + let merged = if b.authority.is_some() && b.path.is_empty() { + format!("/{}", r.path) + } else { + format!("{}{}", b.path.rfind('/').and_then(|i| b.path.get(..=i)).unwrap_or(""), r.path) + }; + UriParts { path: remove_dots(&merged), query: r.query, ..b } + }; + t.to_string() +} + +/// A URI's parts (RFC 3986 §3): `scheme:`, `//authority`, path and `?query`; the fragment left out. +struct UriParts<'a> { + scheme: Option<&'a str>, + authority: Option<&'a str>, + path: String, + query: Option<&'a str>, +} + +impl<'a> UriParts<'a> { + fn of(uri: &'a str) -> Self { + let uri = uri.split('#').next().unwrap_or(""); + // A one-letter scheme is a Windows drive (`C:/…`), not a scheme. + let scheme = uri.split_once(':').map(|(k, _)| k).filter(|k| { + k.len() > 1 && k.starts_with(|c: char| c.is_ascii_alphabetic()) && k.chars().all(|c| c.is_ascii_alphanumeric() || "+-.".contains(c)) + }); + let rest = scheme.and_then(|k| uri.get(k.len().saturating_add(1)..)).unwrap_or(uri); + let (authority, rest) = match rest.strip_prefix("//") { + Some(r) => { + let end = r.find(['/', '?']).unwrap_or(r.len()); + (r.get(..end), r.get(end..).unwrap_or("")) + } + None => (None, rest), + }; + let (path, query) = match rest.split_once('?') { + Some((p, q)) => (p, Some(q)), + None => (rest, None), + }; + UriParts { scheme, authority, path: path.to_string(), query } + } +} + +impl std::fmt::Display for UriParts<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(s) = self.scheme { + write!(f, "{s}:")?; + } + if let Some(a) = self.authority { + write!(f, "//{a}")?; + } + f.write_str(&self.path)?; + if let Some(q) = self.query { + write!(f, "?{q}")?; + } + Ok(()) + } +} + +/// `path` without its `.` and `..` segments (RFC 3986 §5.2.4); `..` stops at the root. +fn remove_dots(path: &str) -> String { + let mut out: Vec<&str> = Vec::new(); + let mut segments = path.split('/').peekable(); + while let Some(seg) = segments.next() { + match seg { + "." => {} + ".." => { + if out.len() > 1 || out.first().is_some_and(|f| !f.is_empty()) { + out.pop(); + } + } + s => out.push(s), + } + if segments.peek().is_none() && matches!(seg, "." | "..") { + out.push(""); + } + } + out.join("/") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Limits, parse}; + + const PNG_1X1: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + + fn page_and_logo(page: &str) -> crate::Message { + let m = format!( + "MIME-Version: 1.0\r\nContent-Type: multipart/related; boundary=b\r\n\r\n\ + --b\r\nContent-Location: {page}\r\nContent-Type: text/html\r\n\r\n

x

\r\n\ + --b\r\nContent-Location: https://example.test/images/logo.png\r\nContent-Transfer-Encoding: base64\r\nContent-Type: image/png\r\n\r\n{PNG_1X1}\r\n--b--\r\n" + ); + parse(m.as_bytes(), &Limits::default()) + } + + #[test] + fn references_relative_to_the_page_resolve_as_uris() { + // The page in `reports/`, its picture in `images/` (RFC 2557 §5). + let m = page_and_logo("https://example.test/reports/page.html"); + let resources = Resources::of(&m.root); + let base = "https://example.test/reports/page.html"; + for src in [ + "../images/logo.png", + "/images/logo.png", + "./../images/./logo.png", + "//example.test/images/logo.png", + "https://example.test/images/logo.png", + " HTTPS://EXAMPLE.TEST/images/LOGO.png ", + // Nothing past the root: `..` there stays at the root. + "../../../images/logo.png", + ] { + let part = resources.find(src, base); + assert_eq!(part.map(|p| p.media_type()), Some("image/png"), "{src}"); + assert!(part.is_some_and(|p| p.data().starts_with(b"\x89PNG"))); + } + assert!(resources.find("images/logo.png", base).is_none()); + assert!(resources.find("cid:logo", base).is_none()); + assert!(resources.by_location("https://example.test/reports/page.html").is_some()); + } + + #[test] + fn locations_resolve_as_rfc_3986_says() { + assert_eq!(resolve_location("https://h/a/b/c.html", "../d/./e.png#f"), "https://h/a/d/e.png"); + assert_eq!(resolve_location("file:///C:/x/page.htm", "page_files/dot.png"), "file:///C:/x/page_files/dot.png"); + assert_eq!(resolve_location("", "dot.png"), "dot.png"); + // A host with no path, and a base whose query holds slashes (§5.2.3: the path merges, + // not the query; an authority with an empty path gets its `/`). + assert_eq!(resolve_location("https://example.test", "logo.png"), "https://example.test/logo.png"); + assert_eq!(resolve_location("https://example.test/page.html?redirect=/other/", "logo.png"), "https://example.test/logo.png"); + assert_eq!(resolve_location("https://h/a/p.html?q", "?r"), "https://h/a/p.html?r"); + assert_eq!(resolve_location("https://h/a/p.html?q", "x.png?v=1"), "https://h/a/x.png?v=1"); + assert_eq!(resolve_location("https://h/a/p.html", "mailto:x@y"), "mailto:x@y"); + assert_eq!(resolve_location("https://h/a/p.html", ".."), "https://h/"); + assert_eq!(resolve_location("https://h/a/p.html", ""), "https://h/a/p.html"); + } + + #[test] + fn the_start_part_is_the_root_and_cid_names_pictures() { + let m = format!( + "MIME-Version: 1.0\nContent-Type: multipart/related; type=\"text/html\"; start=\"\"; boundary=b\n\n\ + --b\nContent-Type: text/html\n\n

Not this one

\n\ + --b\nContent-ID: \nContent-Type: text/html\n\n

This one

\n\ + --b\nContent-ID: \nContent-Transfer-Encoding: base64\nContent-Type: image/png\n\n{PNG_1X1}\n\ + --b\nContent-ID: \nContent-Type: text/plain\n\nA second part of the same id.\n--b--\n" + ); + let m = parse(m.as_bytes(), &Limits::default()); + assert_eq!(start_part(&m.root).map(Part::text).as_deref(), Some("

This one

")); + let resources = Resources::of(&m.root); + assert_eq!(resources.find("CID:dot", "").map(|p| p.media_type()), Some("image/png")); + assert_eq!(resources.by_content_id("page").map(|p| p.media_type()), Some("text/html")); + assert!(resources.by_content_id("").is_none() && resources.find("cid:", "").is_none()); + // No `start`, or one that names nothing. + assert!(start_part(&page_and_logo("https://example.test/p.html").root).is_none()); + let m = parse(b"Content-Type: multipart/related; start=\"\"; boundary=b\n\n--b\nContent-ID: \n\nx\n--b--\n", &Limits::default()); + assert!(start_part(&m.root).is_none()); + } + + proptest::proptest! { + #[test] + fn any_reference_resolves(base in "[a-c:/?#.]{0,24}", reference in "[a-c:/?#.]{0,24}") { + let resolved = resolve_location(&base, &reference); + proptest::prop_assert!(resolved.len() <= base.len() + reference.len() + 1); + } + + #[test] + fn any_text_resolves(base in proptest::prelude::any::(), reference in proptest::prelude::any::()) { + let _ = resolve_location(&base, &reference); + let _ = Resources::default().find(&reference, &base); + } + } +} diff --git a/crates/mime/src/transfer.rs b/crates/mime/src/transfer.rs new file mode 100644 index 00000000..de55a3ac --- /dev/null +++ b/crates/mime/src/transfer.rs @@ -0,0 +1,312 @@ +//! Content transfer encodings (RFC 2045): base64 and quoted-printable. + +/// Longest encoded line, without its line break. +const LINE: usize = 76; + +/// Decode base64, skipping anything that isn't base64 (line breaks, spaces, junk). Padding ends +/// a group, so several padded chunks one after another decode as they should. +pub fn base64_decode(input: &[u8]) -> Vec { + fn flush(out: &mut Vec, acc: u32, n: u8) { + match n { + 2 => out.push((acc >> 4) as u8), + 3 => { + out.push((acc >> 10) as u8); + out.push((acc >> 2) as u8); + } + _ => {} + } + } + let mut out = Vec::with_capacity(input.len() / 4 * 3); + let (mut acc, mut n) = (0u32, 0u8); + for &c in input { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + b'/' => 63, + b'=' => { + flush(&mut out, acc, n); + (acc, n) = (0, 0); + continue; + } + _ => continue, + }; + acc = (acc << 6) | u32::from(v); + n += 1; + if n == 4 { + out.extend_from_slice(&[(acc >> 16) as u8, (acc >> 8) as u8, acc as u8]); + (acc, n) = (0, 0); + } + } + flush(&mut out, acc, n); + out +} + +/// Base64 on one line (RFC 4648 section 4: `A-Za-z0-9+/`, padded with `=`). +pub fn base64(data: &[u8]) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let symbol = |n: u32| char::from(ALPHABET.get((n & 63) as usize).copied().unwrap_or(b'A')); + let mut out = String::with_capacity(data.len().div_ceil(3).saturating_mul(4)); + for group in data.chunks(3) { + let byte = |i: usize| u32::from(group.get(i).copied().unwrap_or(0)); + let n = byte(0) << 16 | byte(1) << 8 | byte(2); + out.push(symbol(n >> 18)); + out.push(symbol(n >> 12)); + out.push(if group.len() > 1 { symbol(n >> 6) } else { '=' }); + out.push(if group.len() > 2 { symbol(n) } else { '=' }); + } + out +} + +/// Base64 in lines of 76 characters, CRLF between them (none after the last). +pub fn base64_lines(data: &[u8]) -> Vec { + let encoded = base64(data).into_bytes(); + let mut out = Vec::with_capacity(encoded.len() + encoded.len() / LINE * 2); + for (i, line) in encoded.chunks(LINE).enumerate() { + if i > 0 { + out.extend_from_slice(b"\r\n"); + } + out.extend_from_slice(line); + } + out +} + +fn hex(b: u8) -> Option { + char::from(b).to_digit(16).map(|d| d as u8) +} + +/// Decode quoted-printable: `=XX` is a byte, `=` at the end of a line joins it to the next, and +/// white space at the end of a line is padding. A `=` that is neither stays as it is. +pub fn qp_decode(input: &[u8]) -> Vec { + let mut out = Vec::with_capacity(input.len()); + let mut i = 0; + while let Some(&c) = input.get(i) { + match c { + b'=' => { + if let (Some(h), Some(l)) = (input.get(i + 1).copied().and_then(hex), input.get(i + 2).copied().and_then(hex)) { + out.push((h << 4) | l); + i += 3; + continue; + } + let mut j = i + 1; + while matches!(input.get(j), Some(b' ' | b'\t')) { + j += 1; + } + match input.get(j) { + Some(b'\r') if input.get(j + 1) == Some(&b'\n') => i = j + 2, + Some(b'\r' | b'\n') => i = j + 1, + None => i = j, + Some(_) => { + out.push(b'='); + i += 1; + } + } + } + b' ' | b'\t' => { + let mut j = i; + while matches!(input.get(j), Some(b' ' | b'\t')) { + j += 1; + } + if !matches!(input.get(j), Some(b'\r' | b'\n') | None) { + out.extend_from_slice(input.get(i..j).unwrap_or_default()); + } + i = j; + } + c => { + out.push(c); + i += 1; + } + } + } + out +} + +/// Encode text as quoted-printable. `\n` is the line break and is written as CRLF; lines are at +/// most 76 characters. White space before a line break, a `From ` at the start of a line (which +/// mbox files would have to quote) and everything outside printable ASCII are encoded. +pub fn qp_encode(input: &[u8]) -> Vec { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut out = Vec::with_capacity(input.len() + input.len() / 8); + let mut col = 0; + for (i, &c) in input.iter().enumerate() { + if c == b'\n' { + out.extend_from_slice(b"\r\n"); + col = 0; + continue; + } + let next = input.get(i + 1); + let literal = match c { + b'=' => false, + b' ' | b'\t' => !matches!(next, Some(b'\n') | None), + b'F' if col == 0 => input.get(i..i + 5) != Some(b"From "), + b'.' if col == 0 => !matches!(next, Some(b'\n') | None), + 33..=126 => true, + _ => false, + }; + let width = if literal { 1 } else { 3 }; + // Keep room for the `=` of a soft line break. + if col + width > LINE - 1 { + out.extend_from_slice(b"=\r\n"); + col = 0; + } + if literal { + out.push(c); + } else { + out.extend_from_slice(&[ + b'=', + HEX.get(usize::from(c >> 4)).copied().unwrap_or(b'0'), + HEX.get(usize::from(c & 15)).copied().unwrap_or(b'0'), + ]); + } + col += width; + } + out +} + +/// Undo a content transfer encoding named as `Content-Transfer-Encoding` names it (any letter +/// case): `base64` and `quoted-printable`; `7bit`, `8bit`, `binary` and anything unknown are the +/// bytes themselves. +pub fn decode(body: &[u8], encoding: &str) -> Vec { + let encoding = encoding.trim(); + if encoding.eq_ignore_ascii_case("base64") { + base64_decode(body) + } else if encoding.eq_ignore_ascii_case("quoted-printable") { + qp_decode(body) + } else { + body.to_vec() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn base64_is_the_rfc_4648_vectors() { + for (plain, coded) in + [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] + { + assert_eq!(base64(plain.as_bytes()), coded); + assert_eq!(base64_decode(coded.as_bytes()), plain.as_bytes()); + } + assert_eq!(base64(&[0xfb, 0xff]), "+/8="); + } + + #[test] + fn transfer_encodings_by_name() { + assert_eq!(decode(b"SGk=", " Base64 "), b"Hi"); + assert_eq!(decode(b"a=3Db", "Quoted-Printable"), b"a=b"); + for identity in ["7bit", "8bit", "binary", "", "x-unknown"] { + assert_eq!(decode(b"a=3Db", identity), b"a=3Db"); + } + } + + #[test] + fn base64_plain_and_padded() { + assert_eq!(base64_decode(b"SGVsbG8="), b"Hello"); + assert_eq!(base64_decode(b"SGVsbG8"), b"Hello"); + assert_eq!(base64_decode(b"SGVsbA=="), b"Hell"); + assert_eq!(base64_decode(b"SGVs"), b"Hel"); + assert_eq!(base64_decode(b""), b""); + assert_eq!(base64_decode(b"="), b""); + assert_eq!(base64_decode(b"S"), b""); + } + + #[test] + fn base64_skips_junk_and_line_breaks() { + assert_eq!(base64_decode(b"SGVs\r\nbG8g\r\nd29y bGQ=\r\n"), b"Hello world"); + assert_eq!(base64_decode(b" S*G(V)s!b~G8\x00\xFF= "), b"Hello"); + // Two padded chunks, as some senders write. + assert_eq!(base64_decode(b"SGk=SGk="), b"HiHi"); + } + + #[test] + fn base64_lines_wrap_at_76() { + let data: Vec = (0..200u8).collect(); + let lines = base64_lines(&data); + let text = String::from_utf8(lines.clone()).unwrap(); + assert!(text.split("\r\n").all(|l| l.len() <= 76 && !l.is_empty())); + assert_eq!(text.split("\r\n").next().map(str::len), Some(76)); + assert!(!text.ends_with('\n')); + assert_eq!(base64_decode(&lines), data); + assert_eq!(base64_lines(b""), b""); + } + + #[test] + fn quoted_printable_decodes() { + assert_eq!(qp_decode(b"caf=E9 =3D ok"), b"caf\xE9 = ok"); + assert_eq!(qp_decode(b"lower=e9"), b"lower\xE9"); + assert_eq!(qp_decode(b"soft=\r\nbreak=\nand= \t\r\nmore"), b"softbreakandmore"); + assert_eq!(qp_decode(b"keep\r\nhard\nbreaks"), b"keep\r\nhard\nbreaks"); + assert_eq!(qp_decode(b"pad \r\nded\t\nend "), b"pad\r\nded\nend"); + assert_eq!(qp_decode(b"a b"), b"a b"); + } + + #[test] + fn quoted_printable_tolerates_stray_equals() { + assert_eq!(qp_decode(b"a=b"), b"a=b"); + assert_eq!(qp_decode(b"a=ZZ"), b"a=ZZ"); + assert_eq!(qp_decode(b"a=4"), b"a=4"); + assert_eq!(qp_decode(b"x=="), b"x="); + assert_eq!(qp_decode(b"end="), b"end"); + assert_eq!(qp_decode(b"=\r"), b""); + } + + #[test] + fn quoted_printable_encodes() { + assert_eq!(qp_encode("café = ok\nnext".as_bytes()), b"caf=C3=A9 =3D ok\r\nnext"); + assert_eq!(qp_encode(b"trailing \nand\t\nend "), b"trailing=20\r\nand=09\r\nend=20"); + assert_eq!(qp_encode(b"From me\nFrom you\n Fromage"), b"=46rom me\r\n=46rom you\r\n Fromage"); + assert_eq!(qp_encode(b".\n.x\n."), b"=2E\r\n.x\r\n=2E"); + assert_eq!(qp_encode(b"a\rb\x00"), b"a=0Db=00"); + assert_eq!(qp_encode(b""), b""); + } + + #[test] + fn quoted_printable_lines_stay_within_76() { + let text = format!("{}\n{}", "x".repeat(300), "é".repeat(100)); + let encoded = qp_encode(text.as_bytes()); + let s = String::from_utf8(encoded.clone()).unwrap(); + for line in s.split("\r\n") { + assert!(line.len() <= 76, "{}", line.len()); + } + assert_eq!(qp_decode(&encoded), text.replace('\n', "\r\n").as_bytes()); + } + + #[test] + fn long_runs_decode_quickly() { + let spaces = [b' '; 300_000]; + assert_eq!(qp_decode(&spaces), b""); + let mut padded = spaces.to_vec(); + padded.push(b'x'); + assert_eq!(qp_decode(&padded).len(), 300_001); + // Every `=` but the last is literal; the last one is a soft break at the end. + let equals = [b'='; 300_000]; + assert_eq!(qp_decode(&equals).len(), 299_999); + let mixed = b"= ".repeat(150_000); + let _ = qp_decode(&mixed); + } + + proptest::proptest! { + #[test] + fn base64_round_trips(data in proptest::collection::vec(proptest::prelude::any::(), 0..300)) { + proptest::prop_assert_eq!(base64_decode(&base64_lines(&data)), data); + } + + #[test] + fn quoted_printable_round_trips(text in proptest::prelude::any::()) { + let text = text.replace('\r', ""); + let encoded = qp_encode(text.as_bytes()); + proptest::prop_assert!(encoded.split(|b| *b == b'\n').all(|l| l.len() <= 77)); + proptest::prop_assert!(encoded.iter().all(|b| matches!(*b, b'\r' | b'\n' | b'\t' | 32..=126))); + proptest::prop_assert_eq!(qp_decode(&encoded), text.replace('\n', "\r\n").into_bytes()); + } + + #[test] + fn decoders_take_any_bytes(data in proptest::collection::vec(proptest::prelude::any::(), 0..300)) { + let _ = base64_decode(&data); + let _ = qp_decode(&data); + } + } +} diff --git a/crates/mime/src/words.rs b/crates/mime/src/words.rs new file mode 100644 index 00000000..cfa35e1f --- /dev/null +++ b/crates/mime/src/words.rs @@ -0,0 +1,306 @@ +//! RFC 2047 encoded-words: non-ASCII text in header fields (`=?UTF-8?B?…?=`). + +use crate::charset; +use crate::transfer::{base64, base64_decode}; + +/// Most text bytes in one encoded-word: 45 bytes are 60 base64 characters, 72 with the wrapping. +pub const WORD_BYTES: usize = 45; + +/// Longest character set name read. +const MAX_CHARSET: usize = 40; + +/// RFC 2047 encoded-words to text. +/// +/// Reads the `B` and `Q` encodings in every character set of the WHATWG Encoding Standard +/// ([`charset::decode`]); a set it doesn't name is read as UTF-8, with U+FFFD for what doesn't fit. +/// White space between +/// two encoded-words is dropped, as the RFC says. Words in a row that share a character set are +/// decoded together, so a character split across two words still comes out whole. Anything that +/// isn't a well-formed encoded-word is left as it is. +pub fn decode_words(s: &str) -> String { + if !s.contains("=?") { + return s.to_string(); + } + fn flush(out: &mut String, pending: &mut Option<(String, Vec)>) { + if let Some((charset, bytes)) = pending.take() { + out.push_str(&charset::decode(&bytes, &charset)); + } + } + let mut out = String::with_capacity(s.len()); + let mut pending: Option<(String, Vec)> = None; + let mut rest = s; + while let Some(i) = rest.find("=?") { + let before = rest.get(..i).unwrap_or(""); + let tail = rest.get(i..).unwrap_or(""); + match word(tail) { + Some((charset, bytes, used)) => { + if pending.is_none() || !before.chars().all(|c| matches!(c, ' ' | '\t' | '\r' | '\n')) { + flush(&mut out, &mut pending); + out.push_str(before); + } + match &mut pending { + Some((current, buffer)) if *current == charset => buffer.extend_from_slice(&bytes), + _ => { + flush(&mut out, &mut pending); + pending = Some((charset, bytes)); + } + } + rest = tail.get(used..).unwrap_or(""); + } + None => { + flush(&mut out, &mut pending); + out.push_str(before); + out.push_str("=?"); + rest = tail.get(2..).unwrap_or(""); + } + } + } + flush(&mut out, &mut pending); + out.push_str(rest); + out +} + +/// One encoded-word at the start of `s`: its character set (lower case), its bytes, and its length. +fn word(s: &str) -> Option<(String, Vec, usize)> { + let body = s.strip_prefix("=?")?; + let charset_len = body.bytes().take(MAX_CHARSET + 1).position(|b| b == b'?')?; + let charset = body.get(..charset_len)?; + if charset.is_empty() || charset.bytes().any(|b| b.is_ascii_whitespace() || b.is_ascii_control()) { + return None; + } + let after = body.get(charset_len + 1..)?.as_bytes(); + let encoding = after.first()?.to_ascii_uppercase(); + if after.get(1) != Some(&b'?') { + return None; + } + let text = after.get(2..)?; + let text_len = text.iter().position(|b| *b == b'?')?; + if text.get(text_len + 1) != Some(&b'=') { + return None; + } + let text = text.get(..text_len)?; + // An encoded-word is one word: white space inside means this is something else. + if text.iter().any(u8::is_ascii_whitespace) { + return None; + } + let bytes = match encoding { + b'B' => base64_decode(text), + b'Q' => q_decode(text), + _ => return None, + }; + // `charset*language` (RFC 2231): the language is not needed. + let charset = charset.split('*').next().unwrap_or(charset).to_ascii_lowercase(); + Some((charset, bytes, 2 + charset_len + 3 + text_len + 2)) +} + +/// The `Q` encoding: `_` is a space, `=XX` a byte. +fn q_decode(text: &[u8]) -> Vec { + let hex = |b: Option<&u8>| b.and_then(|b| char::from(*b).to_digit(16)).map(|d| d as u8); + let mut out = Vec::with_capacity(text.len()); + let mut i = 0; + while let Some(&c) = text.get(i) { + match (c, hex(text.get(i + 1)), hex(text.get(i + 2))) { + (b'=', Some(h), Some(l)) => { + out.push((h << 4) | l); + i += 3; + } + (b'_', ..) => { + out.push(b' '); + i += 1; + } + _ => { + out.push(c); + i += 1; + } + } + } + out +} + +/// Whether text can go in a header field as it is: printable ASCII that doesn't look like an encoded-word. +pub fn is_plain(s: &str) -> bool { + s.bytes().all(|b| (0x20..=0x7E).contains(&b)) && !s.contains("=?") +} + +/// Text in a header-safe form. +/// +/// Printable ASCII is returned unchanged (unless it holds `=?`, which a reader would take for an +/// encoded-word). Anything else becomes UTF-8 `B` encoded-words of at most 75 characters each, +/// separated by single spaces, so a header can be folded between them. The result never holds a +/// line break; [`decode_words`] gives the text back exactly. +pub fn encode_words(s: &str) -> String { + if is_plain(s) { s.to_string() } else { encode_b(s, WORD_BYTES) } +} + +/// UTF-8 `B` encoded-words whatever the text; the first word holds at most `first` bytes of it +/// (to leave room for a header name), the others [`WORD_BYTES`]. +pub fn encode_b(s: &str, first: usize) -> String { + let mut out = String::with_capacity(s.len().saturating_add(s.len() / 2).saturating_add(16)); + let mut limit = first.clamp(4, WORD_BYTES); + let mut start = 0; + let mut end = 0; + let emit = |chunk: &str, out: &mut String| { + if !out.is_empty() { + out.push(' '); + } + out.push_str("=?UTF-8?B?"); + out.push_str(&base64(chunk.as_bytes())); + out.push_str("?="); + }; + for (i, c) in s.char_indices() { + if i + c.len_utf8() - start > limit && i > start { + emit(s.get(start..i).unwrap_or(""), &mut out); + start = i; + limit = WORD_BYTES; + } + end = i + c.len_utf8(); + } + if end > start || s.is_empty() { + emit(s.get(start..end).unwrap_or(""), &mut out); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_b_and_q() { + assert_eq!(decode_words("=?UTF-8?B?SMOpbGxv?="), "Héllo"); + assert_eq!(decode_words("=?utf-8?q?H=C3=A9llo_world?="), "Héllo world"); + assert_eq!(decode_words("=?ISO-8859-1?Q?caf=E9?="), "café"); + assert_eq!(decode_words("=?iso-8859-15?Q?=A4_5?="), "€ 5"); + assert_eq!(decode_words("=?windows-1252?Q?=93hi=94?="), "“hi”"); + assert_eq!(decode_words("=?US-ASCII?Q?plain?="), "plain"); + assert_eq!(decode_words("=?UTF-8*en?Q?lang?="), "lang"); + } + + #[test] + fn text_around_words_is_kept() { + assert_eq!(decode_words("Re: =?UTF-8?Q?caf=C3=A9?= today"), "Re: café today"); + assert_eq!(decode_words("no words here"), "no words here"); + assert_eq!(decode_words(""), ""); + assert_eq!(decode_words("a =?UTF-8?Q?b?= c =?UTF-8?Q?d?= e"), "a b c d e"); + assert_eq!(decode_words("\"=?UTF-8?Q?Ad=C3=A1?=\" "), "\"Adá\" "); + } + + #[test] + fn space_between_words_is_dropped() { + assert_eq!(decode_words("=?UTF-8?Q?a?= =?UTF-8?Q?b?="), "ab"); + assert_eq!(decode_words("=?UTF-8?Q?a?=\r\n\t=?UTF-8?Q?b?= =?ISO-8859-1?Q?=E9?="), "abé"); + assert_eq!(decode_words("=?UTF-8?Q?a?==?UTF-8?Q?b?="), "ab"); + assert_eq!(decode_words("=?UTF-8?Q?a?= x =?UTF-8?Q?b?="), "a x b"); + assert_eq!(decode_words("=?UTF-8?Q?a_?= =?UTF-8?Q?b?="), "a b"); + } + + #[test] + fn a_character_split_across_words_is_whole() { + // "é" is C3 A9; one byte in each word. + assert_eq!(decode_words("=?UTF-8?Q?caf=C3?= =?UTF-8?Q?=A9!?="), "café!"); + assert_eq!(decode_words("=?UTF-8?B?w6nD?= =?utf-8?B?qQ==?="), "éé"); + } + + #[test] + fn malformed_words_are_left_alone() { + for s in [ + "=?", + "=?UTF-8", + "=?UTF-8?", + "=?UTF-8?Q", + "=?UTF-8?Q?", + "=?UTF-8?Q?abc", + "=?UTF-8?Q?abc?", + "=?UTF-8?X?abc?=", + "=??Q?abc?=", + "=?UTF-8?Q?a b?=", + "=? ?Q?a?=", + "a=?b", + "=?=?=?=?", + ] { + assert_eq!(decode_words(s), s, "{s}"); + } + assert_eq!(decode_words("=?x?=?UTF-8?Q?ok?="), "=?x?ok"); + } + + #[test] + fn unknown_charsets_and_bad_bytes_do_not_fail() { + assert_eq!(decode_words("=?koi8-r?Q?abc?="), "abc"); + assert_eq!(decode_words("=?UTF-8?Q?=FF?="), "\u{fffd}"); + assert_eq!(decode_words("=?UTF-8?B?!!!!?="), ""); + assert_eq!(decode_words("=?UTF-8?Q?=ZZ=4?="), "=ZZ=4"); + } + + #[test] + fn ascii_is_not_encoded() { + assert_eq!(encode_words("Lunch on Friday?"), "Lunch on Friday?"); + assert_eq!(encode_words(""), ""); + assert_eq!(encode_words("a = b ? c"), "a = b ? c"); + } + + #[test] + fn non_ascii_becomes_b_words() { + assert_eq!(encode_words("Héllo"), "=?UTF-8?B?SMOpbGxv?="); + assert_eq!(decode_words(&encode_words("naïve café")), "naïve café"); + // Control characters and things that look like encoded-words too. + assert!(encode_words("tab\there").starts_with("=?UTF-8?B?")); + assert!(encode_words("=?UTF-8?Q?fake?=").starts_with("=?UTF-8?B?")); + assert_eq!(decode_words(&encode_words("=?UTF-8?Q?fake?=")), "=?UTF-8?Q?fake?="); + } + + #[test] + fn long_text_folds_into_words_of_at_most_75() { + let text = "Ünïcödé ".repeat(40); + let encoded = encode_words(&text); + let words: Vec<&str> = encoded.split(' ').collect(); + assert!(words.len() > 5); + for w in &words { + assert!(w.len() <= 75, "{}", w.len()); + assert!(w.starts_with("=?UTF-8?B?") && w.ends_with("?=")); + } + assert!(!encoded.contains(['\r', '\n'])); + assert_eq!(decode_words(&encoded), text); + // Four-byte characters are never cut. + let emoji = "😀".repeat(50); + assert_eq!(decode_words(&encode_words(&emoji)), emoji); + } + + #[test] + fn the_first_word_can_be_shorter() { + let text = "é".repeat(40); + let encoded = encode_b(&text, 10); + let first = encoded.split(' ').next().unwrap(); + assert_eq!(first, "=?UTF-8?B?w6nDqcOpw6nDqQ==?="); + assert_eq!(decode_words(&encoded), text); + assert_eq!(decode_words(&encode_b("", 45)), ""); + assert_eq!(decode_words(&encode_b("x", 0)), "x"); + } + + #[test] + fn hostile_input_is_linear() { + for s in [ + "=?".repeat(200_000), + "=?a?Q?".repeat(80_000), + "=?a?Q?b?= ".repeat(40_000), + format!("=?{}", "a".repeat(400_000)), + "=?UTF-8?Q?".to_string() + &"x".repeat(400_000), + ] { + let _ = decode_words(&s); + } + } + + proptest::proptest! { + #[test] + fn any_text_round_trips(s in proptest::prelude::any::()) { + let encoded = encode_words(&s); + proptest::prop_assert!(!encoded.contains(['\r', '\n', '\0']) || is_plain(&s)); + proptest::prop_assert_eq!(decode_words(&encoded), s.clone()); + proptest::prop_assert_eq!(decode_words(&encode_b(&s, 7)), s); + } + + #[test] + fn any_text_decodes(s in "[=?a-zA-Z0-9_ -]{0,40}") { + let _ = decode_words(&s); + } + } +} diff --git a/crates/mime/src/write.rs b/crates/mime/src/write.rs new file mode 100644 index 00000000..91a08fc9 --- /dev/null +++ b/crates/mime/src/write.rs @@ -0,0 +1,335 @@ +//! MIME entities to bytes: text and file parts, multiparts and their boundaries, and the +//! header values that need encoding. + +use crate::header::{clean, is_field_name, write_field}; +use crate::words::{WORD_BYTES, encode_b, is_plain}; +use crate::{charset, transfer}; + +/// Longest physical line a body may have without a transfer encoding (RFC 5322 §2.1.1). +const MAX_LINE: usize = 998; + +/// Longest piece of a file name parameter, so that its header line folds within 78 characters. +const NAME_PIECE: usize = 52; + +/// Longest boundary prefix kept (a boundary is at most 70 characters, RFC 2046 §5.1.1). +const MAX_PREFIX: usize = 32; + +/// Free text for a header field called `name` (`Subject`, `Comments`): as it is when it is +/// plain, else UTF-8 encoded-words, the first one short enough to fit on the line after the +/// name. Text that begins or ends with a space is encoded too, because readers trim header +/// values, and so is a word too long to fold. CR, LF and NUL are dropped. +pub fn encode_unstructured(name: &str, value: &str) -> String { + let value = clean(value); + if is_plain(&value) && !value.starts_with(' ') && !value.ends_with(' ') && !unfoldable(&value) { + return value; + } + // Leave room for the name on the first line. + let room = 76usize.saturating_sub(name.len().saturating_add(2 + 12)) / 4 * 3; + encode_b(&value, if room < 9 { WORD_BYTES } else { room }) +} + +/// A value that is written as it stands when it is printable ASCII (and tabs), `=?` and all, +/// for a field whose readers don't decode encoded-words (a structured or a trace field kept +/// from a message read); anything else as [`encode_unstructured`] text, so that what is +/// written stays seven-bit. +pub fn seven_bit_value(name: &str, value: &str) -> String { + let value = clean(value); + let edge = |c: char| c == ' ' || c == '\t'; + if value.bytes().all(|b| b == b'\t' || (0x20..=0x7E).contains(&b)) && !value.starts_with(edge) && !value.ends_with(edge) && !unfoldable(&value) { + return value; + } + encode_unstructured(name, &value) +} + +/// Whether a word of `value` is too long for any line it could be folded onto. +fn unfoldable(value: &str) -> bool { + value.split(' ').any(|word| word.len() > MAX_LINE - 100) +} + +/// A message id or content id without anything that could end it early: no control characters, +/// no white space, no angle brackets. Written between `<` and `>`. +pub fn clean_id(raw: &str) -> String { + raw.chars().filter(|c| !c.is_control() && !c.is_whitespace() && !matches!(c, '<' | '>')).collect() +} + +/// A media type that is safe to write for a leaf part: `type/subtype` in lower case. Anything +/// malformed, and `multipart/*` (which would make readers look for parts), becomes +/// `application/octet-stream`. +pub fn safe_media_type(media_type: &str) -> String { + let t = media_type.trim().to_ascii_lowercase(); + match t.split_once('/') { + Some((kind, sub)) if is_type_token(kind) && is_type_token(sub) && kind != "multipart" => t, + _ => "application/octet-stream".to_string(), + } +} + +/// Whether `s` can be a media type's type or subtype (RFC 6838 §4.2). +fn is_type_token(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"!#$&^_.+-".contains(&b)) +} + +/// A quoted-string: only `"` and `\` need a backslash. CR, LF and NUL are dropped. +pub fn quoted(s: &str) -> String { + format!("\"{}\"", clean(s).replace('\\', "\\\\").replace('"', "\\\"")) +} + +/// The `filename` parameters of `Content-Disposition`, each with its `; ` before it: a quoted +/// name when it is plain ASCII, else the RFC 2231 form (`filename*=utf-8''…`, in numbered pieces +/// when long) after an ASCII stand-in for readers that don't know it. +pub fn file_name_params(name: &str) -> String { + let name = clean(name); + let name = name.as_str(); + if is_plain(name) && name.len() <= 200 { + return format!("; filename={}", quoted(name)); + } + let stand_in: String = + name.chars().map(|c| if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { c } else { '_' }).take(NAME_PIECE).collect(); + let mut out = format!("; filename=\"{stand_in}\""); + let mut pieces: Vec = vec![String::new()]; + for b in name.bytes() { + let token = if b.is_ascii_alphanumeric() || b"!#$&+-.^_`|~".contains(&b) { char::from(b).to_string() } else { format!("%{b:02X}") }; + match pieces.last_mut() { + Some(piece) if piece.len().saturating_add(token.len()) <= NAME_PIECE => piece.push_str(&token), + _ => pieces.push(token), + } + } + match pieces.as_slice() { + [only] => out.push_str(&format!("; filename*=utf-8''{only}")), + many => { + for (i, piece) in many.iter().enumerate() { + out.push_str(&format!("; filename*{i}*={}{piece}", if i == 0 { "utf-8''" } else { "" })); + } + } + } + out +} + +/// A file to write as a part ([`Entity::file`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct File<'a> { + /// Its media type; made safe with [`safe_media_type`]. + pub media_type: &'a str, + /// Its name; no name parameters are written when it is empty. (A `text/plain` or + /// `text/html` part without a name is read as body text by mail readers: give it one.) + pub name: &'a str, + /// `Content-Disposition: inline`, not `attachment`. + pub inline: bool, + /// Its `Content-ID`, without angle brackets; none is written when it is empty. + pub content_id: &'a str, + pub data: &'a [u8], +} + +/// A MIME entity ready to be written: its content fields and its encoded body. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Entity { + /// The fields that say what the body is (`Content-Type`, `Content-Transfer-Encoding`…), + /// names and values. More can be pushed (`Content-Location`); every value is written + /// through [`write_field`], which no value can break out of. + pub fields: Vec<(String, String)>, + /// The body as it goes on the wire, without a line end after it. + pub content: Vec, +} + +impl Entity { + /// `text/; charset=utf-8`: `7bit` when the text is ASCII in lines short enough, + /// `quoted-printable` otherwise. Every line end is written as CRLF. A `subtype` that is no + /// token becomes `plain`. + pub fn text(subtype: &str, text: &str) -> Entity { + let subtype = if is_type_token(subtype) { subtype.to_ascii_lowercase() } else { "plain".to_string() }; + let text = charset::unix_lines(text); + let seven_bit = text.bytes().all(|b| matches!(b, b'\n' | b'\t' | 0x20..=0x7E)) && text.split('\n').all(|line| line.len() <= MAX_LINE); + let (encoding, content) = + if seven_bit { ("7bit", text.replace('\n', "\r\n").into_bytes()) } else { ("quoted-printable", transfer::qp_encode(text.as_bytes())) }; + Entity { + fields: vec![("Content-Type".into(), format!("text/{subtype}; charset=utf-8")), ("Content-Transfer-Encoding".into(), encoding.into())], + content, + } + } + + /// A file: base64 in 76-column lines, with its name in both `Content-Type` (`name`, as + /// encoded-words, which is all old readers know) and `Content-Disposition` (`filename`, + /// [`file_name_params`]). An attached message (`message/rfc822`) that is clean 7-bit text + /// goes as it is, as RFC 2046 §5.2.1 asks. + pub fn file(file: &File<'_>) -> Entity { + let mime = safe_media_type(file.media_type); + let name = clean(file.name); + let mut content_type = mime.clone(); + let mut disposition = String::from(if file.inline { "inline" } else { "attachment" }); + if !name.is_empty() { + content_type.push_str("; name="); + // The first word is shorter, to fit on a line after ` name="`. + content_type.push_str("ed(&if is_plain(&name) { name.clone() } else { encode_b(&name, 42) })); + disposition.push_str(&file_name_params(&name)); + } + let (encoding, content) = if mime == "message/rfc822" && is_seven_bit_message(file.data) { + ("7bit", file.data.to_vec()) + } else { + ("base64", transfer::base64_lines(file.data)) + }; + let mut fields = vec![ + ("Content-Type".to_string(), content_type), + ("Content-Transfer-Encoding".to_string(), encoding.to_string()), + ("Content-Disposition".to_string(), disposition), + ]; + let content_id = clean_id(file.content_id); + if !content_id.is_empty() { + fields.push(("Content-ID".to_string(), format!("<{content_id}>"))); + } + Entity { fields, content } + } + + /// `multipart/` holding `parts`, with a boundary from `boundaries` that occurs in + /// none of them. `params` (`("type", "text/html")`) go between the type and the boundary, + /// each value as a quoted-string; a parameter whose name is no token is left out, and a + /// `subtype` that is no token becomes `mixed`. + pub fn multipart(subtype: &str, params: &[(&str, &str)], parts: &[Entity], boundaries: &mut Boundaries) -> Entity { + let subtype = if is_type_token(subtype) { subtype.to_ascii_lowercase() } else { "mixed".to_string() }; + let children: Vec> = parts.iter().map(Entity::to_bytes).collect(); + let boundary = boundaries.next(&children); + let size = children + .iter() + .map(|c| c.len().saturating_add(boundary.len()).saturating_add(8)) + .fold(boundary.len().saturating_add(8), usize::saturating_add); + let mut content = Vec::with_capacity(size); + for child in &children { + content.extend_from_slice(b"--"); + content.extend_from_slice(boundary.as_bytes()); + content.extend_from_slice(b"\r\n"); + content.extend_from_slice(child); + content.extend_from_slice(b"\r\n"); + } + content.extend_from_slice(b"--"); + content.extend_from_slice(boundary.as_bytes()); + content.extend_from_slice(b"--"); + let mut content_type = format!("multipart/{subtype};"); + for (name, value) in params.iter().filter(|(name, _)| is_type_token(name) && !name.eq_ignore_ascii_case("boundary")) { + content_type.push_str(&format!(" {name}={};", quoted(value))); + } + content_type.push_str(&format!(" boundary=\"{boundary}\"")); + Entity { fields: vec![("Content-Type".to_string(), content_type)], content } + } + + /// The entity as it stands inside a multipart: its fields, an empty line, its body. + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(self.content.len().saturating_add(256)); + for (name, value) in &self.fields { + write_field(&mut out, name, value); + } + out.extend_from_slice(b"\r\n"); + out.extend_from_slice(&self.content); + out + } +} + +/// Write a whole message with CRLF line ends: `headers` in order, `MIME-Version: 1.0`, the +/// content fields of `body`, an empty line, and the body. +/// +/// Every value goes through [`write_field`]: CR, LF and NUL are dropped and long lines are +/// folded, so no value can add a header field; a header whose name isn't a field name is left +/// out, and so are a `MIME-Version` and any `Content-…` field among `headers` (the body says +/// those). Values are otherwise written as given: the caller encodes text +/// ([`encode_unstructured`]), addresses ([`crate::format_address_list`]) and ids ([`clean_id`]). +pub fn write_message(headers: &[(&str, &str)], body: &Entity) -> Vec { + let mut out = Vec::new(); + for (name, value) in headers { + let lower = name.to_ascii_lowercase(); + if is_field_name(name.as_bytes()) && lower != "mime-version" && !lower.starts_with("content-") { + write_field(&mut out, name, value); + } + } + write_field(&mut out, "MIME-Version", "1.0"); + out.extend_from_slice(&body.to_bytes()); + out.extend_from_slice(b"\r\n"); + out +} + +/// Whether bytes can travel as they are: ASCII without NUL, CRLF line ends only, no long lines. +fn is_seven_bit_message(data: &[u8]) -> bool { + let mut line = 0usize; + let mut prev = 0u8; + for (i, &b) in data.iter().enumerate() { + match b { + 0 | 0x80..=0xFF => return false, + b'\n' if prev != b'\r' => return false, + b'\n' => line = 0, + b'\r' if data.get(i.saturating_add(1)) != Some(&b'\n') => return false, + b'\r' => {} + _ => { + line = line.saturating_add(1); + if line > MAX_LINE { + return false; + } + } + } + prev = b; + } + true +} + +/// Where the boundaries of one message's multiparts come from. +/// +/// A boundary is a prefix, an FNV-1a hash of the parts and a counter (which sets the multiparts +/// of one message apart, and moves on when a part happens to hold the boundary), so the same +/// content always gets the same boundary and the same bytes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Boundaries { + prefix: String, + counter: u32, +} + +impl Default for Boundaries { + fn default() -> Self { + Boundaries::new("=_craft_") + } +} + +impl Boundaries { + /// Boundaries that start with `prefix`, of which only the characters a boundary may hold + /// without quoting trouble are kept (letters, digits, `=`, `_`, `-`, `.`), at most 32. + pub fn new(prefix: &str) -> Self { + let prefix = prefix.chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '=' | '_' | '-' | '.')).take(MAX_PREFIX).collect(); + Boundaries { prefix, counter: 0 } + } + + /// A boundary that occurs in none of `children`. + fn next(&mut self, children: &[Vec]) -> String { + const PRIME: u64 = 0x0000_0100_0000_01b3; + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for child in children { + for b in child { + hash = (hash ^ u64::from(*b)).wrapping_mul(PRIME); + } + // The edge between two parts counts too. + hash = (hash ^ 0x1ff).wrapping_mul(PRIME); + } + self.free(hash, children) + } + + /// The first boundary for `hash`, counting up, that none of the parts holds. + fn free(&mut self, hash: u64, children: &[Vec]) -> String { + loop { + let candidate = format!("{}{hash:016x}_{}", self.prefix, self.counter); + self.counter = self.counter.wrapping_add(1); + if !children.iter().any(|c| contains(c, candidate.as_bytes())) { + return candidate; + } + } + } +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + let Some((first, rest)) = needle.split_first() else { return true }; + let mut from = 0usize; + while let Some(i) = haystack.get(from..).and_then(|h| h.iter().position(|b| b == first)) { + let at = from.saturating_add(i).saturating_add(1); + if haystack.get(at..at.saturating_add(rest.len())) == Some(rest) { + return true; + } + from = at; + } + false +} + +#[cfg(test)] +#[path = "write_tests.rs"] +mod tests; diff --git a/crates/mime/src/write_tests.rs b/crates/mime/src/write_tests.rs new file mode 100644 index 00000000..6647c9f8 --- /dev/null +++ b/crates/mime/src/write_tests.rs @@ -0,0 +1,284 @@ +//! The writer: MailCraft's writer tests, for the parts of it that are format. + +use super::*; +use crate::{Limits, parse}; + +fn text(bytes: Vec) -> String { + String::from_utf8(bytes).unwrap_or_default() +} + +fn field<'a>(entity: &'a Entity, name: &str) -> Option<&'a str> { + entity.fields.iter().find(|(n, _)| n == name).map(|(_, v)| v.as_str()) +} + +#[test] +fn a_plain_message_byte_for_byte() { + let subject = encode_unstructured("Subject", "Lunch on Friday?"); + let from = crate::format_address_list(&[crate::Mailbox { name: "Ada Lovelace".into(), email: "ada@example.org".into() }]); + let headers = [ + ("Date", "Tue, 01 Jul 2003 08:52:37 +0000"), + ("From", from.as_str()), + ("Subject", subject.as_str()), + ("Message-ID", ""), + ]; + let want = "Date: Tue, 01 Jul 2003 08:52:37 +0000\r\n\ + From: Ada Lovelace \r\n\ + Subject: Lunch on Friday?\r\n\ + Message-ID: \r\n\ + MIME-Version: 1.0\r\n\ + Content-Type: text/plain; charset=utf-8\r\n\ + Content-Transfer-Encoding: 7bit\r\n\ + \r\n\ + Hello Bob,\r\n\r\nLunch?\r\n\r\n"; + assert_eq!(text(write_message(&headers, &Entity::text("plain", "Hello Bob,\n\nLunch?\n"))), want); +} + +#[test] +fn the_body_says_what_it_is_not_the_headers() { + let headers = + [("X-Note", "kept"), ("Content-Type", "text/evil"), ("mime-version", "9"), ("Bad Name", "x"), ("Content-Transfer-Encoding", "base64")]; + let out = text(write_message(&headers, &Entity::text("plain", "x"))); + assert_eq!(out, "X-Note: kept\r\nMIME-Version: 1.0\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 7bit\r\n\r\nx\r\n"); +} + +#[test] +fn long_and_non_ascii_subjects_fold() { + let written = |subject: &str| { + let mut out = Vec::new(); + assert!(write_field(&mut out, "Subject", &encode_unstructured("Subject", subject))); + text(out) + }; + let out = written(&"Ünïcödé subject ".repeat(12)); + assert!(out.starts_with("Subject: =?UTF-8?B?") && out.split("\r\n").all(|l| l.len() <= 78), "{out}"); + assert_eq!( + crate::decode_words(crate::split_headers(out.as_bytes(), true, &Limits::default()).0.get("subject").unwrap_or("")), + "Ünïcödé subject ".repeat(12) + ); + let out = written("plain words ".repeat(30).trim()); + assert!(out.starts_with("Subject: plain words plain words") && out.split("\r\n").all(|l| l.len() <= 78)); + // A subject that could not fold is encoded rather than written as one endless line. + assert!(written(&"x".repeat(3000)).split("\r\n").all(|l| l.len() <= 78)); + // Readers trim values: text with a space at an end is encoded, to keep it. + assert!(encode_unstructured("Subject", " x").starts_with("=?UTF-8?B?")); + assert_eq!(crate::decode_words(&encode_unstructured("Subject", "x ")), "x "); + assert_eq!(encode_unstructured("Subject", "a\r\nb\0"), "ab"); +} + +#[test] +fn kept_values_stay_seven_bit() { + assert_eq!(seven_bit_value("Received", "from a\tby b; =?x?="), "from a\tby b; =?x?="); + assert!(seven_bit_value("X-Note", "ñ").starts_with("=?UTF-8?B?")); + assert!(seven_bit_value("X-Note", "\tx").starts_with("=?UTF-8?B?")); + assert!(seven_bit_value("X-Note", &"x".repeat(3000)).split(' ').all(|w| w.len() <= 75)); + assert_eq!(seven_bit_value("X-Note", "a\r\nb"), "ab"); +} + +#[test] +fn text_encodings() { + let part = |t: &str| { + let p = Entity::text("plain", t); + (field(&p, "Content-Transfer-Encoding").unwrap_or("").to_string(), text(p.content)) + }; + assert_eq!(part("plain\ntext\r\nmixed\rends"), ("7bit".into(), "plain\r\ntext\r\nmixed\r\nends".into())); + assert_eq!(part("café"), ("quoted-printable".into(), "caf=C3=A9".into())); + assert_eq!(part("nul\0"), ("quoted-printable".into(), "nul=00".into())); + let (encoding, content) = part(&"x".repeat(1200)); + assert_eq!(encoding, "quoted-printable"); + assert!(content.split("\r\n").all(|l| l.len() <= 76)); + assert_eq!(part(&"x".repeat(998)).0, "7bit"); + assert_eq!(part("").0, "7bit"); + assert_eq!(field(&Entity::text("HTML", "x"), "Content-Type"), Some("text/html; charset=utf-8")); + assert_eq!(field(&Entity::text("html\r\nX-Evil: 1", "x"), "Content-Type"), Some("text/plain; charset=utf-8")); +} + +#[test] +fn alternative_related_and_mixed_nest_as_they_should() { + let mut boundaries = Boundaries::new("=_mc_"); + let logo = Entity::file(&File { + media_type: "image/png", + name: "logo.png", + inline: true, + content_id: "logo@example.org", + data: &[0x89, b'P', b'N', b'G'], + }); + let report = Entity::file(&File { media_type: "application/pdf", name: "report.pdf", data: b"%PDF-1.4\n", ..File::default() }); + let rich = Entity::multipart( + "related", + &[("type", "text/html")], + &[Entity::text("html", "

Hello

"), logo], + &mut boundaries, + ); + let forms = Entity::multipart("alternative", &[], &[Entity::text("plain", "Hello Bob,\n\nLunch?\n"), rich], &mut boundaries); + let root = Entity::multipart("mixed", &[], &[forms, report], &mut boundaries); + let bytes = write_message(&[("Subject", "Files")], &root); + let out = text(bytes.clone()); + let types: Vec<&str> = out.split("\r\n").filter(|l| l.starts_with("Content-Type:")).collect(); + assert_eq!(types.len(), 7, "{out}"); + assert!(types[0].starts_with("Content-Type: multipart/mixed; boundary=\"=_mc_")); + assert!(types[1].starts_with("Content-Type: multipart/alternative; boundary=\"=_mc_")); + assert_eq!(types[2], "Content-Type: text/plain; charset=utf-8"); + assert!(types[3].starts_with("Content-Type: multipart/related; type=\"text/html\";")); + assert_eq!(types[4], "Content-Type: text/html; charset=utf-8"); + assert_eq!(types[5], "Content-Type: image/png; name=\"logo.png\""); + assert_eq!(types[6], "Content-Type: application/pdf; name=\"report.pdf\""); + assert!(out.contains("Content-Transfer-Encoding: base64\r\nContent-Disposition: inline; filename=\"logo.png\"\r\nContent-ID: \r\n\r\niVBORw==\r\n--=_mc_")); + assert!(out.contains("Content-Disposition: attachment; filename=\"report.pdf\"\r\n\r\nJVBERi0xLjQK\r\n--=_mc_")); + assert!(out.ends_with("--\r\n")); + // Three multiparts, three different boundaries. + let mut found: Vec<&str> = out.split("boundary=\"").skip(1).filter_map(|s| s.split('"').next()).collect(); + found.sort_unstable(); + found.dedup(); + assert_eq!(found.len(), 3); + + // And the reader finds it all again. + let m = parse(&bytes, &Limits::default()); + let leaves = m.root.leaves(); + let got: Vec<(&str, Vec)> = leaves.iter().map(|p| (p.media_type(), p.data().to_vec())).collect(); + assert_eq!( + got, + [ + ("text/plain", b"Hello Bob,\r\n\r\nLunch?\r\n".to_vec()), + ("text/html", b"

Hello

".to_vec()), + ("image/png", vec![0x89, b'P', b'N', b'G']), + ("application/pdf", b"%PDF-1.4\n".to_vec()), + ] + ); + assert_eq!(leaves[2].content_id(), Some("logo@example.org")); + assert_eq!((leaves[2].disposition().value.as_str(), leaves[3].disposition().value.as_str()), ("inline", "attachment")); + assert_eq!(crate::Resources::of(&m.root).find("cid:logo@example.org", "").map(|p| p.media_type()), Some("image/png")); +} + +#[test] +fn file_names() { + assert_eq!(file_name_params("report.pdf"), "; filename=\"report.pdf\""); + assert_eq!(file_name_params("my \"best\" file\\x.txt"), r#"; filename="my \"best\" file\\x.txt""#); + assert_eq!(file_name_params("résumé €.txt"), "; filename=\"r_sum___.txt\"; filename*=utf-8''r%C3%A9sum%C3%A9%20%E2%82%AC.txt"); + // Something that looks like an encoded-word must not be read as one. + assert!(file_name_params("=?UTF-8?Q?x?=.txt").contains("filename*=utf-8''%3D%3FUTF-8%3FQ%3Fx%3F%3D.txt")); + let long = file_name_params(&"ü".repeat(40)); + assert!(long.contains("; filename*0*=utf-8''%C3%BC") && long.contains("; filename*4*=%") && !long.contains("filename*5"), "{long}"); + // Each piece fits a folded line: a space before it, a semicolon after. + for piece in long.split("; ") { + assert!(piece.len() + 2 <= 78, "{piece}"); + } + let file = Entity::file(&File { media_type: "image/png", name: &"ü".repeat(200), ..File::default() }); + assert!(text(file.to_bytes()).split("\r\n").all(|l| l.len() <= 78)); + // Every name reads back, from either place. + for name in ["report.pdf", "my \"best\" file\\x.txt", "résumé €.txt", "=?UTF-8?Q?x?=.txt", &"ü".repeat(200), "a;b=c.txt", "tab\there"] { + let bytes = write_message(&[], &Entity::file(&File { media_type: "application/pdf", name, ..File::default() })); + let m = parse(&bytes, &Limits::default()); + assert_eq!(m.root.file_name().as_deref(), Some(name)); + assert_eq!(m.root.content_type.text("name").as_deref(), Some(name)); + } +} + +#[test] +fn unsafe_media_types_and_nameless_files() { + let part = |media_type: &str, name: &str| Entity::file(&File { media_type, name, ..File::default() }); + assert_eq!(field(&part("multipart/mixed; boundary=x", "a.bin"), "Content-Type"), Some("application/octet-stream; name=\"a.bin\"")); + assert_eq!(field(&part("", ""), "Content-Type"), Some("application/octet-stream")); + assert_eq!(field(&part("application/pdf", ""), "Content-Disposition"), Some("attachment")); + assert_eq!(field(&part("image/png", "ünï.png"), "Content-Type"), Some("image/png; name=\"=?UTF-8?B?w7xuw68ucG5n?=\"")); + assert_eq!(field(&part("image/png", ""), "Content-ID"), None); + assert_eq!(safe_media_type(" Application/PDF "), "application/pdf"); + assert_eq!(safe_media_type("image/svg+xml"), "image/svg+xml"); + for bad in ["", "pdf", "a/b/c", "multipart/mixed", "MULTIPART/related", "text/plain; charset=x", "a b/c", "text/pl\r\nain", "/x", "x/"] { + assert_eq!(safe_media_type(bad), "application/octet-stream", "{bad}"); + } +} + +#[test] +fn attached_messages_travel_as_they_are_when_they_can() { + let part = |data: &[u8]| { + let p = Entity::file(&File { media_type: "message/rfc822", name: "m.eml", data, ..File::default() }); + (field(&p, "Content-Transfer-Encoding").unwrap_or("").to_string(), p.content) + }; + assert_eq!(part(b"Subject: x\r\n\r\nbody\r\n"), ("7bit".into(), b"Subject: x\r\n\r\nbody\r\n".to_vec())); + assert_eq!(part(b"").0, "7bit"); + assert_eq!(part(&[b'x'; 998]).0, "7bit"); + // Anything that could not travel as it is: base64, which loses nothing. + assert_eq!(part("Subject: x\r\n\r\ncafé".as_bytes()), ("base64".into(), b"U3ViamVjdDogeA0KDQpjYWbDqQ==".to_vec())); + assert_eq!(part(b"Subject: x\n\nbare lf").0, "base64"); + assert_eq!(part(b"bare\rcr").0, "base64"); + assert_eq!(part(b"nul\0").0, "base64"); + assert_eq!(part(&[b'x'; 999]).0, "base64"); + // Other files are always base64, however plain they are. + let p = Entity::file(&File { media_type: "text/plain", name: "a.txt", data: b"plain", ..File::default() }); + assert_eq!(field(&p, "Content-Transfer-Encoding"), Some("base64")); +} + +#[test] +fn everything_written_is_seven_bit() { + let mut boundaries = Boundaries::default(); + let parts = [ + Entity::text("plain", "Grüße\n"), + Entity::text("html", "

Grüße

"), + Entity::file(&File { media_type: "message/rfc822", name: "ü.eml", data: "Subject: é\r\n\r\nü".as_bytes(), ..File::default() }), + Entity::file(&File { media_type: "application/octet-stream", name: "b.bin", data: &[0, 128, 255], content_id: "ïd", ..File::default() }), + ]; + let root = Entity::multipart("mixed", &[("type", "tëxt")], &parts, &mut boundaries); + let subject = encode_unstructured("Subject", "Grüße 🙋"); + let from = crate::format_address_list(&[crate::Mailbox { name: "Ünï Cödé".into(), email: "ada@example.org".into() }]); + let note = seven_bit_value("X-Note", "ñ"); + let out = write_message(&[("From", &from), ("Subject", &subject), ("X-Note", ¬e)], &root); + // What a caller writes into a parameter or an id as it is, is the caller's to encode. + let out = text(out).replace("tëxt", "").replace("ïd", ""); + assert!(out.is_ascii(), "{out}"); +} + +#[test] +fn boundaries_are_deterministic_and_avoid_the_content() { + let write = |body: &str| { + let mut boundaries = Boundaries::default(); + let root = Entity::multipart("alternative", &[], &[Entity::text("plain", body), Entity::text("html", "

x

")], &mut boundaries); + text(write_message(&[], &root)) + }; + assert_eq!(write("x"), write("x")); + let first = write("x"); + let boundary = first.split("boundary=\"").nth(1).and_then(|s| s.split('"').next()).unwrap_or_default().to_string(); + assert!(boundary.starts_with("=_craft_") && boundary.len() <= 70); + // Put the boundary that text will get in the text: the writer must pick another. + let mut body = "before\n--x\nafter".to_string(); + for _ in 0..3 { + let out = write(&body); + let boundary = out.split("boundary=\"").nth(1).and_then(|s| s.split('"').next()).unwrap_or_default().to_string(); + assert_eq!(out.matches(&format!("--{boundary}")).count(), 3, "{out}"); + body = format!("before\n--{boundary}\nafter"); + } +} + +#[test] +fn a_boundary_found_in_a_part_is_passed_over() { + let mut boundaries = Boundaries::new("=_mc_"); + assert_eq!(boundaries.free(42, &[b"nothing here".to_vec()]), "=_mc_000000000000002a_0"); + assert_eq!(boundaries.counter, 1); + // The parts hold the first two candidates, one of them in the second part. + let parts = [b"text --=_mc_000000000000002a_1 text".to_vec(), b"=_mc_000000000000002a_2".to_vec()]; + assert_eq!(boundaries.free(42, &parts), "=_mc_000000000000002a_3"); + assert_eq!(boundaries.counter, 4); + // The counter sets apart two multiparts with the same content. + let same = [b"same".to_vec()]; + assert_ne!(boundaries.next(&same), boundaries.next(&same)); + // A prefix can't carry what would break the parameter or the delimiter line. + let hostile = Boundaries::new("a\"\r\n b;c=d\0é--").next(&[]); + assert!(hostile.starts_with("ab") && hostile.bytes().all(|b| b.is_ascii_alphanumeric() || b"=_-.".contains(&b)), "{hostile}"); + assert!(Boundaries::new(&"x".repeat(500)).next(&[]).len() <= 70); +} + +#[test] +fn substring_search() { + assert!(contains(b"hello world", b"lo w")); + assert!(contains(b"aaab", b"aab")); + assert!(contains(b"abc", b"")); + assert!(contains(b"abc", b"abc")); + assert!(!contains(b"abc", b"abcd")); + assert!(!contains(b"", b"a")); + assert!(!contains(b"ababab", b"abb")); +} + +#[test] +fn ids_lose_what_could_end_them() { + assert_eq!(clean_id("id@example.org>\r\nX-Evil: 1\r\n Vec { + let mut boundaries = Boundaries::default(); + let picture = Entity::file(&File { media_type: "image/png", name: "lögo (1).png", inline: true, content_id: "logo@example.org", data: PNG }); + let mut page = Entity::text("html", "

Grüße

\n"); + page.fields.push(("Content-Location".into(), "https://example.test/reports/page.html".into())); + let rich = Entity::multipart("related", &[("type", "text/html"), ("start", "")], &[page, picture], &mut boundaries); + let forms = Entity::multipart("alternative", &[], &[Entity::text("plain", "Grüße\n\nFrom here on.\n"), rich], &mut boundaries); + let inner = craft_mime::write_message(&[("Subject", "Inner")], &Entity::text("plain", "inner")); + let attached = Entity::file(&File { media_type: "message/rfc822", name: "inner.eml", data: &inner, ..File::default() }); + let root = Entity::multipart("mixed", &[], &[forms, attached], &mut boundaries); + let to = craft_mime::format_address_list(&[ + Mailbox { name: "Hopper, Grace".into(), email: "grace@example.org".into() }, + Mailbox { name: "Renée".into(), email: "renee@example.org".into() }, + ]); + let subject = craft_mime::encode_unstructured("Subject", &"Ünïcödé subject ".repeat(8)); + craft_mime::write_message( + &[("Date", "Tue, 01 Jul 2003 10:52:37 +0200"), ("To", &to), ("Subject", &subject), ("Message-ID", "")], + &root, + ) +} + +/// A hand-written MHT archive with LF line ends, a comment in its content type and RFC 2231 +/// parameters. +fn archive() -> Vec { + b"From nobody Tue Jul 1 08:52:37 2003\n\ +MIME-Version: 1.0\n\ +Content-Type: multipart/related (a \"comment\"; boundary=wrong);\n\ +\ttype=\"text/html\"; start=\"\"; boundary=\"----=_B\"\n\ +\n\ +preamble\n\ +------=_B\n\ +Content-ID: \n\ +Content-Location: file:///C:/page.htm\n\ +Content-Transfer-Encoding: quoted-printable\n\ +Content-Type: text/html; charset=\"windows-1250\"\n\ +\n\ +

=E8lanak, jedan =\n\ +red.

\n\ +------=_B\n\ +Content-Location: file:///C:/page_files/dot.png\n\ +Content-Transfer-Encoding: base64\n\ +Content-Type: image/png\n\ +Content-Disposition: inline; filename*0*=utf-8''d%C3%B6;\n\ +\x20filename*1=\"t.png\"\n\ +\n\ +iVBORw0KGgoAAAANSUhEUg==\n\ +------=_B--\n\ +epilogue\n" + .to_vec() +} + +/// Everything a caller can ask of a message, and what must hold whatever the bytes were. +fn exercise(bytes: &[u8], limits: &Limits) -> Message { + let message = craft_mime::parse(bytes, limits); + let mut stack = vec![(&message.root, 0usize)]; + let (mut parts, mut decoded) = (0usize, 0usize); + while let Some((part, depth)) = stack.pop() { + parts += 1; + decoded += part.data().len(); + assert!(depth <= limits.max_depth.min(Limits::HARD_MAX_DEPTH)); + assert!(part.headers.0.len() <= limits.max_header_fields); + assert!( + part.headers.0.iter().all(|f| craft_mime::is_field_name(f.name.as_bytes()) && f.value.len() <= limits.max_header_value.saturating_mul(3)) + ); + let _ = (part.text(), part.file_name(), part.content_id(), part.content_location(), part.disposition(), part.transfer_encoding()); + for field in &part.headers.0 { + let _ = (craft_mime::decode_words(&field.value), craft_mime::parse_address_list(&field.value), craft_mime::parse_date(&field.value)); + } + stack.extend(part.parts().iter().map(|p| (p, depth + 1))); + } + assert!(parts <= limits.max_parts.max(1)); + assert!(decoded <= limits.max_decoded.min(bytes.len())); + let resources = craft_mime::Resources::of(&message.root); + let _ = ( + craft_mime::start_part(&message.root), + resources.find("page_files/dot.png", "file:///C:/page.htm"), + resources.find("cid:logo@example.org", ""), + ); + assert_eq!(craft_mime::parse(bytes, limits), message, "reading is deterministic"); + message +} + +const SMALL: Limits = Limits { + max_input: 700, + max_header_fields: 3, + max_header_value: 40, + max_header_block: 120, + max_depth: 1, + max_parts: 3, + max_decoded: 50, + ..Limits::DEFAULT +}; + +#[test] +fn the_samples_read_whole() { + let m = exercise(&sample(), &Limits::default()); + assert!(!m.truncated); + let types: Vec<&str> = m.root.leaves().iter().map(|p| p.media_type()).collect(); + assert_eq!(types, ["text/plain", "text/html", "image/png", "message/rfc822"]); + assert_eq!(m.root.leaves()[2].file_name().as_deref(), Some("lögo (1).png")); + assert_eq!(m.root.leaves()[2].data(), PNG); + + let m = exercise(&archive(), &Limits::default()); + assert!(!m.truncated); + assert_eq!( + craft_mime::start_part(&m.root).map(Part::text).as_deref(), + Some("

članak, jedan red.

") + ); + let picture = craft_mime::Resources::of(&m.root).find("page_files/dot.png", "file:///C:/page.htm"); + assert_eq!(picture.and_then(Part::file_name).as_deref(), Some("döt.png")); + assert!(picture.is_some_and(|p| p.data().starts_with(b"\x89PNG"))); +} + +#[test] +fn every_truncation_reads() { + for bytes in [sample(), archive()] { + for end in 0..=bytes.len() { + let cut = bytes.get(..end).unwrap_or_default(); + exercise(cut, &Limits::default()); + exercise(cut, &SMALL); + // An mbox reader takes any prefix too. + assert!(craft_mime::mbox_messages(cut).map(|m| m.len()).sum::() <= cut.len()); + } + } +} + +#[test] +fn mutations_read() { + // A small xorshift generator: the same mutations every run. + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + const INTERESTING: &[u8] = b"\0\r\n\t \"\\()<>;:=?-_%*'/@,.\x7f\x80\xff"; + for original in [sample(), archive()] { + for round in 0..4000u32 { + let mut bytes = original.clone(); + for _ in 0..=(round % 4) { + let at = (next() as usize) % bytes.len().max(1); + let pick = INTERESTING.get((next() as usize) % INTERESTING.len()).copied().unwrap_or(0); + match next() % 5 { + 0 => { + if let Some(b) = bytes.get_mut(at) { + *b = pick; + } + } + 1 => bytes.insert(at.min(bytes.len()), pick), + 2 if !bytes.is_empty() => { + bytes.remove(at); + } + // Repeat a slice of the message somewhere else (boundaries, header lines). + 3 => { + let len = (next() as usize) % 80; + let slice = original.get(at..at.saturating_add(len).min(original.len())).unwrap_or_default().to_vec(); + let to = (next() as usize) % bytes.len().max(1); + bytes.splice(to..to, slice); + } + _ => { + if let Some(b) = bytes.get_mut(at) { + *b ^= 1 << (next() % 8); + } + } + } + } + exercise(&bytes, &Limits::default()); + exercise(&bytes, &SMALL); + } + } +} + +#[test] +fn large_hostile_shapes_stay_linear() { + // Each of these is a megabyte or so; quadratic work on any would not finish. + let n = 200_000; + let shapes: Vec> = vec![ + b"X: ".iter().copied().chain(b"=?".repeat(n)).collect(), + format!("Content-Type: multipart/mixed; boundary=b\n\n{}", "--b\n".repeat(n)).into_bytes(), + format!("Content-Type: multipart/mixed; boundary=b\n\n{}", "--b--x\n--bb\n".repeat(n / 2)).into_bytes(), + format!("Content-Type: text/plain{}\n\nx", "; a*0*=''%41".repeat(n / 8)).into_bytes(), + format!("Content-Type: text/plain; a=\"b\"{}\n\nx", "(".repeat(n)).into_bytes(), + format!("To: {}\n\nx", "(<\"\\".repeat(n / 2)).into_bytes(), + format!("A: 1\n{}\nbody", " fold\n".repeat(n)).into_bytes(), + "A: 1\n".repeat(n).into_bytes(), + format!("Content-Transfer-Encoding: quoted-printable\n\n{}", "= ".repeat(n)).into_bytes(), + format!("Content-Transfer-Encoding: base64\n\n{}", "=A".repeat(n)).into_bytes(), + "From \n\n".repeat(n).into_bytes(), + ]; + for bytes in &shapes { + exercise(bytes, &Limits::default()); + let _ = craft_mime::mbox_messages(bytes).count(); + } +} + +/// The header block of a written message, as lines; no line of it may start a field the caller +/// didn't write. +fn header_lines(bytes: &[u8]) -> Vec { + let text = String::from_utf8_lossy(bytes).into_owned(); + assert!(!text.contains('\0'), "{text:?}"); + // Every CR is followed by LF and every LF follows a CR: no bare line break got through. + assert_eq!(text.matches('\r').count(), text.matches("\r\n").count()); + assert_eq!(text.matches('\n').count(), text.matches("\r\n").count()); + text.split("\r\n\r\n").next().unwrap_or_default().split("\r\n").map(str::to_string).collect() +} + +const EVIL: &[&str] = &[ + "x\r\nBcc: eve@example.org", + "x\nX-Evil: 1", + "x\rX-Evil: 2", + "x\r\n\r\n", + "x\0\r\n X-Evil: 3", + "x\"\r\nX-Evil: 4", + "x>\r\nX-Evil: 5\r\n", craft_mime::clean_id(evil)); + for subject in [evil.to_string(), craft_mime::encode_unstructured("Subject", evil), craft_mime::encode_words(evil)] { + let note = craft_mime::seven_bit_value("X-Note", evil); + let headers = [("Subject", subject.as_str()), ("X-Note", note.as_str()), ("To", to.as_str()), ("Message-ID", id.as_str())]; + let out = craft_mime::write_message(&headers, &Entity::text("plain", "body")); + assert_no_injection(&out, ALLOWED); + let m = craft_mime::parse(&out, &Limits::default()); + assert_eq!(m.root.data(), b"body\r\n", "the body starts where the writer put it"); + assert_eq!(craft_mime::parse_address_list(m.root.headers.get("to").unwrap_or_default()).len(), 1, "{to}"); + } + } +} + +#[test] +fn header_names_cannot_inject_fields() { + for name in ["X-A\r\nBcc", "X-A: 1\r\nBcc", "X A", "", "X-A\0", "Bcc:", "X-É"] { + let out = craft_mime::write_message(&[(name, "eve@example.org")], &Entity::text("plain", "body")); + assert_no_injection(&out, &["MIME-Version", "Content-Type", "Content-Transfer-Encoding"]); + let mut entity = Entity::text("plain", "body"); + entity.fields.push((name.to_string(), "eve@example.org".to_string())); + assert_no_injection(&craft_mime::write_message(&[], &entity), &["MIME-Version", "Content-Type", "Content-Transfer-Encoding"]); + } +} + +#[test] +fn part_values_cannot_inject_fields_or_parts() { + const ALLOWED: &[&str] = &["MIME-Version", "Content-Type", "Content-Transfer-Encoding", "Content-Disposition", "Content-ID", "Content-Location"]; + for evil in EVIL { + let mut file = Entity::file(&File { media_type: evil, name: evil, inline: false, content_id: evil, data: b"data" }); + file.fields.push(("Content-Location".into(), evil.to_string())); + assert_no_injection(&craft_mime::write_message(&[], &file), ALLOWED); + assert_no_injection(&craft_mime::write_message(&[], &Entity::text(evil, "x")), ALLOWED); + + // In a multipart: the subtype, a parameter's name and value, the boundary prefix, and + // a body that holds what looks like a delimiter. + let mut boundaries = Boundaries::new(evil); + let text = Entity::text("plain", &format!("--{evil}\n--=_craft_\n{evil}")); + let root = Entity::multipart(evil, &[("type", evil), (evil, "x"), ("boundary", "mine")], &[text, file], &mut boundaries); + let out = craft_mime::write_message(&[], &root); + assert_no_injection(&out, ALLOWED); + let m = craft_mime::parse(&out, &Limits::default()); + assert_eq!(m.root.parts().len(), 2, "{}", String::from_utf8_lossy(&out)); + assert_eq!(m.root.parts()[1].data(), b"data"); + for part in m.root.parts() { + assert!(part.headers.0.iter().all(|f| ALLOWED.iter().any(|a| a.eq_ignore_ascii_case(&f.name))), "{:?}", part.headers); + } + } +} + +#[test] +fn mbox_values_cannot_inject_messages() { + let date = craft_mime::parse_date("Tue, 1 Jul 2003 08:52:37 +0000").unwrap_or_default(); + for evil in EVIL { + let mut out = Vec::new(); + let sender = format!("a@example.org{evil}\nFrom eve@example.org Tue Jul 1 08:52:37 2003"); + let message = format!("Subject: x\r\n\r\nFrom the top\r\n\r\nFrom eve@example.org Tue Jul 1 08:52:37 2003\r\n{evil}\r\n"); + craft_mime::write_mbox_message(&mut out, &sender, &date, message.as_bytes()); + let back: Vec> = craft_mime::mbox_messages(&out).collect(); + assert_eq!(back, [message.into_bytes()]); + } +}