diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index a6493818..7b32c6f1 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -12,7 +12,7 @@ author, source and license. Library code and original assets are under the repos | `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 | | `crates/text/assets/hyphen/en-us.dic` | Grady Ward (Moby Hyphenator II), converted by DesignCraft | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204) | Public domain | | `crates/text/assets/hyphen/en-us.pat` | DesignCraft contributors, trained from the Moby Hyphenator II list | `crates/text/examples/hyphgen.rs` | CC0-1.0 | -| Patterns embedded through `hypher` (optional `craft-text` language features) | hyph-utf8 authors, per language | [tex-hyphen](https://github.com/hyphenation/tex-hyphen), as packaged in [hypher](https://github.com/typst/hypher/tree/main/patterns) | Per file header: LPPL, MIT, BSD-3-Clause, Unlicense or a permit-any-use notice (or a choice including one); Slovak (`hyph-sk.tex`, feature `slovak-gpl` only) is GPL-2.0-or-later; Spanish (`hyph-es.tex` 5.0, Javier Bezos, CervanTeX) is MIT | +| Patterns embedded through `hypher` (optional `craft-text` language features) | hyph-utf8 authors, per language | [tex-hyphen](https://github.com/hyphenation/tex-hyphen), as packaged in [hypher](https://github.com/typst/hypher/tree/main/patterns) | Per file header: LPPL, MIT, BSD-3-Clause, Unlicense or a permit-any-use notice (or a choice including one); Slovak (`hyph-sk.tex`, embedded whenever `hypher/slovak` is on: feature `slovak-gpl`, or another dependency through feature unification) is GPL-2.0-or-later; Spanish (`hyph-es.tex` 5.0, Javier Bezos, CervanTeX) is MIT | | Word-break dictionaries and LSTM models embedded through `icu_segmenter_data` (optional `craft-text` features `thai`, `lao`, `khmer`, `burmese`) | Unicode, Inc. and ICU4X contributors | [ICU4X](https://github.com/unicode-org/icu4x) | Unicode-3.0 | Original assets are covered by the repository licence. diff --git a/NOTICE b/NOTICE index 699ee37c..9d0e267a 100644 --- a/NOTICE +++ b/NOTICE @@ -29,9 +29,11 @@ Hyphenation patterns for other languages, embedded through the hypher crate only Unlicense, a notice permitting any use, or a choice of licenses that includes one of these. An application that enables a language ships that file's copyright and license notice. The exception is Slovak (hyph-sk.tex, Copyright (C) 1992 Jana Chlebíková), under the GNU General - Public License, version 2 or later (SPDX: GPL-2.0-or-later). It is compiled in only with the + Public License, version 2 or later (SPDX: GPL-2.0-or-later). craft-text asks for it through the feature "slovak-gpl", which "all-languages" does not include; an application that enables it - ships GPL-licensed data. The Spanish patterns (feature "spanish") are hyph-es.tex 5.0. + ships GPL-licensed data. Cargo unifies features, so another dependency that enables hypher's + "slovak" feature (hypher's default features do) embeds the same data; a build that must exclude + it checks its dependency graph. The Spanish patterns (feature "spanish") are hyph-es.tex 5.0. Copyright (C) 1993, 1997 Javier Bezos, 2001-2019 Javier Bezos, CervanTeX. MIT License: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without diff --git a/crates/text/Cargo.toml b/crates/text/Cargo.toml index b9d54a48..ae32a115 100644 --- a/crates/text/Cargo.toml +++ b/crates/text/Cargo.toml @@ -138,7 +138,9 @@ russian = ["hyphen", "hypher/russian"] sanskrit = ["hyphen", "hypher/sanskrit"] serbian = ["hyphen", "hypher/serbian"] # Slovak patterns are under the GPL, version 2 or later (hyph-sk.tex), unlike every other language -# here, so `all-languages` leaves them out. A build that enables this ships GPL-licensed data. +# here, so `all-languages` leaves them out. A build that enables this ships GPL-licensed data. Any +# other dependency that enables `hypher/slovak` (hypher's defaults do) embeds them too, through +# feature unification. slovak-gpl = ["hyphen", "hypher/slovak"] slovenian = ["hyphen", "hypher/slovenian"] spanish = ["hyphen", "hypher/spanish"] diff --git a/crates/text/README.md b/crates/text/README.md index 2694c9b5..585945b7 100644 --- a/crates/text/README.md +++ b/crates/text/README.md @@ -42,8 +42,11 @@ Each has a cargo feature named as in hypher (`spanish`, `german`, `norwegian`, enables 46 of them. Patterns for languages whose feature is off are not compiled in. Slovak's patterns are under the GPL, version 2 or later; the patterns of every other language are -under a permissive license. Their feature is `slovak-gpl`, and `all-languages` leaves it out: a build that -enables it ships GPL-licensed data. +under a permissive license. craft-text asks for Slovak through the feature `slovak-gpl`, which +`all-languages` leaves out; a build that enables it ships GPL-licensed data. Cargo unifies features, +so another dependency that enables hypher's `slovak` feature (hypher's default features do) embeds +the same data. A build that must exclude it checks its dependency graph +(`cargo tree -e features -i hypher`). ```rust use craft_text::hyphen::{Lang, Limits, hyphen_points_in}; diff --git a/crates/text/src/hyphen/mod.rs b/crates/text/src/hyphen/mod.rs index f098bdbb..deac28a2 100644 --- a/crates/text/src/hyphen/mod.rs +++ b/crates/text/src/hyphen/mod.rs @@ -161,16 +161,30 @@ impl Lang { } /// The language of a locale code split into lowercase `words`: a language (2 or 3 letters), then -/// optionally a region (2 letters or 3 digits) and a reform year (4 digits). +/// optionally a script (4 letters), a region (2 letters or 3 digits) and a reform year (4 digits), +/// in that order. fn locale_language<'a>(words: &[&'a str]) -> Option<&'a str> { - let letters = |w: &str, n: std::ops::RangeInclusive| n.contains(&w.len()) && w.bytes().all(|b| b.is_ascii_lowercase()); - let digits = |w: &str, n: usize| w.len() == n && w.bytes().all(|b| b.is_ascii_digit()); - match *words { - [lang] if letters(lang, 2..=3) => Some(lang), - [lang, region] if letters(lang, 2..=3) && (letters(region, 2..=2) || digits(region, 3)) => Some(lang), - [lang, region, year] if letters(lang, 2..=3) && (letters(region, 2..=2) || digits(region, 3)) && digits(year, 4) => Some(lang), - _ => None, + fn letters(w: &str, n: std::ops::RangeInclusive) -> bool { + n.contains(&w.len()) && w.bytes().all(|b| b.is_ascii_lowercase()) } + fn digits(w: &str, n: usize) -> bool { + w.len() == n && w.bytes().all(|b| b.is_ascii_digit()) + } + let (&lang, mut rest) = words.split_first()?; + if !letters(lang, 2..=3) { + return None; + } + let script = |w: &str| letters(w, 4..=4); + let region = |w: &str| letters(w, 2..=2) || digits(w, 3); + let year = |w: &str| digits(w, 4); + for subtag in [script as fn(&str) -> bool, region, year] { + if let Some((&w, tail)) = rest.split_first() + && subtag(w) + { + rest = tail; + } + } + rest.is_empty().then_some(lang) } /// Where a word's break points came from. diff --git a/crates/text/src/hyphen/tests.rs b/crates/text/src/hyphen/tests.rs index be36e4de..8c1f1af8 100644 --- a/crates/text/src/hyphen/tests.rs +++ b/crates/text/src/hyphen/tests.rs @@ -70,7 +70,7 @@ fn limits_are_respected() { #[test] fn language_names_and_codes() { - for name in ["English", "English: USA", "english", "$ID/English: UK", "en", "en_US", "en-GB", "EN"] { + for name in ["English", "English: USA", "english", "$ID/English: UK", "en", "en_US", "en-GB", "EN", "en-Latn-US"] { assert_eq!(Lang::for_language(name), Some(Lang::English), "{name}"); } // Each language resolves by name and by code exactly when its feature is on. @@ -88,6 +88,10 @@ fn language_names_and_codes() { format!("{code}_{upper}"), format!("{code}-419"), format!("{code}_{upper}_2006"), + format!("{code}-Latn"), + format!("{code}-Latn-{upper}"), + format!("{code}-1996"), + format!("{code}-Latn-1996"), ] { assert_eq!(Lang::for_language(&spelling), want, "{spelling}"); }