mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 19:14:26 +00:00
craft-ai: shared AI providers — BYO endpoints and one-click local models (#1181)
This commit is contained in:
Generated
+667
-18
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,7 @@ craft-plist = { path = "crates/plist" }
|
||||
craft-platform = { path = "crates/platform" }
|
||||
craft-aep = { path = "crates/aep" }
|
||||
craft-affinity = { path = "crates/affinity" }
|
||||
craft-ai = { path = "crates/ai" }
|
||||
craft-ai-core = { path = "crates/ai-core" }
|
||||
craft-anicraft-format = { path = "crates/anicraft-format" }
|
||||
craft-pdf = { path = "crates/pdf", version = "=0.1.0" }
|
||||
|
||||
@@ -381,6 +381,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
nearest and bilinear sampling that repeats or clamps, resizing and display conversion. Sizes are
|
||||
bounded by `Limits`. From MaterialCraft.
|
||||
- [`craft-ui`](crates/ui/README.md): SVG catalogs, icon buttons, tabs, pane controls, transactional docking, print settings and theme helpers for egui frontends;
|
||||
- [`craft-ai`](crates/ai/README.md) *(spec)*: AI provider abstraction — BYO API keys (OpenAI-compatible) and one-click curated local models with capability probing, resumable checksummed downloads, and document segmentation that protects markup while translating prose;
|
||||
assets, themes, localization and command handling stay in each app. Requires Rust 1.95 or later.
|
||||
- [`craft-vcard`](crates/vcard/README.md): vCard contacts, from MailCraft: vCard 2.1, 3.0 and 4.0 read
|
||||
(legacy character sets, quoted-printable, labels and Apple label groups, `KEY` certificates and
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "craft-ai"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "AI provider abstraction for the Craft suite — BYO API keys and one-click local models"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
dirs = "7"
|
||||
sysinfo = "0.39"
|
||||
ureq = { version = "3", features = ["json"] }
|
||||
sha2 = "0.11"
|
||||
candle-core = { version = "0.9", optional = true }
|
||||
candle-nn = { version = "0.9", optional = true }
|
||||
candle-transformers = { version = "0.9", optional = true }
|
||||
tokenizers = { version = "0.22", optional = true, default-features = false, features = ["onig"] }
|
||||
|
||||
[features]
|
||||
# Local GGUF inference on candle. Heavy dependency tree; apps opt in.
|
||||
local = ["dep:candle-core", "dep:candle-nn", "dep:candle-transformers", "dep:tokenizers"]
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,145 @@
|
||||
# craft-ai — AI providers for the Craft suite
|
||||
|
||||
Shared AI abstraction for every Craft app: translation, rewriting, summaries,
|
||||
vision captions — one crate, one settings store, one model cache.
|
||||
|
||||
Status: **spec** — this file is the contract; implementation follows.
|
||||
|
||||
## The feature this powers
|
||||
|
||||
A per-app **AI tab** offering document-wide operations — the flagship being
|
||||
"translate this whole document" — backed by either:
|
||||
|
||||
1. **BYOT** — the user's own API token against an OpenAI-compatible endpoint
|
||||
(OpenAI, Azure, Ollama, LM Studio, any compatible proxy), or
|
||||
2. **Local model** — a one-click download of a curated open model, chosen for
|
||||
the machine it's running on.
|
||||
|
||||
The user experience is deliberately *informed*: every step reports what is
|
||||
happening, why this model was picked, what it will cost in disk/RAM, and that
|
||||
nothing leaves the machine in local mode.
|
||||
|
||||
## Modules
|
||||
|
||||
### `probe` — capability detection
|
||||
|
||||
Runs once, caches the result:
|
||||
|
||||
| Signal | Source |
|
||||
| --- | --- |
|
||||
| Total/free RAM | `sysinfo` or `std::alloc`-safe OS calls |
|
||||
| CPU threads + features | `std::thread`, `is_x86_feature_detected!` (AVX2 gates Q4 speed) |
|
||||
| GPU + VRAM | `wgpu` adapter info (apps already link wgpu — reuse, don't re-probe) |
|
||||
| Free disk at the model cache dir | `fs2`/`std::fs` |
|
||||
| Network reachability | optional HEAD to the model host, skipped offline |
|
||||
|
||||
Output: `MachineCaps { ram_gb, vram_gb, threads, avx2, disk_free_gb }`.
|
||||
|
||||
### `registry` — the curated model list
|
||||
|
||||
A small hand-maintained table (Rust consts, not runtime-fetched — supply
|
||||
chain stays auditable). Each entry: `id`, display name, HF repo + file,
|
||||
`sha256`, size, min RAM, recommended RAM, license id, quality tier,
|
||||
languages covered.
|
||||
|
||||
Curation rule — **permissive licenses only** (the suite is MIT OR Apache-2.0
|
||||
and commercial users run it):
|
||||
|
||||
- ✅ Qwen2.5-0.5B / 1.5B / 3B Instruct (Apache-2.0)
|
||||
- ✅ SmolLM2-1.7B (Apache-2.0)
|
||||
- ✅ Gemma-3 (Gemma Terms — allowed, flag in UI)
|
||||
- ✅ Helsinki-NLP opus-mt-* per-language pairs (CC-BY/MIT)
|
||||
- ❌ NLLB-200, M2M-100 — CC-BY-NC, excluded
|
||||
- ❌ Llama-* — custom license acceptable? default: exclude until reviewed
|
||||
|
||||
`registry::suggest(&caps)` picks the best fitting model: prefers 3B when
|
||||
VRAM ≥ 6 GB or RAM ≥ 16 GB, falls to 1.5B at 8 GB, 0.5B below. Always shows
|
||||
alternatives — the suggestion is a default, not a cage.
|
||||
|
||||
### `provider` — inference backends
|
||||
|
||||
```rust
|
||||
trait Provider {
|
||||
fn name(&self) -> &str;
|
||||
fn complete(&self, prompt: &str, opts: &GenOpts) -> Result<String, AiError>;
|
||||
fn test(&self) -> Result<ProviderInfo, AiError>;
|
||||
}
|
||||
```
|
||||
|
||||
- `OpenAiCompat` — POST `/v1/chat/completions`, configurable endpoint+token+
|
||||
model. Covers OpenAI, Ollama (`http://localhost:11434/v1`), LM Studio,
|
||||
Groq, OpenRouter… Token stored via the OS keyring (`keyring` crate) or a
|
||||
0600-permissioned config file — never in project files, never logged.
|
||||
- `LocalGguf` — GGUF inference. First implementation target: **candle**
|
||||
(pure Rust, Apache/MIT — keeps the no-C++ toolchain rule). `llama-cpp-2`
|
||||
stays a possible perf upgrade later, gated behind a feature flag.
|
||||
|
||||
### `fetch` — one-click download
|
||||
|
||||
- Resumable HTTP GET (range requests), streamed to `*.part` then renamed —
|
||||
a half-downloaded GGUF can never be mistaken for a valid one.
|
||||
- SHA-256 checked against the registry entry; mismatch = hard fail, delete.
|
||||
- Progress events (`Download { bytes_done, bytes_total, bytes_per_sec }`)
|
||||
feed the UI's live status line.
|
||||
- Cache dir is **shared across the suite**:
|
||||
`%LOCALAPPDATA%\artcraft\models` (Windows), `~/Library/Application
|
||||
Support/artcraft/models` (macOS), `~/.cache/artcraft/models` (Linux).
|
||||
|
||||
### `translate` — document segmentation
|
||||
|
||||
The model is the easy part; the hard part is *what not to touch*. The
|
||||
translator walks the document as segments, not raw text:
|
||||
|
||||
- **Protected**: LaTeX commands/envs/math (`\cite{…}`, `$…$`), code fences,
|
||||
placeholders (`{count}`), URLs, citation keys, markup tags.
|
||||
- **Translated**: prose runs, captions, headings, table cells.
|
||||
- Per-app adapters declare the grammar: TexCraft protects `\command{…}` and
|
||||
math mode; WordCraft protects field codes; MatCraft protects code.
|
||||
- Batch-friendly: segments → prompt with numbered items → parse numbered
|
||||
output; verify count matches, retry split halves on mismatch.
|
||||
- Deterministic options: temperature 0-ish, glossary hook (`HashMap<term,
|
||||
translation>`) applied post-hoc.
|
||||
|
||||
## The AI tab (per app, `ui-egui/src/ai.rs`)
|
||||
|
||||
First-open flow — the "smooth informed" sequence:
|
||||
|
||||
1. Two cards: **Use an API provider** / **Run on this PC**.
|
||||
2. BYOT card: endpoint, token, model fields + **Test connection** → green
|
||||
check with model list echo.
|
||||
3. Local card: **Detect this PC** → probe runs →
|
||||
`Suggested: Qwen2.5-1.5B-Instruct (Q4_K_M, 1.1 GB) — fits your 16 GB RAM;
|
||||
est. ~8 tok/s` with an "Other models ▸" disclosure showing the registry.
|
||||
4. **Download** → progress bar with MB/%/speed → `Verifying checksum…` →
|
||||
`Loading…` → `Ready — local, offline, nothing leaves this PC`.
|
||||
5. Then the operations: "Translate document to [language ▾]", "Summarize
|
||||
selection", custom prompt box. Status line narrates each step;
|
||||
expandable activity log for the curious.
|
||||
6. Language detected vs document language → gentle offer chip:
|
||||
`This document is French — translate to English?`
|
||||
|
||||
## Non-negotiables
|
||||
|
||||
- **Honesty**: the UI always says which provider is active, what model,
|
||||
and "cloud — your document text is sent to the endpoint" vs "local —
|
||||
nothing leaves this PC". No silent cloud calls; AI is strictly opt-in.
|
||||
- **No crash**: model load failures, corrupt GGUFs, OOM — all `Result`,
|
||||
all surfaced as messages.
|
||||
- **Rust only**: candle over llama.cpp bindings preserves the wasm story
|
||||
(local inference on wasm is future-possible with `candle` + WGPU backend).
|
||||
- **Licensing**: registry carries SPDX ids; the UI shows them; CC-BY-NC and
|
||||
unreviewed custom licenses are excluded by policy.
|
||||
- **Everything is a command**: `ai.translate_doc`, `ai.status`,
|
||||
`ai.models.list`, `ai.model.download`, `ai.provider.test` — reachable
|
||||
from the app's control channel and MCP, same as every Craft convention.
|
||||
|
||||
## Milestones
|
||||
|
||||
| # | Deliverable | Done when |
|
||||
| --- | --- | --- |
|
||||
| A1 | `probe` + `registry` + `fetch` | CLI-free test: probe reports real caps; `suggest` picks sanely per fixture; a tiny model downloads + sha256 verifies |
|
||||
| A2 | `OpenAiCompat` provider | `translate` a string through Ollama/OpenAI with mocked HTTP tests |
|
||||
| A3 | `LocalGguf` on candle | Qwen2.5-0.5B loads and answers on CPU in a test (model fixture fetched in-test, skipped without network) |
|
||||
| A4 | `translate` segmentation | LaTeX/code/placeholder round-trip tests — protected spans byte-identical |
|
||||
| A5 | AI tab in one pilot app (suggest TexCraft or WordCraft) | download → translate document end-to-end, live status UX |
|
||||
| A6 | Commands + MCP surface | `ai.*` commands reachable via control channel |
|
||||
@@ -0,0 +1,52 @@
|
||||
//! The crate's error type: everything fails with `Result`, never a panic.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
/// Every failure a caller can act on: the variant says what went wrong,
|
||||
/// the message says why.
|
||||
#[derive(Debug)]
|
||||
pub enum AiError {
|
||||
/// The HTTP layer failed (DNS, TLS, status code, body).
|
||||
Http(String),
|
||||
/// A downloaded file's SHA-256 did not match the expected digest.
|
||||
Checksum { expected: String, got: String },
|
||||
/// The local disk refused us (permissions, full, vanished mid-write).
|
||||
Io(std::io::Error),
|
||||
/// The provider returned a response we could not parse.
|
||||
BadResponse(String),
|
||||
/// The provider rejected the request (bad token, quota, model name).
|
||||
Rejected(String),
|
||||
/// No provider is configured — the UI should route to setup.
|
||||
NotConfigured,
|
||||
/// Local model inference failed (load, OOM, bad GGUF).
|
||||
Inference(String),
|
||||
/// The operation was cancelled by the user.
|
||||
Cancelled,
|
||||
/// The same cache entry is already being downloaded by another
|
||||
/// caller — only one may hold the `.part` file at a time.
|
||||
Busy,
|
||||
}
|
||||
|
||||
impl fmt::Display for AiError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Http(m) => write!(f, "network error: {m}"),
|
||||
Self::Checksum { expected, got } => write!(f, "checksum mismatch: expected {expected}, got {got}"),
|
||||
Self::Io(e) => write!(f, "disk error: {e}"),
|
||||
Self::BadResponse(m) => write!(f, "unreadable response: {m}"),
|
||||
Self::Rejected(m) => write!(f, "provider rejected the request: {m}"),
|
||||
Self::NotConfigured => write!(f, "no AI provider configured"),
|
||||
Self::Inference(m) => write!(f, "local model error: {m}"),
|
||||
Self::Cancelled => write!(f, "cancelled"),
|
||||
Self::Busy => write!(f, "a download of this model is already in progress"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AiError {}
|
||||
|
||||
impl From<std::io::Error> for AiError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Self::Io(e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
//! One-click model download: resumable, `.part`-suffixed while in flight
|
||||
//! (a half-downloaded GGUF is never mistaken for a valid one), SHA-256
|
||||
//! verified, then atomically renamed.
|
||||
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::Digest;
|
||||
|
||||
use crate::error::AiError;
|
||||
use crate::registry::ModelEntry;
|
||||
|
||||
/// Progress + lifecycle events, fed to the UI's status line.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Event {
|
||||
/// We know the total; download is starting or resuming.
|
||||
Started { total: u64, resumed_from: u64 },
|
||||
/// Bytes on disk so far. `bytes_per_sec` is a running average.
|
||||
Progress { done: u64, total: u64, bytes_per_sec: u64 },
|
||||
/// Body finished; hashing the file before we trust it.
|
||||
Verifying,
|
||||
/// Download is complete. `verified` is false when no reference digest
|
||||
/// existed anywhere (registry table and the host's ETag) — the file is
|
||||
/// kept but the UI should say "unverified" rather than pretend.
|
||||
Done { verified: bool },
|
||||
}
|
||||
|
||||
/// A handle to abort a download: `cancel` returns false → `AiError::Cancelled`.
|
||||
pub type Cancel<'a> = &'a dyn Fn() -> bool;
|
||||
|
||||
/// ureq's `timeout_recv_body` is a *total* budget, not a stall timer, so a
|
||||
/// single fixed cap would kill slow-but-alive downloads of multi-GB models.
|
||||
/// Instead the GET's body deadline scales with the registry's declared
|
||||
/// size: a floor rate of 256 KiB/s, between 5 min and 6 h. A dead
|
||||
/// connection can therefore never block the read loop indefinitely, and a
|
||||
/// retry still resumes from the `.part` file.
|
||||
const MIN_BODY_RATE: u64 = 256 * 1024;
|
||||
const BODY_BUDGET_MIN: Duration = Duration::from_secs(5 * 60);
|
||||
const BODY_BUDGET_MAX: Duration = Duration::from_secs(6 * 60 * 60);
|
||||
|
||||
fn body_agent(remaining_bytes: u64) -> ureq::Agent {
|
||||
let budget = Duration::from_secs(remaining_bytes / MIN_BODY_RATE).clamp(BODY_BUDGET_MIN, BODY_BUDGET_MAX);
|
||||
ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(Duration::from_secs(15)))
|
||||
.timeout_send_request(Some(Duration::from_secs(30)))
|
||||
.timeout_recv_response(Some(Duration::from_secs(60)))
|
||||
.timeout_recv_body(Some(budget))
|
||||
.build()
|
||||
.into()
|
||||
}
|
||||
|
||||
/// OS-level lock guarding one cache entry against concurrent downloads;
|
||||
/// released automatically on close/process exit, so no stale lock files.
|
||||
fn lock_entry(part_path: &Path) -> Result<File, AiError> {
|
||||
let mut lock_name = part_path.as_os_str().to_os_string();
|
||||
lock_name.push(".lock");
|
||||
let lock = OpenOptions::new().write(true).create(true).truncate(false).open(PathBuf::from(lock_name))?;
|
||||
match lock.try_lock() {
|
||||
Ok(()) => Ok(lock),
|
||||
Err(std::fs::TryLockError::WouldBlock) => Err(AiError::Busy),
|
||||
Err(std::fs::TryLockError::Error(e)) => Err(AiError::Io(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Download `entry` into `cache_dir`, calling `on` with progress events.
|
||||
/// Resumes an existing `*.part` via HTTP Range. Returns the final path.
|
||||
pub fn download(entry: &ModelEntry, cache_dir: &Path, on: impl Fn(&Event), cancel: Option<Cancel<'_>>) -> Result<PathBuf, AiError> {
|
||||
if cancel.is_some_and(|c| !c()) {
|
||||
return Err(AiError::Cancelled);
|
||||
}
|
||||
fs::create_dir_all(cache_dir)?;
|
||||
let final_path = cache_dir.join(entry.file);
|
||||
if final_path.is_file() {
|
||||
// Already on disk — verify once, reuse if good.
|
||||
on(&Event::Verifying);
|
||||
if let Some(expected) = expected_sha256(entry)? {
|
||||
let got = sha256_file(&final_path)?;
|
||||
if got.eq_ignore_ascii_case(&expected) {
|
||||
on(&Event::Done { verified: true });
|
||||
return Ok(final_path);
|
||||
}
|
||||
return Err(AiError::Checksum { expected, got });
|
||||
}
|
||||
on(&Event::Done { verified: false });
|
||||
return Ok(final_path);
|
||||
}
|
||||
|
||||
let part_path = cache_dir.join(format!("{}.part", entry.file));
|
||||
// One downloader per cache entry: two concurrent calls would read the
|
||||
// same resume offset and append the same bytes twice, corrupting the
|
||||
// `.part`. The lock is held through the final rename.
|
||||
let _entry_lock = lock_entry(&part_path)?;
|
||||
let mut resumed_from = part_path.metadata().map_or(0, |m| m.len());
|
||||
|
||||
let expected = expected_sha256(entry)?;
|
||||
|
||||
let get = |range_from: u64| {
|
||||
// Budget for the *full* size, not the remainder: a 200 that
|
||||
// ignores Range means we re-download the whole model.
|
||||
let mut req = body_agent(entry.size_mb.saturating_mul(1 << 20)).get(entry.url);
|
||||
if range_from > 0 {
|
||||
req = req.header("Range", &format!("bytes={range_from}-"));
|
||||
}
|
||||
req.call()
|
||||
};
|
||||
let mut resp = match get(resumed_from) {
|
||||
Ok(r) => r,
|
||||
// A 416 means the server found our resume offset unsatisfiable —
|
||||
// the common cause is a `.part` that already holds the complete
|
||||
// file (cancelled between the last byte and the rename). If it
|
||||
// verifies, finish the job; otherwise restart from scratch.
|
||||
Err(ureq::Error::StatusCode(416)) if resumed_from > 0 => {
|
||||
if let Some(expected) = &expected
|
||||
&& sha256_file(&part_path).is_ok_and(|got| got.eq_ignore_ascii_case(expected))
|
||||
{
|
||||
on(&Event::Verifying);
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
on(&Event::Done { verified: true });
|
||||
return Ok(final_path);
|
||||
}
|
||||
fs::remove_file(&part_path)?;
|
||||
resumed_from = 0;
|
||||
get(0).map_err(http_err)?
|
||||
}
|
||||
Err(e) => return Err(http_err(e)),
|
||||
};
|
||||
|
||||
// A 200 to a Range request means the server ignored resume — truncate.
|
||||
let honored_resume = resumed_from > 0 && resp.status() == 206;
|
||||
let total = content_length(&resp).map(|len| if honored_resume { len + resumed_from } else { len });
|
||||
|
||||
on(&Event::Started { total: total.unwrap_or(0), resumed_from: if honored_resume { resumed_from } else { 0 } });
|
||||
|
||||
let mut file = if honored_resume { OpenOptions::new().append(true).open(&part_path)? } else { File::create(&part_path)? };
|
||||
|
||||
let mut body = resp.body_mut().as_reader();
|
||||
let mut buf = vec![0u8; 256 * 1024];
|
||||
let mut done = if honored_resume { resumed_from } else { 0 };
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
if cancel.is_some_and(|c| !c()) {
|
||||
return Err(AiError::Cancelled);
|
||||
}
|
||||
let n = match body.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => n,
|
||||
Err(e) => return Err(AiError::Io(e)),
|
||||
};
|
||||
file.write_all(&buf[..n])?;
|
||||
done += n as u64;
|
||||
let secs = started.elapsed().as_secs_f64().max(0.001);
|
||||
on(&Event::Progress { done, total: total.unwrap_or(0), bytes_per_sec: (done.saturating_sub(resumed_from) as f64 / secs) as u64 });
|
||||
}
|
||||
file.flush()?;
|
||||
drop(file);
|
||||
|
||||
// A short read means the connection dropped mid-body — keep the .part
|
||||
// for resume and report it as an HTTP failure, not a success.
|
||||
if let Some(t) = total
|
||||
&& done < t
|
||||
{
|
||||
return Err(AiError::Http(format!("connection dropped at {done}/{t} bytes — retry to resume")));
|
||||
}
|
||||
|
||||
on(&Event::Verifying);
|
||||
let verified = if let Some(expected) = expected {
|
||||
let got = sha256_file(&part_path)?;
|
||||
if !got.eq_ignore_ascii_case(&expected) {
|
||||
// A corrupt download is deleted — it can never silently pass.
|
||||
let _ = fs::remove_file(&part_path);
|
||||
return Err(AiError::Checksum { expected, got });
|
||||
}
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
on(&Event::Done { verified });
|
||||
Ok(final_path)
|
||||
}
|
||||
|
||||
/// The digest we hold this file to: our registry hash if pinned, else the
|
||||
/// ETag of the resolved URL (Hugging Face serves the LFS sha256 there).
|
||||
fn expected_sha256(entry: &ModelEntry) -> Result<Option<String>, AiError> {
|
||||
if let Some(s) = entry.sha256 {
|
||||
return Ok(Some(s.to_string()));
|
||||
}
|
||||
// No redirects: HF serves the LFS sha256 in `x-linked-etag` on the
|
||||
// 302 itself; following it would return the CDN's own etag instead.
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.max_redirects(0)
|
||||
.timeout_connect(Some(Duration::from_secs(15)))
|
||||
.timeout_recv_response(Some(Duration::from_secs(30)))
|
||||
.build()
|
||||
.into();
|
||||
match agent.head(entry.url).call() {
|
||||
Ok(resp) => Ok(etag(&resp)),
|
||||
// HEAD unsupported/unreachable → proceed without verification.
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn etag(resp: &ureq::http::Response<ureq::Body>) -> Option<String> {
|
||||
// Prefer the redirect target's etag (the LFS object), fall back to
|
||||
// the response's own. Quotes stripped; weak etags ("W/…") ignored.
|
||||
for name in ["x-linked-etag", "etag"] {
|
||||
if let Some(v) = resp.headers().get(name).and_then(|v| v.to_str().ok()) {
|
||||
let v = v.trim().trim_matches('"');
|
||||
if !v.starts_with("W/") && v.len() == 64 && v.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn content_length(resp: &ureq::http::Response<ureq::Body>) -> Option<u64> {
|
||||
resp.headers()
|
||||
.get("content-length")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.trim().parse().ok())
|
||||
// A redirect's own length is meaningless; only trust it after
|
||||
// ureq followed redirects (it does — so the final body length).
|
||||
.filter(|&n: &u64| n > 0)
|
||||
}
|
||||
|
||||
fn sha256_file(path: &Path) -> Result<String, AiError> {
|
||||
let mut file = File::open(path)?;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
let mut buf = vec![0u8; 256 * 1024];
|
||||
loop {
|
||||
let n = file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(hasher.finalize().iter().map(|b| format!("{b:02x}")).collect())
|
||||
}
|
||||
|
||||
fn http_err(e: ureq::Error) -> AiError {
|
||||
match e {
|
||||
ureq::Error::StatusCode(code) => AiError::Rejected(format!("HTTP {code}")),
|
||||
other => AiError::Http(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a model would land once downloaded.
|
||||
pub fn model_path(entry: &ModelEntry, cache_dir: &Path) -> PathBuf {
|
||||
cache_dir.join(entry.file)
|
||||
}
|
||||
|
||||
/// Is `entry` already fully downloaded?
|
||||
pub fn is_downloaded(entry: &ModelEntry, cache_dir: &Path) -> bool {
|
||||
cache_dir.join(entry.file).is_file()
|
||||
}
|
||||
|
||||
/// `tokenizer.json` beside the GGUF: `<file>.tokenizer.json`.
|
||||
pub fn tokenizer_path(entry: &ModelEntry, cache_dir: &Path) -> PathBuf {
|
||||
cache_dir.join(format!("{}.tokenizer.json", entry.file))
|
||||
}
|
||||
|
||||
/// The companion tokenizer — small (a few MB), so no resume loop, but the
|
||||
/// same `.part`-then-rename rule applies.
|
||||
pub fn download_tokenizer(entry: &ModelEntry, cache_dir: &Path) -> Result<PathBuf, AiError> {
|
||||
let dest = tokenizer_path(entry, cache_dir);
|
||||
if dest.is_file() {
|
||||
return Ok(dest);
|
||||
}
|
||||
let tmp = dest.with_extension("part");
|
||||
let mut resp = body_agent(32 * 1024 * 1024).get(entry.tokenizer_url).call().map_err(http_err)?;
|
||||
let bytes = resp.body_mut().read_to_vec().map_err(|e| AiError::Http(e.to_string()))?;
|
||||
fs::write(&tmp, bytes)?;
|
||||
fs::rename(&tmp, &dest)?;
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// Remove `entry`'s file (and any `.part` remnant). Missing is fine;
|
||||
/// other failures (permissions, locked) are surfaced to the caller.
|
||||
pub fn delete(entry: &ModelEntry, cache_dir: &Path) -> Result<(), AiError> {
|
||||
for p in [cache_dir.join(entry.file), cache_dir.join(format!("{}.part", entry.file))] {
|
||||
match fs::remove_file(&p) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => return Err(AiError::Io(e)),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::registry::Arch;
|
||||
|
||||
fn entry() -> &'static ModelEntry {
|
||||
static E: ModelEntry = ModelEntry {
|
||||
id: "t",
|
||||
name: "t",
|
||||
url: "",
|
||||
file: "m.gguf",
|
||||
tokenizer_url: "",
|
||||
sha256: None,
|
||||
size_mb: 1,
|
||||
min_ram_gb: 0.0,
|
||||
rec_ram_gb: 0.0,
|
||||
license: "MIT",
|
||||
quality: crate::registry::Quality::Tiny,
|
||||
arch: Arch::Llama,
|
||||
langs: "",
|
||||
};
|
||||
&E
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn second_lock_on_same_entry_is_busy() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let part = dir.path().join("m.gguf.part");
|
||||
let _lock = lock_entry(&part).expect("first lock");
|
||||
assert!(matches!(lock_entry(&part), Err(AiError::Busy)));
|
||||
// Releasing lets the next caller in.
|
||||
drop(_lock);
|
||||
assert!(lock_entry(&part).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reports_failures_beyond_not_found() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
// Absent paths are fine.
|
||||
delete(entry(), dir.path()).expect("missing ok");
|
||||
// A directory where the file would be fails removal on every OS.
|
||||
fs::create_dir(dir.path().join("m.gguf")).expect("mkdir");
|
||||
assert!(matches!(delete(entry(), dir.path()), Err(AiError::Io(_))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! craft-ai — AI provider abstraction for the Craft suite.
|
||||
//!
|
||||
//! See `README.md` for the design: BYO API provider (OpenAI-compatible) +
|
||||
//! one-click local models with capability detection.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod error;
|
||||
mod provider;
|
||||
mod registry;
|
||||
mod translate;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod fetch;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[cfg(feature = "local")]
|
||||
pub mod local;
|
||||
// `MachineCaps` is portable data the registry needs on every target; only
|
||||
// the OS probe itself is native-only.
|
||||
pub mod probe;
|
||||
|
||||
pub use error::AiError;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use provider::OpenAiCompat;
|
||||
pub use provider::{GenOpts, Provider, ProviderInfo};
|
||||
pub use registry::{Arch, MODELS, ModelEntry, Quality, by_id, fits, suggest};
|
||||
pub use translate::{PdfGrammar, Segment, TranslateGrammar, is_protected_token, segment, translate_segments};
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Local GGUF inference on candle (pure Rust — keeps the no-C++ rule and
|
||||
//! the door open for wasm later). CPU only for now; GPU comes from the
|
||||
//! app's wgpu adapter in a later milestone.
|
||||
//!
|
||||
//! Both supported architectures speak ChatML — one prompt template covers
|
||||
//! Qwen2.5 and SmolLM2.
|
||||
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
|
||||
use candle_core::quantized::gguf_file;
|
||||
use candle_core::{Device, IndexOp, Tensor};
|
||||
use candle_transformers::generation::LogitsProcessor;
|
||||
use candle_transformers::models::{quantized_llama, quantized_qwen2};
|
||||
use tokenizers::Tokenizer;
|
||||
|
||||
use crate::error::AiError;
|
||||
use crate::provider::{GenOpts, Provider, ProviderInfo};
|
||||
use crate::registry::{Arch, ModelEntry};
|
||||
|
||||
enum Weights {
|
||||
Qwen2(quantized_qwen2::ModelWeights),
|
||||
Llama(quantized_llama::ModelWeights),
|
||||
}
|
||||
|
||||
impl Weights {
|
||||
fn forward(&mut self, ids: &Tensor, pos: usize) -> Result<Tensor, candle_core::Error> {
|
||||
match self {
|
||||
Self::Qwen2(w) => w.forward(ids, pos),
|
||||
Self::Llama(w) => w.forward(ids, pos),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A downloaded GGUF loaded and ready. Construction is the expensive
|
||||
/// part; `complete` is then just a forward loop.
|
||||
pub struct LocalGguf {
|
||||
weights: Weights,
|
||||
tokenizer: Tokenizer,
|
||||
device: Device,
|
||||
eos: u32,
|
||||
entry: &'static ModelEntry,
|
||||
}
|
||||
|
||||
const SEED: u64 = 0xC0FFEE;
|
||||
|
||||
impl LocalGguf {
|
||||
/// Load the GGUF + tokenizer.json of `entry` from the model cache.
|
||||
/// All failures are `Inference` errors — a corrupt file, an OOM, a
|
||||
/// wrong-arch GGUF, all land in the status line, never in a panic.
|
||||
pub fn load(entry: &'static ModelEntry, gguf_path: &Path, tokenizer_path: &Path) -> Result<Self, AiError> {
|
||||
let device = Device::Cpu;
|
||||
let mut file = File::open(gguf_path).map_err(AiError::Io)?;
|
||||
// Malformed GGUF metadata (e.g. `general.alignment = 0`) makes
|
||||
// candle's parser panic on a division — hostile input must come
|
||||
// back as an error, not a crash.
|
||||
// Malformed GGUF metadata (e.g. `general.alignment = 0`, or a
|
||||
// zero `attention.head_count` reaching a division in the weight
|
||||
// constructors) makes candle panic — hostile input must come
|
||||
// back as an error, not a crash. The guard covers parsing *and*
|
||||
// weight construction.
|
||||
let weights = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<Weights, AiError> {
|
||||
let content = gguf_file::Content::read(&mut file).map_err(|e| AiError::Inference(format!("bad GGUF: {e}")))?;
|
||||
match entry.arch {
|
||||
Arch::Qwen2 => Ok(Weights::Qwen2(
|
||||
quantized_qwen2::ModelWeights::from_gguf(content, &mut file, &device)
|
||||
.map_err(|e| AiError::Inference(format!("cannot load {}: {e}", entry.file)))?,
|
||||
)),
|
||||
Arch::Llama => Ok(Weights::Llama(
|
||||
quantized_llama::ModelWeights::from_gguf(content, &mut file, &device)
|
||||
.map_err(|e| AiError::Inference(format!("cannot load {}: {e}", entry.file)))?,
|
||||
)),
|
||||
}
|
||||
}))
|
||||
.map_err(|_| AiError::Inference("GGUF loader panicked on malformed metadata".to_owned()))??;
|
||||
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(|e| AiError::Inference(format!("bad tokenizer.json: {e}")))?;
|
||||
let eos = tokenizer.token_to_id("<|im_end|>").ok_or_else(|| AiError::Inference("tokenizer has no EOS token".into()))?;
|
||||
Ok(Self { weights, tokenizer, device, eos, entry })
|
||||
}
|
||||
|
||||
/// ChatML wrap — Qwen2.5 and SmolLM2 both ship this template.
|
||||
fn chat(prompt: &str) -> String {
|
||||
format!(
|
||||
"<|im_start|>system\nYou are a precise translation engine. Follow the instructions exactly.<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n",
|
||||
)
|
||||
}
|
||||
|
||||
fn generate(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError> {
|
||||
let text = Self::chat(prompt);
|
||||
let encoding = self.tokenizer.encode(text, true).map_err(|e| AiError::Inference(format!("encode failed: {e}")))?;
|
||||
let mut tokens: Vec<u32> = encoding.get_ids().to_vec();
|
||||
|
||||
let temp = if opts.temperature <= 0.02 { None } else { Some(f64::from(opts.temperature)) };
|
||||
let mut sampler = LogitsProcessor::new(SEED, temp, None);
|
||||
|
||||
let mut pos = 0usize;
|
||||
let mut generated: Vec<u32> = Vec::with_capacity(opts.max_tokens.min(4096) as usize);
|
||||
let mut hit_eos = false;
|
||||
for _ in 0..opts.max_tokens {
|
||||
let input_len = tokens.len() - pos;
|
||||
let input =
|
||||
Tensor::new(&tokens[pos..], &self.device).and_then(|t| t.reshape((1, t.dims1()?))).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
let logits = self.weights.forward(&input, pos).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
pos += input_len;
|
||||
// `forward` already returns the last sequence position's
|
||||
// logits [batch, vocab] — index the batch row only.
|
||||
let logits = logits.i(0).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
let next = sampler.sample(&logits).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
if next == self.eos {
|
||||
hit_eos = true;
|
||||
break;
|
||||
}
|
||||
tokens.push(next);
|
||||
generated.push(next);
|
||||
}
|
||||
// No EOS inside the budget → the reply was cut mid-translation;
|
||||
// partial output would silently truncate the document.
|
||||
if !hit_eos {
|
||||
return Err(AiError::Inference(format!("generation hit max_tokens ({}) without EOS", opts.max_tokens)));
|
||||
}
|
||||
// Decode the generated ids through the tokenizer: byte-level BPE
|
||||
// vocab strings can't be concatenated (spaces/newlines/non-ASCII
|
||||
// bytes stay encoded, and UTF-8 can split across tokens).
|
||||
self.tokenizer.decode(&generated, true).map_err(|e| AiError::Inference(format!("decode failed: {e}")))
|
||||
}
|
||||
}
|
||||
|
||||
impl Provider for LocalGguf {
|
||||
fn name(&self) -> String {
|
||||
format!("local: {}", self.entry.name)
|
||||
}
|
||||
|
||||
fn complete(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError> {
|
||||
self.generate(prompt, opts).map(|s| s.trim().to_string())
|
||||
}
|
||||
|
||||
fn test(&mut self) -> Result<ProviderInfo, AiError> {
|
||||
Ok(ProviderInfo { label: format!("loaded: {}", self.entry.name), models: vec![self.entry.id.to_string()] })
|
||||
}
|
||||
|
||||
fn is_local(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//! Machine capability detection — runs once, cache the result.
|
||||
//!
|
||||
//! GPU/VRAM is deliberately *not* probed here: apps already link wgpu, so
|
||||
//! they read their own adapter info and fill `MachineCaps::vram_gb` after
|
||||
//! calling [`probe`]. Re-probing the GPU would double the surface for a
|
||||
//! bug that reports nonsense VRAM on hybrid laptops.
|
||||
|
||||
/// What this machine can comfortably run.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct MachineCaps {
|
||||
/// Total physical RAM, in GiB.
|
||||
pub ram_gb: f32,
|
||||
/// Free space on the volume holding the model cache, in GiB.
|
||||
pub disk_free_gb: f32,
|
||||
/// Logical CPU threads.
|
||||
pub threads: usize,
|
||||
/// AVX2 detected — without it quantized GGUF inference crawls.
|
||||
pub avx2: bool,
|
||||
/// Dedicated VRAM in GiB, when the app supplied it from its wgpu adapter.
|
||||
/// `None` means "unknown or no discrete GPU" — never guessed.
|
||||
pub vram_gb: Option<f32>,
|
||||
}
|
||||
|
||||
impl MachineCaps {
|
||||
/// Rough throughput hint for a model of `size_mb`: tokens/sec on CPU.
|
||||
/// Deliberately conservative — an honest underestimate beats an
|
||||
/// optimistic one that makes the app look slow.
|
||||
pub fn est_tokens_per_sec(&self, size_mb: u64) -> f32 {
|
||||
let base = if self.avx2 { 6.0 } else { 1.2 };
|
||||
// ~0.5B params ≈ 350 MiB Q4; speed scales inversely with size.
|
||||
let scale = 350.0 / (size_mb.max(1) as f32);
|
||||
let thread_boost = (self.threads as f32 / 4.0).min(1.5);
|
||||
(base * scale * thread_boost).max(0.3)
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe the real machine once. `cache_dir` is the future model-cache
|
||||
/// location — disk space is measured on *its* volume, which may not be
|
||||
/// the OS volume.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn probe(cache_dir: &std::path::Path) -> MachineCaps {
|
||||
let mut sys = sysinfo::System::new();
|
||||
sys.refresh_memory();
|
||||
let ram_gb = sys.total_memory() as f32 / (1 << 30) as f32;
|
||||
|
||||
let disk_free_gb = disk_free(cache_dir);
|
||||
|
||||
MachineCaps { ram_gb, disk_free_gb, threads: std::thread::available_parallelism().map_or(1, |n| n.get()), avx2: avx2(), vram_gb: None }
|
||||
}
|
||||
|
||||
#[cfg(all(not(target_arch = "wasm32"), any(target_arch = "x86_64", target_arch = "x86")))]
|
||||
fn avx2() -> bool {
|
||||
std::arch::is_x86_feature_detected!("avx2")
|
||||
}
|
||||
|
||||
#[cfg(all(not(target_arch = "wasm32"), not(any(target_arch = "x86_64", target_arch = "x86"))))]
|
||||
fn avx2() -> bool {
|
||||
// AArch64 NEON is mandatory, so the honest answer is "fast enough".
|
||||
true
|
||||
}
|
||||
|
||||
/// Free bytes on the volume containing `dir`, walking up to the nearest
|
||||
/// existing ancestor if `dir` doesn't exist yet (the common case).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn disk_free(dir: &std::path::Path) -> f32 {
|
||||
// Canonicalize the existing ancestor: a relative or symlinked
|
||||
// `cache_dir` must still match an absolute mount point.
|
||||
let probe: std::path::PathBuf =
|
||||
dir.ancestors().find(|p| p.exists()).and_then(|p| p.canonicalize().ok()).unwrap_or_else(|| std::path::PathBuf::from("."));
|
||||
let disks = sysinfo::Disks::new_with_refreshed_list();
|
||||
// The longest mount point that prefixes our dir owns it.
|
||||
let best = disks.iter().filter(|d| probe.starts_with(d.mount_point())).max_by_key(|d| d.mount_point().as_os_str().len());
|
||||
best.map_or(0.0, |d| d.available_space() as f32 / (1 << 30) as f32)
|
||||
}
|
||||
|
||||
/// The suite-shared model cache (`%LOCALAPPDATA%\artcraft\models` etc.).
|
||||
/// `None` on platforms without a cache dir — the caller should refuse
|
||||
/// local models rather than splashing files into the cwd.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn models_dir() -> Option<std::path::PathBuf> {
|
||||
dirs::cache_dir().map(|d| d.join("artcraft").join("models"))
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
//! Inference backends behind one trait. Apps hold a `Box<dyn Provider>`
|
||||
//! and never care whether the bytes went to OpenAI, Ollama or a local GGUF.
|
||||
|
||||
use crate::error::AiError;
|
||||
|
||||
/// Generation parameters — kept minimal on purpose; the UI exposes
|
||||
/// "translate", not a sampler playground.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GenOpts {
|
||||
/// Max tokens to generate.
|
||||
pub max_tokens: u32,
|
||||
/// Sampling temperature; translation wants ~0.
|
||||
pub temperature: f32,
|
||||
}
|
||||
|
||||
impl Default for GenOpts {
|
||||
fn default() -> Self {
|
||||
Self { max_tokens: 2048, temperature: 0.1 }
|
||||
}
|
||||
}
|
||||
|
||||
/// What `test()` learned about a provider — shown next to the green check.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ProviderInfo {
|
||||
/// Human label, e.g. "Ollama at http://localhost:11434".
|
||||
pub label: String,
|
||||
/// Model ids the endpoint advertised (may be empty if it doesn't say).
|
||||
pub models: Vec<String>,
|
||||
}
|
||||
|
||||
/// An AI backend the suite can use. `Send` so apps can move a loaded
|
||||
/// provider onto a worker thread for long generations.
|
||||
pub trait Provider: Send {
|
||||
/// Short name for the status line ("local: Qwen2.5 1.5B", "OpenAI-compatible").
|
||||
fn name(&self) -> String;
|
||||
/// One completion. Prompt construction is the caller's job — this
|
||||
/// trait is deliberately text-in/text-out so every consumer (translate,
|
||||
/// summarize, custom prompt) shares one path.
|
||||
fn complete(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError>;
|
||||
/// Can we reach it? Never sends document text — just a handshake.
|
||||
fn test(&mut self) -> Result<ProviderInfo, AiError>;
|
||||
/// True when document text stays on this machine.
|
||||
fn is_local(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Any OpenAI-compatible `/v1/chat/completions` endpoint: OpenAI, Azure,
|
||||
/// Ollama (`http://localhost:11434/v1`), LM Studio, Groq, OpenRouter…
|
||||
/// The token is a plain string — storage (OS keyring, settings file) is
|
||||
/// the app's choice, and it is never logged or echoed into errors.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub struct OpenAiCompat {
|
||||
/// Base URL ending at the API root, e.g. `https://api.openai.com/v1`.
|
||||
pub endpoint: String,
|
||||
/// Bearer token; `None` for unauthenticated local endpoints (Ollama).
|
||||
pub token: Option<String>,
|
||||
/// Model id sent in every request.
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl OpenAiCompat {
|
||||
pub fn new(endpoint: impl Into<String>, token: Option<String>, model: impl Into<String>) -> Self {
|
||||
Self { endpoint: endpoint.into(), token, model: model.into() }
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}/{}", self.endpoint.trim_end_matches('/'), path)
|
||||
}
|
||||
|
||||
/// ureq sets no timeouts by default — an endpoint that accepts the
|
||||
/// socket and never replies would block the caller forever, so every
|
||||
/// request goes through an agent with finite deadlines. The response
|
||||
/// budgets are generous: a non-streamed chat completion only sends
|
||||
/// headers once generation is done, which can take minutes.
|
||||
fn agent() -> ureq::Agent {
|
||||
ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(std::time::Duration::from_secs(15)))
|
||||
.timeout_send_request(Some(std::time::Duration::from_secs(30)))
|
||||
.timeout_send_body(Some(std::time::Duration::from_secs(60)))
|
||||
.timeout_recv_response(Some(std::time::Duration::from_secs(300)))
|
||||
.timeout_recv_body(Some(std::time::Duration::from_secs(300)))
|
||||
.build()
|
||||
.into()
|
||||
}
|
||||
|
||||
fn auth(&self, req: ureq::RequestBuilder<ureq::typestate::WithBody>) -> ureq::RequestBuilder<ureq::typestate::WithBody> {
|
||||
match &self.token {
|
||||
Some(t) if !t.is_empty() => req.header("Authorization", format!("Bearer {t}")),
|
||||
_ => req,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl Provider for OpenAiCompat {
|
||||
fn name(&self) -> String {
|
||||
format!("{} ({})", self.model, self.endpoint)
|
||||
}
|
||||
|
||||
fn complete(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError> {
|
||||
if self.endpoint.trim().is_empty() {
|
||||
return Err(AiError::NotConfigured);
|
||||
}
|
||||
let body = serde_json::json!({
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": opts.temperature,
|
||||
"max_tokens": opts.max_tokens,
|
||||
});
|
||||
let mut resp =
|
||||
self.auth(Self::agent().post(self.url("chat/completions"))).header("Content-Type", "application/json").send_json(body).map_err(|e| {
|
||||
match e {
|
||||
ureq::Error::StatusCode(401 | 403) => AiError::Rejected("authentication failed — check the API token".into()),
|
||||
ureq::Error::StatusCode(404) => AiError::Rejected("model or endpoint not found — check both fields".into()),
|
||||
ureq::Error::StatusCode(c) => AiError::Rejected(format!("HTTP {c}")),
|
||||
other => AiError::Http(other.to_string()),
|
||||
}
|
||||
})?;
|
||||
let v: serde_json::Value = resp.body_mut().read_json().map_err(|e| AiError::BadResponse(e.to_string()))?;
|
||||
// A completion cut off at max_tokens ("length") or aborted is not
|
||||
// a translation — returning it would silently truncate the doc.
|
||||
match v.pointer("/choices/0/finish_reason").and_then(serde_json::Value::as_str) {
|
||||
Some("stop") | Some("tool_calls") | None => {}
|
||||
Some(reason) => return Err(AiError::BadResponse(format!("incomplete completion (finish_reason={reason})"))),
|
||||
}
|
||||
v.pointer("/choices/0/message/content")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| AiError::BadResponse("no choices[0].message.content in response".into()))
|
||||
}
|
||||
|
||||
fn test(&mut self) -> Result<ProviderInfo, AiError> {
|
||||
if self.endpoint.trim().is_empty() {
|
||||
return Err(AiError::NotConfigured);
|
||||
}
|
||||
// GET /v1/models — a handshake only, no user text leaves the app.
|
||||
let mut req = Self::agent().get(self.url("models"));
|
||||
if let Some(t) = &self.token
|
||||
&& !t.is_empty()
|
||||
{
|
||||
req = req.header("Authorization", format!("Bearer {t}"));
|
||||
}
|
||||
let mut resp = req.call().map_err(|e| match e {
|
||||
ureq::Error::StatusCode(401 | 403) => AiError::Rejected("authentication failed — check the API token".into()),
|
||||
ureq::Error::StatusCode(c) => AiError::Rejected(format!("HTTP {c}")),
|
||||
other => AiError::Http(other.to_string()),
|
||||
})?;
|
||||
let v: serde_json::Value = resp.body_mut().read_json().map_err(|e| AiError::BadResponse(e.to_string()))?;
|
||||
let models = v
|
||||
.get("data")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.map(|arr| arr.iter().filter_map(|m| m.get("id").and_then(serde_json::Value::as_str).map(str::to_string)).collect())
|
||||
.unwrap_or_default();
|
||||
Ok(ProviderInfo { label: format!("reachable: {}", self.endpoint), models })
|
||||
}
|
||||
|
||||
fn is_local(&self) -> bool {
|
||||
// Local endpoints keep text on the machine; we can't verify a
|
||||
// remote URL's forwarding, so honesty means: loopback → local.
|
||||
// Parse the authority properly — a prefix match would call
|
||||
// `localhost.example.com` or `127.0.0.1.example.com` "local"
|
||||
// while document text leaves the machine.
|
||||
let e = self.endpoint.trim();
|
||||
let after_scheme = e.split_once("://").map_or(e, |(_, rest)| rest);
|
||||
let authority = after_scheme.split(['/', '?', '#']).next().unwrap_or("");
|
||||
let host_port = authority.rsplit('@').next().unwrap_or(""); // strip userinfo
|
||||
let host = if let Some(rest) = host_port.strip_prefix('[') {
|
||||
rest.split(']').next().unwrap_or("")
|
||||
} else {
|
||||
host_port.split(':').next().unwrap_or("")
|
||||
};
|
||||
host.eq_ignore_ascii_case("localhost") || host.parse::<std::net::IpAddr>().is_ok_and(|ip| ip.is_loopback())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(target_arch = "wasm32")))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn is_local_accepts_only_real_loopback() {
|
||||
let local = |ep: &str| OpenAiCompat::new(ep, None, "m").is_local();
|
||||
assert!(local("http://localhost:11434/v1"));
|
||||
assert!(local("https://localhost/v1"));
|
||||
assert!(local("http://127.0.0.1:8080/v1"));
|
||||
assert!(local("http://127.99.0.1/v1")); // whole 127/8 is loopback
|
||||
assert!(local("http://[::1]:11434/v1"));
|
||||
// Lookalike hostnames must NOT report local — document text would
|
||||
// leave the machine while the UI promises it stays.
|
||||
assert!(!local("https://localhost.example.com/v1"));
|
||||
assert!(!local("https://127.0.0.1.example.com/v1"));
|
||||
assert!(!local("https://localhost.evil.io/v1"));
|
||||
assert!(!local("https://api.openai.com/v1"));
|
||||
assert!(!local("https://example.com/localhost/v1"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! The curated model registry — a hand-maintained table, compiled in.
|
||||
//! Runtime-fetching this list would make our supply chain un-auditable.
|
||||
//!
|
||||
//! Curation rule: **permissive licenses only** (the suite is MIT OR
|
||||
//! Apache-2.0 and commercial users run it). CC-BY-NC models like NLLB-200
|
||||
//! are excluded by policy; their absence is deliberate.
|
||||
//!
|
||||
//! `sha256` may be `None`: Hugging Face serves the LFS digest in the
|
||||
//! `X-Linked-ETag` header of the resolve redirect, which [`crate::fetch`]
|
||||
//! uses as the expected checksum. We only hard-code a hash when we've
|
||||
//! verified it ourselves — never a guessed one.
|
||||
|
||||
use crate::probe::MachineCaps;
|
||||
|
||||
/// Model quality tier — higher is better output, bigger download.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Quality {
|
||||
Tiny,
|
||||
Small,
|
||||
Medium,
|
||||
}
|
||||
|
||||
/// Which candle architecture loads the GGUF.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Arch {
|
||||
Qwen2,
|
||||
Llama,
|
||||
}
|
||||
|
||||
/// One downloadable model.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ModelEntry {
|
||||
/// Stable identifier stored in settings (`"qwen2.5-1.5b-q4km"`).
|
||||
pub id: &'static str,
|
||||
/// Display name for the picker.
|
||||
pub name: &'static str,
|
||||
/// Canonical download URL (Hugging Face `resolve` endpoint).
|
||||
pub url: &'static str,
|
||||
/// Filename inside the model cache.
|
||||
pub file: &'static str,
|
||||
/// Companion tokenizer (needed by local inference; the GGUF ships
|
||||
/// vocab only, not the chat-time tokenizer.json).
|
||||
pub tokenizer_url: &'static str,
|
||||
/// Verified SHA-256 of the GGUF, when we have one.
|
||||
pub sha256: Option<&'static str>,
|
||||
/// Download size, MiB — shown to the user before they commit.
|
||||
pub size_mb: u64,
|
||||
/// Below this RAM the model is refused outright.
|
||||
pub min_ram_gb: f32,
|
||||
/// Recommended RAM for comfortable use.
|
||||
pub rec_ram_gb: f32,
|
||||
/// SPDX license id, displayed in the UI.
|
||||
pub license: &'static str,
|
||||
pub quality: Quality,
|
||||
pub arch: Arch,
|
||||
/// Human-readable language coverage for the picker.
|
||||
pub langs: &'static str,
|
||||
}
|
||||
|
||||
/// The registry, smallest first.
|
||||
pub static MODELS: &[ModelEntry] = &[
|
||||
ModelEntry {
|
||||
id: "qwen2.5-0.5b-q4km",
|
||||
name: "Qwen2.5 0.5B Instruct (Q4_K_M)",
|
||||
url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_k_m.gguf",
|
||||
file: "qwen2.5-0.5b-instruct-q4_k_m.gguf",
|
||||
tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer.json",
|
||||
sha256: None,
|
||||
size_mb: 389,
|
||||
min_ram_gb: 2.0,
|
||||
rec_ram_gb: 4.0,
|
||||
license: "Apache-2.0",
|
||||
quality: Quality::Tiny,
|
||||
arch: Arch::Qwen2,
|
||||
langs: "29+ languages incl. EN/ZH/FR/DE/ES/JA/KO/RU",
|
||||
},
|
||||
ModelEntry {
|
||||
id: "smollm2-1.7b-q4km",
|
||||
name: "SmolLM2 1.7B Instruct (Q4_K_M)",
|
||||
url: "https://huggingface.co/bartowski/SmolLM2-1.7B-Instruct-GGUF/resolve/main/SmolLM2-1.7B-Instruct-Q4_K_M.gguf",
|
||||
file: "SmolLM2-1.7B-Instruct-Q4_K_M.gguf",
|
||||
tokenizer_url: "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct/resolve/main/tokenizer.json",
|
||||
sha256: None,
|
||||
size_mb: 1033,
|
||||
min_ram_gb: 4.0,
|
||||
rec_ram_gb: 8.0,
|
||||
license: "Apache-2.0",
|
||||
quality: Quality::Small,
|
||||
arch: Arch::Llama,
|
||||
langs: "primarily EN/FR/DE/ES/IT",
|
||||
},
|
||||
ModelEntry {
|
||||
id: "qwen2.5-1.5b-q4km",
|
||||
name: "Qwen2.5 1.5B Instruct (Q4_K_M)",
|
||||
url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q4_k_m.gguf",
|
||||
file: "qwen2.5-1.5b-instruct-q4_k_m.gguf",
|
||||
tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/tokenizer.json",
|
||||
sha256: None,
|
||||
size_mb: 986,
|
||||
min_ram_gb: 4.0,
|
||||
rec_ram_gb: 8.0,
|
||||
license: "Apache-2.0",
|
||||
quality: Quality::Small,
|
||||
arch: Arch::Qwen2,
|
||||
langs: "29+ languages incl. EN/ZH/FR/DE/ES/JA/KO/RU",
|
||||
},
|
||||
// NOTE: Qwen2.5-3B was removed — its `qwen-research` license is
|
||||
// non-commercial and breaks the permissive-only rule above. No
|
||||
// permissively-licensed Medium-tier candidate is currently curated.
|
||||
];
|
||||
|
||||
/// Look up a model by stable id.
|
||||
pub fn by_id(id: &str) -> Option<&'static ModelEntry> {
|
||||
MODELS.iter().find(|m| m.id == id)
|
||||
}
|
||||
|
||||
/// Whether `entry` can run at all on this machine.
|
||||
pub fn fits(caps: &MachineCaps, entry: &ModelEntry) -> bool {
|
||||
let ram_ok = caps.ram_gb >= entry.min_ram_gb;
|
||||
let disk_ok = caps.disk_free_gb >= entry.size_mb as f32 / 1024.0 * 1.3; // file + headroom
|
||||
ram_ok && disk_ok
|
||||
}
|
||||
|
||||
/// The default suggestion for `caps`: best quality that fits, preferring
|
||||
/// 3B on roomy machines, 1.5B mid-range, 0.5B on small ones. Returns
|
||||
/// `None` when nothing fits — the UI then says so honestly instead of
|
||||
/// offering a download that can't run.
|
||||
pub fn suggest(caps: &MachineCaps) -> Option<&'static ModelEntry> {
|
||||
let capable: Vec<&ModelEntry> = MODELS.iter().filter(|m| fits(caps, m)).collect();
|
||||
let best_allowed = if caps.vram_gb.is_some_and(|v| v >= 6.0) || caps.ram_gb >= 16.0 {
|
||||
Quality::Medium
|
||||
} else if caps.ram_gb >= 8.0 {
|
||||
Quality::Small
|
||||
} else {
|
||||
Quality::Tiny
|
||||
};
|
||||
capable.into_iter().filter(|m| m.quality <= best_allowed).max_by_key(|m| m.quality)
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
//! Document translation as *segments*, never raw text: the model sees
|
||||
//! only translatable runs, and protected spans (URLs, placeholders,
|
||||
//! markup, numbers-only runs) come back byte-identical because we never
|
||||
//! sent them.
|
||||
//!
|
||||
//! Each app declares its grammar via [`TranslateGrammar`]; the default
|
||||
//! [`PdfGrammar`] suits extracted PDF text.
|
||||
|
||||
use crate::error::AiError;
|
||||
use crate::provider::{GenOpts, Provider};
|
||||
|
||||
/// A unit of text the translator decided on.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Segment {
|
||||
/// Sent to the model; replace with the translation.
|
||||
Text(String),
|
||||
/// Never sent; re-inserted verbatim.
|
||||
Protected(String),
|
||||
}
|
||||
|
||||
impl Segment {
|
||||
/// The source text, translated or not.
|
||||
pub fn source(&self) -> &str {
|
||||
match self {
|
||||
Self::Text(s) | Self::Protected(s) => s,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which runs of a document's text must never be translated.
|
||||
pub trait TranslateGrammar {
|
||||
/// True when `token` (already delimited — a word, symbol, or
|
||||
/// whitespace-free run) is structural and must survive verbatim.
|
||||
fn is_protected(&self, token: &str) -> bool;
|
||||
}
|
||||
|
||||
/// Grammar for text extracted from a PDF: protects URLs, e-mail
|
||||
/// addresses, `{placeholders}`, `<markup>`, and runs with no letters
|
||||
/// (page furniture, digits, punctuation).
|
||||
pub struct PdfGrammar;
|
||||
|
||||
impl TranslateGrammar for PdfGrammar {
|
||||
fn is_protected(&self, token: &str) -> bool {
|
||||
is_protected_token(token)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared protected-token test, useful to other apps' grammars.
|
||||
pub fn is_protected_token(token: &str) -> bool {
|
||||
let t = token.trim();
|
||||
if t.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if !t.chars().any(|c| c.is_alphabetic()) {
|
||||
return true; // numbers, punctuation, symbols
|
||||
}
|
||||
let lower = t.to_ascii_lowercase();
|
||||
lower.starts_with("http://")
|
||||
|| lower.starts_with("https://")
|
||||
|| lower.starts_with("www.")
|
||||
|| lower.contains('@') && lower.contains('.') && !lower.contains(' ')
|
||||
|| (t.starts_with('{') && t.ends_with('}'))
|
||||
|| (t.starts_with('<') && t.ends_with('>'))
|
||||
}
|
||||
|
||||
/// Split `text` into segments on whitespace boundaries, grouping
|
||||
/// consecutive protected or translatable tokens. `<markup>` spans are
|
||||
/// carved out first — a tag with attributes contains whitespace, and a
|
||||
/// whole `<em>Hello</em>` run must keep "Hello" translatable while the
|
||||
/// tags themselves stay byte-identical.
|
||||
pub fn segment(text: &str, grammar: &dyn TranslateGrammar) -> Vec<Segment> {
|
||||
fn flush(out: &mut Vec<Segment>, buf: &mut String, protected: bool) {
|
||||
if buf.is_empty() {
|
||||
return;
|
||||
}
|
||||
let seg = if protected { Segment::Protected(std::mem::take(buf)) } else { Segment::Text(std::mem::take(buf)) };
|
||||
out.push(seg);
|
||||
}
|
||||
|
||||
let mut out: Vec<Segment> = Vec::new();
|
||||
let mut buf = String::new();
|
||||
let mut buf_protected = false;
|
||||
let mut started = false;
|
||||
|
||||
for (token, is_tag) in split_tokens(text) {
|
||||
let ws = token.chars().all(char::is_whitespace);
|
||||
// Whitespace glues to whatever the current run is; markup tags
|
||||
// are always protected.
|
||||
let this_protected = if ws { buf_protected } else { is_tag || grammar.is_protected(token) };
|
||||
if started && this_protected != buf_protected {
|
||||
flush(&mut out, &mut buf, buf_protected);
|
||||
}
|
||||
buf_protected = this_protected;
|
||||
started = true;
|
||||
buf.push_str(token);
|
||||
}
|
||||
flush(&mut out, &mut buf, buf_protected);
|
||||
out
|
||||
}
|
||||
|
||||
/// Tokenize into `(slice, is_tag)`: markup tags are whole `<…>` spans
|
||||
/// (attributes included), everything else alternates between maximal
|
||||
/// whitespace and non-whitespace runs.
|
||||
fn split_tokens(text: &str) -> Vec<(&str, bool)> {
|
||||
let mut out = Vec::new();
|
||||
let mut pos = 0usize;
|
||||
let bytes = text.as_bytes();
|
||||
while pos < bytes.len() {
|
||||
if bytes[pos] == b'<'
|
||||
&& let Some(tag_len) = tag_len(&text[pos..])
|
||||
{
|
||||
out.push((&text[pos..pos + tag_len], true));
|
||||
pos += tag_len;
|
||||
continue;
|
||||
}
|
||||
// Advance a non-tag run up to the next '<' that starts a tag;
|
||||
// bare '<' characters just stay inside the run.
|
||||
let mut scan = pos;
|
||||
let mut run_end = bytes.len();
|
||||
while let Some(lt) = bytes[scan..].iter().position(|&b| b == b'<') {
|
||||
let at = scan + lt;
|
||||
if tag_len(&text[at..]).is_some() {
|
||||
run_end = at;
|
||||
break;
|
||||
}
|
||||
scan = at + 1;
|
||||
}
|
||||
for piece in split_words_ws(&text[pos..run_end]) {
|
||||
out.push((piece, false));
|
||||
}
|
||||
pos = run_end;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Length of a `<…>` tag starting at `s[0]`, or `None` when `<` does not
|
||||
/// begin markup (bare `<`, `<3`, `< hello`). Tags start `</`, `<?`, `<!`
|
||||
/// or `<`+letter and close at the first `>` before any `<` or newline;
|
||||
/// the scan is capped so pathological input can't stall it.
|
||||
fn tag_len(s: &str) -> Option<usize> {
|
||||
const MAX_TAG: usize = 512;
|
||||
let second = s.as_bytes().get(1)?;
|
||||
let opener_ok = second.is_ascii_alphabetic() || matches!(second, b'/' | b'!' | b'?');
|
||||
if !opener_ok {
|
||||
return None;
|
||||
}
|
||||
s.char_indices().skip(2).take(MAX_TAG).find(|(_, c)| matches!(c, '>' | '<' | '\n')).and_then(|(i, c)| (c == '>').then_some(i + 1))
|
||||
}
|
||||
|
||||
/// Split into alternating word/whitespace runs (like `split_inclusive`
|
||||
/// but grouping every maximal run of whitespace or non-whitespace).
|
||||
fn split_words_ws(text: &str) -> Vec<&str> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0usize;
|
||||
let mut prev_ws: Option<bool> = None;
|
||||
for (i, c) in text.char_indices() {
|
||||
let ws = c.is_whitespace();
|
||||
if let Some(p) = prev_ws
|
||||
&& p != ws
|
||||
{
|
||||
out.push(&text[start..i]);
|
||||
start = i;
|
||||
}
|
||||
prev_ws = Some(ws);
|
||||
}
|
||||
if start < text.len() {
|
||||
out.push(&text[start..]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// How many segments one prompt carries — keeps responses snappy and
|
||||
/// makes a numbering slip cheap to retry.
|
||||
const BATCH: usize = 20;
|
||||
/// …and how many bytes one prompt carries. A document with no protected
|
||||
/// tokens is one `Text` segment of arbitrary length — without a size
|
||||
/// bound the request can exceed `max_tokens` and come back truncated
|
||||
/// (which providers must reject, not return as success).
|
||||
const BATCH_BYTES: usize = 4 * 1024;
|
||||
/// A single `Text` run larger than this is chopped into pieces before
|
||||
/// batching, so no one request is unbounded.
|
||||
const RUN_BYTES: usize = 3 * 1024;
|
||||
|
||||
/// A translatable piece of one segment: the whitespace-stripped `core`
|
||||
/// is sent to the model, `lead`/`trail` are restored verbatim around the
|
||||
/// translation so `Visitez` never glues to `https://a.io`.
|
||||
struct Run<'a> {
|
||||
seg: usize,
|
||||
lead: &'a str,
|
||||
core: &'a str,
|
||||
trail: &'a str,
|
||||
/// False for whitespace-only pieces: never sent to the model, but
|
||||
/// still reassembled into the segment.
|
||||
send: bool,
|
||||
}
|
||||
|
||||
/// `(leading ws, core, trailing ws)` of `s`, all on char boundaries.
|
||||
fn ws_edges(s: &str) -> (&str, &str, &str) {
|
||||
let lead = s.len() - s.trim_start().len();
|
||||
let tail = s.len() - s.trim_end().len();
|
||||
if lead + tail >= s.len() {
|
||||
return (s, "", ""); // all-whitespace run: nothing to translate
|
||||
}
|
||||
(&s[..lead], &s[lead..s.len() - tail], &s[s.len() - tail..])
|
||||
}
|
||||
|
||||
/// Chop `s` into ≤ `max`-byte pieces on whitespace boundaries (the last
|
||||
/// ws before the limit; a run with no ws is hard-cut at the limit). The
|
||||
/// boundary whitespace stays inside the pieces — nothing is lost.
|
||||
fn split_run(s: &str, max: usize) -> Vec<&str> {
|
||||
let mut out = Vec::new();
|
||||
let mut rest = s;
|
||||
while rest.len() > max {
|
||||
let mut ws_cut = None;
|
||||
let mut boundary = 0usize;
|
||||
for (i, c) in rest.char_indices() {
|
||||
if i >= max {
|
||||
break;
|
||||
}
|
||||
boundary = i + c.len_utf8();
|
||||
if c.is_whitespace() {
|
||||
ws_cut = Some(i); // cut *before* the ws — it leads the next piece
|
||||
}
|
||||
}
|
||||
let cut = match ws_cut.filter(|w| *w > max / 4) {
|
||||
Some(w) => w,
|
||||
None => boundary.max(1),
|
||||
};
|
||||
out.push(&rest[..cut]);
|
||||
rest = &rest[cut..];
|
||||
}
|
||||
if !rest.is_empty() {
|
||||
out.push(rest);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Translate every `Text` segment to `target_lang`, leaving `Protected`
|
||||
/// untouched. Batched with numbered items; a count mismatch retries the
|
||||
/// offending half rather than misaligning the whole document.
|
||||
pub fn translate_segments(provider: &mut dyn Provider, segs: &[Segment], target_lang: &str) -> Result<Vec<String>, AiError> {
|
||||
// Source whitespace never reaches the model: each Text segment is
|
||||
// split edge-to-edge, and oversized runs are chopped further.
|
||||
let mut runs: Vec<Run<'_>> = Vec::new();
|
||||
for (i, s) in segs.iter().enumerate() {
|
||||
let Segment::Text(src) = s else { continue };
|
||||
for piece in split_run(src, RUN_BYTES) {
|
||||
let (lead, core, trail) = ws_edges(piece);
|
||||
// Pure-whitespace pieces (possible after a long run is chopped)
|
||||
// keep their bytes in the output but skip the provider.
|
||||
runs.push(Run { seg: i, lead, core, trail, send: !core.is_empty() });
|
||||
}
|
||||
}
|
||||
let run_segs: std::collections::HashSet<usize> = runs.iter().map(|r| r.seg).collect();
|
||||
|
||||
let mut out: Vec<String> = segs.iter().map(|s| s.source().to_string()).collect();
|
||||
for &i in &run_segs {
|
||||
out[i].clear(); // rebuilt from translated runs below
|
||||
}
|
||||
let mut i = 0;
|
||||
while i < runs.len() {
|
||||
// Whitespace-only pieces restore verbatim, in order, and never
|
||||
// reach the model.
|
||||
while i < runs.len() && !runs[i].send {
|
||||
let r = &runs[i];
|
||||
out[r.seg].push_str(r.lead);
|
||||
out[r.seg].push_str(r.trail);
|
||||
i += 1;
|
||||
}
|
||||
// A batch is bounded by item count *and* total prompt bytes.
|
||||
let mut bytes = 0usize;
|
||||
let start = i;
|
||||
while i < runs.len() && runs[i].send && i - start < BATCH && bytes + runs[i].core.len() <= BATCH_BYTES {
|
||||
bytes += runs[i].core.len();
|
||||
i += 1;
|
||||
}
|
||||
if i == start {
|
||||
if i < runs.len() {
|
||||
i += 1; // never spin on an oversized single item
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let chunk = &runs[start..i];
|
||||
let translated = translate_batch(provider, chunk, target_lang)?;
|
||||
for (run, t) in chunk.iter().zip(translated) {
|
||||
out[run.seg].push_str(run.lead);
|
||||
out[run.seg].push_str(&t);
|
||||
out[run.seg].push_str(run.trail);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One numbered batch; splits recursively on count mismatch so a single
|
||||
/// malformed reply can't corrupt neighbors.
|
||||
fn translate_batch(provider: &mut dyn Provider, runs: &[Run<'_>], lang: &str) -> Result<Vec<String>, AiError> {
|
||||
if runs.len() == 1 {
|
||||
return Ok(vec![translate_one(provider, runs[0].core, lang)?]);
|
||||
}
|
||||
let prompt = numbered_prompt(&runs.iter().map(|r| r.core.to_string()).collect::<Vec<_>>(), lang);
|
||||
let reply = provider.complete(&prompt, &GenOpts::default())?;
|
||||
match parse_numbered(&reply, runs.len()) {
|
||||
Some(v) => Ok(v),
|
||||
None => {
|
||||
let mid = runs.len() / 2;
|
||||
let mut a = translate_batch(provider, &runs[..mid], lang)?;
|
||||
let b = translate_batch(provider, &runs[mid..], lang)?;
|
||||
a.extend(b);
|
||||
Ok(a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_one(provider: &mut dyn Provider, source: &str, lang: &str) -> Result<String, AiError> {
|
||||
let prompt = format!("Translate the following text to {lang}. Reply with ONLY the translation, no quotes, no notes.\n\n{source}");
|
||||
Ok(provider.complete(&prompt, &GenOpts::default())?.trim().to_string())
|
||||
}
|
||||
|
||||
/// `"1. …"`-style numbered prompt — models follow it far more reliably
|
||||
/// than delimiter schemes.
|
||||
fn numbered_prompt(sources: &[String], lang: &str) -> String {
|
||||
let mut p = format!(
|
||||
"Translate each numbered item to {lang}. Keep the numbering. \
|
||||
Preserve all formatting, placeholders like {{name}}, URLs and markup verbatim. \
|
||||
Reply with ONLY the numbered translations.\n\n"
|
||||
);
|
||||
for (i, s) in sources.iter().enumerate() {
|
||||
p.push_str(&format!("{}. {}\n", i + 1, s.replace('\n', " ")));
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Parse `"1. …\n2. …"` back into items; `None` when the reply is
|
||||
/// malformed — duplicates, out-of-range numbers, or a missing item all
|
||||
/// mean the numbering slipped, and the caller retries/splits rather
|
||||
/// than accepting corrupted text. Non-numbered lines after an item are
|
||||
/// continuations of it; non-numbered chatter before item 1 is rejected
|
||||
/// the same way, so nothing is silently dropped.
|
||||
fn parse_numbered(reply: &str, want: usize) -> Option<Vec<String>> {
|
||||
let mut out = vec![String::new(); want];
|
||||
let mut seen = vec![false; want];
|
||||
let mut current: Option<usize> = None;
|
||||
for line in reply.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match numbered_line(line) {
|
||||
Some(idx) if idx >= 1 && idx <= want => {
|
||||
if seen[idx - 1] {
|
||||
return None; // "1. un\n2. deux\n1. trois" overwrites — reject
|
||||
}
|
||||
seen[idx - 1] = true;
|
||||
let rest = line.split_once(['.', ')']).map_or("", |(_, r)| r).trim().to_string();
|
||||
out[idx - 1] = rest;
|
||||
current = Some(idx - 1);
|
||||
}
|
||||
Some(_) => return None, // numbered but out of range — malformed reply
|
||||
// A wrapped/multi-line item: keep the continuation, not
|
||||
// "1. Première phrase.\nSuite" dropping "Suite". Unnumbered
|
||||
// preamble is rejected — not our format.
|
||||
None => {
|
||||
let i = current?;
|
||||
out[i].push('\n');
|
||||
out[i].push_str(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
(seen.iter().all(|s| *s) && out.iter().all(|s| !s.is_empty())).then_some(out)
|
||||
}
|
||||
|
||||
/// `Some(idx)` when `line` opens as `"12."` / `"12)"` — a 1-3 digit
|
||||
/// number followed by `'.'` or `')'`. Anything else is `None`, so
|
||||
/// "Section 2. notes" is a continuation, not item 2.
|
||||
fn numbered_line(line: &str) -> Option<usize> {
|
||||
let line = line.trim_start();
|
||||
let digits = line.bytes().take_while(|b| b.is_ascii_digit()).count();
|
||||
if digits == 0 || digits > 3 || !matches!(line.as_bytes().get(digits), Some(b'.' | b')')) {
|
||||
return None;
|
||||
}
|
||||
line.get(..digits)?.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn split_words_ws_groups_runs() {
|
||||
assert_eq!(split_words_ws("a b\tc"), vec!["a", " ", "b", "\t", "c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pdf_grammar_protects() {
|
||||
let g = PdfGrammar;
|
||||
assert!(g.is_protected("https://x.io/a"));
|
||||
assert!(g.is_protected("{count}"));
|
||||
assert!(g.is_protected("42.5"));
|
||||
assert!(!g.is_protected("Bonjour"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segmentation_glues_whitespace_and_protects_spans() {
|
||||
let segs = segment("Visit https://a.io now {n} times", &PdfGrammar);
|
||||
// "Visit " text, protected url, " now " text, protected {n}, " times" text
|
||||
let texts: Vec<&Segment> = segs.iter().filter(|s| matches!(s, Segment::Text(_))).collect();
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.trim() == "https://a.io")));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.trim() == "{n}")));
|
||||
assert!(texts.iter().any(|s| s.source().contains("Visit")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_roundtrip() {
|
||||
let r = "1. un\n2. deux\n3. trois";
|
||||
assert_eq!(parse_numbered(r, 3), Some(vec!["un".into(), "deux".into(), "trois".into()]));
|
||||
assert_eq!(parse_numbered("1. un\n3. trois", 3), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_rejects_duplicates_and_overflow() {
|
||||
assert_eq!(parse_numbered("1. un\n2. deux\n1. trois", 2), None);
|
||||
assert_eq!(parse_numbered("1. un\n2. deux\n3. trois", 2), None);
|
||||
assert_eq!(parse_numbered("0. zero\n1. un\n2. deux", 2), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_keeps_continuation_lines() {
|
||||
let r = "1. Première phrase.\nSuite du paragraphe\n2. Deuxième élément.";
|
||||
assert_eq!(parse_numbered(r, 2), Some(vec!["Première phrase.\nSuite du paragraphe".into(), "Deuxième élément.".into()]));
|
||||
// Preamble chatter that isn't a numbered item is rejected, not
|
||||
// silently dropped.
|
||||
assert_eq!(parse_numbered("Sure, here you go:\n1. un\n2. deux", 2), None);
|
||||
// "Section 2. notes" looks numbered but isn't → continuation.
|
||||
let r2 = "1. un\nSection 2. notes\n2. deux";
|
||||
assert_eq!(parse_numbered(r2, 2), Some(vec!["un\nSection 2. notes".into(), "deux".into()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markup_tags_split_from_their_text() {
|
||||
let segs = segment("<em>Hello</em> world", &PdfGrammar);
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p == "<em>")));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.trim() == "</em>")));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Text(t) if t.contains("Hello"))));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Text(t) if t.contains("world"))));
|
||||
// Tags containing spaces stay whole and protected.
|
||||
let segs = segment("<a href=\"x\">link</a>", &PdfGrammar);
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p == "<a href=\"x\">")));
|
||||
// A bare '<' is text, not a tag opener — words around it stay
|
||||
// translatable (the lone '<' itself is protected as punctuation).
|
||||
let segs = segment("a < b and <3 hearts", &PdfGrammar);
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Text(t) if t.contains("hearts"))));
|
||||
assert!(!segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.contains('<') && p.contains('>'))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ws_edges_and_split_run_preserve_every_byte() {
|
||||
assert_eq!(ws_edges(" héllo \n"), (" ", "héllo", " \n"));
|
||||
assert_eq!(ws_edges(" "), (" ", "", ""));
|
||||
let s = "alpha beta gamma delta epsilon";
|
||||
let pieces = split_run(s, 12);
|
||||
assert!(pieces.iter().all(|p| p.len() <= 12));
|
||||
assert_eq!(pieces.concat(), s); // boundary whitespace survives
|
||||
// A run with no whitespace is hard-cut at a char boundary.
|
||||
let long = "x".repeat(50);
|
||||
let pieces = split_run(&long, 20);
|
||||
assert!(pieces.iter().all(|p| p.len() <= 20));
|
||||
assert_eq!(pieces.concat(), long);
|
||||
}
|
||||
|
||||
/// A provider that echoes numbered replies — exercises the batching,
|
||||
/// whitespace restore and reassembly end to end without a network.
|
||||
struct Echo;
|
||||
|
||||
impl Provider for Echo {
|
||||
fn name(&self) -> String {
|
||||
"echo".into()
|
||||
}
|
||||
|
||||
fn complete(&mut self, prompt: &str, _opts: &GenOpts) -> Result<String, AiError> {
|
||||
if prompt.starts_with("Translate each numbered item") {
|
||||
// Parse our own prompt's numbering and echo each item.
|
||||
let body = prompt.split("\n\n").nth(1).unwrap_or("");
|
||||
let mut out = String::new();
|
||||
for line in body.lines() {
|
||||
out.push_str(line);
|
||||
out.push('\n');
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
Ok(prompt.rsplit("\n\n").next().unwrap_or(prompt).to_string())
|
||||
}
|
||||
|
||||
fn test(&mut self) -> Result<crate::provider::ProviderInfo, AiError> {
|
||||
Ok(crate::provider::ProviderInfo { label: "echo".into(), models: vec![] })
|
||||
}
|
||||
|
||||
fn is_local(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_accepts_paren_markers() {
|
||||
// "1) …" replies contain sentence-ending periods inside items —
|
||||
// the leading delimiter must be picked before inspecting text.
|
||||
let r = "1) Première phrase.\nSuite.\n2) Deuxième élément.";
|
||||
assert_eq!(parse_numbered(r, 2), Some(vec!["Première phrase.\nSuite.".into(), "Deuxième élément.".into()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_only_pieces_survive_reconstruction() {
|
||||
let segs = vec![Segment::Text(format!("{}Hello", " ".repeat(RUN_BYTES)))];
|
||||
let mut echo = Echo;
|
||||
let out = translate_segments(&mut echo, &segs, "fr").unwrap_or_default();
|
||||
assert_eq!(out.concat(), format!("{}Hello", " ".repeat(RUN_BYTES)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translate_restores_whitespace_around_protected_spans() {
|
||||
let segs = segment("Visit https://a.io now", &PdfGrammar);
|
||||
let mut echo = Echo;
|
||||
let out = translate_segments(&mut echo, &segs, "fr").unwrap_or_default();
|
||||
assert_eq!(out.concat(), "Visit https://a.io now");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user