diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..873f4f7aa --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[lints] +cargo.unused_workspace_dependencies = "allow" diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..2408084cf --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +reviews: + auto_review: + # Dependabot bumps are checked by CI; skip CodeRabbit reviews on them. + ignore_usernames: + - "dependabot[bot]" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..a854bbdb9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,16 @@ +# Font-build test fixtures are compared byte for byte: a checkout must not convert them. +crates/fonts-build/tests/fixture/** -text + +# The pinned font index is hashed and compiled in as committed: a checkout must not convert its +# line endings, or `Index::bundled` fails its checksum on Windows. +crates/font-store/index/sources.json -text +crates/font-store/index/indexes/*.gz binary + +# Copies of the vendored krilla source diff, one per patch (see vendor/krilla/README.craft.md). +vendor/krilla/patches/*.patch linguist-generated=true + +# Has a pinned digest and is embedded as bytes by craft-font-store. +crates/font-store/index/sources.json -text + +# The build fixture verifies these license files are copied byte-for-byte. +crates/fonts-build/tests/fixture/craft-fonts/fonts/**/OFL.txt -text diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..52414ce11 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,32 @@ +version: 2 +updates: + # Maintain Rust / Cargo dependencies + - package-ecosystem: "cargo" + directories: + - "/" + - "/crates/pdf/read-only-contract" + schedule: + interval: "daily" + time: "06:00" + timezone: "America/New_York" + # Consolidate updates into a single PR rather than one PR per crate + groups: + cargo-dependencies: + patterns: + - "*" + commit-message: + prefix: "chore(deps)" + + # Maintain GitHub Actions workflows + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + time: "06:00" + timezone: "America/New_York" + groups: + actions: + patterns: + - "*" + commit-message: + prefix: "ci(deps)" diff --git a/.github/workflows/ai-core.yml b/.github/workflows/ai-core.yml new file mode 100644 index 000000000..cc8d738ac --- /dev/null +++ b/.github/workflows/ai-core.yml @@ -0,0 +1,44 @@ +name: AI model + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/ai-core/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ai-core.yml'] + pull_request: + paths: ['crates/ai-core/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ai-core.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: ai-core-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ai-core: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-ai-core --all-targets --locked -- -D warnings + - run: cargo test -p craft-ai-core --locked + # The model promises to compile for the web. + - run: cargo check -p craft-ai-core --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/atomic.yml b/.github/workflows/atomic.yml index 25423780e..1a20fa679 100644 --- a/.github/workflows/atomic.yml +++ b/.github/workflows/atomic.yml @@ -11,18 +11,30 @@ on: permissions: contents: read +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: atomic-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: atomic: timeout-minutes: 15 strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable components: rustfmt, clippy - name: Install ACL tools if: runner.os == 'Linux' @@ -33,3 +45,14 @@ jobs: - name: Check inherited Linux ACLs if: runner.os == 'Linux' run: cargo test -p craft-atomic --locked -- --ignored --nocapture + + wasm: + name: Atomic WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - run: cargo check -p craft-atomic --target wasm32-unknown-unknown + diff --git a/.github/workflows/audio.yml b/.github/workflows/audio.yml new file mode 100644 index 000000000..0517efe6f --- /dev/null +++ b/.github/workflows/audio.yml @@ -0,0 +1,163 @@ +name: Shared audio + +on: + push: + branches: [main] + paths: ["crates/audio/**", "crates/media/**", "crates/media-testkit/**", "scripts/fetch-opus-vectors.sh", "Cargo.toml", "Cargo.lock", ".github/workflows/audio.yml"] + pull_request: + paths: ["crates/audio/**", "crates/media/**", "crates/media-testkit/**", "scripts/fetch-opus-vectors.sh", "Cargo.toml", "Cargo.lock", ".github/workflows/audio.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-audio-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + timeout-minutes: 15 + name: Audio lint, feature sets, docs, WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-audio-lint + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-audio --check + # Each feature on its own, so code that needs another feature fails here. + - name: Clippy, each feature alone + run: | + for f in "" decode decode-native symphonia encode lossless-encode pcm-encode flac-encode peaks aac-decode aac-encode "aac-encode,rayon" opus-decode opus-encode ac3-decode mpa-decode serde; do + echo "::group::features: ${f:-none}" + cargo clippy -p craft-audio --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings + echo "::endgroup::" + done + - run: cargo clippy -p craft-audio --all-targets --locked --all-features -- -D warnings + # missing_docs is a warning in the crate; -D warnings makes it (and broken links) fail here. + - run: cargo doc -p craft-audio --all-features --no-deps --locked + env: + RUSTDOCFLAGS: "-D warnings" + # Partial feature sets must document too (their links to other features are allowed to dangle). + - run: cargo doc -p craft-audio --no-default-features --features decode-native --no-deps --locked + env: + RUSTDOCFLAGS: "-D warnings" + - run: cargo doc -p craft-audio --no-default-features --features lossless-encode --no-deps --locked + env: + RUSTDOCFLAGS: "-D warnings" + - run: cargo check -p craft-audio --all-features --locked --target wasm32-unknown-unknown + - run: cargo check -p craft-audio --no-default-features --features lossless-encode --locked --target wasm32-unknown-unknown + msrv: + timeout-minutes: 10 + name: Audio on the minimum Rust version + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The workspace's rust-version. + - run: rustup toolchain install 1.95.0 --profile minimal + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-audio-msrv + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo +1.95.0 check -p craft-audio -p craft-media --all-features --locked + features: + timeout-minutes: 10 + name: Audio tests without symphonia and lossless-encode only + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-audio-features + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo test -p craft-audio --no-default-features --features lossless-encode --locked + - run: cargo test -p craft-audio --no-default-features --features pcm-encode --locked --lib + - run: cargo test -p craft-audio --no-default-features --features flac-encode --locked --lib + - run: cargo test -p craft-audio --no-default-features --features decode-native --locked + test: + timeout-minutes: 20 + name: Audio tests with ffmpeg oracles and the Opus conformance vectors + # Pinned for the ffmpeg its apt ships (8.0), so the files the oracle tests write do not change + # with the runner image. + runs-on: ubuntu-26.04 + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-audio-test + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/media-fixtures/opus/vectors + # The archive's SHA-256 (scripts/fetch-opus-vectors.sh checks it). + key: opus-testvectors-rfc8251-6b26a22f + - name: Build tests while installing ffmpeg and fetching the Opus vectors + run: | + # The man-db index rebuild after dpkg is slow on hosted runners. + sudo rm -f /var/lib/man-db/auto-update + # A mirror can stall for most of the job's time limit, and the next try usually gets a good one. + install_ffmpeg() { + for attempt in 1 2 3; do + # timeout runs as root, so a stalled apt-get is killed too and frees the dpkg lock. + sudo timeout -k 10 180 sh -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ffmpeg' && return 0 + echo "apt: attempt $attempt failed with exit code $?" + sudo timeout -k 10 60 dpkg --configure -a || true + done + return 1 + } + # The vectors are fetched after ffmpeg, so the two downloads do not slow each other. + { + install_ffmpeg > "$RUNNER_TEMP/apt.log" 2>&1 || exit 1 + scripts/fetch-opus-vectors.sh "$RUNNER_TEMP/media-fixtures/opus/vectors" > "$RUNNER_TEMP/vectors.log" 2>&1 || exit 2 + } & + fetch=$! + cargo test -p craft-audio --all-features --locked --no-run + wait "$fetch" || case $? in + 1) cat "$RUNNER_TEMP/apt.log"; exit 1 ;; + *) cat "$RUNNER_TEMP/vectors.log"; exit 1 ;; + esac + ffmpeg -hide_banner -version | sed -n 1p + - run: cargo test -p craft-audio --all-features --locked + env: + CRAFT_MEDIA_REQUIRE_ORACLES: "1" + CRAFT_MEDIA_FIXTURES_DIR: ${{ runner.temp }}/media-fixtures diff --git a/.github/workflows/cms.yml b/.github/workflows/cms.yml new file mode 100644 index 000000000..a15a48c9d --- /dev/null +++ b/.github/workflows/cms.yml @@ -0,0 +1,26 @@ +name: Colour management + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/cms/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/cms.yml'] + pull_request: + paths: ['crates/cms/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/cms.yml'] + +permissions: + contents: read + +jobs: + cms: + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - run: cargo clippy -p craft-cms --all-targets --all-features --locked -- -D warnings + # Release: the FOGRA39 fit and ink-limit tests take about 35 s in debug and 4 s here. Tests that need + # installed system profiles print "skipped:" when none are present. + - run: cargo test -p craft-cms --all-features --release --locked -- --nocapture diff --git a/.github/workflows/crw.yml b/.github/workflows/crw.yml new file mode 100644 index 000000000..8fec98661 --- /dev/null +++ b/.github/workflows/crw.yml @@ -0,0 +1,101 @@ +name: Shared CRW + +on: + push: + branches: [main] + paths: ["crates/crw/**", "scripts/fetch-crw-samples.py", "Cargo.toml", "Cargo.lock", ".github/workflows/crw.yml"] + pull_request: + paths: ["crates/crw/**", "scripts/fetch-crw-samples.py", "Cargo.toml", "Cargo.lock", ".github/workflows/crw.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-crw-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + native: + timeout-minutes: 20 + name: CRW lint and tests (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-crw-${{ matrix.os }} + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-crw --check + - run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings + - run: cargo test -p craft-crw --locked + + wasm: + timeout-minutes: 10 + name: CRW WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-crw-wasm + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown + + samples: + timeout-minutes: 20 + name: CRW sample files decode exactly + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-crw-samples + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Cache the CC0 sample files + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/crw-samples + key: crw-samples-${{ hashFiles('crates/crw/tests/samples.json') }} + - name: Fetch the pinned CC0 sample files (raw.pixls.us) + run: python scripts/fetch-crw-samples.py "$RUNNER_TEMP/crw-samples" + - run: cargo test -p craft-crw --release --locked --test samples -- --nocapture + env: + CRAFT_CRW_SAMPLES: ${{ runner.temp }}/crw-samples + CRAFT_CRW_REQUIRE_SAMPLES: "1" diff --git a/.github/workflows/dialogs.yml b/.github/workflows/dialogs.yml new file mode 100644 index 000000000..97a931aec --- /dev/null +++ b/.github/workflows/dialogs.yml @@ -0,0 +1,42 @@ +name: Dialogs + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/dialogs/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/dialogs.yml'] + pull_request: + paths: ['crates/dialogs/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/dialogs.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: dialogs-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + dialogs: + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + # The token isn't needed after checkout; cargo runs build scripts and proc macros from + # dependencies, so don't leave it in .git/config for that third-party code to read. + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: clippy + # The macOS code is linted on Linux: it compiles for the target without the SDK. + targets: aarch64-apple-darwin + - run: cargo clippy -p craft-dialogs --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-dialogs --target aarch64-apple-darwin --locked -- -D warnings + - run: cargo test -p craft-dialogs --locked diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 000000000..a0c619c1e --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,47 @@ +name: Craft documents + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/doc/**', 'crates/pcraft-format/**', 'crates/vectorcraft-format/**', 'crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/doc.yml'] + pull_request: + paths: ['crates/doc/**', 'crates/pcraft-format/**', 'crates/vectorcraft-format/**', 'crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/doc.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: doc-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + doc: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-doc --all-targets --all-features --locked -- -D warnings + - run: cargo test -p craft-doc --locked -- --nocapture + - run: cargo test -p craft-doc --all-features --locked -- --nocapture + - run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/docpath.yml b/.github/workflows/docpath.yml new file mode 100644 index 000000000..571b65eca --- /dev/null +++ b/.github/workflows/docpath.yml @@ -0,0 +1,41 @@ +name: Document paths + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml'] + pull_request: + paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: docpath-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + docpath: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy -p craft-docpath --all-targets --locked -- -D warnings + - run: cargo test -p craft-docpath --locked -- --nocapture diff --git a/.github/workflows/ecproj-format.yml b/.github/workflows/ecproj-format.yml new file mode 100644 index 000000000..16bab05c7 --- /dev/null +++ b/.github/workflows/ecproj-format.yml @@ -0,0 +1,45 @@ +name: EffectCraft project format + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ecproj-format.yml'] + pull_request: + paths: ['crates/ecproj-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/ecproj-format.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: ecproj-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ecproj-format: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings + - run: cargo test -p craft-ecproj-format --locked -- --nocapture + - run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/font-store.yml b/.github/workflows/font-store.yml new file mode 100644 index 000000000..f246f621a --- /dev/null +++ b/.github/workflows/font-store.yml @@ -0,0 +1,47 @@ +name: Font store + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/font-store/**', 'crates/font-catalog/**', 'crates/atomic/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/font-store.yml'] + pull_request: + paths: ['crates/font-store/**', 'crates/font-catalog/**', 'crates/atomic/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/font-store.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: font-store-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + font-store: + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings + - run: cargo test -p craft-font-store --locked + - run: cargo clippy -p craft-font-catalog --all-targets --locked -- -D warnings + - run: cargo test -p craft-font-catalog --locked + - name: Check the web build + if: runner.os == 'Linux' + run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/formats.yml b/.github/workflows/formats.yml new file mode 100644 index 000000000..e19902e8c --- /dev/null +++ b/.github/workflows/formats.yml @@ -0,0 +1,111 @@ +name: Shared format crates + +on: + push: + branches: [main] + paths: + - "crates/xml/**" + - "crates/mime/**" + - "crates/cfb/**" + - "crates/mesh/**" + - "crates/metafile/**" + - "crates/dxf/**" + - "crates/gcode/**" + - "crates/svg-paths/**" + - "crates/silhouette-studio/**" + - "crates/tiff/**" + - "crates/heif/**" + - "crates/cms/**" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/formats.yml" + pull_request: + paths: + - "crates/xml/**" + - "crates/mime/**" + - "crates/cfb/**" + - "crates/mesh/**" + - "crates/metafile/**" + - "crates/dxf/**" + - "crates/gcode/**" + - "crates/svg-paths/**" + - "crates/silhouette-studio/**" + - "crates/tiff/**" + - "crates/heif/**" + - "crates/cms/**" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/formats.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-formats-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check: + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + # `features`: each is linted alone, so code that needs another feature fails here. + - { crate: craft-xml, features: "" } + - { crate: craft-mime, features: "" } + - { crate: craft-cfb, features: "" } + - { crate: craft-mesh, features: "stl obj ply gltf" } + - { crate: craft-metafile, features: "write" } + - { crate: craft-dxf, features: "" } + - { crate: craft-gcode, features: "" } + - { crate: craft-svg-paths, features: "" } + - { crate: craft-silhouette-studio, features: "" } + - { crate: craft-tiff, features: "" } + - { crate: craft-heif, features: "testdata" } + - { crate: craft-cms, features: "" } + name: ${{ matrix.crate }} lint, tests, WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: ${{ matrix.crate }} + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p "$CRATE" --check + env: + CRATE: ${{ matrix.crate }} + - name: Clippy, default features and each feature alone + env: + CRATE: ${{ matrix.crate }} + FEATURES: ${{ matrix.features }} + run: | + cargo clippy -p "$CRATE" --all-targets --locked -- -D warnings + for f in $FEATURES; do + echo "::group::feature: $f" + cargo clippy -p "$CRATE" --all-targets --locked --no-default-features --features "$f" -- -D warnings + echo "::endgroup::" + done + - run: cargo clippy -p "$CRATE" --all-targets --locked --all-features -- -D warnings + env: + CRATE: ${{ matrix.crate }} + - run: cargo test -p "$CRATE" --all-features --locked + env: + CRATE: ${{ matrix.crate }} + - run: cargo check -p "$CRATE" --all-features --locked --target wasm32-unknown-unknown + env: + CRATE: ${{ matrix.crate }} diff --git a/.github/workflows/house-rules.yml b/.github/workflows/house-rules.yml new file mode 100644 index 000000000..de64ad3ee --- /dev/null +++ b/.github/workflows/house-rules.yml @@ -0,0 +1,28 @@ +name: House rules + +on: + push: + branches: [main] + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + pull_request: + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: house-rules-${{ github.ref }} + cancel-in-progress: true + +jobs: + house-rules: + uses: storytold/craft-common-actions/.github/workflows/house-rules.yml@main + with: + rust: true diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml new file mode 100644 index 000000000..5f717aeff --- /dev/null +++ b/.github/workflows/i18n.yml @@ -0,0 +1,41 @@ +name: Translations + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] + pull_request: + paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: i18n-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + i18n: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings + - run: cargo test -p craft-i18n --features collate --locked -- --nocapture diff --git a/.github/workflows/jpeg.yml b/.github/workflows/jpeg.yml new file mode 100644 index 000000000..900baf678 --- /dev/null +++ b/.github/workflows/jpeg.yml @@ -0,0 +1,46 @@ +name: Shared JPEG + +on: + push: + branches: [main] + paths: ["crates/jpeg/**", "Cargo.toml", "Cargo.lock", ".github/workflows/jpeg.yml"] + pull_request: + paths: ["crates/jpeg/**", "Cargo.toml", "Cargo.lock", ".github/workflows/jpeg.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-jpeg-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check: + timeout-minutes: 10 + name: JPEG lint, tests, WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-jpeg + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-jpeg --check + - run: cargo clippy -p craft-jpeg --all-targets --locked -- -D warnings + - run: cargo test -p craft-jpeg --locked + - run: cargo check -p craft-jpeg --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/keymap.yml b/.github/workflows/keymap.yml new file mode 100644 index 000000000..aba535265 --- /dev/null +++ b/.github/workflows/keymap.yml @@ -0,0 +1,115 @@ +name: Keymap + +on: + push: + branches: [main] + paths: ["crates/keymap/**", "crates/settings/**", "crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/keymap.yml"] + pull_request: + paths: ["crates/keymap/**", "crates/settings/**", "crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/keymap.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: keymap-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + linux: + timeout-minutes: 15 + name: Keymap (Linux) + runs-on: ubuntu-latest + env: + CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Checkout test fonts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: storytold/craft-fonts + ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + path: .test-inputs/craft-fonts + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-keymap-linux + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-keymap --check + - run: cargo clippy -p craft-keymap --all-targets --no-default-features --locked -- -D warnings + # `snapshot` covers `ui` and `egui` and compiles the dialog screenshots, which need a GPU + # to run, so the test steps leave them out. + - run: cargo clippy -p craft-keymap --all-targets --features snapshot --locked -- -D warnings + # Doctests run once, in the `ui` step. + - name: Test core + run: cargo test -p craft-keymap --no-default-features --lib --tests --locked + - name: Test egui dispatch and the shortcuts dialog + run: cargo test -p craft-keymap --features ui --locked + native: + timeout-minutes: 15 + name: Keymap (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["macos-latest","windows-latest"]') || fromJSON('["windows-latest"]') }} + runs-on: ${{ matrix.os }} + env: + CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Checkout test fonts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: storytold/craft-fonts + ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + path: .test-inputs/craft-fonts + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-keymap-${{ matrix.os }} + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo test -p craft-keymap --features ui --locked + wasm: + timeout-minutes: 10 + name: Keymap WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-keymap-wasm + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo check -p craft-keymap --features ui --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml new file mode 100644 index 000000000..cb3912d54 --- /dev/null +++ b/.github/workflows/media.yml @@ -0,0 +1,84 @@ +name: Shared media + +on: + push: + branches: [main] + paths: ["crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"] + pull_request: + paths: ["crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/media.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-media-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + timeout-minutes: 10 + name: Media lint, feature sets, WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-media-lint + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-media --check + - run: cargo clippy -p craft-media --all-targets --locked --no-default-features -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features bitstream -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features ogg -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings + - run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings + - run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown + test: + timeout-minutes: 15 + name: Media tests with ffmpeg oracles + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-media-test + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Build tests while installing ffmpeg + run: | + # The man-db index rebuild after dpkg is slow on hosted runners. + sudo rm -f /var/lib/man-db/auto-update + (sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg) > "$RUNNER_TEMP/apt.log" 2>&1 & + apt=$! + cargo test -p craft-media --all-features --locked --no-run + wait "$apt" || { cat "$RUNNER_TEMP/apt.log"; exit 1; } + ffmpeg -hide_banner -version | sed -n 1p + - run: cargo test -p craft-media --all-features --locked + env: + CRAFT_MEDIA_REQUIRE_ORACLES: "1" + CRAFT_MEDIA_FIXTURES_DIR: ${{ runner.temp }}/media-fixtures diff --git a/.github/workflows/menu.yml b/.github/workflows/menu.yml new file mode 100644 index 000000000..6d0e082cb --- /dev/null +++ b/.github/workflows/menu.yml @@ -0,0 +1,39 @@ +name: Menu + +on: + push: + branches: [main] + paths: ["crates/menu/**", "crates/keymap/**", "crates/quit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/menu.yml"] + pull_request: + paths: ["crates/menu/**", "crates/keymap/**", "crates/quit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/menu.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: menu-${{ github.ref }} + cancel-in-progress: true + +jobs: + linux: + timeout-minutes: 15 + name: Menu (Linux) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + targets: aarch64-apple-darwin, wasm32-unknown-unknown + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: craft-menu-linux + - run: cargo fmt -p craft-menu --check + - run: cargo clippy -p craft-menu --all-targets --locked -- -D warnings + # The macOS backend type-checks from Linux; it needs no linker or SDK. + - run: cargo clippy -p craft-menu --lib --locked --target aarch64-apple-darwin -- -D warnings + - run: cargo clippy -p craft-menu --lib --locked --target wasm32-unknown-unknown -- -D warnings + - run: cargo test -p craft-menu --locked diff --git a/.github/workflows/pcraft-format.yml b/.github/workflows/pcraft-format.yml new file mode 100644 index 000000000..177bb4593 --- /dev/null +++ b/.github/workflows/pcraft-format.yml @@ -0,0 +1,45 @@ +name: PhotoCraft format + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/pcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/pcraft-format.yml'] + pull_request: + paths: ['crates/pcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/pcraft-format.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: pcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + pcraft-format: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: "1.95" + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings + - run: cargo test -p craft-pcraft-format --locked -- --nocapture + - run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml index e0d9fcf1b..a494d085c 100644 --- a/.github/workflows/pdf.yml +++ b/.github/workflows/pdf.yml @@ -3,17 +3,20 @@ name: Shared PDF on: push: branches: [main] - paths: ["crates/pdf/**", "vendor/hayro*/**", "vendor/krilla/**", "scripts/test-pdf-backends.sh", "scripts/*fetch-pdf-test-fonts.py", "Cargo.toml", "Cargo.lock", ".github/workflows/pdf.yml"] + paths: ["crates/pdf/**", "crates/plate-test/**", "crates/prepress/**", "crates/cms/**", "vendor/hayro*/**", "vendor/krilla/**", "vendor/linesweeper/**","scripts/test-pdf-backends.sh", "scripts/*fetch-pdf-test-fonts.py", "Cargo.toml", "Cargo.lock", ".github/workflows/pdf.yml"] pull_request: - paths: ["crates/pdf/**", "vendor/hayro*/**", "vendor/krilla/**", "scripts/test-pdf-backends.sh", "scripts/*fetch-pdf-test-fonts.py", "Cargo.toml", "Cargo.lock", ".github/workflows/pdf.yml"] + paths: ["crates/pdf/**", "crates/plate-test/**", "crates/prepress/**", "crates/cms/**", "vendor/hayro*/**", "vendor/krilla/**", "vendor/linesweeper/**","scripts/test-pdf-backends.sh", "scripts/*fetch-pdf-test-fonts.py", "Cargo.toml", "Cargo.lock", ".github/workflows/pdf.yml"] workflow_dispatch: permissions: contents: read concurrency: - group: shared-pdf-${{ github.ref }} - cancel-in-progress: true + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-pdf-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: native: @@ -21,19 +24,26 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} runs-on: ${{ matrix.os }} env: RUSTFLAGS: --cfg craft_pdf_unicode_fixtures steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: craft-pdf - - uses: actions/setup-python@v5 + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Provision licensed PDF test fonts @@ -50,11 +60,18 @@ jobs: name: PDF vendored contracts runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: craft-pdf-backends + save-if: ${{ github.ref == 'refs/heads/main' }} - run: bash scripts/test-pdf-backends.sh wasm: name: PDF WASM runtime @@ -63,14 +80,20 @@ jobs: RUSTFLAGS: --cfg craft_pdf_unicode_fixtures CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable targets: wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: craft-pdf-wasm - - uses: actions/setup-python@v5 + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Provision licensed PDF test fonts @@ -79,10 +102,46 @@ jobs: python scripts/test-fetch-pdf-test-fonts.py python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - - name: Install the matching WASM test runner + - name: Read the wasm-bindgen version from Cargo.lock + id: wasm-bindgen shell: bash run: | runner_version=$(python3 -c 'import tomllib; d=tomllib.load(open("Cargo.lock", "rb")); print(next(p["version"] for p in d["package"] if p["name"] == "wasm-bindgen"))') - cargo install wasm-bindgen-cli --version "$runner_version" --locked + echo "version=$runner_version" >> "$GITHUB_OUTPUT" + - name: Install the matching WASM test runner + # A prebuilt, checksum-verified binary instead of building it from source on every run. + # A version the pinned action does not list yet falls back to `cargo install --locked`. + uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32 + with: + tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} + fallback: cargo-install - run: cargo test -p craft-pdf --all-features --locked --target wasm32-unknown-unknown - run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked --target wasm32-unknown-unknown + plate-test: + name: Plate tests and craft-prepress + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: clippy + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-plate-test + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Provision licensed PDF test fonts + shell: bash + run: | + python scripts/test-fetch-pdf-test-fonts.py + python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" + - run: cargo clippy -p craft-plate-test -p craft-prepress --all-targets --all-features --locked -- -D warnings + - run: cargo test -p craft-plate-test -p craft-prepress --all-features --locked diff --git a/.github/workflows/print.yml b/.github/workflows/print.yml index ea0696e3f..3838e6c42 100644 --- a/.github/workflows/print.yml +++ b/.github/workflows/print.yml @@ -8,18 +8,29 @@ on: paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml'] permissions: contents: read +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: print-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: print: timeout-minutes: 30 strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable components: rustfmt, clippy - run: cargo fmt --check - run: cargo clippy -p craft-print --all-targets -- -D warnings diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml new file mode 100644 index 000000000..0035b1c04 --- /dev/null +++ b/.github/workflows/proof.yml @@ -0,0 +1,77 @@ +name: Proofing + +on: + push: + branches: [main] + paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"] + pull_request: + paths: ["crates/proof/**", "Cargo.toml", "Cargo.lock", ".github/workflows/proof.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: craft-proof-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check: + timeout-minutes: 20 + name: Proofing lint, tests with and without Language data, WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The Language data (craft-i18n `language-data/`), embedded by crates/proof/build.rs. + # Pinned to storytold/craft-i18n#114's head (#109's SCOWL dictionaries + judgment/judgement, + # CP2.1); TODO(CP2.1): move to #114's merge commit once it merges. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: storytold/craft-i18n + ref: c239f934ace90b7939afaceb23b472ceeafd6b8a + path: craft-i18n + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-proof + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-proof --check + - name: Without Language data (a fork, docs.rs, offline) + run: | + cargo clippy -p craft-proof --all-targets --locked -- -D warnings + cargo test -p craft-proof --locked + cargo check -p craft-proof --locked --target wasm32-unknown-unknown + - name: A release without Language data fails + run: | + if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then + echo "::error::CRAFT_I18N_REQUIRED=1 without CRAFT_I18N_DIR built"; exit 1 + fi + grep -q "CRAFT_I18N_REQUIRED=1 but CRAFT_I18N_DIR is unset" build.log + - name: With Language data + env: + CRAFT_I18N_DIR: ${{ github.workspace }}/craft-i18n + CRAFT_I18N_REQUIRED: "1" + run: | + cargo clippy -p craft-proof --all-targets --locked -- -D warnings + cargo test -p craft-proof --locked + cargo check -p craft-proof --locked --target wasm32-unknown-unknown + - name: The English-only build (CRAFT_PROOF_EMBED=en, CP4.4's default once Language packs ship) + env: + CRAFT_I18N_DIR: ${{ github.workspace }}/craft-i18n + CRAFT_I18N_REQUIRED: "1" + CRAFT_PROOF_EMBED: en + run: cargo test -p craft-proof --locked diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml new file mode 100644 index 000000000..654fe4c02 --- /dev/null +++ b/.github/workflows/sonarcloud.yml @@ -0,0 +1,27 @@ +name: SonarCloud + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: sonarcloud-${{ github.ref }} + # A push to main is the run that publishes coverage; superseded runs there would never finish. + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + sonarcloud: + # Track shared CI improvements on main. + uses: storytold/craft-common-actions/.github/workflows/sonarcloud.yml@main + with: + pdf-test-fonts: true + # The codec and format code behind features that are off by default. Without these the + # coverage build never compiles it, and new code there reads as untested. + cargo-features: >- + craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write + secrets: inherit diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 79845a449..0892f5a2c 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -3,17 +3,20 @@ name: Shared UI on: push: branches: [main] - paths: ["crates/ui/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] + paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] pull_request: - paths: ["crates/ui/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] + paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] workflow_dispatch: permissions: contents: read concurrency: - group: shared-ui-${{ github.ref }} - cancel-in-progress: true + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-ui-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: native: @@ -22,22 +25,29 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 - name: Install software Vulkan renderer if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y mesa-vulkan-drivers libvulkan1 echo 'WGPU_BACKEND=vulkan' >> "$GITHUB_ENV" - - uses: dtolnay/rust-toolchain@stable + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 with: + toolchain: stable components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: craft-ui + save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo fmt --check - run: cargo clippy -p craft-ui --all-targets --locked -- -D warnings - run: cargo clippy -p craft-ui --all-targets --all-features --locked -- -D warnings @@ -48,11 +58,17 @@ jobs: name: UI WASM runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 with: + egress-policy: audit + - uses: actions/checkout@v7 + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable targets: wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: craft-ui-wasm + save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo check -p craft-ui --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml new file mode 100644 index 000000000..f9c8e1498 --- /dev/null +++ b/.github/workflows/update.yml @@ -0,0 +1,48 @@ +name: App updates + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/update/**', 'Cargo.toml', 'Cargo.lock', 'rustfmt.toml', 'clippy.toml', '.github/workflows/update.yml'] + pull_request: + paths: ['crates/update/**', 'Cargo.toml', 'Cargo.lock', 'rustfmt.toml', 'clippy.toml', '.github/workflows/update.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: update-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + update: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The token isn't needed after checkout; cargo runs build scripts and proc macros from + # dependencies, so don't leave it in .git/config for that third-party code to read. + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + - run: cargo fmt --check + # Default (no egui) and the egui feature both build and lint cleanly. + - run: cargo clippy -p craft-update --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-update --all-targets --locked --features egui -- -D warnings + - run: cargo test -p craft-update --locked -- --nocapture + - run: cargo test -p craft-update --locked --features egui -- --nocapture diff --git a/.github/workflows/vectorcraft-format.yml b/.github/workflows/vectorcraft-format.yml new file mode 100644 index 000000000..b2b20463a --- /dev/null +++ b/.github/workflows/vectorcraft-format.yml @@ -0,0 +1,45 @@ +name: VectorCraft format + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/vectorcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/vectorcraft-format.yml'] + pull_request: + paths: ['crates/vectorcraft-format/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/vectorcraft-format.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: vectorcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + vectorcraft-format: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: "1.95" + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - run: cargo fmt --check + - run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings + - run: cargo test -p craft-vectorcraft-format --locked -- --nocapture + - run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml new file mode 100644 index 000000000..ec1c16eda --- /dev/null +++ b/.github/workflows/video.yml @@ -0,0 +1,80 @@ +name: Shared video + +on: + push: + branches: [main] + paths: ["crates/video/**", "crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/video.yml"] + pull_request: + paths: ["crates/video/**", "crates/media/**", "crates/media-testkit/**", "Cargo.toml", "Cargo.lock", ".github/workflows/video.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: shared-video-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + timeout-minutes: 20 + name: Video lint, feature sets, docs, WASM + runs-on: ubuntu-latest + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: "1.95" + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-video-lint + - run: cargo fmt -p craft-video --check + # Each feature on its own, so code that needs another feature fails here. + - name: Clippy, each feature alone + run: | + for f in "" dnx vp8 prores h264 hevc vp9 vp9-decode vp9-encode av1 threads; do + echo "::group::features: ${f:-none}" + cargo clippy -p craft-video --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings + echo "::endgroup::" + done + - run: cargo clippy -p craft-video --all-targets --locked --all-features -- -D warnings + - run: cargo doc -p craft-video --all-features --no-deps --locked + env: + RUSTDOCFLAGS: "-D warnings" + - run: cargo check -p craft-video --all-features --locked --target wasm32-unknown-unknown + test: + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + name: Video tests (${{ matrix.os }}) + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: "1.95" + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-video-${{ matrix.os }} + - name: Install ffmpeg (Linux) + if: runner.os == 'Linux' + run: | + sudo rm -f /var/lib/man-db/auto-update + sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg + - run: cargo test -p craft-video --all-features --locked diff --git a/.github/workflows/web-api.yml b/.github/workflows/web-api.yml new file mode 100644 index 000000000..2460fb569 --- /dev/null +++ b/.github/workflows/web-api.yml @@ -0,0 +1,57 @@ +name: Web API (page command API) + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/web-api/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/web-api.yml'] + pull_request: + paths: ['crates/web-api/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/web-api.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: web-api-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + web-api: + # One OS: the crate is platform-independent JSON handling plus wasm32; no platform code. + timeout-minutes: 30 + runs-on: ubuntu-latest + env: + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - name: Read the wasm-bindgen version from Cargo.lock + id: wasm-bindgen + shell: bash + run: | + runner_version=$(python3 -c 'import tomllib; d=tomllib.load(open("Cargo.lock", "rb")); print(next(p["version"] for p in d["package"] if p["name"] == "wasm-bindgen"))') + echo "version=$runner_version" >> "$GITHUB_OUTPUT" + - name: Install the matching WASM test runner + # A prebuilt, checksum-verified binary instead of building it from source on every run. + # A version the pinned action does not list yet falls back to `cargo install --locked`. + uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32 + with: + tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} + fallback: cargo-install + - run: cargo fmt --check + - run: cargo clippy -p craft-web-api --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-web-api --all-targets --locked --target wasm32-unknown-unknown -- -D warnings + - run: cargo test -p craft-web-api --locked + - run: cargo test -p craft-web-api --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/web-host.yml b/.github/workflows/web-host.yml new file mode 100644 index 000000000..b780ee76b --- /dev/null +++ b/.github/workflows/web-host.yml @@ -0,0 +1,62 @@ +name: Shared web host + +on: + push: + branches: [main] + paths: ["crates/web-host/**", "Cargo.toml", "Cargo.lock", ".github/workflows/web-host.yml"] + pull_request: + paths: ["crates/web-host/**", "Cargo.toml", "Cargo.lock", ".github/workflows/web-host.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it + # always gets to save the cache. + group: shared-web-host-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + check: + timeout-minutes: 20 + name: Web host lint, tests, WASM tests + runs-on: ubuntu-latest + env: + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + targets: wasm32-unknown-unknown + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 + with: + key: craft-web-host + save-if: ${{ github.ref == 'refs/heads/main' }} + - run: cargo fmt -p craft-web-host --check + - run: cargo clippy -p craft-web-host --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-web-host --all-targets --locked --target wasm32-unknown-unknown -- -D warnings + - run: cargo test -p craft-web-host --locked + - name: Read the wasm-bindgen version from Cargo.lock + id: wasm-bindgen + shell: bash + run: | + runner_version=$(grep -A1 '^name = "wasm-bindgen"$' Cargo.lock | sed -n 's/^version = "\(.*\)"$/\1/p') + echo "version=$runner_version" >> "$GITHUB_OUTPUT" + - name: Install the matching WASM test runner + # A prebuilt, checksum-verified binary instead of building it from source on every run. + # A version the pinned action does not list yet falls back to `cargo install --locked`. + uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32 + with: + tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} + fallback: cargo-install + - run: cargo test -p craft-web-host --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/window.yml b/.github/workflows/window.yml new file mode 100644 index 000000000..f570ef5ce --- /dev/null +++ b/.github/workflows/window.yml @@ -0,0 +1,50 @@ +name: Window decorations + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/window/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/window.yml'] + pull_request: + paths: ['crates/window/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/window.yml'] + +permissions: + contents: read + +concurrency: + # Pull requests share one group per branch, so a newer push cancels the older run. Every other + # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. + group: window-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + window: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # macOS runners are kept for releases: macOS runs only when started by hand. + os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} + runs-on: ${{ matrix.os }} + steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + - uses: actions/checkout@v7 + with: + # The token isn't needed after checkout; cargo runs build scripts and proc macros from + # dependencies, so don't leave it in .git/config for that third-party code to read. + persist-credentials: false + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 + with: + toolchain: stable + components: rustfmt, clippy + - run: cargo fmt --check + # Default (no egui) and the egui feature both build and lint cleanly. + - run: cargo clippy -p craft-window --all-targets --locked -- -D warnings + - run: cargo clippy -p craft-window --all-targets --locked --features egui -- -D warnings + - run: cargo test -p craft-window --locked -- --nocapture + - run: cargo test -p craft-window --locked --features egui -- --nocapture + - run: cargo clippy -p craft-window --all-targets --locked --features restore -- -D warnings + - run: cargo test -p craft-window --locked --features restore,egui -- --nocapture diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml new file mode 100644 index 000000000..a7a3bb587 --- /dev/null +++ b/.github/workflows/workspace.yml @@ -0,0 +1,79 @@ +name: Workspace + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + native: + name: Workspace + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest","windows-latest","macos-latest"]' + timeout-minutes: 120 + tools: "" + packages: mesa-vulkan-drivers libvulkan1 + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + linux-env: WGPU_BACKEND=vulkan + windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1 + env: CRAFT_PRINT_REQUIRE_QUEUE=0 + setup-command: | + set -euo pipefail + corpus_fetcher=.test-inputs/photocraft + mkdir -p "$corpus_fetcher" + git -C "$corpus_fetcher" init + git -C "$corpus_fetcher" remote add origin https://github.com/storytold/PhotoCraft.git + git -C "$corpus_fetcher" fetch --quiet --depth=1 origin 74330157cb601573ae4523a95bb9132822fc3918 + git -C "$corpus_fetcher" checkout --quiet --detach FETCH_HEAD + (cd "$corpus_fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite) + mkdir -p corpus + cp -a "$corpus_fetcher/corpus/." corpus/ + python scripts/test-fetch-pdf-test-fonts.py + python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV" + printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" + command: | + cargo fmt --check + cargo clippy --workspace --all-targets --locked -- -D warnings + cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + cargo test --workspace --locked + cargo test --workspace --all-features --locked + windows-command: | + powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' + cargo fmt --check + cargo clippy --workspace --all-targets --locked -- -D warnings + cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + cargo test --workspace --locked + cargo test --workspace --all-features --locked + + msrv: + name: Rust 1.95 + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest"]' + toolchain: 1.95.0 + targets: "" + tools: "" + craft-fonts-ref: "" + require-craft-fonts: false + timeout-minutes: 15 + dev-debug: "" + command: cargo check --workspace --lib --all-features --locked + + wasm: + name: UI and settings WASM + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest"]' + targets: wasm32-unknown-unknown + tools: "" + craft-fonts-ref: "" + require-craft-fonts: false + timeout-minutes: 15 + command: cargo check -p craft-ui -p craft-settings --all-features --locked --target wasm32-unknown-unknown diff --git a/.gitignore b/.gitignore index 9108c660c..3c2518872 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ plan/ plans/ /target +# Also a sub-crate's own build directory: `crates/pdf/read-only-contract/target/` is not covered +# by the anchored `/target` above, and 803 of its files reached a commit once (#185 review). +target/ .DS_Store /.claude/ .idea/ +/vendor/*/target diff --git a/AGENTS.md b/AGENTS.md index 03e7ff50e..aa4561e67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,14 +1,63 @@ + +## House rules + +These apply in every Craft repository. A later rule may be stricter, not weaker. The source is `standards/house-rules-core.md` in `storytold/craft-contrib`. Propose changes there, not in a copy. + +1. **A person is accountable.** Whoever runs an agent answers for what it opens and for its review comments. +2. **Check, then claim.** Before writing code, read the issue comments, search pull requests for `#N` and the keywords, and check recent commits on `main`. Comment on the issue first. Release a claim you will not finish. +3. **Verify, don't guess.** Open a definition before using it. Confirm the cause in the code, even when someone names a location. +4. **Keep fixes narrow and tested.** Fix what the issue describes, where the bug is. Add a test that fails without the fix. +5. **Review the diff before opening a pull request.** Identifiers exist as used, the test fails without the fix, nothing unrelated changed, and the description is true. +6. **Look for a better pull request.** If another fixes the same thing, close yours when it is better or equal, and say why. +7. **Report only what ran.** Say what you tested and what you could not run. Without a local build or test, say so and rely on CI before marking ready. +8. **`main` changes through a pull request.** Do not push to it. Branch protection enforces this. There is no exception. +9. **Stop and ask only about** licensing outside the allowlist, publishing, releases, a new remote, secrets, spending money, destructive git, and anything else hard to undo. Otherwise keep working. `plan/` is a gitignored notebook, not rules. Without it, do not ask for it and do not stop. +10. **Keep secrets and personal details out** of the repository, logs and requests. +11. **CI stays shared.** Workflows call `storytold/craft-common-actions`. Do not change CI in a feature or fix pull request. If CI cannot cover a change, open an issue there. +12. **Guards say why.** A check that looks removable says what it prevents. Read that before changing it. +13. **Instructions must be readable by everyone.** No notes about one machine, and no links to files a reader cannot open. `craftrules` is retired. Do not link to it. +14. **Keep pull requests moving.** Concurrent pull requests are fine; unanswered actionable feedback is not. Respond in 7 days, a reminder at 14, closure eligible at 30, from when an actionable request waits on the author. Suggestions and unconfirmed bot findings do not start the clock. A fix, explanation or agreed plan resets it; unrelated activity does not. Waiting on a reviewer or maintainer pauses it and does not count against the author. A maintainer decides closure; it is never automatic. Reuse search results, do not poll, and stop at a rate limit. +15. **No agent attribution.** A pull request does not say an agent wrote it. A repository rule may not require that. + + +## Rust Craft repositories + +These apply to native Craft apps and to shared Rust libraries. Select them by repository type. A repository that merely contains some Rust does not take this block. Rules later in a repository's file may be stricter, but not weaker. The source is `standards/house-rules-rust.md` in [storytold/craft-contrib](https://github.com/storytold/craft-contrib). + +1. **Never crash.** Non-test code does not panic and has no `unsafe`, except where the repository file names an exception. Return errors, treat input as hostile, and bound recursion and allocations. New production crate roots copy the `deny` attribute from an existing crate: `clippy::unwrap_used`, `clippy::expect_used`, `clippy::panic`, `clippy::unimplemented`, `clippy::todo`, `clippy::unreachable`. Every crash fix comes with a test that panicked before the fix. +2. **Clean room.** Learn from observed behaviour and public specifications. Do not copy another product's code, assets, strings or artwork. Check a dependency's licence before adding it. +3. **Assets and fonts.** Every asset is original or suitably licensed, and is recorded in `ATTRIBUTION.md` in the same change. Font files live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), not in the app repository. +4. **Rust only for native Craft apps.** No handwritten JavaScript or TypeScript, and no Tauri, Electron or webview shell, in those apps. Shared tooling follows its documented technical scope and maintainer-approved exceptions. Craft PR Review's existing Tauri and TypeScript UI is one such exception. It does not relax this rule for native apps. Preserve any existing web build. +5. **Time goes through `chrono`.** Do not hand-roll calendar conversion. For dates beyond what `chrono` can represent, follow [standards/time.md](https://github.com/storytold/craft-contrib/blob/main/standards/time.md). +6. **Contributor credits.** Add or edit only your own person's entry in `contributors/people.toml`, and only after asking them. Never guess a name or hand-edit `contributors.json`. + +## Licence allowlist + +Dependencies and assets use SPDX identifiers. An expression is allowed when every identifier is in this list. A repository may name a shorter list. It may not add an identifier here by editing its own copy of these rules. + +- `ISC` +- `MIT` +- `BSD-2-Clause` +- `BSD-3-Clause` +- `Apache-2.0` +- `OFL-1.1` +- `Ubuntu-font-1.0` +- `Bitstream-Vera` +- `CC0-1.0` +- `CC-BY-4.0` +- `CC-BY-SA-4.0` +- `LicenseRef-Public-Domain` + +`LicenseRef-Storyteller-Trademark` is not in this list. It covers Storyteller brand marks only, where a repository already says so. + + # AGENTS.md -Shared Rust libraries for the Craft Apps. The shared standards live in -[craftrules](https://github.com/storytold/craftrules) (`../../craftrules` on a dev machine): read its -`AGENTS.md`, then `standards/never-crash.md`, `standards/engineering.md` and `standards/licensing.md`. +Shared Rust libraries for the Craft Apps. -- **Never crash.** Every crate's `lib.rs` starts with - `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]`. - No `unsafe` (the workspace forbids it). - **New crates** go in `crates/`, named `craft-`, inherit the workspace package fields - and lints, and are listed in `[workspace.dependencies]`. + and lints, and are listed in `[workspace.dependencies]`. A new crate's `Cargo.toml` includes + `[lints] workspace = true`. - **No app code.** A crate here must be useful to more than one app and must not depend on any app. - **Before merging:** `cargo fmt --check`, `cargo clippy --workspace --all-targets -- -D warnings`, `cargo test --workspace`. diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 34cd2ee23..32bae195d 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -6,9 +6,32 @@ author, source and license. Library code and original assets are under the repos | File | Author | Source | License | |---|---|---|---| +| `licenses/dwg/acadrust-MPL-2.0.txt`, `licenses/dwg/acadrust-THIRD_PARTY_NOTICES.md` | `acadrust` contributors; ACadSharp authors for the noted ported routines | [acadrust 0.6.6](https://github.com/hakanaktt/acadrust), used unmodified as `craft-dwg`'s optional native dependency | MPL-2.0; embedded ACadSharp notice is MIT; see both retained files | | `crates/cms/profiles/photocraft-coated-cmyk.icc` | PhotoCraft contributors; generated by `craft_cms::synth::coated_cmyk` | [PhotoCraft `crates/cms`](https://github.com/storytold/photocraft/tree/b3222ee8e7e6bcb4799c37f904c595f20f9baf5e/crates/cms) | CC0-1.0 | | `crates/cms/profiles/craft-coated-fogra39.icc` | craft-libs contributors; generated by `crates/cms/tests/fogra39.rs` from the FOGRA39 data below | this repository | CC0-1.0 | +| `crates/font-store/index/sources.json` (unmodified) | craft-font-index contributors | [aiongg/craft-font-index@5a9a7f8](https://github.com/aiongg/craft-font-index/tree/5a9a7f8038c8f3e2e1583a79c92a816e8e71b34f) | Apache-2.0 | +| `crates/font-store/index/indexes/google-fonts.json.gz` (unmodified) | craft-font-index contributors; generated from the METADATA.pb files of [google/fonts](https://github.com/google/fonts) at `45e3d5269e4a17d823f6d84470bc084dc5ae46f6` | [aiongg/craft-font-index@5a9a7f8](https://github.com/aiongg/craft-font-index/tree/5a9a7f8038c8f3e2e1583a79c92a816e8e71b34f) | Apache-2.0 | +| `crates/font-store/index/indexes/noto.json.gz` (unmodified) | craft-font-index contributors; generated from [notofonts/notofonts.github.io](https://github.com/notofonts/notofonts.github.io) at `93fac8abebe6c7fed0c5b7d2456d8fa484fa146c` and [noto-cjk](https://github.com/notofonts/noto-cjk) releases | [aiongg/craft-font-index@5a9a7f8](https://github.com/aiongg/craft-font-index/tree/5a9a7f8038c8f3e2e1583a79c92a816e8e71b34f) | Apache-2.0 | | `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) | +| `vendor/linesweeper/tests/regression/`, `vendor/linesweeper/proptest-regressions/`, the `.snap` files under `vendor/linesweeper/src/` and the path data in `vendor/linesweeper/benches/painted_dreams.rs` (all unmodified; hashes in `vendor/linesweeper/SOURCE.json`) | As published in the package, whose manifest names Joe Neeman as author and records no other source | [linesweeper 0.5.0](https://crates.io/crates/linesweeper/0.5.0) | MIT OR Apache-2.0 (the package's `license` field) | +| `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 | +| `crates/text/assets/hyphen/en-us.dic` | Grady Ward (Moby Hyphenator II), converted by DesignCraft | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204) | Public domain | +| `crates/proof/data/en-words.dic` | Grady Ward (Moby Hyphenator II word list); break points dropped by craft-proof contributors with `crates/proof/examples/en_words.rs` | [Project Gutenberg eBook #3204](https://www.gutenberg.org/ebooks/3204), via craft-i18n `language-data/en-us/en-us.dic` | Public domain | +| `crates/proof/data/en-extra.txt` | WordCraft contributors (original list, no other dictionary copied) | craft-i18n `language-data/en-us/en-extra.txt` | MIT OR Apache-2.0 | +| `crates/text/assets/hyphen/en-us.pat` | DesignCraft contributors, trained from the Moby Hyphenator II list | `crates/text/examples/hyphgen.rs` | CC0-1.0 | +| Patterns embedded through `hypher` (optional `craft-text` language features) | hyph-utf8 authors, per language | [tex-hyphen](https://github.com/hyphenation/tex-hyphen), as packaged in [hypher](https://github.com/typst/hypher/tree/main/patterns) | Per file header: LPPL, MIT, BSD-3-Clause, Unlicense or a permit-any-use notice (or a choice including one); Slovak (`hyph-sk.tex`, embedded whenever `hypher/slovak` is on: feature `slovak-gpl`, or another dependency through feature unification) is GPL-2.0-or-later; Spanish (`hyph-es.tex` 5.0, Javier Bezos, CervanTeX) is MIT | +| Word-break dictionaries and LSTM models embedded through `icu_segmenter_data` (optional `craft-text` features `thai`, `lao`, `khmer`, `burmese`) | Unicode, Inc. and ICU4X contributors | [ICU4X](https://github.com/unicode-org/icu4x) | Unicode-3.0 | +| `crates/ui/tests/fonts/NotoSansArabic-test.ttf` (subset to the test strings, default weight set to 100, width axis dropped) | The Noto Project Authors | [google/fonts@7085eb8 `ofl/notosansarabic`](https://github.com/google/fonts/tree/7085eb89a950e85db5b166b7a58d414544b4140c/ofl/notosansarabic), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansArabic.txt`) | +| `crates/ui/tests/fonts/NotoSansCJKsc-test.otf` (subset to the test strings) | The Noto Project Authors (Adobe, Google) | [notofonts/noto-cjk@f8d1575 `Sans/OTF/SimplifiedChinese`](https://github.com/notofonts/noto-cjk/tree/f8d157532fbfaeda587e826d4cd5b21a49186f7c/Sans/OTF/SimplifiedChinese), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansCJK.txt`) | +| `crates/ui/tests/fonts/NotoSansArabic-screens.ttf` (instance at weight 400 and width 100, subset to the screen test strings) | The Noto Project Authors | [google/fonts@7085eb8 `ofl/notosansarabic`](https://github.com/google/fonts/tree/7085eb89a950e85db5b166b7a58d414544b4140c/ofl/notosansarabic), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansArabic.txt`) | +| `crates/ui/tests/fonts/NotoSansCJKsc-screens.otf` (subset to printable ASCII and the screen test strings) | The Noto Project Authors (Adobe, Google) | [notofonts/noto-cjk@f8d1575 `Sans/OTF/SimplifiedChinese`](https://github.com/notofonts/noto-cjk/tree/f8d157532fbfaeda587e826d4cd5b21a49186f7c/Sans/OTF/SimplifiedChinese), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansCJK.txt`) | +| `crates/ui/tests/fonts/NotoSansThai-screens.ttf` (instance at weight 400 and width 100, subset to the screen test strings) | The Noto Project Authors | [google/fonts@8b0a1d0 `ofl/notosansthai`](https://github.com/google/fonts/tree/8b0a1d0f5983c89bc2b93f1b5fb55f9e252744b5/ofl/notosansthai), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansThai.txt`) | +| `crates/crw/tests/samples.json` (manifest only; the 11 raw files are not bundled, `scripts/fetch-crw-samples.py` downloads them outside the checkout) | The photographers who contributed each file to raw.pixls.us (Canon EOS D30, D60, 10D ×3, 300D ×2, Digital Rebel, PowerShot G1, G5, G6) | [raw.pixls.us](https://raw.pixls.us/), each file's `getfile.php` URL and SHA-256 in the manifest | CC0-1.0 (as published by raw.pixls.us for each file) | +| `crates/crw/src/tables.rs`, `crates/crw/clean-room/logs/table_sel*.json` | craft-crw contributors (derived in LightCraft, 2026-10, by known-plaintext alignment of the sample files' entropy streams; `crates/crw/clean-room/02-crw-format.md`) | this repository | MIT OR Apache-2.0 | +| `crates/crw/clean-room/logs/private-corpus.tsv` (file names, sizes and SHA-256 of the maintainer's own EOS D60 files; no image content) | craft-crw contributors (LightCraft derivation record) | this repository | MIT OR Apache-2.0 | +| `crates/prc/src/acof/table.rs` (the rows of the PRC `Double` prefix-code table, reformatted as Rust tuples) | Kristof Ralovich (prc-rs); the table is defined by ISO 14739-1 | [ralovich/prc-rs@287b671](https://github.com/ralovich/prc-rs/blob/287b671bb5b31b05efb5ec1b98f7c359b9e4dd7d/src/double.rs), through [PdfCraft `crates/prc`](https://github.com/storytold/pdfcraft/tree/80e26e674d1aa33078e83c8139875047f3289eb6/crates/prc) | MIT (`licenses/prc-rs/LICENSE.txt`; the notice is also in the file header) | +| `crates/math/tests/corpus/equations.tsv`, `crates/math/tests/corpus/latex.tsv` (unmodified) | shuff57 (equaSHion) | [storytold/craft-libs#29](https://github.com/storytold/craft-libs/pull/29) at `7a6ef6b9`, `corpus/` | MIT OR Apache-2.0 | +| `crates/math/tests/corpus/omml/*.xml` (unmodified) | WordCraft and DeckCraft contributors, collected by shuff57 (equaSHion) | [storytold/craft-libs#29](https://github.com/storytold/craft-libs/pull/29) at `7a6ef6b9`, `corpus/omml/` | MIT OR Apache-2.0 | Original assets are covered by the repository licence. @@ -28,14 +51,15 @@ The profile licence is retained in `licenses/pdf-writer/LICENSE-ICC.txt`. | `krilla 0.8.2 / icc/sRGB-v4.icc` | Clinton Ingram | https://github.com/LaurenzV/krilla/blob/3ffdf0588cf98050aad6edba51ca70162e1fb5b5/crates/krilla/icc/sRGB-v4.icc | CC0-1.0 | `c56e1685d888f5edb92fe07f2750f387f8fe8e91b32ff8fb0b56bfbbb9458353` | PDF writer colour profiles | The four Krilla ICC entries above are also stored unchanged in `vendor/krilla/icc`. -The vendored source patch and per-file hashes are recorded in `vendor/krilla/README.craft.md` and `vendor/krilla/SOURCE.json`. +The vendored source patches are in `vendor/krilla/patches`, described in `vendor/krilla/README.craft.md`; per-file hashes are in `vendor/krilla/SOURCE.json`. ## Assets retained with the optional PDF reader -The shared reader centralizes the four existing patched Hayro source trees from PdfCraft. +The shared reader centralizes the five patched Hayro source trees from PdfCraft and the +patched crates.io `hayro-postscript` 0.1.0 scanner, which bundles no assets. The standard-14 fonts, CMYK profile and generated metrics listed below remain byte-identical -in `vendor/hayro-interpret`; no application document fonts are added. CMaps remain a pinned -registry dependency. The removed upstream LAB.icc profile is not restored. Source hashes +in `vendor/hayro-interpret`, and the CMaps in `vendor/hayro-cmap/assets`; no application +document fonts are added. The removed upstream LAB.icc profile is not restored. Source hashes and the exact transfer revision are recorded in each vendored crate's SOURCE.json. | Crate / file | Author | Source | Licence | SHA-256 | Use | @@ -55,5 +79,18 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso | `hayro-interpret 0.7.0 / assets/FoxitSerifItalic.pfb` | Foxit Software Inc. / PDFium Authors | https://github.com/storytold/pdfcraft/blob/7437bde378f43856eada40af25b983812e615e8a/vendor/hayro-interpret/assets/FoxitSerifItalic.pfb | BSD-3-Clause | `610ae0687198045c4db5d1a6650fd5e92536631706382eb8b72e38df578d0ae9` | Renders non-embedded standard-14 fonts (pinned hayro-interpret) | | `hayro-interpret 0.7.0 / assets/FoxitSymbol.pfb` | Foxit Software Inc. / PDFium Authors | https://github.com/storytold/pdfcraft/blob/7437bde378f43856eada40af25b983812e615e8a/vendor/hayro-interpret/assets/FoxitSymbol.pfb | BSD-3-Clause | `47967d055530e7357088a08403115425643ec2cdfd6201ba8af0fbd7116c1539` | Renders non-embedded standard-14 fonts (pinned hayro-interpret) | | `hayro-interpret 0.7.0 / assets/CGATS001Compat-v2-micro.icc` | Clinton Ingram (saucecontrol/Compact-ICC-Profiles) | https://github.com/storytold/pdfcraft/blob/7437bde378f43856eada40af25b983812e615e8a/vendor/hayro-interpret/assets/CGATS001Compat-v2-micro.icc | CC0-1.0 | `73e1ba37d2bad5bab2a964f40a9eed96209666efc067c3322626214bbef234a0` | Default CMYK colour conversion (pinned hayro-interpret) | -| `hayro-cmap 0.1.0 / assets/cmaps.brotli` | Adobe | https://crates.io/crates/hayro-cmap/0.1.0 | BSD-3-Clause | `ddf37af521d904d5646c2fc37b2b7e912dc1b56f0d210190b61f598154758e46` | Character-code mapping tables required to read CJK PDFs. Non-visual dependency data; existing embedding unchanged | +| `hayro-cmap 0.1.0 / assets/cmaps.brotli` | Adobe | https://github.com/storytold/pdfcraft/blob/a2ec27b4775f53c40cad3456936d9406611e3a8d/vendor/hayro-cmap/assets/cmaps.brotli | BSD-3-Clause | `ddf37af521d904d5646c2fc37b2b7e912dc1b56f0d210190b61f598154758e46` | Character-code mapping tables required to read CJK PDFs. Non-visual dependency data; existing embedding unchanged | | `hayro-interpret 0.7.0 / src/font/generated/metrics.rs` | Adobe (Core 14 AFM metrics, PDF specification encodings); tables by The Hayro Authors | https://github.com/storytold/pdfcraft/blob/7437bde378f43856eada40af25b983812e615e8a/vendor/hayro-interpret/src/font/generated/metrics.rs | MIT OR Apache-2.0 | `2ec9e608573eda459de1e1d9bda3bb494315e36bee885acc88912a11fe0caa41` | Glyph widths and encodings for non-embedded standard fonts. Non-visual dependency data; existing embedding unchanged | +| `crates/audio/src/ac3/golden_frames.bin` | FilmCraft contributors (original work: 12 AC-3 and E-AC-3 syncframes that ffmpeg's `ac3` and `eac3` encoders made from tones and seeded noise synthesised by ffmpeg's lavfi generators; no third-party or recorded audio) | [FilmCraft `crates/ac3`](https://github.com/storytold/filmcraft/tree/1c9c2e82/crates/ac3/src) | MIT OR Apache-2.0 | +| `crates/audio/tests/fixtures/` | SoundCraft and craft-libs contributors (original work: 0.1 s test tones from ffmpeg's `sine` generator and hand-built silent MPEG frames; no third-party or recorded audio; commands in `crates/audio/tests/fixtures/README.md`) | [SoundCraft `crates/audio-io/tests/fixtures`](https://github.com/storytold/soundcraft/tree/e86d09a/crates/audio-io/tests/fixtures); `tone.mov`, `tone.mka`, `tone.opus`, `tone.ac3` and `tone.eac3` made in this repository | MIT OR Apache-2.0 | +| `crates/image-codecs/src/codecs/xpm/x11_colors.rs` | X.Org Foundation and X11 contributors (X11 `rgb.txt` colour names and values, normalised into a lookup table) | , via [PhotoCraft `crates/codecs`](https://github.com/storytold/photocraft/tree/5000240c6/crates/codecs/src/codecs/xpm) | MIT (X.Org app/rgb) | +| `crates/raster-codecs/src/xpm/x11_colors.rs` | X.Org Foundation and X11 contributors (X11 `rgb.txt` colour names and values, normalised into a lookup table) | , via [PhotoCraft `crates/codecs`](https://github.com/storytold/photocraft/tree/63b641b55e54bf8f5c83efd977fdb16a2c255431/crates/codecs/src/codecs/xpm) | MIT (X.Org app/rgb) | +| `crates/psd/tests/fixtures/atn/*.atn` (seven synthetic action sets, encoded field by field; `crates/psd/tests/fixtures/atn/README.md`) | PhotoCraft contributors | [PhotoCraft `crates/psd/tests/fixtures/atn`](https://github.com/storytold/photocraft/tree/63b641b55e54bf8f5c83efd977fdb16a2c255431/crates/psd/tests/fixtures/atn) | MIT OR Apache-2.0 | +| `crates/lottie/tests/fixtures/` | EffectCraft contributors (original work: four hand-written Lottie JSON documents, written from the public Lottie format description; not exported from any Adobe product or third-party file) | [EffectCraft `crates/lottie/tests/fixtures`](https://github.com/storytold/effectcraft/tree/e18f48fb/crates/lottie/tests/fixtures) | MIT OR Apache-2.0 | +| `crates/pkcs7/tests/data/` | PdfCraft contributors (original work: throw-away test keys and certificates made with OpenSSL 3 for these tests, as seven PKCS #12 files with the password `test` and two PEM certificates; they protect nothing; commands in `crates/pkcs7/tests/data/README.md`) | [PdfCraft `crates/sign/tests/data`](https://github.com/storytold/pdfcraft/tree/1d7a9f9f/crates/sign/tests/data) | MIT OR Apache-2.0 | +| `crates/affinity/tests/support/corpus.tsv`, `crates/affinity/tests/support/affinity3-signatures.txt` (manifest and measured signatures only; the 51 documents are not bundled, `scripts/fetch-affinity-corpus.sh` downloads them outside the checkout) | The authors of each document: samuel-etver (vector-art), Seth A. Robinson (Patchy fixtures), Nick Beeuwsaert (AFDesignLoad), Jeremy Cantu (Branding), eviltwo (AssetStoreTemplate), Square (leakcanary) | each file's pinned upstream URL and SHA-256 in the manifest; list carried over from [VectorCraft `xtask/src/affinity_corpus.rs`](https://github.com/storytold/vectorcraft/blob/6e873b806c9d5b3f2b55f651df57bd97933b579a/xtask/src/affinity_corpus.rs) | CC0-1.0, MIT or Apache-2.0 (per file, in the manifest) | +| `crates/raw/docs/coverage/adobe-cameras.tsv` | Facts (maker, model, extension, profile, minimum Camera Raw version) parsed by LightCraft contributors from Adobe's published list | [Adobe Camera Raw supported cameras](https://helpx.adobe.com/camera-raw/desktop/dng-and-file-formats/camera-raw-plug-supported-cameras.html), Wayback capture 20261003133011; moved from [LightCraft `docs/raw`](https://github.com/storytold/lightcraft/tree/5ff567a4a2e067bef1b6bce152c11e7f727ccbaf/docs/raw) | Facts, not copyrightable expression; the file is MIT OR Apache-2.0 | +| `crates/raw/docs/coverage/{coverage,coverage-sweep,evidence,usage-weights}.tsv`, `coverage.py` | LightCraft contributors | [LightCraft `docs/raw`](https://github.com/storytold/lightcraft/tree/5ff567a4a2e067bef1b6bce152c11e7f727ccbaf/docs/raw) | MIT OR Apache-2.0 | +| `crates/raw/docs/raf-corpus.sha256` (hashes only; the files are not bundled) | The photographers who contributed each file to raw.pixls.us | [raw.pixls.us](https://raw.pixls.us/) | CC0-1.0 | +| `crates/drawingml/src/preset_text.txt` | DeckCraft contributors (Nikita Rakitin); generated by `crates/drawingml/tests/fixtures/make_preset_guides.py` from the guide formulas of ECMA-376 Part 1 `presetShapeDefinitions.xml` | [storytold/deckcraft#650](https://github.com/storytold/deckcraft/pull/650) | MIT OR Apache-2.0 (the formulas are the format's definition in ECMA-376) | +| `crates/drawingml/tests/fixtures/preset_table.txt` | DeckCraft contributors (Nikita Rakitin); generated by `crates/drawingml/tests/fixtures/make_preset_table.py` from ECMA-376 Part 1 `presetShapeDefinitions.xml` | [storytold/deckcraft#650](https://github.com/storytold/deckcraft/pull/650) | MIT OR Apache-2.0 (the formulas are the format's definition in ECMA-376) | diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..31364ecf9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + +All crates share the workspace version. See [the versioning policy](docs/versioning.md). + +## v0.1.0 — 2026-10-12 + +First coordinated Git release of craft-libs. Applications can pin a named baseline instead of +following development commits. This is an initial 0.x release: compatibility is maintained +within `0.1.x`; changes requiring consumer migration use a new minor version. + +### Included crates + +- **craft-atomic:** same-directory staged file replacement, exclusive creation, streaming + writes, transient rename retries and fault-injection support, extracted from PhotoCraft. +- **craft-settings:** app-owned TOML settings with platform/portable paths, atomic saves, + backups, damaged-file recovery, schema migrations and preservation of unknown fields. +- **craft-quit:** native macOS termination, optional egui close handling and an optional + application menu. Save decisions and prompts stay with the app. +- **craft-print:** CUPS and native Windows PDF submission, printer discovery, copies, duplex, + colour and scaling options. Includes bounded PDF sheet handling and the pinned Windows + completion fix. PDF generation stays with the app. +- **craft-ui:** egui icon buttons, tabs, splits, resize handles, toolbar overflow, transactional + docking, theme helpers and print-settings controls. Commands, content, localization and + application state stay with the consumer. Docking serialization is opt-in through `serde`. + +### Requirements and adoption + +- Rust **1.95+**, edition 2024. Public UI integrations use **egui 0.36**. +- Use `git = "https://github.com/storytold/craft-libs", tag = "v0.1.0"` for each required crate; + use the same tag everywhere. The [README](README.md#use-a-released-version) has a complete + manifest example and a commit-pin alternative. Commit application lockfiles. +- No automatic migration from an arbitrary earlier Git revision is implied. Compare the API + you used with this release and test affected workflows before changing pins. +- For `craft-quit`, run the close hook last in `eframe::App::logic`, after the app's unsaved-work + decision. Finish asynchronous saves before requesting close; termination does not execute + code after `run_native`. See the crate's API documentation before adoption. +- This release is distributed through Git only. No crates.io packages are published by it. + +### Platform coverage and known limits + +The release gate checks the workspace on Linux, macOS and Windows with default and all features, +plus Linux minimum-compiler and UI/settings WASM checks. UI tests include offscreen visual +regressions. Windows print tests exercise a temporary paused spooler queue; those checks do +not establish physical printer fidelity or every application's native GUI behavior. + +- `craft-ui` supports native and WASM use. Font coverage, bidirectional text and screen-reader + behavior still depend on the application and upstream stack; see [#17](https://github.com/storytold/craft-libs/issues/17). +- `craft-settings` persistence is native-only. WASM exposes settings traits, document helpers + and error types. Concurrent writers are best-effort, last-writer-wins; strict coordination + needs an application-owned lock. +- `craft-atomic` does not preserve all filesystem metadata during replacement. Directory sync + is best effort; tests do not prove physical power-loss durability. See its README's contract. +- `craft-quit`'s native behavior is macOS-specific; other platforms keep their normal close + path. Earlier hardware testing of the close hook is recorded in + [#3](https://github.com/storytold/craft-libs/issues/3); it does not qualify every consuming app. +- `craft-print` requires CUPS command-line tools on Linux/macOS or the native Windows spooler. + Browser submission is unsupported. Windows accepts a bounded subset of print-ready PDFs; + unsupported inputs and driver paper/scaling mismatches return errors. Printing is synchronous, + so call it from a worker. Physical output needs hardware testing. +- Windows printing retains a pinned Git dependency on the maintained `winprint` fork. Its + build dependencies require the license/advisory policy exceptions documented in + [craft-print's README](crates/print/README.md). Registry publication is deferred. + +Planned crates and open feature PRs are not part of this release. Existing commit pins remain +valid, and consumers do not need to update simply because `main` changes. diff --git a/CLAUDE.md b/CLAUDE.md index 11908a75d..43c994c2d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1 +1 @@ -Read `AGENTS.md`. +@AGENTS.md diff --git a/Cargo.lock b/Cargo.lock index 8461f6b35..a1b1ccab3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,22 +3,48 @@ version = 4 [[package]] -name = "accesskit" -version = "0.24.1" +name = "acadrust" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b7f7f85a7e5f68090000ed7622545829afd484d210358702ae4cb97dd0c320" +checksum = "8923c99ea1f151cb7fc17ce2fc14f8b1e330fa22611e535602d985c0cc3c89a2" dependencies = [ + "anyhow", + "bitflags 2.13.2", + "byteorder", + "encoding_rs", + "flate2", + "foldhash 0.2.0", + "indexmap", + "itoa", + "memmap2", + "nom 7.1.3", + "oem_cp", + "once_cell", + "rayon", + "ryu", + "thiserror 2.0.21", + "web-time", +] + +[[package]] +name = "accesskit" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad442f58ee04714aaa0ba0a2768c1ea1935b29507bb22ddece8cbbc76db02932" +dependencies = [ + "enumn", + "serde", "uuid", ] [[package]] name = "accesskit_consumer" -version = "0.35.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53cf47daed85312e763fbf85ceca136e0d7abc68e0a7e12abe11f48172bc3b10" +checksum = "8a45e2bb300bea6d5f33a82c3e051c3ddc6f41c209750f68a0acbe991a284344" dependencies = [ "accesskit", - "hashbrown 0.16.1", + "hashbrown 0.17.1", ] [[package]] @@ -27,6 +53,62 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "bytes", + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.1", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes 0.8.4", + "cipher 0.4.4", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-kw" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69fa2b352dcefb5f7f3a5fb840e02665d311d878955380515e4fd50095dd3d8c" +dependencies = [ + "aes 0.8.4", +] + [[package]] name = "ahash" version = "0.8.12" @@ -35,23 +117,57 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "once_cell", + "serde", "version_check", "zerocopy", ] +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +[[package]] +name = "alloc-no-stdlib" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" + [[package]] name = "alloc-stdlib" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" dependencies = [ - "alloc-no-stdlib", + "alloc-no-stdlib 2.0.4", +] + +[[package]] +name = "alloc-stdlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" +dependencies = [ + "alloc-no-stdlib 3.0.0", ] [[package]] @@ -60,6 +176,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "android-activity" version = "0.6.1" @@ -95,23 +217,39 @@ dependencies = [ ] [[package]] -name = "arboard" -version = "3.6.1" +name = "anyhow" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ - "clipboard-win", - "image", - "log", - "objc2 0.6.5", - "objc2-app-kit 0.3.2", - "objc2-core-foundation", - "objc2-core-graphics", - "objc2-foundation 0.3.2", - "parking_lot", - "percent-encoding", - "windows-sys 0.60.2", - "x11rb", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.2.17", + "password-hash", + "zeroize", ] [[package]] @@ -125,6 +263,24 @@ name = "arrayvec" version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +dependencies = [ + "serde", +] + +[[package]] +name = "as-raw-xcb-connection" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] [[package]] name = "ash" @@ -135,6 +291,126 @@ dependencies = [ "libloading", ] +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.5", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.5", +] + +[[package]] +name = "async-recursion" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.5", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.92" @@ -158,6 +434,49 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey", + "rayon", + "thiserror 2.0.21", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + [[package]] name = "aws-lc-rs" version = "1.18.1" @@ -182,6 +501,18 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.22.1" @@ -206,7 +537,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", +] + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec 0.10.1", ] [[package]] @@ -215,6 +555,43 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + +[[package]] +name = "bit_field" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb2907cc03d08cdb1c6724ad0670ce67c4a71e1c45a6d7635224011714cfc4b1" + +[[package]] +name = "bitfields" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6e59298da389bc0649c7463856b34c6e17fe542f88939426ede4436c6b1195" +dependencies = [ + "bitfields-impl", +] + +[[package]] +name = "bitfields-impl" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c044f98f86f15414668d6c8187c7e4fadab1ad2b31680f648703e0fe07c555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "thiserror 2.0.21", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -227,6 +604,49 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -236,6 +656,33 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -254,15 +701,97 @@ dependencies = [ "objc2 0.6.5", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "blowfish" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" +dependencies = [ + "byteorder", + "cipher 0.4.4", +] + +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "bp256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f98c2d9e8e0796b24f84b7ae14159b91ae6ae4b85f74f7655a678a56ad5e1e" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "primefield", + "primeorder 0.14.0", +] + +[[package]] +name = "bp384" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc61f3a96b44a33fbb971e9917b81f4207d6dae9a6b0715845c785a48e73d9c2" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "primefield", + "primeorder 0.14.0", +] + [[package]] name = "brotli" version = "8.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", + "alloc-no-stdlib 2.0.4", + "alloc-stdlib 0.2.4", + "brotli-decompressor 5.0.3", +] + +[[package]] +name = "brotli" +version = "9.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" +dependencies = [ + "alloc-no-stdlib 3.0.0", + "alloc-stdlib 0.3.0", + "brotli-decompressor 6.3.1", ] [[package]] @@ -271,10 +800,35 @@ version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "alloc-no-stdlib 2.0.4", + "alloc-stdlib 0.2.4", ] +[[package]] +name = "brotli-decompressor" +version = "6.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f" +dependencies = [ + "alloc-no-stdlib 3.0.0", + "alloc-stdlib 0.3.0", +] + +[[package]] +name = "buffer-redux" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "431a9cc8d7efa49bc326729264537f5e60affce816c66edf434350778c9f4f54" +dependencies = [ + "memchr", +] + +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + [[package]] name = "bumpalo" version = "3.20.3" @@ -301,6 +855,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "byteorder-lite" version = "0.1.0" @@ -328,27 +888,31 @@ dependencies = [ ] [[package]] -name = "calloop" -version = "0.14.5" +name = "camellia" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b32313e937c8721c9d17ee9bf60699a6402b4b7c2cb6ac1bee0ffcaa4b4e5e" +checksum = "3264e2574e9ef2b53ce6f536dea83a69ac0bc600b762d1523ff83fe07230ce30" dependencies = [ - "bitflags 2.13.2", - "polling", - "rustix 1.1.5", - "tracing", + "byteorder", + "cipher 0.4.4", ] [[package]] -name = "calloop-wayland-source" -version = "0.4.1" +name = "cap-primitives" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" dependencies = [ - "calloop 0.14.5", + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", "rustix 1.1.5", - "wayland-backend", - "wayland-client", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", ] [[package]] @@ -357,6 +921,24 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cast5" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b07d673db1ccf000e90f54b819db9e75a8348d6eb056e9b8ab53231b7a9911" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "cc" version = "1.6.0" @@ -369,6 +951,26 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfb" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f13298d9178622b7916ec5d01d470425c04273c8ed07e87b1c38b34c4f4612b" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + +[[package]] +name = "cfb-mode" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "738b8d467867f80a71351933f70461f5b56f24d5c93e0cf216e59229c968d330" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -381,6 +983,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + [[package]] name = "chacha20" version = "0.10.2" @@ -392,6 +1003,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chardetng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.45" @@ -407,12 +1029,34 @@ dependencies = [ ] [[package]] -name = "clipboard-win" -version = "5.4.1" +name = "cipher" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "error-code", + "crypto-common 0.1.7", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.6", +] + +[[package]] +name = "cmac" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8543454e3c3f5126effff9cd44d562af4e31fb8ce1cc0d3dcd8f084515dbc1aa" +dependencies = [ + "cipher 0.4.4", + "dbl", + "digest 0.10.7", ] [[package]] @@ -424,6 +1068,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cmpv2" version = "0.2.0" @@ -431,8 +1081,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "961b955a666e25ee5a1091d219128d6e6401e3dab84efb1a2bf6b4035d797b39" dependencies = [ "crmf", - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", "x509-cert", ] @@ -442,12 +1092,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ - "const-oid", - "der", - "spki", + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", "x509-cert", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -474,6 +1133,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "combine" version = "4.6.8" @@ -499,6 +1168,27 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -564,6 +1254,12 @@ dependencies = [ "libm", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -582,6 +1278,57 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-aaf" +version = "0.1.0" +dependencies = [ + "craft-audio", + "craft-cfb", + "craft-timeline", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-aep" +version = "0.1.0" + +[[package]] +name = "craft-affinity" +version = "0.1.0" +dependencies = [ + "crc32fast", + "flate2", + "image", + "proptest", + "ruzstd", + "sha2 0.10.9", +] + +[[package]] +name = "craft-ai-core" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", +] + +[[package]] +name = "craft-anicraft-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "craft-assuan" +version = "0.1.0" +dependencies = [ + "proptest", + "thiserror 2.0.21", +] + [[package]] name = "craft-atomic" version = "0.1.0" @@ -589,28 +1336,1006 @@ dependencies = [ "rustix 1.1.5", ] +[[package]] +name = "craft-audio" +version = "0.1.0" +dependencies = [ + "craft-media", + "craft-media-testkit", + "log", + "proptest", + "rayon", + "serde", + "serde_json", + "symphonia", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-captions" +version = "0.1.0" +dependencies = [ + "craft-timeline", + "craft-xml", + "proptest", + "roxmltree 0.21.2", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-cfb" +version = "0.1.0" +dependencies = [ + "craft-media-testkit", +] + [[package]] name = "craft-cms" version = "0.1.0" dependencies = [ + "md-5 0.10.6", "moxcms", "rayon", ] +[[package]] +name = "craft-codepage" +version = "0.1.0" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "craft-crs" +version = "0.1.0" +dependencies = [ + "craft-metadata", + "craft-xml", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-crw" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-csv" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-dav" +version = "0.1.0" +dependencies = [ + "craft-xml", + "proptest", +] + +[[package]] +name = "craft-designcraft-format" +version = "0.1.0" +dependencies = [ + "craft-document-bundle", + "craft-eps", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-dialogs" +version = "0.1.0" +dependencies = [ + "block2 0.6.2", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "raw-window-handle", + "rfd", +] + +[[package]] +name = "craft-doc" +version = "0.1.0" +dependencies = [ + "blake3", + "craft-ecproj-format", + "craft-pcraft-format", + "craft-vectorcraft-format", + "flate2", + "proptest", + "ruzstd", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-doc-binary" +version = "0.1.0" +dependencies = [ + "cfb", + "chrono", + "craft-cfb", + "craft-codepage", + "craft-wordmodel", + "image", + "log", + "miniz_oxide 0.9.1", + "proptest", + "serde_json", + "sha1 0.11.0", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-docpath" +version = "0.1.0" + +[[package]] +name = "craft-document-bundle" +version = "0.1.0" +dependencies = [ + "craft-zip", + "serde_json", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-docx" +version = "0.1.0" +dependencies = [ + "base64 0.23.1", + "chrono", + "craft-cfb", + "craft-drawingml", + "craft-math", + "craft-offcrypto", + "craft-opc", + "craft-wordmodel", + "craft-xml", + "image", + "log", + "proptest", + "quick-xml 0.41.0", + "serde_json", + "thiserror 2.0.21", + "unicode-bidi", + "zeroize", + "zip", +] + +[[package]] +name = "craft-dpx" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-drawingml" +version = "0.1.0" +dependencies = [ + "craft-xml", + "kurbo 0.13.1", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "craft-dwg" +version = "0.1.0" +dependencies = [ + "acadrust", + "log", +] + +[[package]] +name = "craft-dxf" +version = "0.1.0" +dependencies = [ + "craft-codepage", + "encoding_rs", + "kurbo 0.13.1", + "log", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-ecproj-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-eps" +version = "0.1.0" +dependencies = [ + "kurbo 0.13.1", + "thiserror 2.0.21", + "vectorcraft-doc", + "vectorcraft-eps", +] + +[[package]] +name = "craft-epub" +version = "0.1.0" +dependencies = [ + "craft-xml", + "craft-zip", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-ews" +version = "0.1.0" +dependencies = [ + "craft-xml", + "proptest", +] + +[[package]] +name = "craft-exr" +version = "0.1.0" +dependencies = [ + "exr", + "flate2", + "half", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-fig" +version = "0.1.0" +dependencies = [ + "craft-figma-rest", + "craft-kiwi", + "craft-zip", + "flate2", + "kurbo 0.13.1", + "ruzstd", + "serde_json", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-figcraft-format" +version = "0.1.0" +dependencies = [ + "craft-zip", + "serde", + "serde_json", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-figma-rest" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-file-formats" +version = "0.1.0" +dependencies = [ + "craft-eps", + "craft-idml", + "craft-image-codecs", + "craft-kra", + "craft-ora", + "craft-pcraft-bundle", + "craft-raw", + "craft-svg", + "craft-zip", + "serde", + "serde_json", +] + +[[package]] +name = "craft-file-import" +version = "0.1.0" +dependencies = [ + "craft-dxf", + "craft-eps", + "craft-file-formats", + "craft-pdf 0.1.0", + "craft-raster-codecs", + "craft-svg", + "kurbo 0.13.1", + "thiserror 2.0.21", + "vectorcraft-geom", +] + +[[package]] +name = "craft-flags" +version = "0.1.0" +dependencies = [ + "craft-atomic", + "dirs", + "tempfile", + "toml_edit", +] + +[[package]] +name = "craft-flow" +version = "0.1.0" +dependencies = [ + "craft-wordmodel", + "image", + "log", +] + +[[package]] +name = "craft-font-catalog" +version = "0.1.0" +dependencies = [ + "fontique", + "skrifa 0.47.0", + "thiserror 2.0.21", + "windows-registry", + "write-fonts 0.54.1", +] + +[[package]] +name = "craft-font-manager" +version = "0.1.0" +dependencies = [ + "craft-font-catalog", + "craft-font-store", + "craft-i18n", + "craft-ui", + "egui", + "egui_kittest", + "egui_system_font_provider", + "epaint_default_fonts", + "image", + "log", + "serde", +] + +[[package]] +name = "craft-font-store" +version = "0.1.0" +dependencies = [ + "cap-primitives", + "craft-atomic", + "craft-font-catalog", + "flate2", + "log", + "ruzstd", + "serde", + "serde_json", + "sha2 0.10.9", + "skrifa 0.47.0", + "tar", + "thiserror 2.0.21", + "toml", + "ureq", + "write-fonts 0.54.1", +] + +[[package]] +name = "craft-fonts-build" +version = "0.1.0" +dependencies = [ + "craft-fonts-build-consumer", + "skera", + "skrifa 0.48.0", +] + +[[package]] +name = "craft-fonts-build-consumer" +version = "0.1.0" +dependencies = [ + "craft-fonts-build", +] + +[[package]] +name = "craft-gainmap" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-gcode" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-gif" +version = "0.1.0" +dependencies = [ + "gif", + "image", +] + +[[package]] +name = "craft-hdr" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-heif" +version = "0.1.0" +dependencies = [ + "heic-rs", + "rayon", +] + +[[package]] +name = "craft-html" +version = "0.1.0" +dependencies = [ + "craft-codepage", + "craft-flow", + "craft-math", + "craft-mime", + "craft-wordmodel", + "encoding_rs", + "proptest", +] + +[[package]] +name = "craft-i18n" +version = "0.1.0" +dependencies = [ + "fixed_decimal", + "icu_collator", + "icu_decimal", + "icu_locale_core", + "icu_locale_fallback", + "icu_normalizer", + "icu_plurals", + "icu_properties", + "icu_provider", + "icu_provider_adapters", + "icu_provider_blob", + "log", + "objc2-foundation 0.3.2", + "sys-locale", + "unicode-bidi", + "windows-registry", + "writeable", +] + +[[package]] +name = "craft-ical" +version = "0.1.0" +dependencies = [ + "chrono", + "proptest", +] + +[[package]] +name = "craft-idml" +version = "0.1.0" +dependencies = [ + "craft-xml", + "craft-zip", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-image-codecs" +version = "0.1.0" +dependencies = [ + "craft-heif", + "craft-jpeg", + "craft-metadata", + "exr", + "flate2", + "half", + "image", + "image-webp", + "jpeg-encoder 0.6.1", + "jxl-oxide", + "png 0.18.1", + "proptest", + "rayon", + "serde_json", + "thiserror 2.0.21", + "tiff", + "zune-core", + "zune-jpeg", + "zune-jpegxl", +] + +[[package]] +name = "craft-indd" +version = "0.1.0" +dependencies = [ + "indd", +] + +[[package]] +name = "craft-jmap" +version = "0.1.0" +dependencies = [ + "proptest", + "serde_json", +] + +[[package]] +name = "craft-jpeg" +version = "0.1.0" +dependencies = [ + "jpeg-decoder", + "jpeg-encoder 0.6.1", + "rayon", + "thiserror 2.0.21", + "zune-jpeg", +] + +[[package]] +name = "craft-jxl" +version = "0.1.0" +dependencies = [ + "craft-cms", + "jixel", + "jxl-oxide", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-keymap" +version = "0.1.0" +dependencies = [ + "craft-i18n", + "craft-settings", + "craft-ui", + "egui", + "egui_kittest", + "image", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", +] + +[[package]] +name = "craft-kiwi" +version = "0.1.0" + +[[package]] +name = "craft-kra" +version = "0.1.0" +dependencies = [ + "craft-xml", + "craft-zip", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-latex" +version = "0.1.0" +dependencies = [ + "craft-flow", + "craft-markdown", + "craft-math", + "craft-mime", + "craft-wordmodel", + "proptest", +] + +[[package]] +name = "craft-lottie" +version = "0.1.0" +dependencies = [ + "craft-zip", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-markdown" +version = "0.1.0" +dependencies = [ + "craft-flow", + "craft-html", + "craft-wordmodel", +] + +[[package]] +name = "craft-materialx" +version = "0.1.0" +dependencies = [ + "craft-xml", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-math" +version = "0.1.0" +dependencies = [ + "craft-xml", + "proptest", + "serde", + "serde_json", +] + +[[package]] +name = "craft-media" +version = "0.1.0" +dependencies = [ + "chrono", + "craft-media-testkit", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-media-testkit" +version = "0.1.0" + +[[package]] +name = "craft-menu" +version = "0.1.0" +dependencies = [ + "craft-keymap", + "craft-quit", + "egui", + "muda", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", +] + +[[package]] +name = "craft-mesh" +version = "0.1.0" +dependencies = [ + "craft-opc", + "craft-xml", + "miniz_oxide 0.9.1", + "serde_json", +] + +[[package]] +name = "craft-metadata" +version = "0.1.0" +dependencies = [ + "brotli 9.1.1", + "brotli-decompressor 6.3.1", + "chrono", + "craft-tiff", + "proptest", + "quick-xml 0.41.0", + "serde", + "serde_json", +] + +[[package]] +name = "craft-metafile" +version = "0.1.0" +dependencies = [ + "encoding_rs", + "image", + "kurbo 0.13.1", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-midi" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-mime" +version = "0.1.0" +dependencies = [ + "chardetng", + "chrono", + "craft-codepage", + "encoding_rs", + "proptest", +] + +[[package]] +name = "craft-odf" +version = "0.1.0" +dependencies = [ + "aes 0.9.4", + "base64 0.23.1", + "chrono", + "craft-flow", + "craft-math", + "craft-metafile", + "craft-wordmodel", + "craft-xml", + "craft-zip", + "flate2", + "getrandom 0.3.4", + "hmac 0.13.0", + "image", + "js-sys", + "log", + "sha1 0.11.0", + "sha2 0.11.1", + "thiserror 2.0.21", + "zeroize", + "zip", +] + +[[package]] +name = "craft-ods" +version = "0.1.0" +dependencies = [ + "craft-odf", + "craft-sheet", + "craft-xml", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-offcrypto" +version = "0.1.0" +dependencies = [ + "aes 0.9.4", + "base64 0.23.1", + "craft-cfb", + "craft-xml", + "getrandom 0.3.4", + "hmac 0.13.0", + "sha1 0.11.0", + "sha2 0.11.1", + "thiserror 2.0.21", + "zeroize", +] + +[[package]] +name = "craft-opc" +version = "0.1.0" +dependencies = [ + "craft-xml", + "craft-zip", + "log", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-openpgp" +version = "0.1.0" +dependencies = [ + "ed25519-dalek 2.2.0", + "pgp", + "proptest", + "rand 0.8.8", + "rsa 0.9.10", + "sha1 0.10.7", + "sha2 0.10.9", + "sha3 0.10.9", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-ora" +version = "0.1.0" +dependencies = [ + "craft-xml", + "craft-zip", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-pcraft-bundle" +version = "0.1.0" +dependencies = [ + "blake3", + "craft-atomic", + "craft-pcraft-format", + "craft-zip", + "ruzstd", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-pcraft-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-pdf" version = "0.1.0" dependencies = [ + "aes 0.9.4", "bytemuck", + "chrono", + "craft-stream-codecs", + "flate2", "hayro", "hayro-cmap", "hayro-interpret", "hayro-syntax", + "js-sys", "krilla", "kurbo 0.13.1", + "md-5 0.11.0", + "proptest", + "roxmltree 0.21.2", + "serde_json", + "sha2 0.11.1", + "stringprep", "thiserror 2.0.21", "ttf-parser", + "wasm-bindgen", "wasm-bindgen-test", + "zeroize", +] + +[[package]] +name = "craft-pdf" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=92d392cff132eaf53e13afff77f38b5c11b73aa0#92d392cff132eaf53e13afff77f38b5c11b73aa0" +dependencies = [ + "hayro-interpret", + "hayro-syntax", + "krilla", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-pdn" +version = "0.1.0" +dependencies = [ + "flate2", + "proptest", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-penpot" +version = "0.1.0" +dependencies = [ + "craft-zip", + "serde", + "serde_json", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-pkcs7" +version = "0.1.0" +dependencies = [ + "aes 0.9.4", + "aws-lc-rs", + "base64 0.23.1", + "bp256", + "bp384", + "cbc", + "chrono", + "crypto-bigint 0.7.11", + "des 0.9.0", + "ecdsa 0.17.0", + "ed25519-dalek 3.0.0", + "getrandom 0.4.3", + "hmac 0.13.0", + "p256 0.14.0", + "p384 0.14.0", + "p521 0.14.0", + "pbkdf2", + "rc2", + "ripemd 0.2.0", + "rsa 0.10.0-rc.18", + "sha1 0.11.0", + "sha2 0.11.1", + "sha3 0.12.0", + "thiserror 2.0.21", + "zeroize", +] + +[[package]] +name = "craft-plate-test" +version = "0.1.0" +dependencies = [ + "craft-cms", + "craft-pdf 0.1.0", + "hayro-syntax", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-platform" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-io", + "async-process", + "egui", + "futures-lite", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", + "zbus", +] + +[[package]] +name = "craft-plotter" +version = "0.1.0" +dependencies = [ + "serde", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-pptx" +version = "0.1.0" +dependencies = [ + "craft-drawingml", + "craft-opc", + "craft-xml", + "kurbo 0.13.1", + "log", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-prc" +version = "0.1.0" +dependencies = [ + "flate2", + "proptest", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-prepress" +version = "0.1.0" +dependencies = [ + "craft-cms", + "craft-pdf 0.1.0", + "craft-plate-test", + "kurbo 0.13.1", + "linesweeper", + "thiserror 2.0.21", + "ttf-parser", ] [[package]] @@ -618,13 +2343,57 @@ name = "craft-print" version = "0.1.0" dependencies = [ "craft-winprint-gdi", - "miniz_oxide 0.8.9", - "roxmltree", + "log", + "miniz_oxide 0.9.1", + "roxmltree 0.21.2", "tempfile", "winprint", "winsafe", ] +[[package]] +name = "craft-proof" +version = "0.1.0" +dependencies = [ + "log", + "miniz_oxide 0.9.1", + "proptest", + "toml", + "unicode-normalization", +] + +[[package]] +name = "craft-ps" +version = "0.1.0" +dependencies = [ + "craft-eps", + "kurbo 0.13.1", + "miniz_oxide 0.8.9", + "pdf-writer", + "skrifa 0.47.0", + "weezl 0.1.12", +] + +[[package]] +name = "craft-psd" +version = "0.1.0" +dependencies = [ + "craft-psd", + "encoding_rs", + "flate2", + "proptest", + "rayon", + "serde", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-pst" +version = "0.1.0" +dependencies = [ + "proptest", +] + [[package]] name = "craft-quit" version = "0.1.0" @@ -636,6 +2405,66 @@ dependencies = [ "objc2-app-kit 0.3.2", ] +[[package]] +name = "craft-raster-codecs" +version = "0.1.0" +dependencies = [ + "base64 0.23.1", + "craft-image-codecs", + "image", + "rayon", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-raw" +version = "0.1.0" +dependencies = [ + "chrono", + "craft-crw", + "craft-metadata", + "craft-tiff", + "jpeg-encoder 0.7.1", + "jxl-oxide", + "miniz_oxide 0.9.1", + "proptest", + "rayon", + "serde", + "sha2 0.10.9", + "thiserror 2.0.21", + "zune-core", + "zune-jpeg", + "zune-jpegxl", +] + +[[package]] +name = "craft-rtf" +version = "0.1.0" +dependencies = [ + "craft-codepage", + "craft-flow", + "craft-metafile", + "craft-mime", + "craft-wordmodel", + "encoding_rs", + "image", + "proptest", +] + +[[package]] +name = "craft-scraft-format" +version = "0.1.0" +dependencies = [ + "craft-audio", + "craft-midi", + "craft-time", + "craft-zip", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-settings" version = "0.1.0" @@ -648,25 +2477,375 @@ dependencies = [ ] [[package]] -name = "craft-ui" +name = "craft-sf2" version = "0.1.0" dependencies = [ - "egui", - "egui_extras", - "egui_kittest", - "image", + "craft-sf2", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-sheet" +version = "0.1.0" + +[[package]] +name = "craft-silhouette-studio" +version = "0.1.0" +dependencies = [ + "kurbo 0.13.1", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-sketch" +version = "0.1.0" +dependencies = [ + "craft-figma-rest", + "craft-zip", + "kurbo 0.13.1", + "serde_json", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "craft-stencilcraft-document" +version = "0.1.0" +dependencies = [ + "craft-atomic", + "craft-stencilcraft-format", + "craft-svg", + "kurbo 0.13.1", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-stencilcraft-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-stream-codecs" +version = "0.1.0" +dependencies = [ + "fax", + "hayro-ccitt", + "hayro-jbig2", + "proptest", + "thiserror 2.0.21", + "weezl 0.2.1", +] + +[[package]] +name = "craft-svg" +version = "0.1.0" +dependencies = [ + "craft-image-codecs", + "craft-xml", + "flate2", + "kurbo 0.13.1", + "resvg 0.48.1", + "roxmltree 0.21.2", + "thiserror 2.0.21", + "usvg 0.48.1", + "vectorcraft-geom", + "vectorcraft-pathops", +] + +[[package]] +name = "craft-svg-paths" +version = "0.1.0" +dependencies = [ + "craft-svg", + "kurbo 0.13.1", +] + +[[package]] +name = "craft-text" +version = "0.1.0" +dependencies = [ + "hypher", + "icu_segmenter", + "miniz_oxide 0.8.9", + "unicode-bidi", + "unicode-bidi-mirroring", + "unicode-general-category", + "unicode-linebreak", + "unicode-segmentation", +] + +[[package]] +name = "craft-tga" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-tiff" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-time" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-timeline" +version = "0.1.0" +dependencies = [ + "craft-time", + "craft-xml", + "proptest", "serde", "serde_json", ] +[[package]] +name = "craft-ui" +version = "0.1.0" +dependencies = [ + "craft-fonts-build", + "craft-i18n", + "eframe", + "egui", + "egui_extras", + "egui_kittest", + "epaint", + "fontique", + "image", + "js-sys", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "serde", + "serde_json", + "skrifa 0.44.0", + "unicode-script", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", +] + +[[package]] +name = "craft-update" +version = "0.1.0" +dependencies = [ + "chrono", + "dunce", + "egui", + "egui_kittest", + "epaint_default_fonts", + "flate2", + "getrandom 0.3.4", + "image", + "log", + "plist", + "rustix 1.1.5", + "semver", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "ureq", + "verifysign", + "winreg", + "winsafe", + "zip", +] + +[[package]] +name = "craft-vcard" +version = "0.1.0" +dependencies = [ + "chrono", + "craft-mime", + "proptest", +] + +[[package]] +name = "craft-vectorcraft-format" +version = "0.1.0" +dependencies = [ + "flate2", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-vectordrawable" +version = "0.1.0" +dependencies = [ + "craft-xml", + "kurbo 0.13.1", +] + +[[package]] +name = "craft-video" +version = "0.1.0" +dependencies = [ + "craft-media", + "craft-media-testkit", + "rayon", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-web-api" +version = "0.1.0" +dependencies = [ + "js-sys", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", +] + +[[package]] +name = "craft-web-host" +version = "0.1.0" +dependencies = [ + "wasm-bindgen", + "wasm-bindgen-test", + "web-sys", +] + +[[package]] +name = "craft-webp" +version = "0.1.0" +dependencies = [ + "craft-video", + "image", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-window" +version = "0.1.0" +dependencies = [ + "craft-atomic", + "craft-settings", + "egui", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "serde", + "serde_json", + "tempfile", + "wayland-client", +] + [[package]] name = "craft-winprint-gdi" version = "0.1.0" -source = "git+https://github.com/storytold/winprint.rs?rev=417ac3c4d14d3da5485a99c7d5ce8f05664ec312#417ac3c4d14d3da5485a99c7d5ce8f05664ec312" +source = "git+https://github.com/storytold/winprint.rs?rev=64e7afd4d1483512385012c340ecaaaf0f3beb7f#64e7afd4d1483512385012c340ecaaaf0f3beb7f" dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "craft-wordmodel" +version = "0.1.0" +dependencies = [ + "base64 0.23.1", + "craft-math", + "log", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", + "unicode-segmentation", +] + +[[package]] +name = "craft-xcf" +version = "0.1.0" +dependencies = [ + "flate2", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-xfl" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-xls" +version = "0.1.0" +dependencies = [ + "craft-cfb", + "craft-sheet", + "md-5 0.10.6", + "miniz_oxide 0.9.1", + "sha1 0.10.7", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-xlsx" +version = "0.1.0" +dependencies = [ + "craft-drawingml", + "craft-opc", + "craft-sheet", + "craft-xml", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-xml" +version = "0.1.0" +dependencies = [ + "quick-xml 0.41.0", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-y4m" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "craft-zip" +version = "0.1.0" +dependencies = [ + "craft-zip", + "flate2", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "crc24" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd121741cf3eb82c08dd3023eb55bf2665e5f60ec20f89760cf836ae4562e6a0" + [[package]] name = "crc32fast" version = "1.5.2" @@ -683,8 +2862,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36fe21b96d5b87f5de4b5b7202ec41c00110ac817ce6728fe75fb2fe5962ed92" dependencies = [ "cms", - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", "x509-cert", ] @@ -728,6 +2907,35 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914c195c4d7fbc5ed7f88d14cbf85914d5c664967658d43404b263913a477e82" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "serdect 0.4.3", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -735,31 +2943,187 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "crypto-primes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" +dependencies = [ + "crypto-bigint 0.7.11", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", + "subtle", +] + [[package]] name = "cursor-icon" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "cx448" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c0cf476284b03eb6c10e78787b21c7abb7d7d43cb2f02532ba6b831ed892fa" +dependencies = [ + "crypto-bigint 0.5.5", + "elliptic-curve 0.13.8", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "serdect 0.3.0", + "sha3 0.10.9", + "signature 2.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + [[package]] name = "data-url" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" +[[package]] +name = "dbl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9" +dependencies = [ + "generic-array", +] + [[package]] name = "der" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" +dependencies = [ + "const-oid 0.10.3", + "pem-rfc7468 1.0.0", "zeroize", ] @@ -774,14 +3138,119 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "des" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher 0.5.2", +] + +[[package]] +name = "dify" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ce0fb972943b4e88cd03b8f92953df0c71bb05e0bde8e5b684895d808013cc" +dependencies = [ + "anyhow", + "colored", + "getopts", + "image", + "rayon", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.3", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -818,6 +3287,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.13.2", + "block2 0.6.2", + "libc", "objc2 0.6.5", ] @@ -862,6 +3333,22 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dsa" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48bc224a9084ad760195584ce5abb3c2c34a225fa312a128ad245a6b412b7689" +dependencies = [ + "digest 0.10.7", + "num-bigint-dig", + "num-traits", + "pkcs8 0.10.2", + "rfc6979 0.4.0", + "sha2 0.10.9", + "signature 2.2.0", + "zeroize", +] + [[package]] name = "dunce" version = "1.0.5" @@ -869,20 +3356,108 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] -name = "ecolor" -version = "0.36.2" +name = "eax" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0656a7530b9e2841ee7a5f983965bdd9840c32f4012f759cfd44b912cced6b34" +checksum = "9954fabd903b82b9d7a68f65f97dc96dd9ad368e40ccc907a7c19d53e6bfac28" +dependencies = [ + "aead", + "cipher 0.4.4", + "cmac", + "ctr", + "subtle", +] + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der 0.7.10", + "digest 0.10.7", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", +] + +[[package]] +name = "ecdsa" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" +dependencies = [ + "der 0.8.3", + "digest 0.11.3", + "elliptic-curve 0.14.3", + "rfc6979 0.6.0", + "signature 3.0.0", + "spki 0.8.1", + "zeroize", +] + +[[package]] +name = "ecolor" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "bytemuck", "emath", + "serde", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.2.0", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "signature 3.0.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek 5.0.0", + "ed25519 3.0.0", + "sha2 0.11.1", + "subtle", + "zeroize", ] [[package]] name = "eframe" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ad41fac6e3149abb4866e93df37dbd33baa95db525c3368c944584e95a5dc2" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "ahash", "bytemuck", @@ -890,6 +3465,9 @@ dependencies = [ "egui", "egui-wgpu", "egui-winit", + "egui_glow", + "glutin", + "glutin-winit", "image", "js-sys", "log", @@ -898,9 +3476,10 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "pollster", + "pollster 1.0.1", "profiling", "raw-window-handle", + "send_wrapper", "static_assertions", "wasm-bindgen", "web-sys", @@ -912,19 +3491,19 @@ dependencies = [ [[package]] name = "egui" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc938cc27cd911415e1e4d151bc56ff9df063d8db081374e87aeb805ea74b2ba" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "accesskit", "ahash", "bitflags 2.13.2", "emath", "epaint", - "itertools", + "itertools 0.15.0", "log", "nohash-hasher", "profiling", + "serde", "smallvec", "unicode-segmentation", "web-sys", @@ -932,9 +3511,8 @@ dependencies = [ [[package]] name = "egui-wgpu" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cef1f6d1bba40b09a4d0d62cb822b81030922cfbf65c5373398c3a3caafc5bd" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "ahash", "bytemuck", @@ -952,57 +3530,85 @@ dependencies = [ [[package]] name = "egui-winit" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98466000559d66ba4db786a30448b01483946add4188e7f10f522a27b4fb89cc" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ - "arboard", + "block2 0.6.2", "bytemuck", "egui", "log", "objc2 0.6.5", + "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "objc2-ui-kit 0.3.2", "profiling", "raw-window-handle", - "smithay-clipboard", "web-time", + "windows-sys 0.61.2", "winit", ] [[package]] name = "egui_extras" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cbdc785020861553f230f23d330ba1fb72613b9d8e0e457f3bdc8ba6db6dcf" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "ahash", "egui", "enum-map", - "itertools", + "itertools 0.15.0", "log", "profiling", - "resvg", + "resvg 0.45.1", +] + +[[package]] +name = "egui_glow" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" +dependencies = [ + "bytemuck", + "egui", + "glow", + "log", + "profiling", + "winit", ] [[package]] name = "egui_kittest" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb3f3effb2726fee23449293e67acbf6cc6ce35525b1722bd5743bb348114d5" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ + "dify", "eframe", "egui", "egui-wgpu", "image", "kittest", "log", - "pollster", + "open", + "pollster 1.0.1", "serde", + "tempfile", "toml", "wgpu", ] +[[package]] +name = "egui_system_font_provider" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" +dependencies = [ + "epaint", + "fontique", + "log", + "poll-promise", + "skrifa 0.44.0", + "sys-locale", + "unicode-script", +] + [[package]] name = "either" version = "1.19.0" @@ -1010,14 +3616,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] -name = "emath" -version = "0.36.2" +name = "elliptic-curve" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c8d4141cf0f60fb2f7aefaf352d3ffa5171ef0ab09731c64391801e95e48f1" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct 0.2.0", + "base64ct", + "crypto-bigint 0.5.5", + "digest 0.10.7", + "ff 0.13.1", + "generic-array", + "group 0.13.0", + "hkdf", + "pem-rfc7468 0.7.0", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "sec1 0.7.3", + "serde_json", + "serdect 0.2.0", + "subtle", + "tap", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.11", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", + "subtle", + "zeroize", +] + +[[package]] +name = "emath" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "bytemuck", + "serde", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encoding_rs" version = "0.8.42" @@ -1033,19 +3696,57 @@ dependencies = [ ] [[package]] -name = "enum-map" -version = "2.7.3" +name = "endi" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enum-map" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4532960034c15ba2f55e9eee631040ecc473c32e6c4b200828c759840f13a06" dependencies = [ "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.17.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +checksum = "7127272f64b3e098f7d81f350d672081b879bff896d219c42370dc858b1af9ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enumn" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" dependencies = [ "proc-macro2", "quote", @@ -1054,34 +3755,55 @@ dependencies = [ [[package]] name = "epaint" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ec0308dc23130fc1a7d4623a8534c2ed069c6dc84e258ee4f3de99615aa1a3" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" dependencies = [ "ahash", "bytemuck", "ecolor", "emath", "epaint_default_fonts", - "font-types 0.12.6", - "harfrust", + "font-types 0.12.7", + "harfrust 0.12.0", "log", "nohash-hasher", "parking_lot", "profiling", "self_cell", + "serde", "skrifa 0.44.0", "smallvec", + "unicode-bidi", "unicode-general-category", + "unicode-properties", "unicode-segmentation", - "vello_cpu 0.1.0", + "vello_cpu 0.2.0", ] [[package]] name = "epaint_default_fonts" -version = "0.36.2" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" + +[[package]] +name = "equator" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "equivalent" @@ -1099,12 +3821,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "error-code" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" - [[package]] name = "euclid" version = "0.22.14" @@ -1114,6 +3830,48 @@ dependencies = [ "num-traits", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "exr" +version = "1.74.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb29d0be702adf92c645d9b2f93472906ba83d400e59f98adb3770ce5c18c2f6" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide 0.8.9", + "multiversion", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.5.0" @@ -1141,6 +3899,39 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "bitvec", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "filetime" version = "0.2.29" @@ -1157,6 +3948,17 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fixed_decimal" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2214fb00f669ee96bee5bb29dfb1ddabe058b2bc426817efc79afa03c9bc55ae" +dependencies = [ + "displaydoc", + "smallvec", + "writeable", +] + [[package]] name = "flagset" version = "0.4.7" @@ -1211,6 +4013,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -1228,11 +4036,55 @@ dependencies = [ [[package]] name = "font-types" -version = "0.12.6" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c92d9081c7ab3bd83ef1eecdfad090f3d966e3ce74106779664114cfe1a6274" +checksum = "835ae9d781464a79026cde387f9cbc650c9a4909d7cf7e658b127e3c35d2f35d" dependencies = [ "bytemuck", + "serde", +] + +[[package]] +name = "fontconfig-parser" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc773e24e02d4ddd8395fd30dc147524273a83e54e0f312d986ea30de5f5646" +dependencies = [ + "roxmltree 0.20.0", +] + +[[package]] +name = "fontdb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2660c5e9157bf76d2db1294e4a9feba604ef610819a3b591088d0d8392a3290f" +dependencies = [ + "fontconfig-parser", + "log", + "slotmap", + "tinyvec", +] + +[[package]] +name = "fontique" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6688bc1294fe7117d788937b6c53480169b29c566954af490830d4c09da9516a" +dependencies = [ + "hashbrown 0.17.1", + "linebender_resource_handle", + "memmap2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-text", + "objc2-foundation 0.3.2", + "parlance", + "read-fonts 0.41.0", + "roxmltree 0.21.2", + "smallvec", + "windows", + "windows-core", + "yeslogic-fontconfig-sys", ] [[package]] @@ -1271,12 +4123,29 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix 1.1.5", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.34" @@ -1299,6 +4168,19 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-sink" version = "0.3.34" @@ -1334,6 +4216,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -1346,6 +4229,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1359,6 +4251,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -1368,11 +4274,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 6.0.0", "rand_core 0.10.1", "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.14.2" @@ -1380,7 +4296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" dependencies = [ "color_quant", - "weezl", + "weezl 0.1.12", ] [[package]] @@ -1401,7 +4317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d99fc21d493812643aae86d53b7bbd02f376434a90317e8a790bc209fdd6605e" dependencies = [ "bytemuck", - "foldhash", + "foldhash 0.2.0", "hashbrown 0.17.1", "log", "peniko", @@ -1413,18 +4329,19 @@ dependencies = [ [[package]] name = "glifo" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" +checksum = "282a26c1e23de04bdab3e34a21b6f877a479b96737792516b9b8b8f69b6661be" dependencies = [ "bytemuck", - "foldhash", + "foldhash 0.2.0", "hashbrown 0.17.1", "log", "peniko", + "png 0.18.1", "skrifa 0.44.0", "smallvec", - "vello_common 0.1.0", + "vello_common 0.2.0", ] [[package]] @@ -1439,6 +4356,62 @@ dependencies = [ "web-sys", ] +[[package]] +name = "glutin" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325" +dependencies = [ + "bitflags 2.13.2", + "cfg_aliases", + "cgl", + "dispatch2", + "glutin_egl_sys", + "glutin_glx_sys", + "glutin_wgl_sys", + "libloading", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "raw-window-handle", + "windows-sys 0.52.0", + "x11-dl", +] + +[[package]] +name = "glutin-winit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85edca7075f8fc728f28cb8fbb111a96c3b89e930574369e3e9c27eb75d3788f" +dependencies = [ + "cfg_aliases", + "glutin", + "raw-window-handle", + "winit", +] + +[[package]] +name = "glutin_egl_sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4680ba6195f424febdc3ba46e7a42a0e58743f2edb115297b86d7f8ecc02d2" +dependencies = [ + "gl_generator", + "windows-sys 0.52.0", +] + +[[package]] +name = "glutin_glx_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7bb2938045a88b612499fbcba375a77198e01306f52272e692f8c1f3751185" +dependencies = [ + "gl_generator", + "x11-dl", +] + [[package]] name = "glutin_wgl_sys" version = "0.6.1" @@ -1462,6 +4435,28 @@ dependencies = [ "windows", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff 0.13.1", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "guillotiere" version = "0.7.0" @@ -1514,6 +4509,28 @@ dependencies = [ "smallvec", ] +[[package]] +name = "harfrust" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "948d0741125ba89cd3e1c23e5642415b6ade7e1d29d67ba25fb925b533e989d6" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "once_cell", + "read-fonts 0.43.3", + "smallvec", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1522,7 +4539,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1531,7 +4548,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1556,10 +4573,8 @@ checksum = "9f4d0e94ddd48749f06bbe4e5389fb9799a0c45bcaf00495042076ef05e3241a" [[package]] name = "hayro-cmap" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d285dc30731c8485de5fa732fbdf2b3affdf01e4da7c2135022ca6fa664bf6" dependencies = [ - "brotli", + "brotli 8.0.4", "hayro-postscript", ] @@ -1572,7 +4587,7 @@ dependencies = [ "hayro-syntax", "kurbo 0.13.1", "moxcms", - "phf", + "phf 0.13.1", "rustc-hash 2.1.3", "siphasher", "skrifa 0.47.0", @@ -1600,8 +4615,6 @@ dependencies = [ [[package]] name = "hayro-postscript" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "885c5ef0654933139a9b9546fc2c69e18d37f38aa2520f079092b1be18f1fcaa" [[package]] name = "hayro-syntax" @@ -1635,6 +4648,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heic-rs" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8883c3bfa6b4ff19a40e52a6dc051c5612cbe21bf408fc6d4875e8dfd777cb66" +dependencies = [ + "rayon", +] + [[package]] name = "hermit-abi" version = "0.5.3" @@ -1647,6 +4669,33 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -1686,6 +4735,17 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "subtle", + "typenum", + "zeroize", +] + [[package]] name = "hyper" version = "1.12.0" @@ -1748,6 +4808,12 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "hypher" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f2b4f6b5da34d400d4c112eadcc14512cc5d04307a8eee182586d9bebd7ce6" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1772,6 +4838,24 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collator" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_normalizer", + "icu_properties", + "icu_provider", + "serde", + "smallvec", + "utf16_iter", + "utf8_iter", + "zerovec", +] + [[package]] name = "icu_collections" version = "2.3.0" @@ -1780,12 +4864,30 @@ checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", + "serde", "utf8_iter", "yoke", "zerofrom", "zerovec", ] +[[package]] +name = "icu_decimal" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb8f655bba2d0c0459e43a5b0e6cd3cd388109898fb3d85d048165e8b0abd08" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_locale_core", + "icu_pattern", + "icu_plurals", + "icu_provider", + "serde", + "writeable", + "zerovec", +] + [[package]] name = "icu_locale_core" version = "2.3.0" @@ -1794,11 +4896,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -1809,7 +4932,11 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", + "serde", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] @@ -1819,6 +4946,32 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +[[package]] +name = "icu_pattern" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b3db611446bd92c246b03e4aa0bafa71ddb194af431ddbb42b9d70e4bfce1a0" +dependencies = [ + "displaydoc", + "either", + "serde", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_plurals" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e475e6766ef87b1d3c1f97be6363995b0bfaeddbc789671615df598a97ff0593" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_provider", + "serde", + "zerovec", +] + [[package]] name = "icu_properties" version = "2.3.0" @@ -1830,6 +4983,7 @@ dependencies = [ "icu_locale_core", "icu_properties_data", "icu_provider", + "serde", "zerotrie", "zerovec", ] @@ -1848,6 +5002,9 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "postcard", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -1855,6 +5012,68 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_provider_adapters" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df887bc4f55dbc773c7c45a0c7dfa75be886f7b7812830a71ab7271cc1801820" +dependencies = [ + "icu_locale_fallback", + "icu_provider", +] + +[[package]] +name = "icu_provider_blob" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916b44fac319d3f58e4d8d805a627df19e27937b582cd99fcde3bb6b7ce1b0e" +dependencies = [ + "icu_provider", + "postcard", + "serde", + "writeable", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "core_maths", + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + +[[package]] +name = "idea" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "075557004419d7f2031b8bb7f44bb43e55a83ca7b63076a8fb8fe75753836477" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1884,10 +5103,18 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", + "color_quant", + "gif", + "image-webp", "moxcms", "num-traits", "png 0.18.1", + "qoi", + "ravif", + "rgb", "tiff", + "zune-core", + "zune-jpeg", ] [[package]] @@ -1897,7 +5124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" dependencies = [ "byteorder-lite", - "quick-error", + "quick-error 2.0.1", ] [[package]] @@ -1912,6 +5139,23 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09e54e57b4c48b40f7aec75635392b12b3421fa26fe8b4332e63138ed278459c" +[[package]] +name = "imagesize" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" + +[[package]] +name = "imgref" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e44b0a4eaa4c82f441d50a963f2d5f05a787240aeee097597033e72accfd22f" + +[[package]] +name = "indd" +version = "0.1.0" +source = "git+https://github.com/storytold/indd-utils?rev=7f3060e5bce78d21ea70e0d5eacb96edce15dfbe#7f3060e5bce78d21ea70e0d5eacb96edce15dfbe" + [[package]] name = "indexmap" version = "2.14.2" @@ -1922,12 +5166,92 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "inout" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a580c13f0ffc335ef42dd3f6b1a41b726051c5e1656d9a3aa6fddb3c5dfbbf9" +dependencies = [ + "block-padding 0.4.2", + "hybrid-array", +] + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.59.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + [[package]] name = "ipnet" version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.15.0" @@ -1943,6 +5267,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jixel" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278a75e12caacec34be774d62ad05e45fc0b6a52d1f4e9f7fbc0209ecdf794f3" + [[package]] name = "jni" version = "0.22.4" @@ -2011,6 +5341,27 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" +dependencies = [ + "rayon", +] + +[[package]] +name = "jpeg-encoder" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b454d911ac55068f53495488d8ccd0646eaa540c033a28ee15b07838afafb01f" + +[[package]] +name = "jpeg-encoder" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0370574b86f7eca156b9f298392b5e69a23f8c86f3f865add60bbc2e79467a6" + [[package]] name = "js-sys" version = "0.3.106" @@ -2022,6 +5373,217 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jxl-bitstream" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b480e752277e29eb4054f69546887a9b84656fe78c08f54ba5850ced98a378fe" +dependencies = [ + "tracing", +] + +[[package]] +name = "jxl-coding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd972bcd125e776f1eb241ac50e39f956095a1c2770c64736c968f8946bd9a3c" +dependencies = [ + "jxl-bitstream", + "tracing", +] + +[[package]] +name = "jxl-color" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f316b1358c1711755b3ee8e8cb5c4a1dad12e796233088a7a513440782de80b2" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-image", + "jxl-oxide-common", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-frame" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d967c6fd669c7c01060b5022d8835fa82fd46b06ffc98b549f17600a097c2b3" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-grid" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01671307879a033bfa52e6e8784b941aca770b3f3a7d33830b455b6844f793fb" +dependencies = [ + "tracing", +] + +[[package]] +name = "jxl-image" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f752d62577c702a94dbbce4045caf08cb58639e8a4d56464b40ecf33ffe565" +dependencies = [ + "jxl-bitstream", + "jxl-grid", + "jxl-oxide-common", + "tracing", +] + +[[package]] +name = "jxl-jbr" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35d032bcec660647828527ff42c6f5776d2fd44b8357f9f6d9ac6dc07218e46" +dependencies = [ + "brotli-decompressor 5.0.3", + "jxl-bitstream", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-modular" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2f045b24c738dd91d482be385512b512721ae08a671bd4b27bf1c47f215235" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-oxide-common", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-oxide" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36c662923f47586880211f3bc7c0d83fb3a9b410d278c7bde93450748abeef3" +dependencies = [ + "brotli-decompressor 5.0.3", + "jxl-bitstream", + "jxl-color", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-jbr", + "jxl-oxide-common", + "jxl-render", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-oxide-common" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62394c5021b3a9e7e0dbb2d639d555d019090c9946c39f6d3b09d390db4157b" +dependencies = [ + "jxl-bitstream", +] + +[[package]] +name = "jxl-render" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34386bfdb6a19b5a30cc9beb4d475d537422c31ae8c39bb69640fcce3fcaf19" +dependencies = [ + "bytemuck", + "jxl-bitstream", + "jxl-coding", + "jxl-color", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-threadpool" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f15eb830aa77a7f21148d72e153562a26bfe570139bd4922eab1908dd499d3" +dependencies = [ + "rayon", + "rayon-core", + "tracing", +] + +[[package]] +name = "jxl-vardct" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce72a18c6d3a47172ab6c479be2bdb56f22066b5d7092663f03b4490820b4511" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "once_cell", + "sha2 0.10.9", + "signature 2.2.0", +] + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + [[package]] name = "keyboard-types" version = "0.8.3" @@ -2051,9 +5613,9 @@ checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" [[package]] name = "kittest" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ceaa75eb0036a32b6b9833962eb18137449e9817e2e586006471925b727fd5" +checksum = "0bb81a721ba76aa095b7f45deb180504f786904e56ca4b8e141f66eb89a78b75" dependencies = [ "accesskit", "accesskit_consumer", @@ -2106,15 +5668,41 @@ dependencies = [ "arrayvec", "euclid", "polycool", + "serde", "smallvec", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libloading" version = "0.8.9" @@ -2149,6 +5737,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" +[[package]] +name = "linesweeper" +version = "0.5.0" +dependencies = [ + "arrayvec", + "kurbo 0.13.1", + "polycool", + "rustc-hash 2.1.3", + "smallvec", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2166,6 +5765,9 @@ name = "litemap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +dependencies = [ + "serde_core", +] [[package]] name = "litrs" @@ -2188,12 +5790,56 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru-slab" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2202,13 +5848,22 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2225,6 +5880,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2284,6 +5945,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multiversion" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb8a68be21d0755f83bfe47e79b896cc732687cf1903f0e50585e8d6ecc53076" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05e0c3c3e4edaa21d87c645852ed0b0c3663752bcc6ec43353bd6189dce35e1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "target-features", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -2297,7 +5980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130" dependencies = [ "arrayvec", - "bit-set", + "bit-set 0.10.0", "bitflags 2.13.2", "cfg-if", "cfg_aliases", @@ -2358,12 +6041,64 @@ dependencies = [ "jni-sys 0.3.1", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nohash-hasher" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2373,6 +6108,89 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "serde", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2453,9 +6271,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ "bitflags 2.13.2", + "block2 0.6.2", "objc2 0.6.5", "objc2-core-foundation", - "objc2-core-graphics", "objc2-foundation 0.3.2", ] @@ -2512,7 +6330,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags 2.13.2", "dispatch2", "objc2 0.6.5", "objc2-core-foundation", @@ -2543,6 +6360,16 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-text" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" +dependencies = [ + "bitflags 2.13.2", + "objc2-core-foundation", +] + [[package]] name = "objc2-encode" version = "4.1.0" @@ -2715,6 +6542,27 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "ocb3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c196e0276c471c843dd5777e7543a36a298a4be942a2a688d8111cd43390dedb" +dependencies = [ + "aead", + "cipher 0.4.4", + "ctr", + "subtle", +] + +[[package]] +name = "oem_cp" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7959d3349f484aec36462f98226983578bf7d33f9b034c32967489c7000e1f" +dependencies = [ + "phf 0.11.3", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2727,6 +6575,22 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "open" +version = "5.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8c8b8e725ad26404d6896eb2f064294a0a7b4f1d9120210d85544684c4ca95" +dependencies = [ + "is-wsl", + "libc", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -2749,6 +6613,15 @@ dependencies = [ "libredox", ] +[[package]] +name = "ordered-channel" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95be4d57809897b5a7539fc15a7dfe0e84141bc3dfaa2e9b1b27caa90acf61ab" +dependencies = [ + "crossbeam-channel", +] + [[package]] name = "ordered-float" version = "5.5.0" @@ -2758,6 +6631,101 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "sha2 0.10.9", +] + +[[package]] +name = "p256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.1", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "fiat-crypto 0.3.0", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.1", +] + +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "rand_core 0.6.4", + "sha2 0.10.9", +] + +[[package]] +name = "p521" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad64cc32c2dc466317c12ee5853e61f159f9eab1fe7efade0395dc2e7b43449" +dependencies = [ + "base16ct 1.0.0", + "ecdsa 0.17.0", + "elliptic-curve 0.14.3", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.1", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -2781,6 +6749,45 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parlance" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73072a9470554c8e794778acece2312661218aaabb635600957d952cf01c716e" + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", + "hmac 0.13.0", +] + [[package]] name = "pdf-writer" version = "0.15.0" @@ -2812,6 +6819,15 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "peniko" version = "0.6.1" @@ -2831,6 +6847,86 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pgp" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae70f4d9325a391db30d115d6191a7bb67cec856aa0bba131290f0c4e09a532a" +dependencies = [ + "aead", + "aes 0.8.4", + "aes-gcm", + "aes-kw", + "argon2", + "base64 0.22.1", + "bitfields", + "block-padding 0.3.3", + "blowfish", + "buffer-redux", + "byteorder", + "bytes", + "camellia", + "cast5", + "cfb-mode", + "cipher 0.4.4", + "const-oid 0.9.6", + "crc24", + "curve25519-dalek 4.1.3", + "cx448", + "derive_builder", + "derive_more", + "des 0.8.1", + "digest 0.10.7", + "dsa", + "eax", + "ecdsa 0.16.9", + "ed25519-dalek 2.2.0", + "elliptic-curve 0.13.8", + "flate2", + "generic-array", + "getrandom 0.2.17", + "hex", + "hkdf", + "idea", + "k256", + "log", + "md-5 0.10.6", + "memchr", + "nom 8.0.0", + "num-bigint-dig", + "num-traits", + "num_enum", + "ocb3", + "p256 0.13.2", + "p384 0.13.1", + "p521 0.13.3", + "rand 0.8.8", + "replace_with", + "ripemd 0.1.3", + "rsa 0.9.10", + "sha1 0.10.7", + "sha1-checked", + "sha2 0.10.9", + "sha3 0.10.9", + "signature 2.2.0", + "smallvec", + "snafu", + "subtle", + "twofish", + "web-time", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.13.1" @@ -2838,7 +6934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ "phf_macros", - "phf_shared", + "phf_shared 0.13.1", ] [[package]] @@ -2848,7 +6944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ "fastrand", - "phf_shared", + "phf_shared 0.13.1", ] [[package]] @@ -2858,12 +6954,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.13.1", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -2915,6 +7020,58 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs1" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6250dd7321adbe93d12b750aee0ced159fd13e77f625edcdf1cb6f83cd99f650" +dependencies = [ + "der 0.8.3", + "spki 0.8.1", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.3", + "spki 0.8.1", +] + [[package]] name = "pkg-config" version = "0.3.34" @@ -2927,6 +7084,19 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "plist" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327a4cb2ad49dc48adfdc9da89e22c64d488b508c67a044d71ba729e4521a285" +dependencies = [ + "base64 0.23.1", + "indexmap", + "quick-xml 0.42.0", + "serde", + "time", +] + [[package]] name = "png" version = "0.17.16" @@ -2953,6 +7123,16 @@ dependencies = [ "miniz_oxide 0.8.9", ] +[[package]] +name = "poll-promise" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6a58fecbf9da8965bcdb20ce4fd29788d1acee68ddbb64f0ba1b81bccdb7df" +dependencies = [ + "document-features", + "static_assertions", +] + [[package]] name = "polling" version = "3.11.0" @@ -2967,6 +7147,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "pollster" version = "1.0.1" @@ -2982,6 +7168,18 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -2997,21 +7195,86 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "presser" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint 0.7.11", + "crypto-common 0.2.2", + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve 0.13.8", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve 0.14.3", + "once_cell", + "primefield", + "serdect 0.4.3", + "wnaf", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -3059,6 +7322,39 @@ name = "profiling" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set 0.11.1", + "bit-vec 0.10.1", + "bitflags 2.13.2", + "chacha20", + "core_detect", + "num-traits", + "rand 0.10.3", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] [[package]] name = "pxfm" @@ -3066,6 +7362,21 @@ version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-error" version = "2.0.1" @@ -3081,6 +7392,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.12" @@ -3111,7 +7431,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash 2.1.3", @@ -3147,12 +7467,45 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.3" @@ -3164,11 +7517,43 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] [[package]] name = "rand_core" @@ -3185,12 +7570,70 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "range-alloc" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.14.0", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.5", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror 2.0.21", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error 2.0.1", + "rav1e", + "rgb", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -3229,6 +7672,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rc2" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceda21af1ae61033b63175653a1af86cae399d79cd03ca80ba347eb3a6c4a7fe" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "read-fonts" version = "0.39.2" @@ -3246,7 +7698,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" dependencies = [ "bytemuck", - "font-types 0.12.6", + "font-types 0.12.7", + "once_cell", +] + +[[package]] +name = "read-fonts" +version = "0.43.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005c8acf251756c478b0bf402885bfd88a1476020c4c7e6060edc1aa68da38ea" +dependencies = [ + "bytemuck", + "font-types 0.12.7", "once_cell", ] @@ -3257,10 +7720,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5a0cf4bb6cbce4c29756270d189a5e400ac05e91192161d83a0bcb6e68492e8" dependencies = [ "bytemuck", - "font-types 0.12.6", + "font-types 0.12.7", "once_cell", ] +[[package]] +name = "read-fonts" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4edc8db0d972de88e53a085b1f01fb3fbdae7bb0cb7a0ce2b2f8285a412fe3ef" +dependencies = [ + "bytemuck", + "foldhash 0.2.0", + "font-types 0.12.7", + "once_cell", + "yoke", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -3298,12 +7774,30 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "renderdoc-sys" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" +[[package]] +name = "replace_with" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51743d3e274e2b18df81c4dc6caf8a5b8e15dbe799e0dca05c7617380094e884" + [[package]] name = "reqwest" version = "0.13.5" @@ -3356,9 +7850,71 @@ dependencies = [ "log", "pico-args", "rgb", - "svgtypes", - "tiny-skia", - "usvg", + "svgtypes 0.15.3", + "tiny-skia 0.11.4", + "usvg 0.45.1", +] + +[[package]] +name = "resvg" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e3803f97b999e80cbf7c6ecdd07a8102204d92e1633cf48783720c521196bd" +dependencies = [ + "bytemuck", + "gif", + "image-webp", + "log", + "pico-args", + "rgb", + "svgtypes 0.16.2", + "tiny-skia 0.12.0", + "usvg 0.48.1", + "zune-jpeg", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint 0.7.11", + "hmac 0.13.0", +] + +[[package]] +name = "rfd" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20dafead71c16a34e1ff357ddefc8afc11e7d51d6d2b9fbd07eaa48e3e540220" +dependencies = [ + "block2 0.6.2", + "dispatch2", + "js-sys", + "libc", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "percent-encoding", + "pollster 0.4.0", + "raw-window-handle", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3384,12 +7940,76 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ripemd" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "ripemd" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd4211456b4172d7e44261920c25acf07367c4f04bb5f5d54fc21b090d9b159" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "roxmltree" version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" +[[package]] +name = "roxmltree" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907" +dependencies = [ + "memchr", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1 0.7.5", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + +[[package]] +name = "rsa" +version = "0.10.0-rc.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" +dependencies = [ + "const-oid 0.10.3", + "crypto-bigint 0.7.11", + "crypto-primes", + "digest 0.11.3", + "pkcs1 0.8.0", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "signature 3.0.0", + "zeroize", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -3438,13 +8058,25 @@ dependencies = [ ] [[package]] -name = "rustls" -version = "0.23.45" +name = "rustix-linux-procfs" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix 1.1.5", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3518,6 +8150,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error 1.2.3", + "tempfile", + "wait-timeout", +] + [[package]] name = "rustybuzz" version = "0.20.1" @@ -3536,6 +8180,12 @@ dependencies = [ "unicode-script", ] +[[package]] +name = "ruzstd" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" + [[package]] name = "ryu" version = "1.0.23" @@ -3566,18 +8216,41 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct 0.2.0", + "der 0.7.10", + "generic-array", + "pkcs8 0.10.2", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.3", + "hybrid-array", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -3612,6 +8285,16 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" @@ -3645,9 +8328,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -3667,6 +8350,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "serde_spanned" version = "1.1.2" @@ -3688,6 +8382,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serdect" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" +dependencies = [ + "base16ct 0.2.0", + "serde", +] + +[[package]] +name = "serdect" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42f67da2385b51a5f9652db9c93d78aeaf7610bf5ec366080b6de810604af53" +dependencies = [ + "base16ct 0.2.0", + "serde", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", + "serde", +] + [[package]] name = "sha1" version = "0.10.7" @@ -3696,7 +8420,29 @@ checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1 0.10.7", + "zeroize", ] [[package]] @@ -3707,7 +8453,39 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak 0.1.6", +] + +[[package]] +name = "sha3" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.2", + "sponge-cursor", ] [[package]] @@ -3716,15 +8494,36 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ + "digest 0.10.7", "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "sigstore-bundle" version = "0.6.6" @@ -3751,15 +8550,15 @@ checksum = "da90c3d9af638898a5fdc347479b18f950bb0dde11ecf2244f94cd28135da53b" dependencies = [ "aws-lc-rs", "base64 0.22.1", - "const-oid", - "der", - "digest", + "const-oid 0.9.6", + "der 0.7.10", + "digest 0.10.7", "pem", "rand_core 0.10.1", - "sha2", - "signature", + "sha2 0.10.9", + "signature 2.2.0", "sigstore-types", - "spki", + "spki 0.7.3", "thiserror 2.0.21", "tracing", "x509-cert", @@ -3825,10 +8624,10 @@ dependencies = [ "chrono", "cmpv2", "cms", - "const-oid", - "der", + "const-oid 0.9.6", + "der 0.7.10", "hex", - "rand", + "rand 0.10.3", "reqwest", "rustls-pki-types", "rustls-webpki", @@ -3864,7 +8663,7 @@ dependencies = [ "base64 0.22.1", "chrono", "cms", - "const-oid", + "const-oid 0.9.6", "hex", "pem", "rustls-pki-types", @@ -3901,6 +8700,15 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3922,6 +8730,18 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "skera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531045338e36ad05b9652912a47ba38a935d0eaa0293c3e33d8112cfa414c14b" +dependencies = [ + "hashbrown 0.17.1", + "skrifa 0.48.0", + "thiserror 2.0.21", + "write-fonts 0.54.1", +] + [[package]] name = "skrifa" version = "0.42.1" @@ -3952,6 +8772,16 @@ dependencies = [ "read-fonts 0.44.0", ] +[[package]] +name = "skrifa" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5213b4825bac3587046330052d61bfac7bd3bd0d203c758d7c2ef726fcea034" +dependencies = [ + "bytemuck", + "read-fonts 0.45.1", +] + [[package]] name = "slab" version = "0.4.12" @@ -3972,43 +8802,8 @@ name = "smallvec" version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" - -[[package]] -name = "smithay-client-toolkit" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0512da38f5e2b31201a93524adb8d3136276fa4fe4aafab4e1f727a82b534cc0" dependencies = [ - "bitflags 2.13.2", - "calloop 0.14.5", - "calloop-wayland-source", - "cursor-icon", - "libc", - "log", - "memmap2", - "rustix 1.1.5", - "thiserror 2.0.21", - "wayland-backend", - "wayland-client", - "wayland-csd-frame", - "wayland-cursor", - "wayland-protocols", - "wayland-protocols-experimental", - "wayland-protocols-misc", - "wayland-protocols-wlr", - "wayland-scanner", - "xkeysym", -] - -[[package]] -name = "smithay-clipboard" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71704c03f739f7745053bde45fa203a46c58d25bc5c4efba1d9a60e9dba81226" -dependencies = [ - "libc", - "smithay-client-toolkit", - "wayland-backend", + "serde", ] [[package]] @@ -4020,6 +8815,27 @@ dependencies = [ "serde", ] +[[package]] +name = "snafu" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "socket2" version = "0.6.5" @@ -4030,6 +8846,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -4046,9 +8868,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.3", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4070,6 +8908,16 @@ dependencies = [ "float-cmp", ] +[[package]] +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" +dependencies = [ + "hashbrown 0.15.5", + "serde", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -4081,6 +8929,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.28.0" @@ -4111,7 +8965,7 @@ dependencies = [ "kurbo 0.13.1", "rustc-hash 2.1.3", "skrifa 0.42.1", - "write-fonts", + "write-fonts 0.48.1", ] [[package]] @@ -4130,6 +8984,213 @@ dependencies = [ "siphasher", ] +[[package]] +name = "svgtypes" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f12b12995e3638c70618c1facb1ce65302f6a6f706b675f07c58a5d7391120e" +dependencies = [ + "kurbo 0.13.1", + "siphasher", +] + +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6405d5c34ff6f8f7ca08a4101efe66d21e180f7322ef9359900a0e55698a7892" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445932ecb0c59362fde9c082dd63a75380650c5077fdb8b80b8cac850442a74c" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a78f96f3cf62932d0c497959c0cc2100ef36b03c7ca1417b260f432a482d8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d90b4fcf796137cc683c538282804ff9629f8ad9dbfd881fcbba331ac4e986" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab64327ee1920531c5bf86cf7e9cd1a599d57013ddc30691da62683cefc65191" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5495e7f7e3c7035328d82b6d6e377eef289bb0c4105bdeb557fc93a833f994" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -4182,6 +9243,18 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "js-sys", + "libc", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "system-configuration" version = "0.7.0" @@ -4203,6 +9276,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.46" @@ -4214,6 +9293,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "tempfile" version = "3.27.0" @@ -4276,6 +9361,15 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "tiff" version = "0.11.3" @@ -4285,11 +9379,41 @@ dependencies = [ "fax", "flate2", "half", - "quick-error", - "weezl", + "quick-error 2.0.1", + "weezl 0.1.12", "zune-jpeg", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-skia" version = "0.11.4" @@ -4305,6 +9429,21 @@ dependencies = [ "tiny-skia-path 0.11.4", ] +[[package]] +name = "tiny-skia" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47ffee5eaaf5527f630fb0e356b90ebdec84d5d18d937c5e440350f88c5a91ea" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "png 0.18.1", + "tiny-skia-path 0.12.0", +] + [[package]] name = "tiny-skia-path" version = "0.11.4" @@ -4334,6 +9473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -4408,10 +9548,12 @@ version = "1.1.8+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" dependencies = [ + "indexmap", "serde_core", "serde_spanned", "toml_datetime", "toml_parser", + "toml_writer", "winnow", ] @@ -4505,7 +9647,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4546,6 +9687,15 @@ dependencies = [ "core_maths", ] +[[package]] +name = "twofish" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78e83a30223c757c3947cd144a31014ff04298d8719ae10d03c31c0448c8013" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "type-map" version = "0.5.1" @@ -4555,12 +9705,35 @@ dependencies = [ "rustc-hash 2.1.3", ] +[[package]] +name = "typed-path" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys 0.61.2", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -4591,6 +9764,12 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.25" @@ -4618,12 +9797,34 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-vo" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94" + [[package]] name = "unicode-width" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -4636,6 +9837,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" +dependencies = [ + "base64 0.23.1", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -4661,15 +9892,54 @@ dependencies = [ "kurbo 0.11.3", "log", "pico-args", - "roxmltree", + "roxmltree 0.20.0", "simplecss", "siphasher", "strict-num", - "svgtypes", + "svgtypes 0.15.3", "tiny-skia-path 0.11.4", "xmlwriter", ] +[[package]] +name = "usvg" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "977d0a4abdef933f424a99fe09f95576e089b90aebc6f016a3bc813762493e91" +dependencies = [ + "base64 0.23.1", + "data-url", + "fontdb", + "harfrust 0.12.0", + "imagesize 0.15.0", + "kurbo 0.13.1", + "log", + "pico-args", + "roxmltree 0.21.2", + "simplecss", + "siphasher", + "skrifa 0.44.0", + "strict-num", + "svgtypes 0.16.2", + "tiny-skia-path 0.12.0", + "unicode-bidi", + "unicode-script", + "unicode-vo", + "xmlwriter", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4681,6 +9951,237 @@ name = "uuid" version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" +dependencies = [ + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + +[[package]] +name = "vectorcraft-brush" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "kurbo 0.13.1", + "serde", + "serde_json", + "vectorcraft-color", + "vectorcraft-doc", + "vectorcraft-geom", +] + +[[package]] +name = "vectorcraft-color" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "kurbo 0.13.1", + "moxcms", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "vectorcraft-doc" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "chrono", + "hayro-ccitt", + "image", + "kurbo 0.13.1", + "serde", + "serde_json", + "thiserror 2.0.21", + "tiff", + "vectorcraft-color", + "vectorcraft-geom", + "zune-jpeg", +] + +[[package]] +name = "vectorcraft-effects" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "image", + "kurbo 0.13.1", + "serde_json", + "vectorcraft-doc", + "vectorcraft-geom", + "vectorcraft-pathops", + "vectorcraft-plugins", + "vectorcraft-text", + "vectorcraft-three-d", +] + +[[package]] +name = "vectorcraft-eps" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "flate2", + "hayro-ccitt", + "image", + "kurbo 0.13.1", + "memchr", + "ruzstd", + "serde_json", + "vectorcraft-brush", + "vectorcraft-color", + "vectorcraft-doc", + "vectorcraft-effects", + "vectorcraft-geom", + "vectorcraft-metafile", + "vectorcraft-pdf", + "vectorcraft-render", + "vectorcraft-text", +] + +[[package]] +name = "vectorcraft-geom" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "kurbo 0.13.1", + "serde", + "thiserror 2.0.21", +] + +[[package]] +name = "vectorcraft-metafile" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "image", + "kurbo 0.13.1", + "vectorcraft-brush", + "vectorcraft-color", + "vectorcraft-doc", + "vectorcraft-effects", + "vectorcraft-geom", + "vectorcraft-text", +] + +[[package]] +name = "vectorcraft-pathops" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "kurbo 0.13.1", + "linesweeper", + "thiserror 2.0.21", + "vectorcraft-geom", +] + +[[package]] +name = "vectorcraft-pdf" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "aes 0.8.4", + "chrono", + "craft-pdf 0.1.0 (git+https://github.com/storytold/craft-libs?rev=92d392cff132eaf53e13afff77f38b5c11b73aa0)", + "flate2", + "getrandom 0.4.3", + "image", + "jpeg-encoder 0.6.1", + "kurbo 0.13.1", + "log", + "md-5 0.10.6", + "roxmltree 0.21.2", + "serde", + "sha2 0.10.9", + "skrifa 0.47.0", + "smallvec", + "thiserror 2.0.21", + "vectorcraft-brush", + "vectorcraft-color", + "vectorcraft-doc", + "vectorcraft-effects", + "vectorcraft-geom", + "vectorcraft-text", + "weezl 0.2.1", +] + +[[package]] +name = "vectorcraft-plugins" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", + "vectorcraft-color", + "vectorcraft-doc", + "vectorcraft-geom", + "wasmi", +] + +[[package]] +name = "vectorcraft-render" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "flate2", + "gif", + "image", + "jpeg-encoder 0.6.1", + "log", + "png 0.18.1", + "vectorcraft-brush", + "vectorcraft-color", + "vectorcraft-doc", + "vectorcraft-effects", + "vectorcraft-geom", + "vectorcraft-text", + "vello_common 0.2.0", + "vello_cpu 0.2.0", + "weezl 0.2.1", + "winsafe", +] + +[[package]] +name = "vectorcraft-text" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "harfrust 0.13.3", + "icu_segmenter", + "kurbo 0.13.1", + "log", + "skrifa 0.47.0", + "subsetter", + "unicode-bidi", + "unicode-general-category", + "unicode-script", + "unicode-segmentation", + "vectorcraft-doc", + "vectorcraft-geom", + "vello_cpu 0.2.0", + "windows-registry", +] + +[[package]] +name = "vectorcraft-three-d" +version = "0.18.0" +source = "git+https://github.com/storytold/vectorcraft.git?rev=6b3a6b8788ebcb420d69449199cbdf323888d27e#6b3a6b8788ebcb420d69449199cbdf323888d27e" +dependencies = [ + "vectorcraft-geom", + "vectorcraft-pathops", +] [[package]] name = "vello_common" @@ -4718,15 +10219,16 @@ dependencies = [ [[package]] name = "vello_common" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e9aed918117e8152c9eddfd8362d73c465c23f26c44786aa331707b8a64fa2" +checksum = "bbb2141a2bca6e6d598e471fd4d1d7eed8e020aad6a28187edda07f091a325dd" dependencies = [ "bytemuck", "fearless_simd", "guillotiere", "log", "peniko", + "png 0.18.1", "smallvec", "thiserror 2.0.21", ] @@ -4746,14 +10248,29 @@ dependencies = [ [[package]] name = "vello_cpu" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac7349e1f55f6b801c7c277958df4ea53e7f20f21e8014910ad888b2ecda93ea" +checksum = "ace506ca414548966fcf3c283be94eb1cad8254de488791ee698bf438b0890a9" dependencies = [ "bytemuck", - "glifo 0.2.0", + "crossbeam-channel", + "glifo 0.3.0", "hashbrown 0.17.1", - "vello_common 0.1.0", + "ordered-channel", + "png 0.18.1", + "rayon", + "thread_local", + "vello_common 0.2.0", +] + +[[package]] +name = "verifysign" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6acfcc42a8d44f340d001193d97ef56bf33210b51ad760ae8308c97d5241ca51" +dependencies = [ + "core-foundation 0.10.1", + "windows-sys 0.61.2", ] [[package]] @@ -4762,6 +10279,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -4787,6 +10313,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.129" @@ -4882,6 +10417,56 @@ version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f692aa943ccd88363733b77063f32cfed5bc6cbea8e6e8b251b302f881606fe" +[[package]] +name = "wasmi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78693fcdd618e0fc34af59c6b8efa9ac5d58c68df940beff4bedddb6acfe7c27" +dependencies = [ + "spin", + "wasmi_collections", + "wasmi_core", + "wasmi_ir", + "wasmparser", +] + +[[package]] +name = "wasmi_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8be2aa467cf2d29e96ff759472c36eeb44a3c81c67fc9cb76c9a24c519c557" +dependencies = [ + "string-interner", +] + +[[package]] +name = "wasmi_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69372d5fda3ea3d1e0aa6603c7888110e0187e88ea17cd8fc2e2df0a0e1f37fa" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmi_ir" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f17b774caa13c618c7244f1ee51fe23c5e7b8538a471fa46d9949779758aed6" +dependencies = [ + "wasmi_core", +] + +[[package]] +name = "wasmparser" +version = "0.228.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abf1132c1fdf747d56bbc1bb52152400c70f336870f968b85e89ea422198ae3" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + [[package]] name = "wayland-backend" version = "0.3.17" @@ -4891,7 +10476,6 @@ dependencies = [ "cc", "downcast-rs", "rustix 1.1.5", - "scoped-tls", "smallvec", "wayland-sys", ] @@ -4908,79 +10492,6 @@ dependencies = [ "wayland-scanner", ] -[[package]] -name = "wayland-csd-frame" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625c5029dbd43d25e6aa9615e88b829a5cad13b2819c4ae129fdbb7c31ab4c7e" -dependencies = [ - "bitflags 2.13.2", - "cursor-icon", - "wayland-backend", -] - -[[package]] -name = "wayland-cursor" -version = "0.31.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" -dependencies = [ - "rustix 1.1.5", - "wayland-client", - "xcursor", -] - -[[package]] -name = "wayland-protocols" -version = "0.32.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" -dependencies = [ - "bitflags 2.13.2", - "wayland-backend", - "wayland-client", - "wayland-scanner", -] - -[[package]] -name = "wayland-protocols-experimental" -version = "20250721.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a1f863128dcaaec790d7b4b396cc9b9a7a079e878e18c47e6c2d2c5a8dcbb1" -dependencies = [ - "bitflags 2.13.2", - "wayland-backend", - "wayland-client", - "wayland-protocols", - "wayland-scanner", -] - -[[package]] -name = "wayland-protocols-misc" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" -dependencies = [ - "bitflags 2.13.2", - "wayland-backend", - "wayland-client", - "wayland-protocols", - "wayland-scanner", -] - -[[package]] -name = "wayland-protocols-wlr" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" -dependencies = [ - "bitflags 2.13.2", - "wayland-backend", - "wayland-client", - "wayland-protocols", - "wayland-scanner", -] - [[package]] name = "wayland-scanner" version = "0.31.11" @@ -4988,7 +10499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" dependencies = [ "proc-macro2", - "quick-xml", + "quick-xml 0.41.0", "quote", ] @@ -5033,12 +10544,27 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "weezl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + [[package]] name = "wgpu" version = "30.0.1" @@ -5076,8 +10602,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14" dependencies = [ "arrayvec", - "bit-set", - "bit-vec", + "bit-set 0.10.0", + "bit-vec 0.9.1", "bitflags 2.13.2", "bytemuck", "cfg_aliases", @@ -5149,7 +10675,7 @@ dependencies = [ "android_system_properties", "arrayvec", "ash", - "bit-set", + "bit-set 0.10.0", "bitflags 2.13.2", "block2 0.6.2", "bytemuck", @@ -5349,7 +10875,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -5358,16 +10884,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -5385,31 +10902,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -5427,107 +10927,59 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winit" version = "0.30.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6755fa58a9f8350bd1e472d4c3fcc25f824ec358933bba33306d0b63df5978d" +source = "git+https://github.com/storytold/winit.git?rev=a517b3846ee8457d84ee4975110bfda6797c44dc#a517b3846ee8457d84ee4975110bfda6797c44dc" dependencies = [ "android-activity", "atomic-waker", "bitflags 2.13.2", "block2 0.5.1", - "calloop 0.13.0", + "bytemuck", + "calloop", "cfg_aliases", "concurrent-queue", "core-foundation 0.9.4", @@ -5542,6 +10994,7 @@ dependencies = [ "objc2-foundation 0.2.2", "objc2-ui-kit 0.2.2", "orbclient", + "percent-encoding", "pin-project", "raw-window-handle", "redox_syscall 0.4.1", @@ -5554,6 +11007,8 @@ dependencies = [ "web-sys", "web-time", "windows-sys 0.52.0", + "x11-dl", + "x11rb", "xkbcommon-dl", ] @@ -5569,7 +11024,7 @@ dependencies = [ [[package]] name = "winprint" version = "0.2.1" -source = "git+https://github.com/storytold/winprint.rs?rev=5f80b9baf6b58937c401656634e42d33f00ddf49#5f80b9baf6b58937c401656634e42d33f00ddf49" +source = "git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02#72951d7455ebcab569fbd9386c513d3d9f53fb02" dependencies = [ "flate2", "fmt-derive", @@ -5584,15 +11039,54 @@ dependencies = [ "windows", "windows-core", "windows-implement", + "windows-numerics", "xml", ] +[[package]] +name = "winreg" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" +dependencies = [ + "cfg-if", + "windows-sys 0.59.0", +] + [[package]] name = "winsafe" version = "0.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ef0ffc427f045c0cc9ebffd6f4f91153dcd2a1547b5c3c29ee3f541e16e95c6" +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.2", + "windows-sys 0.59.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "primefield", +] + [[package]] name = "write-fonts" version = "0.48.1" @@ -5606,11 +11100,53 @@ dependencies = [ "read-fonts 0.39.2", ] +[[package]] +name = "write-fonts" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55716cfa7e8bcdb90289e07dfac478ccb2e3ec0f9a604fe682c7a03a7984d705" +dependencies = [ + "font-types 0.12.7", + "indexmap", + "kurbo 0.13.1", + "log", + "read-fonts 0.45.1", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +dependencies = [ + "either", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] [[package]] name = "x11rb" @@ -5618,7 +11154,11 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" dependencies = [ + "as-raw-xcb-connection", "gethostname", + "libc", + "libloading", + "once_cell", "rustix 1.1.5", "x11rb-protocol", ] @@ -5629,17 +11169,29 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek 4.1.3", + "rand_core 0.6.4", + "serde", + "zeroize", +] + [[package]] name = "x509-cert" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", - "der", - "sha1", - "signature", - "spki", + "const-oid 0.9.6", + "der 0.7.10", + "sha1 0.10.7", + "signature 2.2.0", + "spki 0.7.3", "tls_codec", ] @@ -5651,7 +11203,7 @@ checksum = "f5ceece934a21607055b7ac5c25adb56a2ff559804b10705dc674d1d838c15e1" dependencies = [ "cmpv2", "cms", - "der", + "der 0.7.10", ] [[package]] @@ -5664,12 +11216,6 @@ dependencies = [ "rustix 1.1.5", ] -[[package]] -name = "xcursor" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "163b33ed8786455e2fa5d72f554057ce3f3182425434f756cd39c99839d88e23" - [[package]] name = "xkbcommon-dl" version = "0.4.2" @@ -5713,6 +11259,23 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9440ea3e5aeabb0ac63af70daf835274065238cdd0cec83418f417eae38bacee" +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + +[[package]] +name = "yeslogic-fontconfig-sys" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8b8abf912b9a29ff112e1671c97c33636903d13a69712037190e6805af4f76" +dependencies = [ + "dlib", + "once_cell", + "pkg-config", +] + [[package]] name = "yoke" version = "0.8.3" @@ -5736,6 +11299,76 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zbus" +version = "5.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db4be7c075cb421e4b7ee645541604239bd243ba7c357511f4ff3a74b555907" +dependencies = [ + "async-broadcast", + "async-executor", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-lite", + "hex", + "libc", + "ordered-stream", + "rustix 1.1.5", + "serde", + "serde_repr", + "tracing", + "uds_windows", + "uuid", + "windows-sys 0.61.2", + "winnow", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "5.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990635d09ade6df1868f72f8cac69a876a90981e8bd3c40b1be413f8dc88f40" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", + "zbus_names", + "zvariant", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "4.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812a12be4ead7a59e18ea8f047d08add0066b73bc62fc1467066a2472ae088dd" +dependencies = [ + "serde", + "winnow", + "zvariant", +] + +[[package]] +name = "zcheapstr" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1afec51604565183aeb5c54c20aeab286120d4e4460f7f76e3e8bb8c0d99473" +dependencies = [ + "serde", +] + [[package]] name = "zerocopy" version = "0.8.62" @@ -5804,8 +11437,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", + "litemap", + "serde_core", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -5814,6 +11450,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -5830,6 +11467,22 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "zip" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704" +dependencies = [ + "crc32fast", + "flate2", + "getrandom 0.4.3", + "indexmap", + "memchr", + "nix", + "typed-path", + "windows-sys 0.61.2", +] + [[package]] name = "zlib-rs" version = "0.6.8" @@ -5848,6 +11501,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + [[package]] name = "zune-jpeg" version = "0.5.15" @@ -5856,3 +11518,63 @@ checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" dependencies = [ "zune-core", ] + +[[package]] +name = "zune-jpegxl" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cba11ecd07cc23351500e840ae03841b7e4997923ec8e4832ee3ae199ea0b6b4" +dependencies = [ + "zune-core", +] + +[[package]] +name = "zvariant" +version = "5.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80bf6d6cd6a4b601acb64cfb8284cdc643fcfa75ef26ae3c46ec8aa4e8a1fbb7" +dependencies = [ + "endi", + "enumflags2", + "serde", + "winnow", + "zcheapstr", + "zvariant_derive", + "zvariant_utils", +] + +[[package]] +name = "zvariant_derive" +version = "5.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2672456ba1cb54b5eaec136e63135b3c7495fb938e53b28a62e633496d486d" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad0294361a320b694a328460dc73add56c306150f5cb6bfafc44446120008a3" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn 3.0.6", + "winnow", +] + +[[patch.unused]] +name = "egui_demo_lib" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" + +[[patch.unused]] +name = "egui_inspection" +version = "0.37.0" +source = "git+https://github.com/storytold/egui.git?rev=6d518fc799681571963842f95b55a6b68431b15a#6d518fc799681571963842f95b55a6b68431b15a" diff --git a/Cargo.toml b/Cargo.toml index ed556fc45..38b82c843 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "3" members = ["crates/*"] -exclude = ["vendor/krilla", "vendor/hayro", "vendor/hayro-interpret", "vendor/hayro-syntax", "vendor/hayro-jbig2"] +exclude = ["vendor/krilla", "vendor/hayro", "vendor/hayro-interpret", "vendor/hayro-syntax", "vendor/hayro-jbig2", "vendor/hayro-cmap", "vendor/hayro-postscript", "vendor/linesweeper"] [workspace.package] version = "0.1.0" @@ -11,28 +11,182 @@ rust-version = "1.95" repository = "https://github.com/storytold/craft-libs" [workspace.dependencies] +acadrust = "=0.6.6" +log = "0.4" # internal: add each crate here as `craft- = { path = "crates/" }` +craft-platform = { path = "crates/platform" } +craft-aep = { path = "crates/aep" } +craft-affinity = { path = "crates/affinity" } +craft-ai-core = { path = "crates/ai-core" } +craft-anicraft-format = { path = "crates/anicraft-format" } craft-pdf = { path = "crates/pdf", version = "=0.1.0" } +craft-ps = { path = "crates/ps", version = "=0.1.0" } +craft-pdn = { path = "crates/pdn" } +craft-plate-test = { path = "crates/plate-test" } +pdf-writer = "0.15" +miniz_oxide = "0.8" +weezl = "0.1" +skrifa = "0.47" krilla = { version = "=0.8.2", default-features = false, features = ["raster-images"] } hayro = { version = "=0.7.1", default-features = false } hayro-interpret = { version = "=0.7.0", default-features = false, features = ["embed-fonts", "embed-cmaps"] } hayro-syntax = { version = "=0.7.2", default-features = false, features = ["std"] } bytemuck = "1" +kurbo = "0.13" thiserror = "2" +craft-aaf = { path = "crates/aaf" } craft-atomic = { path = "crates/atomic" } +craft-captions = { path = "crates/captions" } +craft-cfb = { path = "crates/cfb" } +craft-codepage = { path = "crates/codepage" } craft-cms = { path = "crates/cms" } +craft-crs = { path = "crates/crs" } +craft-crw = { path = "crates/crw" } +craft-csv = { path = "crates/csv" } +craft-dav = { path = "crates/dav" } +craft-dialogs = { path = "crates/dialogs" } +craft-doc-binary = { path = "crates/doc-binary" } +craft-docx = { path = "crates/docx" } +craft-document-bundle = { path = "crates/document-bundle" } +craft-designcraft-format = { path = "crates/designcraft-format" } +craft-dxf = { path = "crates/dxf" } +craft-drawingml = { path = "crates/drawingml" } +craft-eps = { path = "crates/eps" } +craft-file-formats = { path = "crates/file-formats" } +craft-file-import = { path = "crates/file-import" } +vectorcraft-doc = { git = "https://github.com/storytold/vectorcraft.git", rev = "6b3a6b8788ebcb420d69449199cbdf323888d27e" } +vectorcraft-eps = { git = "https://github.com/storytold/vectorcraft.git", rev = "6b3a6b8788ebcb420d69449199cbdf323888d27e" } +vectorcraft-geom = { git = "https://github.com/storytold/vectorcraft.git", rev = "6b3a6b8788ebcb420d69449199cbdf323888d27e" } +vectorcraft-pathops = { git = "https://github.com/storytold/vectorcraft.git", rev = "6b3a6b8788ebcb420d69449199cbdf323888d27e" } +craft-doc = { path = "crates/doc" } +craft-docpath = { path = "crates/docpath" } +craft-dpx = { path = "crates/dpx" } +craft-dwg = { path = "crates/dwg" } +craft-ecproj-format = { path = "crates/ecproj-format" } +craft-figma-rest = { path = "crates/figma-rest" } +craft-exr = { path = "crates/exr" } +craft-flags = { path = "crates/flags" } +craft-font-catalog = { path = "crates/font-catalog" } +craft-font-manager = { path = "crates/font-manager" } +craft-font-store = { path = "crates/font-store" } +craft-gainmap = { path = "crates/gainmap" } +craft-flow = { path = "crates/flow" } +craft-gcode = { path = "crates/gcode" } +craft-gif = { path = "crates/gif" } +craft-hdr = { path = "crates/hdr" } +craft-mesh = { path = "crates/mesh" } +craft-html = { path = "crates/html" } +craft-heif = { path = "crates/heif" } +craft-fig = { path = "crates/fig" } +craft-fonts-build = { path = "crates/fonts-build" } +craft-i18n = { path = "crates/i18n" } +craft-idml = { path = "crates/idml" } +craft-indd = { path = "crates/indd" } +craft-jmap = { path = "crates/jmap" } +craft-ical = { path = "crates/ical" } +craft-jpeg = { path = "crates/jpeg" } +craft-jxl = { path = "crates/jxl" } +craft-kiwi = { path = "crates/kiwi" } +craft-keymap = { path = "crates/keymap" } +craft-assuan = { path = "crates/assuan" } +craft-kra = { path = "crates/kra" } +craft-audio = { path = "crates/audio" } +# No default features, so craft-audio's `encode` builds craft-media without `isobmff`. +craft-latex = { path = "crates/latex" } +craft-markdown = { path = "crates/markdown" } +craft-math = { path = "crates/math" } +craft-media = { path = "crates/media", default-features = false } +craft-materialx = { path = "crates/materialx" } +craft-media-testkit = { path = "crates/media-testkit" } +craft-midi = { path = "crates/midi" } +craft-sf2 = { path = "crates/sf2" } +craft-mime = { path = "crates/mime" } +craft-odf = { path = "crates/odf" } +craft-ods = { path = "crates/ods" } +craft-prc = { path = "crates/prc" } +craft-lottie = { path = "crates/lottie" } +craft-menu = { path = "crates/menu" } +craft-metafile = { path = "crates/metafile" } +craft-offcrypto = { path = "crates/offcrypto" } +craft-opc = { path = "crates/opc" } +craft-openpgp = { path = "crates/openpgp" } +craft-ora = { path = "crates/ora" } +craft-pptx = { path = "crates/pptx" } +craft-penpot = { path = "crates/penpot" } +craft-pcraft-format = { path = "crates/pcraft-format" } +craft-pcraft-bundle = { path = "crates/pcraft-bundle" } +craft-pkcs7 = { path = "crates/pkcs7" } +craft-plotter = { path = "crates/plotter" } +craft-prepress = { path = "crates/prepress" } craft-print = { path = "crates/print" } +craft-proof = { path = "crates/proof" } +craft-raster-codecs = { path = "crates/raster-codecs" } +craft-image-codecs = { path = "crates/image-codecs" } +craft-psd = { path = "crates/psd" } +craft-pst = { path = "crates/pst" } craft-quit = { path = "crates/quit" } +craft-rtf = { path = "crates/rtf" } +craft-raw = { path = "crates/raw" } +craft-sheet = { path = "crates/sheet" } +craft-sketch = { path = "crates/sketch" } craft-settings = { path = "crates/settings" } +craft-scraft-format = { path = "crates/scraft-format" } +craft-time = { path = "crates/time" } +craft-silhouette-studio = { path = "crates/silhouette-studio" } +craft-svg = { path = "crates/svg" } +craft-stencilcraft-format = { path = "crates/stencilcraft-format" } +craft-stencilcraft-document = { path = "crates/stencilcraft-document" } +craft-svg-paths = { path = "crates/svg-paths" } +craft-stream-codecs = { path = "crates/stream-codecs" } +craft-text = { path = "crates/text" } +craft-tga = { path = "crates/tga" } +craft-timeline = { path = "crates/timeline" } +craft-tiff = { path = "crates/tiff" } +craft-metadata = { path = "crates/metadata" } craft-ui = { path = "crates/ui" } -egui = { version = "0.36", default-features = false } +craft-update = { path = "crates/update" } +craft-vcard = { path = "crates/vcard" } +craft-vectorcraft-format = { path = "crates/vectorcraft-format" } +craft-vectordrawable = { path = "crates/vectordrawable" } +craft-xls = { path = "crates/xls" } +craft-xfl = { path = "crates/xfl" } +craft-wordmodel = { path = "crates/wordmodel" } +craft-xlsx = { path = "crates/xlsx" } +indd = { git = "https://github.com/storytold/indd-utils", rev = "7f3060e5bce78d21ea70e0d5eacb96edce15dfbe" } +craft-xcf = { path = "crates/xcf" } +craft-xml = { path = "crates/xml" } +craft-zip = { path = "crates/zip" } +craft-y4m = { path = "crates/y4m" } +craft-epub = { path = "crates/epub" } +craft-ews = { path = "crates/ews" } +craft-video = { path = "crates/video" } +craft-web-api = { path = "crates/web-api" } +craft-window = { path = "crates/window" } +craft-web-host = { path = "crates/web-host" } +craft-webp = { path = "crates/webp" } +egui = { version = "0.37", default-features = false } +md5 = { package = "md-5", version = "0.10", default-features = false } +jxl-oxide = { version = "0.12", default-features = false } +jixel = "=0.3.4" rayon = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +proptest = "1" +# Character sets of the WHATWG Encoding Standard (craft-mime). (Apache-2.0 OR MIT) AND BSD-3-Clause. +encoding_rs = "0.8" +# All date/time conversion goes through chrono (no hand-rolled calendar code beyond the 400-year range extension; AGENTS.md). +chrono = { version = "0.4.45", default-features = false, features = ["std"] } [workspace.lints.rust] unsafe_code = "forbid" +unused_workspace_dependencies = "allow" unexpected_cfgs = { level = "warn", check-cfg = ["cfg(craft_pdf_unicode_fixtures)"] } [workspace.lints.clippy] +cognitive_complexity = "deny" +too_many_arguments = "deny" +unused_workspace_dependencies = "allow" +wildcard_imports = "deny" dbg_macro = "warn" todo = "warn" @@ -43,3 +197,37 @@ hayro = { path = "vendor/hayro" } hayro-interpret = { path = "vendor/hayro-interpret" } hayro-syntax = { path = "vendor/hayro-syntax" } hayro-jbig2 = { path = "vendor/hayro-jbig2" } +hayro-cmap = { path = "vendor/hayro-cmap" } +hayro-postscript = { path = "vendor/hayro-postscript" } +linesweeper = { path = "vendor/linesweeper" } +ecolor = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +emath = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +epaint = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +epaint_default_fonts = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui-winit = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui-wgpu = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui_glow = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui_extras = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui_kittest = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui_system_font_provider = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +eframe = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui_demo_lib = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +egui_inspection = { git = "https://github.com/storytold/egui.git", rev = "6d518fc799681571963842f95b55a6b68431b15a" } +winit = { git = "https://github.com/storytold/winit.git", rev = "a517b3846ee8457d84ee4975110bfda6797c44dc" } + +# symphonia's FLAC decoder restores LPC samples as `residual + (prediction as i32)`. In a valid +# 32-bit stream the prediction can leave the i32 range while the sum does not, so a debug build +# panics where a release build wraps to the right sample. Apps that decode 32-bit FLAC through +# craft-audio's `decode` need the same override in their own workspace. +[profile.dev.package.symphonia-bundle-flac] +overflow-checks = false + +# Codec hot loops: the oracle and conformance tests of a debug build take minutes below opt-level 3. +[profile.dev.package.craft-video] +opt-level = 3 + +# The object-stream parser follows nesting 384 levels deep: unoptimised, its frames overflow the +# 2 MiB stack of a test thread. The apps this reader came from build their crates at this level. +[profile.dev.package.craft-affinity] +opt-level = 1 diff --git a/NOTICE b/NOTICE index 9467671f3..e1cffb26c 100644 --- a/NOTICE +++ b/NOTICE @@ -5,6 +5,17 @@ Licensed under the MIT License or the Apache License, Version 2.0, at your optio Third-party material +Optional DWG bridge + +`craft-dwg` uses `acadrust` 0.6.6 unmodified as a separate dependency for DWG reading and writing. +It is licensed under the Mozilla Public License 2.0. The license text and upstream third-party +notices are retained in `licenses/dwg/`; the crate is available from crates.io and +https://github.com/hakanaktt/acadrust. + +The byte-level bridge source was moved from CADCraft `crates/dwg` at revision +`664837df8925f932e90a7ae37c58d2f54e7a1320`. The bridge itself is MIT OR Apache-2.0; its only +MPL-2.0 component is the unmodified `acadrust` dependency. + This product bundles third-party material under its own licenses, listed below. The per-file list (author, source, license) is in ATTRIBUTION.md. @@ -19,11 +30,101 @@ crates/cms/tests/data/FOGRA39.txt: FOGRA39 characterisation data, distributed un the data are distributed unmodified and Fogra is identified as the source. The designation FOGRAxx may be used solely to identify the respective reference data. Such use does not imply certification, approval or endorsement by Fogra." +crates/text/assets/hyphen/en-us.dic: public domain, derived from the Moby Hyphenator II list by + Grady Ward (Project Gutenberg eBook #3204). +crates/proof/data/en-words.dic: public domain, the words of the Moby Hyphenator II list by Grady + Ward (Project Gutenberg eBook #3204), without its break points. +crates/proof/data/en-extra.txt: MIT OR Apache-2.0, WordCraft contributors' own list of modern + English words (a copy of craft-i18n's language-data/en-us/en-extra.txt). +craft-proof Language data, embedded at build time from craft-i18n's language-data/ when + CRAFT_I18N_DIR is set (CRAFT_PROOF_EMBED picks the locales; by default English and the German, + Polish, Portuguese, Italian and Russian data). Each file keeps its licence header; the + licences are listed in craft-i18n's ATTRIBUTION.md, and a pattern file's notice ships with any + application that embeds it. The Portuguese hyphenation patterns (language-data/pt/ + hyphenation.pat, hyph-utf8 hyph-pt 1.4) are under the BSD 3-clause licence: + Copyright (C) 1987, 1994, 1996, 2015 Pedro J. de Rezende, 1996, 2015 J. Joao Dias Almeida, + 2024 Leonardo Araujo and Aline Benevides. + Redistribution and use in source and binary forms, with or without modification, are + permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + * Neither the name of the University of Campinas, of the University of Minho nor the names of + its contributors may be used to endorse or promote products derived from this software + without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PEDRO J. DE REZENDE OR J.JOAO + DIAS ALMEIDA BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +crates/text/assets/hyphen/en-us.pat: CC0-1.0, trained from en-us.dic by DesignCraft's trainer. +Hyphenation patterns for other languages, embedded through the hypher crate only when the + matching craft-text feature is enabled (none are by default). They come from hyph-utf8, + https://github.com/hyphenation/tex-hyphen, as packaged in https://github.com/typst/hypher + (patterns/). Each file is under the license stated in its header: LPPL, MIT, BSD-3-Clause, the + Unlicense, a notice permitting any use, or a choice of licenses that includes one of these. An + application that enables a language ships that file's copyright and license notice. The + exception is Slovak (hyph-sk.tex, Copyright (C) 1992 Jana Chlebíková), under the GNU General + Public License, version 2 or later (SPDX: GPL-2.0-or-later). craft-text asks for it through the + feature "slovak-gpl", which "all-languages" does not include; an application that enables it + ships GPL-licensed data. Cargo unifies features, so another dependency that enables hypher's + "slovak" feature (hypher's default features do) embeds the same data; a build that must exclude + it checks its dependency graph. The Spanish patterns (feature "spanish") are hyph-es.tex 5.0. + Copyright (C) 1993, 1997 Javier Bezos, 2001-2019 Javier Bezos, CervanTeX. MIT License: + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: The above copyright notice + and this permission notice shall be included in all copies or substantial portions of the + Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE + AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +- Word-break dictionaries and LSTM models for Thai, Lao, Khmer and Burmese, embedded through the + icu_segmenter_data crate only when craft-text's "thai", "lao", "khmer" or "burmese" feature is + enabled (none are by default). From ICU4X, https://github.com/unicode-org/icu4x. + Copyright (c) 2020-2024 Unicode, Inc. Unicode License v3 (SPDX: Unicode-3.0); portions adapted + from ICU4C, (c) 1995-2016 International Business Machines Corporation and others. An + application that enables one of these features ships the Unicode License v3 copyright and + permission notice, https://www.unicode.org/license.txt. The optional PDF writer bundles four CC0 ICC profiles by Clinton Ingram. See ATTRIBUTION.md for per-file sources and hashes. Source provenance +crates/image-codecs (craft-image-codecs) is moved from PhotoCraft's standalone `photocraft-codecs` +crate, including its bounded decoders, encoders, format registry, image model, metadata handling, +orientation, fidelity warnings, and tests, from revision 5000240c6: +https://github.com/storytold/photocraft +Copyright (c) 2026 ArtCraft Team and the PhotoCraft contributors. The format implementations are +clean-room work based on their public specifications; dependency licences and X.Org's `rgb.txt` +attribution are listed in ATTRIBUTION.md. + +crates/raw (craft-raw) is moved from LightCraft, with behaviour ported from two other readers: +- LightCraft, crates/raw (src/lib.rs, dng.rs, dngwrite.rs, tiffraw.rs, unpack.rs, ljpeg.rs, jxl.rs, + opcodes.rs, profile.rs, gaintable.rs, semantic.rs, preview.rs, vendor/crw.rs, vendor/hasselblad.rs, + vendor.rs (shared level helpers), vendor/mrw.rs, vendor/erf.rs, vendor/mef.rs, vendor/kodak.rs, vendor/pef.rs, + vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, vendor/nef.rs, vendor/nefc.rs, vendor/nef_lens.rs, docs/nikon-lens-corrections.md, vendor/rw2.rs, vendor/raf.rs, vendor/rafc.rs, vendor/raf_lens.rs, docs/raf-compression.md, docs/raf-corpus.sha256, vendor/orf.rs, docs/orf-compression.md, vendor/arw.rs, vendor/arw6.rs, vendor/arw6_curve.rs, vendor/arw6_testenc.rs, vendor/arw_lens.rs, vendor/arw_lens_rm6.rs, vendor/sr2.rs, llvc.rs, lens_metadata.rs, docs/arw6-compression.md, docs/sony-lens-corrections.md, docs/sony-ilce7rm6-lens-corrections.md, vendor/cr3.rs, vendor/crx.rs, vendor/crx_wavelet.rs, sparse.rs, saturation.rs (the CR3 white extension), docs/cr3.md, docs/raw/, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), + revision 5ff567a4a2e067bef1b6bce152c11e7f727ccbaf. + https://github.com/storytold/lightcraft + Copyright (c) 2026 ArtCraft Team and the LightCraft contributors. +- PhotoCraft, crates/raw/src/dng.rs, hasselblad.rs and tests/it/dng.rs (default-crop defaults, + white-level fallbacks, the 8-bit rule for lossy JPEG, Hasselblad's per-site black levels), + revision c1b22bdaae2a1359ba3d164eca4d4b0c7378b547. + https://github.com/storytold/photocraft + Copyright (c) 2026 ArtCraft Team and the PhotoCraft contributors. +- FilmCraft, crates/dng/src/lib.rs (the CinemaDNG frame tags and time code), revision + 4bec22bd03bd1d995c9cf72e02a284f96d3d72cd. + https://github.com/storytold/filmcraft + Copyright (c) 2026 ArtCraft Team and the FilmCraft contributors. The SVG icon preparation in crates/ui is adapted from: - PhotoCraft, crates/ui-egui/src/icons.rs, revision e7a83ca342cdff568254f85f442b9cc8afb08066. https://github.com/storytold/photocraft @@ -53,6 +154,13 @@ The transactional panel model and drop-zone semantics in crates/ui/src/docking d FilmCraft/EffectCraft split, tab and accordion layouts and VectorCraft floating groups listed above. The generic transaction validation, optional renderer and workspace-location helpers are implemented here without dependencies on application engines or native window helpers. +The hyphenation code, data and generator in crates/text (src/hyphen/, assets/hyphen/ and +examples/hyphgen.rs) are adapted from: +- DesignCraft, crates/compose/src/hyphen/, crates/compose/examples/hyphgen.rs and + assets/hyphenation/, revision e161878e4c7744fa06f9a2eede8103cb4c296355 (above). +- WordCraft, crates/proof/src/hyphen.rs, revision e4b7d155f15a7784197ba34eeea0b54490d70eb2. + https://github.com/storytold/wordcraft + Copyright (c) 2026 ArtCraft Team and the WordCraft contributors. All sources listed above are licensed MIT OR Apache-2.0. No icon artwork is copied here. The PDF spooler in crates/print is extracted from: @@ -75,6 +183,31 @@ PrintTicket and its driver-paper preference): Copyright (c) 2026 ArtCraft Team and the PdfCraft contributors. Licensed MIT OR Apache-2.0. +The menu layout and native-menu driver in crates/menu are extracted from: +- PhotoCraft, crates/ui-egui/src/native_menu.rs, revision 6f85054288a564f9fbbccb99268ffdb34e581584. + https://github.com/storytold/photocraft + Copyright (c) 2026 ArtCraft Team and the PhotoCraft contributors. + Licensed MIT OR Apache-2.0. PhotoCraft's menus, translations and command handling are not copied. +The macOS backend in crates/menu/src/mac.rs is adapted from: +- PhotoCraft, apps/photocraft/src/mac_menu.rs, revision 6f85054288a564f9fbbccb99268ffdb34e581584. + https://github.com/storytold/photocraft + Copyright (c) 2026 ArtCraft Team and the PhotoCraft contributors. +- VectorCraft, apps/vectorcraft/src/mac_menu.rs, revision 0d0ea352df99044589a7ce0efc7004f7a5b6b7c0 + (closing an open menu on a click or Escape, and its tests, in crates/menu/src/driver.rs). + https://github.com/storytold/vectorcraft + Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors. + Both licensed MIT OR Apache-2.0. +The egui menu bar in crates/menu/src/bar (its menu navigation, scrolling, submenu hover +intent, keyboard access and mnemonics, and their tests) is ported from: +- PhotoCraft, crates/ui-egui/src/menu_nav.rs and menu_nav_tests.rs, menu_keys.rs and + menu_keys_tests.rs, and the menu bar in crates/ui-egui/src/menus.rs, revision + c36e46c3873ee39f299cc50a3dcd28494c78366d. + https://github.com/storytold/photocraft + Copyright (c) 2026 ArtCraft Team and the PhotoCraft contributors, among them aiongg, artman67, + BFlat, Brandon Thomas, Brennan Stehling, Daniel Lošák, David Mann, Fabio Lissi, FredApps, + Higgs32584, itsF4LCON, jason4short, Korg1ch, MarekSpodBiedry and Nigel Ihza Firdausy. + Licensed MIT OR Apache-2.0. PhotoCraft's menus, theme and Help search are not copied. + The colour management module in crates/cms is extracted from: - PhotoCraft, crates/cms, revision b3222ee8e7e6bcb4799c37f904c595f20f9baf5e. https://github.com/storytold/photocraft @@ -91,18 +224,74 @@ The optional writer embeds Compact ICC Profiles by Clinton Ingram See licenses/pdf-writer/LICENSE-ICC.txt and ATTRIBUTION.md for sources and hashes. The writer adds no font files. Its four existing ICC profiles are retained under vendor/krilla/icc. -Krilla is vendored with a native unwind-cleanup patch; see vendor/krilla/README.craft.md +Krilla is vendored with the local patches in vendor/krilla/patches; see vendor/krilla/README.craft.md and vendor/krilla/SOURCE.json. Its upstream MIT/Apache-2.0 licences and CC0 profiles are unchanged. Optional PDF reader -Hayro 0.7.1, hayro-interpret 0.7.0, hayro-syntax 0.7.2 and hayro-jbig2 0.3.0 -(https://github.com/LaurenzV/hayro), MIT OR Apache-2.0, retain their upstream licence files. -These patched sources are transferred from PdfCraft revision -7437bde378f43856eada40af25b983812e615e8a (https://github.com/storytold/pdfcraft). +Hayro 0.7.1, hayro-interpret 0.7.0, hayro-syntax 0.7.2, hayro-jbig2 0.3.0, hayro-cmap 0.1.0 +and hayro-postscript 0.1.0 (https://github.com/LaurenzV/hayro), MIT OR Apache-2.0, retain their +upstream licence files. The first five are patched sources transferred from PdfCraft +(https://github.com/storytold/pdfcraft): hayro-cmap from revision +a2ec27b4775f53c40cad3456936d9406611e3a8d, the others from revision +ec0a965074f376a5bf610435948e4e5184bf3299. hayro-postscript is the crates.io 0.1.0 package +(upstream revision a2354aebaf86f5ab13e5fd6b6b0521306488bf9d) with one local patch. See vendor/README.craft-reader.md and each crate's SOURCE.json for provenance and local changes. +The hayro-interpret plate mode (separations, overprint and output-intent simulation) is ported +from PdfCraft pull request #2770 by Guillem Hermida, commit +6fcf2a4aca29e943e28cb69017ff8f87d00656c6 (https://github.com/storytold/pdfcraft/pull/2770), +MIT OR Apache-2.0; crates/pdf/tests/plates.rs and tests/support/synthetic_icc.rs adapt its tests. +Knockout transparency groups in hayro-interpret and hayro are ported from PdfCraft pull request +#2974 by mtlcl410, commit bae69819594a6385a027fec1f67e6b3c36c259b4 +(https://github.com/storytold/pdfcraft/pull/2974), MIT OR Apache-2.0; +crates/pdf/tests/knockout.rs adapts its test. The reader retains the existing Foxit/PDFium standard-14 font assets (BSD-3-Clause), Compact ICC Profiles by Clinton Ingram (CC0-1.0), and dependency CMaps (BSD-3-Clause). Their licences and per-file hashes are retained in vendor/hayro-interpret/assets, -licenses/pdf-backend and ATTRIBUTION.md. These are existing reader dependency assets; +vendor/hayro-cmap/assets, licenses/pdf-backend and ATTRIBUTION.md. These are existing reader dependency assets; applications retain their own document-font selection and licensing policy. + +linesweeper + +linesweeper 0.5.0 (https://crates.io/crates/linesweeper/0.5.0), MIT OR Apache-2.0, is stored in +vendor/linesweeper with its upstream license files and one local patch. Its manifest names Joe Neeman +as author, and its LICENSE-MIT reads "Copyright (c) 2018 Raph Levien". See +vendor/linesweeper/README.craft.md and vendor/linesweeper/SOURCE.json for provenance, the local patch +and the files left out. + +Platform lifecycle source provenance + +The optional crates/platform services adapt the preserved local platform candidate +9b135779a1e7136438a217c18afe1693ea23f1ab (dialog outcomes and browser resource ownership), +and current application implementations: +- PdfCraft crates/platform/src/desktop_theme.rs, a8cd797a917b1007544575c7280c9a0ce6a17e8f. +- CADCraft apps/cadcraft/src/desktop_theme.rs, 0cc66ad3066e71f2c3d4cacb7de905e79ba1c95b. +- ArtCraft Launcher apps/artcraft-launcher/src/system_theme.rs, f1676dfb41c226b5be490721f486ee6427419d80. +- Earlier PhotoCraft, FilmCraft and EffectCraft appearance services at revisions + 0c72d95425dece90ef9a1cceb49e3315c96e22d5, 794a25fc9ed2e3f6fab51a254625bc9b99a88015 + and 1b4d47f34f2b1c1b4a89f4b7348889def8fd648b respectively. +Sources: https://github.com/storytold/pdfcraft, https://github.com/storytold/cadcraft, +https://github.com/storytold/craft-launcher, https://github.com/storytold/photocraft, +https://github.com/storytold/filmcraft, https://github.com/storytold/effectcraft. +Copyright (c) 2026 ArtCraft Team and the respective application and craft-libs contributors. +All adapted source is licensed MIT OR Apache-2.0. + +The Canon CRW decoder in crates/crw (src/lib.rs, src/tables.rs, src/tests.rs, tests/) and its +derivation record (crates/crw/clean-room/) come from: +- LightCraft, crates/raw/src/vendor/crw.rs, crates/raw/src/vendor/crw_tables.rs, + crates/raw/tests/crw_oracle.rs, crates/raw/tests/it/vendor_robust.rs and clean-room/crw/, + https://github.com/storytold/lightcraft/pull/3021, revision 6ca439b5155820d2befeacd81ec0c6a80b2bd3bc + (decoder ported at 70c755e42a65661401142819278d51b0c5977f7f; the record is identical at both). + https://github.com/storytold/lightcraft + Copyright (c) 2026 ArtCraft Team and the LightCraft contributors. +Its white-level plateau estimate (saturation_plateau) is LightCraft's vendor::white_from_data at the +same revision. The CRW compression was derived clean-room from CC0 sample files and the +maintainer's own photographs by black-box comparison with a reference decoder's command-line +output; no raw-decoder source was consulted (crates/crw/clean-room/sources.md). + +crates/audio/src/alac/ (the ALAC decoder and encoder) is ported from Apple's ALAC reference +implementation, https://github.com/macosforge/alac at c38887c5c5e64a4b31108733bd79ca9b2496d987 +(codec/), Copyright (c) 2011 Apple Inc., licensed under the Apache License, Version 2.0 +(LICENSE-APACHE). The files are modified: translated to Rust, with bounds checks on hostile input, +wrapping integer arithmetic, the unrolled predictor and per-width matrix routines folded into one, +and the average bit rate filled in. Those files are Apache-2.0 only. diff --git a/README.md b/README.md index a5db4f308..d094847d0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ Craft App Libraries =================== +Scanned by SonarCloud + This repo contains libraries used by the various Craft Apps. It's a Rust monorepo workspace, and the libraries can be included by any Craft Apps that need them. In the future this may grow to include non-Rust library code. @@ -15,29 +17,372 @@ A Cargo workspace (edition 2024, resolver 3). Each library is a crate under `cra (`license.workspace = true`, and so on) and `[lints] workspace = true`. Register each crate in `[workspace.dependencies]` in the root `Cargo.toml`. -Apps depend on a crate by git: +## Use a released version + +The first coordinated release is **[v0.1.0](https://github.com/storytold/craft-libs/releases/tag/v0.1.0)**. +It gives consumers a fixed baseline while development continues on `main`. All five crates +share this version. They require Rust 1.95 or later; the UI integrations use egui 0.36. + +Add only the crates you need, using the same tag for every craft-libs dependency: ```toml -craft- = { git = "https://github.com/storytold/craft-libs", rev = "" } +[dependencies] +craft-ui = { git = "https://github.com/storytold/craft-libs", tag = "v0.1.0" } +craft-settings = { git = "https://github.com/storytold/craft-libs", tag = "v0.1.0" } +# Optional, for egui close handling and the macOS application menu: +craft-quit = { git = "https://github.com/storytold/craft-libs", tag = "v0.1.0", features = ["menu"] } ``` +`craft-atomic` and `craft-print` use the same dependency syntax. Enable `craft-ui`'s +`serde` feature if you need to serialize its docking layout. These are Git releases; +crates.io publication is deferred. + +Release tags are never moved or reused. For a commit pin, resolve the tag with +`git ls-remote https://github.com/storytold/craft-libs 'refs/tags/v0.1.0^{}'` and replace +`tag = "v0.1.0"` with `rev = ""` on every craft-libs dependency. +Use either `tag` or `rev`, not both. An existing commit pin keeps working; switching to this +release may require adapting to changes since that commit. + +Commit your application's `Cargo.lock` and build with `cargo build --locked` for reproducible +Rust dependency resolution. A Git pin fixes craft-libs source, not every transitive dependency. +A library's own lockfile does not constrain its downstream consumers. Upgrade deliberately: +read the release notes, update all craft-libs pins together, refresh the lockfile with +`cargo check`, and run your application's tests and affected workflows before committing. + +See the [release notes](CHANGELOG.md) for included functionality and known limits, and the +[versioning policy](docs/versioning.md) for compatibility guarantees and release checks. + ## Libraries +- [`craft-platform`](crates/platform/README.md): optional native dialog lifecycle, Linux/BSD desktop appearance observation and browser download ownership; application policy stays in the apps. + +- [`craft-aep`](crates/aep/README.md): After Effects projects: the `.aep` (RIFX) and `.aepx` + container as a lossless chunk tree that writes back byte for byte, and a typed, read-only view + of compositions, layers, properties, keyframes, effects, masks, shapes, text and footage. + Clean room, from EffectCraft. Mapping onto an app's document stays in the app. +- [`craft-ai-core`](crates/ai-core/README.md): the data model and contract for AI in the Craft Apps (connections, models, tools, requests, jobs, consent, history, route spec). Pure, no I/O, wasm-safe. Contract in [docs/ai-shared-layer.md](docs/ai-shared-layer.md). - [`craft-pdf`](crates/pdf/README.md): independently optional PDF parsing, interpretation, rasterization and guarded authoring. Apps retain layout, fonts, editing and export policy; writer-only consumers do not enable the shared reader. +- [`craft-pdn`](crates/pdn/README.md): Paint.NET documents (`.pdn`, PDN3), read: the layer list, blend modes and chunked gzip pixel blocks, with a data-only MS-NRBF (.NET binary serialization) reader. From PhotoCraft. +- [`craft-aaf`](crates/aaf/README.md): clean-room AAF (Edit Protocol, structured storage) and OMF Interchange 2.0 (Bento) readers and writers over one mob-style composition model with exact times, clip gain, markers, nested compositions and embedded PCM essence. From FilmCraft. +- [`craft-assuan`](crates/assuan/README.md): the Assuan protocol and gpg-agent's operations (key lookup, signing a digest, decrypting a session key) over any `Read + Write` stream, with a bounded canonical S-expression reader. No sockets; everything the agent sends is capped. +- [`craft-affinity`](crates/affinity/README.md): bounded reader for native Affinity documents (`.af`, `.afdesign`, `.afphoto`, `.afpub`): archive, object stream and a neutral document model with layers, artboards, vectors, text, tables, images and masks. Read only; mapping onto an app's document stays in the app. - [`craft-atomic`](crates/atomic/README.md): Crash-safe atomic file writing and replacement with same-directory staging, directory fsync, Apple `F_FULLFSYNC` fallback, transient lock backoff, and fault injection for the Craft suite. +- [`craft-captions`](crates/captions/README.md): caption and subtitle files behind one neutral + model (cues with exact tick times): SubRip, WebVTT, Scenarist SCC, MacCaption MCC, EBU STL and + TTML / IMSC1 / DFXP readers and writers, and CEA-608 / CEA-708 byte-pair coding. Carved from + FilmCraft. +- [`craft-cfb`](crates/cfb/README.md): clean-room Compound File Binary ([MS-CFB], OLE structured + storage) reader and writer: the container of legacy Office files, Outlook messages, encrypted + OOXML packages and AAF. Bounded on hostile input; what the streams hold stays with each format. +- [`craft-dav`](crates/dav/README.md): WebDAV multistatus documents (RFC 4918) with the CalDAV and + CardDAV namespaces, from MailCraft: read fail-closed into a bounded `craft-xml` tree with fixed + prefixes, and the successful properties of each response. No I/O. - [`craft-cms`](crates/cms/README.md): pure-Rust ICC v2/v4 colour management: profile parsing and writing, transforms, rendering intents, black point compensation, soft proofing, gamut checks, 3D LUTs and built-in profiles (sRGB, Display P3, synthetic coated CMYK and others). +- [`craft-crs`](crates/crs/README.md): Adobe Camera Raw settings (the `crs:` namespace of an XMP packet or `.xmp` sidecar, read through craft-metadata): the settings by name, numbers, tone curve points, local corrections and the crop, read and written. From PhotoCraft. +- [`craft-codepage`](crates/codepage/README.md): legacy 8-bit code pages, one table each: the single-byte Windows pages, Mac OS Roman and fifteen DOS pages, keyed by Windows code page number, with the RTF/Word font charsets. craft-rtf, craft-dxf and craft-mime decode through it. +- [`craft-crw`](crates/crw/README.md): clean-room Canon CRW (CIFF) raw decoder, from LightCraft: + the heap container, the entropy coding derived by black-box analysis (frozen code tables), sensor + black and white levels and as-shot white balance. The complete derivation record (plan, sources + log, scripts, verification) is in `crates/crw/clean-room/`. Colour stays in each app. +- [`craft-csv`](crates/csv/README.md): delimited text (CSV, TSV) and SYLK: encoding detection (UTF-8, UTF-16, Windows-1252 or Windows-1256), delimiter sniffing, a record parser with a cap on fields per record, field quoting for writers, and a SYLK (`.slk`) reader with R1C1 formulas rewritten to A1. +- [`craft-dialogs`](crates/dialogs/README.md): native file and message dialogs that never block + the event loop: queued requests shown at the end of the frame and answered on a later frame, + free-standing Open and Folder panels and Save sheets on macOS, case-insensitive filters. What + to do with the answer stays in each app. +- [`craft-doc`](crates/doc/README.md): reads PhotoCraft `.pcraft`, VectorCraft `.vectorcraft` + and EffectCraft `.ecproj` documents without the apps: the kind and version, the previews and + PDF they carry, and their pages and layers. Rendering and editing stay in each app. +- [`craft-doc-binary`](crates/doc-binary/README.md): the Word 97-2003 binary format (`.doc`, [MS-DOC]) read into + `craft-wordmodel`, with XOR, RC4 and RC4 CryptoAPI password protection. WordCraft's reader, on `craft-cfb`. +- [`craft-docpath`](crates/docpath/src/lib.rs): decides whether a path or `file:` URL stored in a + document may be opened. Paths stored in documents are followed only when they are local or on the + document's own network share; other network paths, device paths and non-file URLs are refused. +- [`craft-dpx`](crates/dpx/src/lib.rs): DPX (SMPTE ST 268) and Kodak Cineon frames, from PhotoCraft, + FilmCraft and EffectCraft: header fields and code values at 8, 10, 12 and 16 bits in both byte + orders, filled and packed, RGB, RGBA, ABGR, luma and Y'CbCr descriptors, and a writer for both + formats. Bounded on hostile input. Colour conversion and orientation stay in each app. +- [`craft-drawingml`](crates/drawingml/README.md): the DrawingML that PPTX, DOCX and XLSX share: + EMU units and `a:xfrm`, colours with theme slots and colour transforms, preset shape outlines + (`a:prstGeom`), custom geometry with guide formulas (`a:custGeom`) and preset text warps + (`a:prstTxWarp`). From DeckCraft. Takes `craft-xml` trees; packages stay with each format. +- [`craft-eps`](crates/eps/README.md): Encapsulated PostScript files as a container: the binary + preview header (read and write), the DSC header comments and bounding boxes (also `(atend)`), + the EPSI preview bitmap, ASCII85 data, and telling EPS, PostScript and Illustrator files + (PostScript- or PDF-based) apart. No PostScript interpreter. +- [`craft-docx`](crates/docx/README.md): DOCX (WordprocessingML) read and written on `craft-wordmodel`, with + Word XML (Flat OPC) and Word 2003 XML, password-protected packages and Restrict Editing hashes. + WordCraft's DOCX crate on `craft-opc`, `craft-xml` and `craft-offcrypto`. +- [`craft-dxf`](crates/dxf/src/lib.rs): DXF tag reader and writer, from CADCraft: ASCII and + binary DXF as (group code, value) tags, split into sections and entities, with pre-2007 code + pages and version detection, and written back; and `model`, the drawing above the tags (header + variables, tables, blocks, layouts, entities) and its writer (R12 to 2018), merged from CADCraft's and VectorCraft's readers. Bounded + on hostile input. Mapping the model onto a document stays in each app. +- [`craft-dwg`](crates/dwg/README.md): bounded byte-level DWG↔DXF conversion with diagnostics, + capped output and an optional child-process worker; the `acadrust` dependency remains isolated + behind bytes and is unavailable on wasm32 and FreeBSD/aarch64. +- [`craft-ecproj-format`](crates/ecproj-format/README.md): the envelope of EffectCraft `.ecproj` + project files: `savedBy`, the schema version and its check, and the file object with the + project and its saved view as type parameters. The project model stays in EffectCraft. +- [`craft-fig`](crates/fig/README.md): clean-room reader and writer for the `.fig` container + (Figma Design and FigJam local copies): the ZIP archive, the chunked canvas with its kiwi schema + and message, images, and the vector network and geometry blobs. Bounded on hostile input. + Carved from FigCraft. +- [`craft-epub`](crates/epub/README.md): EPUB packages: the OCF ZIP container, the package document (metadata, manifest, spine) and the navigation document, written as EPUB 3 (reflowable and fixed layout) and read (EPUB 2 and 3) with bounded entry sizes. Content documents are the caller's. +- [`craft-figma-rest`](crates/figma-rest/README.md): typed model of the Figma REST API file JSON + (document tree, paints, effects, text styles, layout, component metadata), read and written with + serde without losing fields it has no name for. The neutral model the `.fig` and `.sketch` + readers map onto, with their extensions documented. +- [`craft-ews`](crates/ews/README.md): the SOAP layer of Exchange Web Services, from MailCraft, without + sockets: the request envelope and an escaping element builder, and the answer envelope, faults + and response messages with their codes, read with `craft-xml`. The operations and everything + over HTTP stay in each app. +- [`craft-exr`](crates/exr/src/lib.rs): OpenEXR pieces shared by PhotoCraft and EffectCraft: layer + and channel names, the channels that make a colour picture, and Cryptomatte IDs, metadata keys, + manifests and rank channels. The optional `decode` feature adds bounded part listing, flat + decoding (scan lines, tiles, every compression of the `exr` crate) and writing. +- [`craft-flags`](crates/flags/README.md): Feature flags shared by the Craft apps and the ArtCraft Launcher: one per-user `ArtCraft/flags.toml`, a `CRAFT_FLAGS` override, every flag off by default (`AI-MCP`, `AI-MODELS`). +- [`craft-flow`](crates/flow/README.md): the flow model every text format maps to (paragraphs with a kind, lists and formatted spans; tables; pictures, fields, notes, equations), converted to and from a craft-wordmodel `Document`. +- [`craft-font-catalog`](crates/font-catalog/README.md): the installed fonts on every platform + (font folders, fontconfig and Flatpak host folders, CoreText, DirectWrite, the font store), + lookup by family, style, localized, legacy, full and PostScript names, family loads shared + between threads, and rescans when fonts are installed or removed. +- [`craft-font-manager`](crates/font-manager/README.md): Craft Font Manager as a library: search + and filters over the Google Fonts index, installed fonts by family, the font store interface, + and (`egui` feature) `FontsPanel`, the whole interface as one panel for the launcher's Fonts tab + and the standalone app. +- [`craft-font-store`](crates/font-store/README.md): the suite's font store: the pinned Google + Fonts and Noto indexes plus indexes the user adds (never curated), exact lookup by family and + style or PostScript name, and installs from each source's URLs (jsDelivr, then raw GitHub) + checked by size, sha256 and parsing, with the licence, a manifest, and removal. Off until the + user turns downloading on; agents can't. +- [`craft-fonts-build`](crates/fonts-build/README.md): embeds the optional craft-fonts faces + from an app's `build.rs` (`CRAFT_FONTS_DIR`, `CRAFT_FONTS_REQUIRED`, script filters) as a table + of `EmbeddedFace`s. No dependencies; the app picks the faces. +- [`craft-gainmap`](crates/gainmap/README.md): HDR gain maps, from LightCraft: the ISO 21496-1 + metadata and Adobe `hdrgm` XMP (read and write), measuring a gain map from SDR and HDR + renditions, applying one, and Apple's legacy maps. Encoding and the container stay in each app. +- [`craft-gcode`](crates/gcode/src/lib.rs): Grbl-style G-code writer: polylines in + millimetres to `G0`/`G1` moves with laser power and feed, for cutting lasers and plotters, as a + commented file or as bare lines for streaming. Its `grbl` module has the real-time command bytes + and the status and acknowledgement parsers. +- [`craft-gif`](crates/gif/src/lib.rs): GIF block reader, from PhotoCraft: the logical screen, + every frame's position, delay, disposal and transparent colour, the loop count and the first + frame's colour table, without decompressing pixels. Tells an animation from a still and whether + a file can hold the picture it declares before a decoder runs. +- [`craft-hdr`](crates/hdr/src/lib.rs): Radiance HDR (RGBE) picture reader, from FilmCraft: flat, + old and new run-length scan lines and all eight axis orders, to linear float RGB. Scan lines are + validated before the picture is allocated. +- [`craft-heif`](crates/heif/README.md): optional HEIF/HEIC still-image decoder, from LightCraft: + heic-rs reads the container and HEVC pictures, and this crate converts to 16-bit RGB to match + libheif (VUI range and matrix, chroma upsampling, `imir`, ICC on grid tiles). Also orientation, + alpha, EXIF/XMP and thumbnails. It never panics, and apps put it behind a non-default feature. +- [`craft-html`](crates/html/README.md): the bottom layer of reading and writing HTML: a tolerant tokenizer (text, start tags, end tags; elements to drop or to read as raw text are options), character references with the legacy names browsers accept without `;`, escaping, and the character set a page declares. With the `flow` feature, HTML documents read into and written from the craft-flow model. +- [`craft-i18n`](crates/i18n/README.md): translations over the embedded craft-i18n catalogs: + lookup with fallback (`pt-br` → `pt` → English), plurals, language names, the system locale and + a pseudo locale. The catalogs stay a fully optional build input; apps keep `tl!` and their + language setting. +- [`craft-raw`](crates/raw/README.md): camera raw files, the file-format layer, from LightCraft with + what PhotoCraft's and FilmCraft's readers added: container parsing and decompression to a neutral + raw image (samples, CFA, black and white levels, active area, crop, orientation, colour tags, + opcode lists, embedded previews), one Cargo feature per format. DNG reading and writing, + Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through + `craft-crw`) so far; demosaicing and everything after stay with the develop + pipeline. +- [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft + (writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any + origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on + hostile input. +- [`craft-idml`](crates/idml/README.md): IDML (InDesign Markup Language) packages: the ZIP container read with bounded part sizes and lenient part names, `designmap.xml` with its includes in place, the `` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written. +- [`craft-ical`](crates/ical/README.md): iCalendar (RFC 5545) building blocks, from MailCraft: content + lines, DATE/DATE-TIME values, durations, TEXT escaping, the common subset of recurrence rules, + and `VTIMEZONE` definitions read and written. No time-zone database; what events and to-dos + mean stays in each app. Bounded and panic-free on hostile input. +- [`craft-stream-codecs`](crates/stream-codecs/README.md): byte-stream and bi-level image codecs that + PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, + CCITT Group 3 and 4 decoding and Group 4 encoding, JBIG2 decoding and lossless encoding. Every + decoder is bounded and returns its partial output on damage. From PdfCraft; a PDF filter chain + is built on it in `craft-pdf`. +- [`craft-timeline`](crates/timeline/README.md): editorial timeline interchange: exact media time and SMPTE timecode, import/export reports, media path and `file://` URL handling, Avid Log Exchange (ALE) and CMX 3600 EDL documents, format detection. From FilmCraft. +- [`craft-time`](crates/time/README.md): exact rational ticks and SMPTE time across media, with a `soundcraft` compatibility module retaining SoundCraft's sample-index timebase, tempo maps, grids and session serialization semantics. +- [`craft-scraft-format`](crates/scraft-format/README.md): SoundCraft's typed `.scraft` session schema, versioned reader/writer, hostile-input validation, and `.scraftz` bundle reader/writer. Engine and UI mapping stays in SoundCraft. +- [`craft-metadata`](crates/metadata/README.md): photo metadata, from LightCraft: EXIF (read and write, + maker-note lens and focus point, every tag by name), IPTC-IIM, XMP packets (read, write, and a merge + that keeps what another program wrote), GPX track logs, and the blocks that carry them in JPEG, PNG, + WebP, TIFF-based, RAF and CR3 files. One `Metadata` model for all of them. Built on `craft-tiff`. +- [`craft-tiff`](crates/tiff/README.md): TIFF and BigTIFF IFD reader and writer, from LightCraft: + every field type, the IFD chain, SubIFDs and Exif/GPS/Interop pointer IFDs, maker-note IFDs + (Canon, Nikon, Sony, Fujifilm, Olympus, Panasonic, Pentax), strip and tile layout. Bounded and + panic-free on hostile input. Pixel decompression stays with the callers. +- [`craft-metafile`](crates/metafile/README.md): Windows metafiles, from DeckCraft, WordCraft and + VectorCraft: a WMF, EMF and EMF+ reader into a neutral picture (paths with line styles, pattern + fills, bitmaps, text, clip regions and clip paths) and, behind `write`, EMF and WMF writers. + Bounded and panic-free on hostile input. Mapping the picture onto a document stays in each app. +- [`craft-psd`](crates/psd/README.md): Adobe Photoshop PSD and PSB reader and writer, from + PhotoCraft: a lossless format-level model (layers, masks, tagged blocks, image resources, + descriptors, all four compressions, lazy reading from a file) that writes an unmodified file + back byte for byte, a builder for new files, and the Photoshop resource files that share its + primitives (`.abr`, `.asl`, `.aco`, `.act`, `.ase`, `.grd`, `.pat`, `.atn`, `.csh`, `.shc`) and + the Photoshop blocks of a layered TIFF. Bounded and panic-free on hostile input. Compositing + and the mapping onto a document stay in each app. +- [`craft-jmap`](crates/jmap/README.md): the JMAP protocol core (RFC 8620), from MailCraft, without + sockets: session, requests of method calls with back-references, responses, ids, address + templates, credentials and the judgement of HTTP answers, over any HTTP implementation. Bounded + on hostile answers; what a program asks and makes of the answers stays in it. +- [`craft-jpeg`](crates/jpeg/README.md): clean-room JPEG decoder for 12-bit and arithmetic-coded + frames (T.81 SOF0/SOF1 at 12 bits, SOF9, SOF10), band decoding, and an optimized 8-bit baseline + encoder, from PhotoCraft; includes a frame probe for routing. Metadata interpretation stays in + each app. +- [`craft-raster-codecs`](crates/raster-codecs/README.md): small raster formats, one module each, + from PhotoCraft: DDS (BC1 to BC3, read and write), DICOM, FITS (read and write), IFF/ILBM (read + and write), PCX, PNM/PAM/PFM (read and write) and XPM, into one flat image type. Bounded and + panic-free on hostile input. Colour, orientation and the document mapping stay in each app. +- [`craft-kiwi`](crates/kiwi/README.md): clean-room decoder and encoder for the kiwi binary schema + format (the encoding inside `.fig` files): a schema of enums, structs and messages, and messages + decoded with it into a tree keyed by field name. Bounded on hostile input. Carved from FigCraft. +- [`craft-keymap`](crates/keymap/README.md): keyboard bindings: chords and their text forms, + scopes, conflicts, user overrides stored through craft-settings and import of other + applications' shortcut sets. Command catalogs, file readers and the shortcut editor UI stay + in each app. +- [`craft-audio`](crates/audio/README.md): audio file I/O and codecs (WAV, AIFF, FLAC, AAC, Opus, + AC-3/E-AC-3, MP1/MP2, MP3 encoding), streaming and seeking, and packet decoding for any demuxer. See its + README for formats and features. +- [`craft-latex`](crates/latex/README.md): LaTeX documents read (the everyday subset) and written (a standalone article), to and from the craft-flow model. +- [`craft-markdown`](crates/markdown/README.md): Markdown read and written (a CommonMark subset with GitHub tables and strikethrough), to and from the craft-flow model. +- [`craft-math`](crates/math/README.md): equations as one tree (the Office Math object model) with its syntaxes, read and written: OMML, presentation MathML, LaTeX math and the UnicodeMath linear format. +- [`craft-media`](crates/media/README.md): bit-level I/O (`bitstream`), clean-room MP4/MOV + (`isobmff`), Matroska/WebM (`matroska`) and MXF (`mxf`) demuxers and muxers, Ogg (`ogg`), AVI (`avi`) and + MPEG-TS/PS (`mpegts`) demuxers, from FilmCraft, and an ISO BMFF box reader. Containers only; codecs are in `craft-audio`, and + video codecs will go in a separate crate. +- [`craft-lottie`](crates/lottie/README.md): Lottie (Bodymovin) JSON: the document model of + lottie-spec 1.0 (animation, assets, layers by type, shape items, masks, effects, text, markers) + with a bounded reader and a writer that keep what they do not know, and the encodings of its + values (keyframes, ease tangents, bezier paths, gradients, data URIs). From EffectCraft; mapping + onto an app's document stays in the app. +- [`craft-menu`](crates/menu/README.md): an app's menus as a macOS menu bar: the Mac layout + (app menu, Window, Help, system-key clashes), a platform backend trait, and key equivalents + fed back to egui as the input egui-winit would have made. Commands and their enabled state + stay in each app. +- [`craft-mesh`](crates/mesh/README.md): 3D mesh file formats behind one neutral model (meshes + of polygons with per-corner indices, materials, a node hierarchy): STL, OBJ with MTL, PLY, + glTF 2.0 / GLB and 3MF, one feature each. Readers are bounded by `Limits`, never open a file themselves + glTF 2.0 / GLB and FBX, one feature each. Readers are bounded by `Limits`, never open a file themselves + and never panic. Carved from EffectCraft, SolidCraft, TextureCraft and MaterialCraft. +- [`craft-midi`](crates/midi/README.md): Standard MIDI Files: a bounded reader for formats 0, 1 and + 2 and a format-1 writer, over a neutral model of notes and channel events in ticks at 960 per + quarter note. From SoundCraft. +- [`craft-mime`](crates/mime/README.md): Internet messages and MIME, read and written: RFC 5322 + header fields, MIME parts and multiparts, encoded-words, RFC 2231 parameters, base64 and + quoted-printable, character sets, address lists, dates, mbox files and the `multipart/related` + lookups of MHT web archives. What a message means (body, attachments, flags) stays in each app. +- [`craft-openpgp`](crates/openpgp/README.md): OpenPGP as mail carries it: PGP/MIME and cleartext + Digital signatures checked, attached Public keys and `Autocrypt:` headers read, a secret key + file inspected without using it. Built on rPGP; no I/O, and every size and count is capped. +- [`craft-odf`](crates/odf/README.md): OpenDocument packages (ODF 1.3 Part 2), read and written: the ZIP container with its `mimetype` and manifest within limits, password protection (AES-256-CBC with PBKDF2, as LibreOffice writes it) and flat single-file documents. With the `text` feature, OpenDocument Text documents read into and written from the craft-flow model. +- [`craft-prc`](crates/prc/README.md): a reader for PRC (ISO 14739-1), the 3D model format of PDF 3D + annotations: container, bit-level primitives, schema, tessellations and the product tree as + placed, coloured triangle meshes and a model tree. Bounded on hostile input. From PdfCraft; + rendering stays in the app. +- [`craft-offcrypto`](crates/offcrypto/README.md): clean-room Office document cryptography ([MS-OFFCRYPTO]): password-encrypted OOXML packages on `craft-cfb` (agile and standard encryption read, with the integrity check enforced; agile written), unsupported schemes named, and the password hashes of document and sheet protection. +- [`craft-ods`](crates/ods/README.md): OpenDocument spreadsheets (`.ods`, `.ots`) on `craft-odf`: the tables of `content.xml` with typed values, displayed text, formulas as text, style names, repeats expanded and merged cells, with dates and durations as serial numbers. Bounded cells, merges and text; strict XML. +- [`craft-opc`](crates/opc/README.md): Open Packaging Conventions, the ZIP container of DOCX, XLSX and PPTX + files: bounded readers (every part up front, or on demand), part names, relationships, content + types and a deterministic package writer. What the parts hold stays with each format's crate. +- [`craft-pcraft-format`](crates/pcraft-format/README.md): the envelope of PhotoCraft's `.pcraft` + bundle: `Manifest` with the document as a type parameter, `TileRef`, the entry paths, the + manifest decode limits and the migration runner. The document model stays in PhotoCraft. +- [`craft-pkcs7`](crates/pkcs7/README.md): the formats of digital signatures, read, verified and + written: ASN.1 DER, X.509 certificates and chains, CMS SignedData (PKCS #7), PKCS #12 identity + files, RFC 3161 timestamps, and CRL / OCSP revocation evidence. What a signature means for a + document or a message stays in each app. +- [`craft-penpot`](crates/penpot/README.md): clean-room reader and writer for the Penpot 2.x export + container (`.penpot`): the ZIP archive, its manifest, the per-file, per-page and per-shape JSON + members and media objects, as the JSON the file holds. Bounded on hostile input. Carved from + FigCraft. +- [`craft-plotter`](crates/plotter/README.md): cutter and pen-plotter command languages: HP-GL, + DMPL and GP-GL (plain and the Silhouette dialect, three generations, registration marks) from a + job in millimetres, plus the Silhouette device queries and reply parsers. No I/O. +- [`craft-pptx`](crates/pptx/README.md): PresentationML, the model of a `.pptx` presentation: + masters, layouts, slides, shapes, placeholders and their inheritance, transitions and animations, + comments, sections, media and kept parts. From DeckCraft. DrawingML comes from `craft-drawingml`. - [`craft-print`](crates/print/README.md): PDF submission to CUPS and Windows, printer discovery, duplex/copies/colour and driver preferences. PDF generation stays in each app. +- [`craft-proof`](crates/proof/README.md): proofing per language (spelling, suggestions, grammar, + hyphenation, a Hunspell reader, user words, Language packs). The word lists and patterns are + craft-i18n's `language-data/`, embedded at build time from `CRAFT_I18N_DIR`: English and the + carry-over locales by default, the set `CRAFT_PROOF_EMBED` names otherwise (`en` for English + only); other languages come from installed Language packs. + +- [`craft-pst`](crates/pst/README.md): Outlook personal folders files (`.pst`, [MS-PST]) read over any + `Read + Seek`, from MailCraft: the node database, property and table contexts, folders, messages, + recipients, attachments, attached messages and named properties, ANSI and Unicode, encoded or not, + with a password. Salvages damaged files. Bounded and panic-free on hostile input. What an item + means stays in each app. - [`craft-quit`](crates/quit/src/lib.rs): native macOS quit, egui close handling and an optional app menu. Unsaved-document decisions and save prompts stay in each app. +- [`craft-rtf`](crates/rtf/README.md): Rich Text Format: a tokenizer, text decoding in the file's code pages + (`\uN`, `\'hh`, single-byte and East Asian multibyte pages), escaping and dates, and a reader and writer + for a small neutral model (paragraphs, character formatting, outline levels, tables). - [`craft-settings`](crates/settings/README.md): `config.toml` settings storage for every app: platform folders, portable mode and overrides, atomic saves while running, backup, damaged-file recovery and schema migrations. Policy in [docs/settings-storage.md](docs/settings-storage.md). +- [`craft-sheet`](crates/sheet/README.md): spreadsheet vocabulary shared by `craft-xls`, `craft-xlsx` and `craft-ods`: cell positions and A1 text, cell values and error values, sheet-name quoting, Excel's built-in number formats and indexed colour palette. Not a file format; no dependencies. +- [`craft-sf2`](crates/sf2/README.md): SoundFont 2 (`.sf2`) reader: the sample pool and, per preset, the key and velocity regions a note sounds, with tuning, loop points, pan, level and volume envelope resolved from the preset and instrument zones. Bounded on hostile input. From LoopCraft. +- [`craft-silhouette-studio`](crates/silhouette-studio/src/lib.rs): bounded, fail-closed reader for Silhouette Studio `.studio` and `.studio3` documents: outlines, line colours and page size in millimetres. Protected designs are refused. +- [`craft-stencilcraft-format`](crates/stencilcraft-format/README.md): the versioned `.stencilcraft` JSON wire schema and envelope: required members, version 1 defaults, optional fields, and header checks. +- `craft-stencilcraft-document`: StencilCraft's typed document model, geometry helpers, validation, and native `.stencilcraft` codec. +- [`craft-sketch`](crates/sketch/README.md): clean-room reader for the `.sketch` container (Sketch 48 and newer): the ZIP archive, the version cut, `document.json`, the pages in document order and images, as the JSON the file holds. Bounded on hostile input. Carved from FigCraft. +- [`craft-svg`](crates/svg/src/lib.rs): SVG. `sniff` recognises SVG and SVGZ data; `svgz` unpacks + SVGZ within a byte limit, decodes UTF-8, UTF-16 and ISO 8859-1, and compresses; `guard` bounds + element nesting and entity expansion before any XML reader sees the text; `paths` reads geometry + (shapes, transforms, ``/`` instances and the CSS cascade as Bézier paths in + millimetres, each tagged by the caller); `color` parses CSS colours; `write` writes documents + (elements with escaping, path data, numbers). Feature `tree`: the full parser, usvg at the one + version every Craft app uses, re-exported behind an entry point that runs those checks first + (plus nesting with references followed, attribute counts, bounded embedded SVGs), and a budget + check for drawing a tree with resvg. Apps take usvg from here, not as their own dependency. +- [`craft-svg-paths`](crates/svg-paths/src/lib.rs): the former name of `craft-svg`'s `paths`, + `color` and `write`; re-exports them for existing users. +- [`craft-text`](crates/text/README.md): text processing below the composer. Its `hyphen` module finds + hyphenation points, with a bundled US English dictionary and patterns and optional hypher patterns + for 47 more languages (one cargo feature each). - [`craft-ui`](crates/ui/README.md): SVG catalogs, icon buttons, tabs, pane controls, transactional docking, print settings and theme helpers for egui frontends; assets, themes, localization and command handling stay in each app. Requires Rust 1.95 or later. +- [`craft-vcard`](crates/vcard/README.md): vCard contacts, from MailCraft: vCard 2.1, 3.0 and 4.0 read + (legacy character sets, quoted-printable, labels and Apple label groups, `KEY` certificates and + OpenPGP public keys), vCard 3.0 written. Bounded and panic-free on hostile input. The mapping onto + an app's contact type stays in each app. + +- [`craft-vectorcraft-format`](crates/vectorcraft-format/README.md): the envelope of VectorCraft's + `.vectorcraft` file: its members, versions, embedded images and the data after the JSON, with the + document as a type parameter. The document and its migrations stay in VectorCraft. +- [`craft-vectordrawable`](crates/vectordrawable/src/lib.rs): Android Vector Drawable writer: a model of + paths, groups, clip outlines, strokes and linear, radial and sweep gradients; the XML as Android + Studio formats it; and the mapping of a gradient on any affine frame to what a drawable can say. + What a drawable cannot draw is the caller's to bake in or leave out. No reader. +- [`craft-web-api`](crates/web-api/README.md): the page command API every Craft web build offers its page — one + `window.` shape (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, `screenshot`) over the app's + JSON control channel. +- [`craft-web-host`](crates/web-host/README.md): the host messages between a Craft app's web build and the page that hosts it (`?host=parent` or `data-host="page"`): one contract and one bridge for every app. +- [`craft-webp`](crates/webp/src/lib.rs): the WebP container, from EffectCraft (reader) and + PhotoCraft (writer): canvas size, alpha and every animation frame's timing from the chunk + headers without decoding, and the RIFF container around an encoded VP8 frame with alpha, ICC, + EXIF and XMP chunks. The VP8 and VP8L codecs are not part of it. +- [`craft-xls`](crates/xls/README.md): clean-room Excel 97-2003 workbook (`.xls`, BIFF8) reader: worksheets, cell values, formulas as A1 text (shared and array formulas included), cell formats with resolved palette colours, merged cells, column and row records, defined names, print areas and calculation settings, with limits on records, cells and text. +- [`craft-wordmodel`](crates/wordmodel/README.md): a word-processing document model (stories, paragraphs, + runs, tables, sections, styles, numbering, fields, notes, comments, revisions, drawings, math): what the + text-format crates read into and write from. WordCraft's model without its editing code. +- [`craft-xlsx`](crates/xlsx/README.md): Excel workbooks (`.xlsx`, `.xlsm`, SpreadsheetML, transitional and strict) and binary workbooks (`.xlsb`) on `craft-opc`: the workbook part, shared strings, the styles tables and worksheet rows streamed one at a time, with limits on the package, the XML and the binary records. +- [`craft-ora`](crates/ora/README.md): OpenRaster documents (`.ora`): the ZIP archive and the `stack.xml` layer tree as written, read with bounded sizes and written. From PhotoCraft. +- [`craft-y4m`](crates/y4m/src/lib.rs): YUV4MPEG2 (`.y4m`) reader, from FilmCraft: the stream + header, colour space layouts at 8 to 16 bits, a frame index built from the `FRAME` lines without + loading the file, and plane access. Bounded on hostile input. +- [`craft-kra`](crates/kra/README.md): Krita documents (`.kra`, `.krz`), read: the ZIP archive, the `maindoc.xml` node tree as written, and the tiled pixel files with their LZF compression, a tile at a time. From PhotoCraft. +- [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. +- [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. +- [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities and compatibility boundaries across the Craft suite. - [Theme infrastructure](docs/theme-infrastructure.md): eight-app comparison and the small shared diff --git a/clippy.toml b/clippy.toml index 4b6c79f55..8f54197d8 100644 --- a/clippy.toml +++ b/clippy.toml @@ -2,3 +2,7 @@ allow-unwrap-in-tests = true allow-expect-in-tests = true allow-panic-in-tests = true allow-indexing-slicing-in-tests = true +cognitive-complexity-threshold = 30 +disallowed-types = [ + { path = "egui::ComboBox", reason = "use craft_ui::ComboBox: egui's combo box menu can't grow back while open (egui#5225)" }, +] diff --git a/crates/aaf/Cargo.toml b/crates/aaf/Cargo.toml new file mode 100644 index 000000000..397032961 --- /dev/null +++ b/crates/aaf/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "craft-aaf" +description = "Clean-room AAF (Edit Protocol, structured storage) and OMF Interchange 2.0 (Bento) readers and writers over one mob-style composition model, with embedded PCM essence." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-cfb = { workspace = true } +craft-timeline = { workspace = true } +# WAV and AIFF around embedded PCM essence: the one WAV/AIFF implementation, not a second one here. +craft-audio = { workspace = true, features = ["decode-native", "pcm-encode"] } +thiserror = { workspace = true } diff --git a/crates/aaf/README.md b/crates/aaf/README.md new file mode 100644 index 000000000..7d3d44767 --- /dev/null +++ b/crates/aaf/README.md @@ -0,0 +1,67 @@ +# craft-aaf + +AAF and OMF, the two mob-based editorial interchange formats, read and written through one +composition model. + +| Module | What | +|---|---| +| `model` | `Document`: compositions (tracks of fillers, source clips and overlapping transitions; clip gain, constant or keyframed; markers; start timecode), nested compositions, and the media `Source`s they reference (file path or embedded PCM, picture size and rate, sound channels / rate / bits, start timecode, markers). Times are exact `craft_timeline::time::Tick`s. | +| `aaf` | `read`, `write`, `sniff`: AAF Edit Protocol in structured storage (`craft-cfb`). | +| `omf` | `read`, `write`, `sniff`: OMF Interchange 2.0 in a Bento container. Sound only. | + +No file I/O: documents are bytes and media references are strings. An application maps its own +timeline onto `model` once and gets both formats. Embedded audio is PCM in the model; the WAVE and +AIFF wrappers the formats use come from `craft-audio`. + +## Specifications + +Clean-room: written from public specifications only. The AAF SDK, pyaaf2, the OMF Toolkit and +other implementations were not consulted. Where a specification leaves a layout open (how a +nested sequence is written, how several video tracks share a slot), what Premiere Pro does was +read off files it exports. + +| Format | Document | Edition | Used for | +|---|---|---|---| +| AAF | AMWA *AAF Object Specification* | v1.1 (2005) | classes, properties, mob / slot / segment model, sequences and transitions, operation groups, parameters, definitions, descriptors, locators, essence data | +| AAF | AMWA *AAF Low-Level Container Specification* | v1.0.1 | objects as structured storages, `properties` streams, stored forms, strong / weak reference collections and their index streams, `referenced properties` | +| AAF | AMWA *AAF Edit Protocol* | v1.0 (AMWA AS-01) | top-level composition, mob chains (composition → master → file → physical source), operation definitions, usage codes | +| AAF | SMPTE RP 224 / RP 210 registers | — | data definition, class and type labels | +| AAF | Microsoft [MS-CFB] | rev. 10.0 | the container, see `craft-cfb` | +| OMF | Avid *OMF Interchange Specification* | Version 2.0 (1997) | classes (`HEAD`, `CMOB`, `MMOB`, `SMOB`, `MSLT`, `TRKD`, `SEQU`, `SCLP`, `FILL`, `TRAN`, `EFFE`, `ESLT`, `CVAL`, `VVAL`, `TCCP`, `WAVD`, `AIFD`, `WAVE`, `AIFC`, locators), property and type names | +| OMF | Apple *Bento Specification* | revision 1.0d5 | container label, TOC encoding, objects / properties / types, references | + +## What is read and written + +AAF: a top-level composition mob with a timecode slot, a timeline slot per track (sequences of +fillers, source clips and transitions; `VideoDissolve`, `VideoFadeToBlack`, `SMPTEVideoWipe`, +`MonoAudioDissolve`), clip gain as a `MonoAudioGain` operation group with a constant or varying +amplitude, markers as comment markers in an event slot, a composition mob for every nested +composition, and per media item a master mob, file source mobs (CDCI or PCM descriptor with a +network locator, or `EssenceData` with embedded PCM) and a tape source mob with the start +timecode. The reader also takes sequences inside sequences, selectors, essence groups, legacy data +definitions, and Premiere Pro's layout (all video tracks as layers of one slot, nested compositions +tagged top-level). 512-byte or 4096-byte sectors. + +OMF: a composition mob with a timecode slot and a sound slot per track, in samples; transitions as +`SimpleMonoAudioDissolve`; gain as `MonoAudioGain`; master, file (`WAVD` / `AIFD`, embedded `WAVE` +data or a locator) and tape mobs. + +Not represented: speed changes, video effects other than transitions, OMF picture and markers. + +Private tags, first written by FilmCraft and kept so existing files read back: AAF tagged values +`FilmCraft Effect`, `FilmCraft Frame Size`, `FilmCraft Audio Sample Rate`; OMF properties +`FilmCraft:SCLP:ClipName`, `FilmCraft:EFFE:EffectID`. + +## Hostile input + +A damaged or crafted file gives an error or a report entry, never a panic, a hang or an allocation +sized by a count in the file: weak-collection counts are checked against the index stream, zero-size +keys are refused, a storage owned twice is an error, nesting of segments and of compositions is +bounded, times are clamped to about three days. `src/aaf/hostile_tests.rs` and +`src/roundtrip_tests.rs` (truncation at every 97th byte, 400 random 8-byte corruptions per format) +cover them. + +## Origin + +Moved from FilmCraft's `filmcraft-interchange` (`aaf`, `omf`, the model types of `comp`). The +mapping between the model and an application's project stays in the application. diff --git a/crates/aaf/src/aaf.rs b/crates/aaf/src/aaf.rs new file mode 100644 index 000000000..05894f94e --- /dev/null +++ b/crates/aaf/src/aaf.rs @@ -0,0 +1,84 @@ +//! AAF (Advanced Authoring Format) import and export, following the AAF Edit Protocol. +//! +//! Written from the public AMWA specifications: *AAF Object Specification v1.1*, *AAF Low-Level +//! Container Specification v1.0.1* (structured storage mapping, see [`store`]) and *AAF Edit +//! Protocol v1.0*; the container is Microsoft's Compound File Binary format ([`craft_cfb`]). +//! +//! A written file holds one top-level **CompositionMob** (Edit Protocol usage code +//! "top level") with +//! +//! - a timecode slot (sequence start timecode, rounded rate, drop frame), +//! - one timeline slot per video track (picture) and per audio track (sound; one per channel when +//! broken out to mono), each a **Sequence** of **Filler**s, **SourceClip**s and **Transition**s +//! (Cross Dissolve → `VideoDissolve_2`, Dip to Black → `VideoFadeToBlack`, audio crossfades → +//! `MonoAudioDissolve`; the model's transition name is kept in a tagged value), +//! - clip volume as a `MonoAudioGain` **OperationGroup** around the source clip with an +//! `Amplitude` **ConstantValue** or **VaryingValue** (keyframes as control points), +//! - sequence markers as **CommentMarker**s in an event slot; +//! +//! a **CompositionMob** without a usage code for every nested sequence, built the same way (a +//! clip of the nested sequence is a **SourceClip** that points at that mob, at its first slot of +//! the clip's kind, with its start in that slot's edit units; Premiere Pro writes nested sequences +//! this way); +//! +//! and for every media item a **MasterMob**, a file **SourceMob** per essence (a CDCI or PCM +//! descriptor with a network locator to the file, or `EssenceData` holding embedded PCM) and a +//! tape **SourceMob** carrying the media's start timecode. Clip and media markers become comment +//! markers on the master mob. +//! +//! Import reads the same structures (and the common variations: sequences inside sequences, +//! selectors, operation groups around clips, legacy data definitions, essence groups, and Premiere +//! Pro's one video slot holding a nested scope with a segment per video track) into the model: +//! compositions (one that others use is a nested composition), sources (with their path, or their +//! embedded audio as PCM) and markers. +//! +//! Private tags. Three tagged values are not in any register; they were first written by FilmCraft +//! and keep their names so that existing files read back: `FilmCraft Effect` (a transition's +//! name), `FilmCraft Frame Size` and `FilmCraft Audio Sample Rate` (on the composition). + +mod ids; +mod read; +pub(crate) mod store; +mod write; + +use crate::model::Document; +use crate::{Identity, Report, Result}; + +/// How to write an AAF file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WriteOptions<'a> { + pub identity: Identity<'a>, + /// Write 512-byte sectors (compound file version 3) instead of 4096-byte sectors. + pub small_sectors: bool, +} + +/// Write `doc` as an AAF file: its first composition is the top-level one, with every nested +/// composition and every source it uses. Times that are not on whole edit units are rounded and +/// reported. +pub fn write(doc: &Document, opts: &WriteOptions<'_>, report: &mut Report) -> Result> { + let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 }; + write::write(doc, version, &opts.identity, report) +} + +/// Read an AAF file: its top-level compositions, the compositions nested in them and their +/// sources. What cannot be represented is skipped and named in `report`. +pub fn read(bytes: &[u8], report: &mut Report) -> Result { + read::read(bytes, report) +} + +/// Whether `bytes` look like an AAF file (a compound file whose root has the AAF root class or a +/// `Header-2` storage). +pub fn sniff(bytes: &[u8]) -> bool { + if !craft_cfb::sniff(bytes) { + return false; + } + match craft_cfb::CompoundFile::open(bytes) { + Ok(cf) => cf.root().clsid == ids::ROOT || cf.find("Header-2").is_ok(), + Err(_) => false, + } +} + +#[cfg(test)] +mod hostile_tests; +#[cfg(test)] +mod tests; diff --git a/crates/aaf/src/aaf/hostile_tests.rs b/crates/aaf/src/aaf/hostile_tests.rs new file mode 100644 index 000000000..853b0000c --- /dev/null +++ b/crates/aaf/src/aaf/hostile_tests.rs @@ -0,0 +1,110 @@ +//! Hostile object trees: a crafted file must give an error, never a crash, a hang or an +//! allocation sized by a count in the file (AGENTS.md §0). + +use craft_cfb::{Version, Writer}; + +use super::store::{self, SF_STRONG_VECTOR, SF_WEAK_SET, SF_WEAK_VECTOR, Value, utf16z}; + +const PID: u16 = 0x0007; + +/// A root object whose one property `PID` is a weak reference vector or set (stored form `sf`) +/// named `Keys`, with `index` as its `Keys index` stream. +fn weak_collection_file(sf: u16, index: Vec) -> Vec { + let name = utf16z("Keys"); + // byte order 'L', format version, 1 entry { pid, stored form, value length }, the value + let mut props = vec![0x4C, 0x20]; + for x in [1, PID, sf, name.len() as u16] { + props.extend_from_slice(&x.to_le_bytes()); + } + props.extend_from_slice(&name); + let mut w = Writer::new(Version::V3); + w.stream(Writer::ROOT, "properties", props).unwrap(); + w.stream(Writer::ROOT, "Keys index", index).unwrap(); + w.finish() +} + +/// A weak collection index: u32 count, u16 tag, u16 key pid, u8 key size, then the keys. +fn index(count: u32, size: u8, keys: &[u8]) -> Vec { + let mut v = count.to_le_bytes().to_vec(); + v.extend_from_slice(&0u16.to_le_bytes()); + v.extend_from_slice(&0x1B01u16.to_le_bytes()); + v.push(size); + v.extend_from_slice(keys); + v +} + +fn read_keys(sf: u16, index: Vec) -> Result>, String> { + match store::read(&weak_collection_file(sf, index))?.get(PID) { + Some(Value::WeakVec(_, _, keys)) => Ok(keys.clone()), + other => panic!("not a weak collection: {other:?}"), + } +} + +/// With a key size of 0 every key fits, so the count alone drove the loop: up to 4.29 billion +/// empty keys (about 100 GB) from a 9-byte index. A million is enough to show it, cheaply. +#[test] +fn zero_size_weak_collection_keys_are_an_error() { + for sf in [SF_WEAK_VECTOR, SF_WEAK_SET] { + match read_keys(sf, index(1_000_000, 0, &[])) { + Err(e) => assert!(e.contains("zero-size"), "{e}"), + Ok(keys) => panic!("read {} empty keys instead of an error", keys.len()), + } + } +} + +#[test] +fn weak_collection_count_beyond_the_index_is_an_error() { + for sf in [SF_WEAK_VECTOR, SF_WEAK_SET] { + let e = read_keys(sf, index(u32::MAX, 4, &[1, 2, 3, 4, 5, 6, 7, 8])).unwrap_err(); + assert!(e.contains("short weak collection index"), "{e}"); + } +} + +#[test] +fn weak_collections_within_the_index_are_read() { + for sf in [SF_WEAK_VECTOR, SF_WEAK_SET] { + assert_eq!(read_keys(sf, index(0, 0, &[])).unwrap(), Vec::>::new()); + assert_eq!(read_keys(sf, index(2, 4, &[1, 2, 3, 4, 5, 6, 7, 8])).unwrap(), [vec![1, 2, 3, 4], vec![5, 6, 7, 8]]); + } +} + +/// A strong vector whose entries all name the same child storage would rebuild that subtree once +/// per entry; a storage has one owner, so the repeat is an error. +#[test] +fn a_storage_owned_twice_is_an_error() { + let name = utf16z("Items"); + let mut props = vec![0x4C, 0x20]; + for x in [1, PID, SF_STRONG_VECTOR, name.len() as u16] { + props.extend_from_slice(&x.to_le_bytes()); + } + props.extend_from_slice(&name); + // count, next free key, last key, then the entry keys + let mut idx = 3u32.to_le_bytes().to_vec(); + idx.extend_from_slice(&[0; 8]); + for _ in 0..3 { + idx.extend_from_slice(&1u32.to_le_bytes()); + } + let mut w = Writer::new(Version::V3); + w.stream(Writer::ROOT, "properties", props).unwrap(); + w.stream(Writer::ROOT, "Items index", idx).unwrap(); + let child = w.storage(Writer::ROOT, "Items{1}", [0; 16]).unwrap(); + w.stream(child, "properties", vec![0x4C, 0x20, 0, 0]).unwrap(); + let e = store::read(&w.finish()).unwrap_err(); + assert!(e.contains("more than one reference"), "{e}"); +} + +/// Objects nest through strong references; a chain deeper than the reader follows is an error, +/// and one within it reads. +#[test] +fn objects_nested_too_deeply_are_an_error() { + let chain = |depth: usize| { + let mut o = store::Obj::new([1; 16]); + for _ in 0..depth { + o = store::Obj::new([1; 16]).with(PID, Value::Strong(Box::new(o))); + } + store::write(&o, Version::V4).unwrap() + }; + assert!(store::read(&chain(150)).is_ok()); + let e = store::read(&chain(250)).unwrap_err(); + assert!(e.contains("nested too deeply"), "{e}"); +} diff --git a/crates/aaf/src/aaf/ids.rs b/crates/aaf/src/aaf/ids.rs new file mode 100644 index 000000000..70a184060 --- /dev/null +++ b/crates/aaf/src/aaf/ids.rs @@ -0,0 +1,262 @@ +//! Class ids, property ids and definition ids of the AAF object model subset read and +//! writes (AAF Object Specification v1.1, AAF Edit Protocol). Class and data definition AUIDs are +//! SMPTE labels (RP 224 / RP 210 registers) in AUID byte order; property ids are the AAF / ST 377-1 +//! local tags. + +use super::store::Auid; + +/// AUID of a SMPTE universal label: the label's last 8 bytes become Data1–Data3 (stored +/// little-endian), its first 8 bytes Data4. +pub(crate) const fn ul(b: [u8; 16]) -> Auid { + [b[11], b[10], b[9], b[8], b[13], b[12], b[15], b[14], b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]] +} + +/// A GUID `{d1-d2-d3-d4}` in AUID byte order. +pub(crate) const fn guid(d1: u32, d2: u16, d3: u16, d4: [u8; 8]) -> Auid { + let a = d1.to_le_bytes(); + let b = d2.to_le_bytes(); + let c = d3.to_le_bytes(); + [a[0], a[1], a[2], a[3], b[0], b[1], c[0], c[1], d4[0], d4[1], d4[2], d4[3], d4[4], d4[5], d4[6], d4[7]] +} + +/// Class AUID: SMPTE label 06.0E.2B.34.02.06.01.01.0D.01.01.01.01.01.xx.00. +pub(crate) const fn class(xx: u8) -> Auid { + ul([0x06, 0x0E, 0x2B, 0x34, 0x02, 0x06, 0x01, 0x01, 0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, xx, 0x00]) +} + +/// The root object's class (the storage class of the file's root). +pub(crate) const ROOT: Auid = guid(0xB3B398A5, 0x1C90, 0x11D4, [0x80, 0x53, 0x08, 0x00, 0x36, 0x21, 0x08, 0x04]); +/// The meta-dictionary (class 0x25 of the meta-model, 06.0E.2B.34.02.06.01.01.0D.01.01.01.02.25.00.00). +pub(crate) const META_DICTIONARY: Auid = ul([0x06, 0x0E, 0x2B, 0x34, 0x02, 0x06, 0x01, 0x01, 0x0D, 0x01, 0x01, 0x01, 0x02, 0x25, 0x00, 0x00]); + +#[allow(dead_code)] +pub(crate) mod cls { + use super::class; + use crate::aaf::store::Auid; + pub const COMMENT_MARKER: Auid = class(0x08); + pub const FILLER: Auid = class(0x09); + pub const OPERATION_GROUP: Auid = class(0x0A); + pub const NESTED_SCOPE: Auid = class(0x0B); + pub const SCOPE_REFERENCE: Auid = class(0x0D); + pub const SELECTOR: Auid = class(0x0E); + pub const SEQUENCE: Auid = class(0x0F); + pub const SOURCE_CLIP: Auid = class(0x11); + pub const TIMECODE: Auid = class(0x14); + pub const TRANSITION: Auid = class(0x17); + pub const CONTENT_STORAGE: Auid = class(0x18); + pub const CONTROL_POINT: Auid = class(0x19); + pub const DATA_DEFINITION: Auid = class(0x1B); + pub const OPERATION_DEFINITION: Auid = class(0x1C); + pub const PARAMETER_DEFINITION: Auid = class(0x1D); + pub const CONTAINER_DEFINITION: Auid = class(0x20); + pub const INTERPOLATION_DEFINITION: Auid = class(0x21); + pub const DICTIONARY: Auid = class(0x22); + pub const ESSENCE_DATA: Auid = class(0x23); + pub const CDCI_DESCRIPTOR: Auid = class(0x28); + pub const TAPE_DESCRIPTOR: Auid = class(0x2E); + pub const HEADER: Auid = class(0x2F); + pub const IDENTIFICATION: Auid = class(0x30); + pub const NETWORK_LOCATOR: Auid = class(0x32); + pub const COMPOSITION_MOB: Auid = class(0x35); + pub const MASTER_MOB: Auid = class(0x36); + pub const SOURCE_MOB: Auid = class(0x37); + pub const EVENT_MOB_SLOT: Auid = class(0x39); + pub const TIMELINE_MOB_SLOT: Auid = class(0x3B); + pub const CONSTANT_VALUE: Auid = class(0x3D); + pub const VARYING_VALUE: Auid = class(0x3E); + pub const TAGGED_VALUE: Auid = class(0x3F); + pub const DESCRIPTIVE_MARKER: Auid = class(0x41); + pub const PCM_DESCRIPTOR: Auid = class(0x48); + pub const IMPORT_DESCRIPTOR: Auid = class(0x4A); + /// Classes with a `Locator` and media properties read as file descriptors. + pub const WAVE_DESCRIPTOR: Auid = class(0x2C); + pub const AIFC_DESCRIPTOR: Auid = class(0x26); + pub const SOUND_DESCRIPTOR: Auid = class(0x42); + pub const RGBA_DESCRIPTOR: Auid = class(0x29); + pub const MULTIPLE_DESCRIPTOR: Auid = class(0x44); +} + +/// Property ids. +#[allow(dead_code)] +pub(crate) mod pid { + // Root + pub const ROOT_META_DICTIONARY: u16 = 0x0001; + pub const ROOT_HEADER: u16 = 0x0002; + // MetaDictionary and MetaDefinition + pub const META_CLASS_DEFINITIONS: u16 = 0x0003; + pub const META_TYPE_DEFINITIONS: u16 = 0x0004; + pub const META_IDENTIFICATION: u16 = 0x0005; + // Header + pub const BYTE_ORDER: u16 = 0x3B01; + pub const LAST_MODIFIED: u16 = 0x3B02; + pub const CONTENT: u16 = 0x3B03; + pub const DICTIONARY: u16 = 0x3B04; + pub const VERSION: u16 = 0x3B05; + pub const IDENTIFICATION_LIST: u16 = 0x3B06; + pub const OBJECT_MODEL_VERSION: u16 = 0x3B07; + pub const OPERATIONAL_PATTERN: u16 = 0x3B09; + // Identification + pub const COMPANY_NAME: u16 = 0x3C01; + pub const PRODUCT_NAME: u16 = 0x3C02; + pub const PRODUCT_VERSION_STRING: u16 = 0x3C04; + pub const PRODUCT_ID: u16 = 0x3C05; + pub const DATE: u16 = 0x3C06; + pub const PLATFORM: u16 = 0x3C08; + pub const GENERATION_AUID: u16 = 0x3C09; + // ContentStorage + pub const MOBS: u16 = 0x1901; + pub const ESSENCE_DATA: u16 = 0x1902; + // Dictionary + pub const OPERATION_DEFINITIONS: u16 = 0x2603; + pub const PARAMETER_DEFINITIONS: u16 = 0x2604; + pub const DATA_DEFINITIONS: u16 = 0x2605; + pub const CONTAINER_DEFINITIONS: u16 = 0x2608; + pub const INTERPOLATION_DEFINITIONS: u16 = 0x2609; + // DefinitionObject + pub const IDENTIFICATION: u16 = 0x1B01; + pub const NAME: u16 = 0x1B02; + pub const DESCRIPTION: u16 = 0x1B03; + // OperationDefinition + pub const OPDEF_DATA_DEFINITION: u16 = 0x1E01; + pub const OPDEF_IS_TIME_WARP: u16 = 0x1E02; + pub const OPDEF_NUMBER_INPUTS: u16 = 0x1E07; + pub const OPDEF_PARAMETERS_DEFINED: u16 = 0x1E09; + // ParameterDefinition + pub const PARAMDEF_DISPLAY_UNITS: u16 = 0x1F03; + // Mob + pub const MOB_ID: u16 = 0x4401; + pub const MOB_NAME: u16 = 0x4402; + pub const SLOTS: u16 = 0x4403; + pub const MOB_LAST_MODIFIED: u16 = 0x4404; + pub const MOB_CREATION_TIME: u16 = 0x4405; + pub const MOB_USER_COMMENTS: u16 = 0x4406; + pub const USAGE_CODE: u16 = 0x4408; + // SourceMob + pub const ESSENCE_DESCRIPTION: u16 = 0x4701; + // MobSlot + pub const SLOT_ID: u16 = 0x4801; + pub const SLOT_NAME: u16 = 0x4802; + pub const SEGMENT: u16 = 0x4803; + pub const PHYSICAL_TRACK_NUMBER: u16 = 0x4804; + // TimelineMobSlot / EventMobSlot + pub const EDIT_RATE: u16 = 0x4B01; + pub const ORIGIN: u16 = 0x4B02; + pub const EVENT_EDIT_RATE: u16 = 0x4901; + // Component + pub const DATA_DEFINITION: u16 = 0x0201; + pub const LENGTH: u16 = 0x0202; + pub const COMPONENT_USER_COMMENTS: u16 = 0x0204; + // Sequence + pub const COMPONENTS: u16 = 0x1001; + // SourceReference / SourceClip + pub const SOURCE_ID: u16 = 0x1101; + pub const SOURCE_MOB_SLOT_ID: u16 = 0x1102; + pub const START_TIME: u16 = 0x1201; + // Event + pub const POSITION: u16 = 0x0601; + pub const COMMENT: u16 = 0x0602; + // Timecode + pub const TC_START: u16 = 0x1501; + pub const TC_FPS: u16 = 0x1502; + pub const TC_DROP: u16 = 0x1503; + // Transition + pub const OPERATION_GROUP: u16 = 0x1801; + pub const CUT_POINT: u16 = 0x1802; + // OperationGroup + pub const OPERATION: u16 = 0x0B01; + pub const INPUT_SEGMENTS: u16 = 0x0B02; + pub const PARAMETERS: u16 = 0x0B03; + // Selector / NestedScope + pub const SELECTED: u16 = 0x0F01; + pub const NESTED_SLOTS: u16 = 0x0C01; + // Parameter / ConstantValue / VaryingValue / ControlPoint + pub const PARAMETER_DEFINITION: u16 = 0x4C01; + pub const CONSTANT_VALUE: u16 = 0x4D01; + pub const INTERPOLATION: u16 = 0x4E01; + pub const POINT_LIST: u16 = 0x4E02; + pub const CP_VALUE: u16 = 0x1A02; + pub const CP_TIME: u16 = 0x1A03; + pub const CP_EDIT_HINT: u16 = 0x1A04; + // TaggedValue + pub const TAG_NAME: u16 = 0x5001; + pub const TAG_VALUE: u16 = 0x5003; + // EssenceDescriptor / FileDescriptor + pub const LOCATOR: u16 = 0x2F01; + pub const SAMPLE_RATE: u16 = 0x3001; + pub const FILE_LENGTH: u16 = 0x3002; + pub const CONTAINER_FORMAT: u16 = 0x3004; + pub const STORED_HEIGHT: u16 = 0x3202; + pub const STORED_WIDTH: u16 = 0x3203; + pub const FRAME_LAYOUT: u16 = 0x320C; + pub const VIDEO_LINE_MAP: u16 = 0x320D; + pub const IMAGE_ASPECT_RATIO: u16 = 0x320E; + pub const COMPONENT_WIDTH: u16 = 0x3301; + pub const HORIZONTAL_SUBSAMPLING: u16 = 0x3302; + pub const QUANTIZATION_BITS: u16 = 0x3D01; + pub const LOCKED: u16 = 0x3D02; + pub const AUDIO_SAMPLING_RATE: u16 = 0x3D03; + pub const CHANNELS: u16 = 0x3D07; + pub const AVERAGE_BPS: u16 = 0x3D09; + pub const BLOCK_ALIGN: u16 = 0x3D0A; + // NetworkLocator + pub const URL_STRING: u16 = 0x4001; + // EssenceData + pub const ESSENCE_MOB_ID: u16 = 0x2701; + pub const ESSENCE_STREAM: u16 = 0x2702; +} + +/// Data definitions (SMPTE RP 224 labels 06.0E.2B.34.04.01.01.01.01.03.02.xx.yy). +pub(crate) mod ddef { + use super::{guid, ul}; + use crate::aaf::store::Auid; + pub const PICTURE: Auid = ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x01, 0x01, 0x03, 0x02, 0x02, 0x01, 0x00, 0x00, 0x00]); + pub const SOUND: Auid = ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x01, 0x01, 0x03, 0x02, 0x02, 0x02, 0x00, 0x00, 0x00]); + pub const TIMECODE: Auid = ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x01, 0x01, 0x03, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00]); + pub const DESCRIPTIVE_METADATA: Auid = ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x01, 0x01, 0x03, 0x02, 0x01, 0x10, 0x00, 0x00, 0x00]); + /// Pre-SMPTE ("legacy") data definitions written by older applications. + pub const LEGACY_PICTURE: Auid = guid(0x6F3C8CE1, 0x6CEF, 0x11D2, [0x80, 0x7D, 0x00, 0x60, 0x08, 0x14, 0x3E, 0x6F]); + pub const LEGACY_SOUND: Auid = guid(0x78E1EBE1, 0x6CEF, 0x11D2, [0x80, 0x7D, 0x00, 0x60, 0x08, 0x14, 0x3E, 0x6F]); + pub const LEGACY_TIMECODE: Auid = guid(0x7F275E81, 0x77E5, 0x11D2, [0x80, 0x7F, 0x00, 0x60, 0x08, 0x14, 0x3E, 0x6F]); +} + +/// Operation, parameter, interpolation and container definitions. +pub(crate) mod def { + use super::guid; + use crate::aaf::store::Auid; + const AV: [u8; 8] = [0x8A, 0x09, 0x00, 0x60, 0x08, 0x14, 0x3E, 0x6F]; + pub const VIDEO_DISSOLVE: Auid = guid(0x0C3BEA41, 0xFC05, 0x11D2, AV); + pub const VIDEO_FADE_TO_BLACK: Auid = guid(0x0C3BEA40, 0xFC05, 0x11D2, AV); + pub const SMPTE_VIDEO_WIPE: Auid = guid(0x0C3BEA43, 0xFC05, 0x11D2, AV); + pub const MONO_AUDIO_DISSOLVE: Auid = guid(0x0C3BEA44, 0xFC05, 0x11D2, AV); + const AG: [u8; 8] = [0x8A, 0x38, 0x00, 0x50, 0x04, 0x0E, 0xF7, 0xD2]; + pub const MONO_AUDIO_GAIN: Auid = guid(0x9D2EA893, 0x0968, 0x11D3, AG); + const PD: [u8; 8] = [0x8A, 0x4C, 0x00, 0x50, 0x04, 0x0E, 0xF7, 0xD2]; + pub const PARAM_LEVEL: Auid = guid(0xE4962320, 0x2267, 0x11D3, PD); + pub const PARAM_AMPLITUDE: Auid = guid(0xE4962321, 0x2267, 0x11D3, PD); + const IP: [u8; 8] = [0x80, 0xA9, 0x00, 0x60, 0x08, 0x14, 0x3E, 0x6F]; + pub const INTERP_LINEAR: Auid = guid(0x5B6C85A4, 0x0EDE, 0x11D3, IP); + pub const INTERP_CONSTANT: Auid = guid(0x5B6C85A5, 0x0EDE, 0x11D3, IP); + const CD: [u8; 8] = [0x80, 0x9B, 0x00, 0x60, 0x08, 0x14, 0x3E, 0x6F]; + pub const CONTAINER_AAF: Auid = guid(0x4313B571, 0xD8BA, 0x11D2, CD); + pub const CONTAINER_EXTERNAL: Auid = guid(0x4313B572, 0xD8BA, 0x11D2, CD); + /// Type ids of indirect values (SMPTE RP 210 type labels). + pub const TYPE_RATIONAL: Auid = super::ul([0x06, 0x0E, 0x2B, 0x34, 0x01, 0x04, 0x01, 0x01, 0x03, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00]); + pub const TYPE_STRING: Auid = super::ul([0x06, 0x0E, 0x2B, 0x34, 0x01, 0x04, 0x01, 0x01, 0x01, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00]); + /// Edit Protocol operational pattern (`AAFEditProtocol`) and mob usage codes (`Usage_TopLevel`), + /// as in the SMPTE Labels register. + pub const OP_EDIT_PROTOCOL: Auid = super::ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x05, 0x0D, 0x01, 0x12, 0x01, 0x01, 0x00, 0x00, 0x00]); + pub const USAGE_TOP_LEVEL: Auid = super::ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x01, 0x0D, 0x01, 0x01, 0x02, 0x01, 0x01, 0x07, 0x00]); + /// The label FilmCraft 0.16 and earlier wrote as the top-level usage code (in no register). + /// Import reads it as top-level too. + pub const USAGE_TOP_LEVEL_OLD: Auid = super::ul([0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x05, 0x0D, 0x01, 0x12, 0x01, 0x02, 0x03, 0x00, 0x00]); +} + +/// Weak reference target paths (pids from the root object). +pub(crate) mod path { + use super::pid; + pub const DATA_DEFS: [u16; 3] = [pid::ROOT_HEADER, pid::DICTIONARY, pid::DATA_DEFINITIONS]; + pub const OPERATION_DEFS: [u16; 3] = [pid::ROOT_HEADER, pid::DICTIONARY, pid::OPERATION_DEFINITIONS]; + pub const PARAMETER_DEFS: [u16; 3] = [pid::ROOT_HEADER, pid::DICTIONARY, pid::PARAMETER_DEFINITIONS]; + pub const INTERPOLATION_DEFS: [u16; 3] = [pid::ROOT_HEADER, pid::DICTIONARY, pid::INTERPOLATION_DEFINITIONS]; + pub const CONTAINER_DEFS: [u16; 3] = [pid::ROOT_HEADER, pid::DICTIONARY, pid::CONTAINER_DEFINITIONS]; +} diff --git a/crates/aaf/src/aaf/read.rs b/crates/aaf/src/aaf/read.rs new file mode 100644 index 000000000..31c841c5d --- /dev/null +++ b/crates/aaf/src/aaf/read.rs @@ -0,0 +1,759 @@ +//! AAF objects → composition model. + +use std::collections::{HashMap, HashSet}; + +use craft_timeline::time::{FrameRate, Tick}; + +use super::ids::{cls, ddef, def, pid}; +use super::store::{self, Auid, Obj, Value, rational_of, utf16_of}; +use crate::model::{CClip, CItem, CKind, CMarker, CTrack, CTransition, Composition, Document, Gain, Source, units_to_ticks}; +use crate::{Error, Report, Result}; + +type MobId = [u8; 32]; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum DataKind { + Picture, + Sound, + Timecode, + Other, +} + +fn data_kind(o: &Obj) -> DataKind { + match o.weak_key(pid::DATA_DEFINITION).and_then(|k| ::try_from(k).ok()) { + Some(k) if k == ddef::PICTURE || k == ddef::LEGACY_PICTURE => DataKind::Picture, + Some(k) if k == ddef::SOUND || k == ddef::LEGACY_SOUND => DataKind::Sound, + Some(k) if k == ddef::TIMECODE || k == ddef::LEGACY_TIMECODE => DataKind::Timecode, + Some(_) => DataKind::Other, + None => DataKind::Other, + } +} + +fn len_of(o: &Obj) -> i64 { + o.i64(pid::LENGTH).unwrap_or(0).max(0) +} + +/// The value of an indirect property: (type id, bytes). +fn indirect(d: &[u8]) -> Option<(Auid, &[u8])> { + if d.len() >= 17 && (d[0] == 0x4C || d[0] == 0x42) { + return Some((d[1..17].try_into().ok()?, &d[17..])); + } + if d.len() >= 16 { + return Some((d[..16].try_into().ok()?, &d[16..])); + } + None +} + +fn tagged_values(o: &Obj, p: u16) -> Vec<(String, String)> { + o.objs(p) + .iter() + .filter_map(|t| { + let name = t.string(pid::TAG_NAME)?; + let (ty, v) = indirect(t.data(pid::TAG_VALUE)?)?; + let value = if ty == def::TYPE_STRING || v.len() % 2 == 0 { utf16_of(v) } else { String::new() }; + Some((name, value)) + }) + .collect() +} + +fn indirect_f64(d: &[u8]) -> Option { + let (_, v) = indirect(d)?; + match v.len() { + 8 => rational_of(v).filter(|r| r.1 != 0).map(|(n, d)| n as f64 / d as f64), + 4 => Some(i32::from_le_bytes(v.try_into().ok()?) as f64), + _ => None, + } +} + +struct R<'a> { + report: &'a mut Report, + mobs: HashMap, + essence: HashMap, + doc: Document, + /// (file mob id, slot id, master key) → source index + sources: HashMap<(MobId, u32, String), usize>, + /// Composition mobs read as nested compositions: their index in `doc.nested`. + nests: HashMap, + /// The compositions being read, outermost first. + open: Vec, +} + +struct NestClip<'a> { + mob_id: MobId, + slot_id: u32, + rate: (i64, i64), + kind: CKind, + name: String, + mob: &'a Obj, +} + +/// Compositions nested deeper than this are read as gaps. +const MAX_NESTING: usize = 16; + +pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result { + let root = store::read(bytes).map_err(|e| Error::Parse { format: "AAF", message: e })?; + let header = root.strong(pid::ROOT_HEADER).ok_or_else(|| Error::Parse { format: "AAF", message: "no header object".into() })?; + let content = header.strong(pid::CONTENT).ok_or_else(|| Error::Parse { format: "AAF", message: "no content storage".into() })?; + let mut mobs = HashMap::new(); + for m in content.objs(pid::MOBS) { + if let Some(id) = m.mob_id(pid::MOB_ID) { + mobs.insert(id, m); + } + } + let mut essence = HashMap::new(); + for e in content.objs(pid::ESSENCE_DATA) { + if let (Some(id), Some(d)) = (e.mob_id(pid::ESSENCE_MOB_ID), e.stream(pid::ESSENCE_STREAM)) { + essence.insert(id, d); + } + } + let mut r = R { report, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() }; + // Top-level compositions: those no other composition uses, among the ones tagged top-level + // when any are. (Premiere Pro tags a nested sequence's composition top-level too; it is still + // only a nested sequence.) Compositions that only use each other in a circle all count. The + // label older FilmCraft versions wrote for top-level counts as top-level too. + let comps: Vec<&Obj> = content.objs(pid::MOBS).iter().filter(|m| m.class == cls::COMPOSITION_MOB).collect(); + let mut referenced = HashSet::new(); + for c in &comps { + collect_refs(c, &mut referenced); + } + let is_top = |c: &&Obj| c.auid(pid::USAGE_CODE).is_some_and(|u| u == def::USAGE_TOP_LEVEL || u == def::USAGE_TOP_LEVEL_OLD); + let tagged_top: Vec<&Obj> = comps.iter().copied().filter(is_top).collect(); + let pool = if tagged_top.is_empty() { comps } else { tagged_top }; + let unused: Vec<&Obj> = pool.iter().copied().filter(|c| c.mob_id(pid::MOB_ID).is_none_or(|id| !referenced.contains(&id))).collect(); + let top = if unused.is_empty() { pool } else { unused }; + for c in top { + r.open.extend(c.mob_id(pid::MOB_ID)); + let comp = r.composition(c); + r.open.clear(); + r.doc.compositions.push(comp); + } + if r.doc.compositions.is_empty() { + return Err(Error::Empty); + } + Ok(r.doc) +} + +fn collect_refs(o: &Obj, out: &mut HashSet) { + if o.class == cls::SOURCE_CLIP + && let Some(id) = o.mob_id(pid::SOURCE_ID) + { + out.insert(id); + } + for (_, v) in &o.props { + match v { + Value::Strong(c) => collect_refs(c, out), + Value::StrongVec(v) | Value::StrongSet(v, _) => v.iter().for_each(|c| collect_refs(c, out)), + _ => {} + } + } +} + +fn rate_of(o: &Obj, p: u16) -> Option<(i64, i64)> { + o.rational(p).filter(|r| r.0 > 0 && r.1 > 0) +} + +impl R<'_> { + fn composition(&mut self, c: &Obj) -> Composition { + let name = c.string(pid::MOB_NAME).unwrap_or_else(|| "Sequence".into()); + let slots = c.objs(pid::SLOTS); + let pic_rate = slots + .iter() + .filter(|s| s.class == cls::TIMELINE_MOB_SLOT) + .find(|s| s.strong(pid::SEGMENT).is_some_and(|g| matches!(data_kind(g), DataKind::Picture | DataKind::Timecode))) + .and_then(|s| rate_of(s, pid::EDIT_RATE)) + .or_else(|| slots.iter().find_map(|s| rate_of(s, pid::EDIT_RATE))) + .unwrap_or((25, 1)); + let rate = FrameRate::new(pic_rate.0, pic_rate.1); + let tags = tagged_values(c, pid::MOB_USER_COMMENTS); + let tag = |n: &str| tags.iter().find(|(k, _)| k == n).map(|(_, v)| v.clone()); + let (mut width, mut height) = tag("FilmCraft Frame Size") + .and_then(|s| s.split_once('x').and_then(|(a, b)| Some((a.trim().parse().ok()?, b.trim().parse().ok()?)))) + .unwrap_or((0, 0)); + let mut sample_rate = tag("FilmCraft Audio Sample Rate").and_then(|s| s.parse().ok()).unwrap_or(0u32); + let mut comp = Composition { + name, + aaf_mob_id: c.mob_id(pid::MOB_ID).map(|id| hex(&id)), + rate, + sample_rate: 48_000, + width: 1920, + height: 1080, + start_tc: 0, + drop: false, + tracks: Vec::new(), + markers: Vec::new(), + }; + let (mut nv, mut na) = (0, 0); + for s in slots { + let Some(seg) = s.strong(pid::SEGMENT) else { continue }; + if s.class == cls::EVENT_MOB_SLOT { + let er = rate_of(s, pid::EVENT_EDIT_RATE).unwrap_or(pic_rate); + comp.markers.extend(markers_of(seg, er)); + continue; + } + let er = rate_of(s, pid::EDIT_RATE).unwrap_or(pic_rate); + let origin = s.i64(pid::ORIGIN).unwrap_or(0); + match data_kind(seg) { + DataKind::Timecode => { + if let Some(tc) = find_class(seg, cls::TIMECODE) { + comp.start_tc = tc.i64(pid::TC_START).unwrap_or(0); + comp.drop = tc.u8(pid::TC_DROP).unwrap_or(0) != 0; + } + } + kind @ (DataKind::Picture | DataKind::Sound) => { + let ckind = if kind == DataKind::Picture { CKind::Picture } else { CKind::Sound }; + if ckind == CKind::Sound && sample_rate == 0 && er.1 == 1 && er.0 >= 8000 { + sample_rate = er.0 as u32; + } + // Premiere Pro writes every video track into one slot: a nested scope with a + // segment per track, the lowest track first. (A scope whose last segment + // refers to the others is an effect over layers: one track, as before.) + let layers: Vec<&Obj> = if ckind == CKind::Picture && seg.class == cls::NESTED_SCOPE && !refers_to_scope(seg, 0) { + seg.objs(pid::NESTED_SLOTS).iter().collect() + } else { + Vec::new() + }; + let layered = !layers.is_empty(); + let whole = units_to_ticks(len_of(seg), er.0, er.1); + for layer in if layered { layers } else { vec![seg] } { + let mut items = self.segment(layer, er, ckind, 0); + if layered { + fit(&mut items, whole); + } + if origin > 0 { + // material before the origin is not shown + trim_front(&mut items, units_to_ticks(origin, er.0, er.1)); + } + let number = s.u32(pid::PHYSICAL_TRACK_NUMBER).unwrap_or(0); + let name = s.string(pid::SLOT_NAME).unwrap_or_default(); + let channels = if ckind == CKind::Sound { + items + .iter() + .filter_map(|i| match i { + CItem::Clip(c) => self.doc.sources.get(c.source).map(|s| s.channels), + _ => None, + }) + .max() + .unwrap_or(1) + .min(6) + } else { + 0 + }; + let n = if ckind == CKind::Picture { + nv += 1; + nv + } else { + na += 1; + na + }; + comp.tracks.push(CTrack { kind: ckind, name, number: if number > 0 && !layered { number } else { n }, channels, items }); + } + } + DataKind::Other => self.report.info("a slot of an unsupported data kind was skipped"), + } + } + if width == 0 { + // no size in the file: that of the first picture this composition shows, else of any + let own = comp.tracks.iter().filter(|t| t.kind == CKind::Picture).flat_map(|t| &t.items).find_map(|i| match i { + CItem::Clip(c) => self.doc.sources.get(c.source).filter(|s| s.kind == CKind::Picture && s.width > 0 && s.nested.is_none()), + _ => None, + }); + let pic = own.or_else(|| self.doc.sources.iter().find(|s| s.kind == CKind::Picture && s.width > 0)); + (width, height) = pic.map_or((1920, 1080), |p| (p.width, p.height)); + } + comp.width = width; + comp.height = height; + if sample_rate > 0 { + comp.sample_rate = sample_rate; + } + comp + } + + /// Items of a segment (a sequence or a single component). + fn segment(&mut self, seg: &Obj, rate: (i64, i64), kind: CKind, depth: usize) -> Vec { + if depth > 32 { + self.report.warn("segments nested too deeply were skipped"); + return vec![CItem::Filler(units_to_ticks(len_of(seg), rate.0, rate.1))]; + } + if seg.class == cls::SEQUENCE { + let mut out = Vec::new(); + for c in seg.objs(pid::COMPONENTS) { + out.extend(self.component(c, rate, kind, depth + 1)); + } + return out; + } + self.component(seg, rate, kind, depth + 1) + } + + fn component(&mut self, c: &Obj, rate: (i64, i64), kind: CKind, depth: usize) -> Vec { + let len = units_to_ticks(len_of(c), rate.0, rate.1); + match c.class { + x if x == cls::FILLER => vec![CItem::Filler(len)], + x if x == cls::SOURCE_CLIP => vec![self.clip(c, rate, kind, None)], + x if x == cls::SEQUENCE => self.segment(c, rate, kind, depth), + x if x == cls::TRANSITION => { + let cut = units_to_ticks(c.i64(pid::CUT_POINT).unwrap_or(0), rate.0, rate.1); + let og = c.strong(pid::OPERATION_GROUP); + let effect = og + .and_then(|o| tagged_values(o, pid::COMPONENT_USER_COMMENTS).into_iter().find(|(k, _)| k == "FilmCraft Effect").map(|(_, v)| v)) + .unwrap_or_else(|| { + let op = og.and_then(|o| o.weak_key(pid::OPERATION)).and_then(|k| ::try_from(k).ok()); + match (kind, op) { + (CKind::Sound, _) => "constant_power".into(), + (_, Some(o)) if o == def::VIDEO_FADE_TO_BLACK => "dip_to_black".into(), + (_, Some(o)) if o == def::SMPTE_VIDEO_WIPE => "wipe".into(), + _ => "cross_dissolve".into(), + } + }); + vec![CItem::Transition(CTransition { len, cut: cut.clamp(Tick::ZERO, len), effect })] + } + x if x == cls::OPERATION_GROUP => { + let op = c.weak_key(pid::OPERATION).and_then(|k| ::try_from(k).ok()); + let inputs = c.objs(pid::INPUT_SEGMENTS); + let Some(input) = inputs.first() else { + self.report.info("an effect without input was imported as a gap"); + return vec![CItem::Filler(len)]; + }; + let gain = if op == Some(def::MONO_AUDIO_GAIN) { + self.gain(c, len, rate) + } else { + self.report.warn("effects other than audio gain are not imported (their first input is used)"); + None + }; + let mut items = self.segment(input, rate, kind, depth); + // the operation's length rules + fit(&mut items, len); + if let Some(g) = gain { + match items.as_mut_slice() { + [CItem::Clip(cl)] => cl.gain = Some(g), + _ => self.report.warn("audio gain around several clips was not imported"), + } + } + items + } + x if x == cls::SELECTOR => { + self.report.info("a selector was flattened to its selected input"); + match c.strong(pid::SELECTED) { + Some(s) => { + let mut items = self.segment(s, rate, kind, depth); + fit(&mut items, len); + items + } + None => vec![CItem::Filler(len)], + } + } + x if x == cls::NESTED_SCOPE => { + self.report.info("a nested scope was flattened to its final input"); + match c.objs(pid::NESTED_SLOTS).last() { + Some(s) => { + let mut items = self.segment(s, rate, kind, depth); + fit(&mut items, len); + items + } + None => vec![CItem::Filler(len)], + } + } + _ => { + self.report.info("an unsupported component was imported as a gap"); + vec![CItem::Filler(len)] + } + } + } + + fn gain(&mut self, og: &Obj, len: Tick, _rate: (i64, i64)) -> Option { + let p = og + .objs(pid::PARAMETERS) + .iter() + .find(|p| p.auid(pid::PARAMETER_DEFINITION) == Some(def::PARAM_AMPLITUDE)) + .or_else(|| og.objs(pid::PARAMETERS).first())?; + if p.class == cls::CONSTANT_VALUE { + return p.data(pid::CONSTANT_VALUE).and_then(indirect_f64).map(Gain::Constant); + } + if p.class == cls::VARYING_VALUE { + let linear = p.weak_key(pid::INTERPOLATION).and_then(|k| ::try_from(k).ok()) != Some(def::INTERP_CONSTANT); + let points: Vec<(Tick, f64)> = p + .objs(pid::POINT_LIST) + .iter() + .filter_map(|cp| { + let (n, d) = cp.rational(pid::CP_TIME).filter(|r| r.1 != 0)?; + let v = cp.data(pid::CP_VALUE).and_then(indirect_f64)?; + Some((Tick((len.0 as i128 * n as i128 / d as i128).clamp(-(1i128 << 56), 1i128 << 56) as i64), v)) + }) + .collect(); + return (!points.is_empty()).then_some(Gain::Varying { linear, points }); + } + None + } + + fn clip(&mut self, c: &Obj, rate: (i64, i64), kind: CKind, master_key: Option) -> CItem { + let len = units_to_ticks(len_of(c), rate.0, rate.1); + let tagged_name = tagged_values(c, pid::COMPONENT_USER_COMMENTS).into_iter().find(|(k, _)| k == "Clip Name").map(|(_, v)| v); + let (Some(id), Some(slot)) = (c.mob_id(pid::SOURCE_ID), c.u32(pid::SOURCE_MOB_SLOT_ID)) else { return CItem::Filler(len) }; + let name = tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME))).unwrap_or_default(); + if id == [0; 32] { + return CItem::Filler(len); + } + if let Some(mob) = self.mobs.get(&id).copied().filter(|m| m.class == cls::COMPOSITION_MOB) { + return self.nest_clip(c, NestClip { mob, mob_id: id, slot_id: slot, rate, kind, name }); + } + let source_rate = self + .mobs + .get(&id) + .and_then(|mob| mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot))) + .and_then(|s| rate_of(s, pid::EDIT_RATE)) + .unwrap_or(rate); + let start = units_to_ticks(c.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1); + match self.resolve(id, slot, start, kind, master_key, 0) { + Some((source, t)) => CItem::Clip(CClip { len, source, start: t, gain: None, name }), + None => CItem::Filler(len), + } + } + + /// A source clip that points at a composition: a nested sequence (Premiere Pro writes its + /// nested sequences this way). The composition is read once, however many clips use it. + fn nest_clip(&mut self, c: &Obj, clip: NestClip<'_>) -> CItem { + let NestClip { mob, mob_id: id, slot_id, rate, kind, name } = clip; + let len = units_to_ticks(len_of(c), rate.0, rate.1); + let index = match self.nests.get(&id) { + Some(&i) => i, + None => { + if self.open.contains(&id) || self.open.len() >= MAX_NESTING { + self.report.warn("a composition nested in itself or nested too deeply was imported as a gap"); + return CItem::Filler(len); + } + self.open.push(id); + let comp = self.composition(mob); + self.open.pop(); + let i = self.doc.nested.len(); + self.doc.nested.push(comp); + self.nests.insert(id, i); + i + } + }; + // the start counts edit units of the slot the clip points at + let slot_rate = + mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot_id)).and_then(|s| rate_of(s, pid::EDIT_RATE)).unwrap_or(rate); + let start = units_to_ticks(c.i64(pid::START_TIME).unwrap_or(0), slot_rate.0, slot_rate.1); + let skey = (id, if kind == CKind::Picture { 0 } else { 1 }, "composition".to_string()); + let source = match self.sources.get(&skey) { + Some(&i) => i, + None => { + let Some(comp) = self.doc.nested.get(index) else { return CItem::Filler(len) }; + let length = comp + .tracks + .iter() + .map(|t| { + Tick(t.items.iter().fold(0i64, |a, i| { + if matches!(i, CItem::Transition(_)) { a.saturating_sub(i.len().0) } else { a.saturating_add(i.len().0) } + })) + }) + .max() + .unwrap_or(Tick::ZERO); + let channels = if kind == CKind::Sound { 2 } else { 0 }; + let s = Source { + key: format!("composition:{}", hex(&id)), + name: comp.name.clone(), + kind, + path: None, + channel: None, + channels, + file_channels: channels, + width: comp.width, + height: comp.height, + frame_rate: comp.rate, + sample_rate: comp.sample_rate, + bits: 16, + length, + offset: Tick::ZERO, + start_tc: None, + tc_rate: comp.rate, + embedded: None, + markers: Vec::new(), + nested: Some(index), + }; + let i = self.doc.sources.len(); + self.doc.sources.push(s); + self.sources.insert(skey, i); + i + } + }; + CItem::Clip(CClip { len, source, start, gain: None, name }) + } + + /// Follow a source reference to a file source mob: (source index, file time). + fn resolve(&mut self, id: MobId, slot_id: u32, start: Tick, kind: CKind, master_key: Option, depth: usize) -> Option<(usize, Tick)> { + if depth > 16 { + return None; + } + let Some(mob) = self.mobs.get(&id).copied() else { + self.report.warn("a clip references a mob that is not in the file (imported as a gap)"); + return None; + }; + let slot = mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot_id))?; + let rate = rate_of(slot, pid::EDIT_RATE).unwrap_or((25, 1)); + let seg = slot.strong(pid::SEGMENT)?; + if mob.class == cls::COMPOSITION_MOB { + self.report.warn("nested compositions are not supported (imported as gaps)"); + return None; + } + if mob.class == cls::MASTER_MOB { + let key = master_key.unwrap_or_else(|| hex(&id)); + let clip = first_clip(seg)?; + let next = clip.mob_id(pid::SOURCE_ID)?; + let next_slot = clip.u32(pid::SOURCE_MOB_SLOT_ID)?; + let source_rate = self + .mobs + .get(&next) + .and_then(|m| m.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(next_slot))) + .and_then(|s| rate_of(s, pid::EDIT_RATE)) + .unwrap_or(rate); + let inner = units_to_ticks(clip.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1); + if next == [0; 32] { + return None; + } + let markers = mob + .objs(pid::SLOTS) + .iter() + .filter(|s| s.class == cls::EVENT_MOB_SLOT) + .flat_map(|s| s.strong(pid::SEGMENT).map(|g| markers_of(g, rate_of(s, pid::EVENT_EDIT_RATE).unwrap_or(rate))).unwrap_or_default()) + .collect::>(); + let r = self.resolve(next, next_slot, start + inner, kind, Some(key), depth + 1)?; + let src = &mut self.doc.sources[r.0]; + if src.markers.is_empty() { + src.markers = markers; + } + if src.name.is_empty() + && let Some(n) = mob.string(pid::MOB_NAME) + { + src.name = n; + } + return Some(r); + } + // a source mob: file (has a file descriptor) or physical (tape / import) + let desc = mob.strong(pid::ESSENCE_DESCRIPTION); + let is_file = desc.is_some_and(|d| d.get(pid::SAMPLE_RATE).is_some() || d.get(pid::LOCATOR).is_some() || self.essence.contains_key(&id)); + if !is_file { + return None; + } + let key = master_key.unwrap_or_else(|| hex(&id)); + let skey = (id, slot_id, key.clone()); + if let Some(&i) = self.sources.get(&skey) { + return Some((i, start)); + } + let d = desc?; + let seg_kind = match data_kind(seg) { + DataKind::Picture => CKind::Picture, + DataKind::Sound => CKind::Sound, + _ => kind, + }; + let path = d.objs(pid::LOCATOR).iter().find_map(|l| l.string(pid::URL_STRING)).map(|u| locator_path(&u)); + let file_rate = rate_of(d, pid::SAMPLE_RATE).unwrap_or(rate); + let length = d + .i64(pid::FILE_LENGTH) + .map(|l| units_to_ticks(l, file_rate.0, file_rate.1)) + .unwrap_or_else(|| units_to_ticks(len_of(seg), rate.0, rate.1)); + let sound_slots = mob.objs(pid::SLOTS).iter().filter(|s| s.strong(pid::SEGMENT).is_some_and(|g| data_kind(g) == DataKind::Sound)).count(); + let file_channels = d.u32(pid::CHANNELS).unwrap_or(1).max(1); + let channel = (seg_kind == CKind::Sound && file_channels > 1 && sound_slots > 1) + .then(|| slot.u32(pid::PHYSICAL_TRACK_NUMBER).unwrap_or(slot_id).saturating_sub(1).min(file_channels - 1)); + let mut layout = d.u8(pid::FRAME_LAYOUT).unwrap_or(0); + if layout > 4 { + layout = 0; + } + let stored_h = d.u32(pid::STORED_HEIGHT).unwrap_or(0); + let fr = FrameRate::new(file_rate.0, file_rate.1); + // timecode from the physical source mob + let tc = first_clip(seg).and_then(|clip| { + let tid = clip.mob_id(pid::SOURCE_ID)?; + let tape = self.mobs.get(&tid)?; + let tslot = tape.objs(pid::SLOTS).iter().find_map(|s| { + let g = s.strong(pid::SEGMENT)?; + let t = find_class(g, cls::TIMECODE)?; + Some((t.i64(pid::TC_START).unwrap_or(0), t.u16(pid::TC_FPS).unwrap_or(25).max(1) as i64, rate_of(s, pid::EDIT_RATE))) + })?; + let off = units_to_ticks(clip.i64(pid::START_TIME).unwrap_or(0), rate.0, rate.1); + let tcr = tslot.2.map(|r| FrameRate::new(r.0, r.1)).unwrap_or_else(|| FrameRate::new(tslot.1, 1)); + Some((tslot.0 + tcr.frame_at(off), tcr)) + }); + let mut s = Source { + key, + name: mob.string(pid::MOB_NAME).unwrap_or_default(), + kind: seg_kind, + path, + channel, + channels: if channel.is_some() { 1 } else { file_channels }, + file_channels, + width: d.u32(pid::STORED_WIDTH).unwrap_or(0), + height: if matches!(layout, 1 | 3) { stored_h.saturating_mul(2) } else { stored_h }, + frame_rate: if seg_kind == CKind::Picture { fr } else { FrameRate::FPS_25 }, + sample_rate: d + .rational(pid::AUDIO_SAMPLING_RATE) + .filter(|r| r.1 > 0 && r.0 > 0) + .map(|r| (r.0 / r.1) as u32) + .unwrap_or(file_rate.0 as u32), + bits: d.u32(pid::QUANTIZATION_BITS).unwrap_or(16).clamp(8, 32) as u16, + length, + offset: Tick::ZERO, + start_tc: tc.map(|t| t.0), + tc_rate: tc.map(|t| t.1).unwrap_or(fr), + embedded: None, + markers: Vec::new(), + nested: None, + }; + if s.kind == CKind::Sound { + s.frame_rate = FrameRate::FPS_25; + if let Some(data) = self.essence.get(&id) { + s.embedded = Some(data.to_vec()); + // embedded essence may be a whole WAVE / AIFF file + if let Some((pcm, ch, sr, bits)) = crate::pcm::parse_wav(data) { + s.embedded = Some(pcm); + s.file_channels = ch as u32; + s.channels = if s.channel.is_some() { 1 } else { ch as u32 }; + s.sample_rate = sr; + s.bits = bits; + } + } + } + if s.kind == CKind::Picture && s.width == 0 { + s.width = 1920; + s.height = 1080; + } + let i = self.doc.sources.len(); + self.doc.sources.push(s); + self.sources.insert(skey, i); + Some((i, start)) + } +} + +/// Drop `t` from the front of a track's items (slot origin). +fn trim_front(items: &mut Vec, mut t: Tick) { + while t > Tick::ZERO && !items.is_empty() { + let l = items[0].len(); + if matches!(items[0], CItem::Transition(_)) { + items.remove(0); + continue; + } + if l <= t { + items.remove(0); + t -= l; + } else { + match &mut items[0] { + CItem::Filler(x) => *x -= t, + CItem::Clip(c) => { + c.len -= t; + c.start += t; + } + CItem::Transition(_) => {} + } + t = Tick::ZERO; + } + } +} + +/// Make the items exactly `len` long (an operation group / selector is as long as itself). +fn fit(items: &mut Vec, len: Tick) { + let total = Tick( + items.iter().fold(0i64, |a, i| if matches!(i, CItem::Transition(_)) { a.saturating_sub(i.len().0) } else { a.saturating_add(i.len().0) }), + ); + if total == len || items.is_empty() { + if items.is_empty() { + items.push(CItem::Filler(len)); + } + return; + } + if total < len { + items.push(CItem::Filler(len - total)); + return; + } + let mut excess = total - len; + while excess > Tick::ZERO { + match items.last_mut() { + Some(CItem::Filler(x)) | Some(CItem::Clip(CClip { len: x, .. })) if *x > excess => { + *x -= excess; + excess = Tick::ZERO; + } + Some(CItem::Transition(_)) => { + items.pop(); + } + Some(other) => { + excess -= other.len(); + items.pop(); + } + None => break, + } + } +} + +/// Whether anything inside `o` is a scope reference (a nested scope used as an effect's layers). +fn refers_to_scope(o: &Obj, depth: usize) -> bool { + if o.class == cls::SCOPE_REFERENCE { + return true; + } + if depth > 32 { + return false; + } + o.props.iter().any(|(_, v)| match v { + Value::Strong(c) => refers_to_scope(c, depth + 1), + Value::StrongVec(v) | Value::StrongSet(v, _) => v.iter().any(|c| refers_to_scope(c, depth + 1)), + _ => false, + }) +} + +fn first_clip(seg: &Obj) -> Option<&Obj> { + if seg.class == cls::SOURCE_CLIP { + return Some(seg); + } + if seg.class == cls::SEQUENCE { + return seg.objs(pid::COMPONENTS).iter().find_map(first_clip); + } + // essence group (choices 0x0501) / operation group inputs + seg.objs(0x0501).iter().find_map(first_clip).or_else(|| seg.objs(pid::INPUT_SEGMENTS).iter().find_map(first_clip)) +} + +fn find_class(seg: &Obj, class: Auid) -> Option<&Obj> { + if seg.class == class { + return Some(seg); + } + seg.objs(pid::COMPONENTS).iter().find_map(|c| find_class(c, class)) +} + +fn markers_of(seg: &Obj, rate: (i64, i64)) -> Vec { + let events: Vec<&Obj> = if seg.class == cls::SEQUENCE { seg.objs(pid::COMPONENTS).iter().collect() } else { vec![seg] }; + events + .into_iter() + .filter(|e| e.class == cls::COMMENT_MARKER || e.class == cls::DESCRIPTIVE_MARKER || e.get(pid::POSITION).is_some()) + .map(|e| { + let tags = tagged_values(e, pid::COMPONENT_USER_COMMENTS); + let tag = |n: &str| tags.iter().find(|(k, _)| k == n).map(|(_, v)| v.clone()); + let comment = e.string(pid::COMMENT).unwrap_or_default(); + let start = units_to_ticks(e.i64(pid::POSITION).unwrap_or(0), rate.0, rate.1); + CMarker { + start, + duration: units_to_ticks(e.i64(pid::LENGTH).unwrap_or(0).max(0), rate.0, rate.1), + name: tag("Name").unwrap_or_else(|| comment.clone()), + comment: tag("Comment").unwrap_or_else(|| if tags.is_empty() { String::new() } else { comment.clone() }), + color: tag("Color"), + } + }) + .collect() +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn locator_path(url: &str) -> String { + if url.contains("://") { craft_timeline::path::file_url_to_path(url) } else { craft_timeline::path::decode_uri_path(url) } +} + +#[cfg(test)] +mod locator_tests { + use super::locator_path; + + #[test] + fn decodes_relative_locator_escapes_without_losing_remote_authorities() { + assert_eq!(locator_path("Audio%20Files/Di%C3%A1logo%20%231.wav"), "Audio Files/Diálogo #1.wav"); + assert_eq!(locator_path("file:///tmp/a%20b.wav"), "/tmp/a b.wav"); + assert_eq!(locator_path("file://server/share/a.wav"), "//server/share/a.wav"); + assert_eq!(locator_path("https://example.com/a%20b.wav"), "https://example.com/a%20b.wav"); + } +} diff --git a/crates/aaf/src/aaf/store.rs b/crates/aaf/src/aaf/store.rs new file mode 100644 index 000000000..69b14fcf2 --- /dev/null +++ b/crates/aaf/src/aaf/store.rs @@ -0,0 +1,539 @@ +//! AAF objects in structured storage (the AAF Low-Level Container Specification): every object +//! is a storage whose class id is the object's class AUID, holding a `properties` stream; strong +//! references are child storages, collections have an index stream, weak references name a key +//! in a target collection through the root's `referenced properties` table. +//! +//! `properties` stream (little-endian): +//! +//! ```text +//! u8 byte order (0x4C 'L') u8 format version (0x20) u16 entry count +//! entry count × { u16 pid, u16 stored form, u16 value length } the values, in entry order +//! ``` +//! +//! | stored form | value | +//! |---|---| +//! | 0x82 data | the property's bytes | +//! | 0x22 strong reference | UTF-16 name of the child storage | +//! | 0x32 strong reference vector | UTF-16 collection name; storages `name{key}`, stream `name index`: u32 count, u32 first free key, u32 last free key, count × u32 local key | +//! | 0x3A strong reference set | as a vector; the index adds u16 key pid, u8 key size, and each entry is u32 local key, u32 reference count, key bytes | +//! | 0x02 weak reference | u16 referenced-property tag, u16 key pid, u8 key size, key bytes | +//! | 0x12 / 0x1A weak reference vector / set | UTF-16 collection name; stream `name index`: u32 count, u16 tag, u16 key pid, u8 key size, count × key | +//! | 0x42 data stream | u8 byte order, UTF-16 name of the stream | +//! +//! `referenced properties` (root stream): u8 byte order, u16 path count, u32 pid count, then the +//! pids of each path (from the root object) terminated by a 0 pid. + +use craft_cfb::{CompoundFile, EntryKind, NodeId, Version, Writer}; + +pub(crate) type Auid = [u8; 16]; + +pub(crate) const SF_DATA: u16 = 0x82; +pub(crate) const SF_DATA_STREAM: u16 = 0x42; +pub(crate) const SF_STRONG: u16 = 0x22; +pub(crate) const SF_STRONG_VECTOR: u16 = 0x32; +pub(crate) const SF_STRONG_SET: u16 = 0x3A; +pub(crate) const SF_WEAK: u16 = 0x02; +pub(crate) const SF_WEAK_VECTOR: u16 = 0x12; +pub(crate) const SF_WEAK_SET: u16 = 0x1A; + +const BYTE_ORDER_LE: u8 = 0x4C; +const FORMAT_VERSION: u8 = 0x20; + +/// A property value. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Value { + Data(Vec), + Strong(Box), + StrongVec(Vec), + /// Objects keyed by the data property `key_pid` of each. + StrongSet(Vec, u16), + /// (path of pids from the root to the target collection, key pid, key) + Weak(Vec, u16, Vec), + WeakVec(Vec, u16, Vec>), + Stream(Vec), +} + +/// An AAF object: its class and properties. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Obj { + pub class: Auid, + pub props: Vec<(u16, Value)>, +} + +impl Obj { + pub fn new(class: Auid) -> Obj { + Obj { class, props: Vec::new() } + } + pub fn set(&mut self, pid: u16, v: Value) -> &mut Obj { + match self.props.iter_mut().find(|(p, _)| *p == pid) { + Some(slot) => slot.1 = v, + None => self.props.push((pid, v)), + } + self + } + pub fn with(mut self, pid: u16, v: Value) -> Obj { + self.set(pid, v); + self + } + pub fn data_prop(mut self, pid: u16, d: Vec) -> Obj { + self.set(pid, Value::Data(d)); + self + } + pub fn get(&self, pid: u16) -> Option<&Value> { + self.props.iter().find(|(p, _)| *p == pid).map(|(_, v)| v) + } + pub fn data(&self, pid: u16) -> Option<&[u8]> { + match self.get(pid)? { + Value::Data(d) => Some(d), + _ => None, + } + } + pub fn strong(&self, pid: u16) -> Option<&Obj> { + match self.get(pid)? { + Value::Strong(o) => Some(o), + _ => None, + } + } + /// Elements of a strong vector or set (empty when absent). + pub fn objs(&self, pid: u16) -> &[Obj] { + match self.get(pid) { + Some(Value::StrongVec(v)) | Some(Value::StrongSet(v, _)) => v, + _ => &[], + } + } + pub fn weak_key(&self, pid: u16) -> Option<&[u8]> { + match self.get(pid)? { + Value::Weak(_, _, k) => Some(k), + _ => None, + } + } + pub fn stream(&self, pid: u16) -> Option<&[u8]> { + match self.get(pid)? { + Value::Stream(d) => Some(d), + _ => None, + } + } + pub fn u8(&self, pid: u16) -> Option { + self.data(pid).and_then(|d| d.first().copied()) + } + pub fn u16(&self, pid: u16) -> Option { + self.data(pid).filter(|d| d.len() >= 2).map(|d| u16::from_le_bytes([d[0], d[1]])) + } + pub fn u32(&self, pid: u16) -> Option { + self.data(pid).filter(|d| d.len() >= 4).map(|d| u32::from_le_bytes([d[0], d[1], d[2], d[3]])) + } + pub fn i64(&self, pid: u16) -> Option { + let d = self.data(pid)?; + match d.len() { + 8.. => Some(i64::from_le_bytes(d[..8].try_into().ok()?)), + 4..=7 => Some(i32::from_le_bytes(d[..4].try_into().ok()?) as i64), + _ => None, + } + } + pub fn rational(&self, pid: u16) -> Option<(i64, i64)> { + self.data(pid).and_then(rational_of) + } + pub fn string(&self, pid: u16) -> Option { + self.data(pid).map(utf16_of) + } + pub fn auid(&self, pid: u16) -> Option { + self.data(pid).and_then(|d| d.get(..16)).and_then(|d| d.try_into().ok()) + } + pub fn mob_id(&self, pid: u16) -> Option<[u8; 32]> { + self.data(pid).and_then(|d| d.get(..32)).and_then(|d| d.try_into().ok()) + } +} + +pub(crate) fn rational_of(d: &[u8]) -> Option<(i64, i64)> { + (d.len() >= 8).then(|| (i32::from_le_bytes([d[0], d[1], d[2], d[3]]) as i64, i32::from_le_bytes([d[4], d[5], d[6], d[7]]) as i64)) +} + +pub(crate) fn rational(num: i64, den: i64) -> Vec { + let mut v = (num.clamp(i32::MIN as i64, i32::MAX as i64) as i32).to_le_bytes().to_vec(); + v.extend_from_slice(&(den.clamp(i32::MIN as i64, i32::MAX as i64) as i32).to_le_bytes()); + v +} + +/// UTF-16LE with a terminating NUL. +pub(crate) fn utf16z(s: &str) -> Vec { + let mut v: Vec = s.encode_utf16().flat_map(u16::to_le_bytes).collect(); + v.extend_from_slice(&[0, 0]); + v +} + +pub(crate) fn utf16_of(d: &[u8]) -> String { + let u: Vec = d.as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes([c[0], c[1]])).take_while(|&c| c != 0).collect(); + String::from_utf16_lossy(&u) +} + +/// Short property names used to name storages and streams (any name is valid: readers follow +/// the names stored in the property values). +fn prop_name(pid: u16) -> &'static str { + match pid { + 0x0001 => "MetaDictionary", + 0x0002 => "Header", + 0x0003 => "ClassDefinitions", + 0x0004 => "TypeDefinitions", + 0x3B03 => "Content", + 0x3B04 => "Dictionary", + 0x3B06 => "IdentificationList", + 0x1901 => "Mobs", + 0x1902 => "EssenceData", + 0x2603 => "OperationDefinitions", + 0x2604 => "ParameterDefinitions", + 0x2605 => "DataDefinitions", + 0x2607 => "CodecDefinitions", + 0x2608 => "ContainerDefinitions", + 0x2609 => "InterpolationDefs", + 0x4403 => "Slots", + 0x4406 => "UserComments", + 0x4803 => "Segment", + 0x1001 => "Components", + 0x1801 => "OperationGroup", + 0x0B02 => "InputSegments", + 0x0B03 => "Parameters", + 0x4E02 => "PointList", + 0x4701 => "EssenceDescription", + 0x2F01 => "Locator", + 0x2702 => "Data", + 0x0204 => "ComponentComments", + 0x1E09 => "ParametersDefined", + _ => "Property", + } +} + +/// The key size of a strong reference set keyed by `key_pid`, for a set with no entry to take it +/// from: AUIDs (definitions) are 16 bytes, MobIDs 32. +fn set_key_size(key_pid: u16) -> usize { + match key_pid { + 0x0005 | 0x1B01 => 16, + 0x4401 | 0x2701 => 32, + _ => 0, + } +} + +fn storage_name(pid: u16) -> String { + let suffix = format!("-{pid:x}"); + let name: String = prop_name(pid).chars().take(22 - suffix.len()).collect(); + format!("{name}{suffix}") +} + +/// Errors while reading the object tree. +fn bad(s: impl Into) -> String { + s.into() +} + +// --------------------------------------------------------------------------------------------- +// Writing +// --------------------------------------------------------------------------------------------- + +struct W { + cfb: Writer, + paths: Vec>, +} + +/// Serialise `root` (the Root object: MetaDictionary 0x0001, Header 0x0002) into a compound file. +pub(crate) fn write(root: &Obj, version: Version) -> Result, String> { + let mut w = W { cfb: Writer::new(version), paths: Vec::new() }; + w.obj(Writer::ROOT, root)?; + let mut rp = vec![BYTE_ORDER_LE]; + rp.extend_from_slice(&(w.paths.len() as u16).to_le_bytes()); + let n: usize = w.paths.iter().map(|p| p.len() + 1).sum(); + rp.extend_from_slice(&(n as u32).to_le_bytes()); + for p in &w.paths { + for pid in p { + rp.extend_from_slice(&pid.to_le_bytes()); + } + rp.extend_from_slice(&0u16.to_le_bytes()); + } + w.cfb.stream(Writer::ROOT, "referenced properties", rp).map_err(|e| e.to_string())?; + Ok(w.cfb.finish()) +} + +impl W { + fn tag(&mut self, path: &[u16]) -> u16 { + match self.paths.iter().position(|p| p == path) { + Some(i) => i as u16, + None => { + self.paths.push(path.to_vec()); + (self.paths.len() - 1) as u16 + } + } + } + + fn obj(&mut self, node: NodeId, o: &Obj) -> Result<(), String> { + self.cfb.set_clsid(node, o.class); + let mut entries: Vec<(u16, u16, Vec)> = Vec::new(); + for (pid, v) in &o.props { + let pid = *pid; + let (sf, value) = match v { + Value::Data(d) => (SF_DATA, d.clone()), + Value::Strong(child) => { + let name = storage_name(pid); + let s = self.cfb.storage(node, &name, child.class).map_err(|e| e.to_string())?; + self.obj(s, child)?; + (SF_STRONG, utf16z(&name)) + } + Value::StrongVec(items) | Value::StrongSet(items, _) => { + let name = storage_name(pid); + let mut index = Vec::new(); + index.extend_from_slice(&(items.len() as u32).to_le_bytes()); + index.extend_from_slice(&(items.len() as u32).to_le_bytes()); // first free key + index.extend_from_slice(&u32::MAX.to_le_bytes()); // last free key + let set_key = if let Value::StrongSet(_, k) = v { Some(*k) } else { None }; + let key_size = set_key.map(|k| items.first().and_then(|i| i.data(k)).map_or_else(|| set_key_size(k), <[u8]>::len)).unwrap_or(0); + if let Some(k) = set_key { + index.extend_from_slice(&k.to_le_bytes()); + index.push(key_size as u8); + } + for (i, item) in items.iter().enumerate() { + let s = self.cfb.storage(node, &format!("{name}{{{i:x}}}"), item.class).map_err(|e| e.to_string())?; + self.obj(s, item)?; + index.extend_from_slice(&(i as u32).to_le_bytes()); + if let Some(k) = set_key { + index.extend_from_slice(&1u32.to_le_bytes()); + let mut key = item.data(k).unwrap_or(&[]).to_vec(); + key.resize(key_size, 0); + index.extend_from_slice(&key); + } + } + self.cfb.stream(node, &format!("{name} index"), index).map_err(|e| e.to_string())?; + (if set_key.is_some() { SF_STRONG_SET } else { SF_STRONG_VECTOR }, utf16z(&name)) + } + Value::Weak(path, key_pid, key) => { + let mut d = self.tag(path).to_le_bytes().to_vec(); + d.extend_from_slice(&key_pid.to_le_bytes()); + d.push(key.len() as u8); + d.extend_from_slice(key); + (SF_WEAK, d) + } + Value::WeakVec(path, key_pid, keys) => { + let name = storage_name(pid); + let tag = self.tag(path); + let size = keys.first().map_or(0, Vec::len); + let mut index = (keys.len() as u32).to_le_bytes().to_vec(); + index.extend_from_slice(&tag.to_le_bytes()); + index.extend_from_slice(&key_pid.to_le_bytes()); + index.push(size as u8); + for k in keys { + let mut k = k.clone(); + k.resize(size, 0); + index.extend_from_slice(&k); + } + self.cfb.stream(node, &format!("{name} index"), index).map_err(|e| e.to_string())?; + (SF_WEAK_VECTOR, utf16z(&name)) + } + Value::Stream(data) => { + let name = storage_name(pid); + self.cfb.stream(node, &name, data.clone()).map_err(|e| e.to_string())?; + let mut d = vec![BYTE_ORDER_LE]; + d.extend_from_slice(&utf16z(&name)); + (SF_DATA_STREAM, d) + } + }; + if value.len() > u16::MAX as usize { + return Err(format!("property {pid:#06x} is {} bytes (the limit is 65535)", value.len())); + } + entries.push((pid, sf, value)); + } + let mut props = vec![BYTE_ORDER_LE, FORMAT_VERSION]; + props.extend_from_slice(&(entries.len() as u16).to_le_bytes()); + for (pid, sf, v) in &entries { + props.extend_from_slice(&pid.to_le_bytes()); + props.extend_from_slice(&sf.to_le_bytes()); + props.extend_from_slice(&(v.len() as u16).to_le_bytes()); + } + for (_, _, v) in &entries { + props.extend_from_slice(v); + } + self.cfb.stream(node, "properties", props).map_err(|e| e.to_string())?; + Ok(()) + } +} + +// --------------------------------------------------------------------------------------------- +// Reading +// --------------------------------------------------------------------------------------------- + +struct R<'a> { + cf: CompoundFile<'a>, + paths: Vec>, + objects: usize, + materialized_stream_bytes: usize, + /// Storages already read: a storage is owned by one parent, so a second strong reference to it + /// (which would rebuild its whole subtree again) is refused. + owned: std::collections::HashSet, + /// Bytes of weak-reference target paths copied into the tree so far. + path_bytes: usize, +} + +const MAX_DEPTH: usize = 200; +const MAX_OBJECTS: usize = 2_000_000; +const MAX_MATERIALIZED_STREAM_BYTES: usize = 256 << 20; +/// Each weak reference carries its own copy of its target path (real paths are a handful of +/// property ids); the copies together may not exceed this. +const MAX_PATH_BYTES: usize = 64 << 20; + +/// Parse the object tree of an AAF compound file (the root object). +pub(crate) fn read(bytes: &[u8]) -> Result { + let cf = CompoundFile::open(bytes).map_err(|e| e.to_string())?; + cf.validate_root_chain().map_err(|e| e.to_string())?; + let mut paths = Vec::new(); + if let Ok(rp) = cf.read_path("referenced properties") + && rp.len() >= 7 + { + let count = u16::from_le_bytes([rp[1], rp[2]]) as usize; + let mut cur = Vec::new(); + for c in rp[7..].as_chunks::<2>().0 { + let pid = u16::from_le_bytes([c[0], c[1]]); + if pid == 0 { + paths.push(std::mem::take(&mut cur)); + if paths.len() == count { + break; + } + } else { + cur.push(pid); + } + } + } + let mut r = R { cf, paths, objects: 0, materialized_stream_bytes: 0, owned: Default::default(), path_bytes: 0 }; + r.obj(0, 0) +} + +fn rd_u16(b: &[u8], at: usize) -> Option { + b.get(at..at + 2).map(|s| u16::from_le_bytes([s[0], s[1]])) +} +fn rd_u32(b: &[u8], at: usize) -> Option { + b.get(at..at + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]])) +} + +impl R<'_> { + fn child(&self, parent: usize, name: &str) -> Result { + self.cf.child(parent, name).ok_or_else(|| bad(format!("missing storage or stream \"{name}\""))) + } + + fn stream(&mut self, parent: usize, name: &str) -> Result, String> { + let id = self.child(parent, name)?; + let data = self.cf.read(id).map_err(|e| e.to_string())?; + self.materialized_stream_bytes = self + .materialized_stream_bytes + .checked_add(data.len()) + .filter(|&n| n <= MAX_MATERIALIZED_STREAM_BYTES) + .ok_or_else(|| bad("AAF materialized stream data exceeds the memory limit"))?; + Ok(data) + } + + /// A copy of weak-reference path `tag`, counted against [`MAX_PATH_BYTES`]. + fn path(&mut self, tag: u16) -> Result, String> { + let path = self.paths.get(tag as usize).cloned().unwrap_or_default(); + self.path_bytes = self.path_bytes.saturating_add(path.len() * 2); + if self.path_bytes > MAX_PATH_BYTES { + return Err(bad("weak reference paths exceed the memory limit")); + } + Ok(path) + } + + fn obj(&mut self, entry: usize, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(bad("objects nested too deeply")); + } + if !self.owned.insert(entry) { + return Err(bad("a storage is owned by more than one reference")); + } + self.objects += 1; + if self.objects > MAX_OBJECTS { + return Err(bad("too many objects")); + } + let e = self.cf.entry(entry).ok_or_else(|| bad("bad directory entry"))?; + if !matches!(e.kind, EntryKind::Root | EntryKind::Storage) { + return Err(bad(format!("\"{}\" is not a storage", e.name))); + } + let mut o = Obj::new(e.clsid); + let props = self.stream(entry, "properties")?; + if props.len() < 4 { + return Err(bad("short properties stream")); + } + if props[0] != BYTE_ORDER_LE { + return Err(bad("big-endian AAF files are not supported")); + } + let n = rd_u16(&props, 2).unwrap_or(0) as usize; + let mut at = 4 + n * 6; + if at > props.len() { + return Err(bad("truncated property index")); + } + for i in 0..n { + let (Some(pid), Some(sf), Some(len)) = (rd_u16(&props, 4 + i * 6), rd_u16(&props, 6 + i * 6), rd_u16(&props, 8 + i * 6)) else { + return Err(bad("truncated property index")); + }; + if o.props.iter().any(|(previous, _)| *previous == pid) { + return Err(bad("duplicate property id")); + } + let v = props.get(at..at + len as usize).ok_or_else(|| bad("truncated property value"))?; + at += len as usize; + let value = match sf { + SF_STRONG => { + let c = self.child(entry, &utf16_of(v))?; + Value::Strong(Box::new(self.obj(c, depth + 1)?)) + } + SF_STRONG_VECTOR | SF_STRONG_SET => { + let name = utf16_of(v); + let idx = self.stream(entry, &format!("{name} index"))?; + let count = rd_u32(&idx, 0).ok_or_else(|| bad("short collection index"))? as usize; + let mut items = Vec::new(); + if sf == SF_STRONG_VECTOR { + for k in 0..count { + let key = rd_u32(&idx, 12 + k * 4).ok_or_else(|| bad("short vector index"))?; + let c = self.child(entry, &format!("{name}{{{key:x}}}"))?; + items.push(self.obj(c, depth + 1)?); + } + Value::StrongVec(items) + } else { + let key_pid = rd_u16(&idx, 12).ok_or_else(|| bad("short set index"))?; + let key_size = *idx.get(14).ok_or_else(|| bad("short set index"))? as usize; + let stride = 8 + key_size; + for k in 0..count { + let key = rd_u32(&idx, 15 + k * stride).ok_or_else(|| bad("short set index"))?; + let c = self.child(entry, &format!("{name}{{{key:x}}}"))?; + items.push(self.obj(c, depth + 1)?); + } + Value::StrongSet(items, key_pid) + } + } + SF_WEAK => { + let (Some(tag), Some(key_pid), Some(&size)) = (rd_u16(v, 0), rd_u16(v, 2), v.get(4)) else { + return Err(bad("short weak reference")); + }; + let key = v.get(5..5 + size as usize).ok_or_else(|| bad("short weak reference"))?.to_vec(); + Value::Weak(self.path(tag)?, key_pid, key) + } + SF_WEAK_VECTOR | SF_WEAK_SET => { + let name = utf16_of(v); + let idx = self.stream(entry, &format!("{name} index"))?; + let (Some(count), Some(tag), Some(key_pid), Some(&size)) = (rd_u32(&idx, 0), rd_u16(&idx, 4), rd_u16(&idx, 6), idx.get(8)) else { + return Err(bad("short weak collection index")); + }; + let size = size as usize; + if count > 0 && size == 0 { + return Err(bad("zero-size weak collection key")); + } + // every key must lie in the index, so its length (not the count) bounds the loop + if (count as usize).checked_mul(size).and_then(|n| n.checked_add(9)).is_none_or(|end| end > idx.len()) { + return Err(bad("short weak collection index")); + } + let mut keys = Vec::new(); + for k in 0..count as usize { + keys.push(idx.get(9 + k * size..9 + (k + 1) * size).ok_or_else(|| bad("short weak collection index"))?.to_vec()); + } + Value::WeakVec(self.path(tag)?, key_pid, keys) + } + SF_DATA_STREAM => { + let name = utf16_of(v.get(1..).unwrap_or(&[])); + Value::Stream(self.stream(entry, &name)?) + } + _ => Value::Data(v.to_vec()), + }; + o.props.push((pid, value)); + } + Ok(o) + } +} diff --git a/crates/aaf/src/aaf/tests.rs b/crates/aaf/src/aaf/tests.rs new file mode 100644 index 000000000..3e7794471 --- /dev/null +++ b/crates/aaf/src/aaf/tests.rs @@ -0,0 +1,559 @@ +//! Structural checks of written AAF files against the required properties and rules of the AAF +//! Object Specification v1.1 and the Edit Protocol, read back through the compound file reader. + +use std::collections::HashSet; + +use craft_timeline::time::FrameRate; + +use super::ids::{self, cls, ddef, def, path, pid}; +use super::store::{self, Obj, Value}; +use super::*; +use crate::model::{CItem, CKind}; +use crate::testdoc::{IDENTITY, clip, composition, picture_and_sound_cut, source, track}; + +fn document() -> Document { + // two 48-frame clips from source frames 10 and 200, a 12-frame transition + picture_and_sound_cut(FrameRate::FPS_24, 48, 12, (10, 200)) +} + +fn written(doc: &Document, small_sectors: bool) -> Vec { + let mut report = Report::default(); + let bytes = write(doc, &WriteOptions { identity: IDENTITY, small_sectors }, &mut report).unwrap(); + assert!(!report.has_warnings(), "{report}"); + bytes +} + +fn read_clean(bytes: &[u8]) -> Document { + let mut report = Report::default(); + let doc = read(bytes, &mut report).unwrap(); + assert!(!report.has_warnings(), "{report}"); + doc +} + +fn walk<'a>(o: &'a Obj, out: &mut Vec<&'a Obj>) { + out.push(o); + for (_, v) in &o.props { + match v { + Value::Strong(c) => walk(c, out), + Value::StrongVec(v) | Value::StrongSet(v, _) => v.iter().for_each(|c| walk(c, out)), + _ => {} + } + } +} + +fn keys(dict: &Obj, p: u16) -> HashSet> { + dict.objs(p).iter().filter_map(|d| d.data(pid::IDENTIFICATION).map(<[u8]>::to_vec)).collect() +} + +/// filmcraft#323: readers that load the meta-dictionary (pyaaf2) need its ClassDefinitions and +/// TypeDefinitions sets. They are empty: everything written uses the baseline classes and types +/// those readers already know. +#[test] +fn the_meta_dictionary_has_its_class_and_type_definition_sets() { + for small in [false, true] { + let bytes = written(&document(), small); + let cf = craft_cfb::CompoundFile::open(&bytes).unwrap(); + assert_eq!(cf.entries[cf.find("MetaDictionary-1").unwrap()].clsid, ids::META_DICTIONARY); + for name in ["ClassDefinitions-3", "TypeDefinitions-4"] { + // u32 count, u32 first free key, u32 last free key, u16 key pid, u8 key size + let index = cf.read_path(&format!("MetaDictionary-1/{name} index")).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(index.len(), 15, "{name}: an empty set"); + assert_eq!(index[0..4], 0u32.to_le_bytes(), "{name}: no entries"); + assert_eq!(u16::from_le_bytes([index[12], index[13]]), pid::META_IDENTIFICATION, "{name}: keyed by the definition's AUID"); + assert_eq!(index[14], 16, "{name}: 16-byte keys"); + } + let root = store::read(&bytes).unwrap(); + let meta = root.strong(pid::ROOT_META_DICTIONARY).unwrap(); + for set in [pid::META_CLASS_DEFINITIONS, pid::META_TYPE_DEFINITIONS] { + assert!(matches!(meta.get(set), Some(Value::StrongSet(items, k)) if items.is_empty() && *k == pid::META_IDENTIFICATION), "{set:#06x}"); + } + } +} + +#[test] +fn written_files_have_the_required_structure() { + for small in [false, true] { + let bytes = written(&document(), small); + assert_written_file(&bytes, small); + } +} + +/// The top-level composition is tagged `Usage_TopLevel`, which the SMPTE Labels register gives as +/// urn:smpte:ul:060e2b34.04010101.0d010102.01010700, stored as the AUID +/// {0d010102-0101-0700-060e-2b3404010101}. Readers that look for it (LibAAF) find no sequence +/// without it. +#[test] +fn the_top_level_composition_has_the_registered_usage_code() { + let bytes = written(&document(), false); + let root = store::read(&bytes).unwrap(); + let content = root.strong(pid::ROOT_HEADER).unwrap().strong(pid::CONTENT).unwrap(); + let comp = content.objs(pid::MOBS).iter().find(|m| m.class == cls::COMPOSITION_MOB).unwrap(); + let usage = [0x02, 0x01, 0x01, 0x0D, 0x01, 0x01, 0x00, 0x07, 0x06, 0x0E, 0x2B, 0x34, 0x04, 0x01, 0x01, 0x01]; + assert_eq!(comp.data(pid::USAGE_CODE), Some(&usage[..])); +} + +/// Files from FilmCraft 0.16 and earlier tag the top-level composition with the unregistered label +/// 060e2b34.04010105.0d011201.02030000. They still import: the composition no other one uses is +/// the edit. +#[test] +fn files_with_the_old_usage_code_still_import() { + let bytes = written(&document(), false); + let mut root = store::read(&bytes).unwrap(); + let content = strong_mut(strong_mut(&mut root, pid::ROOT_HEADER), pid::CONTENT); + for mob in objs_mut(content, pid::MOBS).iter_mut().filter(|m| m.class == cls::COMPOSITION_MOB) { + mob.set(pid::USAGE_CODE, Value::Data(def::USAGE_TOP_LEVEL_OLD.to_vec())); + } + let old_file = store::write(&root, craft_cfb::Version::V4).unwrap(); + let got = read_clean(&old_file); + assert_eq!(got.compositions.len(), 1); + assert_eq!(got, read_clean(&bytes)); +} + +/// A file that holds one edit tagged with each label (a tool merged a composition from an older +/// file into a newer one) imports both edits. +#[test] +fn edits_tagged_with_either_usage_code_both_import() { + let bytes = written(&document(), false); + let mut root = store::read(&bytes).unwrap(); + let content = strong_mut(strong_mut(&mut root, pid::ROOT_HEADER), pid::CONTENT); + let mobs = objs_mut(content, pid::MOBS); + let mut old = mobs.iter().find(|m| m.class == cls::COMPOSITION_MOB).unwrap().clone(); + let mut id = old.mob_id(pid::MOB_ID).unwrap(); + id[31] ^= 0xFF; + old.set(pid::MOB_ID, Value::Data(id.to_vec())); + old.set(pid::MOB_NAME, Value::Data(store::utf16z("Old"))); + old.set(pid::USAGE_CODE, Value::Data(def::USAGE_TOP_LEVEL_OLD.to_vec())); + mobs.push(old); + let mixed = store::write(&root, craft_cfb::Version::V4).unwrap(); + let document = read_clean(&mixed); + let mut ids: Vec<_> = document.compositions.iter().map(|c| c.aaf_mob_id.as_deref().unwrap()).collect(); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|id| id.len() == 64 && id.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()))); + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), 2, "each composition keeps its own MobID"); + let mut names: Vec = document.compositions.into_iter().map(|c| c.name).collect(); + names.sort(); + assert_eq!(names, ["Old", "S"]); +} + +fn assert_written_file(bytes: &[u8], small: bool) { + assert_compound_file(bytes, small); + let root = store::read(bytes).unwrap(); + assert!(root.strong(pid::ROOT_META_DICTIONARY).is_some()); + let header = root.strong(pid::ROOT_HEADER).unwrap(); + let content = assert_header(header); + let dict = header.strong(pid::DICTIONARY).unwrap(); + let definitions = Definitions { + data: keys(dict, pid::DATA_DEFINITIONS), + operations: keys(dict, pid::OPERATION_DEFINITIONS), + interpolation: keys(dict, pid::INTERPOLATION_DEFINITIONS), + containers: keys(dict, pid::CONTAINER_DEFINITIONS), + parameters: keys(dict, pid::PARAMETER_DEFINITIONS), + }; + let mobs = content.objs(pid::MOBS); + let comp = assert_mob_set(content, mobs); + for mob in mobs { + assert_mob(mob, mobs, &definitions); + } + assert_composition(comp); +} + +fn assert_compound_file(bytes: &[u8], small: bool) { + let cf = craft_cfb::CompoundFile::open(bytes).unwrap(); + assert_eq!(cf.version, if small { 3 } else { 4 }); + assert_eq!(cf.root().clsid, ids::ROOT); + let references = cf.read_path("referenced properties").unwrap(); + let pids: Vec = references[7..].as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes([c[0], c[1]])).collect(); + assert!(pids.windows(3).any(|w| w == path::DATA_DEFS)); + let header = cf.find("Header-2").unwrap(); + assert_eq!(cf.entries[header].clsid, cls::HEADER); + assert!(cf.read_path("Header-2/properties").unwrap().starts_with(&[0x4C, 0x20])); +} + +fn assert_header(header: &Obj) -> &Obj { + for req in [pid::BYTE_ORDER, pid::LAST_MODIFIED, pid::CONTENT, pid::DICTIONARY, pid::VERSION, pid::IDENTIFICATION_LIST, pid::OBJECT_MODEL_VERSION] + { + assert!(header.get(req).is_some(), "Header property {req:#06x}"); + } + assert_eq!(header.auid(pid::OPERATIONAL_PATTERN), Some(def::OP_EDIT_PROTOCOL)); + let ident = &header.objs(pid::IDENTIFICATION_LIST)[0]; + for req in [pid::COMPANY_NAME, pid::PRODUCT_NAME, pid::PRODUCT_VERSION_STRING, pid::PRODUCT_ID, pid::DATE, pid::GENERATION_AUID] { + assert!(ident.get(req).is_some(), "Identification property {req:#06x}"); + } + header.strong(pid::CONTENT).unwrap() +} + +struct Definitions { + data: HashSet>, + operations: HashSet>, + interpolation: HashSet>, + containers: HashSet>, + parameters: HashSet>, +} + +fn assert_mob_set<'a>(content: &Obj, mobs: &'a [Obj]) -> &'a Obj { + let mob_ids: Vec<[u8; 32]> = mobs.iter().map(|m| m.mob_id(pid::MOB_ID).unwrap()).collect(); + assert_eq!(mob_ids.iter().collect::>().len(), mob_ids.len(), "unique mob ids"); + assert!(matches!(content.get(pid::MOBS), Some(Value::StrongSet(_, k)) if *k == pid::MOB_ID)); + let compositions: Vec<&Obj> = mobs.iter().filter(|m| m.class == cls::COMPOSITION_MOB).collect(); + assert_eq!(compositions.len(), 1); + assert_eq!(compositions[0].auid(pid::USAGE_CODE), Some(def::USAGE_TOP_LEVEL)); + assert_eq!(mobs.iter().filter(|m| m.class == cls::MASTER_MOB).count(), 1); + assert_eq!(mobs.iter().filter(|m| m.class == cls::SOURCE_MOB).count(), 3, "picture file, sound file, tape"); + compositions[0] +} + +fn assert_mob(mob: &Obj, mobs: &[Obj], definitions: &Definitions) { + for req in [pid::MOB_ID, pid::MOB_NAME, pid::SLOTS, pid::MOB_LAST_MODIFIED, pid::MOB_CREATION_TIME] { + assert!(mob.get(req).is_some(), "Mob property {req:#06x}"); + } + assert_source_descriptor(mob, &definitions.containers); + for slot in mob.objs(pid::SLOTS) { + assert!(slot.get(pid::SLOT_ID).is_some() && slot.get(pid::SEGMENT).is_some()); + if slot.class == cls::TIMELINE_MOB_SLOT { + assert!(slot.rational(pid::EDIT_RATE).is_some_and(|r| r.0 > 0 && r.1 > 0)); + assert!(slot.get(pid::ORIGIN).is_some()); + } + } + let mut all = Vec::new(); + walk(mob, &mut all); + for component in all { + assert_component(component, mobs, definitions); + } +} + +fn assert_source_descriptor(mob: &Obj, containers: &HashSet>) { + if mob.class != cls::SOURCE_MOB { + return; + } + let descriptor = mob.strong(pid::ESSENCE_DESCRIPTION).expect("SourceMob::EssenceDescription"); + if descriptor.class != cls::TAPE_DESCRIPTOR { + for req in [pid::SAMPLE_RATE, pid::FILE_LENGTH, pid::CONTAINER_FORMAT] { + assert!(descriptor.get(req).is_some(), "FileDescriptor property {req:#06x}"); + } + assert!(containers.contains(descriptor.weak_key(pid::CONTAINER_FORMAT).unwrap())); + let url = descriptor.objs(pid::LOCATOR)[0].string(pid::URL_STRING).unwrap(); + assert_eq!(url, "file:///m/a.mov"); + } + if descriptor.class == cls::PCM_DESCRIPTOR { + assert_eq!(descriptor.u32(pid::CHANNELS), Some(2)); + assert_eq!(descriptor.u32(pid::QUANTIZATION_BITS), Some(24)); + assert_eq!(descriptor.u16(pid::BLOCK_ALIGN), Some(6)); + } +} + +fn assert_component(o: &Obj, mobs: &[Obj], definitions: &Definitions) { + if is_component(o) { + let dd = o.weak_key(pid::DATA_DEFINITION).expect("Component::DataDefinition"); + assert!(definitions.data.contains(dd), "data definition in the dictionary"); + let event_seq = o.class == cls::SEQUENCE && dd == ddef::DESCRIPTIVE_METADATA; + assert!(event_seq || o.get(pid::LENGTH).is_some(), "Component::Length"); + } + if o.class == cls::SOURCE_CLIP { + assert_source_clip(o, mobs); + } else if o.class == cls::OPERATION_GROUP { + assert_operation_group(o, definitions); + } else if o.class == cls::SEQUENCE && o.get(pid::LENGTH).is_some() { + assert_sequence_length(o); + } +} + +fn is_component(o: &Obj) -> bool { + [cls::FILLER, cls::SOURCE_CLIP, cls::SEQUENCE, cls::TRANSITION, cls::OPERATION_GROUP, cls::TIMECODE, cls::COMMENT_MARKER].contains(&o.class) +} + +fn assert_source_clip(clip: &Obj, mobs: &[Obj]) { + let id = clip.mob_id(pid::SOURCE_ID).unwrap(); + let slot = clip.u32(pid::SOURCE_MOB_SLOT_ID).unwrap(); + assert!(clip.get(pid::START_TIME).is_some()); + if id != [0; 32] { + let target = mobs.iter().find(|m| m.mob_id(pid::MOB_ID) == Some(id)).expect("source clip target in the file"); + assert!(target.objs(pid::SLOTS).iter().any(|s| s.u32(pid::SLOT_ID) == Some(slot)), "target slot exists"); + } +} + +fn assert_operation_group(group: &Obj, definitions: &Definitions) { + assert!(definitions.operations.contains(group.weak_key(pid::OPERATION).unwrap())); + for parameter in group.objs(pid::PARAMETERS) { + assert!(definitions.parameters.contains(parameter.auid(pid::PARAMETER_DEFINITION).unwrap().as_slice())); + if parameter.class == cls::VARYING_VALUE { + assert!(definitions.interpolation.contains(parameter.weak_key(pid::INTERPOLATION).unwrap())); + assert!(!parameter.objs(pid::POINT_LIST).is_empty()); + } + } +} + +fn assert_sequence_length(sequence: &Obj) { + let components = sequence.objs(pid::COMPONENTS); + let sum = components + .iter() + .enumerate() + .map(|(i, component)| { + let len = component.i64(pid::LENGTH).unwrap(); + if component.class == cls::TRANSITION { + assert!(i > 0 && i + 1 < components.len(), "transition at a sequence end"); + assert!(components[i - 1].class != cls::TRANSITION && components[i + 1].class != cls::TRANSITION); + assert!(components[i - 1].i64(pid::LENGTH).unwrap() >= len && components[i + 1].i64(pid::LENGTH).unwrap() >= len); + let cut = component.i64(pid::CUT_POINT).unwrap(); + assert!((0..=len).contains(&cut)); + assert!(component.strong(pid::OPERATION_GROUP).is_some()); + -len + } else { + len + } + }) + .sum::(); + assert_eq!(sum, sequence.i64(pid::LENGTH).unwrap(), "sequence length = segments − transitions"); +} + +fn assert_composition(comp: &Obj) { + let slots = comp.objs(pid::SLOTS); + let tc = slots[0].strong(pid::SEGMENT).unwrap(); + assert_eq!(tc.class, cls::TIMECODE); + assert_eq!(tc.u16(pid::TC_FPS), Some(24)); + let video = slots[1].strong(pid::SEGMENT).unwrap().objs(pid::COMPONENTS); + assert_eq!(video.iter().map(|c| c.class).collect::>(), vec![cls::SOURCE_CLIP, cls::TRANSITION, cls::SOURCE_CLIP]); + assert_eq!(video[0].i64(pid::LENGTH), Some(54), "outgoing clip extends to the transition end"); + assert_eq!(video[2].i64(pid::LENGTH), Some(54), "incoming clip starts at the transition start"); + assert_eq!(video[2].i64(pid::START_TIME), Some(194)); + assert_eq!(video[1].i64(pid::CUT_POINT), Some(6)); + assert_eq!(video[1].strong(pid::OPERATION_GROUP).unwrap().weak_key(pid::OPERATION), Some(&def::VIDEO_DISSOLVE[..])); + assert_audio_composition_slot(&slots[2]); +} + +fn assert_audio_composition_slot(slot: &Obj) { + let audio = slot.strong(pid::SEGMENT).unwrap().objs(pid::COMPONENTS); + assert_eq!(slot.rational(pid::EDIT_RATE), Some((48_000, 1))); + assert_eq!(audio[2].class, cls::OPERATION_GROUP); + assert_eq!(audio[2].weak_key(pid::OPERATION), Some(&def::MONO_AUDIO_GAIN[..])); + let amplitude = audio[2].objs(pid::PARAMETERS)[0].data(pid::CONSTANT_VALUE).unwrap(); + let (numerator, denominator) = store::rational_of(&litude[17..]).unwrap(); + assert!((numerator as f64 / denominator as f64 - 10f64.powf(-3.0 / 20.0)).abs() < 1e-4); + assert_eq!(audio[1].strong(pid::OPERATION_GROUP).unwrap().weak_key(pid::OPERATION), Some(&def::MONO_AUDIO_DISSOLVE[..])); +} + +#[test] +fn store_round_trips_every_stored_form() { + let leaf = |n: u8| Obj::new([n; 16]).data_prop(0x1B01, vec![n; 16]); + let root = Obj::new(ids::ROOT) + .data_prop(0x0003, vec![1, 2, 3]) + .with(0x0004, Value::Strong(Box::new(leaf(1).with(0x0005, Value::StrongVec(vec![leaf(2), leaf(3)]))))) + .with( + 0x0002, + Value::Strong(Box::new( + Obj::new([9; 16]) + .with(0x3B04, Value::Strong(Box::new(Obj::new([8; 16]).with(0x2605, Value::StrongSet(vec![leaf(4), leaf(5)], 0x1B01))))), + )), + ) + .with(0x0006, Value::Weak(path::DATA_DEFS.to_vec(), 0x1B01, vec![5; 16])) + .with(0x0007, Value::WeakVec(path::DATA_DEFS.to_vec(), 0x1B01, vec![vec![4; 16], vec![5; 16]])) + .with(0x0008, Value::Stream((0..10_000u32).map(|i| i as u8).collect())); + for v in [craft_cfb::Version::V3, craft_cfb::Version::V4] { + let bytes = store::write(&root, v).unwrap(); + assert_eq!(store::read(&bytes).unwrap(), root); + } + // a data value longer than a property can hold is an error, not a panic + assert!(store::write(&Obj::new(ids::ROOT).data_prop(1, vec![0; 70_000]), craft_cfb::Version::V4).is_err()); +} + +/// The property `p` of `o` as a list of objects, to change. +fn objs_mut(o: &mut Obj, p: u16) -> &mut Vec { + match o.props.iter_mut().find(|(k, _)| *k == p).map(|(_, v)| v) { + Some(Value::StrongVec(v)) | Some(Value::StrongSet(v, _)) => v, + _ => panic!("no object list {p:#06x}"), + } +} + +fn strong_mut(o: &mut Obj, p: u16) -> &mut Obj { + match o.props.iter_mut().find(|(k, _)| *k == p).map(|(_, v)| v) { + Some(Value::Strong(v)) => v, + _ => panic!("no object {p:#06x}"), + } +} + +fn is_picture(o: &Obj) -> bool { + o.weak_key(pid::DATA_DEFINITION) == Some(&ddef::PICTURE[..]) +} + +/// How Premiere Pro 26.5.2 lays a sequence with a nested sequence out in AAF (seen in a file it +/// exported): every composition is tagged top-level, the nested one too; all video tracks of a +/// composition are in ONE slot, a nested scope with a segment per track, lowest first; a clip of +/// the nested sequence is a source clip that points at its composition. Our own file is rewritten +/// into that layout and must read back as the same edit. +#[test] +fn premiere_style_layers_and_nested_compositions_are_read() { + let r = FrameRate::FPS_24; + // "Outer": media on V1 and V3, the sequence "S" nested on V2 with its sound on A1 + let inner = document(); + let mut sources = inner.sources.clone(); + for kind in [CKind::Picture, CKind::Sound] { + let mut nest = source("seq:2", "S", kind, r, 16, None); + (nest.path, nest.nested, nest.length, nest.width, nest.height) = (None, Some(0), r.tick_of(96), 1920, 1080); + sources.push(nest); + } + let gap = |frames: i64| CItem::Filler(r.tick_of(frames)); + let outer = composition( + "Outer", + r, + vec![ + track(CKind::Picture, "V1", 1, vec![clip(r, 0, 5, 30, "a")]), + track(CKind::Picture, "V2", 2, vec![gap(10), clip(r, 2, 12, 40, "S")]), + track(CKind::Picture, "V3", 3, vec![gap(60), clip(r, 0, 100, 20, "a")]), + track(CKind::Sound, "A1", 1, vec![gap(10), clip(r, 3, 12, 40, "S")]), + ], + ); + let ours = written(&Document { compositions: vec![outer], sources, nested: inner.compositions }, false); + + let mut root = store::read(&ours).unwrap(); + let content = strong_mut(strong_mut(&mut root, pid::ROOT_HEADER), pid::CONTENT); + let mut compositions = 0; + for mob in objs_mut(content, pid::MOBS).iter_mut().filter(|m| m.class == cls::COMPOSITION_MOB) { + compositions += 1; + mob.set(pid::USAGE_CODE, Value::Data(def::USAGE_TOP_LEVEL.to_vec())); + let slots = objs_mut(mob, pid::SLOTS); + let picture: Vec = slots.iter().enumerate().filter(|(_, s)| s.strong(pid::SEGMENT).is_some_and(is_picture)).map(|(i, _)| i).collect(); + let layers: Vec = picture.iter().map(|&i| slots[i].strong(pid::SEGMENT).unwrap().clone()).collect(); + let length = layers.iter().filter_map(|l| l.i64(pid::LENGTH)).max().unwrap_or(0); + let scope = Obj::new(cls::NESTED_SCOPE) + .with(pid::DATA_DEFINITION, Value::Weak(path::DATA_DEFS.to_vec(), pid::IDENTIFICATION, ddef::PICTURE.to_vec())) + .data_prop(pid::LENGTH, length.to_le_bytes().to_vec()) + .with(pid::NESTED_SLOTS, Value::StrongVec(layers)); + // the first video slot keeps its id (clips of the nested sequence point at it) and holds + // every layer; the other video slots go + slots[picture[0]].set(pid::SEGMENT, Value::Strong(Box::new(scope))); + for &i in picture[1..].iter().rev() { + slots.remove(i); + } + } + assert_eq!(compositions, 2); + let theirs = store::write(&root, craft_cfb::Version::V4).unwrap(); + assert_ne!(theirs, ours); + + let want = read_clean(&ours); + let got = read_clean(&theirs); + // one edit; the nested composition is one it uses, not a second edit + assert_eq!(got.compositions.len(), 1); + assert_eq!(got.nested.len(), 1); + // layers of one slot share its name and are all as long as the scope: the same edit apart + // from picture track names and gaps after the last clip + let edit = |mut doc: Document| { + for t in doc.compositions.iter_mut().chain(&mut doc.nested).flat_map(|c| &mut c.tracks).filter(|t| t.kind == CKind::Picture) { + t.name.clear(); + while matches!(t.items.last(), Some(CItem::Filler(_))) { + t.items.pop(); + } + } + doc + }; + assert_eq!(edit(got.clone()), edit(want)); + // and that edit is the one that was written: three video tracks, the nest on the second + let q = &got.compositions[0]; + let picture: Vec<_> = q.tracks.iter().filter(|t| t.kind == CKind::Picture).collect(); + assert_eq!(picture.iter().map(|t| t.items.iter().filter(|i| matches!(i, CItem::Clip(_))).count()).collect::>(), [1, 1, 1]); + let nest_of = |kind: CKind| { + let t = q.tracks.iter().filter(|t| t.kind == kind).nth(if kind == CKind::Picture { 1 } else { 0 }).unwrap(); + let [CItem::Filler(gap), CItem::Clip(c), ..] = &t.items[..] else { panic!("{:?}", t.items) }; + (*gap, c.len, c.start, got.sources[c.source].nested, got.sources[c.source].name.clone()) + }; + assert_eq!(nest_of(CKind::Picture), (r.tick_of(10), r.tick_of(40), r.tick_of(12), Some(0), "S".to_string())); + assert_eq!(nest_of(CKind::Sound), nest_of(CKind::Picture)); + assert_eq!(got.nested[0].name, "S"); +} + +/// A slot's name is the track's name, whatever it starts with; a track without one reads back +/// without one. +#[test] +fn audio_tracks_keep_their_names() { + for name in ["Dialogue.Dialogue-1", "Dialogue.Dialogue-1 L", "", "é", "音声", "🎤 Dialogue", "Ä1"] { + let mut doc = document(); + doc.compositions[0].tracks[1].name = name.into(); + let back = read_clean(&written(&doc, false)); + assert_eq!(back.compositions[0].tracks[1].name, name); + } +} + +#[test] +fn unnamed_source_clip_uses_the_referenced_master_name() { + let mut doc = crate::testdoc::sound_only(None); + let name = doc.sources[0].name.clone(); + for item in &mut doc.compositions[0].tracks[0].items { + if let CItem::Clip(clip) = item { + clip.name = name.clone(); + } + } + let mut root = store::read(&written(&doc, false)).unwrap(); + let content = strong_mut(strong_mut(&mut root, pid::ROOT_HEADER), pid::CONTENT); + let master = objs_mut(content, pid::MOBS).iter_mut().find(|m| m.class == cls::MASTER_MOB).unwrap(); + master.set(pid::MOB_NAME, Value::Data(store::utf16z("Master edit"))); + let bytes = store::write(&root, craft_cfb::Version::V4).unwrap(); + let back = read_clean(&bytes); + let clip = back.compositions[0].tracks[0] + .items + .iter() + .find_map(|item| match item { + CItem::Clip(c) => Some(c), + _ => None, + }) + .unwrap(); + assert_eq!(clip.name, "Master edit"); +} + +#[test] +fn sound_source_offsets_use_audio_sample_rate_on_disk() { + let doc = crate::testdoc::sound_only(None); + let bytes = written(&doc, false); + let root = store::read(&bytes).unwrap(); + let mut all = Vec::new(); + walk(&root, &mut all); + let starts: Vec = all.iter().filter(|o| o.class == cls::SOURCE_CLIP).filter_map(|o| o.i64(pid::START_TIME)).collect(); + assert!(starts.contains(&19_200), "expected 0.4 seconds at 48 kHz: {starts:?}"); + let back = read_clean(&bytes); + let clip = back.compositions[0].tracks[0] + .items + .iter() + .find_map(|i| match i { + CItem::Clip(c) => Some(c), + _ => None, + }) + .unwrap(); + assert_eq!(clip.start, craft_timeline::time::FrameRate::FPS_25.tick_of(10)); +} + +#[test] +fn flattened_selectors_and_nested_scopes_are_reported() { + for (class, property, message) in + [(cls::SELECTOR, pid::SELECTED, "selector was flattened"), (cls::NESTED_SCOPE, pid::NESTED_SLOTS, "nested scope was flattened")] + { + let mut root = store::read(&written(&document(), false)).unwrap(); + let content = strong_mut(strong_mut(&mut root, pid::ROOT_HEADER), pid::CONTENT); + let comp = objs_mut(content, pid::MOBS).iter_mut().find(|mob| mob.class == cls::COMPOSITION_MOB).unwrap(); + let slot = objs_mut(comp, pid::SLOTS) + .iter_mut() + .find(|slot| { + slot.strong(pid::SEGMENT).is_some_and(|s| s.class == cls::SEQUENCE && s.weak_key(pid::DATA_DEFINITION) == Some(&ddef::SOUND[..])) + }) + .unwrap(); + let segment = strong_mut(slot, pid::SEGMENT); + let child = segment.clone(); + let mut wrapper = Obj::new(class) + .with(pid::DATA_DEFINITION, child.get(pid::DATA_DEFINITION).unwrap().clone()) + .with(pid::LENGTH, child.get(pid::LENGTH).unwrap().clone()); + let value = if class == cls::SELECTOR { Value::Strong(Box::new(child)) } else { Value::StrongVec(vec![child]) }; + wrapper.set(property, value); + *segment = wrapper; + let bytes = store::write(&root, craft_cfb::Version::V4).unwrap(); + let mut report = Report::default(); + read(&bytes, &mut report).unwrap(); + assert!(report.mentions(message), "{report}"); + } +} + +#[test] +fn relative_media_locator_round_trips_escaped_characters() { + let mut doc = document(); + doc.sources[0].path = Some("Audio Files/100%20 #1.wav".into()); + let bytes = written(&doc, false); + let got = read_clean(&bytes); + assert!(got.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav"))); +} diff --git a/crates/aaf/src/aaf/write.rs b/crates/aaf/src/aaf/write.rs new file mode 100644 index 000000000..8c9aae88e --- /dev/null +++ b/crates/aaf/src/aaf/write.rs @@ -0,0 +1,804 @@ +//! Composition model → AAF objects. + +use std::collections::HashMap; + +use craft_timeline::time::{FrameRate, Tick}; + +use super::ids::{META_DICTIONARY, ROOT, cls, ddef, def, path, pid}; +use super::store::{self, Auid, Obj, Value, rational, utf16z}; +use crate::model::{CItem, CKind, CMarker, Composition, Document, Gain, Source, ticks_to_units}; +use crate::{Error, Identity, Report, Result}; + +/// UMID-style mob id (SMPTE 330 basic UMID): label, length 0x13, instance 0, material number. +pub(crate) fn mob_id(seed: u64, n: u64) -> [u8; 32] { + let mut id = [0u8; 32]; + id[..12].copy_from_slice(&[0x06, 0x0A, 0x2B, 0x34, 0x01, 0x01, 0x01, 0x05, 0x01, 0x01, 0x0F, 0x20]); + id[12] = 0x13; + let a = mix(seed ^ n.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + let b = mix(a ^ 0xD6E8_FEB8_6659_FD93); + id[16..24].copy_from_slice(&a.to_le_bytes()); + id[24..32].copy_from_slice(&b.to_le_bytes()); + // a "random" material number per RFC 4122 v4 layout keeps it a valid UUID + id[22] = (id[22] & 0x0F) | 0x40; + id[24] = (id[24] & 0x3F) | 0x80; + id +} + +fn mix(mut z: u64) -> u64 { + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +pub(crate) fn fnv(s: &[u8], h: u64) -> u64 { + s.iter().fold(h, |h, &b| (h ^ b as u64).wrapping_mul(0x0000_0100_0000_01B3)) +} + +fn timestamp() -> Vec { + // 1970-01-01 00:00:00.00: written files are reproducible + let mut v = 1970i16.to_le_bytes().to_vec(); + v.extend_from_slice(&[1, 1, 0, 0, 0, 0]); + v +} + +fn weak(path: [u16; 3], key: Auid) -> Value { + Value::Weak(path.to_vec(), pid::IDENTIFICATION, key.to_vec()) +} + +fn data_def(kind: CKind) -> Auid { + match kind { + CKind::Picture => ddef::PICTURE, + CKind::Sound => ddef::SOUND, + } +} + +/// An indirect value: byte order, type id, value. +pub(crate) fn indirect(ty: Auid, v: &[u8]) -> Vec { + let mut d = vec![0x4C]; + d.extend_from_slice(&ty); + d.extend_from_slice(v); + d +} + +fn tagged(name: &str, value: &str) -> Obj { + Obj::new(cls::TAGGED_VALUE).data_prop(pid::TAG_NAME, utf16z(name)).data_prop(pid::TAG_VALUE, indirect(def::TYPE_STRING, &utf16z(value))) +} + +fn gain_rational(g: f64) -> Vec { + rational((g.max(0.0) * 65536.0).round().min(i32::MAX as f64) as i64, 65536) +} + +fn definition(class: Auid, id: Auid, name: &str) -> Obj { + Obj::new(class).data_prop(pid::IDENTIFICATION, id.to_vec()).data_prop(pid::NAME, utf16z(name)).data_prop(pid::DESCRIPTION, utf16z("")) +} + +struct W<'a> { + report: &'a mut Report, + inexact: bool, +} + +struct IdGen { + seed: u64, + next: u64, +} + +impl IdGen { + fn for_document(doc: &Document, comp: &Composition) -> Self { + let mut seed = fnv(comp.name.as_bytes(), 0xCBF2_9CE4_8422_2325); + for source in &doc.sources { + seed = fnv(source.path.as_deref().unwrap_or("").as_bytes(), fnv(source.key.as_bytes(), seed)); + } + seed = fnv(&(comp.tracks.len() as u64).to_le_bytes(), seed); + Self { seed, next: 0 } + } + + fn next(&mut self) -> [u8; 32] { + self.next += 1; + mob_id(self.seed, self.next) + } +} + +struct MediaOutput { + mobs: Vec, + essence: Vec, + master_ref: HashMap, +} + +type FileMob = ((Option, usize, CKind), [u8; 32], Vec, usize); + +struct GroupSlots { + master_id: [u8; 32], + tape_id: [u8; 32], + timecode: Option<(i64, FrameRate)>, + master_slots: Vec, + tape_slots: Vec, + files: Vec, + master_ref: HashMap, +} + +struct GroupSlotBuilder<'a, 'b> { + doc: &'a Document, + master_id: [u8; 32], + tape_id: [u8; 32], + ids: &'b mut IdGen, + files: &'b mut Vec, + master_slots: &'b mut Vec, + tape_slots: &'b mut Vec, + master_ref: &'b mut HashMap, +} + +struct FileSlotInput<'a> { + source: &'a Source, + file_slot: u32, + length: i64, + has_timecode: bool, + tape_id: [u8; 32], + member_index: usize, + slots: &'a mut Vec, + rate: (i64, i64), +} + +impl W<'_> { + /// Edit units at `rate` of tick `t`. + fn u(&mut self, t: Tick, rate: (i64, i64)) -> i64 { + let (n, exact) = ticks_to_units(t, rate.0, rate.1); + if !exact { + self.inexact = true; + } + n + } +} + +fn source_rate(s: &Source) -> (i64, i64) { + match s.kind { + CKind::Picture => (s.frame_rate.num, s.frame_rate.den), + CKind::Sound => (s.sample_rate.max(1) as i64, 1), + } +} + +fn write_media(doc: &Document, w: &mut W<'_>, ids: &mut IdGen) -> MediaOutput { + let mut groups: Vec<(String, Vec)> = Vec::new(); + for (index, source) in doc.sources.iter().enumerate() { + if source.nested.is_some() { + continue; + } + match groups.iter_mut().find(|(key, _)| *key == source.key) { + Some((_, members)) => members.push(index), + None => groups.push((source.key.clone(), vec![index])), + } + } + + let mut output = MediaOutput { mobs: Vec::new(), essence: Vec::new(), master_ref: HashMap::new() }; + for (_, members) in groups { + write_source_group(doc, &members, w, ids, &mut output); + } + output +} + +fn write_source_group(doc: &Document, members: &[usize], w: &mut W<'_>, ids: &mut IdGen, output: &mut MediaOutput) { + let first = &doc.sources[members[0]]; + let slots = build_group_slots(doc, members, w, ids); + output.master_ref.extend(slots.master_ref); + let mut master_slots = slots.master_slots; + add_master_markers(doc, members, w, &mut master_slots); + output.mobs.push( + Obj::new(cls::MASTER_MOB) + .data_prop(pid::MOB_ID, slots.master_id.to_vec()) + .data_prop(pid::MOB_NAME, utf16z(&first.name)) + .data_prop(pid::MOB_LAST_MODIFIED, timestamp()) + .data_prop(pid::MOB_CREATION_TIME, timestamp()) + .with(pid::SLOTS, Value::StrongVec(master_slots)), + ); + let tape_id = slots.tape_id; + let timecode = slots.timecode; + let tape_slots = slots.tape_slots; + add_file_mobs(doc, slots.files, w, output); + add_tape_mob(first, tape_id, timecode, tape_slots, output); +} + +fn build_group_slots(doc: &Document, members: &[usize], w: &mut W<'_>, ids: &mut IdGen) -> GroupSlots { + let master_id = ids.next(); + let tape_id = ids.next(); + let timecode = members.iter().find_map(|&i| doc.sources[i].start_tc.map(|t| (t, doc.sources[i].tc_rate))); + let mut tape_slots = Vec::new(); + if let Some((start, rate)) = timecode { + tape_slots.push(timecode_slot(members, doc, w, start, rate)); + } + let mut files: Vec = Vec::new(); + let mut master_slots = Vec::new(); + let mut master_ref = HashMap::new(); + let mut builder = GroupSlotBuilder { + doc, + master_id, + tape_id, + ids, + files: &mut files, + master_slots: &mut master_slots, + tape_slots: &mut tape_slots, + master_ref: &mut master_ref, + }; + for (member_index, &source_index) in members.iter().enumerate() { + builder.add_source(source_index, member_index, timecode.is_some(), w); + } + GroupSlots { master_id, tape_id, timecode, master_slots, tape_slots, files, master_ref } +} + +fn timecode_slot(members: &[usize], doc: &Document, w: &mut W<'_>, start: i64, rate: FrameRate) -> Obj { + let edit_rate = (rate.num, rate.den); + let len = members.iter().map(|&i| doc.sources[i].length).max().unwrap_or(Tick::ZERO); + let units = w.u(len, edit_rate); + timeline_slot( + 1, + "TC1", + 0, + edit_rate, + Obj::new(cls::TIMECODE) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, ddef::TIMECODE)) + .data_prop(pid::LENGTH, units.to_le_bytes().to_vec()) + .data_prop(pid::TC_START, start.to_le_bytes().to_vec()) + .data_prop(pid::TC_FPS, (rate.timecode_base().clamp(1, u16::MAX as i64) as u16).to_le_bytes().to_vec()) + .data_prop(pid::TC_DROP, vec![0]), + ) +} + +impl GroupSlotBuilder<'_, '_> { + fn add_source(&mut self, source_index: usize, member_index: usize, has_timecode: bool, w: &mut W<'_>) { + let source = &self.doc.sources[source_index]; + let rate = source_rate(source); + let embedded_key = if source.embedded.is_some() { source_index } else { usize::MAX }; + let file_key = (source.path.clone(), embedded_key, source.kind); + let file_slot = source.channel.map_or(1, |channel| channel + 1); + let len_units = w.u(source.length, rate); + let file_index = match self.files.iter().position(|file| file.0 == file_key) { + Some(index) => index, + None => { + self.files.push((file_key, self.ids.next(), Vec::new(), source_index)); + self.files.len() - 1 + } + }; + let file_id = self.files[file_index].1; + add_file_slot(FileSlotInput { + source, + file_slot, + length: len_units, + has_timecode, + tape_id: self.tape_id, + member_index, + slots: &mut self.files[file_index].2, + rate, + }); + if has_timecode { + let tape_slot_id = 2 + member_index as u32; + self.tape_slots.push(timeline_slot( + tape_slot_id, + &slot_name(source.kind, member_index as u32 + 1), + member_index as u32 + 1, + rate, + source_clip(source.kind, len_units, [0; 32], 0, 0), + )); + } + let master_slot = member_index as u32 + 1; + self.master_slots.push(timeline_slot( + master_slot, + &slot_name(source.kind, master_slot), + master_slot, + rate, + source_clip(source.kind, len_units, file_id, file_slot, 0), + )); + self.master_ref.insert(source_index, (self.master_id, master_slot)); + } +} + +fn add_file_slot(input: FileSlotInput<'_>) { + let FileSlotInput { source, file_slot, length, has_timecode, tape_id, member_index, slots, rate } = input; + if slots.iter().any(|slot| slot.u32(pid::SLOT_ID) == Some(file_slot)) { + return; + } + let tape_slot_id = if has_timecode { 2 + member_index as u32 } else { 0 }; + let mob_id = if has_timecode { tape_id } else { [0; 32] }; + let clip = source_clip(source.kind, length, mob_id, tape_slot_id, 0); + slots.push(timeline_slot(file_slot, &slot_name(source.kind, file_slot), file_slot, rate, clip)); +} + +fn add_master_markers(doc: &Document, members: &[usize], w: &mut W<'_>, master_slots: &mut Vec) { + let first = &doc.sources[members[0]]; + let marker_rate = + members.iter().map(|&i| &doc.sources[i]).find(|source| source.kind == CKind::Picture).map(source_rate).unwrap_or_else(|| source_rate(first)); + let markers: Vec<&CMarker> = members.iter().flat_map(|&i| doc.sources[i].markers.iter()).collect(); + if markers.is_empty() { + return; + } + let mut unique: Vec = markers.into_iter().cloned().collect(); + unique.sort_by(|a, b| (a.start, &a.name).cmp(&(b.start, &b.name))); + unique.dedup(); + master_slots.push(event_slot(w, members.len() as u32 + 1, marker_rate, &unique)); +} + +fn add_file_mobs(doc: &Document, files: Vec, w: &mut W<'_>, output: &mut MediaOutput) { + for (_, file_id, slots, source_index) in files { + let source = &doc.sources[source_index]; + let desc = descriptor(w, source); + output.mobs.push( + Obj::new(cls::SOURCE_MOB) + .data_prop(pid::MOB_ID, file_id.to_vec()) + .data_prop(pid::MOB_NAME, utf16z(&source.name)) + .data_prop(pid::MOB_LAST_MODIFIED, timestamp()) + .data_prop(pid::MOB_CREATION_TIME, timestamp()) + .with(pid::SLOTS, Value::StrongVec(slots)) + .with(pid::ESSENCE_DESCRIPTION, Value::Strong(Box::new(desc))), + ); + if let Some(data) = &source.embedded { + output.essence.push( + Obj::new(cls::ESSENCE_DATA).data_prop(pid::ESSENCE_MOB_ID, file_id.to_vec()).with(pid::ESSENCE_STREAM, Value::Stream(data.clone())), + ); + } + } +} + +fn add_tape_mob(first: &Source, tape_id: [u8; 32], timecode: Option<(i64, FrameRate)>, tape_slots: Vec, output: &mut MediaOutput) { + let Some(_) = timecode else { return }; + let reel = first.path.as_deref().map(craft_timeline::path::file_stem).unwrap_or(&first.name).to_string(); + output.mobs.push( + Obj::new(cls::SOURCE_MOB) + .data_prop(pid::MOB_ID, tape_id.to_vec()) + .data_prop(pid::MOB_NAME, utf16z(&reel)) + .data_prop(pid::MOB_LAST_MODIFIED, timestamp()) + .data_prop(pid::MOB_CREATION_TIME, timestamp()) + .with(pid::SLOTS, Value::StrongVec(tape_slots)) + .with(pid::ESSENCE_DESCRIPTION, Value::Strong(Box::new(Obj::new(cls::TAPE_DESCRIPTOR)))), + ); +} + +/// Serialise a composition document. +pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Identity<'_>, report: &mut Report) -> Result> { + let comp = doc.compositions.first().ok_or(Error::Empty)?; + let mut w = W { report, inexact: false }; + let mut ids = IdGen::for_document(doc, comp); + let MediaOutput { mut mobs, essence, master_ref } = write_media(doc, &mut w, &mut ids); + let mut used_ops: Vec = Vec::new(); + let mut uses_varying = false; + + // ---- nested compositions: a clip refers to one as it does to a master mob, through the + // composition's first track of the clip's kind (a broken-out channel: that channel's track) + let mut refs = Refs { mobs: master_ref, rates: doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() }; + let nested_ids: Vec<[u8; 32]> = doc.nested.iter().map(|_| ids.next()).collect(); + for (si, s) in doc.sources.iter().enumerate() { + let Some((comp, id)) = s.nested.and_then(|n| Some((doc.nested.get(n)?, nested_ids.get(n)?))) else { continue }; + let of_kind: Vec = comp.tracks.iter().enumerate().filter(|(_, t)| t.kind == s.kind).map(|(i, _)| i).collect(); + let Some(&track) = s.channel.and_then(|c| of_kind.get(c as usize)).or(of_kind.first()) else { continue }; + let rate = match s.kind { + CKind::Picture => (comp.rate.num, comp.rate.den), + CKind::Sound => (comp.sample_rate.max(1) as i64, 1), + }; + refs.mobs.insert(si, (*id, track as u32 + 2)); + refs.rates.insert(si, rate); + } + + // ---- the compositions: the exported sequence first + let comp_id = ids.next(); + let composition_mobs = { + let mut composition_writer = CompositionWriter { w: &mut w, doc, refs: &refs, used_ops: &mut used_ops, uses_varying: &mut uses_varying }; + let mut output = vec![composition_writer.write(comp, comp_id, true)]; + output.extend(doc.nested.iter().zip(&nested_ids).map(|(nested, id)| composition_writer.write(nested, *id, false))); + output + }; + mobs.splice(0..0, composition_mobs); + if w.inexact { + w.report.info("some times are not on whole edit units and were rounded"); + } + + // ---- dictionary + let mut ddefs = Vec::new(); + for (id, name) in + [(ddef::PICTURE, "Picture"), (ddef::SOUND, "Sound"), (ddef::TIMECODE, "Timecode"), (ddef::DESCRIPTIVE_METADATA, "DescriptiveMetadata")] + { + ddefs.push(definition(cls::DATA_DEFINITION, id, name)); + } + let mut opdefs = Vec::new(); + for op in &used_ops { + let (name, dd, inputs) = match *op { + x if x == def::VIDEO_DISSOLVE => ("Video Dissolve", ddef::PICTURE, 2), + x if x == def::VIDEO_FADE_TO_BLACK => ("Video Fade To Black", ddef::PICTURE, 1), + x if x == def::SMPTE_VIDEO_WIPE => ("SMPTE Video Wipe", ddef::PICTURE, 2), + x if x == def::MONO_AUDIO_DISSOLVE => ("Mono Audio Dissolve", ddef::SOUND, 2), + _ => ("Mono Audio Gain", ddef::SOUND, 1i32), + }; + let params: Vec> = if *op == def::MONO_AUDIO_GAIN { + vec![def::PARAM_AMPLITUDE.to_vec()] + } else if *op == def::VIDEO_DISSOLVE { + vec![def::PARAM_LEVEL.to_vec()] + } else { + Vec::new() + }; + let mut o = definition(cls::OPERATION_DEFINITION, *op, name) + .with(pid::OPDEF_DATA_DEFINITION, weak(path::DATA_DEFS, dd)) + .data_prop(pid::OPDEF_IS_TIME_WARP, vec![0]) + .data_prop(pid::OPDEF_NUMBER_INPUTS, inputs.to_le_bytes().to_vec()); + if !params.is_empty() { + o.set(pid::OPDEF_PARAMETERS_DEFINED, Value::WeakVec(path::PARAMETER_DEFS.to_vec(), pid::IDENTIFICATION, params)); + } + opdefs.push(o); + } + let pdefs = vec![ + definition(cls::PARAMETER_DEFINITION, def::PARAM_LEVEL, "Level").data_prop(pid::PARAMDEF_DISPLAY_UNITS, utf16z("")), + definition(cls::PARAMETER_DEFINITION, def::PARAM_AMPLITUDE, "Amplitude").data_prop(pid::PARAMDEF_DISPLAY_UNITS, utf16z("")), + ]; + let idefs = if uses_varying || !used_ops.is_empty() { + vec![ + definition(cls::INTERPOLATION_DEFINITION, def::INTERP_LINEAR, "Linear"), + definition(cls::INTERPOLATION_DEFINITION, def::INTERP_CONSTANT, "Constant"), + ] + } else { + Vec::new() + }; + let cdefs = vec![ + definition(cls::CONTAINER_DEFINITION, def::CONTAINER_EXTERNAL, "External Container"), + definition(cls::CONTAINER_DEFINITION, def::CONTAINER_AAF, "AAF Container"), + ]; + let mut dict = Obj::new(cls::DICTIONARY) + .with(pid::DATA_DEFINITIONS, Value::StrongSet(ddefs, pid::IDENTIFICATION)) + .with(pid::PARAMETER_DEFINITIONS, Value::StrongSet(pdefs, pid::IDENTIFICATION)) + .with(pid::CONTAINER_DEFINITIONS, Value::StrongSet(cdefs, pid::IDENTIFICATION)); + if !opdefs.is_empty() { + dict.set(pid::OPERATION_DEFINITIONS, Value::StrongSet(opdefs, pid::IDENTIFICATION)); + } + if !idefs.is_empty() { + dict.set(pid::INTERPOLATION_DEFINITIONS, Value::StrongSet(idefs, pid::IDENTIFICATION)); + } + + let mut content = Obj::new(cls::CONTENT_STORAGE).with(pid::MOBS, Value::StrongSet(mobs, pid::MOB_ID)); + if !essence.is_empty() { + content.set(pid::ESSENCE_DATA, Value::StrongSet(essence, pid::ESSENCE_MOB_ID)); + } + let ident = Obj::new(cls::IDENTIFICATION) + .data_prop(pid::COMPANY_NAME, utf16z(identity.company)) + .data_prop(pid::PRODUCT_NAME, utf16z(identity.product)) + .data_prop(pid::PRODUCT_VERSION_STRING, utf16z(identity.version)) + .data_prop(pid::PRODUCT_ID, identity.product_id.to_vec()) + .data_prop(pid::DATE, timestamp()) + .data_prop(pid::PLATFORM, utf16z(identity.platform)) + .data_prop(pid::GENERATION_AUID, mob_id(ids.seed, 0)[16..].to_vec()); + let header = Obj::new(cls::HEADER) + .data_prop(pid::BYTE_ORDER, 0x4949i16.to_le_bytes().to_vec()) + .data_prop(pid::LAST_MODIFIED, timestamp()) + .data_prop(pid::VERSION, vec![1, 1]) + .data_prop(pid::OBJECT_MODEL_VERSION, 1u32.to_le_bytes().to_vec()) + .data_prop(pid::OPERATIONAL_PATTERN, def::OP_EDIT_PROTOCOL.to_vec()) + .with(pid::CONTENT, Value::Strong(Box::new(content))) + .with(pid::DICTIONARY, Value::Strong(Box::new(dict))) + .with(pid::IDENTIFICATION_LIST, Value::StrongVec(vec![ident])); + // The meta-dictionary holds only extensions to the baseline object model, and everything written + // here is baseline, so its class and type definition sets are empty; readers that load them + // (pyaaf2) still need the sets to be there (filmcraft#323). + let meta = Obj::new(META_DICTIONARY) + .with(pid::META_CLASS_DEFINITIONS, Value::StrongSet(Vec::new(), pid::META_IDENTIFICATION)) + .with(pid::META_TYPE_DEFINITIONS, Value::StrongSet(Vec::new(), pid::META_IDENTIFICATION)); + let root = Obj::new(ROOT).with(pid::ROOT_META_DICTIONARY, Value::Strong(Box::new(meta))).with(pid::ROOT_HEADER, Value::Strong(Box::new(header))); + store::write(&root, version).map_err(Error::Other) +} + +/// Where clips find their sources: source index → (mob id, slot id), and for nested compositions +/// the edit rate of that slot. +#[derive(Default)] +struct Refs { + mobs: HashMap, + rates: HashMap, +} + +/// The composition mob of `comp`. Slot 1 is its timecode; its tracks follow from slot 2 in order. +struct CompositionWriter<'a, 'b, 'c> { + w: &'a mut W<'c>, + doc: &'b Document, + refs: &'b Refs, + used_ops: &'a mut Vec, + uses_varying: &'a mut bool, +} + +impl CompositionWriter<'_, '_, '_> { + fn write(&mut self, comp: &Composition, id: [u8; 32], top: bool) -> Obj { + let w = &mut *self.w; + let doc = self.doc; + let refs = self.refs; + let used_ops = &mut *self.used_ops; + let uses_varying = &mut *self.uses_varying; + let crate_rate = (comp.rate.num, comp.rate.den); + let mut slots = Vec::new(); + let total = comp + .tracks + .iter() + .map(|t| { + let mut c = Tick::ZERO; + for i in &t.items { + match i { + CItem::Transition(x) => c -= x.len, + other => c += other.len(), + } + } + c + }) + .max() + .unwrap_or(Tick::ZERO); + let total_units = w.u(total, crate_rate); + slots.push(timeline_slot( + 1, + "TC1", + 0, + crate_rate, + Obj::new(cls::TIMECODE) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, ddef::TIMECODE)) + .data_prop(pid::LENGTH, total_units.to_le_bytes().to_vec()) + .data_prop(pid::TC_START, comp.start_tc.to_le_bytes().to_vec()) + .data_prop(pid::TC_FPS, (comp.rate.timecode_base().clamp(1, u16::MAX as i64) as u16).to_le_bytes().to_vec()) + .data_prop(pid::TC_DROP, vec![comp.drop as u8]), + )); + let mut slot_id = 2u32; + for t in &comp.tracks { + let rate = match t.kind { + CKind::Picture => crate_rate, + CKind::Sound => (comp.sample_rate.max(1) as i64, 1), + }; + let dd = data_def(t.kind); + let mut comps = Vec::new(); + let mut cursor = Tick::ZERO; + for it in &t.items { + match it { + CItem::Filler(l) => { + let n = w.u(cursor + *l, rate) - w.u(cursor, rate); + cursor += *l; + comps.push( + Obj::new(cls::FILLER) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, dd)) + .data_prop(pid::LENGTH, n.to_le_bytes().to_vec()), + ); + } + CItem::Clip(c) => { + let n = w.u(cursor + c.len, rate) - w.u(cursor, rate); + let s = &doc.sources[c.source]; + let (mid, mslot) = refs.mobs.get(&c.source).copied().unwrap_or(([0; 32], 0)); + // (a nested composition counts the start in its own slot's edit units) + let start = w.u(c.start, refs.rates.get(&c.source).copied().unwrap_or(rate)); + let mut sc = source_clip(t.kind, n, mid, mslot, start); + if !c.name.is_empty() && c.name != s.name { + sc.set(pid::COMPONENT_USER_COMMENTS, Value::StrongVec(vec![tagged("Clip Name", &c.name)])); + } + let seg = match (&c.gain, t.kind) { + (Some(g), CKind::Sound) => { + if !used_ops.contains(&def::MONO_AUDIO_GAIN) { + used_ops.push(def::MONO_AUDIO_GAIN); + } + let param = match g { + Gain::Constant(a) => Obj::new(cls::CONSTANT_VALUE) + .data_prop(pid::PARAMETER_DEFINITION, def::PARAM_AMPLITUDE.to_vec()) + .data_prop(pid::CONSTANT_VALUE, indirect(def::TYPE_RATIONAL, &gain_rational(*a))), + Gain::Varying { linear, points } => { + *uses_varying = true; + let pts = points + .iter() + .map(|(off, a)| { + let o = w.u(cursor + *off, rate) - w.u(cursor, rate); + Obj::new(cls::CONTROL_POINT) + .data_prop(pid::CP_TIME, rational(o, n.max(1))) + .data_prop(pid::CP_VALUE, indirect(def::TYPE_RATIONAL, &gain_rational(*a))) + .data_prop(pid::CP_EDIT_HINT, vec![0]) + }) + .collect(); + Obj::new(cls::VARYING_VALUE) + .data_prop(pid::PARAMETER_DEFINITION, def::PARAM_AMPLITUDE.to_vec()) + .with( + pid::INTERPOLATION, + weak(path::INTERPOLATION_DEFS, if *linear { def::INTERP_LINEAR } else { def::INTERP_CONSTANT }), + ) + .with(pid::POINT_LIST, Value::StrongVec(pts)) + } + }; + Obj::new(cls::OPERATION_GROUP) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, dd)) + .data_prop(pid::LENGTH, n.to_le_bytes().to_vec()) + .with(pid::OPERATION, weak(path::OPERATION_DEFS, def::MONO_AUDIO_GAIN)) + .with(pid::INPUT_SEGMENTS, Value::StrongVec(vec![sc])) + .with(pid::PARAMETERS, Value::StrongVec(vec![param])) + } + _ => sc, + }; + comps.push(seg); + cursor += c.len; + } + CItem::Transition(x) => { + let ts = cursor - x.len; + let n = w.u(cursor, rate) - w.u(ts, rate); + let cut = w.u(ts + x.cut, rate) - w.u(ts, rate); + cursor = ts; + let op = transition_op(t.kind, &x.effect); + if !used_ops.contains(&op) { + used_ops.push(op); + } + let mut og = Obj::new(cls::OPERATION_GROUP) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, dd)) + .data_prop(pid::LENGTH, n.to_le_bytes().to_vec()) + .with(pid::OPERATION, weak(path::OPERATION_DEFS, op)) + .with(pid::COMPONENT_USER_COMMENTS, Value::StrongVec(vec![tagged("FilmCraft Effect", &x.effect)])); + if op == def::VIDEO_DISSOLVE { + *uses_varying = true; + let pts = [(0, 0), (1, 1)] + .iter() + .map(|&(t, v)| { + Obj::new(cls::CONTROL_POINT) + .data_prop(pid::CP_TIME, rational(t, 1)) + .data_prop(pid::CP_VALUE, indirect(def::TYPE_RATIONAL, &rational(v, 1))) + .data_prop(pid::CP_EDIT_HINT, vec![0]) + }) + .collect(); + og.set( + pid::PARAMETERS, + Value::StrongVec(vec![ + Obj::new(cls::VARYING_VALUE) + .data_prop(pid::PARAMETER_DEFINITION, def::PARAM_LEVEL.to_vec()) + .with(pid::INTERPOLATION, weak(path::INTERPOLATION_DEFS, def::INTERP_LINEAR)) + .with(pid::POINT_LIST, Value::StrongVec(pts)), + ]), + ); + } + comps.push( + Obj::new(cls::TRANSITION) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, dd)) + .data_prop(pid::LENGTH, n.to_le_bytes().to_vec()) + .data_prop(pid::CUT_POINT, cut.to_le_bytes().to_vec()) + .with(pid::OPERATION_GROUP, Value::Strong(Box::new(og))), + ); + } + } + } + let len: i64 = comps + .iter() + .map(|c| { + let l = c.i64(pid::LENGTH).unwrap_or(0); + if c.class == cls::TRANSITION { -l } else { l } + }) + .sum(); + let seq = Obj::new(cls::SEQUENCE) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, dd)) + .data_prop(pid::LENGTH, len.to_le_bytes().to_vec()) + .with(pid::COMPONENTS, Value::StrongVec(comps)); + slots.push(timeline_slot(slot_id, &t.name, t.number, rate, seq)); + slot_id += 1; + } + if !comp.markers.is_empty() { + slots.push(event_slot(w, slot_id, crate_rate, &comp.markers)); + } + let mut mob = Obj::new(cls::COMPOSITION_MOB) + .data_prop(pid::MOB_ID, id.to_vec()) + .data_prop(pid::MOB_NAME, utf16z(&comp.name)) + .data_prop(pid::MOB_LAST_MODIFIED, timestamp()) + .data_prop(pid::MOB_CREATION_TIME, timestamp()); + // only the exported sequence is a top-level composition; a nested sequence is found through + // the clips that use it + if top { + mob.set(pid::USAGE_CODE, Value::Data(def::USAGE_TOP_LEVEL.to_vec())); + } + mob.with(pid::SLOTS, Value::StrongVec(slots)).with( + pid::MOB_USER_COMMENTS, + Value::StrongVec(vec![ + tagged("FilmCraft Frame Size", &format!("{}x{}", comp.width, comp.height)), + tagged("FilmCraft Audio Sample Rate", &comp.sample_rate.to_string()), + ]), + ) + } +} + +fn slot_name(kind: CKind, n: u32) -> String { + match kind { + CKind::Picture => format!("V{n}"), + CKind::Sound => format!("A{n}"), + } +} + +fn timeline_slot(id: u32, name: &str, number: u32, rate: (i64, i64), segment: Obj) -> Obj { + let mut o = Obj::new(cls::TIMELINE_MOB_SLOT) + .data_prop(pid::SLOT_ID, id.to_le_bytes().to_vec()) + .data_prop(pid::SLOT_NAME, utf16z(name)) + .data_prop(pid::EDIT_RATE, rational(rate.0, rate.1)) + .data_prop(pid::ORIGIN, 0i64.to_le_bytes().to_vec()) + .with(pid::SEGMENT, Value::Strong(Box::new(segment))); + if number > 0 { + o.set(pid::PHYSICAL_TRACK_NUMBER, Value::Data(number.to_le_bytes().to_vec())); + } + o +} + +fn source_clip(kind: CKind, len: i64, source: [u8; 32], slot: u32, start: i64) -> Obj { + Obj::new(cls::SOURCE_CLIP) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, data_def(kind))) + .data_prop(pid::LENGTH, len.to_le_bytes().to_vec()) + .data_prop(pid::SOURCE_ID, source.to_vec()) + .data_prop(pid::SOURCE_MOB_SLOT_ID, slot.to_le_bytes().to_vec()) + .data_prop(pid::START_TIME, start.to_le_bytes().to_vec()) +} + +fn event_slot(w: &mut W, id: u32, rate: (i64, i64), markers: &[CMarker]) -> Obj { + let comps = markers + .iter() + .map(|m| { + let pos = w.u(m.start, rate); + let len = w.u(m.start + m.duration, rate) - pos; + let mut tags = vec![tagged("Name", &m.name), tagged("Comment", &m.comment)]; + if let Some(c) = &m.color { + tags.push(tagged("Color", c)); + } + Obj::new(cls::COMMENT_MARKER) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, ddef::DESCRIPTIVE_METADATA)) + .data_prop(pid::LENGTH, len.to_le_bytes().to_vec()) + .data_prop(pid::POSITION, pos.to_le_bytes().to_vec()) + .data_prop(pid::COMMENT, utf16z(if m.comment.is_empty() { &m.name } else { &m.comment })) + .with(pid::COMPONENT_USER_COMMENTS, Value::StrongVec(tags)) + }) + .collect(); + Obj::new(cls::EVENT_MOB_SLOT) + .data_prop(pid::SLOT_ID, id.to_le_bytes().to_vec()) + .data_prop(pid::SLOT_NAME, utf16z("Markers")) + .data_prop(pid::EVENT_EDIT_RATE, rational(rate.0, rate.1)) + .with( + pid::SEGMENT, + Value::Strong(Box::new( + Obj::new(cls::SEQUENCE) + .with(pid::DATA_DEFINITION, weak(path::DATA_DEFS, ddef::DESCRIPTIVE_METADATA)) + .with(pid::COMPONENTS, Value::StrongVec(comps)), + )), + ) +} + +fn transition_op(kind: CKind, effect: &str) -> Auid { + match kind { + CKind::Sound => def::MONO_AUDIO_DISSOLVE, + CKind::Picture => match effect { + "dip_to_black" => def::VIDEO_FADE_TO_BLACK, + e if e.contains("wipe") => def::SMPTE_VIDEO_WIPE, + _ => def::VIDEO_DISSOLVE, + }, + } +} + +fn descriptor(w: &mut W, s: &Source) -> Obj { + let rate = source_rate(s); + let len = w.u(s.length, rate); + let container = if s.embedded.is_some() { def::CONTAINER_AAF } else { def::CONTAINER_EXTERNAL }; + let mut d = match s.kind { + CKind::Picture => Obj::new(cls::CDCI_DESCRIPTOR) + .data_prop(pid::STORED_WIDTH, s.width.to_le_bytes().to_vec()) + .data_prop(pid::STORED_HEIGHT, s.height.to_le_bytes().to_vec()) + .data_prop(pid::FRAME_LAYOUT, vec![0]) + .data_prop(pid::IMAGE_ASPECT_RATIO, rational(s.width.max(1) as i64, s.height.max(1) as i64)) + .data_prop(pid::VIDEO_LINE_MAP, { + let mut v = 2u32.to_le_bytes().to_vec(); + v.extend_from_slice(&4u32.to_le_bytes()); + v.extend_from_slice(&0i32.to_le_bytes()); + v.extend_from_slice(&0i32.to_le_bytes()); + v + }) + .data_prop(pid::COMPONENT_WIDTH, 8u32.to_le_bytes().to_vec()) + .data_prop(pid::HORIZONTAL_SUBSAMPLING, 2u32.to_le_bytes().to_vec()), + CKind::Sound => { + // Channel counts and rates come from the project: saturate, as these fields can't + // hold more anyway. + let block = s.file_channels.max(1).saturating_mul((s.bits as u32).div_ceil(8)); + Obj::new(cls::PCM_DESCRIPTOR) + .data_prop(pid::AUDIO_SAMPLING_RATE, rational(s.sample_rate as i64, 1)) + .data_prop(pid::LOCKED, vec![1]) + .data_prop(pid::CHANNELS, s.file_channels.max(1).to_le_bytes().to_vec()) + .data_prop(pid::QUANTIZATION_BITS, (s.bits as u32).to_le_bytes().to_vec()) + .data_prop(pid::BLOCK_ALIGN, u16::try_from(block).unwrap_or(u16::MAX).to_le_bytes().to_vec()) + .data_prop(pid::AVERAGE_BPS, block.saturating_mul(s.sample_rate).to_le_bytes().to_vec()) + } + }; + d.set(pid::SAMPLE_RATE, Value::Data(rational(rate.0, rate.1))); + d.set(pid::FILE_LENGTH, Value::Data(len.to_le_bytes().to_vec())); + d.set(pid::CONTAINER_FORMAT, weak(path::CONTAINER_DEFS, container)); + if let Some(p) = &s.path { + let url = if craft_timeline::path::is_absolute(p) { + craft_timeline::path::path_to_file_url(p, false) + } else { + craft_timeline::path::encode_uri_path(p) + }; + d.set(pid::LOCATOR, Value::StrongVec(vec![Obj::new(cls::NETWORK_LOCATOR).data_prop(pid::URL_STRING, utf16z(&url))])); + } + d +} diff --git a/crates/aaf/src/lib.rs b/crates/aaf/src/lib.rs new file mode 100644 index 000000000..ee7448bc0 --- /dev/null +++ b/crates/aaf/src/lib.rs @@ -0,0 +1,60 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] +//! AAF and OMF: the two mob-based editorial interchange formats, read and written through one +//! composition model. +//! +//! - [`model`]: a [`model::Document`] of compositions (tracks of fillers, source clips and +//! overlapping transitions, with clip gain and markers) and the media sources they reference, +//! with exact times ([`craft_timeline::time::Tick`]). +//! - [`aaf`]: Advanced Authoring Format, Edit Protocol, in structured storage (`craft-cfb`). +//! - [`omf`]: OMF Interchange 2.0 in a Bento container; sound only. +//! +//! Both read into and write from the same model, so an application maps its timeline onto the +//! model once. Audio essence can be embedded: the model carries it as PCM, and the WAVE / AIFF +//! wrappers the formats use are craft-audio's. +//! +//! Clean-room: written from the public specifications only (AMWA *AAF Object Specification* v1.1, +//! *AAF Low-Level Container Specification* v1.0.1, *AAF Edit Protocol* v1.0, SMPTE RP 224 / RP 210 +//! labels; Avid *OMF Interchange Specification* 2.0, Apple *Bento Specification* 1.0d5). No file +//! I/O: documents are bytes and media references are strings. +//! +//! Input is untrusted: a damaged or crafted file gives an [`Error`] or a [`Report`] entry, never +//! a panic, a hang or an allocation sized by a count in the file. + +pub mod aaf; +pub mod model; +pub mod omf; +mod pcm; +#[cfg(test)] +mod roundtrip_tests; +#[cfg(test)] +mod testdoc; + +pub use craft_timeline::report::Report; + +/// Error from a reader or writer. Unsupported *features* never error; they go in the [`Report`]. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The bytes are not a readable document of the format. + #[error("{format} parse error: {message}")] + Parse { format: &'static str, message: String }, + /// The document has no composition. + #[error("the document contains no timeline")] + Empty, + /// The document could not be written. + #[error("{0}")] + Other(String), +} + +pub type Result = std::result::Result; + +/// Who a written file says made it (the AAF `Identification` object, the OMF `IDNT` object). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Identity<'a> { + pub company: &'a str, + pub product: &'a str, + /// The product's version, as text. + pub version: &'a str, + pub platform: &'a str, + /// AAF `ProductID`: an AUID naming the product. + pub product_id: [u8; 16], +} diff --git a/crates/aaf/src/model.rs b/crates/aaf/src/model.rs new file mode 100644 index 000000000..e6bc79140 --- /dev/null +++ b/crates/aaf/src/model.rs @@ -0,0 +1,189 @@ +//! A format-neutral mob-style composition model shared by AAF and OMF. +//! +//! Both formats describe a timeline as a composition of slots (tracks) whose segments are +//! sequences of fillers, source clips and transitions, where a transition *overlaps* the end of +//! the segment before it and the start of the segment after it (the sequence's length is the sum of +//! its segment lengths minus its transition lengths). Source clips reference media through master +//! mobs and file source mobs: here, a [`Source`] per essence, grouped into one media item by its +//! key. All times are ticks; the format serialisers convert to and from edit units. + +use craft_timeline::time::{FrameRate, TICKS_PER_SECOND, Tick}; + +/// Picture or sound. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum CKind { + Picture, + Sound, +} + +/// A whole document: compositions and the media they reference. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Document { + /// The top-level compositions. + pub compositions: Vec, + pub sources: Vec, + /// Compositions that are only used inside others (nested sequences); a [`Source`] refers to + /// one by its index here. They may nest further. + pub nested: Vec, +} + +/// How a nested sequence is written. Premiere Pro writes one as a composition of its own in AAF +/// (clips refer to it the way they refer to media) and renders its sound in OMF, which has no +#[derive(Clone, Debug, PartialEq)] +pub struct Composition { + pub name: String, + /// AAF CompositionMob identifier as 64 lowercase hexadecimal digits. OMF has no equivalent. + /// It is a read identity for applications that let a user choose among compositions; writers + /// generate fresh format identifiers and do not preserve it. + pub aaf_mob_id: Option, + pub rate: FrameRate, + pub sample_rate: u32, + pub width: u32, + pub height: u32, + pub start_tc: i64, + pub drop: bool, + pub tracks: Vec, + pub markers: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CTrack { + pub kind: CKind, + pub name: String, + /// Physical track number (1-based, per kind). + pub number: u32, + /// Sound: channels of the track (1 mono, 2 stereo, 6 5.1). + pub channels: u32, + pub items: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum CItem { + Filler(Tick), + Clip(CClip), + Transition(CTransition), +} + +impl CItem { + /// The item's length on the track. + pub fn len(&self) -> Tick { + match self { + CItem::Filler(l) => *l, + CItem::Clip(c) => c.len, + CItem::Transition(t) => t.len, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CClip { + pub len: Tick, + /// Index into [`Document::sources`]. + pub source: usize, + /// Source (file) time of the segment's first frame. + pub start: Tick, + pub gain: Option, + pub name: String, +} + +/// Clip gain (linear amplitude). +#[derive(Clone, Debug, PartialEq)] +pub enum Gain { + Constant(f64), + /// Points at offsets from the segment start; `linear` false = hold (step) interpolation. + Varying { + linear: bool, + points: Vec<(Tick, f64)>, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CTransition { + pub len: Tick, + /// Where the cut lies, from the transition start. + pub cut: Tick, + /// The transition, by a name of the writing application's choosing; it is kept in a private tag + /// and comes back unchanged. Names the formats themselves can say: `cross_dissolve` and + /// `dip_to_black` (picture), any name containing `wipe` (a SMPTE wipe), `constant_power` (sound). + /// A file without the tag reads as one of these. + pub effect: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CMarker { + pub start: Tick, + pub duration: Tick, + pub name: String, + pub comment: String, + pub color: Option, +} + +/// One essence: a file (or embedded data) carrying the picture, or some channels of sound, of a +/// media item. +#[derive(Clone, Debug, PartialEq)] +pub struct Source { + /// Groups the sources of one media item (one master mob). + pub key: String, + pub name: String, + pub kind: CKind, + pub path: Option, + /// Sound: the (0-based) channel of the file this source exposes (broken out to mono), or + /// `None` for all of them. + pub channel: Option, + /// Sound: channels this source exposes (1 when `channel` is set, else `file_channels`). + pub channels: u32, + /// Sound: channels of the file / embedded data. + pub file_channels: u32, + pub width: u32, + pub height: u32, + /// Picture: the frame rate of the media. + pub frame_rate: FrameRate, + pub sample_rate: u32, + pub bits: u16, + pub length: Tick, + /// Media time of the essence's time 0 (trimmed / consolidated essence). + pub offset: Tick, + /// Start timecode in frames at `tc_rate`. + pub start_tc: Option, + pub tc_rate: FrameRate, + pub embedded: Option>, + pub markers: Vec, + /// Not media but a nested composition: its index in [`Document::nested`]. `start` of a clip + /// that uses it is a time in that composition. + pub nested: Option, +} + +impl Source { + /// A picture source with no length: a still image. + pub fn is_still(&self) -> bool { + self.kind == CKind::Picture && self.length == Tick::ZERO + } +} + +/// The largest time read from a document (about 3.3 days). +pub const MAX_TIME: Tick = Tick(1 << 56); + +/// Ticks of `n` edit units at rate `num/den` (rounded to the nearest tick). +pub fn units_to_ticks(n: i64, num: i64, den: i64) -> Tick { + if num <= 0 || den <= 0 { + return Tick::ZERO; + } + let t = n as i128 * den as i128 * TICKS_PER_SECOND as i128; + let q = t.div_euclid(num as i128); + let r = t.rem_euclid(num as i128); + // documents are untrusted: keep every time far from overflow (±3 days) + Tick((q + if r * 2 >= num as i128 { 1 } else { 0 }).clamp(-MAX_TIME.0 as i128, MAX_TIME.0 as i128) as i64) +} + +/// Edit units at rate `num/den` nearest to `t`; the flag is false when `t` is not on a unit. +pub fn ticks_to_units(t: Tick, num: i64, den: i64) -> (i64, bool) { + if num <= 0 || den <= 0 { + return (0, false); + } + let x = t.0 as i128 * num as i128; + let d = den as i128 * TICKS_PER_SECOND as i128; + let q = x.div_euclid(d); + let r = x.rem_euclid(d); + let n = q + if r * 2 >= d { 1 } else { 0 }; + (n.clamp(i64::MIN as i128, i64::MAX as i128) as i64, r == 0) +} diff --git a/crates/aaf/src/omf.rs b/crates/aaf/src/omf.rs new file mode 100644 index 000000000..57e0f756c --- /dev/null +++ b/crates/aaf/src/omf.rs @@ -0,0 +1,47 @@ +//! OMF Interchange 2.0 for audio post: written, and the same structures read. +//! +//! Written from the public *OMF Interchange Specification Version 2.0* (Avid Technology, 1995/97) +//! and Apple's *Bento Specification* revision 1.0d5 for the container ([`bento`]). +//! +//! The file holds a composition mob (`CMOB`) with a timecode slot and one sound slot per audio +//! track (one per channel when broken out to mono): sequences (`SEQU`) of fillers (`FILL`), source +//! clips (`SCLP`) and transitions (`TRAN` with a `SimpleMonoAudioDissolve` effect); clip volume is a +//! `MonoAudioGain` effect (`EFFE`) whose slot −1 is the clip and slot 1 the amplitude +//! (`CVAL` constant or `VVAL` keyframes). Every media item has a master mob (`MMOB`), a file source +//! mob (`SMOB` with a `WAVD` / `AIFD` descriptor: embedded essence in a `WAVE` media data object, +//! or a locator to a separate file) and a tape source mob carrying its timecode. Times are in +//! samples (sound slots use the sample rate as edit rate) so edits are sample-exact. +//! +//! Only sound is written: picture tracks and markers of the model are left out, and a nested +//! composition is not written as one (readers of OMF do not follow compositions). A caller with +//! nested sequences mixes their sound and passes it as media like any other; Premiere Pro's OMF +//! export does the same. +//! +//! Private properties. `FilmCraft:SCLP:ClipName` (a clip's name) and `FilmCraft:EFFE:EffectID` (a +//! transition's name) are not in the specification; they were first written by FilmCraft and keep +//! their names so that existing files read back. + +pub(crate) mod bento; +mod read; +mod write; + +use crate::model::Document; +use crate::{Identity, Report, Result}; + +/// Write the sound of `doc`'s first composition as an OMF 2.0 file. +pub fn write(doc: &Document, identity: &Identity<'_>, report: &mut Report) -> Result> { + write::write(doc, identity, report) +} + +/// Read an OMF 2.0 file (as written by [`write`] and by applications using the same structures). +pub fn read(bytes: &[u8], report: &mut Report) -> Result { + read::read(bytes, report) +} + +/// Whether `bytes` end with a Bento container label (OMF files do). +pub fn sniff(bytes: &[u8]) -> bool { + bento::sniff(bytes) +} + +#[cfg(test)] +mod tests; diff --git a/crates/aaf/src/omf/bento.rs b/crates/aaf/src/omf/bento.rs new file mode 100644 index 000000000..00a7e2a26 --- /dev/null +++ b/crates/aaf/src/omf/bento.rs @@ -0,0 +1,407 @@ +//! The Bento container (Apple, *Bento Specification* revision 1.0d5), the storage layer of OMF +//! Interchange 2.0 files. +//! +//! A Bento container is a heap of value bytes, a table of contents (TOC) describing every value of +//! every object, and a fixed-size label at the very end of the file that locates the TOC: +//! +//! ```text +//! label (24 bytes, big-endian): magic A4 'C' 'M' A5 'H' 'd' 'r' D7, u16 flags, u16 buffer size +//! (KiB), u16 major version (1), u16 minor version (0), +//! u32 TOC offset, u32 TOC size +//! ``` +//! +//! Every value belongs to an (object id, property id, type id) triple. Properties and types are +//! themselves objects carrying a global name (`"OMFI:CPNT:Length"`, `"omfi:Length32"`). The TOC +//! is a stream of one-byte operations, each followed by its operands (big-endian): +//! +//! | op | operands | meaning | +//! |---|---|---| +//! | 1 NewObject | u32 id | following entries belong to this object | +//! | 2 NewProperty | u32 id | … to this property | +//! | 3 NewType | u32 id | starts a new value of this type | +//! | 4 ExplicitGen | u32 | generation of the value | +//! | 5 Offset4Len4 | u32 offset, u32 length | a value segment in the file | +//! | 6 ContdOffset4Len4 | u32 offset, u32 length | a continuation segment | +//! | 7 Offset8Len4 / 8 Contd… | u64 offset, u32 length | segments beyond 4 GiB | +//! | 9–13 Immediate0–4 | 4 bytes | a value of 0–4 bytes stored in the TOC | +//! | 14 ContdImmediate4 | 4 bytes | continuation | +//! | 15 ReferenceListID | u32 id | the object holding this value's reference list | +//! | 24 EndOfBufr | — | skip to the end of the current TOC buffer | +//! | 0xFF Padding | — | ignored | +//! +//! Object references (OMF's `omfi:ObjRef`) are 4-byte keys resolved through the referencing +//! object's reference list (standard property 22: pairs of u32 key, u32 object id). FilmCraft +//! uses the referenced object's id as the key. + +use std::collections::{BTreeMap, HashMap}; + +pub(crate) const MAGIC: [u8; 8] = [0xA4, b'C', b'M', 0xA5, b'H', b'd', b'r', 0xD7]; +pub(crate) const LABEL_SIZE: usize = 24; + +// standard object ids +const TOC_OBJECT: u32 = 1; +const TOC_SEED: u32 = 2; +const TOC_MIN_SEED: u32 = 3; +const TOC_TYPE: u32 = 19; +const TYPE_7BIT_ASCII: u32 = 21; +pub(crate) const OBJ_REFERENCES: u32 = 22; +const GLOBAL_TYPE_NAME: u32 = 23; +const GLOBAL_PROP_NAME: u32 = 24; +const MIN_USER_ID: u32 = 100; + +const OP_NEW_OBJECT: u8 = 1; +const OP_NEW_PROPERTY: u8 = 2; +const OP_NEW_TYPE: u8 = 3; +const OP_EXPLICIT_GEN: u8 = 4; +const OP_OFFSET4_LEN4: u8 = 5; +const OP_CONTD_OFFSET4_LEN4: u8 = 6; +const OP_OFFSET8_LEN4: u8 = 7; +const OP_CONTD_OFFSET8_LEN4: u8 = 8; +const OP_IMMEDIATE0: u8 = 9; +const OP_IMMEDIATE4: u8 = 13; +const OP_CONTD_IMMEDIATE4: u8 = 14; +const OP_REFERENCE_LIST_ID: u8 = 15; +const OP_END_OF_BUFFER: u8 = 24; +const OP_PADDING: u8 = 0xFF; + +const TOC_BUFFER_KIB: u16 = 8; + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Seg { + Imm(Vec), + At(u64, u32), +} + +/// Bento values are metadata, not essence payloads. Limit reassembled segments so repeated +/// references cannot amplify a small OMF into an enormous allocation. +const MAX_VALUE_BYTES: usize = 16 << 20; + +/// Most bytes of global names (property and type names) kept for one container. Real files name a +/// few hundred short strings; without a cap, many objects naming one large shared range would each +/// retain their own copy. +const MAX_NAME_BYTES: usize = 4 << 20; + +/// Builds a Bento container. +pub(crate) struct Writer { + data: Vec, + names: HashMap<(u32, String), u32>, + next: u32, + /// (object, property, type, segments) + entries: Vec<(u32, u32, u32, Vec)>, +} + +impl Writer { + pub fn new() -> Writer { + Writer { data: Vec::new(), names: HashMap::new(), next: MIN_USER_ID, entries: Vec::new() } + } + + fn named(&mut self, kind: u32, name: &str) -> u32 { + if let Some(&id) = self.names.get(&(kind, name.to_string())) { + return id; + } + let id = self.new_object(); + let mut v = name.as_bytes().to_vec(); + v.push(0); + let seg = self.store(&v); + self.entries.push((id, kind, TYPE_7BIT_ASCII, vec![seg])); + self.names.insert((kind, name.to_string()), id); + id + } + + pub fn property(&mut self, name: &str) -> u32 { + self.named(GLOBAL_PROP_NAME, name) + } + + pub fn type_id(&mut self, name: &str) -> u32 { + self.named(GLOBAL_TYPE_NAME, name) + } + + pub fn new_object(&mut self) -> u32 { + let id = self.next; + self.next += 1; + id + } + + fn store(&mut self, v: &[u8]) -> Seg { + if v.len() <= 4 { + return Seg::Imm(v.to_vec()); + } + let at = self.data.len() as u64; + self.data.extend_from_slice(v); + Seg::At(at, v.len() as u32) + } + + /// Set a value of `obj` (values of more than 4 GiB are not supported by OMF). + pub fn set(&mut self, obj: u32, property: &str, ty: &str, value: &[u8]) { + let p = self.property(property); + let t = self.type_id(ty); + let seg = self.store(value); + self.entries.push((obj, p, t, vec![seg])); + } + + /// The reference list of `obj` (keys are object ids). + pub fn set_references(&mut self, obj: u32, targets: &[u32]) { + let mut v = Vec::with_capacity(targets.len() * 8); + for t in targets { + v.extend_from_slice(&t.to_be_bytes()); + v.extend_from_slice(&t.to_be_bytes()); + } + let seg = self.store(&v); + self.entries.push((obj, OBJ_REFERENCES, TYPE_7BIT_ASCII, vec![seg])); + } + + pub fn finish(mut self) -> Vec { + let seed = self.next; + self.entries.push((TOC_OBJECT, TOC_SEED, TOC_TYPE, vec![Seg::Imm(seed.to_be_bytes().to_vec())])); + self.entries.push((TOC_OBJECT, TOC_MIN_SEED, TOC_TYPE, vec![Seg::Imm(MIN_USER_ID.to_be_bytes().to_vec())])); + self.entries.sort_by_key(|e| (e.0, e.1, e.2)); + let mut toc = Vec::new(); + let (mut cur_obj, mut cur_prop) = (u32::MAX, u32::MAX); + for (obj, prop, ty, segs) in &self.entries { + if *obj != cur_obj { + toc.push(OP_NEW_OBJECT); + toc.extend_from_slice(&obj.to_be_bytes()); + cur_obj = *obj; + cur_prop = u32::MAX; + } + if *prop != cur_prop { + toc.push(OP_NEW_PROPERTY); + toc.extend_from_slice(&prop.to_be_bytes()); + cur_prop = *prop; + } + toc.push(OP_NEW_TYPE); + toc.extend_from_slice(&ty.to_be_bytes()); + for (i, s) in segs.iter().enumerate() { + match s { + Seg::Imm(b) => { + toc.push(if i > 0 { OP_CONTD_IMMEDIATE4 } else { OP_IMMEDIATE0 + b.len() as u8 }); + let mut w = [0u8; 4]; + w[..b.len()].copy_from_slice(b); + toc.extend_from_slice(&w); + } + Seg::At(off, len) if *off <= u32::MAX as u64 => { + toc.push(if i > 0 { OP_CONTD_OFFSET4_LEN4 } else { OP_OFFSET4_LEN4 }); + toc.extend_from_slice(&(*off as u32).to_be_bytes()); + toc.extend_from_slice(&len.to_be_bytes()); + } + Seg::At(off, len) => { + toc.push(if i > 0 { OP_CONTD_OFFSET8_LEN4 } else { OP_OFFSET8_LEN4 }); + toc.extend_from_slice(&off.to_be_bytes()); + toc.extend_from_slice(&len.to_be_bytes()); + } + } + } + } + let mut out = self.data; + let toc_at = out.len() as u64; + out.extend_from_slice(&toc); + let mut label = MAGIC.to_vec(); + label.extend_from_slice(&0u16.to_be_bytes()); // flags + label.extend_from_slice(&TOC_BUFFER_KIB.to_be_bytes()); + label.extend_from_slice(&1u16.to_be_bytes()); + label.extend_from_slice(&0u16.to_be_bytes()); + label.extend_from_slice(&(toc_at.min(u32::MAX as u64) as u32).to_be_bytes()); + label.extend_from_slice(&(toc.len() as u32).to_be_bytes()); + out.extend_from_slice(&label); + out + } +} + +/// One value of a parsed object. +#[derive(Clone, Debug)] +pub(crate) struct Prop { + pub prop: u32, + #[allow(dead_code)] + pub ty: u32, + segs: Vec, +} + +/// A parsed Bento container. +pub(crate) struct Container<'a> { + data: &'a [u8], + pub objects: BTreeMap>, + names: HashMap, + by_name: HashMap, +} + +fn be32(b: &[u8], at: usize) -> Option { + b.get(at..at + 4).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]])) +} + +/// Whether `b` ends with a Bento label. +pub(crate) fn sniff(b: &[u8]) -> bool { + b.len() >= LABEL_SIZE && b[b.len() - LABEL_SIZE..b.len() - LABEL_SIZE + 8] == MAGIC +} + +impl<'a> Container<'a> { + pub fn open(data: &'a [u8]) -> Result, String> { + if !sniff(data) { + return Err("no Bento label at the end of the file".into()); + } + let l = &data[data.len() - LABEL_SIZE..]; + let buf = (u16::from_be_bytes([l[10], l[11]]) as usize).max(1) * 1024; + let toc_at = be32(l, 16).unwrap_or(0) as usize; + let toc_len = be32(l, 20).unwrap_or(0) as usize; + let toc = data.get(toc_at..toc_at.checked_add(toc_len).ok_or("bad TOC size")?).ok_or("the TOC lies outside the file")?; + let mut c = Container { data, objects: BTreeMap::new(), names: HashMap::new(), by_name: HashMap::new() }; + let (mut obj, mut prop) = (0u32, 0u32); + let mut i = 0; + while i < toc.len() { + let op = toc[i]; + i += 1; + let need = |n: usize| -> Result<&[u8], String> { toc.get(i..i + n).ok_or_else(|| "truncated TOC".to_string()) }; + match op { + OP_NEW_OBJECT => { + obj = be32(need(4)?, 0).unwrap_or(0); + i += 4; + } + OP_NEW_PROPERTY => { + prop = be32(need(4)?, 0).unwrap_or(0); + i += 4; + } + OP_NEW_TYPE => { + let ty = be32(need(4)?, 0).unwrap_or(0); + i += 4; + c.objects.entry(obj).or_default().push(Prop { prop, ty, segs: Vec::new() }); + } + OP_EXPLICIT_GEN | OP_REFERENCE_LIST_ID => { + need(4)?; + i += 4; + } + OP_OFFSET4_LEN4 | OP_CONTD_OFFSET4_LEN4 => { + let b = need(8)?; + let seg = Seg::At(be32(b, 0).unwrap_or(0) as u64, be32(b, 4).unwrap_or(0)); + i += 8; + c.push_seg(obj, seg)?; + } + OP_OFFSET8_LEN4 | OP_CONTD_OFFSET8_LEN4 => { + let b = need(12)?; + let off = u64::from_be_bytes(b[..8].try_into().map_err(|_| "truncated TOC")?); + let seg = Seg::At(off, be32(b, 8).unwrap_or(0)); + i += 12; + c.push_seg(obj, seg)?; + } + x @ (OP_IMMEDIATE0..=OP_IMMEDIATE4 | OP_CONTD_IMMEDIATE4) => { + let b = need(4)?; + let n = if x == OP_CONTD_IMMEDIATE4 { 4 } else { (x - OP_IMMEDIATE0) as usize }; + let seg = Seg::Imm(b[..n].to_vec()); + i += 4; + c.push_seg(obj, seg)?; + } + OP_END_OF_BUFFER => { + // the rest of this TOC buffer is unused + i = (i - 1) / buf * buf + buf; + } + OP_PADDING | 0 => {} + other => return Err(format!("unknown TOC operation {other}")), + } + } + // global names + let ids: Vec = c.objects.keys().copied().collect(); + let mut name_bytes = 0usize; + for id in ids { + let name = c.objects[&id].iter().find(|p| p.prop == GLOBAL_PROP_NAME || p.prop == GLOBAL_TYPE_NAME).and_then(|p| c.value(p).ok()); + if let Some(n) = name { + name_bytes = name_bytes.saturating_add(n.len()); + if name_bytes > MAX_NAME_BYTES { + return Err("Bento global names exceed the name budget".into()); + } + let s = String::from_utf8_lossy(&n).trim_end_matches('\0').to_string(); + c.by_name.insert(s.clone(), id); + c.names.insert(id, s); + } + } + Ok(c) + } + + fn push_seg(&mut self, obj: u32, seg: Seg) -> Result<(), String> { + let p = self.objects.get_mut(&obj).and_then(|v| v.last_mut()).ok_or("a value without a type in the TOC")?; + if p.segs.len() > 1 << 20 { + return Err("too many value segments".into()); + } + p.segs.push(seg); + Ok(()) + } + + /// The bytes of a value. + pub fn value(&self, p: &Prop) -> Result, String> { + let total = p.segs.iter().try_fold(0usize, |total, s| { + let len = match s { + Seg::Imm(b) => b.len(), + Seg::At(_, len) => *len as usize, + }; + total.checked_add(len).ok_or_else(|| "Bento value length overflows".to_string()) + })?; + if total > MAX_VALUE_BYTES { + return Err("Bento value exceeds the decoded-value budget".into()); + } + let mut out = Vec::new(); + out.try_reserve_exact(total).map_err(|_| "cannot allocate Bento value")?; + for s in &p.segs { + match s { + Seg::Imm(b) => out.extend_from_slice(b), + Seg::At(off, len) => { + let start = usize::try_from(*off).map_err(|_| "value offset too large")?; + let b = + self.data.get(start..start.checked_add(*len as usize).ok_or("bad value length")?).ok_or("a value lies outside the file")?; + out.extend_from_slice(b); + } + } + } + Ok(out) + } + + pub fn id_of(&self, name: &str) -> Option { + self.by_name.get(name).copied() + } + + #[allow(dead_code)] + pub fn name_of(&self, id: u32) -> Option<&str> { + self.names.get(&id).map(String::as_str) + } + + /// The first value of property `name` of `obj`. + pub fn get(&self, obj: u32, name: &str) -> Option> { + let pid = self.id_of(name)?; + let p = self.objects.get(&obj)?.iter().find(|p| p.prop == pid)?; + self.value(p).ok() + } + + /// Resolve an object reference key of `obj` through its reference list. + pub fn resolve(&self, obj: u32, key: u32) -> u32 { + if let Some(p) = self.objects.get(&obj).and_then(|v| v.iter().find(|p| p.prop == OBJ_REFERENCES)) + && let Ok(v) = self.value(p) + { + for pair in v.as_chunks::<8>().0 { + if be32(pair, 0) == Some(key) { + return be32(pair, 4).unwrap_or(key); + } + } + } + key + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn many_objects_cannot_name_themselves_with_one_shared_range() { + let mut w = Writer::new(); + w.data = vec![b'a'; 1 << 20]; + let ids: Vec = (0..8).map(|_| w.new_object()).collect(); + for id in ids { + w.entries.push((id, GLOBAL_PROP_NAME, TYPE_7BIT_ASCII, vec![Seg::At(0, 1 << 20)])); + } + let file = w.finish(); + assert!(Container::open(&file).err().is_some_and(|e| e.contains("name budget"))); + } + + #[test] + fn repeated_segments_cannot_expand_a_value_past_its_budget() { + let p = Prop { prop: 1, ty: 1, segs: vec![Seg::At(0, MAX_VALUE_BYTES as u32), Seg::Imm(vec![0])] }; + let c = Container { data: &[], objects: BTreeMap::new(), names: HashMap::new(), by_name: HashMap::new() }; + assert!(c.value(&p).unwrap_err().contains("budget")); + } +} diff --git a/crates/aaf/src/omf/read.rs b/crates/aaf/src/omf/read.rs new file mode 100644 index 000000000..d45b6ff63 --- /dev/null +++ b/crates/aaf/src/omf/read.rs @@ -0,0 +1,439 @@ +//! OMF 2.0 objects (Bento) → composition model. + +use std::collections::{HashMap, HashSet}; + +use craft_timeline::time::{FrameRate, Tick}; + +use super::bento::Container; +use crate::model::{CClip, CItem, CKind, CTrack, CTransition, Composition, Document, Gain, Source, units_to_ticks}; +use crate::{Error, Report, Result}; + +type Uid = [u8; 12]; + +const MAX_REFERENCE_VISITS: usize = 65_536; + +struct R<'a, 'b> { + c: &'b Container<'a>, + report: &'b mut Report, + /// big-endian values ("MM"); "II" files are little-endian + be: bool, + mobs: HashMap, + media: HashMap>, + doc: Document, + sources: HashMap<(Uid, u32, String), usize>, +} + +pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result { + let perr = |m: String| Error::Parse { format: "OMF", message: m }; + let c = Container::open(bytes).map_err(perr)?; + let class_of = |o: u32| c.get(o, "OMFI:OOBJ:ObjClass").and_then(|v| v.get(..4).map(|b| [b[0], b[1], b[2], b[3]])); + let head = c.objects.keys().copied().find(|&o| class_of(o) == Some(*b"HEAD")).ok_or_else(|| perr("no OMF header object".into()))?; + let be = c.get(head, "OMFI:HEAD:ByteOrder").is_none_or(|v| v.first() != Some(&0x49)); + let mut r = R { c: &c, report, be, mobs: HashMap::new(), media: HashMap::new(), doc: Document::default(), sources: HashMap::new() }; + let mob_list = r.refs(head, "OMFI:HEAD:Mobs"); + let mob_list = if mob_list.is_empty() { + c.objects.keys().copied().filter(|&o| matches!(class_of(o).as_ref(), Some(b"CMOB" | b"MMOB" | b"SMOB"))).collect() + } else { + mob_list + }; + for &m in &mob_list { + if let Some(id) = r.uid(m, "OMFI:MOBJ:MobID") { + r.mobs.insert(id, m); + } + } + let md = r.refs(head, "OMFI:HEAD:MediaData"); + let md = + if md.is_empty() { c.objects.keys().copied().filter(|&o| matches!(class_of(o).as_ref(), Some(b"WAVE" | b"AIFC"))).collect() } else { md }; + for m in md { + let Some(id) = r.uid(m, "OMFI:MDAT:MobID") else { continue }; + if let Some(d) = c.get(m, "OMFI:WAVE:Data") { + r.media.insert(id, d); + } else if let Some(d) = c.get(m, "OMFI:AIFC:Data") { + r.media.insert(id, d); + } + } + let cmobs: Vec = mob_list.iter().copied().filter(|&m| class_of(m) == Some(*b"CMOB")).collect(); + let mut referenced = HashSet::new(); + let mut visited = HashSet::new(); + let mut visits = 0; + for &m in &cmobs { + r.collect_refs(m, &mut referenced, &mut visited, &mut visits, 0); + } + let primary: Vec = r.refs(head, "OMFI:HEAD:PrimaryMobs").into_iter().filter(|m| cmobs.contains(m)).collect(); + let top: Vec = if !primary.is_empty() { + primary + } else { + cmobs.iter().copied().filter(|m| r.uid(*m, "OMFI:MOBJ:MobID").is_none_or(|id| !referenced.contains(&id))).collect() + }; + for m in top { + let comp = r.composition(m); + r.doc.compositions.push(comp); + } + if r.doc.compositions.is_empty() { + return Err(Error::Empty); + } + Ok(r.doc) +} + +impl R<'_, '_> { + fn class(&self, o: u32) -> [u8; 4] { + self.c.get(o, "OMFI:OOBJ:ObjClass").and_then(|v| v.get(..4).map(|b| [b[0], b[1], b[2], b[3]])).unwrap_or(*b"????") + } + fn u32_of(&self, b: &[u8]) -> Option { + let a: [u8; 4] = b.get(..4)?.try_into().ok()?; + Some(if self.be { u32::from_be_bytes(a) } else { u32::from_le_bytes(a) }) + } + fn i32(&self, o: u32, p: &str) -> Option { + self.c.get(o, p).and_then(|v| self.u32_of(&v)).map(|x| x as i32 as i64) + } + fn u32(&self, o: u32, p: &str) -> Option { + self.c.get(o, p).and_then(|v| self.u32_of(&v)) + } + fn u16(&self, o: u32, p: &str) -> Option { + let v = self.c.get(o, p)?; + let a: [u8; 2] = v.get(..2)?.try_into().ok()?; + Some(if self.be { u16::from_be_bytes(a) } else { u16::from_le_bytes(a) }) + } + fn rational(&self, o: u32, p: &str) -> Option<(i64, i64)> { + let v = self.c.get(o, p)?; + self.rational_of(&v) + } + fn rational_of(&self, v: &[u8]) -> Option<(i64, i64)> { + Some((self.u32_of(v)? as i32 as i64, self.u32_of(v.get(4..)?)? as i32 as i64)) + } + fn string(&self, o: u32, p: &str) -> Option { + self.c.get(o, p).map(|v| String::from_utf8_lossy(&v).trim_end_matches('\0').to_string()) + } + fn uid(&self, o: u32, p: &str) -> Option { + self.c.get(o, p).and_then(|v| v.get(..12).and_then(|b| b.try_into().ok())) + } + fn r#ref(&self, o: u32, p: &str) -> Option { + let v = self.c.get(o, p)?; + let key = u32::from_be_bytes(v.get(..4)?.try_into().ok()?); + Some(self.c.resolve(o, key)) + } + fn refs(&self, o: u32, p: &str) -> Vec { + let Some(v) = self.c.get(o, p) else { return Vec::new() }; + if v.len() < 2 { + return Vec::new(); + } + let n = u16::from_be_bytes([v[0], v[1]]) as usize; + v[2..].as_chunks::<4>().0.iter().take(n).map(|k| self.c.resolve(o, u32::from_be_bytes([k[0], k[1], k[2], k[3]]))).collect() + } + fn data_kind(&self, o: u32) -> String { + self.r#ref(o, "OMFI:CPNT:DataKind").and_then(|d| self.string(d, "OMFI:DDEF:DataKindID")).unwrap_or_default() + } + fn len(&self, o: u32) -> i64 { + self.i32(o, "OMFI:CPNT:Length").unwrap_or(0).max(0) + } + + fn collect_refs(&self, o: u32, out: &mut HashSet, visited: &mut HashSet, visits: &mut usize, depth: usize) { + if depth > 64 || !visited.insert(o) || *visits >= MAX_REFERENCE_VISITS { + return; + } + *visits += 1; + if self.class(o) == *b"SCLP" + && let Some(id) = self.uid(o, "OMFI:SCLP:SourceID") + { + out.insert(id); + } + for p in ["OMFI:MOBJ:Slots", "OMFI:SEQU:Components", "OMFI:EFFE:EffectSlots"] { + for c in self.refs(o, p) { + self.collect_refs(c, out, visited, visits, depth + 1); + } + } + for p in ["OMFI:MSLT:Segment", "OMFI:ESLT:ArgValue"] { + if let Some(c) = self.r#ref(o, p) { + self.collect_refs(c, out, visited, visits, depth + 1); + } + } + } + + fn composition(&mut self, m: u32) -> Composition { + let name = self.string(m, "OMFI:MOBJ:Name").unwrap_or_else(|| "Sequence".into()); + let mut comp = Composition { + name, + aaf_mob_id: None, + rate: FrameRate::FPS_25, + sample_rate: 48_000, + width: 1920, + height: 1080, + start_tc: 0, + drop: false, + tracks: Vec::new(), + markers: Vec::new(), + }; + let slots = self.refs(m, "OMFI:MOBJ:Slots"); + let mut na = 0; + for s in slots { + let Some(seg) = self.r#ref(s, "OMFI:MSLT:Segment") else { continue }; + let rate = self.rational(s, "OMFI:MSLT:EditRate").filter(|r| r.0 > 0 && r.1 > 0).unwrap_or((25, 1)); + let kind = self.data_kind(seg); + if self.class(seg) == *b"TCCP" || kind == "omfi:data:Timecode" { + comp.rate = FrameRate::new(rate.0, rate.1); + if self.class(seg) == *b"TCCP" { + comp.start_tc = self.i32(seg, "OMFI:TCCP:Start").unwrap_or(0); + comp.drop = self.c.get(seg, "OMFI:TCCP:Drop").and_then(|v| v.first().copied()).unwrap_or(0) != 0; + } + continue; + } + let ckind = match kind.as_str() { + "omfi:data:Sound" => CKind::Sound, + "omfi:data:Picture" | "omfi:data:PictureWithMatte" => CKind::Picture, + _ => { + self.report.info("an OMF slot of an unsupported data kind was skipped"); + continue; + } + }; + if ckind == CKind::Picture { + self.report.info("OMF picture slots are not imported (audio only)"); + continue; + } + if rate.1 == 1 && rate.0 >= 8000 { + comp.sample_rate = rate.0 as u32; + } + let items = self.segment(seg, rate, 0); + na += 1; + let td = self.r#ref(s, "OMFI:MSLT:TrackDesc"); + let tname = td.and_then(|t| self.string(t, "OMFI:TRKD:TrackName")).unwrap_or_else(|| format!("A{na}")); + let number = td.and_then(|t| self.u32(t, "OMFI:TRKD:PhysicalTrack")).unwrap_or(na); + let channels = items + .iter() + .filter_map(|i| match i { + CItem::Clip(c) => Some(self.doc.sources[c.source].channels), + _ => None, + }) + .max() + .unwrap_or(1) + .min(6); + comp.tracks.push(CTrack { kind: CKind::Sound, name: tname, number, channels, items }); + } + comp + } + + fn segment(&mut self, seg: u32, rate: (i64, i64), depth: usize) -> Vec { + let len = units_to_ticks(self.len(seg), rate.0, rate.1); + if depth > 32 { + return vec![CItem::Filler(len)]; + } + match &self.class(seg) { + b"SEQU" => { + let mut out = Vec::new(); + for c in self.refs(seg, "OMFI:SEQU:Components") { + out.extend(self.segment(c, rate, depth + 1)); + } + out + } + b"FILL" => vec![CItem::Filler(len)], + b"SCLP" => vec![self.clip(seg, rate)], + b"TRAN" => { + let cut = units_to_ticks(self.i32(seg, "OMFI:TRAN:CutPoint").unwrap_or(0), rate.0, rate.1); + let effect = self + .r#ref(seg, "OMFI:TRAN:Effect") + .and_then(|e| self.string(e, "FilmCraft:EFFE:EffectID")) + .unwrap_or_else(|| "constant_power".into()); + vec![CItem::Transition(CTransition { len, cut: cut.clamp(Tick::ZERO, len), effect })] + } + b"EFFE" => { + let kind = self.r#ref(seg, "OMFI:EFFE:EffectKind").and_then(|k| self.string(k, "OMFI:EDEF:EffectID")).unwrap_or_default(); + let slots = self.refs(seg, "OMFI:EFFE:EffectSlots"); + let arg = |r: &Self, id: i64| { + slots.iter().copied().find(|&s| r.i32(s, "OMFI:ESLT:ArgID") == Some(id)).and_then(|s| r.r#ref(s, "OMFI:ESLT:ArgValue")) + }; + let Some(input) = arg(self, -1).or_else(|| arg(self, 1).filter(|_| kind != "omfi:effect:MonoAudioGain")) else { + return vec![CItem::Filler(len)]; + }; + let gain = if kind == "omfi:effect:MonoAudioGain" { + arg(self, 1).and_then(|v| self.gain(v, len)) + } else { + self.report.warn("OMF effects other than audio gain are not imported (their input is used)"); + None + }; + let mut items = self.segment(input, rate, depth + 1); + if let (Some(g), [CItem::Clip(c)]) = (gain, items.as_mut_slice()) { + c.gain = Some(g); + } + items + } + _ => { + self.report.info("an unsupported OMF component was imported as a gap"); + vec![CItem::Filler(len)] + } + } + } + + fn amp(&self, v: &[u8]) -> Option { + self.rational_of(v).filter(|r| r.1 != 0).map(|(n, d)| n as f64 / d as f64) + } + + fn gain(&mut self, v: u32, len: Tick) -> Option { + match &self.class(v) { + b"CVAL" => self.c.get(v, "OMFI:CVAL:Value").and_then(|d| self.amp(&d)).map(Gain::Constant), + b"VVAL" => { + let linear = self.u16(v, "OMFI:VVAL:Interpolation") != Some(1); + let points: Vec<(Tick, f64)> = self + .refs(v, "OMFI:VVAL:PointList") + .into_iter() + .filter_map(|p| { + let (n, d) = self.rational(p, "OMFI:CTLP:Time").filter(|r| r.1 != 0)?; + let a = self.c.get(p, "OMFI:CTLP:Value").and_then(|d| self.amp(&d))?; + Some((Tick((len.0 as i128 * n as i128 / d as i128).clamp(-(1i128 << 56), 1i128 << 56) as i64), a)) + }) + .collect(); + (!points.is_empty()).then_some(Gain::Varying { linear, points }) + } + _ => None, + } + } + + fn clip(&mut self, c: u32, rate: (i64, i64)) -> CItem { + let len = units_to_ticks(self.len(c), rate.0, rate.1); + let start = units_to_ticks(self.i32(c, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1); + let name = self.string(c, "FilmCraft:SCLP:ClipName").unwrap_or_default(); + let (Some(id), Some(track)) = (self.uid(c, "OMFI:SCLP:SourceID"), self.u32(c, "OMFI:SCLP:SourceTrackID")) else { return CItem::Filler(len) }; + if id == [0; 12] { + return CItem::Filler(len); + } + match self.resolve(id, track, start, None, 0) { + Some((source, t)) => CItem::Clip(CClip { len, source, start: t, gain: None, name }), + None => CItem::Filler(len), + } + } + + fn slot_of(&self, mob: u32, track: u32) -> Option { + self.refs(mob, "OMFI:MOBJ:Slots") + .into_iter() + .find(|&s| self.r#ref(s, "OMFI:MSLT:TrackDesc").and_then(|t| self.u32(t, "OMFI:TRKD:TrackID")) == Some(track)) + } + + fn first_clip(&self, seg: u32) -> Option { + match &self.class(seg) { + b"SCLP" => Some(seg), + b"SEQU" => self.refs(seg, "OMFI:SEQU:Components").into_iter().find_map(|c| self.first_clip(c)), + _ => None, + } + } + + fn resolve(&mut self, id: Uid, track: u32, start: Tick, master: Option, depth: usize) -> Option<(usize, Tick)> { + if depth > 16 { + return None; + } + let Some(&mob) = self.mobs.get(&id) else { + self.report.warn("a clip references a mob that is not in the file (imported as a gap)"); + return None; + }; + let slot = self.slot_of(mob, track)?; + let rate = self.rational(slot, "OMFI:MSLT:EditRate").filter(|r| r.0 > 0 && r.1 > 0).unwrap_or((48_000, 1)); + let seg = self.r#ref(slot, "OMFI:MSLT:Segment")?; + match &self.class(mob) { + b"MMOB" => { + let key = master.unwrap_or_else(|| id.iter().map(|b| format!("{b:02x}")).collect()); + let clip = self.first_clip(seg)?; + let inner = units_to_ticks(self.i32(clip, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1); + let next = self.uid(clip, "OMFI:SCLP:SourceID")?; + let next_track = self.u32(clip, "OMFI:SCLP:SourceTrackID")?; + let name = self.string(mob, "OMFI:MOBJ:Name"); + let r = self.resolve(next, next_track, start + inner, Some(key), depth + 1)?; + if let Some(n) = name { + self.doc.sources[r.0].name = n; + } + Some(r) + } + b"SMOB" => { + let desc = self.r#ref(mob, "OMFI:SMOB:MediaDescription")?; + let sr = self.rational(desc, "OMFI:MDFL:SampleRate"); + if sr.is_none() && !self.media.contains_key(&id) { + return None; // a tape / physical source + } + let key = master.unwrap_or_else(|| id.iter().map(|b| format!("{b:02x}")).collect()); + let skey = (id, track, key.clone()); + if let Some(&i) = self.sources.get(&skey) { + return Some((i, start)); + } + let file_rate = sr.filter(|r| r.0 > 0 && r.1 > 0).unwrap_or(rate); + let length = units_to_ticks(self.i32(desc, "OMFI:MDFL:Length").unwrap_or(0), file_rate.0, file_rate.1); + let summary = self.c.get(desc, "OMFI:WAVD:Summary").or_else(|| self.c.get(desc, "OMFI:AIFD:Summary")).unwrap_or_default(); + let fmt = crate::pcm::parse_wav(&summary).or_else(|| crate::pcm::parse_aiff(&summary)).map(|(_, ch, sr, bits)| (ch as u32, sr, bits)); + let mut path = None; + for l in self.refs(desc, "OMFI:MDES:Locator") { + let p = match &self.class(l) { + b"UNXL" => self.string(l, "OMFI:UNXL:PathName"), + b"DOSL" => self.string(l, "OMFI:DOSL:PathName"), + b"MACL" => self.string(l, "OMFI:MACL:PathName"), + b"TXTL" => self.string(l, "OMFI:TXTL:Name"), + b"NETL" => self.string(l, "OMFI:NETL:URLString").map(|u| craft_timeline::path::file_url_to_path(&u)), + _ => None, + }; + if p.is_some() { + path = p; + break; + } + } + let sound_slots = self.refs(mob, "OMFI:MOBJ:Slots").len(); + let (file_channels, sample_rate, bits) = fmt.unwrap_or((1, file_rate.0 as u32, 16)); + let channel = (file_channels > 1 && sound_slots > 1).then(|| track.saturating_sub(1).min(file_channels - 1)); + // timecode of the physical source + let tc = self.first_clip(seg).and_then(|clip| { + let tid = self.uid(clip, "OMFI:SCLP:SourceID")?; + let tape = *self.mobs.get(&tid)?; + let (start_tc, fps, trate) = self.refs(tape, "OMFI:MOBJ:Slots").into_iter().find_map(|s| { + let g = self.r#ref(s, "OMFI:MSLT:Segment")?; + (self.class(g) == *b"TCCP").then(|| { + ( + self.i32(g, "OMFI:TCCP:Start").unwrap_or(0), + self.u16(g, "OMFI:TCCP:FPS").unwrap_or(25), + self.rational(s, "OMFI:MSLT:EditRate"), + ) + }) + })?; + let tcr = trate + .filter(|r| r.0 > 0 && r.1 > 0) + .map(|r| FrameRate::new(r.0, r.1)) + .unwrap_or_else(|| FrameRate::new(fps.max(1) as i64, 1)); + let off = units_to_ticks(self.i32(clip, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1); + Some((start_tc + tcr.frame_at(off), tcr)) + }); + let mut s = Source { + key, + name: self.string(mob, "OMFI:MOBJ:Name").unwrap_or_default(), + kind: CKind::Sound, + path, + channel, + channels: if channel.is_some() { 1 } else { file_channels }, + file_channels, + width: 0, + height: 0, + frame_rate: FrameRate::FPS_25, + sample_rate, + bits, + length, + offset: Tick::ZERO, + start_tc: tc.map(|t| t.0), + tc_rate: tc.map(|t| t.1).unwrap_or(FrameRate::FPS_25), + embedded: None, + markers: Vec::new(), + nested: None, + }; + if let Some(d) = self.media.get(&id) { + if let Some((pcm, ch, sr, b)) = crate::pcm::parse_wav(d).or_else(|| crate::pcm::parse_aiff(d)) { + s.embedded = Some(pcm); + s.file_channels = ch as u32; + s.channels = if s.channel.is_some() { 1 } else { ch as u32 }; + s.sample_rate = sr; + s.bits = b; + } else { + self.report.warn("embedded OMF audio in an unsupported format was skipped"); + } + } + let i = self.doc.sources.len(); + self.doc.sources.push(s); + self.sources.insert(skey, i); + Some((i, start)) + } + _ => { + self.report.warn("nested OMF compositions are not supported (imported as gaps)"); + None + } + } + } +} diff --git a/crates/aaf/src/omf/tests.rs b/crates/aaf/src/omf/tests.rs new file mode 100644 index 000000000..c6f4ef9f0 --- /dev/null +++ b/crates/aaf/src/omf/tests.rs @@ -0,0 +1,135 @@ +//! Structural checks of written OMF files: the Bento label and TOC, and the required properties of +//! the OMF Interchange 2.0 classes written. + +use std::collections::HashMap; + +use super::bento::{self, Container}; +use super::*; +use crate::testdoc::{IDENTITY, sound_only}; + +/// Every object of a class written has the properties the specification requires of it. +fn assert_required_properties(c: &Container<'_>, by_class: &HashMap>) { + let required: &[(&str, &[&str])] = &[ + ( + "HEAD", + &[ + "OMFI:HEAD:ByteOrder", + "OMFI:HEAD:LastModified", + "OMFI:HEAD:Version", + "OMFI:HEAD:Mobs", + "OMFI:HEAD:MediaData", + "OMFI:HEAD:DefinitionObjects", + ], + ), + ("CMOB", &["OMFI:MOBJ:MobID", "OMFI:MOBJ:Slots", "OMFI:MOBJ:LastModified", "OMFI:MOBJ:CreationTime"]), + ("MMOB", &["OMFI:MOBJ:MobID", "OMFI:MOBJ:Slots", "OMFI:MOBJ:LastModified", "OMFI:MOBJ:CreationTime"]), + ("SMOB", &["OMFI:MOBJ:MobID", "OMFI:MOBJ:Slots", "OMFI:SMOB:MediaDescription"]), + ("MSLT", &["OMFI:MSLT:Segment", "OMFI:MSLT:EditRate", "OMFI:MSLT:TrackDesc"]), + ("TRKD", &["OMFI:TRKD:Origin", "OMFI:TRKD:TrackID"]), + ("SEQU", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length", "OMFI:SEQU:Components"]), + ("SCLP", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length", "OMFI:SCLP:SourceID", "OMFI:SCLP:SourceTrackID", "OMFI:SCLP:StartTime"]), + ("FILL", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length"]), + ("TRAN", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length", "OMFI:TRAN:CutPoint", "OMFI:TRAN:Effect"]), + ("EFFE", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length", "OMFI:EFFE:EffectKind"]), + ("ESLT", &["OMFI:ESLT:ArgID", "OMFI:ESLT:ArgValue"]), + ("CVAL", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length", "OMFI:CVAL:Value"]), + ("EDEF", &["OMFI:EDEF:EffectID"]), + ("DDEF", &["OMFI:DDEF:DataKindID"]), + ("TCCP", &["OMFI:CPNT:DataKind", "OMFI:CPNT:Length", "OMFI:TCCP:Start", "OMFI:TCCP:FPS", "OMFI:TCCP:Drop"]), + ("WAVD", &["OMFI:MDFL:IsOMFI", "OMFI:MDFL:SampleRate", "OMFI:MDFL:Length", "OMFI:WAVD:Summary"]), + ("WAVE", &["OMFI:MDAT:MobID", "OMFI:WAVE:Data"]), + ("IDNT", &["OMFI:IDNT:CompanyName", "OMFI:IDNT:ProductName"]), + ]; + for (k, props) in required { + let Some(objs) = by_class.get(*k) else { + assert_eq!(*k, "FILL", "no {k} object"); + continue; + }; + for &o in objs { + for prop in *props { + assert!(c.get(o, prop).is_some(), "{k} {o} lacks {prop}"); + } + } + } +} + +#[test] +fn written_files_have_the_required_structure() { + let mut report = Report::default(); + let bytes = write(&sound_only(Some(vec![7; 480 * 4])), &IDENTITY, &mut report).unwrap(); + assert!(!report.has_warnings(), "{report}"); + // label + let l = &bytes[bytes.len() - bento::LABEL_SIZE..]; + assert_eq!(&l[..8], &bento::MAGIC); + assert_eq!(u16::from_be_bytes([l[12], l[13]]), 1, "Bento major version"); + let c = Container::open(&bytes).unwrap(); + let class = |o: u32| c.get(o, "OMFI:OOBJ:ObjClass").map(|v| String::from_utf8_lossy(&v).to_string()); + let mut by_class: HashMap> = HashMap::new(); + for &o in c.objects.keys() { + if let Some(k) = class(o) { + by_class.entry(k).or_default().push(o); + } + } + assert_required_properties(&c, &by_class); + assert_eq!(by_class["HEAD"].len(), 1); + assert_eq!(by_class["CMOB"].len(), 1); + let head = by_class["HEAD"][0]; + assert_eq!(c.get(head, "OMFI:HEAD:Version"), Some(vec![2, 0])); + assert_eq!(c.get(head, "OMFI:HEAD:ByteOrder"), Some(b"MM".to_vec())); + // object references resolve to objects of the right class + let refs = |o: u32, p: &str| -> Vec { + let v = c.get(o, p).unwrap(); + let n = u16::from_be_bytes([v[0], v[1]]) as usize; + v[2..].as_chunks::<4>().0.iter().take(n).map(|k| c.resolve(o, u32::from_be_bytes([k[0], k[1], k[2], k[3]]))).collect() + }; + let one = |o: u32, p: &str| -> u32 { c.resolve(o, u32::from_be_bytes(c.get(o, p).unwrap()[..4].try_into().unwrap())) }; + for m in refs(head, "OMFI:HEAD:Mobs") { + assert!(matches!(class(m).as_deref(), Some("CMOB" | "MMOB" | "SMOB"))); + for s in refs(m, "OMFI:MOBJ:Slots") { + assert_eq!(class(s).as_deref(), Some("MSLT")); + assert_eq!(class(one(s, "OMFI:MSLT:TrackDesc")).as_deref(), Some("TRKD")); + } + } + // embedded media: WAVE data is a WAVE file of the file mob it names + for w in &by_class["WAVE"] { + let data = c.get(*w, "OMFI:WAVE:Data").unwrap(); + let (pcm, ch, sr, bits) = crate::pcm::parse_wav(&data).unwrap(); + assert_eq!((pcm.len(), ch, sr, bits), (480 * 4, 2, 48_000, 16)); + let id = c.get(*w, "OMFI:MDAT:MobID").unwrap(); + assert!(by_class["SMOB"].iter().any(|s| c.get(*s, "OMFI:MOBJ:MobID") == Some(id.clone()))); + } + // the composition's sound sequence: clip, transition, gain effect around a clip + let cm = by_class["CMOB"][0]; + let slots = refs(cm, "OMFI:MOBJ:Slots"); + assert_eq!(class(one(slots[0], "OMFI:MSLT:Segment")).as_deref(), Some("TCCP")); + let seq = one(slots[1], "OMFI:MSLT:Segment"); + let comps = refs(seq, "OMFI:SEQU:Components"); + let classes: Vec = comps.iter().map(|&o| class(o).unwrap()).collect(); + assert_eq!(classes, vec!["EFFE", "TRAN", "SCLP"]); + let len = |o: u32| i32::from_be_bytes(c.get(o, "OMFI:CPNT:Length").unwrap()[..4].try_into().unwrap()) as i64; + let samples = |f: i64| f * 48_000 / 25; + assert_eq!(len(comps[0]), samples(55)); + assert_eq!(len(comps[1]), samples(10)); + assert_eq!(len(comps[2]), samples(55)); + assert_eq!(len(seq), samples(100)); + assert_eq!(c.get(one(comps[0], "OMFI:EFFE:EffectKind"), "OMFI:EDEF:EffectID"), Some(b"omfi:effect:MonoAudioGain\0".to_vec())); +} + +#[test] +fn bento_values_round_trip() { + let mut w = bento::Writer::new(); + let a = w.new_object(); + let b = w.new_object(); + w.set(a, "Test:Small", "test:Bytes", &[1, 2]); + w.set(a, "Test:Empty", "test:Bytes", &[]); + w.set(b, "Test:Big", "test:Bytes", &vec![9; 100_000]); + w.set_references(a, &[b]); + let bytes = w.finish(); + let c = Container::open(&bytes).unwrap(); + assert_eq!(c.get(a, "Test:Small"), Some(vec![1, 2])); + assert_eq!(c.get(a, "Test:Empty"), Some(vec![])); + assert_eq!(c.get(b, "Test:Big").map(|v| v.len()), Some(100_000)); + assert_eq!(c.resolve(a, b), b); + assert_eq!(c.name_of(c.id_of("Test:Big").unwrap()), Some("Test:Big")); + assert!(Container::open(&bytes[..bytes.len() - 1]).is_err()); +} diff --git a/crates/aaf/src/omf/write.rs b/crates/aaf/src/omf/write.rs new file mode 100644 index 000000000..755994804 --- /dev/null +++ b/crates/aaf/src/omf/write.rs @@ -0,0 +1,419 @@ +//! Composition model → OMF 2.0 objects in a Bento container. + +use std::collections::HashMap; + +use craft_timeline::time::Tick; + +use super::bento; +use crate::model::{CItem, CKind, Document, Gain, Source, ticks_to_units}; +use crate::{Error, Identity, Report, Result}; + +/// An OMF mob id (`omfi:UID`): prefix, major, minor. +pub(crate) type Uid = [u8; 12]; + +pub(crate) fn uid(prefix: u32, major: u32, minor: u32) -> Uid { + let mut u = [0u8; 12]; + u[..4].copy_from_slice(&prefix.to_be_bytes()); + u[4..8].copy_from_slice(&major.to_be_bytes()); + u[8..].copy_from_slice(&minor.to_be_bytes()); + u +} + +struct W<'a> { + b: bento::Writer, + refs: HashMap>, + ddefs: HashMap<&'static str, u32>, + edefs: HashMap<&'static str, u32>, + defs: Vec, + report: &'a mut Report, + inexact: bool, +} + +fn rational(n: i64, d: i64) -> Vec { + let mut v = (n.clamp(i32::MIN as i64, i32::MAX as i64) as i32).to_be_bytes().to_vec(); + v.extend_from_slice(&(d.clamp(i32::MIN as i64, i32::MAX as i64) as i32).to_be_bytes()); + v +} + +fn cstr(s: &str) -> Vec { + let mut v = s.as_bytes().to_vec(); + v.push(0); + v +} + +impl W<'_> { + fn obj(&mut self, class: &[u8; 4]) -> u32 { + let id = self.b.new_object(); + self.b.set(id, "OMFI:OOBJ:ObjClass", "omfi:ClassID", class); + id + } + fn str(&mut self, o: u32, p: &str, v: &str) { + self.b.set(o, p, "omfi:String", &cstr(v)); + } + fn i32(&mut self, o: u32, p: &str, ty: &str, v: i64) { + if v > i32::MAX as i64 || v < i32::MIN as i64 { + self.report.warn("a time does not fit OMF's 32-bit positions and was clamped"); + } + self.b.set(o, p, ty, &(v.clamp(i32::MIN as i64, i32::MAX as i64) as i32).to_be_bytes()); + } + fn r#ref(&mut self, o: u32, p: &str, target: u32) { + self.b.set(o, p, "omfi:ObjRef", &target.to_be_bytes()); + self.refs.entry(o).or_default().push(target); + } + fn refs(&mut self, o: u32, p: &str, targets: &[u32]) { + let mut v = (targets.len() as u16).to_be_bytes().to_vec(); + for t in targets { + v.extend_from_slice(&t.to_be_bytes()); + } + self.b.set(o, p, "omfi:ObjRefArray", &v); + self.refs.entry(o).or_default().extend_from_slice(targets); + } + fn u(&mut self, t: Tick, rate: (i64, i64)) -> i64 { + let (n, exact) = ticks_to_units(t, rate.0, rate.1); + if !exact { + self.inexact = true; + } + n + } + fn ddef(&mut self, kind: &'static str) -> u32 { + if let Some(&d) = self.ddefs.get(kind) { + return d; + } + let d = self.obj(b"DDEF"); + self.b.set(d, "OMFI:DDEF:DataKindID", "omfi:UniqueName", &cstr(kind)); + self.ddefs.insert(kind, d); + self.defs.push(d); + d + } + fn edef(&mut self, id: &'static str, name: &str) -> u32 { + if let Some(&d) = self.edefs.get(id) { + return d; + } + let d = self.obj(b"EDEF"); + self.b.set(d, "OMFI:EDEF:EffectID", "omfi:UniqueName", &cstr(id)); + self.str(d, "OMFI:EDEF:EffectName", name); + self.b.set(d, "OMFI:EDEF:Bypass", "omfi:ArgIDType", &(-1i32).to_be_bytes()); + self.edefs.insert(id, d); + self.defs.push(d); + d + } + fn cpnt(&mut self, o: u32, kind: &'static str, len: i64) { + let d = self.ddef(kind); + self.r#ref(o, "OMFI:CPNT:DataKind", d); + self.i32(o, "OMFI:CPNT:Length", "omfi:Length32", len); + } + fn mob(&mut self, class: &[u8; 4], id: Uid, name: &str, slots: &[u32]) -> u32 { + let m = self.obj(class); + self.b.set(m, "OMFI:MOBJ:MobID", "omfi:UID", &id); + self.str(m, "OMFI:MOBJ:Name", name); + self.refs(m, "OMFI:MOBJ:Slots", slots); + self.b.set(m, "OMFI:MOBJ:LastModified", "omfi:TimeStamp", &[0, 0, 0, 0, 1]); + self.b.set(m, "OMFI:MOBJ:CreationTime", "omfi:TimeStamp", &[0, 0, 0, 0, 1]); + m + } + fn slot(&mut self, rate: (i64, i64), segment: u32, track_id: u32, name: &str, physical: u32) -> u32 { + let t = self.obj(b"TRKD"); + self.i32(t, "OMFI:TRKD:Origin", "omfi:Position32", 0); + self.b.set(t, "OMFI:TRKD:TrackID", "omfi:UInt32", &track_id.to_be_bytes()); + self.str(t, "OMFI:TRKD:TrackName", name); + if physical > 0 { + self.b.set(t, "OMFI:TRKD:PhysicalTrack", "omfi:UInt32", &physical.to_be_bytes()); + } + let s = self.obj(b"MSLT"); + self.b.set(s, "OMFI:MSLT:EditRate", "omfi:Rational", &rational(rate.0, rate.1)); + self.r#ref(s, "OMFI:MSLT:Segment", segment); + self.r#ref(s, "OMFI:MSLT:TrackDesc", t); + s + } + fn sclp(&mut self, kind: &'static str, len: i64, source: Uid, track: u32, start: i64) -> u32 { + let c = self.obj(b"SCLP"); + self.cpnt(c, kind, len); + self.b.set(c, "OMFI:SCLP:SourceID", "omfi:UID", &source); + self.b.set(c, "OMFI:SCLP:SourceTrackID", "omfi:UInt32", &track.to_be_bytes()); + self.i32(c, "OMFI:SCLP:StartTime", "omfi:Position32", start); + c + } + fn tccp(&mut self, len: i64, start: i64, fps: i64, drop: bool) -> u32 { + let t = self.obj(b"TCCP"); + self.cpnt(t, "omfi:data:Timecode", len); + self.i32(t, "OMFI:TCCP:Start", "omfi:Position32", start); + self.b.set(t, "OMFI:TCCP:FPS", "omfi:UInt16", &(fps.clamp(1, u16::MAX as i64) as u16).to_be_bytes()); + self.b.set(t, "OMFI:TCCP:Drop", "omfi:Boolean", &[drop as u8]); + t + } + fn value(&mut self, len: i64, g: &Gain, seg_len: i64, rate: (i64, i64), base: Tick) -> u32 { + match g { + Gain::Constant(a) => { + let v = self.obj(b"CVAL"); + self.cpnt(v, "omfi:data:Rational", len); + self.b.set(v, "OMFI:CVAL:Value", "omfi:DataValue", &(*a)); + v + } + Gain::Varying { linear, points } => { + let mut pts = Vec::new(); + for (off, a) in points { + let o = self.u(base + *off, rate) - self.u(base, rate); + let p = self.obj(b"CTLP"); + self.b.set(p, "OMFI:CTLP:Time", "omfi:Rational", &rational(o, seg_len.max(1))); + self.b.set(p, "OMFI:CTLP:Value", "omfi:DataValue", &(*a)); + self.b.set(p, "OMFI:CTLP:EditHint", "omfi:EditHintType", &[0]); + pts.push(p); + } + let v = self.obj(b"VVAL"); + self.cpnt(v, "omfi:data:Rational", len); + self.b.set(v, "OMFI:VVAL:Interpolation", "omfi:InterpKind", &(if *linear { 2u16 } else { 1u16 }).to_be_bytes()); + self.refs(v, "OMFI:VVAL:PointList", &pts); + v + } + } + } +} + +fn fnv(s: &[u8], h: u64) -> u64 { + s.iter().fold(h, |h, &b| (h ^ b as u64).wrapping_mul(0x0000_0100_0000_01B3)) +} + +fn amp(a: f64) -> Vec { + rational((a.max(0.0) * 65536.0).round().min(i32::MAX as f64) as i64, 65536) +} + +fn rate_of(s: &Source) -> (i64, i64) { + match s.kind { + CKind::Picture => (s.frame_rate.num, s.frame_rate.den), + CKind::Sound => (s.sample_rate.max(1) as i64, 1), + } +} + +pub(crate) fn write(doc: &Document, identity: &Identity<'_>, report: &mut Report) -> Result> { + let comp = doc.compositions.first().ok_or(Error::Empty)?; + let mut w = + W { b: bento::Writer::new(), refs: HashMap::new(), ddefs: HashMap::new(), edefs: HashMap::new(), defs: Vec::new(), report, inexact: false }; + let mut seed = fnv(comp.name.as_bytes(), 0xCBF2_9CE4_8422_2325); + for s in &doc.sources { + seed = fnv(s.path.as_deref().unwrap_or("").as_bytes(), fnv(s.key.as_bytes(), seed)); + } + let major = (seed >> 32) as u32 ^ seed as u32; + let mut minor = 0u32; + let mut new_uid = || { + minor += 1; + uid(0x464C_4D43, major, minor) + }; + let mut mobs = Vec::new(); + let mut media_data = Vec::new(); + let mut master_ref: HashMap = HashMap::new(); + // media + let mut groups: Vec<(String, Vec)> = Vec::new(); + for (i, s) in doc.sources.iter().enumerate() { + if s.kind != CKind::Sound { + continue; + } + match groups.iter_mut().find(|(k, _)| *k == s.key) { + Some((_, v)) => v.push(i), + None => groups.push((s.key.clone(), vec![i])), + } + } + for (_, members) in &groups { + let first = &doc.sources[members[0]]; + let master = new_uid(); + let tc = members.iter().find_map(|&i| doc.sources[i].start_tc.map(|t| (t, doc.sources[i].tc_rate))); + let tape = new_uid(); + let mut tape_slots = Vec::new(); + if let Some((start, r)) = tc { + let len = members.iter().map(|&i| doc.sources[i].length).max().unwrap_or(Tick::ZERO); + let l = w.u(len, (r.num, r.den)); + let t = w.tccp(l, start, r.timecode_base(), false); + tape_slots.push(w.slot((r.num, r.den), t, 1, "TC1", 0)); + } + let mut master_slots = Vec::new(); + for (k, &si) in members.iter().enumerate() { + let s = &doc.sources[si]; + let rate = rate_of(s); + let len = w.u(s.length, rate); + // file mob + let file = new_uid(); + let file_track = s.channel.map_or(1, |c| c + 1); + let tape_track = 2 + k as u32; + let to_tape = + if tc.is_some() { w.sclp("omfi:data:Sound", len, tape, tape_track, 0) } else { w.sclp("omfi:data:Sound", len, uid(0, 0, 0), 0, 0) }; + let fslot = w.slot(rate, to_tape, file_track, &format!("A{file_track}"), file_track); + let desc = descriptor(&mut w, s, len); + let fm = w.mob(b"SMOB", file, &s.name, &[fslot]); + w.r#ref(fm, "OMFI:SMOB:MediaDescription", desc); + mobs.push(fm); + if let Some(pcm) = &s.embedded { + let md = w.obj(b"WAVE"); + w.b.set(md, "OMFI:MDAT:MobID", "omfi:UID", &file); + w.b.set( + md, + "OMFI:WAVE:Data", + "omfi:DataValue", + &craft_audio::pcm_wav_file(pcm, u16::try_from(s.file_channels.max(1)).unwrap_or(u16::MAX), s.sample_rate, s.bits), + ); + media_data.push(md); + } + if tc.is_some() { + let c = w.sclp("omfi:data:Sound", len, uid(0, 0, 0), 0, 0); + tape_slots.push(w.slot(rate, c, tape_track, &format!("A{}", k + 1), k as u32 + 1)); + } + let mclip = w.sclp("omfi:data:Sound", len, file, file_track, 0); + master_slots.push(w.slot(rate, mclip, k as u32 + 1, &format!("A{}", k + 1), k as u32 + 1)); + master_ref.insert(si, (master, k as u32 + 1)); + } + mobs.push(w.mob(b"MMOB", master, &first.name, &master_slots)); + if tc.is_some() { + let d = w.obj(b"MDTP"); + let reel = first.path.as_deref().map(craft_timeline::path::file_stem).unwrap_or(&first.name).to_string(); + let tm = w.mob(b"SMOB", tape, &reel, &tape_slots); + w.r#ref(tm, "OMFI:SMOB:MediaDescription", d); + mobs.push(tm); + } + } + // composition + let crate_rate = (comp.rate.num, comp.rate.den); + let mut slots = Vec::new(); + let total = comp + .tracks + .iter() + .map(|t| t.items.iter().fold(Tick::ZERO, |a, i| if matches!(i, CItem::Transition(_)) { a - i.len() } else { a + i.len() })) + .max() + .unwrap_or(Tick::ZERO); + let tl = w.u(total, crate_rate); + let tcc = w.tccp(tl, comp.start_tc, comp.rate.timecode_base(), comp.drop); + slots.push(w.slot(crate_rate, tcc, 1, "TC1", 0)); + for (track_id, t) in (2..).zip(comp.tracks.iter().filter(|t| t.kind == CKind::Sound)) { + let rate = (comp.sample_rate.max(1) as i64, 1); + let mut comps = Vec::new(); + let mut cursor = Tick::ZERO; + for it in &t.items { + match it { + CItem::Filler(l) => { + let n = w.u(cursor + *l, rate) - w.u(cursor, rate); + cursor += *l; + let f = w.obj(b"FILL"); + w.cpnt(f, "omfi:data:Sound", n); + comps.push(f); + } + CItem::Clip(c) => { + let n = w.u(cursor + c.len, rate) - w.u(cursor, rate); + let (mid, mtrack) = master_ref.get(&c.source).copied().unwrap_or((uid(0, 0, 0), 0)); + let start = w.u(c.start, rate); + let sc = w.sclp("omfi:data:Sound", n, mid, mtrack, start); + if !c.name.is_empty() { + w.str(sc, "FilmCraft:SCLP:ClipName", &c.name); + } + let seg = match &c.gain { + Some(g) => { + let v = w.value(n, g, n, rate, cursor); + let input = w.obj(b"ESLT"); + w.b.set(input, "OMFI:ESLT:ArgID", "omfi:ArgIDType", &(-1i32).to_be_bytes()); + w.r#ref(input, "OMFI:ESLT:ArgValue", sc); + let level = w.obj(b"ESLT"); + w.b.set(level, "OMFI:ESLT:ArgID", "omfi:ArgIDType", &1i32.to_be_bytes()); + w.r#ref(level, "OMFI:ESLT:ArgValue", v); + let kind = w.edef("omfi:effect:MonoAudioGain", "Mono Audio Gain"); + let e = w.obj(b"EFFE"); + w.cpnt(e, "omfi:data:Sound", n); + w.r#ref(e, "OMFI:EFFE:EffectKind", kind); + w.refs(e, "OMFI:EFFE:EffectSlots", &[input, level]); + e + } + None => sc, + }; + comps.push(seg); + cursor += c.len; + } + CItem::Transition(x) => { + let ts = cursor - x.len; + let n = w.u(cursor, rate) - w.u(ts, rate); + let cut = w.u(ts + x.cut, rate) - w.u(ts, rate); + cursor = ts; + let kind = w.edef("omfi:effect:SimpleMonoAudioDissolve", "Simple Mono Audio Dissolve"); + let e = w.obj(b"EFFE"); + w.cpnt(e, "omfi:data:Sound", n); + w.r#ref(e, "OMFI:EFFE:EffectKind", kind); + w.refs(e, "OMFI:EFFE:EffectSlots", &[]); + w.str(e, "FilmCraft:EFFE:EffectID", &x.effect); + let tr = w.obj(b"TRAN"); + w.cpnt(tr, "omfi:data:Sound", n); + w.i32(tr, "OMFI:TRAN:CutPoint", "omfi:Position32", cut); + w.r#ref(tr, "OMFI:TRAN:Effect", e); + comps.push(tr); + } + } + } + let len = w.u(cursor.max(Tick::ZERO), rate); + let seq = w.obj(b"SEQU"); + w.cpnt(seq, "omfi:data:Sound", len); + w.refs(seq, "OMFI:SEQU:Components", &comps); + slots.push(w.slot(rate, seq, track_id, &t.name, t.number)); + } + let cmob = w.mob(b"CMOB", new_uid(), &comp.name, &slots); + mobs.insert(0, cmob); + if w.inexact { + w.report.info("some times are not on whole samples / frames and were rounded"); + } + let ident = w.obj(b"IDNT"); + w.str(ident, "OMFI:IDNT:CompanyName", identity.company); + w.str(ident, "OMFI:IDNT:ProductName", identity.product); + w.str(ident, "OMFI:IDNT:ProductVersionString", identity.version); + w.str(ident, "OMFI:IDNT:Platform", identity.platform); + w.b.set(ident, "OMFI:IDNT:Date", "omfi:TimeStamp", &[0, 0, 0, 0, 1]); + let head = w.obj(b"HEAD"); + w.b.set(head, "OMFI:HEAD:ByteOrder", "omfi:Int16", &0x4D4Di16.to_be_bytes()); + w.b.set(head, "OMFI:HEAD:LastModified", "omfi:TimeStamp", &[0, 0, 0, 0, 1]); + w.b.set(head, "OMFI:HEAD:Version", "omfi:VersionType", &[2, 0]); + w.refs(head, "OMFI:HEAD:Mobs", &mobs); + w.refs(head, "OMFI:HEAD:MediaData", &media_data); + w.refs(head, "OMFI:HEAD:PrimaryMobs", &[cmob]); + let defs = w.defs.clone(); + w.refs(head, "OMFI:HEAD:DefinitionObjects", &defs); + w.refs(head, "OMFI:HEAD:IdentificationList", &[ident]); + let refs = std::mem::take(&mut w.refs); + let mut keys: Vec<_> = refs.keys().copied().collect(); + keys.sort_unstable(); + for k in keys { + let mut t = refs[&k].clone(); + t.sort_unstable(); + t.dedup(); + w.b.set_references(k, &t); + } + Ok(w.b.finish()) +} + +fn descriptor(w: &mut W, s: &Source, len: i64) -> u32 { + let aiff = + s.embedded.is_none() && s.path.as_deref().is_some_and(|p| matches!(craft_timeline::path::extension(p).as_str(), "aif" | "aiff" | "aifc")); + let d = w.obj(if aiff { b"AIFD" } else { b"WAVD" }); + w.b.set(d, "OMFI:MDFL:IsOMFI", "omfi:Boolean", &[s.embedded.is_some() as u8]); + let r = rate_of(s); + w.b.set(d, "OMFI:MDFL:SampleRate", "omfi:Rational", &rational(r.0, r.1)); + w.i32(d, "OMFI:MDFL:Length", "omfi:Length32", len); + let frames = u32::try_from(len.max(0)).unwrap_or(u32::MAX); + // Channel counts come from the project: saturate, as the headers can't hold more anyway. + let bytes_per_frame = s.file_channels.max(1).saturating_mul((s.bits as u32).div_ceil(8)); + let channels = u16::try_from(s.file_channels.max(1)).unwrap_or(u16::MAX); + if aiff { + w.b.set(d, "OMFI:AIFD:Summary", "omfi:DataValue", &craft_audio::pcm_aiff_header(frames, channels, s.sample_rate, s.bits)); + } else { + w.b.set( + d, + "OMFI:WAVD:Summary", + "omfi:DataValue", + &craft_audio::pcm_wav_header(frames.saturating_mul(bytes_per_frame), channels, s.sample_rate, s.bits), + ); + } + if let Some(p) = &s.path { + let mut locs = Vec::new(); + let dos = p.len() > 2 && p.as_bytes()[1] == b':'; + let l = w.obj(if dos { b"DOSL" } else { b"UNXL" }); + w.str(l, if dos { "OMFI:DOSL:PathName" } else { "OMFI:UNXL:PathName" }, p); + locs.push(l); + if craft_timeline::path::is_absolute(p) { + let n = w.obj(b"NETL"); + w.str(n, "OMFI:NETL:URLString", &craft_timeline::path::path_to_file_url(p, false)); + locs.push(n); + } + w.refs(d, "OMFI:MDES:Locator", &locs); + } + d +} diff --git a/crates/aaf/src/pcm.rs b/crates/aaf/src/pcm.rs new file mode 100644 index 000000000..789c976a8 --- /dev/null +++ b/crates/aaf/src/pcm.rs @@ -0,0 +1,11 @@ +//! Embedded PCM essence in and out of WAVE and AIFF wrappers (craft-audio does the containers). + +/// PCM (little-endian, 8-bit unsigned), channels, sample rate and bits of a WAVE file. +pub(crate) fn parse_wav(b: &[u8]) -> Option<(Vec, u16, u32, u16)> { + craft_audio::wav_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits)) +} + +/// The same for an AIFF / AIFF-C (`NONE`, `twos`, `sowt`) file. +pub(crate) fn parse_aiff(b: &[u8]) -> Option<(Vec, u16, u32, u16)> { + craft_audio::aiff_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits)) +} diff --git a/crates/aaf/src/roundtrip_tests.rs b/crates/aaf/src/roundtrip_tests.rs new file mode 100644 index 000000000..7ba6e993a --- /dev/null +++ b/crates/aaf/src/roundtrip_tests.rs @@ -0,0 +1,251 @@ +//! Documents written and read back, in both formats; damaged files. + +use craft_timeline::time::{FrameRate, Tick}; + +use crate::model::{CItem, CKind, CMarker, Composition, Document, Gain}; +use crate::testdoc::{IDENTITY, pcm_ramp, picture_and_sound, sound_only}; +use crate::{Error, Report, aaf, omf}; + +fn aaf_bytes(doc: &Document, small_sectors: bool) -> Vec { + let mut report = Report::default(); + let bytes = aaf::write(doc, &aaf::WriteOptions { identity: IDENTITY, small_sectors }, &mut report).unwrap(); + assert!(report.is_empty(), "{report}"); + bytes +} + +fn omf_bytes(doc: &Document) -> Vec { + let mut report = Report::default(); + let bytes = omf::write(doc, &IDENTITY, &mut report).unwrap(); + assert!(!report.has_warnings(), "{report}"); + bytes +} + +fn read_clean(bytes: &[u8], is_aaf: bool) -> Document { + let mut report = Report::default(); + let doc = if is_aaf { aaf::read(bytes, &mut report) } else { omf::read(bytes, &mut report) }.unwrap(); + assert!(!report.has_warnings(), "{report}"); + doc +} + +/// What a track holds, with each clip's source by what it is rather than by index. Gain is stored +/// as a rational: four decimals survive. +fn shape(doc: &Document) -> Vec<(CKind, String, u32, Vec)> { + let comp = &doc.compositions[0]; + comp.tracks + .iter() + .map(|t| { + let items = t + .items + .iter() + .map(|i| match i { + CItem::Filler(l) => format!("gap {}", l.0), + CItem::Transition(x) => format!("{} {} cut {}", x.effect, x.len.0, x.cut.0), + CItem::Clip(c) => { + let s = &doc.sources[c.source]; + let gain = match &c.gain { + None => "unity".to_string(), + Some(Gain::Constant(g)) => format!("{g:.4}"), + Some(Gain::Varying { linear, points }) => { + format!("{linear} {:?}", points.iter().map(|(t, g)| (t.0, format!("{g:.4}"))).collect::>()) + } + }; + format!("clip {} from {} of {:?} {:?} gain {gain}", c.len.0, c.start.0, s.path, s.kind) + } + }) + .collect(); + (t.kind, t.name.clone(), t.number, items) + }) + .collect() +} + +#[test] +fn aaf_round_trips_the_edit_at_every_rate() { + for (rate, drop) in [(FrameRate::FPS_25, false), (FrameRate::FPS_29_97, true), (FrameRate::FPS_23_976, false), (FrameRate::FPS_59_94, true)] { + let mut doc = picture_and_sound(rate); + let comp = &mut doc.compositions[0]; + (comp.start_tc, comp.drop) = (rate.timecode_base() * 3600, drop); + comp.markers = vec![ + CMarker { start: rate.tick_of(10), duration: Tick::ZERO, name: "Slate".into(), comment: "take 2".into(), color: Some("Green".into()) }, + CMarker { start: rate.tick_of(40), duration: rate.tick_of(5), name: "Range".into(), comment: String::new(), color: None }, + ]; + // keyframed gain on the first sound clip, held gain steps on nothing else + if let Some(CItem::Clip(c)) = comp.tracks[1].items.first_mut() { + c.gain = Some(Gain::Varying { linear: true, points: vec![(Tick::ZERO, 1.0), (rate.tick_of(25), 0.5)] }); + } + doc.sources[0].markers = + vec![CMarker { start: rate.tick_of(30), duration: Tick::ZERO, name: "In media".into(), comment: String::new(), color: None }]; + for small in [false, true] { + let back = read_clean(&aaf_bytes(&doc, small), true); + let (want, got) = (&doc.compositions[0], &back.compositions[0]); + assert_eq!(shape(&back), shape(&doc), "{rate} small {small}"); + assert_eq!((&got.name, got.rate, got.start_tc, got.drop), (&want.name, want.rate, want.start_tc, want.drop), "{rate}"); + assert_eq!((got.width, got.height, got.sample_rate), (1920, 1080, 48_000), "{rate}"); + let markers = |m: &[CMarker]| m.iter().map(|m| (m.start, m.duration, m.name.clone(), m.comment.clone())).collect::>(); + assert_eq!(markers(&got.markers), markers(&want.markers), "{rate}"); + assert_eq!(back.nested, []); + let picture = back.sources.iter().find(|s| s.kind == CKind::Picture).unwrap(); + assert_eq!((picture.width, picture.height, picture.frame_rate, picture.length), (1280, 720, rate, doc.sources[0].length), "{rate}"); + assert_eq!((picture.start_tc, picture.tc_rate), (Some(86_400), rate), "{rate}"); + assert_eq!(markers(&picture.markers), markers(&doc.sources[0].markers), "{rate}"); + let sound = back.sources.iter().find(|s| s.kind == CKind::Sound).unwrap(); + assert_eq!((sound.channels, sound.file_channels, sound.sample_rate, sound.bits), (2, 2, 48_000, 24), "{rate}"); + assert_eq!(sound.key, picture.key, "one media item"); + // what was read, written again, reads as the same edit (mob ids, and so source keys, are new; + // a writer generates fresh format identifiers and never preserves `aaf_mob_id`) + let again = read_clean(&aaf_bytes(&back, small), true); + let no_ids = |c: &[Composition]| { + c.iter() + .cloned() + .map(|mut c| { + c.aaf_mob_id = None; + c + }) + .collect::>() + }; + assert_eq!((shape(&again), no_ids(&again.compositions)), (shape(&back), no_ids(&back.compositions)), "{rate}"); + } + } +} + +#[test] +fn embedded_audio_comes_back_bit_for_bit() { + let pcm = pcm_ramp(480, 2, 3); + let mut doc = sound_only(Some(pcm.clone())); + // trimmed essence: its first sample is media time 0.2 s + doc.sources[0].offset = Tick::from_units(9_600, 48_000); + for is_aaf in [true, false] { + let bytes = if is_aaf { aaf_bytes(&doc, false) } else { omf_bytes(&doc) }; + assert_eq!((aaf::sniff(&bytes), omf::sniff(&bytes)), (is_aaf, !is_aaf)); + let back = read_clean(&bytes, is_aaf); + let s = &back.sources[0]; + assert_eq!(s.embedded.as_deref(), Some(&pcm[..]), "aaf {is_aaf}"); + assert_eq!((s.path.as_deref(), s.file_channels, s.sample_rate, s.bits), (None, 2, 48_000, 16), "aaf {is_aaf}"); + assert_eq!(s.length, Tick::from_units(480, 48_000), "aaf {is_aaf}"); + assert_eq!(shape(&back), shape(&doc), "aaf {is_aaf}"); + } +} + +#[test] +fn omf_round_trips_sound_linked_to_wave_and_aiff_files() { + for path in ["/m/a.wav", "/m/a.aif"] { + let mut doc = sound_only(None); + doc.sources[0].path = Some(path.into()); + let back = read_clean(&omf_bytes(&doc), false); + assert_eq!(shape(&back), shape(&doc), "{path}"); + let s = &back.sources[0]; + assert_eq!((s.file_channels, s.sample_rate, s.bits, s.embedded.as_ref()), (2, 48_000, 16, None), "{path}"); + assert_eq!(back.compositions[0].rate, FrameRate::FPS_25); + } +} + +#[test] +fn omf_leaves_picture_out() { + let doc = picture_and_sound(FrameRate::FPS_25); + let back = read_clean(&omf_bytes(&doc), false); + assert!(back.compositions[0].tracks.iter().all(|t| t.kind == CKind::Sound)); + assert_eq!(shape(&back).len(), 1); +} + +#[test] +fn empty_and_foreign_documents() { + let none = Document::default(); + assert_eq!(aaf::write(&none, &aaf::WriteOptions { identity: IDENTITY, small_sectors: false }, &mut Report::default()), Err(Error::Empty)); + assert_eq!(omf::write(&none, &IDENTITY, &mut Report::default()), Err(Error::Empty)); + let mut empty = sound_only(None); + empty.compositions[0].tracks.clear(); + empty.sources.clear(); + for small in [false, true] { + assert_eq!(read_clean(&aaf_bytes(&empty, small), true).compositions.len(), 1); + } + assert_eq!(read_clean(&omf_bytes(&empty), false).compositions.len(), 1); + assert!(matches!(aaf::read(b"not an aaf", &mut Report::default()), Err(Error::Parse { format: "AAF", .. }))); + assert!(matches!(omf::read(b"not an omf", &mut Report::default()), Err(Error::Parse { format: "OMF", .. }))); + assert!(!aaf::sniff(b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1 but short")); + assert!(!omf::sniff(b"")); +} + +/// Truncation and random corruption: an error or a document, never a panic. +#[test] +fn damaged_documents_never_panic() { + let aaf_file = aaf_bytes(&picture_and_sound(FrameRate::FPS_25), true); + let omf_file = omf_bytes(&sound_only(Some(pcm_ramp(100, 2, 1)))); + for (bytes, is_aaf) in [(&aaf_file, true), (&omf_file, false)] { + let read = |b: &[u8]| { + let mut report = Report::default(); + let _ = if is_aaf { aaf::read(b, &mut report) } else { omf::read(b, &mut report) }; + }; + for cut in (0..bytes.len()).step_by(97) { + read(&bytes[..cut]); + } + let mut seed = 0xA5A5_1234_5678_9ABCu64; + for _ in 0..400 { + let mut g = bytes.clone(); + for _ in 0..8 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let at = (seed as usize) % g.len(); + g[at] = (seed >> 40) as u8; + } + read(&g); + } + } +} + +/// A time that is not on a whole edit unit is rounded, and the report says so. +#[test] +fn off_grid_times_are_rounded_and_reported() { + let mut doc = picture_and_sound(FrameRate::FPS_25); + if let Some(CItem::Clip(c)) = doc.compositions[0].tracks[0].items.first_mut() { + c.start += Tick(7); + } + let mut report = Report::default(); + aaf::write(&doc, &aaf::WriteOptions { identity: IDENTITY, small_sectors: false }, &mut report).unwrap(); + assert!(report.mentions("rounded"), "{report}"); +} + +/// A composition that uses itself (a damaged file) is a gap with a warning, not a recursion. +#[test] +fn a_composition_nested_in_itself_is_a_gap() { + let rate = FrameRate::FPS_25; + let mut doc = picture_and_sound(rate); + let mut inner = doc.compositions[0].clone(); + inner.name = "Inner".into(); + let mut nest = doc.sources[0].clone(); + (nest.key, nest.path, nest.nested, nest.start_tc) = ("seq:9".into(), None, Some(0), None); + doc.sources.push(nest); + let uses_inner = CItem::Clip(crate::model::CClip { len: rate.tick_of(20), source: 2, start: Tick::ZERO, gain: None, name: "Inner".into() }); + inner.tracks[0].items = vec![uses_inner.clone()]; + doc.nested.push(inner); + doc.compositions[0].tracks[0].items = vec![uses_inner]; + let bytes = aaf::write(&doc, &aaf::WriteOptions { identity: IDENTITY, small_sectors: false }, &mut Report::default()).unwrap(); + let mut report = Report::default(); + let back = aaf::read(&bytes, &mut report).unwrap(); + assert!(report.mentions("nested in itself"), "{report}"); + assert_eq!(back.nested.len(), 1); + assert!(matches!(back.nested[0].tracks[0].items[..], [CItem::Filler(_)]), "{:?}", back.nested[0].tracks[0].items); +} + +#[test] +fn aaf_source_start_uses_referenced_slot_rate() { + let doc = sound_only(None); + let bytes = aaf_bytes(&doc, false); + let back = read_clean(&bytes, true); + let original = doc.compositions[0].tracks[0] + .items + .iter() + .find_map(|item| match item { + CItem::Clip(c) => Some(c.start), + _ => None, + }) + .unwrap(); + let actual = back.compositions[0].tracks[0] + .items + .iter() + .find_map(|item| match item { + CItem::Clip(c) => Some(c.start), + _ => None, + }) + .unwrap(); + assert_eq!(actual, original); +} diff --git a/crates/aaf/src/testdoc.rs b/crates/aaf/src/testdoc.rs new file mode 100644 index 000000000..c2740e1e7 --- /dev/null +++ b/crates/aaf/src/testdoc.rs @@ -0,0 +1,117 @@ +//! Documents the tests write: what an editor's two-clip sequence with a dissolve looks like in +//! the model. + +use craft_timeline::time::{FrameRate, Tick}; + +use crate::Identity; +use crate::model::{CClip, CItem, CKind, CTrack, CTransition, Composition, Document, Gain, Source}; + +pub(crate) const IDENTITY: Identity<'static> = + Identity { company: "Craft", product: "craft-aaf tests", version: "1.2.3", platform: "test", product_id: *b"craft-aaf-tests." }; + +/// A source of `kind` for the media item `key` at `path`: 1280×720 picture at `rate`, or stereo +/// 48 kHz sound of `bits` bits; 100 seconds long. +pub(crate) fn source(key: &str, path: &str, kind: CKind, rate: FrameRate, bits: u16, start_tc: Option) -> Source { + let sound = kind == CKind::Sound; + Source { + key: key.into(), + name: path.rsplit('/').next().unwrap_or(path).into(), + kind, + path: Some(path.into()), + channel: None, + channels: if sound { 2 } else { 0 }, + file_channels: if sound { 2 } else { 0 }, + width: if sound { 0 } else { 1280 }, + height: if sound { 0 } else { 720 }, + frame_rate: rate, + sample_rate: 48_000, + bits, + length: rate.tick_of(100 * rate.timecode_base()), + offset: Tick::ZERO, + start_tc, + tc_rate: rate, + embedded: None, + markers: Vec::new(), + nested: None, + } +} + +pub(crate) fn clip(rate: FrameRate, source: usize, start: i64, len: i64, name: &str) -> CItem { + CItem::Clip(CClip { len: rate.tick_of(len), source, start: rate.tick_of(start), gain: None, name: name.into() }) +} + +/// Two clips of `source` cut at frame `cut`, each `cut` frames long on the timeline, joined by a +/// transition of `dur` frames centred on the cut: each clip runs half the transition past the cut. +pub(crate) fn two_clips(rate: FrameRate, source: usize, cut: i64, dur: i64, (in_a, in_b): (i64, i64), effect: &str) -> Vec { + let half = dur / 2; + vec![ + clip(rate, source, in_a, cut + half, "a"), + CItem::Transition(CTransition { len: rate.tick_of(dur), cut: rate.tick_of(half), effect: effect.into() }), + clip(rate, source, in_b - half, cut + half, "a"), + ] +} + +pub(crate) fn composition(name: &str, rate: FrameRate, tracks: Vec) -> Composition { + Composition { + name: name.into(), + aaf_mob_id: None, + rate, + sample_rate: 48_000, + width: 1920, + height: 1080, + start_tc: 0, + drop: false, + tracks, + markers: Vec::new(), + } +} + +pub(crate) fn track(kind: CKind, name: &str, number: u32, items: Vec) -> CTrack { + CTrack { kind, name: name.into(), number, channels: if kind == CKind::Sound { 2 } else { 0 }, items } +} + +/// "S" at `rate`: V1 and A1 each hold two clips of `a.mov`, `cut` frames each on the timeline (from +/// source frames `ins`), with a transition of `dur` frames centred on the cut; the second sound +/// clip is 3 dB down. +pub(crate) fn picture_and_sound_cut(rate: FrameRate, cut: i64, dur: i64, ins: (i64, i64)) -> Document { + let sources = vec![ + source("item:1", "/m/a.mov", CKind::Picture, rate, 24, Some(86_400)), + source("item:1", "/m/a.mov", CKind::Sound, rate, 24, Some(86_400)), + ]; + let mut sound = two_clips(rate, 1, cut, dur, ins, "constant_power"); + if let Some(CItem::Clip(c)) = sound.last_mut() { + c.gain = Some(Gain::Constant(10f64.powf(-3.0 / 20.0))); + } + let tracks = vec![track(CKind::Picture, "V1", 1, two_clips(rate, 0, cut, dur, ins, "cross_dissolve")), track(CKind::Sound, "A1", 1, sound)]; + Document { compositions: vec![composition("S", rate, tracks)], sources, nested: Vec::new() } +} + +/// [`picture_and_sound_cut`] with 50-frame clips from source frames 10 and 205 and a 10-frame +/// transition: every time is a multiple of five frames, which is a whole number of 48 kHz samples +/// at the NTSC rates too. +pub(crate) fn picture_and_sound(rate: FrameRate) -> Document { + picture_and_sound_cut(rate, 50, 10, (10, 205)) +} + +/// 16-bit PCM: `frames` frames of `channels` channels, every sample different from its neighbours. +pub(crate) fn pcm_ramp(frames: usize, channels: usize, seed: i32) -> Vec { + (0..frames * channels).flat_map(|i| ((i as i32 * 37 + seed * 1000) as i16).to_le_bytes()).collect() +} + +/// "S" at 25 fps, sound only: A1 holds two 50-frame clips of `a.wav` (from source frames 10 and +/// 300) with a 10-frame crossfade centred on the cut; the first clip is 6 dB up. The sound is +/// embedded when `embedded` is given (480 frames of 16-bit stereo). +pub(crate) fn sound_only(embedded: Option>) -> Document { + let rate = FrameRate::FPS_25; + let mut s = source("item:1", "/m/a.wav", CKind::Sound, rate, 16, None); + if let Some(pcm) = embedded { + s.path = None; + s.length = Tick::from_units((pcm.len() / 4) as i64, 48_000); + s.embedded = Some(pcm); + } + let mut items = two_clips(rate, 0, 50, 10, (10, 300), "constant_power"); + if let Some(CItem::Clip(c)) = items.first_mut() { + c.gain = Some(Gain::Constant(10f64.powf(6.0 / 20.0))); + } + Document { compositions: vec![composition("S", rate, vec![track(CKind::Sound, "A1", 1, items)])], sources: vec![s], nested: Vec::new() } +} diff --git a/crates/aep/Cargo.toml b/crates/aep/Cargo.toml new file mode 100644 index 000000000..b0ebb9110 --- /dev/null +++ b/crates/aep/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "craft-aep" +description = "After Effects projects: the .aep (RIFX 'Egg!') and .aepx container as a lossless chunk tree with a typed, read-only view (compositions, layers, properties, keyframes, effects, masks, shapes, text, footage), bounded on hostile input" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +# The tool's own tests run with the crate's (examples are only built by default). +[[example]] +name = "stubicc" +test = true diff --git a/crates/aep/README.md b/crates/aep/README.md new file mode 100644 index 000000000..95fcdd1a6 --- /dev/null +++ b/crates/aep/README.md @@ -0,0 +1,104 @@ +# craft-aep + +After Effects projects (`.aep`, `.aepx`): the container as a lossless chunk tree, and a typed, +read-only view of what it holds. Bytes in, a neutral model out, and the tree back to bytes. +Mapping the view onto an application's document (which effect becomes which, units, what is +reported as lost) stays in the application. + +No file access: callers pass bytes, so the crate builds for wasm32. No dependencies. + +| Form | Read | Write | +|---|---|---| +| `.aep` (RIFX `Egg!`) | chunk tree and typed view | the chunk tree, byte for byte | +| `.aepx` (XML) | into the same chunk tree | no | + +## Container + +An `.aep` is a `RIFX` container (big-endian RIFF) with form type `Egg!`: chunks of a 4-byte id, a +big-endian `u32` size and a body padded to an even length; `LIST` chunks start with a 4-byte list +type followed by chunks. `parse` builds the tree, `write` serialises it, and for every input that +parses, `write(parse(x)) == x` byte for byte (container and chunk sizes are recomputed from the +tree; pad bytes, odd bodies and the bytes after the container, an XMP packet, are kept). + +The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a small bounded +XML tokenizer of its own); `parse_any` picks the reader by content, so the typed view takes +either form. The mapping is in [`docs/aep-format.md`](docs/aep-format.md), section 1. + +A reader that only looks does not need the owned tree. `TreeIndex` is the same tree as one flat, +read-only array in depth-first order, borrowed from the input: `TreeIndex::parse` for an `.aep`, +`index_aepx` and `TreeIndex::from_drafts` for an `.aepx` (decoded into one buffer first). It +counts the same chunks against the same `Limits` and returns the same errors as `parse`, and the +typed view reads through either (`Project::new` for the owned tree, `Project::from_index` for the +index). An importer should use the index; it is about twice as fast and allocates far less. + +Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay +opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining +input, nesting and chunk count are bounded (`Limits`), nothing is allocated from a declared size, +and truncated or damaged files return `Err` (or, inside the container, an opaque body). + +## Typed view + +The container layer decodes no chunk body. On top of it, a typed view (`Project`, `Item`, `Comp`, +`Layer`, `Property`, ...) reads the fields whose layout was verified against the values the test +files were made with: layer and composition headers (switches, exact times and time stretch, size, +frame rate, work area, shutter phase, display start, renderer), static transform values in After +Effects' units with the "absent = default" rule, keyframe records (interpolation sides, ease per +component, plain and spatial templates), Time Remapping, expressions with their enable switch, +effect instances and parameter values (with the settings data of Curves, Levels, Hue/Saturation, +Cryptomatte, EXtractoR, IDentifier and Puppet meshes), masks and bezier paths with tangents, shape +layer groups, text documents (paragraph and style runs, box, justification, tracking, faux styles, +caps, leading), markers with duration, strings and cue data, cameras and lights with their +options, track mattes, project settings and the working space's name, the version that saved the +file, footage (stills, audio, image sequences, the layer of a Photoshop file an item shows), solids +and precomp links. Every accessor returns +`Option` or an empty list on damaged data and never panics. Which fields are verified, which are +not decoded, and how it was found is in [`docs/aep-format.md`](docs/aep-format.md). + +Writing a project from an application's document (building every chunk in code) is not here: it +is EffectCraft's exporter, which is written against EffectCraft's own model. + +## Specifications + +Public specifications used for parts the format embeds: + +- ICC.1:2010: the profile description of a working-space profile (a v2 `desc` or a v4 `mluc` + tag), from which the working space's name is read (`settings.rs`). +- RFC 4648 (base64): the profile data in the project's colour settings JSON. + +## Clean room + +Everything here comes from our own observations: project files that we generated ourselves with +After Effects, one change per file, compared byte by byte. No source code or format +documentation of another `.aep` reader or writer was used, and no Adobe asset is used. + +## Tests + +The unit tests and the robustness tests build their input in code. The tests of the typed view +read generated projects (own test data): a few combined projects, one composition per case +([`tests/fixtures/CASES.md`](tests/fixtures/CASES.md)). They are not in this repository but in +[storytold/effectcraft-corpus](https://github.com/storytold/effectcraft-corpus), with their +attribution, the generation notes and the inspection of their embedded content. +`tests/fixtures/corpus.lock` pins the commit and lists the SHA-256 of every file used. To run +against them: + +```sh +git clone https://github.com/storytold/effectcraft-corpus.git +git -C effectcraft-corpus checkout +CRAFT_AEP_FIXTURES=$PWD/effectcraft-corpus/aep CRAFT_AEP_FIXTURES_REQUIRED=1 cargo test -p craft-aep +``` + +The tests reach the projects only through `tests/it/common/mod.rs`, which hands a test its case +as a project of its own cut from the combined file. Without `CRAFT_AEP_FIXTURES` such a test +prints `SKIPPED: fixture missing` and passes; with `CRAFT_AEP_FIXTURES_REQUIRED` set, a +missing project fails it. + +## Tools + +- `examples/aepdump.rs`: prints the chunk tree of a file (ids, sizes, short data, match names): + `cargo run -p craft-aep --example aepdump -- file.aep`. +- `examples/stubicc.rs`: replaces every embedded ICC profile of a saved project by a stub that + keeps only the profile's description: the JSON `colorProfileData` of the working space in an + `.aep` chunk or the `.aepx` text, and a raw profile of an imported image: + `cargo run -p craft-aep --example stubicc -- in.aep out.aep`. A profile it cannot replace stops + it before anything is written. Use it before a generated project becomes test data, so no + vendor colour data stays in it, and check the result independently. diff --git a/crates/aep/docs/aep-format.md b/crates/aep/docs/aep-format.md new file mode 100644 index 000000000..3582276d7 --- /dev/null +++ b/crates/aep/docs/aep-format.md @@ -0,0 +1,1102 @@ +# After Effects project format (`.aep`) + +> **Last reviewed:** 2026-11-05 · **Last updated:** 2026-11-11 · **Change:** footage: the Photoshop layer an item shows and its document's size (`opti` tag `8BPS`, section 3.3, EffectCraft issue 3087); before that: Puppet (`ADBE FreePin3`, 3.7): the property tree from an external scripting listing, **?** until a byte sample exists (#3093); before that: footage: an image sequence without a filename prefix stores only the suffix `Utf8`, and one imported with Force alphabetical order stores its file list in `LIST StVc` / `StVS` (section 3, footage); before that: versions: `svap` is the version that wrote the file, `head` bytes 4 to 7 the version it is written for, and what a Save a Copy As 25.x / 24.x of After Effects 2026 changes (section 1, "Versions"); before that: the stretch numerator is signed (a negative one reverses the layer, 981-983); layer styles (3.12b, 950-980); before that: footage: `sspc@118` of an image sequence is its folder's modification time (860, read-back x1-x3); before that: text: paragraph `/24`, `/35` and style `/68` are neutral for 0 and 1 (732-734, edited in place); before that: keyframes: the ease of mask path keys (599, 700), the spatial speed unit of fractional values (597, 598, 730); before that: text: character and paragraph settings, manual kerning, vertical text, `/nil` is neutral (S17: 538-578, 595, 596); before that: cameras, lights and 3D rotations (S16: the auto Position of a two-node camera is where After Effects renders it, 537; camera and light defaults read back 255, 256; Orientation `otst/otda` 488), Timewarp (529-533), an expression on an effect parameter (534); before that: text: style slot /13 is superscript / subscript (526-528; every scripted text of the corpus is superscript), wrapping box text 517; before that: effects of batch 5 (409-450, 511-516: integer sliders, checkbox defaults in `pard`, arbitrary data of Hue/Saturation and Levels, the parameters of Invert, Fast Box Blur, Gaussian Blur, Drop Shadow, Glow; section 3.7); before that: shape items of batch 5 (451-484, 519: ellipse, fill and stroke opacity, fill rule, group transform, shape layer anchor, trim paths, rectangle roundness, polystar fields, gradient default, nested groups, the defaults of every item; section 3.9); before that: track mattes: mode at `ldta` byte 107 and the matte layer id at byte 160 (cases 402-408, section 3.1); before that: text: automatic leading is verified (400); Exposure (`ADBE Exposure2`, cases 389-397): group start / end parameter types 13 / 14, float slider valid minimum and slider range (section 3.7); before that: text: U+0003 is a paragraph break (386, 387); before that: a stored text Anchor Point is pixels (385, now V); before that: the XML form `.aepx` is read into the same chunk tree (section 1, "The XML form"); the `head` save counter is a `u32` at byte 16; text documents for the importer (stroke colour, automatic leading, the font of each run, the undecoded-slot check, the text-layer Anchor Point, the Source Text record); before that: third round of verified field layouts (explicit-default semantics, type-major bezier ease, exact time stretch and in / out points, Time Remapping state, comp pixel aspect / shutter phase / display start, Point default, Layer Control None, dropdown selection, one `mkif` per mask, polystar and radial gradient records, text styles, camera and light options, marker cue data, audio and image-sequence footage) added to the typed view of `craft-aep` · **Target:** Adobe After Effects 2026 + +What `crates/aep` ([`craft-aep`](../README.md)) reads and writes, and the evidence +for each statement. Everything here was observed on project files we generated ourselves; no +source code, specification or format documentation of any other reader was used (every layout is +backed by our own cases, see section 5). This is the container and a set of +verified field layouts, not a project importer: turning a parsed `.aep` into an application's +document is the application's work. This document was written in EffectCraft, whose importer and +exporter are the first users of the crate; where it names `import/...`, `export/...`, +`aep-import.md`, `aep-export.md`, `effect_map` or `style_map`, those are in +[EffectCraft](https://github.com/storytold/effectcraft) (`crates/aep`, `docs/`). + +Evidence is cited as **case numbers**: the files `NNN-*.aep` of our synthetic corpus, each made by +one scripted change to a base project. Batch 1 (cases 000 to 126) was made in one chain of +Save As; batch 2 (cases 200 to 259) was made in later sessions, each case starting again from the +same loaded base; batch 3 (cases 299 to 354) was made in three further sessions (299 is a Save As +without any edit, the reference for save noise). The base project `000-base` is one composition `C` +(1920x1080, 25 fps, 5 s) holding one red solid layer `L` (1920x1080) in the folder `Solids`. 374 of +these cases (369 `.aep`, and the `.aepx` of `000`, `058`, `060`, `110`, `124`) are the test fixtures. They +are a few combined projects, one composition per case named like the case and rebuilt from the +same recipe (`crates/aep/tests/fixtures/CASES.md`; the generator is in EffectCraft, `crates/aep/tests/fixtures/tools/`), read from the +folder named by `CRAFT_AEP_FIXTURES` (opt-in); `crates/aep/tests/it/common/mod.rs` hands a test its case as a project of its own cut from +the combined file, and a test whose fixture is missing prints `SKIPPED: fixture missing` and passes. The tests named below run on them; +the `.aepx` cases are one pair, `aepx-pair.aep` and `aepx-pair.aepx`. The container +of every fixture is byte-identical to the generated file, with two exceptions that keep the +fixtures small and free of local data: the XMP packet after the container (which grows by 1.7 KB +with every save of the corpus chain) is replaced by the 5.8 KB packet of the base project in all +fixtures but `001` (and inside `` of the five `.aepx` fixtures), and in every fixture that imports a file (a footage item: `126`, `350`-`354` and the later footage cases) the +file path and machine name inside the footage alias JSON are replaced by neutral text of the same +length. + +**Read-back tests.** Some evidence comes from files we wrote ourselves (not After Effects), opened +in After Effects 2026 and read back value by value or rendered: an exporter in development, which is +not in this repository. They are named by an id (`e1`, `f1`, `o1`, `s1`, `t7`, `x1`-`x3`, ...) or by +their local folder (`export-tests\...`), which is not in the repository either. + +Confidence codes in the field tables: **V** = a test in `crates/aep/tests/` asserts that the decoded +value equals the value the case was made with (the corpus manifest); **E** = decoded and consistent +in the case diffs, but no manifest value to assert against (the field is exposed only where a test +pins what the fixtures contain); fields that the corpus does not decide (**?**) or only suggests +(**H**) are not decoded; they are listed in section 4. + +## 1. Container + +| Fact | Evidence | +|---|---| +| The file starts with `RIFX`, a big-endian `u32` size and the form type `Egg!`; all integers in the structure are big-endian (the one exception, `iide`/`idpi`, is little-endian). | All 139 files of batches 1 and 2. | +| The declared size is `4 + the chunks`; it ends before the end of the file. The bytes after it are an XMP packet (``, UTF-8 XML, 5.8 KB in `000`) that grows with every save. | All files; test `header_and_trailer_facts`. | +| A chunk is a 4-byte id, a big-endian `u32` body size and the body; an odd-sized body is followed by one pad byte, always `0` in the corpus (15 622 odd bodies). | Test `odd_sized_chunks_keep_their_pad_byte_even_when_not_zero` keeps any value. | +| The chunks of every file tile the container exactly: no leftover bytes, no chunk larger than its parent. | 139 of 139 files parse without a repair and write back byte for byte. | +| A `LIST` chunk's body is a 4-byte list type followed by chunks. List types seen include `Fold`, `Item`, `Layr`, `tdgp`, `tdbs`, `list`, `Pin ` and `FEE ` (the space is part of the type). | `000`; test `list_types_ending_in_a_space_are_kept`. | +| Lists can be empty (size 4, only the type): `Ewst`, `Pefl`, `CPPl`, `LItm`. | `000`; test `empty_list_and_list_shorter_than_its_type`. | +| Nesting: at most 13 levels in the corpus (the path of a shape layer, 240). | All files; test `nesting_depth_is_below_the_limit`. | +| `LIST btdk` is not a chunk sequence: its body (7775 bytes in `110`) is text. A reader must keep it opaque. Only files with a text layer have it. | `110`-`114`, `243`-`248`; test `raw_text_lists_are_kept_opaque`. | +| Three non-`LIST` ids hold a chunk instead of data: `tdsn`, `pdnm`, `fnam`. Each holds exactly one `Utf8` chunk (a name or label, UTF-8 text, possibly empty). | `tdsn` 31 209 times, `fnam` 40, `pdnm` 32 over the corpus; test `nested_chunk_ids_hold_one_utf8_chunk`. | +| Ids can end in a space; the space is part of the id. In the corpus this happens for the list types `Pin ` and `FEE `. All ids are 4 printable ASCII bytes. | `Pin `, `FEE ` in all files but `001`; tests `list_types_ending_in_a_space_are_kept`, `ids_ending_in_a_space_are_kept`. | +| Every payload that is decoded here is plain: no compression, no encryption. Text is readable, numbers are IEEE big-endian doubles or integers. Entropy and decompression probes (zlib, raw deflate, gzip, lzma, bz2, LZ4 block) over all chunk ids of the corpus found no compressed chunk, and consecutive saves of a project differ in only a few bytes. **One body is not understood: `opti` (footage and render options) has a plain 125-byte header and then a high-entropy body (7.9 bits per byte); whether it is compressed is open.** The high entropy of `sdat` is a flat histogram (five identity ramps 0..255), not random data. | `010` (the layer name `Renamed` is readable in a `Utf8` chunk), `040` (`100.0` and `200.0` as doubles). | + +### How the crate handles it + +- `parse` builds a tree of `Chunk { id, body, pad }`; a body is `Data`, `Chunks` (the three + nested ids), `List { list_type, children }` or `OpaqueList { list_type, data }`. The bytes after + the container are kept as the trailer. +- `write` recomputes every size from the tree. For every input that parses, `write(parse(x)) == x` + byte for byte, including pad bytes, an odd last body without a pad byte, and the trailer. The + tests assert this for all fixtures and for every damaged input that still parses. +- A `LIST` (or nested id) whose body is not a well-formed chunk sequence, or that is nested deeper + than `Limits::max_depth` (64), stays opaque instead of failing the parse. At the top level, a + chunk that does not fit, a wrong form type or a container larger than the input is an error. + Sizes are checked against the remaining input; nothing is allocated from a declared size; the + chunk count is bounded (`Limits::max_chunks`). + +### The XML form (`.aepx`) + +After Effects also saves a project as XML (`.aepx`). Every case of the corpus was saved in both +forms from the same state, the `.aepx` right after the `.aep`. Comparing each pair element by +element with the chunk tree of the `.aep` gives the mapping below. It holds for all 223 pairs of +the corpus (batches 1 to 3 and the 29 cases 355 to 383 of a fourth batch made the same way, which +this document does not use otherwise): the tree read from the `.aepx` is the tree of the `.aep`, and written back as an +`.aep` it gives the same bytes, apart from the save counter and the XMP packet (last row). + +| Fact | Evidence | +|---|---| +| The file is UTF-8 XML with the declaration `` and one root element ``. Its children are the top-level chunks of the `.aep`, in the same order. The `RIFX` header and the `Egg!` form type have no element. Indentation is tabs and line feeds; there are no comments, no entities and no carriage returns in any sample. | All pairs; `000`. | +| A data chunk is an element named by its id, with the body as lowercase hex in the attribute `bdata` and no content: ``. Every body of the `.aep` comes back byte for byte, including large ones (`cdat`, `ldat`, `opti`). | `000` (1 183 `bdata` elements, all equal but `head`, last row), all pairs. | +| A `LIST` is an element named by its list type, holding its chunks as child elements: ``, ``, ``, ``, ``, ``. An empty list is an element without children and without `bdata` (``, ``, ``, ``). The `LIST` id itself is not written. | `000`. | +| A trailing space of an id or list type is dropped: `Pin ` is ``, `FEE ` is ``. A name shorter than 4 characters is padded with spaces to read it back. | `000` (`FEE`, and `Pin` of the solid's footage), `126`. | +| The same name can be a list or a data chunk: `sspc` is a data chunk in a footage item (``) and a list in an effect instance (`` with children). Whether an element has `bdata` decides. | `000` (footage), `071` (Slider Control). | +| The three nested ids `tdsn`, `pdnm` and `fnam` are written like lists: an element with one child, the `Utf8` chunk. | `000` (`tdsn`), `071` (`pdnm`, `fnam`). | +| A `Utf8` chunk is `` with its text as content; an empty one is ``. The text is the chunk body exactly (no trimming). | `000` (`C`, `Solids`, the JSON strings of the colour settings, 19 empty ones), `010` (`Renamed`), `060` (`wiggle(1,10)`). | +| Two numeric chunks are written as text, in a `` child: `numS` (4 bytes, big-endian) as a decimal integer (only `0` is sampled, so whether it is signed is not decided; the reader accepts -2^31 to 2^32-1), and `ppSn` (8 bytes, an `f64`) with nine decimals (`150.000000000`, `0.000000000`). A `ppSn` value that is not a multiple of 10^-9 would be rounded by the XML form (no sample). | `000`, `124` (two compositions: two of each), all pairs. | +| `LIST btdk` (the text document, raw text in the `.aep`, see 3.10) is written like a data chunk: ``, the hex of the bytes after the list type. It is read back as an opaque list. | `110`-`114`, `243`-`248`, `334`-`340`. | +| The footage alias `alas` (a JSON object in the `.aep`) is the element `` inside ``, with one attribute per JSON key, in the same (alphabetical) order: `ascendcount_base`, `ascendcount_target`, `fullpath`, `platform`, `server_name`, `server_volume_name`, `target_is_folder`. The JSON writes the two counts as numbers, `platform` `Win` as `1` and `target_is_folder` `0` / `1` as `false` / `true`, and escapes a backslash of the path as `\\`. Rebuilt this way, the JSON equals the `.aep` body byte for byte. Other platforms and non-ASCII paths were not sampled. | `125`, `126`, `350`-`354` (`352`-`354`: an image sequence, `target_is_folder` `1`). | +| The XMP packet that follows the container of an `.aep` is the CDATA content of ``, a child of the root between the top-level `string` that follows `pdvc` and ``. It is not a chunk. | All pairs. | +| Pad bytes are not written. An odd body gets a zero pad byte when the tree is written as an `.aep`, which is what every `.aep` of the corpus has. | All pairs: the rewritten tree equals the `.aep` byte for byte. | +| The only differences inside the container are save state: the save counter, a big-endian `u32` at byte 16 of `head`, is one higher in the `.aepx` (only byte 19 changes in 222 pairs; in `314` it carries from `0x000000ff` to `0x00000100`), and the XMP packet has one more history entry. | All pairs. | + +**What the XML form leaves out.** In the corpus, nothing: every chunk of every `.aep` has its +element (the count per id agrees in all 223 pairs, apart from the `` children of `numS` and +`ppSn` and the renamed `alas`), and all 239 pairs of the current corpus read into the same chunk +tree (the local test `every_corpus_pair_gives_the_same_chunk_tree`). The reader still tolerates an +`.aepx` that lacks chunks: `elab`, `cmta` and `empd` are not read by the typed view, so a missing one +changes nothing it decodes, and a missing `tdbs` cannot be told apart from a property at its default +(the sparse rule of section 2): its value reads as the default. Elements that have no chunk +counterpart (a name that is not 1 to 4 characters) are skipped with their content. + +### How the crate reads an `.aepx` + +- `read_aepx` returns the same `AepFile` as `parse`; `parse_any` picks the reader by content (a + `RIFX` header, or an optional byte order mark and white space followed by `= 1 (51 200 at 200 %, 38 400 at 150 %) and stays 25 600 at 50 % | 000, 018, 316, 317 | V | +| start | 12 / 16 | i32 / u32 | layer start time = num / den seconds; den = composition timescale (25 600), 1 in a layer made by the precomp operation | 000, 021 (12 800 / 25 600 = 0.5 s), 124 | V | +| in point | 20 / 24 | i32 / u32 | **relative to the start**: absolute = start + in * stretch (the exact ratio above). **Fractional times are exact multiples of 1/25 600 s: 0.3333 s became 8 532/25 600 = 0.33328125 s, and setting the in point moved the out point with it (duration kept)** | 000, 019 (25 600 / 25 600 = 1 s), 021 (stays 0 while the layer starts at 0.5 s), 318 | V | +| out point | 28 / 32 | i32 / u32 | same convention; 5 s = 128 000 / 25 600. **For footage layers the denominator is the footage time base, not 25 600: 48 000 / 48 000 for the 1 s audio layer (the sample rate), 3 / 30 for the three-frame image sequence (the frame rate); start and in point keep 25 600** | 000, 018 (stays 5 s, the layer ends at 10 s), 019 (6 s), 021 (5.5 s), 316, 317 (stays 5 s, ends at 2.5 s / 7.5 s), 318, 351, 353-354 | V | +| name is user-set | 37 bit `0x01` | flag | clear while the layer shows its source's name (000, 124, new solids, default-named camera and light), set when named | 000, 010, 022, 110, 113, 124, 211 | V | +| guide layer | 37 bit `0x02` | flag | | 026 | V | +| adjustment layer | 38 bit `0x02` | flag | | 025 | V | +| 3D layer | 38 bit `0x04` | flag | always set on cameras and lights | 015, 251 | V | +| solo | 38 bit `0x08` | flag | | 014 | V | +| auto-orient | 38 bit `0x40` | flag | orients towards the point of interest: set on two-node cameras (byte 38 is `0x44`) and on all lights, clear on a one-node camera (`0x04`) | 256 (`0x44`), 251 (`0x04`), 252, 254 | V | +| null object | 38 bit `0x80` | flag | set on a layer created as a null | 022, 220 | V | +| **video switch** | 39 bit `0x01` | flag | the eye icon (`enabled`); off gives byte 39 = `06` | 222 | V | +| **audio switch** | 39 bit `0x02` | flag | `audioEnabled` (also changes on a solid); off gives `05` | 223 | V | +| **effects switch** | 39 bit `0x04` | flag | `effectsActive`; off gives `03` | 224 | V | +| motion blur | 39 bit `0x08` | flag | | 017 | V | +| locked | 39 bit `0x20` | flag | | 013 | V | +| shy | 39 bit `0x40` | flag | | 012 | V | +| collapse transformations | 39 bit `0x80` | flag | continuously rasterize; set from creation on shape and text layers | 024, 238 | V | +| source id | 40 | u32 | `idta@16` of the source item (solid, footage or comp); 0 for shape, text and camera layers, `0xffff_ffff` for lights | 000, 022, 124, 126, 252 | V | +| label | 61 | u8 | label colour index (1 red, 5, 8 blue, 15, ...); a footage layer copies the label of its footage item (audio 7, image sequence 3) | 000 (1), 124 (15), 126 (5), 351 (7), 354 (3) | V | +| name copy | 64..95 | ASCII, NUL padded | copy of the user-set name; empty while the layer is unnamed | 000 (empty), 010 (`Renamed`) | V | +| **blend mode** | 99 | u8 | Normal 2, Add 4, Multiply 5, Screen 6, Overlay 7, Difference 26 (each case changes only this byte); set by name (906): Soft Light 8, Hard Light 9, Darken 10, Lighten 11, Exclusion 25, Color Dodge 27, Color Burn 28; by render (907: a script enum sweep wrote codes 2-34, each code rendered over a test image and matched one mode): Classic Difference 12, Hue 13, Saturation 14, Color 15, Luminosity 16, Stencil Alpha 17, Stencil Luma 18, Silhouette Alpha 19, Silhouette Luma 20, Linear Dodge 29, Linear Burn 30, Linear Light 31, Vivid Light 32, Pin Light 33, Hard Mix 34; 23 / 24 render as Color Dodge / Burn there (taken as the classic variants, E); 3 is written for Dissolve and Dancing Dissolve; 21 / 22 are Alpha Add / Luminescent Premul (not told apart). The reader keeps every code beyond the first six raw | 000, 016, 225-228, 906, 907 | V (6 + 22), E (23, 24) | +| layer type | 128 | u32 | 0 AV (solid, footage, precomp, null), 1 light, 2 camera, 3 text, 4 shape | AV: 000, 022, 126; text: 110, 113, 243; shape: 238-242; camera: 251, 253, 256; light: 252, 254 | V | +| parent id | 132 | u32 | 0 = no parent | 000, 022 (L gets the id of the new null layer) | V | +| **light type** | 136 | u32 | 1 = spot, 2 = point (0 on every other layer); parallel and ambient were not sampled and are kept raw | 252, 254 | V | +| **track matte mode** | 107 | u8 | 0 = none, 1 = alpha, 2 = alpha inverted, 3 = luma, 4 = luma inverted; bytes 104..106 are 0 in every file. Setting a matte changes only this byte and the matte layer id on the target layer, and turns the matte layer's video switch off (byte 39 `07` -> `06`) | 402-408 | V | +| **track matte layer** | 160 | u32 | the id (byte 0) of the matte layer, anywhere in the composition: `0x1d` (`M`, above) in 403-406 and 408, `0x1b` (`L`, below its target) in 407; 0 without a matte. A mode with id 0 has no case (After Effects 2026 writes the id; before 23.0 the matte was always the layer directly above) | 403-408 | V | + +API: `LayerHeader` (`is_video_enabled`/`is_enabled`, `is_audio_enabled`, `is_effects_enabled`, +`is_auto_oriented`, `light_type`, `blend_mode`, `stretch` / `stretch_den`, `stretch_ratio`, +`in_point`, `out_point`); the same switches on `Layer`; `Layer::track_matte` +(`TrackMatte`, `MatteMode`; test `tests/matte.rs`). `stretch` is now the +numerator of the ratio (it was the whole multiplier before 50 % and 150 % were sampled). Tests: +`tests/header.rs`, `tests/layer_switches.rs`. The layer name itself is the `Utf8` child of the +`Layr` (empty while the name is the source's); the layer then shows the name of its source item +(`Project::layer_source(...)` then `Item::display_name`, the rule the importer uses). + +### 3.2 Composition header (`Item/cdta`, 204 bytes; the decoder needs the first 176) and renderer + +| Field | Offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---| +| frame duration | 4 | u32 | time units per frame: 1024 at 25 and 30 fps, **800 at 29.97 fps, 1000 at 23.976 fps** | 000, 031, 229, 230 | V | +| timescale | 8 | u32 | time units per second: 25 600 at 25 fps, 30 720 at 30 fps, **23 976 at 29.97 and 23.976 fps**; fps = timescale / frame duration exactly as typed (29.97 is `23976 / 800`, not `30000 / 1001`) | 000, 031, 229, 230 | V | +| work area start | 28 | u32 | in time units | 035 (25 600 = 1 s) | V | +| work area end | 36 | u32 | start + duration in time units; `0xffff_ffff` = the whole composition | 000, 035 (76 800 = 3 s) | V | +| duration | 44 | u32 | in time units. **A duration is snapped to whole frames: 5 s at 29.97 fps is 120 000 units = 150 frames = 5.005005 s, at 23.976 fps 120 frames** | 000, 031, 032 (256 000), 229, 230 | V | +| background | 52..54 | u8 R, G, B | **8 bits per channel**: 0.5 is stored as 128 and reads back as 128/255 = 0.50196 | 000, 033, 231 | V | +| motion blur | 139 bit `0x08` | flag | the composition's switch | 038 | V | +| width, height | 140, 142 | u16, u16 | | 000, 030 (1280 x 720) | V | +| pixel aspect | 144 / 148 | u32 / u32 | numerator / denominator; **0.9091 is stored as the exact ratio 10 / 11 (read-back 0.90909090909091)** | 000 (1 / 1), 037 (2 / 1), 325 (10 / 11) | V | +| frame rate | 156 | u32 | 16.16 fixed point of the float32 rate (25 = `0x0019_0000`, 29.97 = 1 964 114 = 29.9700012, 23.976 = 1 571 291) | 000, 031, 229, 230 | V | +| shutter angle | 172 | u32 | degrees | 000 (180), 034 (90) | V | +| **shutter phase** | 180 | i32 | degrees: 0 by default, -90 = `ffffffa6`; the shutter angle at 172 does not change | 000, 326 | V | +| **display start** | 164 / 168 | u32 / u32 | the comp start timecode as `num / den` seconds: 0 by default; display start frame 100 at 25 fps is 102 400 / 25 600 = 4 s (the denominator is the timescale; at other frame rates it was not sampled) | 000, 327 | V (one sample) | +| renderer | `LIST PRin` / `prin` 4 | 32 ASCII | match name: `ADBE Escher` in the base, `ADBE Calder` in 039 | 000, 039 | V | +| renderer name | `prin` 52 | 32 ASCII | display name: `Classic 3D`, `Advanced 3D` | 000, 039 | V | +| composition name | `Item/Utf8` | text | | 000 (`C`), 124 (`Pre`) | V | + +The inert `ADBE Time Remapping` record of every AV layer has `tduM` = the source's duration in seconds: 5 for 5 s compositions (124, 257, 258), 3 and 2 for a 3 s and a 2 s composition (600, 601), 1 for the 1 s WAV (351), 0.1 for the three-frame sequence (353), 0 on solids and stills (000, 126). Every property's keyframe time scale (`tdb4` bytes 12 to 15) follows the composition (23 976 in every +property of `L` after 229 and 230). The corpus manifest calls the classic renderer +`ADBE Advanced 3d`; the file stores `ADBE Escher` / `Classic 3D`. API: `CompHeader::{time_base, +frame_rate_fixed, duration_units, shutter_phase, display_start}` (`display_start` is the exact +fraction). Tests: `tests/header.rs`, `tests/layer_switches.rs`. + +### 3.3 Items, footage and precomps + +| Field | Chunk path | Offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---|---| +| item kind | `Item/idta` | 0 | u16 | 1 folder, 4 composition, 7 footage | 000 | V | +| item id | `Item/idta` | 16 | u32 | what a layer's source id refers to (`iide` holds the same number little-endian) | 000 (`0x1a` solid, `0x0d` comp), 022, 124 | V | +| item label | `Item/idta` | 58 | u8 | label colour of the item | 000 (1 solid), 124 (15 for `Pre`) | V | +| used comp | `Item/idpi` | 0 | u32 **little-endian** | a composition that uses another as a precomp layer gets one `idpi` per used composition | 124 (`C` uses `Pre`) | V | +| used comp count | `Item/idpc` | 0 | u32 little-endian | the number of `idpi` chunks: one per distinct composition shown (the same one twice counts once); the order of the `idpi` carries no meaning (an After Effects re-save reverses it) | 124, 257, 258, 600, 602 (1), 601 (2) | V | +| footage type tag | `Pin /sspc` | 22 | 4 ASCII | `Soli` for a solid, `png!` for a png | 000, 126 | V | +| footage size | `Pin /sspc` | 30, 34 | u32, u32 | width, height in pixels | 000 (1920 x 1080), 022 (100 x 100 null), 126 (8 x 8), 211, 212, 259 | V | +| footage name | `Pin /opti` | 26 (solid), 58 (file) | ASCII, NUL terminated | the item's own `Utf8` name is empty for footage | 000 (`L`), 022 (`Null 1`), 126 (`test8x8.png`), 211 (`S100`), 259 (`Blue`) | V | +| **solid colour** | `Pin /opti` | 10, 14, 18, 22 | 4 x f32 | ARGB, 0..1: red [1, 0, 0] is `1, 1, 0, 0`, blue [0, 0, 1] is `1, 0, 0, 1` | 000, 259 | V | +| **Photoshop layer** | `Pin /opti` (tag `8BPS`) | 14; 32, 36; 78, 82, 86, 90; 344 | i32 BE; 2 x u32 **LE**; 4 x i32 **LE**; UTF-8, NUL terminated | an item made from one layer of a `.psd`: the layer record's index in the file (bottom of the stack = 0; -1 is the merged image), the document's two sides (which is the width: **?**, the one sample is square), the layer's bounds top, left, bottom, right in document pixels (all 0 for a layer without pixels, such as an adjustment layer) and the layer's name. The item's size (`sspc` 30 / 34) is the bounds' size with Retain Layer Sizes, else the document's. `Footage::psd_layer` | the project of issue 3087: ten layers, each index, bounds and name equal to the `.psd`'s layer record at that position (`Layer 1` `[9, -346, 1083, 1086]`, item 1432 x 1074; `test` `[749, 800, 764, 848]`, item 48 x 15; 32 and 36 hold 1000, 1000 in every record, the `.psd`'s size) | V | +| file path | `Pin /Als2/alas` | key `fullpath` | JSON string | alias JSON of a file footage item | 126 (the fixture holds a neutral path) | V | +| precomp linkage | `Layr/ldta` | 40 | u32 | the layer's source id is the id of the precomp's item | 124 | V | + +Footage items beyond solids and stills (`Pin /sspc`, 222 bytes; audio and image sequences, batch 3): + +| Field | `sspc` offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---| +| file type tag | 22 | 4 ASCII | `png!`, `Soli`, **`WAVE`**; `TPIC` Targa (650), `8BPS` Photoshop (651), `TEXT` for both an Illustrator / PDF file (652) and an EPS (653); all four stills store the duration 0 / 600 | 126, 000, 350, 650-653 | V | +| width, height | 30, 34 | u32, u32 | 0 / 0 for audio; 64 x 64 for the sequence | 350, 352 | V | +| **duration** | 38 / 42 | u32 / u32 | seconds as `num / den`, unreduced: still 0/1, **WAV 48 000/48 000 (the sample rate is the denominator), image sequence 3/30 (the frame rate is the denominator)** | 126, 350, 352 | V | +| native frame rate | 56 | u32 | 30 for the sequence, 0 for stills, audio and solids | 126, 352 | V | +| frame rate (again) | 150 | u32 (unaligned) | the same rate as @56 for sequences and clips: 30 (353, 603), 25 (606, 612); 0 for stills and audio. Conforming a sequence to 25 fps (615 against its reference 614) changes @42 (the duration denominator), @56 and @150 from 30 to 25 together; for clips see @146 | 353, 603, 606, 612, 614, 615 | V | +| conformed rate of a clip | 146 | u32 (unaligned) | the rate a clip is conformed to (Interpret Footage), 0 when not conformed; the effective rate is then also at @150, while @38 / @42 (the duration in native frames over the native rate) and @56 stay native: the 25 fps 25-frame mp4 conformed to 30 stores 25 / 25, @56 25, @146 30, @150 30 (645 against 644), the layer's Time Remapping maximum becomes 25 / 30 s. A conformed image sequence keeps @146 at 0 and stores the new rate at @42, @56 and @150 (615) | 615, 644, 645 | V | +| bits per pixel | 60 | u32 | RGBA still 32, RGB sequence 24, 0 for solids and audio | 126, 352 | V | +| **sample rate** | 160 | f64 | 48 000.0 for the WAV; the same for an mp4 with sound (48 000.0 and 44 100.0 read back from AAC tracks of those rates), absent for an mp4 without (the importer takes a clip's audio from it) | 350, read-back vid-audio | V | +| **file size** | 208 | u32 | bytes of the source file(s): png 149, WAV 96 044 (44 + 48 000 x 2), three png of 168 bytes 504 | 126, 350, 352 | V | +| **sequence numbering** | 172, 176, 180 | u32 x 3 | first and last frame number and the digits of a sequence: 1, 3, 3 for `seq_001.png`..`seq_003.png` (353), 101, 103, 4 for `shot_0101.exr`..`shot_0103.exr` (603); 0 for single files (604, 605, 607). The name around the number is the pair of `Utf8` chunks between `Als2` and `opti` in `Pin ` (`seq_` / `.png`, `shot_` / `.exr`), absent on single files. **An empty prefix is not written**: `0001.png`..`0005.png` leave one `Utf8` (`.png`) there, so a single `Utf8` is the prefix when the first file name in `opti` (byte 58: `0001.png`) starts with it and the suffix when it ends with it; without that evidence the reader takes one starting with `.` as the suffix and reads nothing else. `Item::sequence_info`, `SequenceInfo::file_name` | 353, 603, 604, 605, 607; the prefix-less sequence of issue 3174 (After Effects 25.2) | V | +| **sequence file list** (Force alphabetical order) | `Pin /LIST StVc` | `StVS` (4 bytes, `u32` LE), then `Utf8` x count | a sequence imported with Force alphabetical order has, between `Als2` and `opti`, a `LIST StVc` instead of the prefix and suffix `Utf8`s: `StVS` holds the file count as a **little-endian** `u32` (`05 00 00 00`), followed by one `Utf8` per file name in order (`shot_0001.png`..`shot_0005.png`). The numbering at @172 / @176 is then `0xFFFFFFFF` and the digits @180 0; duration and rate are stored as for a numbered sequence (5/30, 30). The list replaces the numbering; a list without a readable `StVS` count, or with a name that is not a plain file name (`/`, `\`, `:`, a control character such as NUL, `.`, `..`), is not read. Without a stored item name the importer names the item after its first listed file (an assumption: what After Effects shows was not sampled). `SequenceInfo::files`, `SequenceInfo::nth_file_name` | the alphabetical sequence of issue 3174 (After Effects 25.2) | V | +| **sequence folder time** | 118 | u32 (unaligned) | seconds since 1904-01-01 UTC: the modification time of a sequence's folder when After Effects scanned it (860: 0xe6fb443a = 2026-10-19 03:51:22, the folder's and its files' time). With the folder's time After Effects keeps the stored numbering on open; with another (an older time, the first or the last file's time while the folder's differs: x1, x2 against x3) it scans the folder again and warns about a gap in a modal dialog | 860, `export-tests\sparse860\exp` x1-x3 | V | +| still duration denominator | 42 | u32 | 1 for png (126, 607), 600 for jpeg (605), 100 for OpenEXR (604): a still's duration is 0 over a per-format denominator | 126, 604, 605, 607 | V | + +An **image sequence is ONE item** (`opti` holds the first file name `seq_001.png` at 58, 322 bytes like +a still, with the width and height at 20 and 24 and the PNG colour type at 36); its alias JSON points +at the sequence **folder** (`"target_is_folder":true`, `ascendcount_target` 2). The audio item has its +name in the item's `Utf8` (`test-tone.wav`), label 7 (sequence 3, still 5); its `opti` has no display +name. A footage layer shows the item's name while it has no name of its own (351, 354). + +Test: `tests/header.rs` (precomp), `tests/text_markers_footage.rs` and `tests/batch3_objects.rs` +(footage), `tests/layer_switches.rs` (solid colour). API: `Footage::{duration, native_frame_rate, +bits_per_pixel, file_size, audio_sample_rate, is_sequence}`; `Item::sequence_info` (with +`SequenceInfo::frame_count`). + +### 3.4 Properties: sparse storage, static values and units + +A layer's property tree contains a `tdbs` only for a property whose value differs from the default +that After Effects computes for the layer and composition. A reader supplies the defaults: + +| Property (match name) | Stored | Unit in the API | Default when absent | Evidence | Conf. | +|---|---|---|---|---|---| +| `ADBE Position` | f64 x, y, z | pixels | the composition centre `[width / 2, height / 2, 0]` | 000, 040 (`100, 200, 0`), 030 (the comp becomes 1280 x 720, L stays 1920 x 1080: `960, 540, 0` is now stored), 211, 212 (absent on new solids) | V | +| `ADBE Anchor Point` | f64 x, y as a **fraction of the layer's source size**, z in pixels | px = fraction * source size | `0.5, 0.5, 0` (the layer centre) | 044 (`10, 20` px on 1920 x 1080 is `10/1920, 20/1080`), **211 (100 x 100 solid: `10, 20` is stored `0.1, 0.2`), 212 (400 x 300 solid: `100, 50` is stored `0.25, 1/6`)**: the composition size is not the divisor | V | +| `ADBE Anchor Point` of a **text** layer | f64 x, y, z (a text layer has no source) | pixels, as stored | `0, 0, 0` (the text origin) | absent in every other text sample (110-114, 243-248, 334-340, 386); **385 (text anchor set to `10, 20`) stores `10, 20, 0` as f64 pixels**, not a fraction of a size; test `a_stored_text_anchor_is_pixels` | V | +| `ADBE Scale` | f64 fractions | percent (fraction * 100) | 100 % | 000, 041 (`0.5, 0.5, 1.0`) | V | +| `ADBE Rotate Z` | f64 degrees | degrees | 0 | 000, 042 | V | +| `ADBE Opacity` | f64 fraction | percent (fraction * 100) | 100 % | 000, 043 (`0.25`) | V | +| mask feather | f64 x, y | pixels | `[0, 0]` | 090, 091 (`20, 20`), 208 | V | +| mask opacity | f64 fraction | percent | 100 % | 090, 091 (`0.5`) | V | +| mask expansion `ADBE Mask Offset` | f64 | pixels | 0 | 090, 091 (5) | V | +| effect slider | f64 | as set | the `pard` default | 071 (77) | V | +| effect colour | 4 x f64, ARGB 0..255 | ARGB fractions | the `pard` default | 073 (`[0.2, 0.6, 1]` is `255, 51, 153, 255`) | V | +| shape stroke width, round corners radius, light intensity | f64 | as set (pixels, pixels, percent) | not decoded as defaults except the light (100) | 239 (8), 238 (20), 254 (50) | V | +| shape group position, gradient start and end | f64 x, y | pixels | group position `[0, 0]` | 242, 241 | V | +| camera Position, point of interest | f64 x, y, z | pixels | see 3.12 | 251, 253, 256 | V | +| camera Zoom, light colour, cone angle, cone feather | f64 / ARGB 0..255 | pixels, `[A, R, G, B]` fractions, degrees, percent | Zoom 2 666.67 (1920 px comp), colour white, cone angle 90, cone feather 50 | 341, 343-345 | V | + +`Project::layer_anchor_px` multiplies the anchor fraction by the size of the layer's source (a +solid or footage item, or the composition of a precomp). `Transform::text_anchor_px` gives the +anchor of a text layer (pixels as stored, `[0, 0, 0]` when absent). + +**Explicit defaults (batch 3).** A value record exists exactly when the value differs from the +default; setting a value to its default never creates a record and setting it back removes it. A +Gaussian Blur read back as Blurriness **25** when added (the corpus manifest's "0" for rows 214 and +216 was wrong, with a correction note in the manifest); Blurriness set to 0 writes a record, 25 -> 40 -> 25 +leaves the effect subtree byte-identical to the never-touched default (300, 301, and 214 / 078). +The same holds for Fill Opacity 1 -> 1 and 1 -> 0.5 -> 1 (302, 303) and Slider 0 -> 0 (304), and +for the five transform properties set to the value they already had (Position, Opacity, Anchor, +Scale, Rotation, 311-315: the layer tree is byte-identical to the unedited save 299). A writer may +therefore rely on "absent = default" and drop a record that equals the default. Exceptions are +records AE materialises together with a type switch (polygon, radial gradient: 3.9). +`EffectParam::is_default` returns this; `Tracked::source` tells `Default` from `Stored`. + +**Time Remapping (batch 3).** Every layer has an inert `ADBE Time Remapping` property at the root of +its tree (`tdsb` 3, static value 0, no keys). Enabling the remap turns it into a keyed property +(`tdsb` 1, `tdb4` byte 5 bit 0 cleared, byte 68 = 1, no `cdat`) with two plain 48-byte keys whose value +(seconds) is at byte 8: 0 s -> 0 and 5 s -> 5 on a precomp layer (319), 0 s -> 0 and 0.1 s -> 0.1 on an +image-sequence layer (354). It cannot be enabled on a solid. API: `Layer::time_remap` (`TimeRemap`, +`TimeRemapKey`). + +Property record (`LIST tdbs`: `tdsb`, `tdsn`, `tdb4`, then `cdat` or a `LIST list`, optionally a +`Utf8` expression, `tdum`, `tduM`): + +| Field | Chunk path | Offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---|---| +| magic | `tdb4` | 0..2 | `db 99 00` | every property | all fixtures | V | +| components | `tdb4` | 3 | u8 | values in `cdat` / in each key (1, 2, 3, 4 = ARGB) | 040, 041, 043, 073, 091, 206, 208, 210 | V | +| static flag | `tdb4` | 5 bit 0 | flag | set while the property has no keys (Position: 15 static; the bit is clear on an animated property) | 040, 055, 058 | V | +| **template flag** | `tdb4` | 7 | u8 | 0 = plain keyframe template, non-zero = spatial template (3 on Position, 1 on colours and paths) | 204-206, 208, 210 | V | +| time scale | `tdb4` | 12 | u32 | units per second of the keyframe times (25 600; 30 720 at 30 fps; 23 976 at 29.97 and 23.976 fps) | 055, 058, 122, 229, 230 | V | +| has keys | `tdb4` | 68 | u8 | 1 exactly when the property has a `LIST list` | every fixture (test) | V | +| keyed spatial-template flag | `tdb4` | 79 | u8 | 1 on a static colour or point effect value, 0 once it is keyed (359 / 364, 369 / 382); 0 on keyed and static sliders (362 / 363, 391 / 397); meaning **?** | 359, 362-364, 369, 382, 391, 397 | V (values) | +| expression flags | `tdb4` | 76, 77, 120 | u8 | all three are 1 when an expression exists. 76 and 77 are also 1 on a camera's or light's computed (auto) Position, which has no expression | 060, 213, 251 | V | +| **expression disabled** | `tdb4` | 119 | u8 | 1 = the expression is disabled; the text and bytes 76, 77, 120 stay. On the built-in view layers the byte flips by itself after the first save, so it counts only together with an expression | 060 (0), 213 (1) | V | +| state | `tdsb` | 0 | u32 | 1 = written (non-default value, keys or expression), 3 = default state. **A layer, camera, light, shape or text property in the default state holds After Effects' default** or a value with no effect on that layer (survey of 839 corpus and sample files: Material Options, cone 90 / 50 and falloff 500 / 500 of point lights, Position `[0, 0, -1333.33]` of ambient and environment lights, Opacity 1 and Scale `[1, 1, 1]` of cameras and lights, a polygon's Inner Radius 50, Miter Limit 4, the inert Time Remap 0), so the importer skips it. **An effect parameter's record in the default state holds the effective value** (3.7, stored value) | 040, 071, 073 (1); 3 on the unwritten properties of 000; 1200, 1400 (effects) | V (1), E (3), V (effects) | +| static value | `cdat` | 0 | `components` x f64 | the first `components` doubles are the value | 040, 041, 043, 071 | V | +| user name | `tdsn/Utf8` | | text | `-_0_/-` means "no custom name" | 091 (`Mask 1`), 238 (`Group 1`) | V | + +Test: `tests/values.rs`, `tests/typed.rs`, `tests/keyframes.rs`. + +### 3.5 Keyframes (`tdbs/list`: `lhd3` + `ldat`) + +`lhd3` (13 `u32`): word 0 `0x00d0_0bee`, word 2 the record count, word 3 `ceil(count / 4)`, word 4 +the record size, word 7 the count rounded up to a multiple of 4, words 8 to 12 zero; `ldat` is +exactly `count * record size` bytes (cases 050, 055; every list of every fixture). In path lists +(record size 8) words 3 and 7 are allocation capacities in vertices and four times that (12 records: +4 and 16; 9 records: 4 and 16 in the shape paths of 105 and 240, 3 and 12 in the open mask path 233), +not a function of the count. + +**Record header (8 bytes, every kind).** + +| Field | Offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---| +| time | 0 | i32 | in the property's time scale; **layer time** (roving moved key 2 from 25 600 to 26 509 = 1.0355078125 s) | 055 (0, 25 600, 64 000), 056, 058 | V | +| **interpolation in** | 4 | u8 | the incoming side: 1 = linear, 2 = bezier, 3 = hold | 202 (in linear, out hold: `01 03`), 203 (in hold, out linear: `03 01`), 204, 205, 052, 053 | V | +| **interpolation out** | 5 | u8 | the outgoing side, same codes | same | V | +| label | 6..7, bits 6..9 | u16 | keyframe label: label 9 turns the flags `0x0007` into `0x0247` | 057 | V | +| roving | 6..7, bit `0x0020` | u16 | `0x0007` becomes `0x0027` | 056 | V | + +A hold key has no further field (the value is unchanged); a linear key has all ease slots 0. Keys +added through the scripting interface are linear in and out. + +**The template follows the property header (`tdb4` bytes 3 and 7), not the record size.** With +`n` components: + +| Template | `tdb4[7]` | Size | Used by | Evidence | +|---|---|---|---|---| +| plain | 0 | `8 + 40 n`: 48 (n=1), 88 (n=2), 128 (n=3) | Opacity, Rotation, Slider (1); Mask Feather, Rectangle Size (2); Scale (3) | 205, 206, 208, 221 | +| spatial | non-zero | `56 + 24 n`: 128 (n=3), 152 (n=4) | Position (3), colours (4) | 204, 210 | +| mask path | | 64 | the path property of an animated mask (the path itself is in `shap`) | 093 | +| marker | | 16 | `mrst` marker times | 122, 123, 249 | + +A 128-byte record is a Scale key (plain, 3 components) or a Position key (spatial): a reader must +look at the header (206 against 204). Layout of the plain template: value (`n` x `f64`) at 8, then +`4 n` ease doubles in **type-major order**: `[in speed x n][in influence x n][out speed x n][out +influence x n]`, starting at `8 + 8 n` (for `n = 1`: **at 16 in speed, at 24 in influence, at 32 +out speed, at 40 out influence**). Scale key 1 of 305 (`n = 3`, in 0 / 33 %, out 20 / 66 %) is +`0,0,0, .33,.33,.33, .2,.2,.2, .66,.66,.66`; Mask Feather key 1 of 306 (`n = 2`, in 0 / 33, out +15 / 66) is `0,0, .33,.33, 15,15, .66,.66`. Component `i` uses slot `i` of each block (**V** for the +block layout and the units, Scale speed in fractions per second and Mask Feather speed in pixels +per second; **E** that different values per dimension sit in the slot of their dimension, because +the tool that made the corpus applies one ease to all dimensions; **V** since the exporter wrote three different eases for the three Scale dimensions and After Effects read the same three back, aep-export.md R13). `Keyframe::ease_component(i)` +returns one component's `KeyEase`, `Keyframe::ease_components` all of them. Layout of +the spatial template: `u32` at 8 (1 normally, 0 after the spatial auto-bezier and continuous +switches were turned off together, 054), `f64` arc length of the segment to the next key at 16 +(100 in 204 key 1, 0 on the last key and right after creation; safe to ignore), **ease at 24 in +speed, at 32 in influence, at 40 out speed, at 48 out influence**, value (`n` x `f64`) at 56, spatial +in tangent (`n` x `f64`, relative offset from the value) at `56 + 8 n`, out tangent at `56 + 16 n`. + +| Field | Where | Meaning | Evidence | Conf. | +|---|---|---|---|---| +| ease, Position | spatial @24..55 | 204 key 1: in speed 5 / influence 0.2, out speed 50 / influence 0.8; key 2: in 7 / 0.6, out 9 / 0.4. Speed in pixels per second, influence a fraction. **After Effects reports key 1's in speed and the last key's out speed as 0, but the file keeps the requested values (5 and 9): ignore those two slots** | 204, 052 (preset: speed 0, influence 0.33) | V | +| ease, Opacity | plain n=1 @16..47 | 205 key 1: in 0.05 / 0.2, out 0.2 / 0.8; key 2: in 0.3 / 0.6, out 0.09 / 0.4. Speed is in stored units per second, i.e. 100 times smaller than the percent per second shown in the UI; influence a fraction | 205 | V | +| value, Scale | plain n=3 @8 | 206: `[1, 1, 1]` and `[2, 1.5, 1]` (fractions of the manifest percent) | 206 | V | +| value, Rotation, Slider | plain n=1 @8 | degrees / as set (keyed slider 10 and 90) | 221 | V (slider), E (rotation, 207 is not a fixture) | +| value, Mask Feather | plain n=2 @8 | pixels `[0, 0]` and `[40, 20]` | 208 | V | +| value, colour | spatial n=4 @56 | ARGB doubles 0..255: `[255, 255, 0, 0]` and `[255, 0, 127.5, 255]`; flags `0x0001`, tangents zero; the project-level `EfdG` copy of the effect has the same list | 210 | V | +| value, Position | spatial n=3 @56 | pixels | 055 (`[0,0,0]`, `[100,0,0]`, `[200,100,0]`), 056 | V | +| tangents | spatial @`56+8n`, `56+16n` | 054: key 1 out `[30, 60, 0]`, key 2 in `[-30, -60, 0]` | 054 | V | +| arc length | spatial @16 | 100 on key 1, 0 on key 2 | 204 | E | +| **colour distance** | spatial n=4 @16 | for bezier colour keys the Euclidean ARGB distance to the next key, 255 sqrt(2) = 360.6245 for red to blue, 0 on the last key; linear colour keys have 0 (like the Position arc length, filled only for non-linear interpolation). The ease slots of a colour key are one entry (speed 0, influence 0) | 307 against 210 | V | +| **keyed Orientation** | `otst`: `tdbs` key list of 80-byte records (time, interpolation sides, flags at 0..7; `f64` 2.0 at 16 on the first key; the rest 0 for linear keys) and `otky` with one `otda` (three `f64` degrees) per key, in the same order. **After Effects interpolates by a quaternion slerp of the rotation `Rx Ry Rz`**: 703 (keys `[0,0,0]` at 0 s, `[10,20,30]` at 2 s, linear) reads back 0.5 s `[3.434, 4.469, 7.713]`, 1 s `[6.264, 9.353, 15.219]`, 1.5 s `[8.465, 14.560, 22.610]`, which slerp with that order reproduces to 0.001 degree (not per-component: 5 / 10 / 15 at 1 s; the order `Rz Ry Rx` gives `[3.58, 10.64, 14.32]`) | 703 | V (layout, linear), H (ease of bezier keys) | +| **light type** | `ldta` `u32` at 136: 0 parallel (701), 1 spot, 2 point, 3 ambient (702). A light created as an environment light by script without the Advanced 3D renderer is stored as ambient (704) | 701, 702, 704 | V | +| **ease, Anchor Point and effect points** | spatial @24..55 | the value is a fraction of the layer size, and **a speed is pixels per second divided by the layer height**: out speed 50 is 50 / 1080 = 0.046296 on the 1920 x 1080 L (597 Anchor Point, 598 Gradient Ramp Start), 50 / 800 = 0.0625 on a 400 x 800 solid (730; not the width, not the smaller side, not the composition height); influence a fraction. After Effects reads 597 back as 0.5 s [1004.41, 562.21], 1 s [1084.84, 602.42], 1.5 s [1137.13, 628.57] | 597, 598, 730 | V | +| **ease, colour switched to bezier** | spatial n=4 @24..55 | keyframe.set_interpolation bezier on two colour keys stores speed 0 and influence 0 on both sides, and After Effects interpolates them linearly (598 Start Color red to blue: 0.75 / 0.5 / 0.25 red at 0.5 / 1 / 1.5 s); a custom colour ease cannot be set by script | 598 | V | +| **ease, mask path** | 64-byte record @24..55 | in speed, in influence, out speed, out influence as in the spatial template: 599 out / in influence 0.75, 700 0.2 / 0.5, speeds 0 (the path at 0.5 s is 5.42 % / 28.0 % of the way). The `f64` at 16 is 1 (093) or 2 (599, 700) on the first key and 0 on the last (not decoded) | 093, 599, 700 | V (influence), ? (speed unit) | + +API: `Keyframe::{interpolation_in, interpolation_out, kind, value, ease, ease_component, ease_components, spatial}` (the `f64` at byte 16 of a colour key is `SpatialKey::segment_length`); +`Property::keys()` hands the property's template to its records. Marker record (16 bytes): time, +`03 03` hold bytes, 8 stale bytes. Mask path record (64 bytes): the common fields and the ease at byte 24 (599, 700). Test: +`tests/values.rs`, `tests/keyframes.rs`. + +### 3.6 Expressions + +| Field | Chunk path | Meaning | Evidence | Conf. | +|---|---|---|---|---| +| source | `tdbs/Utf8`, after `cdat` / `list` and before `tdum` | raw UTF-8 text, no length prefix, no NUL, no engine tag | 060 (`wiggle(1,10)` on Opacity) | V | +| flags | `tdb4` 76, 77, 120 | all 1 when present, all 0 otherwise | 060, 040 | V | +| **disabled** | `tdb4` 119 | 1 when disabled (`expression.disable_all`); the only byte that differs between 060 and 213 | 060, 213 | V | +| static value | `cdat` | kept while an expression exists (Opacity 1.0) | 060 | V | +| **on an effect parameter** | the parameter's own value record, as on any property: Slider Control's Slider with `time*10` holds the text and the static value 0 | 534 | V | + +The expression engine is project-wide (`LIST ExEn/Utf8`, `javascript-1.0`). API: +`Property::expression_enabled`. Test: `tests/keyframes.rs`. + +### 3.7 Effects (`tdgp[ADBE Effect Parade]/sspc`) + +| Field | Chunk path | Offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---|---| +| match name | the `tdmn` before the `sspc` | | text | `ADBE Slider Control`, `Pseudo/@@S1iPXtJFTHK4QaDbgJvLhw`, ... | 071, 073, 077, 080, 214 | V | +| instance name | `sspc/fnam/Utf8` | | text | the effect's own name (`Slider Control`, `Tint`); it stays the same when the instance is renamed (610) | 071, 073, 077, 080, 610 | V | +| **user name of an instance** | `sspc/tdgp/tdsn/Utf8` (the value group's name) | | text | `-_0_/-` (the placeholder every unnamed group has) while the instance shows the effect's name; the given name otherwise (610: `My Tint`, `fnam` still `Tint`). `Effect::user_name`, `Effect::display_name` | 356, 609, 610 | V | +| **fx switch** | `sspc/tdgp/tdsb` (the value group's flags, `u32`) | | flag | 1 on, 0 switched off in the Effect Controls (`fx-switch.aep`: two Gaussian Blurs on one layer in one save, the first switched off, the second on; read back `enabled` false / true). `Effect::enabled` | fx-switch | V | +| parameter count | `sspc/parT/parn` | 0 | u32 | including the implicit input layer and the built-in entry (3 for a slider) | 071 | V | +| parameter match name | `parT/tdmn` | | text | `-NNNN`; the last is `ADBE Effect Built In Params` | 071 | V | +| parameter type | `parT/pard` | 12 | u32 | 0 layer, 2 fixed slider (16.16), 3 angle, 4 checkbox, 5 colour, 6 point, 7 popup, 9 no data, 10 float slider, 12 path, **13 group start / 14 group end** (Exposure's Master, Red, Green and Blue twirl-downs; the end repeats the group's name; no value), **15 button** (Swap Colors of Tint and Gradient Ramp: empty name, label in `pdnm`) | 071, 073, 077, 080, 214, 215, 356, 368, 389 | V | +| UI flags | `pard` | 4 | u32 | 0x20 greyed out, 0x08 on a group end. **Stale:** After Effects does not reset it on undo (391-397 keep the flags of 390 although Channels is back to Master), so it is UI state and never read for meaning | 389-397 | V (values), E (meaning) | +| parameter name | `pard` | 16 | 32 ASCII | `Slider`, `Color`, `Menu`, `Blurriness`, `Shadow Color`, ... (empty for some checkboxes) | same | V | +| **default** | `pard` | 56 | by type | float slider: f64 (Slider 0, **Blurriness 25**); fixed slider and angle: 16.16 (drop shadow opacity 127.5, direction 135, distance 5, softness 0; Fill opacity 1.0 = 100 %; **Tint Amount to Tint 100, Ramp Scatter 0, Blend With Original 0**); **colour: ARGB bytes (drop shadow `ff000000` black, Fill `ffff0000` red; Ramp Start `ff000000`, End `ffffffff`; Tint Map Black To `00000000` and Map White To `00ffffff`: alpha 0, which is what After Effects reads back)**; popup: u32 index (1); **point: two 16.16 fractions of the layer size at 56 and 60 (Point Control 0.5, 0.5; Ramp Start 0.5, 0 and End 0.5, 1)** | 071, 077, 080, 214, 215, 356, 368, 383 | V | +| **valid maximum** | `pard` | 128 / 108 | 16.16 / f32 | the largest valid value of a fixed slider (128: Tint Amount 100, Ramp Scatter 512, Blend With Original **1.0**) and of a float slider (108: Blurriness 30 000; **Exposure 100, Offset 2, Gamma Correction 10**). The stored scale differs per slider (below), so this is the reader's only hint at it | 356, 368, 078, 389 | V | +| **float slider valid minimum, slider range** | `pard` | 104; 112 / 116 | f32; f32 x 2 | the smallest valid value (Exposure -100, Offset -2, Gamma Correction 0.1 as f32) and the range the slider covers (Exposure -4..4, Offset -0.5..0.5, Gamma 0.1..10); the slider range is repeated as `tdum` / `tduM` (f64) in each value record (391-394). @120 holds the default again as f32 (Gamma 1.0, Blurriness 25), u16 @124 the display decimals (**H**: Exposure 2, Offset 4) | 389, 391-394 | V | +| **definitions of a second instance** | top-level `LIST EfdG` | | | one `EfDf` per effect TYPE (`tdmn` = match name, then a copy of ONE instance's `sspc`, including that instance's value records); the `EfDC` count before them is a little-endian u32. When two layers carry the same effect type, only one instance keeps a full `parT`; **the other has an empty `LIST parT` (4 bytes)** but its own `tdgp` (input layer `-0000` with its own `tdpi`, the button record, and its value records). Its definitions come from the `EfdG` copy by match name, its values from its own records, never from the copy (383: the copy holds S100's Start [0.1, 0.2]; L's Ramp reads the 368 defaults). Which instance keeps the full `parT` is **?** (in 383 S100 is both first in the file and newest). The `EfDf` order follows the stack order (365 / 366); `Pefl/pjef` does not. **Some files have no `EfdG` at all** (seen in 17 projects, all from inside After Effects-made Motion Graphics templates). In each of them every effect type (196 in all; 2 to 14 Color Controls per file) has exactly one instance with a full `parT` and an empty one in every other instance, and the reader takes the others' definitions from that instance (`DefinitionSource::OtherInstance`, #3170). Which instance keeps it is **?** here too | 365, 366, 383; templates | V | +| popup items | `parT/pdnm/Utf8` | | text | items separated by `\|` | 077 (`Item 1\|Item 2\|Item 3`), 214 | V | +| stored value | `sspc/tdgp/-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default.** Some parameters are recorded in the default state (`tdsb` 3) and the record still holds the value After Effects uses: every one of the 25 such records in the corpus and the samples differs from its `pard` default and reads back as recorded (Lumetri Color's hidden controls High Dynamic Range, HDR Specular, the second Look, Legacy Tone And Saturation, Face Detection, Auto Color, Auto Intensity and SemanticAutoTone Progress, 1200; Cryptomatte's Selection Mode on, 1400). No effect record in the default state holds a value that does not count. The reader takes the value in either state (`EffectParam::recorded_value`, `EffectParam::value`); `EffectParam::stored_value` keeps to written records, and a record in any other state (no sample has one) is not read: the importer reports it as damaged data. That other effects record values so is H (no sample shows one) | see below; 1200, 1400 | V (1200, 1400), H (other effects) | + +Stored values by type (`tdsb` 1, or 3 as above): + +| Type | Stored | Evidence | Conf. | +|---|---|---|---| +| float slider | `cdat[0]` as set (Slider 77, Blurriness 40); a keyed slider has 48-byte plain records and no `cdat`. **The value is the float32 After Effects keeps, widened to f64: Offset 0.1 is `0.10000000149011612`, Gamma 2.2 is `2.200000047683716`** | 071, 216, 221, 393, 394, 397 | V | +| **group start / end** | the record always exists (`tdsb` 1, `cdat` 0, descriptor `00010000`), `tdsn` = the group name; no value (scripting: NO_VALUE) | 389-397 | V | +| **fixed slider** | `cdat[0]` as a plain double in the parameter's own scale: Tint Amount to Tint is a **percent** (0 -> 0.0, 37.5 -> 37.5; valid max 100), Gradient Ramp Scatter is **pixels** (12, 250.5; valid max 512), Blend With Original is a **0..1 fraction** (25 % -> 0.25, 100 % -> 1.0; valid max 1.0). Keyed: 48-byte plain records with the plain value (Amount 20 / 80) | 361-363, 378-381 | V | +| colour | ARGB doubles 0..255 (Tint Map Black To [0.2, 0.4, 0.6] -> `[255, 51, 102, 153]`, float32 precision; Ramp Start Color red `[255, 255, 0, 0]`); keyed: 152-byte spatial records with the ARGB value at 56 (210 Fill, 364 Tint Map White To) | 073, 357, 371; keyed: 210, 364 | V | +| **button** | the record always exists (`tdsb` 1, `cdat` 0, descriptor `00010000`) and carries no value (scripting: NO_VALUE). An exception to the sparse rule, like the Layer Control record | 356-383 | V | +| **popup (built-in)** | 1-based index as a double: Ramp Shape Radial = `2.0`; the selection is also written to `pard` @56 (only byte 59 changes, as for the dropdown 324). Default Linear = 1, item count u16 @60 = 2, `pdnm` `Linear Ramp\|Radial Ramp` | 368, 377 | V | +| **checkbox** | `cdat[0] = 1.0` when on, `0.0` when off; **the default is the `u32` at `pard` byte 56** (0 off: Checkbox Control 217, Shadow Only 420, Bypass 389; **1 on: Repeat Edge Pixels of Fast Box Blur and Gaussian Blur, read back 1 in 412 and 417**), and a record exists only when the value differs from it (416 and 419 switch Repeat Edge Pixels off: a record with 0.0, descriptor `00010004`). The corpus manifest's "off" for the Gaussian Blur of 214 was wrong. Exposure's Bypass Linear Light Conversion has an empty `pard` name and its label in `pdnm` (like the buttons) | 217, 389, 396, 412, 416, 417, 419, 420, 426 | V | +| **integer slider** (type 1) | Fast Box Blur's Iterations: default `i32` at `pard` 56 (3), valid maximum `i32` at 128 (50), the value record a plain double (1.0 for 1) with `tdum` / `tduM` 1 / 5 (the slider range) | 412, 414 | V | +| **arbitrary data** (type 11) | the value record is a 4-byte `cdat` 0 followed by a `LIST aRbs` holding the data (`aRbp`); `parT` has the default `aRbp` after the `pard`. Hue/Saturation's Master Hue, Saturation, Lightness and Levels' Input / Output Black and White and Gamma are NOT in their visible parameters (which keep no record) but in this data; see below | 443-446, 511-516 | V | +| **point** | two doubles, a **fraction of the layer size** (`tdb4@24` = the layer's width / height: 16 / 9 in 369, 374, 218, 1 on the 100 x 100 solid of 383; `@16` = 6.25e-12 in all): `[111, 222]` is `[111/1920, 222/1080]` (`tdb4[3]` = 2, `[5]` = 15). **The default is two 16.16 fractions 0.5 in `pard` @56 and @60, relative to the layer: a Point Control on a 100x100 solid reads back [50, 50]**. Gradient Ramp Start / End are stored the same way; values outside the layer give fractions outside 0..1 (`[2000, -100]` -> `[1.0417, -0.0926]`); on a 100x100 solid `[10, 20]` is `[0.1, 0.2]`. **Keyed: 104-byte spatial records (56 + 24 x 2), flags `0x0007`, `u32@8` = 1, the two fractions at 56, tangents zero**; `f64@16` holds a derived distance on linear keys too (1.7397 in 382: ignore it) | 218, 310, 369, 374, 382, 383 | V | +| **angle** | plain degrees: 90 is `90.0` | 219 | V | +| **layer control** | the `-0001` record always exists (`cdat` 0); the sibling `tdpi` (u32) is the **id of the chosen layer** (the new Null, `0x22`). **The default ("Layer 1" on the first layer) and an explicit 1 are the owning layer's own id (320); "None" is `tdpi` 0 (321).** The manifest's "None" for the default (076, 220 rows) was wrong. In the implicit first parameter `tdpi` is the id of the layer that owns the effect | 220, 320, 321 | V | +| **layer parameter** (any effect) | as the Layer Control: the `-NNNN` record always exists with `cdat` 0 and `tdpi` = the id of the chosen layer (Set Matte, Displacement Map, Compound Blur set to the layer below: `tdpi` 340 / 354 / 368 = that layer's id); the effect's implicit `-0000` record holds the owning layer's id | 904, 905 | V | +| **dropdown** | the items are in `pdnm` (`Alpha\|Beta\|...`), `pard` u16 @60 is the item count (5), `pard` @56 the selection (1 by default). Selecting item 3 adds a value record (`cdat[0]` = 3.0, descriptor `00010004`) AND writes 3 into `pard` @56. Setting the items re-creates the effect definition under a new match name (`Pseudo/@@KbKT0rCPQi2A0ytuyGLWSw`; display name `Dropdown Menu Control`) | 077, 323, 324 | V | + +The default of Gaussian Blur: the default-only file is byte-identical to the one with Blurriness 25 +and has no value record; `pard` holds 25 (f64 @56 and f32 @120); setting 40 adds a record and leaves +`pard` at 25, so `pard@56` is the default and not the current value (a popup selection is the one +exception, it overwrites `pard@56`). The corpus manifest's default of 0 was wrong: After Effects +reads a fresh Gaussian Blur as 25 (300, **V**; the conflict is resolved). + +Because a popup selection overwrites `pard@56`, a popup whose definition comes from the `EfdG` copy +has no reliable default in the file (the slot holds the copied instance's selection); the reader +returns no default for it (`EffectParam::pard_default` is `None`), and a consumer uses the effect's +known default. Effect stack order is the order of the `sspc` children of the Effect Parade: moving an +effect (366) reorders them and the `EfDf` copies and changes nothing else (365, 366, **V**). + +**Exposure (`ADBE Exposure2`, 389-397, V).** `parn` = 24: 0 the input layer, 1 `Channels:` (popup +`Master|Individual Channels`, 1-based, default 1), then four groups of five entries, Master 2-6, +Red 7-11, Green 12-16, Blue 17-21, each `[group start (13), Exposure, Offset, Gamma Correction, +group end (14)]` (the channel groups' sliders are named `Red Exposure`, `Red Offset`, `Red Gamma +Correction`, ...), 22 Bypass Linear Light Conversion (checkbox, default off) and 23 the built-in +entry. The three sliders are float sliders with the same `pard` in every group: Exposure (stops) +default 0, valid -100..100, slider -4..4; Offset default 0, valid -2..2, slider -0.5..0.5; Gamma +Correction default **1**, valid and slider 0.1..10. Each one-change case adds exactly one value +record (twice: layer tree and `EfdG`): Channels 2 (390), Exposure 1.5 (391) and -2.25 (392), Offset +0.1 (393), Gamma 2.2 (394), Channels 2 + Red Exposure 1 (395, two records), Bypass 1.0 (396). A +keyed Exposure (397: -1 at 0 s, 2 at 2 s, linear) has plain 48-byte records. With Channels = Master +the per-channel values are stored but inactive. The other byte changes against 389 are only the +stale UI flags `pard@4` and, for a popup selection, `pard@59`. + +**Batch 5 effects (409-450, 511-516, 522-525; V).** Defaults as read back by After Effects for the +fresh effect (MANIFEST), each value change one case. The scale of a fixed slider is told by its +valid maximum (`pard` 128): + +| Effect (match name) | Parameters (`parT` index: type, valid maximum, default) | Evidence | +|---|---|---| +| `ADBE Invert` | 1 Channel: popup of 16 items **including three separators `(-`** (RGB, Red, Green, Blue, `(-`, HLS, Hue, Lightness, Saturation, `(-`, YIQ, Luminance, In Phase Chrominance, Quadrature Chrominance, `(-`, Alpha), default 1, stored 1-based over all 16 (Red = 2); 2 Blend With Original: fixed, max 100 (percent), 50 stored 50.0 | 409-411 | +| `ADBE Box Blur2` (Fast Box Blur) | 1 Blur Radius: fixed, max 30 000, pixels (20); 2 Iterations: integer slider, default 3, max 50; 3 Blur Dimensions: popup (Horizontal and Vertical, Horizontal, Vertical), default 1; 4 Repeat Edge Pixels: checkbox, **default on** | 412-416 | +| `ADBE Gaussian Blur 2` | 1 Blurriness: float, default 25, max 30 000; 2 Blur Dimensions: popup as above (Vertical = 3); 3 Repeat Edge Pixels: checkbox, **default on** | 417-419 (and 214, 216, 300, 301) | +| `ADBE Drop Shadow` | 1 Shadow Color: default black; 2 Opacity: fixed, **max 255** (default 127.5 = 50 %, 204 = 80 %); 3 Direction: angle, default 135; 4 Distance: fixed, max 4 000, pixels, default 5; 5 Softness: fixed, max 30 000, default 0; 6 Shadow Only: checkbox, default off | 420-426 | +| `ADBE Glo2` (Glow) | 1 Glow Based On: popup (Alpha Channel, Color Channels), default 2; 2 Glow Threshold: fixed, **max 255** (default 153 = 60 %); 3 Glow Radius: fixed, max 1 000, default 10; 4 Glow Intensity: fixed, max 255, default 1; 5 Composite Original: popup (On Top, Behind, None), **default 2 (Behind)**; 6 Glow Operation: popup of 25 blend modes starting None, Normal, Add, ..., default 3 (Add); 7 Glow Colors: popup (Original Colors, A & B Colors, Arbitrary Map), default 1; 8 Color Looping: popup (Sawtooth A>B, Sawtooth B>A, Triangle A>B>A, Triangle B>A>B), default 3; 9 Color Loops: fixed, max 127, default 1; 10 Color Phase: angle; 11 A & B Midpoint: fixed, **max 0.99, a 0..1 fraction**, default 0.5; 12 Color A white; 13 Color B black; 14 Glow Dimensions: popup as Blur Dimensions | 427-441 | +| `ADBE HUE SATURATION` | 1 Channel Control: popup (Master, Reds, Yellows, Greens, Cyans, Blues, Magentas); 2 Channel Range: **arbitrary data** (below); 3 Master Hue (angle), 4 Master Saturation, 5 Master Lightness (fixed, max 100): **never stored, the values are in the data**; 6 Colorize: checkbox; 7 Colorize Hue: angle; 8 Colorize Saturation: fixed, max 100, default 25; 9 Colorize Lightness: fixed, max 100 | 443-450 | +| `ADBE Timewarp` | 1 Method: popup (Whole Frames, Frame Mix, Pixel Motion), default 3; 2 Adjust Time By: popup (Speed, Source Frame), default 1; 3 Speed: fixed, max 5 000, default 50, plain (25 stored 25.0; keyed: plain 48-byte records 50 / 200); 4 Source Frame: fixed, max 5 000, plain (12); the tuning, motion blur, matte (`Matte Layer`, `Warp Layer`: layer records always present, 0 = none) and crop parameters follow | 529-533 | +| `ADBE Easy Levels2` (Levels) | 1 Channel: popup (RGB, Red, Green, Blue, Alpha); 2 Histogram: **arbitrary data** (below); 3-7 Input Black, Input White, Gamma, Output Black, Output White: float sliders (max 10 000, Gamma 5), **never stored, the values are in the data**; 8 / 9 Clip To Output Black / White: popup (**On, Off, Off for 32 bpc Color**), default 3 | 511-516 | + +Hue/Saturation data (180 bytes): `i32` master hue at 0 (444: 90), saturation at 4 (445: 50), +lightness at 8 (446: -30; V); then six colour ranges (Reds, Yellows, Greens, Cyans, Blues, +Magentas) of 28 bytes: four `i32` range points in degrees (default Reds 315, 345, 15, 45; Yellows +15, 45, 75, 105; ...; Magentas 255, 285, 315, 345) and the range's hue, saturation, lightness (E: +layout read from the default, no range was changed). The Colorize values are ordinary records +(447-450). Levels data (108 bytes): `u32` `0x000f10a7`, `u32` 1, then five channels (RGB, Red, +Green, Blue, Alpha) of five `f32` input black, input white, gamma, output black, output white; the +RGB values are V (512-516: 0.1, 0.8, 2, 0.2, 0.9 as float32), the other channels E (their defaults +0, 1, 1, 0, 1). After Effects' undo does not revert a change of these values (MANIFEST). API: +`Effect::arbitrary_data`, `HueSaturationData`, `LevelsData`; test `tests/effects_batch5.rs`. + +API: `EffectParam::{stored_value, pard_default, value, is_default, is_animated, keys, valid_max, +valid_min, slider_range, point_px, layer_ref, popup_item_count, popup_selected_index, +popup_selected_item, borrowed_definition}`, `Effect::has_own_definitions`, +`Project::{effect_params, effect_definitions, definition_source}`, `ParamValue`, +`ParamKind::{Button, GroupStart, GroupEnd}`, `DefinitionSource`, `LayerRef`. Tests: +`tests/effects.rs`, `tests/batch3_core.rs`, `tests/batch3_objects.rs`, `tests/effects_batch4.rs` +(356, 357, 361, 363, 364, 366, 368, 369, 371, 374, 377, 378, 380, 382, 383), +`tests/effects_exposure.rs` (389-391, 393-397). + +**Importer block 2026-11-05, #3110 (`1100-quickwins.aep`, After Effects 26.2.1; V for every +row: defaults from the defaults composition, the valid ranges from the minimum and maximum +compositions, the items from one composition per popup item).** Each set value is stored as set +(Decay 20 is `20.0`, Blend With Original 100 % is `1.0`); a value set to the default leaves no record +(HDR Amount 100): + +| Effect (match name) | Parameters (`parT` index: type, valid range, default) | +|---|---| +| `ADBE HDR ToneMap` (HDR Highlight Compression) | 1 Amount: float, 0..100 (slider 0..100), default 100 | +| `ADBE Echo` | 1 Echo Time (seconds): float, -30 000..30 000 (slider -5..5), default -1/30 (float32); 2 Number Of Echoes: integer, max 30 000, default 1; 3 Starting Intensity: float, 0..1, default 1; 4 Decay: float, **valid 0..100, slider 0..1**, default 1; 5 Echo Operator: popup (Add, Maximum, Minimum, Screen, Composite In Back, Composite In Front, Blend), default 1 | +| `ADBE Linear Wipe` | 1 Transition Completion: fixed, max 100, default 0; 2 Wipe Angle: angle, default 90; 3 Feather: fixed, max 32 000, default 0 | +| `ADBE Offset` | 1 Shift Center To: point, default the layer centre; 2 Blend With Original: fixed, **max 1 (a 0..1 fraction)**, default 0 | +| `ADBE Deflicker` (Color Stabilizer) | 1 Stabilize: popup (Brightness, Levels, Curves), **default 2 (Levels)**; 2 Black Point, 3 Mid Point, 4 White Point: points, defaults 0.2 / 0.5 / 0.8 of the width at half the height; 5 Sample Size: fixed, max 50, default 3. The reference frame the Set Frame button picks is not among the parameters | + +**Importer block 2026-11-05, #3112 (`1300-thirdparty.aep`; V for the parameter lists, H for the +data layouts, read from default instances only):** + +| Effect (match name, plug-in version) | Parameters (`parT` index: type, valid range, default) | +|---|---| +| `CS Color Neutralizer` (CC Color Neutralizer, 1.3) | 1 / 7 / 13 Shadows / Midtones / Highlights Unbalance: colours (black, 127 grey, white; pickers: setting one sets the range's sliders, e.g. [0.27, 0.55, 0.85] gives Red 42.94, Green 1.18, Blue -44.12 = (mean - channel) x 150 / 255 on 8-bit values, and leaves no colour record); 2-6, 8-12, 14-18 a group per range (start, Red, Green, Blue, end) with float sliders -100..100 (slider -50..50), default 0; 19 Pinning: float 0..100 (slider 0..50); 20 Blend w. Original: float 0..100; 21-25 the Special group: 22 View popup (Final Result, Correction B to W, Correction Stair), default 1, 23 Black Point float 0..49.9, default 0, 24 White Point float 50.1..100, default 100 | +| `EXtractoR` (2.6.6) | 1 Channel Info (`EXtractoR-0013`): arbitrary data, 688 bytes, below; 2-6 the Process group: 3 Black Point and 4 White Point float sliders, valid ±3.4028235e38 (the float32 range; slider ±1 000), default 0 / 1; 5 UnMult: checkbox (empty `pard` name, the label in `pdnm`), default off | +| `IDentifier` (2.6.6) | 1 Channel Info (`IDentifier-0005`): arbitrary data, 148 bytes, below; 2 Display: popup (Colors, Luma Matte, Alpha Matte, Raw), default 1; 3 ID: integer slider, max 2 147 483 647, default 0 | + +EXtractoR's Channel Info is the plug-in's own **little-endian** layout: `u32` 1 and 28 bytes, then +four 164-byte records (Red, Green, Blue, Alpha), each a kind in byte 0 (1 a channel of the footage by +name: `R`, `G`, `B` by default; 2 the input's own channel, named `(copy)`: Alpha by default) and the +channel name NUL-terminated from record byte 35. EffectCraft imports kind 2 as its EXtractoR's `(copy)`, +the effect input's channel of the same position as earlier effects left it, not as `A` (which names +an OpenEXR footage's own alpha plane). The rest is not read: it differs between instances +with the same channels (uninitialised-looking values, also in bytes 1..4 of a record: `02 00 00 32` +on Alpha in one definition's default, which is why only byte 0 is the kind). **A chosen channel** +(`extractor-07-preset-channel-z`, a preset of our own that reads `Z` into Red, Green and Blue) is kind +1 with its name (`Z`), and the `u32` at record byte 4 is then 13, which looks like the channel's +position in the EXR's sorted channel list of 14 (the defaults hold 0 there: H). IDentifier's data +differs between instances with the same settings except for **the chosen channel's name**, +NUL-terminated from byte 19, which is `Object ID` when no channel is chosen. After Effects lists +only integer (UINT) channels: in 1300's EXR (a FLOAT `ID` channel) it found none ("No ID channels +available"), so that block's preset `identifier-09-preset-channel-id` only sets ID 2; in 1310's +EXR (UINT `objectId` and `IndexOB.X` added) the preset `identifier-09-preset-channel-objectid-id2` +stores `objectId`, with 15 at byte 4 (the channel's position in the sorted list of 16, as for +EXtractoR: H). Without a chosen channel After Effects leaves the layer unchanged (1300 and 1310 +defaults). The other bytes are not read. API: `ExtractorChannels`, +`identifier_channel_type`. + +**Curves and Mesh Warp** (961-965, 985; applied from presets, the data is not scriptable): their +arbitrary-data parameter (type 11) always has a record (`tdbs` with a 4-byte `cdat`) and a `LIST +aRbs` / `aRbp`, also at the default. Curves (`ADBE CurvesCustom-0001`, 1644 bytes): `u16` 1, 1; +five 256-byte tables (RGB, Red, Green, Blue, Alpha) = the natural cubic spline through the points, +rounded (0 of 1280 entries differ); five 72-byte point records of 16 (`u16` in, `u16` out) slots, +the count at 64, an `i32` at 68 (-1 untouched, count - 2 in every edited sample: H). Mesh Warp +(`ADBE MESH WARP-0004`): `u32` rows + 1, `u32` columns + 1 (a 1 x 2-cell read-back test renders as made only in +this order, `export-tests\mesh`; AE does not rebuild the stored mesh when Rows / Columns change, 989), 12 bytes +(zeros, `0d 0d`), then per vertex row-major ten big-endian `f32`: position, handles top, right, +bottom, left as fractions of the layer (H: or of the comp), outward border handles (0, 0); Rows / +Columns / Quality are integer sliders with records only off their default (7, 7, 8). The `-NNNN` +suffix is a parameter id, not the `parT` position (Curves -0002 Channel before -0001; Mesh Warp +-0005 Quality before -0004). API: `CurvesData`, `MeshData`, `curve_lut`. [V; details: corpus FORMAT +S] + +**Lumetri Color** (`ADBE Lumetri`, 1200: 63 compositions in After Effects 2026, 32 bpc, working space +None; by script: defaults, all set, one per section, valid minima and maxima, every popup item the script +can set, checkboxes toggled, keys, an expression; from presets made by hand, `-58` ... `-62`: an Input +LUT, a Look, the RGB curve, Hue vs Sat, the colour wheels): 131 `parT` entries. The `-NNNN` suffix is a +parameter id, not the position (`-0127` ... `-0130` sit between `-0005` and `-0026`), so the importer +keys by suffix. Float sliders (type 10) store the value After Effects shows; the valid ranges (`pard` +104 / 108) are wider than the slider's (112 / 116): Temperature / Tint -300..300 (slider -100..100), +Saturation 0..300 (0..200), Exposure -7..7 (-5..5), Contrast, Highlights, Shadows, Whites, Blacks, +HDR Specular, Tint Balance -150..150 (-100..100), Faded Film 0..150 (0..100), HDR White 100..1000, +HDR Range 100..10000, Refine Blur 0..1000 (0..30), Vignette Amount -5..5 (-3..3); the read-backs of +`-19-min` / `-20-max` equal the stored numbers. **Hidden controls are recorded in the default state** +(`tdsb` 3) and still hold the value After Effects reads back: High Dynamic Range `-0062` (1.0), +HDR Specular `-0064` (22), the second Look popup `-0128` (26.16 as set, read back 26), Legacy Tone And +Saturation `-0097` (its `pard` default reads off as the script saved the project and on once the +project was opened and saved again, `versions/1200-lumetri-26`, `-25x`, `-24x`; a fresh instance reads +back off), Auto Color `-0124`, Face Detection `-0102`, the Auto Intensity `-0123` +(`EffectParam::recorded_value`). Each section has an Active checkbox (blank `pard` name; `-0003` Basic +Correction, `-0023` Creative, `-0037` Curves, `-0118` RGB Curves, `-0104` / `-0105` / `-0107` / `-0109` / +`-0111` the hue / saturation curves, `-0046` Color Wheels, `-0068` HSL Secondary, `-0050` Vignette, on +by default); off, the section does nothing (`-03-sec-basic-correction`, `-17-sec-vignette` render as +`-00-defaults`). Arbitrary data (type 11, a record and `aRbs` in every instance; empty is `fe fe`): +`-0001` an XML copy of the grade written only when the Look changes (not kept current: `-01-set-all` +holds defaults for controls set after the Look; it names the LUT file, `Fuji F125 Kodak 2393 (by +Adobe).cube`); `-0096` the Look's name, UTF-16LE (`SL BLUE ICE`, `Fuji F125 Kodak 2393 (by Adobe)` in +`-59-preset-look-lut`; the popup `-0025` itself has no record, `-0128` can disagree with it: +`-07-sec-creative` names item 20 while `-0128` holds 23), `-0095` likewise for the Input LUT +(`ALEXA_Default_LogC2Rec709`, `-58-preset-input-lut`; the popups `-0005` / `-0127` keep no record, +`-0004`, `-0024` and the assets `-0125` / `-0126` stay `fe fe`: the LUTs are After Effects' own files, +named, not embedded); `-0129` the Look colour space's 16-byte id (the GUID `LookColorSpace` of `-0001` +in Windows layout: the first 4, 2 and 2 bytes little-endian; zero for None; the Color Space popup +`-0130` writes no record of its own, `-22-pop38-1` ... `-53-pop38-33`); +`-0032` Split Toning (32 zero bytes); `-0039` the four RGB curves and `-0042`, `-0106`, `-0108`, +`-0110`, `-0112` the hue / saturation curves, 520 bytes each: `i32` type (3 for the hue curves, 2 for +Luma vs Sat and Sat vs Sat, 0 for an RGB curve), `u32` point count, then little-endian `f64` pairs x, y +sorted by x (bytes after the counted points are stale). RGB: x and y 0..1, (0, 0), (1, 1) by default; +the first of the four is the master curve (`-60-preset-curves-rgb`: (0.491, 0.787) lifts R, G and B +alike, and the render follows a natural cubic spline through the points within 0.003); the order of +the other three is H (the `-0001` XML names `ToneCurvesLuma`, `Red`, `Green`, `Blue`; no sample moves +one). Hue / saturation: x the hue (0 red, 1/3 green) or the luma / saturation 0..1, y the offset from +neutral 0; a fresh curve has no points, the selector colours `-0113` ... `-0117` add three at y = 0, +the hue picked +- 0.0556. Hue vs Sat's y scales the saturation by 1 + y in linear light +(`-61-preset-curves-huesat`: (0.0155, 0.97) takes pure red's linear saturation to 1.96 times, (0.263, +-0.03) and (0.765, -0.02) green, cyan and blue to 0.97); the other four curves' scale is H. `-0047` +Color Wheels, 4 `f64` triples: hue in degrees (0 red, 120 green), brightness slider 0..1 (0.5 +neutral), distance from the centre 0..1; 0, 0.5, 0 each by default; Shadows, Midtones, Highlights, then +a fourth wheel of unknown role (`-62-preset-wheels`: Shadows 229.1, 0.370, 0.547 towards blue with the +slider down; Midtones 0.1, 0.5, 0.395 towards red; Highlights 93.4, 0.727, 0.319 towards yellow-green +with the slider up; the fourth at its default); `-0085` the HSL correction wheels (8 triples, the same +default; layout H); `-0073` the HSL key, 40 bytes: a flag byte (1 once a colour was picked), then hue, +saturation, lightness, each centre, range, tolerance (`f32` LE, fractions) and an enabled byte (fresh: +0.5, 0, 0, on; picking RGB 0.02 0.55 0.85 gives hue centre 0.5605 = that colour's hue and lightness +centre 0.435 = its HSL lightness, range 0.05, tolerance 0.1: `-01-set-all`; whether range is the half +or full width is H); `-0122` the Auto Tone analysis (UTF-16 JSON). The project saved as 25.x and 24.x +copies (`versions/1200-lumetri-25x`, `-24x`: the 58 scripted compositions, saved by After Effects +2026) stores every Lumetri definition, valid range and value as the 26 save does. [V for the sliders, +checkboxes, popups, the `tdsb` 3 values, the master RGB curve, Hue vs Sat, the colour wheels and the +LUT names; H where marked] + +**Cryptomatte plug-in** (match name `Cryptomatte`, the free Cryptomatte plug-in, 2.6.6 in our +sample; read from 88 instances in one production project, not made by us, and from our own After +Effects 2026 sample 1400 (corpus `aep/cryptomatte.aep`: a fresh instance, Selection Mode on, each +Output item, and two presets of our own that choose a layer and pick one object or material in the +plug-in's UI; scripting read-backs and renders per composition); H for what neither varies): +`parT` 0 the input layer; 1 Settings: **arbitrary data** (type 11, `Cryptomatte-0001`, a record and +`aRbs` / `aRbp` in every instance); 2 Output: popup (Colors, Matted Colors, Matted RGBA, Matte +Only), **`pard` default 1 = Colors** (bytes 56..64 `00000001 0004 0001`, in the instance's own and +the project's shared definition; scripting reads 1 from a fresh effect: 1400 `crypto-00-defaults`), +a written record (`tdsb` 1) holding the item number when set (1400: 2, 3, 4; production: 3 in 81, +4 in 7); 3 Selection Mode: checkbox (`pdnm` `fnord!`), no record when off, and when on **a +default-state record (`tdsb` 3) holding 1.0** that After Effects reads back as on and renders +(1400 `crypto-01-set-all`, `crypto-02-chk-toggled`; the general reader takes only `tdsb` 1 +values, so the importer reads this one itself); the built-in entry. As in every project, only the +first instance has its own `parT`; the others borrow the project's (1400). The Settings data is +the plug-in's own layout, **little-endian**: `cry1`; bytes 4..32 differ between instances with +the same settings (zeros in one, pointer-like values in others: not read); the layer name at 32, +NUL-padded to 64 bytes (`CryptoObject`, `CryptoMaterial` in 1400: the EXR layer's name, which is +also its channel prefix there, so the sample cannot tell which of the two the plug-in stores); at +96 a `u32` 1 and a `u32` that is the same in every instance; at 104 the `u32` length of the +selection text including its NUL; at 108 **the MurmurHash3 (x86, 32-bit, seed 0) of the selection +text with its NUL** (V on the three texts of 1400: `<11a97486>` `0x27880767`, `` +`0xf7374a5b`, the empty text `0x514e28b7`; not read); a zero byte at 112; the selection text from +113, the matte list as comma-and-space separated ID hashes `<11a97486>, ` (303 tokens in +production, all in that form). Each token is the 8-digit hash of a picked name as the EXR's +manifest lists it, i.e. the MurmurHash3 of the name (1400: `box_red` `11a97486` picked on +CryptoObject, `mat_paint` `a5deeb9e` on CryptoMaterial). After the text 2 to 3 bytes, zeros or +left-overs of a longer earlier text (1400's presets: 127 bytes, three zeros after the text). **A fresh effect** (the `parT` +default and the record alike, 116 bytes, 1400) is `cry1`, zeros to 96, then `u32`s 1, +`0x514e28b7`, 1, `0x514e28b7` and four zero bytes: no layer and an empty text (length 1, its NUL). +`0x514e28b7` is the hash of a single NUL byte, so the `u32` at 100 looks like the hash of a first +text that is empty in every sample, its length the `u32` 1 at 96 and its NUL the zero byte at 112 +(H: one value). Layer and selection decode the same in the production project and in 1400's two +presets, and After Effects draws exactly the picked objects as selected (`crypto-06`: the box, +`crypto-07`: box and disc, the two objects of `mat_paint`). API: `CryptomatteData` (refuses data without `cry1`, a length past the end or +a non-zero byte at 112). + +**Puppet (`ADBE FreePin3`; ?: external listing, not the corpus, #3093).** No file of the +corpus, the committed fixtures or the samples holds a Puppet. The tree below comes from outside +the corpus: a public scripting listing of the effect's properties (a forum post, an After Effects +release from before the Advanced and Bend pins and Density). It is not bytes we have seen; that +After Effects writes it by the general rule (a `tdmn` and the `LIST tdgp` or `tdbs` it names, +nested in the instance's value group `sspc/tdgp`, as masks, shapes and layer styles are) is +**H**: + +```text +Puppet ADBE FreePin3 + On Transparent ADBE FreePin3 On Transparent + arap ADBE FreePin3 ARAP Group (hidden) + Auto-traced Shapes ADBE FreePin3 Outlines + Mesh ADBE FreePin3 Mesh Group (hidden) + Mesh 1 ADBE FreePin3 Mesh Atom + Mesh ADBE FreePin3 Mesh (hidden: the triangulation) + Triangles ADBE FreePin3 Mesh Tri Count + Expansion ADBE FreePin3 Mesh Expansion + Deform ADBE FreePin3 PosPins + Puppet Pin 1 ADBE FreePin3 PosPin Atom + Vertex Offset ADBE FreePin3 PosPin Vtx Offset (hidden, 2 values) + Vertex Index ADBE FreePin3 PosPin Vtx Index (hidden; 4294967295 = none) + Position ADBE FreePin3 PosPin Position + Overlap ADBE FreePin3 HghtPins (hidden while empty) + Stiffness ADBE FreePin3 StarchPins (hidden while empty) +``` + +A pin's rest place is not a property: it is a vertex of the hidden mesh (Vertex Index) plus an +offset, so it cannot be read without decoding the mesh data. The unit of Position (layer pixels, +or a fraction of the layer as effect points are) is **?**. The match names of the Overlap and +Starch pin atoms and their properties, of the Advanced and Bend pins (Scale, Rotation) and of +Density, Mesh Rotation Refinement and the Puppet Engine are **?**, and so is which of them are +`parT` parameters; whether After Effects stores some `parT` parameters of a fresh Puppet even at +their defaults (the importer would then report them every time) is **?**; whether a Puppet +without pins has an (empty) Mesh Group, or no ARAP group at all, is **?**. The importer +(`import/puppet.rs`) therefore brings the effect in as EffectCraft's Puppet without meshes or +pins and reports, per mesh, Triangles, Expansion and per pin group the pins, how many are keyed +or carry an expression, and their atom match names; every entry outside this tree is named in the +report. The sample that settles it: a fresh Puppet with no pins in one composition and, in +another, a 400 x 800 solid in a larger composition (pixels against fractions) with a Puppet of +three Position pins (one keyed, one moved off its rest place without +keys), one Advanced, one Bend, one Starch and one Overlap pin, Triangles, Expansion and Density +off their defaults, a second mesh, and a scripting read-back of every property's match name and +value, the hidden ones included. Pins added by script are reported to come out disabled, so the +pins are placed by hand or by a preset. + +### 3.8 Masks and bezier paths (`tdgp[ADBE Mask Parade]`) + +| Field | Chunk path | Offset | Type | Meaning | Evidence | Conf. | +|---|---|---|---|---|---|---| +| inverted | `Mask Parade/mkif` | 0 | u8 | | 090 (0), 091 (1) | V | +| **mode** | `mkif` | 4 | u32 | Add 1, Subtract 2, **Intersect 3, Lighten 4, Darken 5, Difference 6, None 0** (329: the same ellipse path as 232) | 090, 091, 234-237, 329 | V | +| name | `tdgp[ADBE Mask Atom]/tdsn/Utf8` | | text | `Mask 1` | 090 | V | +| feather, opacity, expansion | `ADBE Mask Feather`, `ADBE Mask Opacity`, `ADBE Mask Offset` | | see 3.4 | stored only when not at the default; feather can be keyed (plain n=2 records) | 090, 091, 208 | V | +| path property | `om-s[ADBE Mask Shape]/tdbs` | | | animated: `tdb4` byte 68 = 1 and a `list` of 64-byte records (time and common fields only) | 090, 093 | V | +| path shapes | `om-s/omks/LIST shap` | | | one per keyframe, one for a static path | 090, 093 | V | +| path never set | (no `ADBE Mask Shape` entry in the atom) | | | a mask added without a path (scripted `addProperty`) stores no Mask Path at all, also no `om-s`; After Effects reads it back, also after reopening, as the **source rectangle** `[0,0] [0,h] [w,h] [w,0]` (clockwise from the top left, no tangents) and renders it so (901: expansion -100 leaves a 100 px border) | 900, 901 | V | +| box | `shap/shph` | 4, 8, 12, 16 | 4 x f32 | left, top, right, bottom **normalised to the layer (source) size** for masks (560 / 1920, 340 / 1080, 1360 / 1920, 740 / 1080; on a 100x100 solid a 60x40 rectangle has the box 0.2, 0.3, 0.8, 0.7 and decodes exactly with the layer size, not the 1920x1080 composition). It is the box of the vertices **and** of the handles | 090, 232, 309 | V | +| **open path** | `shph` | 3 | u8 | bit `0x08` set = open path (`0x09` against `0x01`); byte 20 is 1 in every sample, open or closed | 233 against 090, 232, 234 | V | +| points | `shap/list` (`lhd3` record size 8) | | f32 x, y | three per vertex in the order **vertex i, out handle i, in handle i + 1**, normalised inside the box (u = (x - left) / (right - left)); the last record is the in handle of vertex 0. A **tangent is the handle minus the vertex**, without a sign change | 090 (rect), 093 (two rects), 232 (ellipse: tangents `0.5523 x radius`, `[-220.91, 0]`, `[0, -110.46]`, ...), 233 (open path, 3 vertices) | V | + +**There is one `mkif` per mask**, placed directly before its atom (two masks: `mkif tdgp mkif tdgp`, +328), so mode and inverted are reported per mask. `mkif` byte 8 is the running mask id counter of the +**layer** in the session (1, 2 for the two masks of 328; 329 then gets 3; undo does not reset it; a new +layer starts at 1 again): exposed as `Mask::counter`, a session counter and not a stable id. API: +`Mask::{mode, inverted, counter}`, `ShapePath::vertices(size)` (vertices in layer pixels: `size` is the size of the layer's source, `Project::layer_source_size`; a text or shape layer has no source size and its mask boxes are already pixels, so the importer passes 1 x 1), `Layer::mask_atoms` (`MaskAtom`: the mask with its atom group, its path property and its shapes by position, for the keys and expressions of the mask properties). Tests: +`tests/masks.rs` (with a synthetic check of the handle order), `tests/batch3_objects.rs`. + +### 3.9 Shape layers (`tdgp[ADBE Root Vectors Group]`) + +A shape layer (type 4) holds `ADBE Vector Group` (user name `Group 1`) with `ADBE Vectors Group` +(the contents, in z-order, `tdsb` 0x401) and `ADBE Vector Transform Group`. Items are named by their +match name and carry their own user name. Colours are ARGB doubles 0..255. All sizes, positions and +path points are in pixels. **An item's eye switch is bit 0 of its group's `tdsb`** (1 on; 986: a +Fill at 0 is not drawn, 987: the same file at 1 draws it; both saved by After Effects from a patched +copy of 850, the switch is not scriptable) [V]. + +| Item (match name) | Property | Stored | Evidence | Conf. | +|---|---|---|---|---| +| `ADBE Vector Shape - Rect` ("Rectangle Path 1") | `ADBE Vector Rect Size` | f64 `[400, 300]` (2 components) | 238-242 | V | +| `ADBE Vector Shape - Group` ("Path 1") | `om-s[ADBE Vector Shape]` | the same `shph` + point list as a mask, with a box in **pixels** relative to the layer origin; vertices `[0,-100] [100,100] [-100,100]`, in tangents `[-10,-20] [30,20] [-60,10]`, out tangents `[40,0] [-20,30] [0,-50]`; the box covers vertices and handles: `-160, -120, 130, 130` | 240 | V | +| `ADBE Vector Graphic - Fill` | `ADBE Vector Fill Color` | `[255, 0, 255, 0]` | 238, 240, 242 | V | +| `ADBE Vector Graphic - Stroke` | `Stroke Color` `[255, 0, 255, 0]`, `Stroke Width` 8, **`Stroke Line Cap` 2.0 (round; butt = 1 is absent)**, **`Stroke Line Join` 2.0 (round; miter = 1 is absent)**, `Stroke Miter Limit` 4 with `tdsb` 3 once the join was touched | doubles | 239 | V | +| `ADBE Vector Filter - RC` ("Round Corners 1") | `ADBE Vector RoundCorner Radius` | f64 20 | 238 | V | +| `ADBE Vector Graphic - G-Fill` ("Gradient Fill 1") | `ADBE Vector Grad Start Pt` `[-200, 0]`, `ADBE Vector Grad End Pt` `[200, 0]` | 2 components, class 15; the gradient type (linear) and the colour stops stay at After Effects' default and are not stored | 241 | V (points) | +| `ADBE Vector Transform Group` | `ADBE Vector Position` | f64 `[50, 50]`; absent = `[0, 0]`, as are all other transform properties at default | 242 | V | + +Batch 3 (all **V**; shape layer `Shape`, label 8): + +| Item | Stored | Evidence | +|---|---|---| +| `ADBE Vector Shape - Star` ("Polystar Path 1") | After Effects' defaults are star, 5 points, outer radius 100, inner radius 50, rotation 0, roundness 0, so **a default star has no property record at all** (the 150 / 70 of batch 1 were stored). A **polygon** stores `ADBE Vector Star Type` 2.0 (`tdsb` 1, descriptor `00020000`) plus `Inner Radius` 50 and `Inner Roundess` 0 (After Effects' own spelling) as `tdsb` 3 default-state records; Points 5 and Outer Radius 100 stay absent | 330, 331 | +| `ADBE Vector Graphic - G-Fill` | `ADBE Vector Grad Type` 2.0 for radial (`tdsb` 1, descriptor `00020000`; linear is 1 and absent); Start Point `[0, 0]` is the default and absent, End Point `[200, 0]` stored; the radial-only `Grad HiLite Length` and `Grad HiLite Angle` are written with `tdsb` 1 and value 0 (absent in the linear 241). Colour stops are in no property | 241, 332 | +| two siblings with one match name | one group holds `Rectangle Path 1` (Size `[400, 300]` stored), `Rectangle Path 2` (Size `[100, 100]` is the default, no record; `ADBE Vector Rect Position` `[300, 0]`, 2 components, class 15) and `Fill 1`, in z-order: a reader keys the items by occurrence, not by match name | 333 | + +Batch 5 (all **V**; shape layer `S`, Group 1 = Ellipse Path 1 [400,300] + Fill 1 green in 451, one change each): every value is stored **as After Effects shows it**, in the shape's own record (`tdsb` 1), and absent at its default: + +| Property (match name) | Stored | Evidence | +|---|---|---| +| `ADBE Vector Shape - Ellipse`: `ADBE Vector Ellipse Size`, `... Ellipse Position` | pixels `[500,200]`, `[100,50]` | 452, 453 | +| `ADBE Vector Graphic - Fill`: `ADBE Vector Fill Opacity`, `ADBE Vector Fill Rule` | percent 50; popup 1-based, even-odd = 2 | 454, 455 | +| `ADBE Vector Transform Group` of a group: `ADBE Vector Anchor`, `Scale`, `Rotation`, `Group Opacity`, `Skew`, `Skew Axis` | pixels `[10,20]`; **percent** `[50,80]` (not a fraction as the layer's Scale); degrees 30; percent 40; 20; degrees 45 | 456-461 | +| layer `ADBE Anchor Point` of a shape layer | pixels `[10,20,0]` as stored (no source size), as on a text layer | 462 | +| `ADBE Vector Graphic - Stroke`: `ADBE Vector Stroke Opacity` | percent 50 (colour `[255,255,0,0]` and width 6 as in 239) | 463, 464 | +| `ADBE Vector Filter - Trim`: `Trim Start`, `Trim End`, `Trim Offset`, `Trim Type` | percent 20, 70; degrees 45; popup 1-based, individually = 2; a new Trim Paths has no record | 465-469 | +| `ADBE Vector Rect Roundness` | pixels 30 | 470, 471 | +| `ADBE Vector Shape - Star`: `Points`, `Rotation`, `Inner Roundess`, `Outer Roundess`, `Position`, `Outer Radius`, `Inner Radius` | 6; degrees 30; percent 20, 40; pixels `[50,-20]`, 150, 30 | 472-479 | +| `ADBE Vector Graphic - G-Fill` added: Start `[0,0]` and End `[100,0]` are the defaults, no record; **the colour stops have no record at their default**, which renders white to black, linear in the encoded values (481: 0.75 / 0.5 / 0.25 at a quarter, half and three quarters) | 480, 481 | +| a second top-level group, a group nested in a group (`ADBE Vector Group` inside `ADBE Vectors Group`), `ADBE Vector Graphic - G-Stroke` (width 10) | the same records as at the top level, in z-order | 482, 483, 484 | +| defaults of every item (519: AE read-back with no record stored): rectangle and ellipse size 100 x 100, position 0, roundness 0; star 5 points, radii 100 / 50, rotation and roundness 0; fill red `[1,0,0]`, opacity 100, rule 1; stroke white, width 2, cap / join 1, miter 4; gradients linear, start `[0,0]`, end `[100,0]`; trim 0 / 100 / 0, type 1; round corners radius 10; group transform anchor / position 0, scale 100, opacity 100; blend mode, composite order, shape direction 1 | none of them has a record | 519 | + +The 3D-extrusion `ADBE Vector Materials Group` of a group changes its `tdsb` from 1 to 3 when a polystar or a gradient is added (472-484), with no property record. + +API: `Layer::shape_groups` (`ShapeGroup`, `ShapeItem`, `Stroke`, and `ShapeGroup::details` with +`ShapeDetail::{Polystar, Gradient, RectPosition}`, `PolystarType`, `GradientType`). Other items +(ellipse, trim paths, repeater, dashes, ...) are listed as `ShapeItem::Other` with their match name; +the polystar is listed as `Other` and the gradient fill as `GradientFill`; both carry their extra +fields in `details` (indexed by `item_index`); see section 4. Tests: `tests/shapes.rs`, +`tests/batch3_objects.rs`. + +### 3.10 Text documents (`btds[ADBE Text Document]/LIST btdk`) + +`btdk` is not a chunk sequence but a PostScript-like dictionary (`/key value` pairs, `<< >>`, `[ ]`, +numbers, `true` / `false`, `/names`, strings as `(` + `fe ff` + UTF-16BE + `)` with `\` escapes). The +verified paths from the top level (keys `/98 /0 /1`; `RUN` = `/1/1[0]/0`): + +| What | Path | Evidence | Conf. | +|---|---|---|---| +| text | `RUN/0`; a trailing `\r` ends the paragraph. **A hard line break is `\r` too: `A\rB` is stored `A\rB\r`** | 110 (`A`), 113 (`Hello`), 243 | V | +| **U+0003 in the text** | stored as it is (`A`, U+0003, `B` is `A\u{3}B\r`), with the paragraph runs of a CR (`[2, 2]`). After Effects lays it out exactly like `A\rB` (387): two composed lines, two paragraphs, baselines at y 0 and 98 (`text.measure` in AE). So U+0003 is a line break that starts a new paragraph; the importer maps it to `\r` | 386, 387; test `etx_is_stored_as_it_is_and_splits_the_paragraph_runs` | V | +| font size | `RUN/6/0[n]/0/0/6/1` (100 by default) | 110, 113 (60) | V | +| fill colour | `RUN/6/0[n]/0/0/6/53/0/1` = `[A R G B]` floats 0..1 | 110 (white), 113 (`[1 1 0.5 0]`), 248 | V | +| **paragraph runs** | `RUN/5/0[n]/1` = characters including the `\r` (`A\rB\r`: `[2, 2]`; `Hello\r`: `[6]`) | 243, 245 | V | +| **style runs** | `RUN/6/0[n]/1` = characters per style run (`A\rB\r`: `[4]`). **A range style splits `Hello\r`: characters 1 to 3 red gives `[1, 2, 3]` with the fills white, `[1 1 0 0]`, white** | 243, 248 | V | +| **justification** | `RUN/5/0[n]/0/0/5/0`: 0 = left, **1 = right (334)**, 2 = centre, **3 = justified with the last line left, 4 = last line right, 5 = last line centred, 6 = every line justified**; the scripting enum value of right (7414) is not what is stored | 245 (0), 246 (2), 334 (1), 565 (3), 567 (4), 566 (5), 568 (6) | V | +| **character settings (S17)** | one `text.set_style` change each against the point text 538 (`HHHH`, Times New Roman 100 px, normal baseline), the slot named is the only one that changes: `/6` horizontal scale as a factor (539: 1.0 -> 0.9), `/7` vertical scale (540: 1.5), `/9` baseline shift in px (541: 41), `/11` automatic kerning **0 none, 1 metrics (default), 2 optical** (542, 543), `/18` ligatures (off by default, 555; switching ligatures off in 551 changes nothing), `/35` baseline direction **1 with the stream (default), 2 rotated (573), 3 across the stream / tate-chu-yoko (576)**, `/36` tsume as a fraction (553: 0.5), `/52` no break (552), `/56` fill drawn (on by default, 546), `/57` stroke drawn (off by default, 544), `/58` **stroke over fill (on by default: 547 setting it on changes nothing, 550 setting it off gives false)**, `/62` line join **0 miter, 1 round** (548), `/63` stroke width in px (**1.0 by default**, 545: 5), `/70` digit set **0 default, 1 Arabic (571), 2 Hindi (578)**; with digit set Arabic the text `1234` has the same layer bounds as with the default (S17, not saved) | 538-578, 595, 596; tests `tests/text_style_slots.rs` | V | +| **manual kerning** | `RUN/8/0` (beside the style runs): one entry `<< /0 << /0 amount >> /1 length >>` per run of characters, amount in 1/1000 em; kerning 50 on `HHHH` gives four entries of length 1 with 50 and one for the closing `\r` without an amount, `RUN/7` = 50, and switches automatic kerning `/11` to 0 | 554 | V | +| **paragraph settings (S17)** | `RUN/5/0[n]/0/0/5/`, one change each against 538: `/1` first-line indent (560: 20), `/2` start indent (561: 30), `/3` end indent (562: 40), `/4` space before (563: 10), `/5` space after (564: 12), all in px; `/8` leading model **0 Roman, 1 Japanese** (572); `/9` automatic hyphenation (**on by default**, 556 off); `/21` Roman hanging punctuation (558); `/29` **every-line composer, off by default** (559 on; 557 setting it off changes nothing, so a new text uses the single-line composer); `/33` direction **0 left to right, 1 right to left** (569). Setting the Universal composer (570) changes nothing; the Latin/CJK composer cannot be set by script | 556-572 | V | +| **vertical text** | `lineOrientation` vertical right to left adds `/0/8/0[0]/0/2/1` = 2 to the frame dictionary (absent for horizontal text) | 575 | V | +| **box (paragraph) text** | frame dictionary `/0/8/0[0]/0`: `/1/0` = 32 floats = 16 x, y pairs tracing the rectangle (corners `[-200,-100] [200,-100] [200,100] [-200,100]`), `/2/0` = 1; both keys are absent for point text. Alignment changes only the glyph geometry, not the frame | 245, 246 (box 400 x 200, top-left `[-200, -100]`), 247 (point text: absent) | V | +| **tracking** | `RUN/6/0[n]/0/0/6/8` = 0, 100 in 247 | 247 | V | +| **faux bold / faux italic** | style slots `.../6/2` and `.../6/3`, `false` -> `true`; the font style stays Regular. **Side effect: the font list `/0/1/0` gains a new first entry (4 -> 5 fonts) with property `/3` = 2 (bold) or 1 (italic), the old entries shift up and the style's font index `/6/0` becomes 1** | 335, 336 | V | +| **caps** | style slot `.../6/12`: 0 normal, **1 small caps, 2 all caps** | 337, 338 | V | +| **superscript / subscript** | style slot `.../6/13`: **1 superscript, 0 normal, 2 subscript** (the only slot that changes between 526, 527 and 528). After Effects lays superscript and subscript out at 58.3 % of the size, 33.3 % of the size above / below the baseline: `HHHH` in Times New Roman 100 px is 288.9 px wide with its baseline at 0 when normal, 168.4 px with its baseline at -33.3 / +33.3 when superscript / subscript (`text.measure`, layer bounds). **Every scripted text of the corpus is superscript** (526: the character state After Effects was in; 243, 517 and the reference 244 too), which is why this slot never showed up as undecoded | 526-528, 243, 517 | V | +| **wrapping box text** | a 600 x 400 box at `[-300, -200]` (as in 245) with a sentence of 70 characters: the document holds the text, size and box only; After Effects composes four lines (`The quick brown fox ` / `jumps over the lazy dog ` / `and keeps running far ` / `away`), the line breaks are not stored | 517 | V | +| **leading** | style slot `.../6/5`: the scripted samples have the explicit leading 98.0 (100 px Times New Roman; `/4` false), leading 60 is 60.0; with automatic leading on, 400 stores 0.01 (see "automatic leading" below) | 339, 400 | V | +| **last run** | the last style run reaches the paragraph end and includes the final `\r` (two range styles on `Hello`: runs `[1, 2, 3]`, fills white / `[1 1 0 0]` / `[1 0 0 1]`, sizes 100 / 100 / 60) | 340 | V | +| font | `.../6/0` = index into the font list `/0/1/0[i]/0/0/0` (`TimesNewRomanPSMT`, ...). The list always holds entries no run uses (`Myriad-Roman`, `MyriadPro-Regular`, `AdobeInvisFont`), so only the fonts the runs point at matter | 110, 335 (index 1 is still `TimesNewRomanPSMT`) | E | +| **stroke colour** | `.../6/54/0/1` = `[A R G B]`, the fill's layout; white in the untouched samples, red `[1 1 0 0]` in 574, blue in 595. The stroke switch, width and order are slots `/57`, `/63`, `/58` (see "character settings") | 110, 113, 574, 595 | V | +| **automatic leading** | style slot `.../6/4`: `true` = automatic leading (lines advance by 1.2 x the size; `/5` is then not used), `false` = `/5` is the leading. **243: `/4` false, `/5` 98, the second line's baseline is stored 98 px below the first (`/1/1[0]/1/2[0]/6[0]/6[0]/6[0]/6[1]/0/0[1]`)**. **400 (Auto Leading switched on for `A` CR `B`, size 100, leading 98; against its unedited save 399): the run style changes in `/4` (false -> true) and `/5` (98 -> 0.01) only, the stored second baseline moves from 98 to 120 px below the first (1.2 x 100), and After Effects measures the same 120 px (`text.measure`)**; the rest of the diff is save noise (`head`, the comp's modification time, the text's `btgu/pgui`). That `/5` holds 0.01 under automatic leading is seen in 400 only | 243, 339 (`/5`); 400 (`/4`); test `automatic_leading_is_slot_4` | V | +| **undecoded slots** | the run style and paragraph dictionaries of an untouched document (244; 110 stores the same bytes) are the reference: a slot that is not decoded and differs from it is listed (`TextDocument::undecoded_style_fields`, `undecoded_paragraph_fields`); a slot the reference lacks is listed only when it is not 0, `false`, empty or `/nil` (a neutral value is taken to carry nothing, so it is not reported as a loss; `/nil` is how 244 stores its paragraph slots `/27`, `/28` and `/36`). Decoded style slots: `/0 /1 /2 /3 /4 /5 /6 /7 /8 /9 /11 /12 /13 /18 /35 /36 /52 /53 /54 /56 /57 /58 /62 /63 /70`; paragraph slots: `/0 /1 /2 /3 /4 /5 /8 /9 /21 /29 /33`. Not found by any scriptable setting (none of the `text.set_style` attributes tried in S17 changes them): style `/20 /38 /78 /87 /88`, paragraph `/31 /32`. **Paragraph `/24`, `/35` and style `/68`** (1, 1 and 0 in a scripted text) were edited in place in the wrapping box text 517 (732: `/24` 0, 733: `/35` 0, 734: `/68` 1, same length); After Effects opens each without a dialog, keeps the value on a re-save and composes the same four lines, baselines and bounds (`text.measure`, `layer.bounds`; 734 renders like 517), so 0 and 1 are not listed (`NEUTRAL_PARAGRAPH_VALUES`, `NEUTRAL_STYLE_VALUES`; their meaning stays unknown, Latin text only). **Paragraph `/34`** is 0 in a scripted text and 7 after Reset Paragraph (727: the only slot that changes); a justified 600 px box text of 88 characters composes the same four lines, baselines and bounds with both values (S18), so 0 and 7 are not listed (`NEUTRAL_PARAGRAPH_VALUES`), other values are | every text sample lists none; 725, 727 | V (reference), E (/34) | +| **Source Text record** | the `tdbs` in `btds[ADBE Text Document]` (next to the `btdk`) has no keys and no expression in every sample; Source Text keyframes (presumably one `btdk` per key) and expressions were not sampled, the reader only reports them (`Layer::source_text_property`, `Layer::text_document_count`) | 110-114, 243-248, 334-340 | E | + +API: `TextDocument` (`paragraphs`, `style_runs`, `text_box`, `paragraph_texts`, `font_of`, +`incomplete_runs`, `undecoded_style_fields`, `undecoded_paragraph_fields`, `kerning`, +`line_orientation`), `StyleRun::{faux_bold, faux_italic, caps, auto_leading, leading, stroke_argb, +font_index, horizontal_scale, vertical_scale, baseline_shift, auto_kern, ligatures, +baseline_direction, tsume, no_break, apply_fill, apply_stroke, stroke_over_fill, line_join, +stroke_width, digit_set}`, `ParagraphRun::{first_line_indent, start_indent, end_indent, space_before, +space_after, leading_type, auto_hyphenate, hanging_roman, every_line_composer, direction}`, `Caps`, +`AutoKern`, `KernRun`, `Justification`. +A damaged run is never dropped silently: a run without a readable length is left out, a run with a +missing size, fill or font index takes them from the run before it, and both count in +`incomplete_runs`. Tests: `tests/text_markers_footage.rs`, `tests/batch3_objects.rs`, +`tests/text_fields.rs`. + +### 3.11 Markers + +`mrst[ADBE Marker]` holds a `tdbs` with a `list` of 16-byte records (the times; `tdb4` byte 12 is the +time scale) and one `LIST mrky/LIST Nmrd` per marker. Layer markers live in the layer's own tree, +composition markers in the `SecL` list named `Markers`. + +| What | Chunk path | Meaning | Evidence | Conf. | +|---|---|---|---|---| +| time | the key list | 25 600 = 1 s | 122, 123, 249 | V | +| **duration** | `Nmrd/NmHd` (20 bytes: u32 4, u32 0, u32 duration, u32 600, u32 0) | duration / 600 = the marker duration in seconds (1 s is 600; 0 for none); 600 is the units per second; **0.5 s and 0.3 s read back exactly (509, 510)** | 249 (1 s), 122 (0), 509, 510 | V | +| **several markers** | the key records and the `Nmrd` lists in the same (time) order: 509 has two composition markers, 1 s `M1` label 2 and 2.5 s `M2` duration 0.5; a fractional time is a plain key time (0.52 s = 13 312 / 25 600, 510) | | 509, 510; test `tests/batch5.rs` | V | +| **strings** | five `Utf8` children of `Nmrd`, in the order **comment, chapter, URL, frame target, cue point name**; unused ones are empty | `LM`, `Chap`, `http://a.example/u`, `_blank`, `CueX` | 249 | V | +| **cue type** | `NmHd` u32 @0: **4 = event cue (also for markers without a cue point name), 5 = navigation cue** | 347 (4), 348 (5) | V | +| **cue parameters** | `NmHd` u32 @4 = the number of key / value pairs; the pairs follow the five fixed strings as `2 x pairs` extra `Utf8` (key, value, key, value): `speaker` = `A`, `n` = `2` with the cue point name `CueE` | 349 | V | +| **label** | `NmHd` u8 @16 (followed by three pad bytes): label 3 is `03 00 00 00`. The label is not in the keyframe flags (unlike the keyframe label of 057) | 347 | V | + +API: `Marker::{label, cue_type, cue_params}`, `CueType`. Tests: `tests/text_markers_footage.rs`, +`tests/batch3_objects.rs`. + +### 3.12 Cameras and lights + +Both have no source item (`ldta@40` is 0 for a camera, `0xffff_ffff` for a light), type 2 or 1, the 3D +bit and byte 38 bit `0x40` (3.1); `ldta@136` is the light type. Their properties sit in the same +groups as on other layers, with these conventions: + +| What | Path | Stored | Evidence | Conf. | +|---|---|---|---|---| +| camera Position, default (auto state) | `ADBE Transform Group/ADBE Position`, `tdb4` 76 and 77 = 1 although no expression exists | `[0, 0, -2666.67]` (x and y are 0 whatever the composition size; z = -zoom). A **one-node** camera reports `[0, 0, -2666.67]` (251). **Correction (S16): a reopened two-node camera reports `[0, 0, -2666.67]` too (256 read back), and After Effects renders it there**: the default two-node camera of 537 above the 3D solid of 486 renders from `[0, 0, -2666.67]` towards the point of interest, not like the composition's default view (537 differs from 486 in 24 % of the grid points, and EffectCraft's import with the stored position matches 537 but for 4 edge points). The `[960, 540, ...]` of the batch-3 notes was not reproduced (535 and 537, created by script, read back the stored value too); the reader keeps only the stored value (`CameraPosition::stored`) | 256, 251, 537 | V | +| camera Position, set by the user | same, `tdb4` 76 and 77 = 0 | the absolute value `[100, 200, -1000]` | 253 | V | +| camera point of interest | `ADBE Anchor Point` | `[960, 540, 0]` in **pixels** (not a fraction); the record's `tdsb` is 1 on a two-node and 3 on the one-node camera (not asserted) | 251, 253, 256 | V (value), E (`tdsb`) | +| light Position, default | same as the camera | `[0, 0, -1333.33]`, as After Effects reports it (255 read back); the point of interest `[960, 540, 0]` | 254, 255 | V | +| light intensity | `ADBE Light Options Group/ADBE Light Intensity` | f64 50.0 percent, `tdsb` 1; absent at the default 100 | 254 (50), 252 (absent) | V | +| light type | `ldta@136` | 1 spot, 2 point | 254, 252 | V | +| **camera Zoom** | `ADBE Camera Options Group/ADBE Camera Zoom` | plain pixels (`tdsb` 1), absent at the default 2 666.67; the auto Position keeps `[0, 0, -2666.67]` and is not recomputed from the zoom | 341 (2000) | V | +| **camera depth of field** | same group: `ADBE Camera Depth of Field` 1.0 (checkbox descriptor `ffff0004`), `ADBE Camera Focus Distance` 800, `ADBE Camera Aperture` 30 (plain numbers), all `tdsb` 1. **Defaults (256 read back): Depth of Field off, Focus Distance = Zoom = 2 666.67, Aperture 25.309 336 332 958 4, Blur Level 100** | 342, 256 | V | +| **light Casts Shadows** | `ADBE Light Options Group/ADBE Casts Shadows` | a set record (`tdsb` 1) holding 0.0 (off) in every light | 252, 254, 255 | V | +| **3D rotations** | `ADBE Rotate X`, `ADBE Rotate Y` | plain degrees, absent at 0 | 486, 487 | V | +| **Orientation** | `LIST otst` = `tdbs` + `LIST otky/otda`: `otda` is three big-endian `f64` degrees; absent at `[0, 0, 0]`. Keyed orientation is not sampled | 488, 485 | V | +| **light colour** | `ADBE Light Options Group/ADBE Light Color` | ARGB doubles 0..255 (`[255, 255, 0, 0]` for red; 4 components, descriptor `0002ffff`), absent = white | 343 | V | +| **light cone** | same group: `ADBE Light Cone Angle` 60 and `ADBE Light Cone Feather 2` 30 (percent), each `tdsb` 1 and absent in the default spot light (90 and 50). Every point light stores them as `tdsb` 3 records with 90 and 50 (a point light created directly equals the converted one of 252, so the "odd" property set is not a conversion artefact) | 344, 345, 346, 252 | V | + +A point light made by converting a spot light has a different property set (`Orientation`, `Rotate +X/Y/Z`, `Cone Angle`, `Cone Feather 2` in the default state) that a spot light lacks (252); whether that +is an artefact of the conversion is untested. API: `Layer::camera_position`; the camera and light +options are read by the importer's table (`import/camera.rs`); `DEFAULT_CAMERA_ZOOM` (2 666.67 is verified for the 1920 px +composition only), `Layer::orientation` (`Orientation`; test `orientation_is_otst_otda` in +`tests/batch5.rs`). Render checks against After Effects (20 px grid, 5 184 points): 486 1 point +off, 535 2, 536 0, 537 4 (anti-aliased edges); **488 (Orientation `[10, 20, 30]`): 272 points +(5.25 %) differ; the import carries the stored angles, and permuted or negated inputs match worse +(6.2 % to 34 %), so the cause is in EffectCraft's renderer and not isolated.** Tests: `tests/layer_switches.rs`, `tests/batch3_objects.rs`. + +### 3.12a Project settings (top-level chunks; cases 620-634) + +Each case is the base reopened and saved with one setting changed (`project.set_settings`), compared +with 620 (the base saved the same way); apart from the save noise (`head` byte 19, `nhed` / `nnhd` +bytes 20-27 / 28-35, `fdta`) only the bytes below changed. + +| setting | where | stored | cases | conf | +|---|---|---|---|---| +| colour depth | `nhed` byte 15 (and `nnhd` byte 24) | 0 = 8, 1 = 16, 2 = 32 bits per channel (the base is 32) | 620, 621, 622, 631 | V | +| working space | `cpid` (16 bytes) and the `Utf8` right after `PwCs` | None: `cpid` all `0xff` and the JSON `{}`; a profile: JSON `{"baseColorProfile":{"colorProfileData":"","colorProfileName":""},"baseProfileType":2}` (`sRGB IEC61966-2.1` in the base, `Rec.709 Gamma 2.4` in 624) and `cpid` a 16-byte id of the profile; After Effects 26.2.1 leaves `colorProfileName` out in a fresh save (settings-c), and the name is then the profile's ICC `desc` tag (v2 `desc` or v4 `mluc`) | 620, 623, 624, 631, 902 | V (None, the name), ? (`cpid`) | +| working gamma | `dwga` `u32` @0 | 0 = 2.2, 1 = 2.4 | 625, 631 | V | +| Linearize Working Space | `lnrp` (1 byte, 1) inserted after `cpid` | absent when off | 626, 632 | V | +| Blend Colors Using 1.0 Gamma | `lnrb` (1 byte, 1) inserted after `cpid`, before `lnrp` | absent when off | 627, 632 | V | +| Compensate for Scene-referred Profiles | `acer` byte 0 | 1 on (base), 0 off | 628 | V | +| time display | `nhed` (and `nnhd`) byte 8 | 0 timecode, 1 frames; feet + frames: byte 8 = 1 and byte 11 = 1, byte 13 = frames per foot (16 = 35 mm, 40 = 16 mm; the base holds 16) | 629, 633, 634 | V | +| frame count start | `nhed` byte 14 (`nnhd` byte 20) | 2 for a count from 0 (base), 1 for a count from 1 | 630 | E (one value each) | + +The audio sample rate of the project is `adfr` (`f64` 48 000 in every case, not varied). API: +`ProjectSettings::read` (`TimeDisplay`, `WorkingSpace`). Test: `tests/it/container/cases_600.rs`. The +fixture of 624 is not in the repository: its working-space JSON holds the profile data After +Effects embeds for that space. + +### 3.12b Layer styles (`tdgp[ADBE Layer Styles]`; cases 950-980) + +Every layer's root group holds `ADBE Layer Styles` (third, after Time Remapping and Transform), also +when no style is used (950). Inside: `ADBE Blend Options Group` (Global Light Angle `ADBE Global +Angle2`, Altitude `ADBE Global Altitude2`, and `ADBE Adv Blend Group` with `ADBE Layer Fill +Opacity2`, `ADBE R/G/B Channel Blend`, `ADBE Blend Interior`, `ADBE Blend Ranges`; 977), then one +switch group per style, `café <3", opts); +let mut text = String::new(); +while let Some(token) = lexer.next() { + match token { + Token::Text(t) => text.push_str(&unescape(t)), + // A style sheet's text, for a caller that reads it. + Token::Start(tag) if tag.name == "style" => assert_eq!(lexer.raw_content("style"), "p{}"), + Token::Start(tag) => assert_eq!(tag.attr("class").as_deref(), Some("x")), + Token::End(name) => assert_eq!(name, "b"), + } +} +assert_eq!(text, "café <3"); +assert_eq!(escape(&text), "café <3"); +``` + +## The tokenizer + +`Lexer` is an iterator of `Token::Text`, `Token::Start(Tag)` and `Token::End(name)`, borrowed +from the input. Text and attribute values come as written; `unescape` and `Tag::attr` decode +them. + +- A `<` that starts no tag is text. A tag, comment or raw element that never ends runs to the + end of the input. +- Comments, doctypes, processing instructions and conditional comments never come out. +- `Options::drop`: elements skipped whole, content and end tag included. +- `Options::raw_text`: elements whose content is one text token, not markup. +- `Lexer::raw_content(name)`: after a start tag, take the element's raw content yourself. +- `Limits`: attributes kept per tag (64) and the longest tag name (64 bytes; a longer one has an + empty name). + +## Character references + +`unescape` decodes named, decimal and hex references. The named set is HTML 4's (Latin-1, Greek, +symbols, arrows, mathematics) and a few more. The 94 legacy names decode without their `;` +(`© 2024`), longest match first, as in browsers; `unescape_attr` leaves such a name alone +when a letter, a digit or `=` follows, so a query string keeps its `©=1`. A numeric reference +to 0x80-0x9F means the Windows-1252 character; 0, surrogates, and numbers past U+10FFFF or too +long to be numbers become U+FFFD. Names are looked at for 32 bytes at most, so the work is +linear. + +`escape` writes `& < > " '` as references; `escape_dq` leaves `'`, for content and attribute +values in double quotes. + +## HTML documents (feature `flow`) + +`craft_html::doc` reads a page or a pasted fragment into the [craft-flow](../flow/README.md) +model and a craft-wordmodel `Document`, and writes one as clean HTML. The reader is a tolerant +tag-soup builder on this crate's tokenizer (a stack of open elements with HTML's implicit closes), +with a CSS subset from `style` attributes and `\n\n\n", + ); + blocks_html(&flow.blocks, &mut out, 0); + out.push_str("\n\n"); + out +} diff --git a/crates/html/src/doc/import.rs b/crates/html/src/doc/import.rs new file mode 100644 index 000000000..c653f4b44 --- /dev/null +++ b/crates/html/src/doc/import.rs @@ -0,0 +1,125 @@ +use super::builder::{Builder, ImageLoader, parse_sheet}; +use super::lexer::{Lexer, Tok}; +use super::math_ruby::{read_math, read_ruby}; +use super::{Document, Flow, Inline, model}; + +/// Parse HTML text into the flow model. +pub fn parse(s: &str) -> Flow { + parse_with(s, &|_| None) +} + +/// [`parse`], loading pictures that aren't `data:` URIs through `images`. +pub fn parse_with(s: &str, images: ImageLoader) -> Flow { + parse_in(s, images, false) +} + +pub(super) fn parse_in(s: &str, images: ImageLoader, page: bool) -> Flow { + // HTML normalizes input newlines before tokenization, not after decoding character references. + let normalized = if s.contains('\r') { + let mut out = String::with_capacity(s.len()); + let mut chunks = s.split('\r'); + out.push_str(chunks.next().unwrap_or_default()); + for chunk in chunks { + out.push('\n'); + out.push_str(chunk.strip_prefix('\n').unwrap_or(chunk)); + } + std::borrow::Cow::Owned(out) + } else { + std::borrow::Cow::Borrowed(s) + }; + // The lexer's offsets (a style element's text is cut out by them) are into the normalized text. + let s: &str = &normalized; + let mut lx = Lexer::new(s); + let mut b = Builder::new(images, page); + let mut guard = 0usize; + while let Some(t) = lx.next() { + guard += 1; + if guard > 50_000_000 { + break; + } + match t { + Tok::Text(t) => b.text(&t), + Tok::Start { name, attrs, self_close } => { + b.close_head_for_start(&name); + if craft_math::mathml::local(&name) == "math" && !self_close { + let el = read_math(&mut lx, name, attrs); + if !b.in_head() + && let Some((linear, display)) = craft_math::mathml::read(&el) + { + b.start_para().inlines.push(Inline::Equation { linear, display }); + } + continue; + } + if name == "ruby" && !self_close { + let r = read_ruby(&mut lx); + if !b.in_head() { + b.push_ruby(&r.base, &r.ruby); + } + continue; + } + if matches!(name.as_str(), "script" | "style" | "noscript" | "template" | "textarea") && !self_close { + let from = lx.pos(); + let to = lx.skip_past_end_tag(&name); + if name == "style" { + parse_sheet(s.get(from..to).unwrap_or(""), &mut b.sheet); + } + continue; + } + b.start(&name, &attrs, self_close); + } + Tok::End(name) => b.end(&name), + } + } + b.finish() +} + +/// Collapse white space inside ruby markup as body text does (HTML's white space only: an +/// ideographic space is text), without the space the element's indentation leaves at the edges. +pub(super) fn collapse_ws(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if matches!(c, ' ' | '\t' | '\n' | '\r' | '\u{000C}') { + if !out.ends_with(' ') { + out.push(' '); + } + } else { + out.push(c); + } + } + out.trim_matches(' ').to_string() +} + +/// Parse HTML into a document. +pub fn import(s: &str) -> Document { + model::begin_import(); + model::to_doc(&parse(s)) +} + +/// [`import`], loading pictures that aren't `data:` URIs through `images`. +pub fn import_with(s: &str, images: ImageLoader) -> Document { + model::begin_import(); + model::to_doc(&parse_with(s, images)) +} + +/// A web page opened as a document, as Word opens one (a file, or a document's HTML +/// `w:altChunk`): its text is Times New Roman 12 pt on single lines, headings bold and black at +/// the sizes browsers give them, and a `p` or heading whose CSS leaves a margin out (or writes it +/// invalidly) gets HTML's automatic spacing there; a margin a span states replaces the one its +/// paragraph states (both as Word's import does). [`import`] reads a pasted fragment, which takes +/// the document's own formatting instead (but its CSS still applies). +pub fn import_page(s: &str, images: ImageLoader) -> Document { + model::begin_import(); + model::to_doc(&parse_in(s, images, true)) +} + +/// [`import_page`] onto `page` (its size and margins), which tables size to: HTML gives no page +/// size, so the caller picks it (the region's paper, #1381). +pub fn import_on(s: &str, images: ImageLoader, page: &craft_wordmodel::SectionProps) -> Document { + model::begin_import(); + model::to_doc_in(&parse_in(s, images, true), page.clone()) +} + +/// The font of a web page's text when its CSS names none. +pub const PAGE_FONT: &str = "Times New Roman"; +/// Heading sizes on a web page, `h1`–`h6`, points. +pub(super) const PAGE_HEADING_SIZES: [f32; 6] = [24.0, 18.0, 13.5, 12.0, 10.0, 7.5]; diff --git a/crates/html/src/doc/lexer.rs b/crates/html/src/doc/lexer.rs new file mode 100644 index 000000000..a29d6d38b --- /dev/null +++ b/crates/html/src/doc/lexer.rs @@ -0,0 +1,50 @@ +//! The crate's tokenizer ([`crate::Lexer`]) as the document reader takes its tokens: text and +//! attribute values decoded, element and attribute names in lower case. + +use crate::{Token, unescape, unescape_attr}; + +#[derive(Debug)] +pub(super) enum Tok { + Text(String), + Start { name: String, attrs: Vec<(String, String)>, self_close: bool }, + End(String), +} + +pub(super) struct Lexer<'a> { + inner: crate::Lexer<'a, 'static>, +} + +impl<'a> Lexer<'a> { + pub(super) fn new(s: &'a str) -> Lexer<'a> { + Lexer { inner: crate::Lexer::new(s) } + } + + /// Where the next token starts. + pub(super) fn pos(&self) -> usize { + self.inner.pos() + } + + /// Read again from `pos` (an earlier [`Lexer::pos`]). + pub(super) fn set_pos(&mut self, pos: usize) { + self.inner.set_pos(pos); + } + + /// Skip past the end tag of raw text element `tag`; returns where that end tag starts (the + /// end of the input when there is none). + pub(super) fn skip_past_end_tag(&mut self, tag: &str) -> usize { + let from = self.inner.pos(); + from + self.inner.raw_content(tag).len() + } + + pub(super) fn next(&mut self) -> Option { + Some(match self.inner.next()? { + Token::Text(t) => Tok::Text(unescape(t)), + Token::Start(tag) => Tok::Start { + attrs: tag.attrs.iter().map(|(n, v)| (n.to_ascii_lowercase(), unescape_attr(v))).collect(), + name: tag.name, + self_close: tag.self_close, + }, + Token::End(name) => Tok::End(name), + }) + } +} diff --git a/crates/html/src/doc/math_ruby.rs b/crates/html/src/doc/math_ruby.rs new file mode 100644 index 000000000..3a81cde50 --- /dev/null +++ b/crates/html/src/doc/math_ruby.rs @@ -0,0 +1,156 @@ +use super::builder::block_kind; +use super::lexer::{Lexer, Tok}; + +/// A `` element (#816): its tokens up to the matching end tag, as a tree. Nesting and size +/// are bounded; an unclosed element ends with the input. +pub(super) fn read_math(lx: &mut Lexer, name: String, attrs: Vec<(String, String)>) -> craft_math::mathml::El { + use craft_math::mathml::{El, Item, MAX_NODES, local}; + let mut stack: Vec = vec![El { name, attrs, children: Vec::new() }]; + let mut count = 0usize; + let close = |stack: &mut Vec| { + if stack.len() > 1 + && let Some(done) = stack.pop() + && let Some(parent) = stack.last_mut() + { + parent.children.push(Item::El(done)); + } + }; + while let Some(t) = lx.next() { + count += 1; + if count > MAX_NODES * 4 { + // Over the cap: drop the rest of this element so its tokens aren't read as HTML text. + skip_math(lx, t); + break; + } + match t { + Tok::Text(t) => { + if let Some(top) = stack.last_mut() + && !t.is_empty() + { + top.children.push(Item::Text(t)); + } + } + Tok::Start { name, attrs, self_close } => { + let el = El { name, attrs, children: Vec::new() }; + if self_close || stack.len() > craft_wordmodel::math::MAX_DEPTH { + if let Some(top) = stack.last_mut() { + top.children.push(Item::El(el)); + } + } else { + stack.push(el); + } + } + Tok::End(name) => { + let name = local(&name).to_string(); + // Close up to the matching open element; a stray end tag is ignored. + if let Some(k) = stack.iter().rposition(|e| local(&e.name) == name) { + if k == 0 { + break; + } + while stack.len() > k { + close(&mut stack); + } + } + } + } + } + while stack.len() > 1 { + close(&mut stack); + } + stack.pop().unwrap_or_default() +} + +/// Skip tokens (starting with `first`) up to and including the `` that closes the element +/// being read; nested `` elements are counted. An unclosed element ends with the input. +fn skip_math(lx: &mut Lexer, first: Tok) { + use craft_math::mathml::local; + let mut depth = 0usize; + let mut t = Some(first); + while let Some(tok) = t { + match tok { + Tok::Start { name, self_close: false, .. } if local(&name) == "math" => depth = depth.saturating_add(1), + Tok::End(name) if local(&name) == "math" => { + if depth == 0 { + return; + } + depth = depth.saturating_sub(1); + } + _ => {} + } + t = lx.next(); + } +} + +/// A `` element's base text and reading (#1681). +#[derive(Default)] +pub(super) struct RubyText { + pub(super) base: String, + pub(super) ruby: String, +} + +/// Which part of a `` element the text being read belongs to. +#[derive(Clone, Copy, PartialEq, Eq)] +enum RubyPart { + /// The text being annotated: an `` child, or the text outside ``/``. + Base, + /// The reading: an `` child. + Ruby, + /// `` fallback parentheses, which are not text: browsers show them only where ruby isn't + /// supported. + Fallback, +} + +/// A block-level tag ends an open `` implicitly, as it does in a browser. +fn ends_ruby(t: &Tok) -> bool { + match t { + Tok::Start { name, .. } | Tok::End(name) => block_kind(name).is_some(), + Tok::Text(_) => false, + } +} + +/// Read a `` element's content: its base text from `` children (or the text outside +/// them), its reading from `` children, and `` fallback parentheses dropped. Tolerant, +/// like the rest of the reader: tags inside are ignored, an unclosed element ends at the next +/// block-level tag (handed back to the caller) or with the input, and the tokens read are capped. +pub(super) fn read_ruby(lx: &mut Lexer) -> RubyText { + /// Most tokens one `` element is read from: a missing end tag must not scan the rest of + /// a hostile file here. + const MAX_TOKENS: usize = 4096; + let mut out = RubyText::default(); + let mut part = RubyPart::Base; + let mut nested = 0usize; + let mut count = 0usize; + while count < MAX_TOKENS { + count += 1; + let start = lx.pos(); + let Some(t) = lx.next() else { break }; + if nested == 0 && ends_ruby(&t) { + // The boundary ends the element: give the token back to the main reader. + lx.set_pos(start); + break; + } + match t { + Tok::Text(t) => match part { + RubyPart::Base => out.base.push_str(&t), + RubyPart::Ruby => out.ruby.push_str(&t), + RubyPart::Fallback => {} + }, + // The self-closing slash is ignored on HTML elements (`` opens an `rt`), as in a + // browser's parser. + Tok::Start { name, self_close, .. } => match name.as_str() { + "rb" => part = RubyPart::Base, + "rt" => part = RubyPart::Ruby, + "rp" => part = RubyPart::Fallback, + "ruby" if !self_close => nested += 1, + _ => {} + }, + Tok::End(name) => match name.as_str() { + "rb" | "rt" | "rp" => part = RubyPart::Base, + "ruby" if nested > 0 => nested -= 1, + "ruby" => break, + _ => {} + }, + } + } + out +} diff --git a/crates/html/src/doc/tests.rs b/crates/html/src/doc/tests.rs new file mode 100644 index 000000000..640ab07d0 --- /dev/null +++ b/crates/html/src/doc/tests.rs @@ -0,0 +1,769 @@ +use super::*; + +#[test] +fn numeric_entity_rejects_sign() { + assert_eq!(unescape("a&#+65;b"), "a&#+65;b"); + assert_eq!(unescape("a&#x+41;b"), "a&#x+41;b"); + assert_eq!(unescape("aAAb"), "aAAb"); +} + +#[test] +fn important_declarations_win_the_cascade() { + let color = |html: &str| fmt_of(paras(&parse(html))[0]).color; + let red = Some(craft_wordmodel::Rgb(255, 0, 0)); + let blue = Some(craft_wordmodel::Rgb(0, 0, 255)); + assert_eq!(color("

x

"), blue); + assert_eq!(color("

x

"), red); + assert_eq!(color("

x

"), blue); + assert_eq!(color("

x

"), red); +} + +#[test] +fn a_declared_charset_decodes_the_page() { + // ISO 8859-2 (Central European): 0xA9 Š, 0xB9 š; named by the caller or the page. + assert_eq!(decode_as(&[0xA9, b' ', 0xB9], Some("iso-8859-2")), "Š š"); + assert!(decode_as(b"

\xA9

", None).contains("

Š

")); + assert_eq!(decode_as(&[0x8A], Some("windows-1252")), "Š"); + // A page that says UTF-16 without a BOM is read as UTF-8, as browsers do. + assert_eq!(decode_as("é".as_bytes(), Some("utf-16")), "é"); +} + +fn texts(f: &Flow) -> Vec { + f.blocks + .iter() + .map(|b| match b { + FBlock::Para(p) => format!("{:?}{:?}:{}", p.kind, p.list.map(|l| (l.ordered, l.level)), p.text()), + FBlock::Table(t) => format!("table{}x{}", t.rows.len(), t.cols()), + }) + .collect() +} + +fn table_style(html: &str) -> Option { + let doc = craft_flow::to_doc(&parse(html)); + doc.body.iter().find_map(|b| match &**b { + craft_wordmodel::Block::Table(t) => Some(t.props.style.clone().unwrap_or_default()), + _ => None, + }) +} + +#[test] +fn borderless_tables_get_no_grid_style() { + assert_eq!(table_style("
ab
").as_deref(), Some("")); + assert_eq!(table_style("
a
").as_deref(), Some("")); + assert_eq!(table_style("
a
").as_deref(), Some("")); + assert_eq!(table_style("
a
").as_deref(), Some("TableGrid")); + assert_eq!(table_style("
a
").as_deref(), Some("TableGrid")); + assert_eq!(table_style("
a
").as_deref(), Some("TableGrid")); + // The last declaration decides: a `style` attribute's over the style sheet's. + assert_eq!(table_style("
a
").as_deref(), Some("TableGrid")); + assert_eq!(table_style("
a
").as_deref(), Some("")); +} + +#[test] +fn borderless_tables_export_without_cell_borders() { + // #438: a table without grid style and borders writes no cell borders and reopens borderless. + for (src, grid) in [ + ("
AB
", false), + ("
AB
", false), + ("
AB
", true), + ] { + let out = export(&craft_flow::to_doc(&parse(src))); + assert_eq!(out.contains("border:1px"), grid, "{src}: {out}"); + assert_eq!(table_style(&out).as_deref(), Some(if grid { "TableGrid" } else { "" }), "{src}"); + assert!(out.contains(">A") && out.contains(">B"), "{out}"); + } +} + +#[test] +fn input_newlines_preserve_preformatted_lines() { + for newline in ["\n", "\r\n", "\r"] { + let f = parse(&format!("
{newline}  Alpha{newline}  Beta{newline}{newline}  Gamma
")); + assert_eq!(texts(&f), ["CodeNone: Alpha", "CodeNone: Beta", "CodeNone:", "CodeNone: Gamma"], "{newline:?}"); + let FBlock::Para(p) = &f.blocks[1] else { panic!() }; + assert!(matches!(p.inlines.last(), Some(Inline::Text(t, f)) if t == "Beta" && f.bold)); + } +} + +#[test] +fn input_newlines_are_normalized_before_character_references() { + let f = parse("

A B\r\nC\rD

A\r\nB\rC

"); + assert_eq!(texts(&f), ["NormalNone:A\rB\nC\nD", "NormalNone:A B C"]); + let f = parse("

α\r\r\nβ γ

link"); + assert_eq!(texts(&f), ["NormalNone:α\n\nβ\rγ", "NormalNone:link"]); + let FBlock::Para(p) = &f.blocks[1] else { panic!() }; + assert!(matches!(p.inlines.first(), Some(Inline::Text(_, f)) if f.link.as_deref() == Some("https://example.test/a\nb\nc\rd"))); +} + +#[test] +fn css_preformatted_paragraphs_keep_edge_spaces() { + // #439: `white-space:pre` keeps leading and trailing spaces like `
`; normal
+    // paragraphs still collapse.
+    let f = parse(
+        "

Alpha Beta

\n Gamma \n

  Alpha  

Alpha Beta

", + ); + assert_eq!(texts(&f), vec!["NormalNone: Alpha Beta ", "NormalNone: Gamma ", "CodeNone: Alpha ", "NormalNone:Alpha Beta"]); +} + +#[test] +fn tag_soup() { + let f = parse( + "T & U

Head

One bold

Two it\n text

  • a
  • b
    1. c
xy
z
q
l1\nl2
", + ); + assert_eq!(f.meta.title, "T & U"); + assert_eq!( + texts(&f), + vec![ + "Heading(1)None:Head", + "NormalNone:One bold", + "NormalNone:Two it text", + "NormalSome((false, 0)):a", + "NormalSome((false, 0)):b", + "NormalSome((true, 1)):c", + "table2x3", + "QuoteNone:q", + "CodeNone:l1", + "CodeNone:l2", + ] + ); + let FBlock::Para(p) = &f.blocks[1] else { panic!() }; + assert!(matches!(&p.inlines[1], Inline::Text(t, f) if t == "bold" && f.bold)); +} + +#[test] +fn omitted_head_keeps_body_text() { + let f = parse("T\n

Hello world

Second

"); + assert_eq!(f.meta.title, "T"); + assert_eq!(texts(&f), vec!["NormalNone:Hello world", "NormalNone:Second"]); + + for body in [ + "

Hello world

", + "
Hello world
", + "Hello world", + "Hello world", + "Hello world", + "Hello world", + ] { + let f = parse(&format!( + "T & U{body}", + )); + assert_eq!(f.meta.title, "T & U"); + assert_eq!(f.meta.author, "Ann"); + assert_eq!(texts(&f), vec!["NormalNone:Hello world"], "{body}"); + } +} + +#[test] +fn styles_and_links() { + let f = parse("

red l
n

"); + let FBlock::Para(p) = &f.blocks[0] else { panic!() }; + assert_eq!(p.align, Some(Align::Center)); + assert!(matches!(&p.inlines[0], Inline::Text(t, f) if t == "red" && f.bold && f.color == Some(craft_wordmodel::Rgb(255,0,0)))); + assert!(p.inlines.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "l" && f.link.as_deref() == Some("http://x")))); + assert!(p.text().ends_with("l\nn")); +} + +/// The paragraphs of every cell of the first table, row by row. +fn cell_paras(f: &Flow) -> Vec>> { + let Some(FBlock::Table(t)) = f.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table") }; + t.rows + .iter() + .map(|r| r.iter().map(|c| c.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.clone()) } else { None }).collect()).collect()) + .collect() +} + +fn all_bold(p: &Para) -> bool { + !p.inlines.is_empty() && p.inlines.iter().all(|i| matches!(i, Inline::Text(_, f) if f.bold)) +} + +fn any_bold(p: &Para) -> bool { + p.inlines.iter().any(|i| matches!(i, Inline::Text(_, f) if f.bold)) +} + +#[test] +fn header_cells_are_bold_unless_their_css_says_otherwise() { + // #1381: a is bold, as in browsers and Word; `font-weight:normal` on it or inside it wins. + let f = parse( + "
HeadPlainInner
abc
", + ); + let rows = cell_paras(&f); + assert!(all_bold(&rows[0][0][0]), "{:?}", rows[0][0]); + assert!(!any_bold(&rows[0][1][0]), "{:?}", rows[0][1]); + assert!(!any_bold(&rows[0][2][0]), "{:?}", rows[0][2]); + assert!(!any_bold(&rows[1][0][0]), "td stays regular"); +} + +#[test] +fn cell_text_align_reaches_the_cell_paragraphs() { + // #1381: `text-align` on a td/th, or inherited from its tr/thead/tbody/table, aligns the + // cell's paragraphs; a paragraph's own text-align inside the cell wins. + let f = parse( + "\ + \ +
rinherited

own

attr
body
table
", + ); + let rows = cell_paras(&f); + assert_eq!(rows[0][0][0].align, Some(Align::Right), "td style"); + assert_eq!(rows[0][1][0].align, Some(Align::Right), "from tr"); + assert_eq!(rows[0][2][0].align, Some(Align::Center), "own paragraph wins"); + assert_eq!(rows[0][3][0].align, Some(Align::Right), "align attribute"); + assert_eq!(rows[1][0][0].align, Some(Align::Justify), "from tbody"); + assert_eq!(rows[2][0][0].align, Some(Align::Center), "from table"); + // `align` on a table places the table; it doesn't align the text in its cells. + let f = parse("
x
"); + assert_eq!(cell_paras(&f)[0][0][0].align, None); +} + +#[test] +fn reporter_sample_header_bold_and_right_aligned_cells() { + // #1381: the reporter's table. + let f = parse( + "\n\n\n\n
IndicatoreValore atteso
Audit4
", + ); + let rows = cell_paras(&f); + assert!(all_bold(&rows[0][0][0]) && all_bold(&rows[0][1][0]), "{:?}", rows[0]); + assert_eq!(rows[1][1][0].align, Some(Align::Right)); + assert_eq!(rows[1][0][0].align, None); +} + +#[test] +fn entities() { + assert_eq!(unescape("a <b> AB &bogus; &"), "a AB &bogus; &"); + assert_eq!(unescape("− © &EURO; ™ ½ ¾"), "− © € ™ ½ ¾"); + assert_eq!(unescape("θ λ Δ ∇ ∑ √ ≠ ≤ ≥ ♥"), "θ λ Δ ∇ ∑ √ ≠ ≤ ≥ ♥"); +} + +/// Browsers decode numeric and legacy named references without their `;` in text, +/// but keep `&name=` and `&nameX` intact inside attribute values (query strings). +#[test] +fn references_without_semicolon() { + assert_eq!(unescape("© 2024 Acme"), "© 2024 Acme"); + assert_eq!(unescape("© 2024 Acme"), "© 2024 Acme"); + assert_eq!(unescape("© 2024 & <"), "© 2024 & <"); + assert_eq!(unescape("&bogus &# &#x &"), "&bogus &# &#x &"); + assert_eq!(unescape(" x"), "\u{00A0}x"); + let f = parse("

l © 2024

"); + let FBlock::Para(p) = &f.blocks[0] else { panic!() }; + assert!(p.inlines.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "l" && f.link.as_deref() == Some("/q?a=1©=2®2=3&b=<")))); + assert!(p.text().ends_with("© 2024")); +} + +#[test] +fn wbr_word_break_opportunity() { + let f = parse("

Supercalifragilistic

"); + let Some(FBlock::Para(p)) = f.blocks.first() else { panic!("expected para") }; + assert_eq!(p.text(), "Super\u{200B}cali\u{200B}fragilistic"); +} + +#[test] +fn page_break_after_and_before() { + let f = parse("

First

Second

"); + let paras: Vec<_> = f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect(); + assert_eq!(paras.len(), 2); + assert!(!paras[0].page_break); + assert!(paras[1].page_break); + + let f2 = parse("

Part 1

Part 2

"); + let paras2: Vec<_> = f2.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect(); + assert_eq!(paras2.len(), 2); + assert!(!paras2[0].page_break); + assert!(paras2[1].page_break); +} + +#[test] +fn font_size_relative_and_absolute() { + let f = parse("largersmallerhuge"); + let inls = match &f.blocks[0] { + FBlock::Para(p) => &p.inlines, + _ => panic!("expected para"), + }; + let sizes: Vec> = inls + .iter() + .map(|i| match i { + Inline::Text(_, fmt) => fmt.size, + _ => None, + }) + .collect(); + assert_eq!(sizes, vec![Some(13.5), Some(10.0), Some(18.0)]); +} + +#[test] +fn decode_with_charset_declarations() { + let iso = b"\xE9l\xE8ve"; + assert!(decode(iso).contains("élève")); + + let win = b"\x80100"; + assert!(decode(win).contains("€100")); + + let gbk = b"\xC4\xE3\xBA\xC3"; + assert!(decode(gbk).contains("你好")); + + let xml = b"hello"; + assert!(decode(xml).contains("hello")); +} + +#[test] +fn raw_text_elements_skip_only_matching_end_tag() { + let f = parse("

Hello

"); + assert_eq!(f.blocks.len(), 1); + let p = match &f.blocks[0] { + FBlock::Para(p) => p, + b => panic!("expected para, got {b:?}"), + }; + assert_eq!(p.text(), "Hello"); + + let f = parse("

World

"); + assert_eq!(f.blocks.len(), 1); + let p = match &f.blocks[0] { + FBlock::Para(p) => p, + b => panic!("expected para, got {b:?}"), + }; + assert_eq!(p.text(), "World"); +} + +#[test] +fn leading_space_survives_round_trip() { + let doc = Document::from_text(" lead"); + let html = export(&doc); + assert!(!html.contains("

lead"), "{html}"); + let back = import(&html); + assert_eq!(back.plain_text(Default::default()).replace('\u{a0}', " "), " lead"); +} + +#[test] +fn css_escapes_read_every_kind_of_newline() { + // A backslash before CR LF, CR, LF or form feed continues the line; after hex digits, one + // of them (CR LF as one) ends the escape. + assert_eq!(css_unescape("A\\\r\nB\\\rC\\\nD\\\u{C}E"), "ABCDE"); + assert_eq!(css_unescape("\\41\r\nB \\42\u{C}C \\43 D"), "AB BC CD"); + assert_eq!(css_unescape("x\\'y\\"), "x'y"); +} + +#[test] +fn font_names_are_written_as_css_strings() { + // A font name could end the CSS string or the declaration and add CSS of its own (#3739). + let fonts = ["Arial\n;position:fixed", "Back\\slash", "O'Brien \"Sans\"", "A,B", "Times New Roman", "M+ 1p"]; + let mut p = Para::default(); + for (i, font) in fonts.iter().enumerate() { + p.push_text(&format!("t{i}"), &Fmt { font: Some((*font).into()), ..Default::default() }); + } + let html = export(&model::to_doc(&Flow { blocks: vec![FBlock::Para(p)], meta: Meta::default() })); + assert!(html.contains("font-family:'Times New Roman'") && html.contains("font-family:'M+ 1p'"), "{html}"); + let styles: Vec<&str> = html.split(" style=\"").skip(1).filter_map(|s| s.split('"').next()).filter(|s| s.contains("font-family")).collect(); + assert_eq!(styles.len(), fonts.len(), "{html}"); + for s in styles { + assert!(!s.contains([';', '\n']) && s.matches('\'').count() == 2, "{s}"); + } + // Read back, each run has its font as it was. + let back = parse(&html); + let runs: Vec<(String, Option)> = back + .blocks + .iter() + .filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }) + .flat_map(|p| p.inlines.iter()) + .filter_map(|i| if let Inline::Text(t, f) = i { Some((t.clone(), f.font.as_deref().map(String::from))) } else { None }) + .collect(); + for (i, font) in fonts.iter().enumerate() { + assert!(runs.contains(&(format!("t{i}"), Some(font.to_string()))), "{font:?} in {runs:?}"); + } +} + +/// A document that reads "I read " then a phonetic guide over `base`, then " daily.". +fn ruby_doc(base: &str, ruby: &str) -> Document { + let props = craft_wordmodel::CharProps::default(); + let mut p = craft_wordmodel::Paragraph::with_text("I read ", props.clone()); + let n = p.len(); + p.insert_object(n, craft_flow::ruby_object(base.into(), ruby.into(), 20.0), &props).unwrap(); + let n = p.len(); + p.insert_text(n, " daily.", &props).unwrap(); + let mut d = Document::new(); + d.body = vec![craft_wordmodel::para_block(p)]; + d +} + +/// The first flow ruby: its base text and reading. +fn ruby_of(f: &Flow) -> Option<(String, String)> { + f.blocks.iter().find_map(|b| match b { + FBlock::Para(p) => p.inlines.iter().find_map(|i| match i { + Inline::Ruby { base, ruby, .. } => Some((base.clone(), ruby.clone())), + _ => None, + }), + _ => None, + }) +} + +/// The first document ruby: its base text and reading. +fn doc_ruby(d: &Document) -> Option<(String, String)> { + d.body.iter().find_map(|b| match &**b { + craft_wordmodel::Block::Para(p) => p.objects.iter().find_map(|o| match o { + craft_wordmodel::InlineObject::Ruby { base, ruby, .. } => Some((base.clone(), ruby.clone())), + _ => None, + }), + _ => None, + }) +} + +/// The first document ruby: base size, ruby size, raise and alignment. +fn doc_ruby_attrs(d: &Document) -> Option<(f32, f32, f32, craft_wordmodel::para::RubyAlign)> { + d.body.iter().find_map(|b| match &**b { + craft_wordmodel::Block::Para(p) => p.objects.iter().find_map(|o| match o { + craft_wordmodel::InlineObject::Ruby { base_size, size, raise, align, .. } => Some((*base_size, *size, *raise, *align)), + _ => None, + }), + _ => None, + }) +} + +#[test] +fn ruby_exports_as_ruby_markup() { + // #1681: the base text with its reading in ``, not plain base text. + let out = export(&ruby_doc("漢字", "かんじ")); + assert!(out.contains("

I read 漢字かんじ daily.

"), "{out}"); + // The base and the reading are escaped like any other text. + let out = export(&ruby_doc("A&B", "xA&Bx<y"), "{out}"); +} + +#[test] +fn ruby_imports_with_its_surrounding_text() { + let f = parse("

I read 漢字かんじ daily.

"); + assert_eq!(ruby_of(&f), Some(("漢字".into(), "かんじ".into()))); + assert_eq!(texts(&f), vec!["NormalNone:I read 漢字 daily."]); + let FBlock::Para(p) = &f.blocks[0] else { panic!() }; + assert!(matches!(&p.inlines[0], Inline::Text(t, _) if t == "I read "), "{:?}", p.inlines); + assert!(matches!(p.inlines.last(), Some(Inline::Text(t, _)) if t == " daily."), "{:?}", p.inlines); + // One document object, holding the base text and the reading. + assert_eq!(doc_ruby(&craft_flow::to_doc(&f)), Some(("漢字".into(), "かんじ".into()))); +} + +#[test] +fn ruby_rb_and_rp_fallbacks_import_as_one_guide() { + // `` holds the base text and `` the fallback parentheses for browsers without + // ruby: the parentheses are not text, and the base pieces join. + let f = parse("

漢字(かんじ)

"); + assert_eq!(ruby_of(&f), Some(("漢字".into(), "かんじ".into()))); + assert_eq!(texts(&f), vec!["NormalNone:漢字"]); + assert_eq!(doc_ruby(&craft_flow::to_doc(&f)), Some(("漢字".into(), "かんじ".into()))); +} + +#[test] +fn ruby_import_uses_the_phonetic_guide_defaults() { + // HTML states no alignment, size or raise: the Phonetic Guide's own defaults over a + // 20 pt run (half the base size, raised clear of the base text, centred)… + let d = crate::doc::import("

漢字かんじ

"); + let Some((base_size, size, raise, align)) = doc_ruby_attrs(&d) else { panic!("no ruby") }; + assert_eq!((base_size, size, align.ooxml()), (20.0, 10.0, "center")); + assert!((raise - 18.0).abs() < 0.001, "{raise}"); + // …and the guide's 10.5 pt fallback base size when the source states none. + let d = crate::doc::import("

漢字かんじ

"); + let Some((base_size, size, raise, _)) = doc_ruby_attrs(&d) else { panic!("no ruby") }; + assert_eq!((base_size, size), (10.5, 5.0)); + assert!((raise - 9.4).abs() < 0.001, "{raise}"); +} + +#[test] +fn ruby_survives_an_html_round_trip() { + let html = export(&ruby_doc("漢字", "かんじ")); + assert_eq!(ruby_of(&parse(&html)), Some(("漢字".into(), "かんじ".into()))); + let back = crate::doc::import(&html); + assert_eq!(doc_ruby(&back), Some(("漢字".into(), "かんじ".into()))); + assert_eq!(back.plain_text(Default::default()), "I read 漢字 daily."); +} + +#[test] +fn ruby_keeps_the_run_formatting() { + let props = craft_wordmodel::CharProps { bold: Some(true), ..Default::default() }; + let mut p = craft_wordmodel::Paragraph::with_text("I read ", props.clone()); + let n = p.len(); + p.insert_object(n, craft_flow::ruby_object("漢字".into(), "かんじ".into(), 20.0), &props).unwrap(); + let mut d = Document::new(); + d.body = vec![craft_wordmodel::para_block(p)]; + assert!(export(&d).contains("漢字かんじ"), "{}", export(&d)); + // Importing it back keeps the formatting for the base text and the reading. + let f = parse("

漢字かんじ

"); + let FBlock::Para(p) = &f.blocks[0] else { panic!() }; + assert!(matches!(&p.inlines[0], Inline::Ruby { fmt, .. } if fmt.bold), "{:?}", p.inlines); +} + +#[test] +fn malformed_ruby_keeps_its_text() { + // No reading, an empty reading, a missing base, an unclosed element and a block boundary + // closing it: none of them panics, and the text inside is kept. + let cases: [(&str, &[&str]); 7] = [ + ("

a 漢字 b

", &["NormalNone:a 漢字 b"]), + ("

a 漢字 b

", &["NormalNone:a 漢字 b"]), + ("

a かんじ b

", &["NormalNone:a かんじ b"]), + ("

a b

", &["NormalNone:a b"]), + ("", &[]), + ("

a 漢字かんじ b

next

", &["NormalNone:a 漢字 b", "NormalNone:next"]), + ("

a 漢字かんじ

next

", &["NormalNone:a 漢字", "NormalNone:next"]), + ]; + for (src, want) in cases { + assert_eq!(texts(&parse(src)), want, "{src}"); + } + // Ruby in ruby: the inner element's text joins the outer guide's. + assert_eq!(ruby_of(&parse("

漢字字じかんじ

")), Some(("漢字字".into(), "じかんじ".into()))); +} + +#[test] +fn small_big_and_css_font_sizes() { + let f = parse( + "

small big huge tiny

", + ); + let FBlock::Para(p) = &f.blocks[0] else { panic!() }; + let sz = |i: usize| match &p.inlines[i] { + Inline::Text(_, f) => f.size, + _ => None, + }; + assert_eq!(sz(0), Some(10.0)); + assert_eq!(sz(2), Some(14.0)); + assert_eq!(sz(4), Some(36.0)); + assert_eq!(sz(6), Some(10.0)); +} + +#[test] +fn css_important_casing_and_spacing() { + let f = parse( + "

Styled

", + ); + assert_eq!(f.blocks.len(), 1); + let p = match &f.blocks[0] { + FBlock::Para(p) => p, + b => panic!("expected para, got {b:?}"), + }; + assert_eq!(p.align, Some(Align::Center)); + match &p.inlines[0] { + Inline::Text(_, fmt) => { + assert_eq!(fmt.color, Some(craft_wordmodel::props::Rgb(0xFF, 0x00, 0x00))); + assert_eq!(fmt.size, Some(16.0)); + assert!(fmt.bold); + } + i => panic!("expected text inline, got {i:?}"), + } +} + +#[test] +fn table_cell_vertical_alignment() { + let f = parse("
topmidbot
"); + let FBlock::Table(t) = &f.blocks[0] else { panic!() }; + assert_eq!(t.rows[0][0].valign, Some(VAlign::Top)); + assert_eq!(t.rows[0][1].valign, Some(VAlign::Center)); + assert_eq!(t.rows[0][2].valign, Some(VAlign::Bottom)); + let doc = craft_flow::to_doc(&f); + let craft_wordmodel::Block::Table(dt) = &*doc.body[0] else { panic!() }; + assert_eq!(dt.rows[0].cells[0].props.valign, Some(VAlign::Top)); + assert_eq!(dt.rows[0].cells[1].props.valign, Some(VAlign::Center)); + assert_eq!(dt.rows[0].cells[2].props.valign, Some(VAlign::Bottom)); + let html = export(&doc); + assert!(html.contains("vertical-align:top")); + assert!(html.contains("vertical-align:middle")); + assert!(html.contains("vertical-align:bottom")); +} + +#[test] +fn many_formatted_leading_breaks_trim_quickly() { + // #3705: a paragraph opening with alternating bold and italic `
` was trimmed one + // span at a time with `Vec::remove(0)`. + let html = format!("

{}x

", "

".repeat(100_000)); + let t = std::time::Instant::now(); + let f = parse(&html); + // About a second on a loaded machine; trimming one span at a time ran past 6 minutes. + assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed()); + let texts: Vec = f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect(); + assert_eq!(texts, ["x"]); +} + +#[test] +fn font_size_at_the_integer_limits_does_not_overflow() { + // `` offsets were added to 3 unchecked, so the extreme values panicked in debug builds. + for size in ["+2147483647", "--2147483648", "-2147483647", "+99999999999"] { + let f = parse(&format!("x")); + assert!(texts(&f).iter().any(|t| t.ends_with(":x")), "size={size}"); + } + // The offsets saturate, then clamp to the 1..=7 scale: 7 is 36 pt, 1 is 7.5 pt. + let size = |html: &str| match &parse(html).blocks[0] { + FBlock::Para(p) => p.inlines.iter().find_map(|i| match i { + Inline::Text(_, fmt) => fmt.size, + _ => None, + }), + _ => None, + }; + assert_eq!(size("x"), Some(36.0)); + assert_eq!(size("x"), Some(36.0)); + assert_eq!(size("x"), Some(7.5)); +} + +#[test] +fn empty_comments_do_not_swallow_the_rest() { + for html in ["

ab

c

", "

ab

c

"] { + let doc = crate::doc::import(html); + assert_eq!(doc.plain_text(Default::default()), "ab\nc", "{html}"); + } +} + +fn paras(f: &Flow) -> Vec<&Para> { + f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect() +} +fn fmt_of(p: &Para) -> Fmt { + p.inlines.iter().find_map(|i| if let Inline::Text(_, f) = i { Some(f.clone()) } else { None }).unwrap_or_default() +} + +#[test] +fn style_sheet_rules_apply_in_cascade_order() { + let f = parse( + "\ +

plain

big

inline wins

digits

", + ); + let ps = paras(&f); + let (plain, big, inline, digits) = (fmt_of(ps[0]), fmt_of(ps[1]), fmt_of(ps[2]), fmt_of(ps[3])); + assert_eq!((plain.color, plain.size, plain.bold, plain.italic), (Some(craft_wordmodel::Rgb(0, 255, 0)), Some(10.0), false, false)); + assert_eq!((big.size, big.bold), (Some(20.0), true)); + assert_eq!(inline.size, Some(8.0)); + assert_eq!(ps[3].align, Some(Align::Center)); + assert_eq!(digits.size, Some(10.0)); +} + +#[test] +fn paragraph_css_states_spacing_indents_and_line_height() { + let f = parse( + "\ +

a

b

c

", + ); + let ps = paras(&f); + assert_eq!( + ps[0].spacing, + Spacing { + before: Some(6.0), + after: Some(3.0), + left: Some(72.0), + right: Some(72.0), + first: Some(-18.0), + line: Some(LineSpacing::Multiple(1.5)), + ..Spacing::default() + } + ); + // Values that don't parse (a comma decimal, `auto`) state nothing. + assert_eq!(ps[1].spacing, Spacing { line: Some(LineSpacing::Exactly(20.0)), ..Spacing::default() }); + assert_eq!(ps[2].spacing, Spacing::default()); +} + +#[test] +fn a_page_opens_as_word_opens_one() { + let html = "\ +

first

Head

kept

own

bare

 

"; + let f = parse_in(html, &|_| None, true); + let ps = paras(&f); + // Times New Roman 12 pt on single lines; headings bold and black at a browser's sizes. + let first = fmt_of(ps[0]); + assert_eq!((first.font.as_deref(), first.size), (Some(PAGE_FONT), Some(12.0))); + assert_eq!(ps[0].spacing.line, Some(LineSpacing::Multiple(1.0))); + let cell = parse_in("
cell
", &|_| None, true); + let FBlock::Table(t) = &cell.blocks[0] else { panic!() }; + let FBlock::Para(cp) = &t.rows[0][0].blocks[0] else { panic!() }; + assert_eq!(cp.spacing.line, Some(LineSpacing::Multiple(1.0))); + let head = fmt_of(ps[1]); + assert_eq!((head.bold, head.size, head.color), (true, Some(24.0), Some(craft_wordmodel::Rgb(0, 0, 0)))); + // A colour the heading inherits stays. + // A generic family names a face on a page; pasted text keeps the document's font. + let generic = parse_in("

a

b

", &|_| None, true); + let fonts: Vec> = paras(&generic).iter().map(|p| fmt_of(p).font.map(|f| f.to_string())).collect(); + assert_eq!(fonts, [Some("Arial".to_string()), Some(PAGE_FONT.to_string())]); + assert_eq!(fmt_of(paras(&parse("

a

"))[0]).font, None); + let inherited = parse_in("

Head

", &|_| None, true); + assert_eq!(fmt_of(paras(&inherited)[0]).color, Some(craft_wordmodel::Rgb(0, 0, 0x80))); + // Automatic spacing where a p or heading leaves its margins out (layout drops it at the + // top of the page and of a cell). + let auto = |p: &Para| (p.spacing.before_auto, p.spacing.after_auto, p.spacing.before, p.spacing.after); + assert_eq!(auto(ps[0]), (true, true, None, None)); + assert_eq!(auto(ps[1]), (true, true, None, None)); + // A span's margin replaces the one its paragraph states; the one it leaves out stays automatic. + assert_eq!(auto(ps[2]), (true, false, None, Some(0.0))); + assert_eq!(auto(ps[3]), (true, false, None, Some(9.75))); + // Other blocks get no automatic spacing. + assert_eq!(auto(ps[4]), (false, false, None, None)); + // The document says so, and the empty paragraph's mark keeps its size. + let d = model::to_doc(&f); + let props = |i: usize| match &*d.body[i] { + craft_wordmodel::Block::Para(p) => p.clone(), + b => panic!("{b:?}"), + }; + assert_eq!((props(0).props.space_before_auto, props(0).props.space_after_auto), (Some(true), Some(true))); + assert_eq!((props(2).props.space_after_auto, props(2).props.space_after), (None, Some(0.0))); + assert_eq!(props(5).mark.size, Some(12.0)); + // `

 

` keeps the size of its space for its line. + assert!(ps[5].inlines.is_empty()); + assert_eq!(ps[5].mark.as_ref().and_then(|m| m.size), Some(12.0)); + // A fragment (a paste) keeps the document's own formatting. + let pasted = parse(html); + assert_eq!(fmt_of(paras(&pasted)[0]).font, None); + assert_eq!(paras(&pasted)[0].spacing, Spacing::default()); +} + +#[test] +fn cells_give_their_columns_widths() { + let f = parse( + "
t
1ab
", + ); + let FBlock::Table(t) = &f.blocks[0] else { panic!("{:?}", f.blocks) }; + assert_eq!(t.widths, [31.5, 150.0, 144.0]); + // A column without a width leaves them all to the layout. + let f = parse("
ab
"); + let FBlock::Table(t) = &f.blocks[0] else { panic!() }; + assert!(t.widths.is_empty()); + // A cell beside a cell spanning rows from above gives its own column its width. + let f = + parse("
ab
c
de
"); + let FBlock::Table(t) = &f.blocks[0] else { panic!() }; + assert_eq!(t.widths, [75.0, 150.0]); +} + +#[test] +fn cells_take_their_css_borders() { + use craft_wordmodel::props::{Border, BorderStyle}; + let f = parse( + "
ab
", + ); + let FBlock::Table(t) = &f.blocks[0] else { panic!() }; + // The style sheet's other cell properties apply too. + assert_eq!(t.rows[0][0].valign, Some(craft_wordmodel::props::VAlign::Bottom)); + let b = t.rows[0][0].borders.expect("borders"); + assert_eq!(b.top, Some(Border::single(0.75))); + assert_eq!(b.bottom, Some(Border { style: BorderStyle::Double, width: 2.0, color: Some(craft_wordmodel::Rgb(255, 0, 0)), space: 0.0 })); + assert_eq!(b.left.map(|l| l.style), Some(BorderStyle::None)); + // A border of no width is none (OOXML has no zero-width line). + assert_eq!(b.right.map(|r| r.style), Some(BorderStyle::None)); + assert_eq!(t.rows[0][1].borders, None); +} + +#[test] +fn style_sheet_edge_cases() { + let f = parse_in( + "\ +

a

bare

b

", + &|_| None, + true, + ); + let ps = paras(&f); + // A rule right after statement at-rules still applies. + assert_eq!(fmt_of(ps[0]).color, Some(craft_wordmodel::Rgb(0, 0, 255))); + // The body's margins aren't the paragraphs' indents. + assert_eq!((ps[1].spacing.left, ps[1].spacing.before), (None, None)); + // Bare numbers, percentages and keywords state no margin: automatic spacing stays. + assert_eq!((ps[2].spacing.before_auto, ps[2].spacing.left), (true, None)); + // The `style` attribute wins even over a sheet with more declarations than are read. + let many = (0..600).map(|i| format!("x-{i}: 1")).collect::>().join(";"); + let f = parse(&format!("

c

")); + assert_eq!(fmt_of(paras(&f)[0]).color, Some(craft_wordmodel::Rgb(0, 255, 0))); +} + +#[test] +fn hostile_style_sheets_never_panic() { + // One declaration block shared by many selectors isn't copied for each. + let selectors = vec!["p"; 20_000].join(","); + let huge = format!("

x

", "x: y;".repeat(200_000)); + let _ = parse_in(&huge, &|_| None, true); + let many = ".a { color: red } ".repeat(20_000); + for css in ["{", "}", "p {", "@media { p { x } ", "p { color: red", "/* open", ".{}", "p..a{}", "\u{0}.é{x:y}", many.as_str()] { + let _ = parse_in(&format!("

x

y"), &|_| None, true); + } +} diff --git a/crates/html/src/entity.rs b/crates/html/src/entity.rs new file mode 100644 index 000000000..8ece41926 --- /dev/null +++ b/crates/html/src/entity.rs @@ -0,0 +1,388 @@ +//! Character references (`&`, `é`, `é`) and escaping. + +/// The Latin-1 names in code point order, from U+00A0 (`nbsp`) to U+00FF (`yuml`). +const LATIN1: [&str; 96] = [ + "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect", "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr", "deg", + "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot", "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest", "Agrave", + "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil", "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml", "ETH", + "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times", "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig", + "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil", "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", + "iuml", "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide", "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", + "thorn", "yuml", +]; + +/// Greek capitals from U+0391; U+03A2 is unassigned. +const GREEK_UPPER: [&str; 25] = [ + "Alpha", "Beta", "Gamma", "Delta", "Epsilon", "Zeta", "Eta", "Theta", "Iota", "Kappa", "Lambda", "Mu", "Nu", "Xi", "Omicron", "Pi", "Rho", "", + "Sigma", "Tau", "Upsilon", "Phi", "Chi", "Psi", "Omega", +]; + +/// Greek small letters from U+03B1. +const GREEK_LOWER: [&str; 25] = [ + "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho", + "sigmaf", "sigma", "tau", "upsilon", "phi", "chi", "psi", "omega", +]; + +/// What a numeric reference to 0x80–0x9F means (HTML Living Standard §13.2.5.80: the +/// Windows-1252 characters, by long tradition). +const C1: [u32; 32] = [ + 0x20AC, 0x81, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x8D, 0x017D, 0x8F, 0x90, 0x2018, 0x2019, + 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178, +]; + +/// Longest entity name looked at. +const MAX_NAME: usize = 32; + +/// The legacy names HTML also decodes without a `;`, sorted by bytes. +const BARE_NAMES: &[&str] = &[ + "AElig", "AMP", "Aacute", "Acirc", "Agrave", "Aring", "Atilde", "Auml", "COPY", "Ccedil", "Eacute", "Ecirc", "Egrave", "Euml", "GT", "Iacute", + "Icirc", "Igrave", "Iuml", "LT", "Ntilde", "Oacute", "Ocirc", "Ograve", "Oslash", "Otilde", "Ouml", "QUOT", "REG", "Uacute", "Ucirc", "Ugrave", + "Uuml", "Yacute", "aacute", "acirc", "aelig", "agrave", "amp", "aring", "atilde", "auml", "ccedil", "cent", "copy", "curren", "deg", "divide", + "eacute", "ecirc", "egrave", "euml", "frac12", "frac14", "frac34", "gt", "iacute", "icirc", "iexcl", "igrave", "iquest", "iuml", "laquo", "lt", + "micro", "middot", "nbsp", "ntilde", "oacute", "ocirc", "ograve", "oslash", "otilde", "ouml", "para", "plusmn", "pound", "quot", "raquo", "reg", + "sect", "shy", "sup1", "sup2", "sup3", "szlig", "times", "uacute", "ucirc", "ugrave", "uuml", "yacute", "yen", "yuml", +]; + +/// Text as HTML, for content and for attribute values in either kind of quote: `&`, `<`, `>`, +/// `"` and `'` become character references. +pub fn escape(text: &str) -> String { + let mut out = String::with_capacity(text.len().saturating_add(text.len() / 8)); + push_escaped(&mut out, text, true); + out +} + +/// [`escape`] that leaves `'` as it is: for content and for attribute values in double quotes. +pub fn escape_dq(text: &str) -> String { + let mut out = String::with_capacity(text.len().saturating_add(text.len() / 8)); + push_escaped(&mut out, text, false); + out +} + +fn push_escaped(out: &mut String, text: &str, apos: bool) { + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' if apos => out.push_str("'"), + c => out.push(c), + } + } +} + +/// The character an entity body names (`amp`, `#38`, `#x26`: what stands between `&` and `;`). +pub fn decode_entity(body: &str) -> Option { + match body.strip_prefix('#') { + Some(_) => reference(body, false).filter(|(_, used)| *used == body.len()).map(|(c, _)| c), + None => named(body), + } +} + +/// Character references in text decoded: named (the HTML 4 set and more), decimal and hex. +/// Anything that is no reference stays as it is. A legacy name decodes without its `;` +/// (`© 2024`), as in browsers. +pub fn unescape(text: &str) -> String { + unescape_in(text, false) +} + +/// [`unescape`] for an attribute value: a legacy name without its `;` stays text when a letter, +/// a digit or `=` follows, so `?a=1©=2` is a query string and not a copyright sign. +pub fn unescape_attr(text: &str) -> String { + unescape_in(text, true) +} + +fn unescape_in(text: &str, attr: bool) -> String { + if !text.contains('&') { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(i) = rest.find('&') { + out.push_str(rest.get(..i).unwrap_or("")); + let after = rest.get(i + 1..).unwrap_or(""); + match reference(after, attr) { + Some((c, used)) => { + out.push(c); + rest = after.get(used..).unwrap_or(""); + } + None => { + out.push('&'); + rest = after; + } + } + } + out.push_str(rest); + out +} + +/// The character a reference names and the bytes it takes, given the text after its `&` +/// (`attr`: inside an attribute value, see [`unescape_attr`]). At most [`MAX_NAME`] bytes are +/// looked at for a name, so a text of stray `&`s is read in linear time. +pub fn reference(s: &str, attr: bool) -> Option<(char, usize)> { + if let Some(num) = s.strip_prefix('#') { + let (digits, radix, skip) = match num.strip_prefix(['x', 'X']) { + Some(hex) => (hex, 16, 2), + None => (num, 10, 1), + }; + let len = digits.bytes().take_while(|b| char::from(*b).is_digit(radix)).count(); + if len == 0 { + return None; + } + // Too many digits is not a character; neither is 0, a surrogate, or anything past U+10FFFF. + let mut code: u32 = 0; + for d in digits.bytes().take(len) { + let d = char::from(d).to_digit(radix).unwrap_or(0); + code = code.checked_mul(radix).and_then(|c| c.checked_add(d)).unwrap_or(u32::MAX); + } + let semicolon = usize::from(digits.as_bytes().get(len) == Some(&b';')); + return Some((numeric(code), skip + len + semicolon)); + } + let len = s.bytes().take(MAX_NAME).take_while(u8::is_ascii_alphanumeric).count(); + if s.as_bytes().get(len) == Some(&b';') + && let Some(c) = s.get(..len).and_then(named) + { + return Some((c, len + 1)); + } + // Without its `;`: the longest legacy name (2 to 6 letters) the text starts with. + let len = (2..=len.min(6)).rev().find(|&l| s.get(..l).is_some_and(|p| BARE_NAMES.binary_search(&p).is_ok()))?; + if attr && s.as_bytes().get(len).is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'=') { + return None; + } + s.get(..len).and_then(named).map(|c| (c, len)) +} + +fn numeric(code: u32) -> char { + let code = match code { + 0x80..=0x9F => C1.get((code - 0x80) as usize).copied().unwrap_or(code), + c => c, + }; + match char::from_u32(code) { + Some('\0') | None => char::REPLACEMENT_CHARACTER, + Some(c) => c, + } +} + +fn named(name: &str) -> Option { + if name.is_empty() { + return None; + } + let c = match name { + "amp" | "AMP" => '&', + "lt" | "LT" => '<', + "gt" | "GT" => '>', + "quot" | "QUOT" => '"', + "apos" | "APOS" => '\'', + "nbsp" | "NBSP" => '\u{00A0}', + "ensp" => '\u{2002}', + "emsp" => '\u{2003}', + "thinsp" => '\u{2009}', + "shy" => '\u{00AD}', + "copy" | "COPY" => '©', + "reg" | "REG" => '®', + "trade" | "TRADE" => '™', + "hellip" => '…', + "mdash" => '—', + "ndash" => '–', + "minus" => '−', + "lsquo" => '‘', + "rsquo" => '’', + "sbquo" => '‚', + "ldquo" => '“', + "rdquo" => '”', + "bdquo" => '„', + "laquo" => '«', + "raquo" => '»', + "bull" => '•', + "middot" => '·', + "deg" => '°', + "plusmn" => '±', + "times" => '×', + "divide" => '÷', + "frac12" => '½', + "frac14" => '¼', + "frac34" => '¾', + "frac13" => '⅓', + "frac23" => '⅔', + "frac18" => '⅛', + "frac38" => '⅜', + "frac58" => '⅝', + "frac78" => '⅞', + "sup1" => '¹', + "sup2" => '²', + "sup3" => '³', + "micro" => 'µ', + "permil" => '‰', + "prime" => '′', + "Prime" => '″', + "oline" => '‾', + "frasl" => '⁄', + "sect" => '§', + "para" => '¶', + "euro" | "EURO" => '€', + "pound" | "POUND" => '£', + "yen" | "YEN" => '¥', + "cent" | "CENT" => '¢', + "curren" => '¤', + "dagger" => '†', + "Dagger" => '‡', + "larr" => '←', + "rarr" => '→', + "uarr" => '↑', + "darr" => '↓', + "harr" => '↔', + "lArr" => '⇐', + "rArr" => '⇒', + "uArr" => '⇑', + "dArr" => '⇓', + "hArr" => '⇔', + "crarr" => '↵', + "forall" => '∀', + "part" => '∂', + "exist" => '∃', + "empty" => '∅', + "nabla" => '∇', + "isin" => '∈', + "notin" => '∉', + "ni" => '∋', + "prod" => '∏', + "sum" => '∑', + "radic" => '√', + "prop" => '∝', + "ang" => '∠', + "and" => '∧', + "or" => '∨', + "cap" => '∩', + "cup" => '∪', + "int" => '∫', + "there4" => '∴', + "sim" => '∼', + "cong" => '≅', + "asymp" | "approx" => '≈', + "ne" => '≠', + "equiv" => '≡', + "le" => '≤', + "ge" => '≥', + "sub" => '⊂', + "sup" => '⊃', + "nsub" => '⊄', + "sube" => '⊆', + "supe" => '⊇', + "oplus" => '⊕', + "otimes" => '⊗', + "perp" => '⊥', + "sdot" => '⋅', + "infin" => '∞', + "loz" => '◊', + "spades" => '♠', + "clubs" => '♣', + "hearts" => '♥', + "diams" => '♦', + "check" => '✓', + "iexcl" => '¡', + "iquest" => '¿', + "szlig" => 'ß', + "auml" => 'ä', + "ouml" => 'ö', + "uuml" => 'ü', + "Auml" => 'Ä', + "Ouml" => 'Ö', + "Uuml" => 'Ü', + "eacute" => 'é', + "egrave" => 'è', + "ecirc" => 'ê', + "euml" => 'ë', + "Eacute" => 'É', + "Egrave" => 'È', + "Ecirc" => 'Ê', + "Euml" => 'Ë', + "aacute" => 'á', + "agrave" => 'à', + "acirc" => 'â', + "atilde" => 'ã', + "Agrave" => 'À', + "Aacute" => 'Á', + "Acirc" => 'Â', + "Atilde" => 'Ã', + "iacute" => 'í', + "igrave" => 'ì', + "icirc" => 'î', + "iuml" => 'ï', + "Iacute" => 'Í', + "Igrave" => 'Ì', + "Icirc" => 'Î', + "Iuml" => 'Ï', + "oacute" => 'ó', + "ograve" => 'ò', + "ocirc" => 'ô', + "otilde" => 'õ', + "Oacute" => 'Ó', + "Ograve" => 'Ò', + "Ocirc" => 'Ô', + "Otilde" => 'Õ', + "uacute" => 'ú', + "ugrave" => 'ù', + "ucirc" => 'û', + "Uacute" => 'Ú', + "Ugrave" => 'Ù', + "Ucirc" => 'Û', + "yacute" => 'ý', + "yuml" => 'ÿ', + "Yacute" => 'Ý', + "ntilde" => 'ñ', + "Ntilde" => 'Ñ', + "ccedil" => 'ç', + "Ccedil" => 'Ç', + "aring" => 'å', + "Aring" => 'Å', + "oslash" => 'ø', + "Oslash" => 'Ø', + "aelig" => 'æ', + "AElig" => 'Æ', + "alpha" => 'α', + "beta" => 'β', + "gamma" => 'γ', + "delta" => 'δ', + "epsilon" => 'ε', + "zeta" => 'ζ', + "eta" => 'η', + "theta" => 'θ', + "iota" => 'ι', + "kappa" => 'κ', + "lambda" => 'λ', + "mu" => 'μ', + "nu" => 'ν', + "xi" => 'ξ', + "pi" => 'π', + "rho" => 'ρ', + "sigma" => 'σ', + "tau" => 'τ', + "upsilon" => 'υ', + "phi" => 'φ', + "chi" => 'χ', + "psi" => 'ψ', + "omega" => 'ω', + "Gamma" => 'Γ', + "Delta" => 'Δ', + "Theta" => 'Θ', + "Lambda" => 'Λ', + "Xi" => 'Ξ', + "Pi" => 'Π', + "Sigma" => 'Σ', + "Upsilon" => 'Υ', + "Phi" => 'Φ', + "Psi" => 'Ψ', + "Omega" => 'Ω', + "zwj" => '\u{200D}', + "zwnj" => '\u{200C}', + "cross" => '\u{2717}', + _ => { + let at = |table: &[&str], base: u32| table.iter().position(|n| *n == name).and_then(|i| char::from_u32(base + i as u32)); + return at(&LATIN1, 0xA0).or_else(|| at(&GREEK_UPPER, 0x391)).or_else(|| at(&GREEK_LOWER, 0x3B1)); + } + }; + Some(c) +} + +#[cfg(test)] +mod tests; diff --git a/crates/html/src/entity/tests.rs b/crates/html/src/entity/tests.rs new file mode 100644 index 000000000..3abd35fdc --- /dev/null +++ b/crates/html/src/entity/tests.rs @@ -0,0 +1,132 @@ +use super::{BARE_NAMES, GREEK_LOWER, GREEK_UPPER, LATIN1, decode_entity, escape, escape_dq, named, reference, unescape, unescape_attr}; + +#[test] +fn escapes_the_five() { + assert_eq!(escape(r#"Tom & 'Jerry'"#), "<a href="x">Tom & 'Jerry'</a>"); + assert_eq!(escape_dq(r#"Tom & 'Jerry'"#), "<a href="x">Tom & 'Jerry'</a>"); + assert_eq!(escape("plain ünïcode"), "plain ünïcode"); + assert_eq!(escape(""), ""); +} + +#[test] +fn bare_names_are_sorted_and_known() { + assert_eq!(BARE_NAMES.len(), 94); + assert!(BARE_NAMES.windows(2).all(|w| w[0] < w[1])); + assert!(BARE_NAMES.iter().all(|n| (2..=6).contains(&n.len()) && named(n).is_some())); +} + +#[test] +fn every_latin_1_and_greek_name_is_its_code_point() { + for (table, base) in [(&LATIN1[..], 0xA0), (&GREEK_UPPER[..], 0x391), (&GREEK_LOWER[..], 0x3B1)] { + for (i, name) in table.iter().enumerate().filter(|(_, n)| !n.is_empty()) { + assert_eq!(named(name), char::from_u32(base + i as u32), "{name}"); + } + } +} + +#[test] +fn named_references() { + assert_eq!(unescape("<b> & "x" 'y'"), " & \"x\" 'y'"); + assert_eq!(unescape("a b"), "a\u{a0}b"); + assert_eq!(unescape("© ® ™ … — –"), "© ® ™ … — –"); + assert_eq!(unescape("‘a’ “b” €5"), "‘a’ “b” €5"); + assert_eq!(unescape("café naïve Über ß ÿ"), "café naïve Über ß ÿ"); + assert_eq!(unescape("αβΩς → ≤ ≠"), "αβΩς → ≤ ≠"); + assert_eq!(unescape("‌¦ÐþΑο✗✓♦"), "\u{200c}¦Ðþ\u{391}ο✗✓♦"); + assert_eq!(unescape("⅓∴⇔◊"), "⅓∴⇔◊"); +} + +#[test] +fn names_are_case_sensitive() { + assert_eq!(unescape("Éé"), "Éé"); + assert_eq!(unescape("&EACUTE;"), "&EACUTE;"); + assert_eq!(unescape("&<"), "&<"); +} + +#[test] +fn numeric_references() { + assert_eq!(unescape("ééé"), "ééé"); + assert_eq!(unescape("😀"), "😀"); + assert_eq!(unescape("😀"), "😀"); + // Without the semicolon. + assert_eq!(unescape("A B."), "A B."); + // Windows-1252 for the C1 range. + assert_eq!(unescape("“hi” € ™"), "“hi” € ™"); +} + +#[test] +fn bad_numbers_become_the_replacement_character() { + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); + assert_eq!(unescape("�"), "\u{fffd}"); +} + +#[test] +fn invalid_references_are_left_alone() { + for s in ["&", "&&", "& ", "&;", "&#;", "&#x;", "&#xZZ;", "&bogus;", "AT&T", "a & b", "x&y=z", "&#", "&#x", "&eacut", "ünï&", "&Alpha", "&hellip"] + { + assert_eq!(unescape(s), s, "{s}"); + assert_eq!(unescape_attr(s), s, "{s}"); + } +} + +#[test] +fn legacy_names_work_without_a_semicolon() { + assert_eq!(unescape("a < b > c"), "a < b > c"); + assert_eq!(unescape("x "), "x\u{a0}"); + assert_eq!(unescape("&"), "&"); + assert_eq!(unescape("© 2024 café!"), "© 2024 café!"); + // The longest name wins: `∉` is one reference; `¥x;` is `¥` and `x;`. + assert_eq!(unescape("∉ ¥x;"), "∉ ¥x;"); + // In text even inside a word, as browsers read it. + assert_eq!(unescape("&lify ?a=1&=2"), "&lify ?a=1&=2"); +} + +#[test] +fn in_an_attribute_a_bare_name_before_a_letter_digit_or_equals_is_text() { + assert_eq!(unescape_attr("?a=1&=2<=3©=4®1"), "?a=1&=2<=3©=4®1"); + assert_eq!(unescape_attr("&lify"), "&lify"); + assert_eq!(unescape_attr("?a=1&b=2"), "?a=1&b=2"); + assert_eq!(unescape_attr("a < b©"), "a < b©"); + // Numeric references need no `;` anywhere. + assert_eq!(unescape_attr("A=1"), "A=1"); +} + +#[test] +fn an_entity_body_decodes_whole_or_not_at_all() { + assert_eq!(decode_entity("amp"), Some('&')); + assert_eq!(decode_entity("#38"), Some('&')); + assert_eq!(decode_entity("#x26"), Some('&')); + assert_eq!(decode_entity("#150"), Some('–')); + for none in ["", "#", "#x", "#38x", "#3 8", "bogus", "amp;"] { + assert_eq!(decode_entity(none), None, "{none}"); + } + assert_eq!(reference("amp;rest", false), Some(('&', 4))); + assert_eq!(reference("#x41;", true), Some(('A', 5))); +} + +#[test] +fn a_long_run_of_ampersands_is_quick() { + let s = "&".repeat(200_000); + assert_eq!(unescape(&s), s); + let s = "&#".repeat(100_000); + assert_eq!(unescape(&s), s); + let s = "&aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".repeat(20_000); + assert_eq!(unescape(&s), s); +} + +proptest::proptest! { + #[test] + fn unescape_undoes_escape(s in proptest::prelude::any::()) { + proptest::prop_assert_eq!(unescape(&escape(&s)), s.clone()); + proptest::prop_assert_eq!(unescape_attr(&escape_dq(&s)), s); + } + + #[test] + fn unescape_never_panics(s in "[&#xX0-9a-zA-Z;= é]{0,60}") { + let _ = unescape(&s); + let _ = unescape_attr(&s); + } +} diff --git a/crates/html/src/lex.rs b/crates/html/src/lex.rs new file mode 100644 index 000000000..ef6b20636 --- /dev/null +++ b/crates/html/src/lex.rs @@ -0,0 +1,359 @@ +//! A tolerant HTML tokenizer: text, start tags and end tags. +//! +//! It follows the general syntax of the HTML Living Standard closely enough for documents, mail +//! and clipboard fragments, and reads anything: a `<` that starts no tag is text, a tag or +//! comment that never ends runs to the end of the input. Comments, doctypes and processing +//! instructions never leave it. + +use crate::entity::unescape_attr; + +/// What a [`Lexer`] caps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Most attributes kept per tag; the rest are read and dropped. Default 64. + pub max_attrs: usize, + /// Longest tag name kept, in bytes; a longer one is no real element and its name is empty. + /// Default 64. + pub max_tag_name: usize, +} + +impl Limits { + pub const DEFAULT: Limits = Limits { max_attrs: 64, max_tag_name: 64 }; +} + +impl Default for Limits { + fn default() -> Limits { + Limits::DEFAULT + } +} + +/// How a [`Lexer`] reads. Element names are lower case. +#[derive(Clone, Copy, Debug, Default)] +pub struct Options<'o> { + pub limits: Limits, + /// Elements that never leave the lexer: the start tag, the content (raw text, not markup) + /// and the end tag are skipped. For `script`, `style`, `title` and the like, when the caller + /// has no use for them. A caller that wants a `style`'s text leaves it out of this list and + /// calls [`Lexer::raw_content`] on its start tag. + pub drop: &'o [&'o str], + /// Elements whose content is text, not markup, and is given: after the start tag comes one + /// [`Token::Text`] with everything up to the end tag. For `textarea`, `xmp`, `plaintext`. + pub raw_text: &'o [&'o str], +} + +/// A start tag. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tag<'a> { + /// Lower case; empty for a name longer than [`Limits::max_tag_name`]. + pub name: String, + /// Names and values as written, in the order written. + pub attrs: Vec<(&'a str, &'a str)>, + /// The tag ends in `/>`. + pub self_close: bool, +} + +impl Tag<'_> { + /// An attribute's value with character references decoded ([`unescape_attr`]); the first + /// of a name wins, as in browsers. Names match in any letter case. + pub fn attr(&self, name: &str) -> Option { + self.raw_attr(name).map(unescape_attr) + } + + /// An attribute's value as written, character references and all. + pub fn raw_attr(&self, name: &str) -> Option<&str> { + self.attrs.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| *v) + } + + /// Whether the tag has the attribute, with or without a value. + pub fn has(&self, name: &str) -> bool { + self.attrs.iter().any(|(n, _)| n.eq_ignore_ascii_case(name)) + } + + /// Every attribute in the order written: the name as written, the value decoded. + pub fn attributes(&self) -> impl Iterator + '_ { + self.attrs.iter().map(|(n, v)| (*n, unescape_attr(v))) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Token<'a> { + /// Character data as written: references are not decoded ([`crate::unescape`]). + Text(&'a str), + Start(Tag<'a>), + /// An end tag's name, lower case (empty when longer than [`Limits::max_tag_name`]). + End(String), +} + +/// The tokenizer: an iterator of [`Token`]s over a string. +pub struct Lexer<'a, 'o> { + src: &'a str, + pos: usize, + opts: Options<'o>, + /// Set after the start tag of a raw-text element: its content comes next. + raw: Option, +} + +impl<'a> Lexer<'a, 'static> { + /// A lexer with [`Options::default`]: nothing dropped, no raw-text elements. + pub fn new(src: &'a str) -> Lexer<'a, 'static> { + Lexer::with(src, Options::default()) + } +} + +impl<'a, 'o> Lexer<'a, 'o> { + pub fn with(src: &'a str, opts: Options<'o>) -> Lexer<'a, 'o> { + Lexer { src, pos: 0, opts, raw: None } + } + + /// How far into the input the tokens given so far go, in bytes. + pub fn pos(&self) -> usize { + self.pos + } + + /// Goes back (or forward) to `pos`, a byte offset an earlier [`Lexer::pos`] gave, to read + /// from there again: for a reader that looked at a token and hands it back. An offset past the + /// end, or inside a character, moves to the next character boundary. + pub fn set_pos(&mut self, pos: usize) { + let mut pos = pos.min(self.src.len()); + while !self.src.is_char_boundary(pos) { + pos += 1; + } + self.pos = pos; + self.raw = None; + } + + /// The raw content of element `name`, whose start tag was the last token: everything up to + /// its end tag (any letter case), or to the end of the input when it has none. The end tag + /// is consumed. For `style` and `script`, whose content is not markup. + pub fn raw_content(&mut self, name: &str) -> &'a str { + self.raw = None; + let b = self.bytes(); + let end = find_end_tag(b, self.pos, name).unwrap_or(b.len()); + let text = self.slice(self.pos, end); + self.pos = find_byte(b, end, b'>').map_or(b.len(), |i| i + 1); + text + } + + fn bytes(&self) -> &'a [u8] { + self.src.as_bytes() + } + + fn slice(&self, start: usize, end: usize) -> &'a str { + self.src.get(start..end).unwrap_or("") + } + + /// A tag name in lower case; empty when it is too long to be a real element. + fn tag_name(&self, raw: &str) -> String { + if raw.len() > self.opts.limits.max_tag_name { String::new() } else { raw.to_ascii_lowercase() } + } + + /// Whether the `<` at `i` starts a tag, a comment or a declaration (rather than being text). + fn markup_at(&self, i: usize) -> bool { + let b = self.bytes(); + match b.get(i + 1) { + Some(c) if c.is_ascii_alphabetic() => true, + Some(b'!' | b'?') => true, + Some(b'/') => matches!(b.get(i + 2), Some(c) if c.is_ascii_alphabetic() || *c == b'>'), + _ => false, + } + } + + /// The end of the text run that starts at `self.pos`: the next `<` that starts markup. + fn text_end(&self) -> usize { + let b = self.bytes(); + let mut i = self.pos + 1; + loop { + match find_byte(b, i, b'<') { + Some(j) if self.markup_at(j) => return j, + Some(j) => i = j + 1, + None => return b.len(), + } + } + } + + /// Skip a comment, doctype or processing instruction at `self.pos`. + fn skip_declaration(&mut self) { + let b = self.bytes(); + let end = if b.get(self.pos..self.pos + 4) == Some(b"").map_or(b.len(), |i| i + 3) + } + } else { + find_byte(b, self.pos, b'>').map_or(b.len(), |i| i + 1) + }; + self.pos = end.min(b.len()); + } + + /// Read the end tag at `self.pos` (``). + fn close_tag(&mut self) -> Option> { + let b = self.bytes(); + let start = self.pos + 2; + let mut i = start; + while let Some(c) = b.get(i) { + if c.is_ascii_whitespace() || *c == b'/' || *c == b'>' { + break; + } + i += 1; + } + let name = self.slice(start, i); + self.pos = find_byte(b, i, b'>').map_or(b.len(), |j| j + 1); + // `` is nothing at all. + (!name.is_empty()).then(|| Token::End(self.tag_name(name))) + } + + /// Read the start tag at `self.pos` (``). + fn open_tag(&mut self) -> Tag<'a> { + let b = self.bytes(); + let start = self.pos + 1; + let mut i = start; + while let Some(c) = b.get(i) { + if c.is_ascii_whitespace() || *c == b'/' || *c == b'>' { + break; + } + i += 1; + } + let name = self.tag_name(self.slice(start, i)); + let mut attrs = Vec::new(); + let mut self_close = false; + loop { + while matches!(b.get(i), Some(c) if c.is_ascii_whitespace() || *c == b'/') { + self_close = b.get(i) == Some(&b'/') && b.get(i + 1) == Some(&b'>'); + i += 1; + } + match b.get(i) { + None => break, + Some(b'>') => { + i += 1; + break; + } + Some(_) => {} + } + // An attribute name runs to white space, `/`, `>` or `=`; a leading `=` is part of it. + let name_start = i; + i += 1; + while let Some(c) = b.get(i) { + if c.is_ascii_whitespace() || matches!(*c, b'/' | b'>' | b'=') { + break; + } + i += 1; + } + let attr = self.slice(name_start, i); + let (value, next) = self.attr_value(i); + i = next; + if attrs.len() < self.opts.limits.max_attrs { + attrs.push((attr, value)); + } + } + self.pos = i.min(b.len()); + Tag { name, attrs, self_close } + } + + /// The value after an attribute name that ends at `i` (empty when it has none), and where + /// reading goes on. + fn attr_value(&self, i: usize) -> (&'a str, usize) { + let b = self.bytes(); + let mut j = i; + while matches!(b.get(j), Some(c) if c.is_ascii_whitespace()) { + j += 1; + } + if b.get(j) != Some(&b'=') { + return ("", j); + } + j += 1; + while matches!(b.get(j), Some(c) if c.is_ascii_whitespace()) { + j += 1; + } + match b.get(j) { + Some(&q) if q == b'"' || q == b'\'' => { + let end = find_byte(b, j + 1, q).unwrap_or(b.len()); + (self.slice(j + 1, end), (end + 1).min(b.len())) + } + _ => { + let mut end = j; + while let Some(c) = b.get(end) { + if c.is_ascii_whitespace() || *c == b'>' { + break; + } + end += 1; + } + (self.slice(j, end), end) + } + } + } +} + +impl<'a> Iterator for Lexer<'a, '_> { + type Item = Token<'a>; + + fn next(&mut self) -> Option> { + loop { + let b = self.bytes(); + if self.pos >= b.len() { + return None; + } + if let Some(name) = self.raw.take() { + let end = find_end_tag(b, self.pos, &name).unwrap_or(b.len()); + let text = self.slice(self.pos, end); + self.pos = end; + if !text.is_empty() { + return Some(Token::Text(text)); + } + continue; + } + if b.get(self.pos) != Some(&b'<') || !self.markup_at(self.pos) { + let start = self.pos; + self.pos = self.text_end(); + return Some(Token::Text(self.slice(start, self.pos))); + } + match b.get(self.pos + 1) { + Some(b'!' | b'?') => self.skip_declaration(), + Some(b'/') => { + if let Some(token) = self.close_tag() { + return Some(token); + } + } + _ => { + let tag = self.open_tag(); + if self.opts.drop.contains(&tag.name.as_str()) { + self.raw_content(&tag.name); + continue; + } + if self.opts.raw_text.contains(&tag.name.as_str()) { + self.raw = Some(tag.name.clone()); + } + return Some(Token::Start(tag)); + } + } + } + } +} + +fn find_byte(b: &[u8], from: usize, byte: u8) -> Option { + b.get(from..)?.iter().position(|c| *c == byte).map(|i| i + from) +} + +fn find_sub(b: &[u8], from: usize, needle: &[u8]) -> Option { + b.get(from..)?.windows(needle.len()).position(|w| w == needle).map(|i| i + from) +} + +/// Where `` or the end), at or after `from`. +fn find_end_tag(b: &[u8], from: usize, name: &str) -> Option { + let mut i = from; + while let Some(lt) = find_byte(b, i, b'<') { + let after = lt + 2 + name.len(); + let named = b.get(lt + 1) == Some(&b'/') && b.get(lt + 2..after).is_some_and(|n| n.eq_ignore_ascii_case(name.as_bytes())); + if named && b.get(after).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')) { + return Some(lt); + } + i = lt + 1; + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/html/src/lex/tests.rs b/crates/html/src/lex/tests.rs new file mode 100644 index 000000000..084caf702 --- /dev/null +++ b/crates/html/src/lex/tests.rs @@ -0,0 +1,192 @@ +use super::{Lexer, Limits, Options, Token}; + +/// As a mail reader sets it up. +const MAIL: Options<'static> = + Options { limits: Limits { max_attrs: 32, max_tag_name: 24 }, drop: &["script", "style", "title", "iframe"], raw_text: &["textarea", "xmp"] }; + +fn show<'a>(lexer: impl Iterator>) -> Vec { + lexer + .map(|t| match t { + Token::Text(t) => format!("T:{t}"), + Token::Start(tag) => { + let attrs: Vec = tag.attrs.iter().map(|(n, v)| format!("{n}={v}")).collect(); + if attrs.is_empty() { format!("<{}>", tag.name) } else { format!("<{} {}>", tag.name, attrs.join(" ")) } + } + Token::End(n) => format!(""), + }) + .collect() +} + +fn tokens(html: &str) -> Vec { + show(Lexer::with(html, MAIL)) +} + +#[test] +fn text_and_tags() { + assert_eq!(tokens("Hi there!"), ["T:Hi ", "", "T:there", "", "T:!"]); + assert_eq!(tokens(""), Vec::::new()); + assert_eq!(tokens("plain"), ["T:plain"]); + assert_eq!(tokens("

"), ["
", "
"]); +} + +#[test] +fn attributes_in_every_quoting_style() { + assert_eq!(tokens(r#""#), [r#""#]); + assert_eq!(tokens("t"), ["", "T:t"]); + assert_eq!(tokens(""), [""]); + assert_eq!(tokens("

"), ["

"]); + // A `>` inside a quoted value does not end the tag. + assert_eq!(tokens(r#"t"#), ["b>", "T:t", ""]); +} + +#[test] +fn attribute_values_are_decoded_on_demand() { + let mut t = Lexer::new(r#""#); + let Some(Token::Start(tag)) = t.next() else { panic!("no tag") }; + assert_eq!(tag.attr("href").as_deref(), Some("?a=1&b=2©=3")); + assert_eq!(tag.raw_attr("href"), Some("?a=1&b=2©=3")); + assert_eq!(tag.attr("missing"), None); + assert!(tag.has("checked") && !tag.has("check")); + assert_eq!(tag.attributes().map(|(n, _)| n).collect::>(), ["HREF", "href", "Checked"]); +} + +#[test] +fn a_tag_that_closes_itself_says_so() { + let closes = |html: &str| match Lexer::new(html).next() { + Some(Token::Start(tag)) => tag.self_close, + other => panic!("{other:?}"), + }; + for yes in ["
", "
", "", "", ""] { + assert!(closes(yes), "{yes}"); + } + // A `/` that ends an unquoted value, or stands anywhere but before the `>`, is not it. + for no in ["
", "", "
", "", "
d"), ["T:a < b and c > d"]); + assert_eq!(tokens("1<2<3"), ["T:1<2<3"]); + assert_eq!(tokens("<"), ["T:<"]); + assert_eq!(tokens("x<"), ["T:x<"]); + assert_eq!(tokens("<3 <- b"), ["T:a", "T:b"]); +} + +#[test] +fn comments_and_declarations_vanish() { + assert_eq!(tokens("ab"), ["T:a", "T:b"]); + assert_eq!(tokens("

x"), ["

", "T:x"]); + assert_eq!(tokens("x"), ["T:x"]); + assert_eq!(tokens("abc"), ["T:a", "T:b", "T:c"]); + assert_eq!(tokens("shown"), ["T:shown"]); + assert_eq!(tokens("shown"), ["T:shown"]); + // An unfinished comment swallows the rest. + assert_eq!(tokens("a", "x".repeat(70_000)); + let mut exif = sample_exif(); + exif.resize(70_000, 0); + img.meta.xmp = Some(big.clone()); + img.meta.exif = Some(exif); + let back = decode(&encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap()).unwrap(); + assert!(back.meta.xmp.is_none() && back.meta.exif.is_none()); + let w = fidelity_warnings(&img, Format::Jpeg); + assert!(w.contains(&FidelityWarning::MetadataTooLarge { what: "XMP", bytes: big.len() }), "{w:?}"); + assert!(w.contains(&FidelityWarning::MetadataTooLarge { what: "EXIF", bytes: 70_000 }), "{w:?}"); +} + +#[test] +fn embed_flags_disable_metadata() { + let img = rich_image(ChannelLayout::Rgb); + let opts = EncodeOptions { embed_icc: false, embed_metadata: false, ..Default::default() }; + for f in [Format::Png, Format::Jpeg, Format::Tiff, Format::WebP] { + let back = decode(&encode(&img, f, &opts).unwrap()).unwrap(); + assert!(back.icc.is_none(), "{f:?}"); + assert!(back.meta.exif.is_none() && back.meta.xmp.is_none(), "{f:?}"); + assert!(back.meta.text.is_empty(), "{f:?}"); + assert!( + fidelity_warnings_with(&img, Format::Bmp, &opts).iter().all(|w| !matches!(w, FidelityWarning::IccDropped | FidelityWarning::ExifDropped)) + ); + } +} + +#[test] +fn png_unicode_text_uses_itxt() { + let mut img = test_image(ChannelLayout::Rgb, SampleType::U8); + img.meta.text = vec![("Title".into(), "Ünïcödé ☃ 雪".into()), ("Author".into(), "latin1 ok é".into())]; + let bytes = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + assert!(bytes.windows(4).any(|w| w == b"iTXt")); + assert!(bytes.windows(4).any(|w| w == b"tEXt")); + let mut back = decode(&bytes).unwrap().meta.text; + let mut want = img.meta.text.clone(); + back.sort(); + want.sort(); + assert_eq!(back, want); +} + +#[test] +fn png_xmp_in_itxt_keyword() { + let mut img = test_image(ChannelLayout::Gray, SampleType::U8); + img.meta.xmp = Some(SAMPLE_XMP.into()); + let bytes = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + assert!(bytes.windows(17).any(|w| w == b"XML:com.adobe.xmp")); + let back = decode(&bytes).unwrap(); + assert_eq!(back.meta.xmp.as_deref(), Some(SAMPLE_XMP)); + assert!(back.meta.text.is_empty(), "XMP must not leak into text"); +} + +#[test] +fn png_exif_chunk_written() { + let mut img = test_image(ChannelLayout::Rgb, SampleType::U8); + img.meta.exif = Some(sample_exif()); + let bytes = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + assert!(bytes.windows(4).any(|w| w == b"eXIf")); + assert!(!bytes.windows(4).any(|w| w == b"iCCP")); +} + +#[test] +fn jpeg_exif_readable_by_oracle() { + let mut img = test_image(ChannelLayout::Rgb, SampleType::U8); + img.meta.exif = Some(sample_exif()); + let bytes = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + use image::ImageDecoder; + let mut d = image::codecs::jpeg::JpegDecoder::new(std::io::Cursor::new(&bytes)).unwrap(); + let exif = d.exif_metadata().unwrap().unwrap(); + assert!(exif.ends_with(&sample_exif())); +} + +#[test] +fn jpeg_dpi_in_jfif() { + let mut img = test_image(ChannelLayout::Gray, SampleType::U8); + img.meta.dpi = Some((96.0, 200.0)); + let back = decode(&encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.meta.dpi, Some((96.0, 200.0))); +} + +#[test] +fn tiff_icc_readable_by_oracle() { + let img = test_image(ChannelLayout::Rgb, SampleType::U16).with_icc(Some(sample_icc(1234))); + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + use image::ImageDecoder; + let mut d = image::codecs::tiff::TiffDecoder::new(std::io::Cursor::new(&bytes)).unwrap(); + assert_eq!(d.icc_profile().unwrap(), img.icc); +} + +#[test] +fn png_icc_readable_by_oracle() { + let img = test_image(ChannelLayout::Rgba, SampleType::U8).with_icc(Some(sample_icc(777))); + let bytes = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + use image::ImageDecoder; + let mut d = image::codecs::png::PngDecoder::new(std::io::Cursor::new(&bytes)).unwrap(); + assert_eq!(d.icc_profile().unwrap(), img.icc); +} + +#[test] +fn metadata_survives_depth_conversion() { + let mut img = test_image(ChannelLayout::Rgb, SampleType::F32).with_icc(Some(sample_icc(300))); + img.meta.xmp = Some(SAMPLE_XMP.into()); + let back = decode(&encode(&img, Format::Png, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.icc, img.icc); + assert_eq!(back.meta.xmp, img.meta.xmp); +} + +/// #2399: a WebP export keeps the resolution even when the image has no EXIF (WebP has no +/// resolution field of its own, so a minimal EXIF block carries it), lossless and lossy alike, +/// and no longer warns that it is dropped. +#[test] +fn webp_keeps_dpi_without_source_exif() { + let mut img = test_image(ChannelLayout::Rgb, SampleType::U8); + img.meta = Metadata { dpi: Some((300.0, 150.0)), ..Default::default() }; + assert!(!fidelity_warnings(&img, Format::WebP).contains(&FidelityWarning::DpiDropped)); + for lossless in [true, false] { + let opts = EncodeOptions { webp_lossless: lossless, ..Default::default() }; + let back = decode(&encode(&img, Format::WebP, &opts).unwrap()).unwrap(); + let (x, y) = back.meta.dpi.unwrap_or_else(|| panic!("lossless {lossless}: dpi lost")); + assert!((x - 300.0).abs() < 0.05 && (y - 150.0).abs() < 0.05, "lossless {lossless}: {x},{y}"); + } + // EXIF that can't be parsed is written as it is, so the resolution is reported as dropped. + img.meta = Metadata { exif: Some(b"Exif\0\0garbage".to_vec()), dpi: Some((300.0, 150.0)), ..Default::default() }; + assert!(fidelity_warnings(&img, Format::WebP).contains(&FidelityWarning::DpiDropped)); + let back = decode(&encode(&img, Format::WebP, &EncodeOptions::default()).unwrap()).unwrap(); + assert!(back.meta.exif.is_some()); + // ...unless the XMP written beside it records the resolution, which readers look at first. + img.meta.xmp = + Some(r#""#.into()); + assert!(!fidelity_warnings(&img, Format::WebP).contains(&FidelityWarning::DpiDropped)); + let back = decode(&encode(&img, Format::WebP, &EncodeOptions::default()).unwrap()).unwrap(); + let (x, y) = back.meta.dpi.expect("dpi from XMP"); + assert!((x - 300.0).abs() < 0.05 && (y - 150.0).abs() < 0.05, "{x},{y}"); + // The default 72 ppi needs no EXIF (a WebP without one opens at 72), so a plain export stays + // a simple-format file; nor does an image without a resolution. + img.meta = Metadata { dpi: Some((72.0, 72.0)), ..Default::default() }; + assert!(!fidelity_warnings(&img, Format::WebP).contains(&FidelityWarning::DpiDropped)); + let bytes = encode(&img, Format::WebP, &EncodeOptions { webp_lossless: false, ..Default::default() }).unwrap(); + assert_eq!(bytes.get(12..16), Some(&b"VP8 "[..])); + // Without a resolution or EXIF there is still no EXIF chunk. + img.meta = Metadata::default(); + let back = decode(&encode(&img, Format::WebP, &EncodeOptions::default()).unwrap()).unwrap(); + assert!(back.meta.exif.is_none()); +} diff --git a/crates/image-codecs/tests/it/orientation.rs b/crates/image-codecs/tests/it/orientation.rs new file mode 100644 index 000000000..d5d3ad90e --- /dev/null +++ b/crates/image-codecs/tests/it/orientation.rs @@ -0,0 +1,676 @@ +//! EXIF / TIFF orientation (#285): files are opened upright, and every export +//! writes Orientation = 1 so the upright pixels are never rotated twice. + +use crate::common; +use common::*; +use craft_image_codecs::*; +use proptest::prelude::{any, prop_assert, prop_assert_eq, proptest}; + +/// A minimal EXIF (TIFF-structured) block: a filler tag, then Orientation. +fn exif(o: u32, big: bool, long: bool) -> Vec { + let u16b = |v: u16| if big { v.to_be_bytes() } else { v.to_le_bytes() }; + let u32b = |v: u32| if big { v.to_be_bytes() } else { v.to_le_bytes() }; + let mut v = if big { b"MM\0*".to_vec() } else { b"II*\0".to_vec() }; + v.extend_from_slice(&u32b(8)); + v.extend_from_slice(&u16b(2)); + // ImageDescription-ish filler (ASCII, 4 bytes inline). + v.extend_from_slice(&u16b(0x010E)); + v.extend_from_slice(&u16b(2)); + v.extend_from_slice(&u32b(4)); + v.extend_from_slice(b"abc\0"); + v.extend_from_slice(&u16b(0x0112)); + if long { + v.extend_from_slice(&u16b(4)); + v.extend_from_slice(&u32b(1)); + v.extend_from_slice(&u32b(o)); + } else { + v.extend_from_slice(&u16b(3)); + v.extend_from_slice(&u32b(1)); + v.extend_from_slice(&u16b(o as u16)); + v.extend_from_slice(&[0, 0]); + } + v.extend_from_slice(&u32b(0)); + v +} + +/// The "upright" test picture: `w`×`h` gray values, distinct per pixel. +fn upright(w: usize, h: usize, value: impl Fn(usize, usize) -> u8) -> Vec { + (0..h).flat_map(|y| (0..w).map(move |x| (x, y))).map(|(x, y)| value(x, y)).collect() +} + +/// What a camera stores for orientation `o` when the scene is `up` (`w`×`h`): the inverse +/// of the turn a viewer applies. Returns the stored pixels and their size. +fn stored(up: &[u8], w: usize, h: usize, o: u16) -> (Vec, usize, usize) { + let (sw, sh) = if o >= 5 { (h, w) } else { (w, h) }; + let mut s = vec![0u8; up.len()]; + for y in 0..h { + for x in 0..w { + // Where upright pixel (x, y) sits in the stored image. + let (sx, sy) = match o { + 1 => (x, y), + 2 => (w - 1 - x, y), // mirrored horizontally + 3 => (w - 1 - x, h - 1 - y), // rotated 180° + 4 => (x, h - 1 - y), // mirrored vertically + 5 => (y, x), // transposed + 6 => (y, w - 1 - x), // viewer turns it 90° clockwise + 7 => (h - 1 - y, w - 1 - x), // transverse + 8 => (h - 1 - y, x), // viewer turns it 90° counter-clockwise + _ => unreachable!(), + }; + s[sy * sw + sx] = up[y * w + x]; + } + } + (s, sw, sh) +} + +/// Inserts a JPEG APP1 EXIF segment right after SOI. +fn jpeg_with_exif(jpeg: &[u8], exif: &[u8]) -> Vec { + let mut seg = b"Exif\0\0".to_vec(); + seg.extend_from_slice(exif); + let mut out = jpeg[..2].to_vec(); + out.extend_from_slice(&[0xFF, 0xE1]); + out.extend_from_slice(&((seg.len() + 2) as u16).to_be_bytes()); + out.extend_from_slice(&seg); + out.extend_from_slice(&jpeg[2..]); + out +} + +/// Inserts a PNG eXIf chunk before the first IDAT. +fn png_with_exif(png: &[u8], exif: &[u8]) -> Vec { + let at = png.windows(4).position(|w| w == b"IDAT").unwrap() - 4; + let mut chunk = (exif.len() as u32).to_be_bytes().to_vec(); + let mut body = b"eXIf".to_vec(); + body.extend_from_slice(exif); + let mut crc = flate2::Crc::new(); + crc.update(&body); + chunk.extend_from_slice(&body); + chunk.extend_from_slice(&crc.sum().to_be_bytes()); + let mut out = png[..at].to_vec(); + out.extend_from_slice(&chunk); + out.extend_from_slice(&png[at..]); + out +} + +fn tiff_with_orientation(data: &[u8], w: u32, h: u32, o: u16) -> Vec { + use tiff::encoder::{TiffEncoder, colortype::Gray8}; + let mut cursor = std::io::Cursor::new(Vec::new()); + let mut enc = TiffEncoder::new(&mut cursor).unwrap(); + let mut im = enc.new_image::(w, h).unwrap(); + im.encoder().write_tag(tiff::tags::Tag::Orientation, o).unwrap(); + im.write_data(data).unwrap(); + cursor.into_inner() +} + +fn gray(w: usize, h: usize, px: Vec) -> Image { + Image::from_u8(w as u32, h as u32, ChannelLayout::Gray, px).unwrap() +} + +/// Exact pixels: distinct values per pixel. +fn exact_scene(w: usize, h: usize) -> Vec { + upright(w, h, |x, y| (y * w + x) as u8 * 3 + 1) +} + +#[test] +fn reads_orientation_in_every_encoding() { + for o in 1..=8 { + for big in [false, true] { + for long in [false, true] { + let e = exif(o, big, long); + assert_eq!(exif_orientation(&e), o as u16, "o={o} big={big} long={long}"); + let mut prefixed = b"Exif\0\0".to_vec(); + prefixed.extend_from_slice(&e); + assert_eq!(exif_orientation(&prefixed), o as u16); + let fixed = upright_exif(&prefixed); + assert_eq!(exif_orientation(&fixed), 1); + assert_eq!(fixed.len(), prefixed.len(), "rewritten in place"); + assert_eq!(&fixed[..6], b"Exif\0\0"); + // Rewritten in the entry's own type and width: exactly `exif(1, ..)`. + assert_eq!(&fixed[6..], &exif(1, big, long)[..], "o={o} big={big} long={long}"); + } + } + } + // Already upright, no tag, or garbage: borrowed unchanged. + assert!(matches!(upright_exif(&exif(1, false, false)), std::borrow::Cow::Borrowed(_))); + assert!(matches!(upright_exif(&sample_exif()), std::borrow::Cow::Borrowed(_))); + assert!(matches!(upright_exif(b"nonsense"), std::borrow::Cow::Borrowed(_))); +} + +#[test] +fn malformed_or_out_of_range_orientation_reads_as_1() { + for bad in [0, 9, 255, 0xFFFF, 0x1_0006] { + assert_eq!(exif_orientation(&exif(bad, false, true)), 1, "{bad}"); + } + let good = exif(6, false, false); + assert_eq!(exif_orientation(&good), 6); + for n in 0..good.len() { + // Any cut, even one that keeps the Orientation entry but loses the next-IFD pointer. + assert_eq!(exif_orientation(&good[..n]), 1, "truncated at {n}"); + assert!(matches!(upright_exif(&good[..n]), std::borrow::Cow::Borrowed(_)), "truncated at {n}"); + } + // IFD offset past the end, a huge entry count, a wrong field type. + let mut far = good.clone(); + far[4..8].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(exif_orientation(&far), 1); + let mut many = good.clone(); + many[8..10].copy_from_slice(&u16::MAX.to_le_bytes()); + assert_eq!(exif_orientation(&many), 1, "an entry count past the end is a truncated table"); + let mut typed = good.clone(); + typed[24..26].copy_from_slice(&2u16.to_le_bytes()); // ASCII + assert_eq!(exif_orientation(&typed), 1); + assert_eq!(exif_orientation(&[]), 1); + assert_eq!(exif_orientation(b"II*\0"), 1); +} + +proptest! { + #[test] + fn random_exif_never_panics(bytes in proptest::collection::vec(any::(), 0..256), le in any::()) { + let mut b = if le { b"II*\0\x08\0\0\0".to_vec() } else { b"MM\0*\0\0\0\x08".to_vec() }; + b.extend_from_slice(&bytes); + let o = exif_orientation(&b); + prop_assert!((1..=8).contains(&o)); + prop_assert_eq!(exif_orientation(&upright_exif(&b)), 1); + let _ = exif_orientation(&bytes); + let _ = upright_exif(&bytes); + } + + #[test] + fn random_xmp_never_panics(s in ".{0,64}", tail in ".{0,16}") { + let x = format!("{s}tiff:Orientation{tail}"); + let _ = upright_xmp(&x); + } +} + +#[test] +fn xmp_orientation_is_rewritten() { + let attr = r#""#; + assert_eq!(upright_xmp(attr), r#""#); + let elem = "83"; + assert_eq!(upright_xmp(elem), "11"); + assert_eq!(upright_xmp("tiff:Orientation='12'"), "tiff:Orientation='1'"); + for same in [r#"tiff:Orientation="1""#, "no orientation", "tiff:Orientation", "tiff:Orientation=\"\"", "tiff:Orientation: 6"] { + assert!(matches!(upright_xmp(same), std::borrow::Cow::Borrowed(_)), "{same}"); + } +} + +#[test] +fn png_opens_upright_for_every_orientation() { + let (w, h) = (5, 3); + let up = exact_scene(w, h); + for o in 1..=8u16 { + let (s, sw, sh) = stored(&up, w, h, o); + let png = encode(&gray(sw, sh, s.clone()), Format::Png, &EncodeOptions::default()).unwrap(); + let file = png_with_exif(&png, &exif(u32::from(o), o % 2 == 0, false)); + let img = decode(&file).unwrap(); + assert_eq!(img.dimensions(), (w as u32, h as u32), "o={o}"); + assert_eq!(img.data(), &up[..], "o={o}"); + assert_eq!(exif_orientation(img.meta.exif.as_deref().unwrap()), 1, "o={o}: metadata says upright"); + // Decoded in steps and finished by `finish_png_decode`, it is the same image. + let stepped = PngDecodeRows::new(file.as_slice(), &Limits::default()).unwrap().step(&mut || true).unwrap().unwrap(); + assert_eq!(finish_png_decode(stepped, &DecodeOptions::default()).unwrap(), img, "o={o}: stepped"); + // Opting out keeps the stored pixels and the tag. + let raw = decode_with(&file, &DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap(); + assert_eq!((raw.dimensions(), raw.data()), ((sw as u32, sh as u32), &s[..]), "o={o}"); + assert_eq!(exif_orientation(raw.meta.exif.as_deref().unwrap()), o); + } +} + +/// An image turned upright a band of rows at a time is the image `oriented` gives, for every +/// orientation, pixel size and shape, and takes a call per band. +#[test] +fn orient_steps_equal_oriented() { + for o in 1..=8u16 { + for (layout, sample) in [ + (ChannelLayout::Gray, SampleType::U8), + (ChannelLayout::GrayA, SampleType::U16), + (ChannelLayout::Rgb, SampleType::U8), + (ChannelLayout::Rgba, SampleType::U16), + (ChannelLayout::Rgba, SampleType::F32), + ] { + for (w, h) in [(1u32, 1u32), (7, 5), (130, 66), (65, 200)] { + let len = w as usize * h as usize * layout.channels() * sample.bytes(); + let data: Vec = (0..len).map(|i| (i.wrapping_mul(2_654_435_761) >> 11) as u8).collect(); + let mut img = Image::from_raw(w, h, layout, sample, data).unwrap(); + img.meta.dpi = Some((100.0, 200.0)); + img.meta.exif = Some(exif(u32::from(o), false, false)); + let want = img.clone().oriented(o).unwrap(); + let mut steps = OrientSteps::new(img, o).unwrap(); + let mut calls = 0; + let got = loop { + calls += 1; + assert!(steps.progress() <= 1.0); + if let Some(img) = steps.step(&mut || false).unwrap() { + break img; + } + }; + assert_eq!(got, want, "o={o} {layout:?} {sample:?} {w}x{h}"); + let rows = if o >= 5 { w } else { h } as usize; + let expected = if (2..=8).contains(&o) { rows.div_ceil(64) } else { 1 }; + assert_eq!(calls, expected, "o={o} {w}x{h}: one call per band of 64 rows"); + } + } + } +} + +/// A PNG turned upright in bands by `begin_png_finish` is the image `decode` gives, for every +/// orientation, and a band of a tall image is not the whole of it. +#[test] +fn png_finishes_upright_in_bands_for_every_orientation() { + let (w, h) = (70, 150); + let up = upright(w, h, |x, y| (x * 7 + y * 13) as u8); + for o in 1..=8u16 { + let (s, sw, sh) = stored(&up, w, h, o); + let png = encode(&gray(sw, sh, s), Format::Png, &EncodeOptions::default()).unwrap(); + let file = png_with_exif(&png, &exif(u32::from(o), o % 2 == 0, false)); + let want = decode(&file).unwrap(); + let img = PngDecodeRows::new(file.as_slice(), &Limits::default()).unwrap().step(&mut || true).unwrap().unwrap(); + let got = match begin_png_finish(img, &DecodeOptions::default()).unwrap() { + PngFinish::Done(img) => { + assert_eq!(o, 1); + *img + } + PngFinish::Orient(mut steps) => { + assert!(o > 1); + let mut calls = 0; + loop { + calls += 1; + if let Some(img) = steps.step(&mut || false).unwrap() { + assert!(calls > 1, "o={o}: more than one band"); + break img; + } + } + } + }; + assert_eq!(got, want, "o={o}"); + } +} + +/// Extra samples (a TIFF's alpha and spot planes) turn with the image, so a channel stays +/// aligned with the pixels for every orientation. +#[test] +fn extra_samples_turn_with_the_image() { + let (w, h) = (4, 7); + let up = exact_scene(w, h); + let up_extra: Vec = up.iter().map(|v| 255 - v).collect(); + for o in 1..=8u16 { + let (s, sw, sh) = stored(&up, w, h, o); + let (s_extra, _, _) = stored(&up_extra, w, h, o); + let mut img = gray(sw, sh, s); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: s_extra }]; + let turned = img.oriented(o).unwrap(); + assert_eq!(turned.data(), &up[..], "o={o}"); + assert_eq!(turned.meta.extra_samples.len(), 1, "o={o}"); + assert_eq!(turned.meta.extra_samples[0].data, up_extra, "o={o}"); + } +} + +#[test] +fn tiff_opens_upright_for_every_orientation() { + let (w, h) = (4, 7); + let up = exact_scene(w, h); + for o in 1..=8u16 { + let (s, sw, sh) = stored(&up, w, h, o); + let img = decode(&tiff_with_orientation(&s, sw as u32, sh as u32, o)).unwrap(); + assert_eq!(img.dimensions(), (w as u32, h as u32), "o={o}"); + assert_eq!(img.data(), &up[..], "o={o}"); + } +} + +#[test] +fn webp_opens_upright() { + let (w, h) = (6, 2); + let up = exact_scene(w, h); + for o in [3, 6, 8] { + let (s, sw, sh) = stored(&up, w, h, o); + let mut out = Vec::new(); + let mut enc = image_webp::WebPEncoder::new(&mut out); + enc.set_exif_metadata(exif(u32::from(o), false, false)); + enc.encode(&s, sw as u32, sh as u32, image_webp::ColorType::L8).unwrap(); + // WebP has no gray mode: it comes back as RGB. + let img = decode(&out).unwrap().convert(ChannelLayout::Gray, SampleType::U8); + assert_eq!((img.dimensions(), img.data()), ((w as u32, h as u32), &up[..]), "o={o}"); + assert_eq!(probe_size(&out).unwrap(), (w as u32, h as u32), "the header probe turns upright too, o={o}"); + } +} + +/// 8×8-block scene so JPEG keeps each block's value: block (bx, by) is distinct. +fn block_scene(w: usize, h: usize) -> Vec { + upright(w, h, |x, y| (20 + 25 * (x / 8 + (w / 8) * (y / 8))) as u8) +} + +fn assert_blocks(img: &Image, up: &[u8], w: usize, h: usize, what: &str) { + assert_eq!(img.dimensions(), (w as u32, h as u32), "{what}"); + let d = img.data(); + for by in 0..h / 8 { + for bx in 0..w / 8 { + let i = (by * 8 + 4) * w + bx * 8 + 4; + assert!((i32::from(d[i]) - i32::from(up[i])).abs() <= 4, "{what}: block ({bx},{by}) is {} not {}", d[i], up[i]); + } + } +} + +#[test] +fn jpeg_opens_upright_for_every_orientation() { + let (w, h) = (32, 16); + let up = block_scene(w, h); + for o in 1..=8u16 { + let (s, sw, sh) = stored(&up, w, h, o); + let jpeg = encode(&gray(sw, sh, s), Format::Jpeg, &EncodeOptions { jpeg_quality: 100, ..Default::default() }).unwrap(); + let file = jpeg_with_exif(&jpeg, &exif(u32::from(o), o > 4, o == 7)); + let img = decode(&file).unwrap(); + assert_blocks(&img, &up, w, h, &format!("o={o}")); + assert_eq!(exif_orientation(img.meta.exif.as_deref().unwrap()), 1); + } +} + +#[test] +fn rgb_jpeg_with_dpi_swaps_resolution_on_a_quarter_turn() { + let img = Image::from_u8(16, 8, ChannelLayout::Rgb, vec![128; 16 * 8 * 3]) + .unwrap() + .with_meta(Metadata { dpi: Some((300.0, 150.0)), ..Default::default() }); + let jpeg = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + let back = decode(&jpeg_with_exif(&jpeg, &exif(6, false, false))).unwrap(); + assert_eq!(back.dimensions(), (8, 16)); + assert_eq!(back.meta.dpi, Some((150.0, 300.0))); +} + +#[test] +fn export_writes_orientation_1_and_keeps_pixels() { + let (w, h) = (32, 16); + let up = block_scene(w, h); + let (s, sw, sh) = stored(&up, w, h, 6); + let jpeg = encode(&gray(sw, sh, s), Format::Jpeg, &EncodeOptions { jpeg_quality: 100, ..Default::default() }).unwrap(); + let opened = decode(&jpeg_with_exif(&jpeg, &exif(6, false, false))).unwrap(); + assert_blocks(&opened, &up, w, h, "opened"); + + // Even an image whose metadata still claims 6 (e.g. from a PSD) is written as 1. + let mut stale = opened.clone(); + stale.meta.exif = Some(exif(6, true, false)); + stale.meta.xmp = Some(r#""#.into()); + for format in [Format::Png, Format::Jpeg, Format::WebP, Format::Tiff] { + let bytes = encode(&stale, format, &EncodeOptions { jpeg_quality: 100, ..Default::default() }).unwrap(); + let raw = decode_with(&bytes, &DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap(); + if let Some(e) = &raw.meta.exif { + assert_eq!(exif_orientation(e), 1, "{format:?}"); + } + if let Some(x) = &raw.meta.xmp { + assert!(x.contains(r#"tiff:Orientation="1""#), "{format:?}: {x}"); + } + assert_eq!(exif_orientation(&bytes), 1, "{format:?} file"); + // Pixels as shown, both with and without applying orientation on reopen. + let back = decode(&bytes).unwrap(); + assert_eq!(back.dimensions(), raw.dimensions(), "{format:?}"); + if format == Format::Jpeg { + assert_blocks(&back, &up, w, h, "jpeg reopen"); + } else { + assert_eq!(back.convert(ChannelLayout::Gray, SampleType::U8).data(), opened.data(), "{format:?} lossless"); + } + } +} + +#[test] +fn round_trip_open_save_reopen_is_stable() { + let (w, h) = (24, 16); + let up = block_scene(w, h); + for o in 1..=8u16 { + let (s, sw, sh) = stored(&up, w, h, o); + let jpeg = encode(&gray(sw, sh, s), Format::Jpeg, &EncodeOptions { jpeg_quality: 100, ..Default::default() }).unwrap(); + let first = decode(&jpeg_with_exif(&jpeg, &exif(u32::from(o), false, false))).unwrap(); + let saved = encode(&first, Format::Jpeg, &EncodeOptions { jpeg_quality: 100, ..Default::default() }).unwrap(); + let second = decode(&saved).unwrap(); + assert_blocks(&second, &up, w, h, &format!("reopened o={o}")); + assert_eq!(exif_orientation(second.meta.exif.as_deref().unwrap()), 1); + } +} + +#[test] +fn malformed_exif_in_a_jpeg_opens_as_stored() { + let img = gray(16, 8, vec![90; 128]); + let jpeg = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + let mut rng = Rng::new(285); + let mut cases = vec![Vec::new(), b"II*\0".to_vec(), b"MM\0*\xff\xff\xff\xff".to_vec(), exif(6, false, false)[..20].to_vec()]; + cases.extend((0..64).map(|i| { + let mut b = if i % 2 == 0 { b"II*\0\x08\0\0\0".to_vec() } else { b"MM\0*\0\0\0\x08".to_vec() }; + b.extend(rng.bytes(i * 7)); + b + })); + for e in cases { + let back = decode(&jpeg_with_exif(&jpeg, &e)).unwrap(); + let o = exif_orientation(&e); + let want = if o >= 5 { (8, 16) } else { (16, 8) }; + assert_eq!(back.dimensions(), want, "{e:?}"); + } +} + +#[test] +fn every_layout_and_depth_orients_and_inverts() { + let inverse = |o: u16| match o { + 6 => 8, + 8 => 6, + o => o, + }; + for layout in ChannelLayout::ALL { + for sample in SampleType::ALL { + let img = test_image(layout, sample); + for o in 0..=9u16 { + let turned = img.clone().oriented(o).unwrap(); + if (2..=8).contains(&o) && o >= 5 { + assert_eq!(turned.dimensions(), (img.height(), img.width())); + } else { + assert_eq!(turned.dimensions(), img.dimensions()); + } + let back = turned.oriented(inverse(o)).unwrap(); + assert_eq!(back.data(), img.data(), "{layout:?} {sample:?} o={o}"); + } + } + } +} + +/// #285 performance budget: turning a 24 MP photo upright on open adds ≤ 50 ms +/// (release build: `cargo test --release -p craft-image-codecs --test it -- orientation:: --ignored`). +#[test] +#[ignore = "timing; run in release"] +fn rotating_24_mp_is_fast() { + let (w, h) = (6000u32, 4000u32); + let img = Image::from_u8(w, h, ChannelLayout::Rgb, (0..w as usize * h as usize * 3).map(|i| i as u8).collect()).unwrap(); + for o in [6u16, 3, 8] { + let mut best = f64::MAX; + for _ in 0..5 { + let c = img.clone(); + let t = std::time::Instant::now(); + let r = c.oriented(o).unwrap(); + best = best.min(t.elapsed().as_secs_f64() * 1000.0); + std::hint::black_box(r); + } + eprintln!("orientation {o}: 24 MP RGB8 in {best:.1} ms"); + assert!(best <= 50.0, "orientation {o} took {best:.1} ms"); + } +} + +/// A TIFF-structured EXIF block: IFD0 at offset 8 holding `entries` +/// (`tag, type, count, 4-byte value field`), a zero next-IFD pointer, then `tail`. +fn ifd(big: bool, entries: &[(u16, u16, u32, [u8; 4])], tail: &[u8]) -> Vec { + let u16b = |v: u16| if big { v.to_be_bytes() } else { v.to_le_bytes() }; + let u32b = |v: u32| if big { v.to_be_bytes() } else { v.to_le_bytes() }; + let mut v = if big { b"MM\0*".to_vec() } else { b"II*\0".to_vec() }; + v.extend_from_slice(&u32b(8)); + v.extend_from_slice(&u16b(entries.len() as u16)); + for &(tag, ty, count, value) in entries { + v.extend_from_slice(&u16b(tag)); + v.extend_from_slice(&u16b(ty)); + v.extend_from_slice(&u32b(count)); + v.extend_from_slice(&value); + } + v.extend_from_slice(&u32b(0)); + v.extend_from_slice(tail); + v +} + +/// The 4-byte value field of a SHORT. +fn short(v: u16, big: bool) -> [u8; 4] { + let b = if big { v.to_be_bytes() } else { v.to_le_bytes() }; + [b[0], b[1], 0, 0] +} + +#[test] +fn duplicate_orientation_entries_first_wins() { + for big in [false, true] { + let o = |v| (0x0112, 3, 1, short(v, big)); + assert_eq!(exif_orientation(&ifd(big, &[o(6), o(3)], &[])), 6); + assert_eq!(exif_orientation(&ifd(big, &[o(3), o(6)], &[])), 3); + assert_eq!(exif_orientation(&ifd(big, &[o(1), o(8)], &[])), 1); + // A malformed first entry is ignored as a whole: a later good one does not count. + let bad_first = ifd(big, &[(0x0112, 3, 2, short(6, big)), o(8)], &[]); + assert_eq!(exif_orientation(&bad_first), 1); + let long = |v: u32| (0x0112, 4, 1, if big { v.to_be_bytes() } else { v.to_le_bytes() }); + let long_first = ifd(big, &[long(6), o(8)], &[]); + assert_eq!(exif_orientation(&long_first), 6, "a LONG first entry counts"); + assert_eq!(&upright_exif(&long_first)[..], &ifd(big, &[long(1), o(1)], &[])[..], "each in its own width"); + let rational_first = ifd(big, &[(0x0112, 5, 1, short(6, big)), o(8)], &[]); + assert_eq!(exif_orientation(&rational_first), 1); + // Upright: every well-formed duplicate becomes 1, so no reader rotates twice. + let dup = ifd(big, &[o(6), o(3)], &[]); + let fixed = upright_exif(&dup); + assert_eq!(fixed.len(), dup.len()); + assert_eq!(&fixed[..], &ifd(big, &[o(1), o(1)], &[])[..]); + // Only a later duplicate disagrees: it is rewritten too. + assert_eq!(&upright_exif(&ifd(big, &[o(1), o(8)], &[]))[..], &ifd(big, &[o(1), o(1)], &[])[..]); + assert!(matches!(upright_exif(&ifd(big, &[o(1), o(1)], &[])), std::borrow::Cow::Borrowed(_))); + } +} + +#[test] +fn strict_ifd_parsing_ignores_the_tag() { + for big in [false, true] { + let o6 = (0x0112, 3, 1, short(6, big)); + let good = ifd(big, &[o6], &[]); + assert_eq!(exif_orientation(&good), 6); + // Truncated before (or inside) the next-IFD pointer. + for cut in 1..=4 { + assert_eq!(exif_orientation(&good[..good.len() - cut]), 1, "cut {cut}"); + } + // IFD offsets inside the 8-byte header. + for off in 0..8u32 { + let mut b = good.clone(); + b[4..8].copy_from_slice(&if big { off.to_be_bytes() } else { off.to_le_bytes() }); + assert_eq!(exif_orientation(&b), 1, "offset {off}"); + assert!(matches!(upright_exif(&b), std::borrow::Cow::Borrowed(_))); + } + // Count must be exactly 1, type must be SHORT or LONG. + let long6 = if big { 6u32.to_be_bytes() } else { 6u32.to_le_bytes() }; + for count in [0, 2, 3, u32::MAX] { + assert_eq!(exif_orientation(&ifd(big, &[(0x0112, 3, count, short(6, big))], &[])), 1, "count {count}"); + assert_eq!(exif_orientation(&ifd(big, &[(0x0112, 4, count, long6)], &[])), 1, "LONG count {count}"); + } + for ty in [1, 2, 5, 6, 7, 8, 9, 0, 0xFFFF] { + let b = ifd(big, &[(0x0112, ty, 1, short(6, big))], &[]); + assert_eq!(exif_orientation(&b), 1, "type {ty}"); + assert!(matches!(upright_exif(&b), std::borrow::Cow::Borrowed(_)), "type {ty}"); + } + } +} + +#[test] +fn upright_exif_keeps_every_other_byte() { + for big in [false, true] { + let u32b = |v: u32| if big { v.to_be_bytes() } else { v.to_le_bytes() }; + // IFD0: Make (12-byte ASCII stored after the IFD), Orientation 6, Software (inline ASCII), + // ExifIFD pointer to a sub-IFD with its own entries. + let ifd_end = 8 + 2 + 4 * 12 + 4; + let make_at = ifd_end as u32; + let sub_at = make_at + 12; + let mut tail = b"PhotoCraftCo".to_vec(); + // Sub-IFD: one DateTimeOriginal-ish SHORT entry, then next-IFD 0. + tail.extend_from_slice(&if big { 1u16.to_be_bytes() } else { 1u16.to_le_bytes() }); + tail.extend_from_slice(&if big { 0x9000u16.to_be_bytes() } else { 0x9000u16.to_le_bytes() }); + tail.extend_from_slice(&if big { 7u16.to_be_bytes() } else { 7u16.to_le_bytes() }); + tail.extend_from_slice(&u32b(4)); + tail.extend_from_slice(b"0232"); + tail.extend_from_slice(&u32b(0)); + let e = ifd( + big, + &[(0x010F, 2, 12, u32b(make_at)), (0x0112, 3, 1, short(6, big)), (0x0131, 2, 4, *b"PC1\0"), (0x8769, 4, 1, u32b(sub_at))], + &tail, + ); + let mut prefixed = b"Exif\0\0".to_vec(); + prefixed.extend_from_slice(&e); + let fixed = upright_exif(&prefixed).into_owned(); + assert_eq!(exif_orientation(&fixed), 1); + assert_eq!(fixed.len(), prefixed.len()); + // Only the Orientation value (2 bytes in the second entry) differs. + let value = 6 + 8 + 2 + 12 + 8; + let changed: Vec = (0..fixed.len()).filter(|&i| fixed[i] != prefixed[i]).collect(); + assert_eq!(changed, vec![if big { value + 1 } else { value }], "big={big}"); + assert_eq!(&fixed[..value], &prefixed[..value]); + assert_eq!(&fixed[value + 2..], &prefixed[value + 2..]); + // The offset-based Make string and the inline Software tag are intact. + assert_eq!(&fixed[6 + make_at as usize..][..12], b"PhotoCraftCo"); + assert!(fixed.windows(4).any(|w| w == b"PC1\0")); + } +} + +#[test] +fn limits_apply_to_the_upright_size() { + // Stored 32×16 landscape; Orientation 6 makes it a 16×32 portrait. + let jpeg = encode(&gray(32, 16, vec![90; 32 * 16]), Format::Jpeg, &EncodeOptions::default()).unwrap(); + let tall = jpeg_with_exif(&jpeg, &exif(6, false, false)); + let limits = Limits { max_width: 32, max_height: 16, ..Limits::default() }; + let opts = DecodeOptions { limits, ..Default::default() }; + assert!(matches!(decode_with(&tall, &opts), Err(CodecError::LimitExceeded(_))), "the upright portrait is too tall"); + // As stored it fits, and a 180° turn keeps the size. + let kept = decode_with(&tall, &DecodeOptions { keep_orientation: true, ..opts.clone() }).unwrap(); + assert_eq!(kept.dimensions(), (32, 16)); + let flipped = decode_with(&jpeg_with_exif(&jpeg, &exif(3, false, false)), &opts).unwrap(); + assert_eq!(flipped.dimensions(), (32, 16)); + // The same holds for a TIFF, whose orientation lives in its own IFD0. + let tiff = tiff_with_orientation(&[7; 32 * 16], 32, 16, 8); + assert!(matches!(decode_with(&tiff, &opts), Err(CodecError::LimitExceeded(_)))); + assert_eq!(decode_with(&tiff, &DecodeOptions::default()).unwrap().dimensions(), (16, 32)); +} + +#[test] +fn long_orientation_reads_and_rewrites_in_both_byte_orders() { + for big in [false, true] { + let u32b = |v: u32| if big { v.to_be_bytes() } else { v.to_le_bytes() }; + let long = |v: u32| (0x0112u16, 4u16, 1u32, u32b(v)); + for o in 1..=8u32 { + let e = ifd(big, &[(0x0131, 2, 4, *b"PC1\0"), long(o)], b"tail"); + assert_eq!(exif_orientation(&e), o as u16, "o={o} big={big}"); + let fixed = upright_exif(&e); + assert_eq!(exif_orientation(&fixed), 1); + // Still a LONG, all 4 value bytes set to 1 in the file's byte order; nothing else moved. + assert_eq!(&fixed[..], &ifd(big, &[(0x0131, 2, 4, *b"PC1\0"), long(1)], b"tail")[..], "o={o} big={big}"); + } + // Out of range, including values that only fit in a LONG, reads as 1 and is left alone. + for bad in [0, 9, 0x1_0006, u32::MAX] { + let e = ifd(big, &[long(bad)], &[]); + assert_eq!(exif_orientation(&e), 1, "{bad}"); + } + // A LONG 6 in a JPEG opens upright. + let jpeg = encode(&gray(32, 16, vec![90; 32 * 16]), Format::Jpeg, &EncodeOptions::default()).unwrap(); + let img = decode(&jpeg_with_exif(&jpeg, &ifd(big, &[long(6)], &[]))).unwrap(); + assert_eq!(img.dimensions(), (16, 32)); + let kept = img.meta.exif.as_deref().unwrap(); + assert_eq!(kept.strip_prefix(b"Exif\0\0").unwrap_or(kept), &ifd(big, &[long(1)], &[])[..]); + } +} + +/// `probe_size` reads the size `decode` returns from the header alone, for every format it +/// covers, and a file cut short inside its header is refused rather than given a size. +#[test] +fn probe_size_matches_decode() { + let img = synth(7, 5, ChannelLayout::Rgb, SampleType::U8, 3, 0.0); + let probed = [Format::WebP, Format::Gif, Format::Bmp, Format::Tga, Format::Qoi, Format::Hdr, Format::Ico, Format::OpenExr]; + assert!(probed.iter().all(|f| rw_formats().contains(f))); + for format in rw_formats() { + let bytes = encode(&img, format, &EncodeOptions::default()).unwrap(); + let decoded = decode(&bytes).unwrap().dimensions(); + match probe_size(&bytes) { + Ok(size) => assert_eq!(size, decoded, "{format:?}"), + Err(_) => assert!(!probed.contains(&format), "{format:?} should be probed"), + } + if probed.contains(&format) { + assert!(probe_size(&bytes[..10]).is_err(), "{format:?} cut inside its header"); + } + } + assert!(probe_size(&[]).is_err()); +} diff --git a/crates/image-codecs/tests/it/pcx.rs b/crates/image-codecs/tests/it/pcx.rs new file mode 100644 index 000000000..fbd0e7e57 --- /dev/null +++ b/crates/image-codecs/tests/it/pcx.rs @@ -0,0 +1,138 @@ +use craft_image_codecs::{ChannelLayout, CodecError, Format, SampleType, caps, decode, decode_as, detect, encode, from_extension}; + +fn header(bits: u8, planes: u8, width: u16, height: u16, stride: u16) -> Vec { + let mut b = vec![0u8; 128]; + b[0] = 0x0a; // ZSoft + b[1] = 5; // PC Paintbrush 3.0+ + b[2] = 1; // RLE + b[3] = bits; + b[8..10].copy_from_slice(&(width - 1).to_le_bytes()); + b[10..12].copy_from_slice(&(height - 1).to_le_bytes()); + b[12..14].copy_from_slice(&300u16.to_le_bytes()); + b[14..16].copy_from_slice(&150u16.to_le_bytes()); + b[65] = planes; + b[66..68].copy_from_slice(&stride.to_le_bytes()); + b[68..70].copy_from_slice(&1u16.to_le_bytes()); + b +} + +fn rle_row(out: &mut Vec, row: &[u8]) { + for &value in row { + if value >= 0xc0 { + out.extend_from_slice(&[0xc1, value]); + } else { + out.push(value); + } + } +} + +#[test] +fn pcx_is_readable_but_not_writable() { + assert_eq!(from_extension("archive.PCX"), Some(Format::Pcx)); + assert!(caps(Format::Pcx).read && !caps(Format::Pcx).write); + let img = craft_image_codecs::Image::from_u8(1, 1, ChannelLayout::Rgb, vec![0, 0, 0]).unwrap(); + assert!(matches!(encode(&img, Format::Pcx, &Default::default()), Err(CodecError::Unsupported { .. }))); +} + +#[test] +fn decodes_three_plane_rgb_and_keeps_dpi() { + let mut b = header(8, 3, 2, 1, 2); + for row in [&[255, 0][..], &[0, 255][..], &[0, 0][..]] { + rle_row(&mut b, row); + } + assert_eq!(detect(&b), Some(Format::Pcx)); + let im = decode(&b).unwrap(); + assert_eq!(im.layout(), ChannelLayout::Rgb); + assert_eq!(im.sample_type(), SampleType::U8); + assert_eq!(im.dimensions(), (2, 1)); + assert_eq!(im.data(), &[255, 0, 0, 0, 255, 0]); + assert_eq!(im.meta.dpi, Some((300.0, 150.0))); +} + +#[test] +fn decodes_256_colour_vga_palette() { + let mut b = header(8, 1, 2, 1, 2); + rle_row(&mut b, &[1, 2]); + b.push(0x0c); + let mut palette = vec![0; 256 * 3]; + palette[3..6].copy_from_slice(&[255, 0, 0]); + palette[6..9].copy_from_slice(&[0, 255, 0]); + b.extend(palette); + assert_eq!(decode(&b).unwrap().data(), &[255, 0, 0, 0, 255, 0]); +} + +#[test] +fn decodes_4bit_and_monochrome_palettes() { + let mut four = header(4, 1, 2, 1, 2); + four[16 + 3..16 + 6].copy_from_slice(&[255, 0, 0]); + four[16 + 6..16 + 9].copy_from_slice(&[0, 0, 255]); + rle_row(&mut four, &[0x12, 0]); + assert_eq!(decode(&four).unwrap().data(), &[255, 0, 0, 0, 0, 255]); + + let mut one = header(1, 1, 2, 1, 2); + one[16 + 3..16 + 6].copy_from_slice(&[255, 0, 0]); + rle_row(&mut one, &[0x80, 0]); + assert_eq!(decode(&one).unwrap().data(), &[255, 0, 0, 0, 0, 0]); +} + +#[test] +fn rejects_truncated_and_overshooting_rle_without_panicking() { + let mut b = header(8, 3, 2, 1, 2); + assert!(decode_as(Format::Pcx, &b).is_err(), "header alone is truncated"); + b.extend([0xc7, 0xff]); // a seven-byte run cannot cross a six-byte scanline + assert!(decode_as(Format::Pcx, &b).is_err()); + let mut b = header(8, 3, 2, 1, 2); + b.extend([0xc1]); // missing run value + assert!(decode_as(Format::Pcx, &b).is_err()); + let mut b = header(8, 3, 2, 1, 2); + b[66..68].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!(detect(&b), None, "invalid stride is not a PCX signature"); + + // A solid image with stride of the form 63k + 1 (64, 127) encodes as k full 63-byte runs + // plus a 1-byte literal (2k + 1 bytes per plane row), which is valid and must decode. + for width in [64u16, 127] { + let mut solid = header(8, 3, width, 1, width); + for _plane in 0..3 { + let mut rem = usize::from(width); + while rem >= 63 { + solid.extend_from_slice(&[0xff, 0x00]); // 63-byte run of 0x00 + rem -= 63; + } + if rem == 1 { + solid.push(0x00); // 1-byte literal 0x00 + } + } + let im = decode(&solid).unwrap(); + assert_eq!(im.dimensions(), (u32::from(width), 1)); + assert!(im.data().iter().all(|&v| v == 0)); + } +} + +#[test] +fn refuses_headers_that_exceed_allocation_budget() { + let mut b = header(8, 3, 3000, 3000, 3000); + // The default decoder rejects the allocation before reading any scanline. + let opts = + craft_image_codecs::DecodeOptions { limits: craft_image_codecs::Limits { max_alloc: 1024, ..Default::default() }, ..Default::default() }; + assert!(matches!(craft_image_codecs::decode_as_with(Format::Pcx, &b, &opts), Err(CodecError::LimitExceeded(_)))); + b[2] = 0; + assert_eq!(detect(&b), None, "unsupported uncompressed encoding"); +} + +#[test] +fn decodes_rgb_runs_across_planes() { + let mut b = header(8, 3, 2, 1, 2); + b.extend([0xc4, 0xff, 0xc2, 0]); + let im = decode(&b).unwrap(); + assert_eq!(im.data(), &[255, 255, 0, 255, 255, 0]); + let png = encode(&im, Format::Png, &Default::default()).unwrap(); + assert_eq!(decode(&png).unwrap().data(), im.data()); +} + +#[test] +fn decodes_four_plane_runs_across_planes() { + let mut b = header(1, 4, 2, 1, 2); + b[16 + 15 * 3..16 + 16 * 3].copy_from_slice(&[255, 255, 0]); + b.extend([0xc8, 0xff]); + assert_eq!(decode(&b).unwrap().data(), &[255, 255, 0, 255, 255, 0]); +} diff --git a/crates/image-codecs/tests/it/png_jpeg.rs b/crates/image-codecs/tests/it/png_jpeg.rs new file mode 100644 index 000000000..0c7bc03d4 --- /dev/null +++ b/crates/image-codecs/tests/it/png_jpeg.rs @@ -0,0 +1,647 @@ +//! Format-specific behaviour: Adam7 PNG, JPEG colour models, oracle +//! cross-checks against the `image` crate, and the optional PngSuite corpus. + +use crate::common; +use common::*; +use craft_image_codecs::*; + +// --------------------------------------------------------------------------- +// PNG Adam7 +// --------------------------------------------------------------------------- + +fn interlaced(img: &Image) -> Vec { + encode(img, Format::Png, &EncodeOptions { png_interlaced: true, ..Default::default() }).unwrap() +} + +fn check_adam7(w: u32, h: u32, layout: ChannelLayout, sample: SampleType) { + let img = synth(w, h, layout, sample, (w * 31 + h) as u64, 0.5); + let a = interlaced(&img); + let b = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + assert_eq!(a[28], 1, "IHDR interlace flag"); + assert_eq!(b[28], 0); + let da = decode(&a).unwrap(); + let db = decode(&b).unwrap(); + assert_eq!(da, db, "{w}x{h} {layout:?} {sample:?}: interlaced != progressive"); + assert_eq!(da.data(), img.data()); + // Oracle: the image crate agrees. + let o = image::load_from_memory_with_format(&a, image::ImageFormat::Png).unwrap(); + let ours = da.convert(ChannelLayout::Rgba, SampleType::U16).to_u16_samples().unwrap(); + assert_eq!(o.into_rgba16().into_raw(), ours); +} + +#[test] +fn adam7_small_sizes_all_layouts() { + for (w, h) in [(1, 1), (2, 2), (3, 3), (5, 1), (1, 9), (7, 5), (8, 8), (9, 17), (33, 31)] { + for l in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + check_adam7(w, h, l, SampleType::U8); + } + } +} + +#[test] +fn adam7_16bit() { + for (w, h) in [(1, 1), (4, 3), (13, 11), (64, 40)] { + for l in [ChannelLayout::Gray, ChannelLayout::Rgba] { + check_adam7(w, h, l, SampleType::U16); + } + } +} + +#[test] +fn adam7_with_metadata() { + let mut img = test_image(ChannelLayout::Rgb, SampleType::U8).with_icc(Some(sample_icc(400))); + img.meta.text.push(("Comment".into(), "interlaced".into())); + let back = decode(&interlaced(&img)).unwrap(); + assert_eq!(back.icc, img.icc); + assert_eq!(back.meta.text, img.meta.text); +} + +#[test] +fn adam7_compressions() { + let img = test_image(ChannelLayout::Rgba, SampleType::U8); + for c in [PngCompression::None, PngCompression::Fast, PngCompression::Best] { + let b = encode(&img, Format::Png, &EncodeOptions { png_interlaced: true, png_compression: c, ..Default::default() }).unwrap(); + assert_eq!(decode(&b).unwrap().data(), img.data()); + } +} + +#[test] +fn png_palette_and_low_bit_depth_expand() { + // Build a 2-bit gray PNG and a palette PNG with tRNS using the png crate. + let mut out = Vec::new(); + { + let mut e = png::Encoder::new(&mut out, 4, 1); + e.set_color(png::ColorType::Grayscale); + e.set_depth(png::BitDepth::Two); + let mut w = e.write_header().unwrap(); + w.write_image_data(&[0b00_01_10_11]).unwrap(); + } + let img = decode(&out).unwrap(); + assert_eq!((img.layout(), img.data()), (ChannelLayout::Gray, &[0u8, 85, 170, 255][..])); + + let mut out = Vec::new(); + { + let mut e = png::Encoder::new(&mut out, 2, 1); + e.set_color(png::ColorType::Indexed); + e.set_depth(png::BitDepth::Eight); + e.set_palette(vec![255, 0, 0, 0, 0, 255]); + e.set_trns(vec![128]); + let mut w = e.write_header().unwrap(); + w.write_image_data(&[0, 1]).unwrap(); + } + let img = decode(&out).unwrap(); + assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgba, &[255u8, 0, 0, 128, 0, 0, 255, 255][..])); +} + +// --------------------------------------------------------------------------- +// JPEG colour models +// --------------------------------------------------------------------------- + +#[test] +fn jpeg_grayscale_decodes_as_gray() { + let img = synth(40, 30, ChannelLayout::Gray, SampleType::U8, 2, 0.02); + let back = decode(&encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Gray); + assert!(psnr(&img, &back) > 35.0); +} + +#[test] +fn jpeg_cmyk_roundtrip_with_adobe_marker() { + let img = synth(40, 30, ChannelLayout::Cmyk, SampleType::U8, 3, 0.02); + let bytes = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_quality: 95, ..Default::default() }).unwrap(); + assert!(bytes.windows(5).any(|w| w == b"Adobe"), "APP14 Adobe marker"); + let back = decode(&bytes).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Cmyk); + assert!(psnr(&img, &back) > 35.0, "{}", psnr(&img, &back)); +} + +#[test] +fn jpeg_ycck_decodes_to_cmyk() { + let img = synth(40, 32, ChannelLayout::Cmyk, SampleType::U8, 4, 0.02); + let mut bytes = Vec::new(); + let mut enc = jpeg_encoder::Encoder::new(&mut bytes, 95); + enc.set_sampling_factor(jpeg_encoder::SamplingFactor::R_4_4_4); + enc.encode(img.data(), 40, 32, jpeg_encoder::ColorType::CmykAsYcck).unwrap(); + let back = decode(&bytes).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Cmyk); + assert!(psnr(&img, &back) > 32.0, "{}", psnr(&img, &back)); +} + +#[test] +fn jpeg_cmyk_flat_colour_exact_ish() { + let img = Image::from_u8(16, 16, ChannelLayout::Cmyk, [10u8, 100, 200, 50].repeat(256)).unwrap(); + let back = decode(&encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap()).unwrap(); + for px in back.data().chunks(4) { + for (a, b) in px.iter().zip([10u8, 100, 200, 50]) { + assert!((*a as i32 - b as i32).abs() <= 2); + } + } +} + +#[test] +fn jpeg_rgb_matches_oracle_decoder() { + let img = test_image(ChannelLayout::Rgb, SampleType::U8); + let bytes = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + let ours = decode(&bytes).unwrap(); + let theirs = image::load_from_memory_with_format(&bytes, image::ImageFormat::Jpeg).unwrap().into_rgb8(); + let max = ours.data().iter().zip(theirs.as_raw()).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap(); + assert!(max <= 1, "max diff {max}"); +} + +#[test] +fn jpeg_progressive_from_other_encoder_decodes() { + let img = test_image(ChannelLayout::Rgb, SampleType::U8); + let mut bytes = Vec::new(); + let mut enc = jpeg_encoder::Encoder::new(&mut bytes, 90); + enc.set_progressive(true); + enc.encode(img.data(), img.width() as u16, img.height() as u16, jpeg_encoder::ColorType::Rgb).unwrap(); + let back = decode(&bytes).unwrap(); + assert!(psnr(&img, &back) > 25.0); +} + +// --------------------------------------------------------------------------- +// Oracle: files we write are readable by the image crate +// --------------------------------------------------------------------------- + +#[test] +fn png_output_matches_oracle_all_layouts() { + for l in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + for s in [SampleType::U8, SampleType::U16] { + let img = test_image(l, s); + let b = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let o = image::load_from_memory(&b).unwrap().into_rgba16().into_raw(); + assert_eq!(o, img.to_rgba16(), "{l:?} {s:?}"); + } + } +} + +#[test] +fn tiff_output_matches_oracle() { + for l in [ChannelLayout::Gray, ChannelLayout::Rgb, ChannelLayout::Rgba] { + for s in [SampleType::U8, SampleType::U16] { + let img = test_image(l, s); + let b = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + let o = image::load_from_memory(&b).unwrap().into_rgba16().into_raw(); + assert_eq!(o, img.to_rgba16(), "{l:?} {s:?}"); + } + } +} + +#[test] +fn tiff_f32_output_matches_oracle() { + let img = test_image(ChannelLayout::Rgba, SampleType::F32); + let b = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: TiffCompression::None, ..Default::default() }).unwrap(); + let o = image::load_from_memory(&b).unwrap().into_rgba32f().into_raw(); + assert_eq!(o, img.to_f32_samples().unwrap()); +} + +// --------------------------------------------------------------------------- +// PNG decoded in steps (PngDecodeRows) +// --------------------------------------------------------------------------- + +/// Decodes `bytes` with [`PngDecodeRows`], going on for `per_call` rows per call (0: unlimited); +/// returns the image and how many calls it took. +fn decode_in_steps(bytes: &[u8], per_call: usize) -> Result<(Image, usize), CodecError> { + let mut rows = PngDecodeRows::new(bytes, &Limits::default())?; + let mut calls = 0; + loop { + calls += 1; + let mut left = per_call; + let mut more = || { + left = left.wrapping_sub(1); + per_call == 0 || left > 0 + }; + if let Some(img) = rows.step(&mut more)? { + return Ok((img, calls)); + } + assert!(calls < 100_000, "the decoder never finished"); + } +} + +fn assert_steps_equal_decode(bytes: &[u8], what: &str) { + let whole = decode(bytes); + for per_call in [1, 7, 0] { + // `decode` also takes the resolution from EXIF or XMP and turns the image upright; + // `finish_png_decode` does that for a stepped decode, as the stepped import does. + let stepped = decode_in_steps(bytes, per_call).and_then(|(img, calls)| Ok((finish_png_decode(img, &DecodeOptions::default())?, calls))); + match (&whole, stepped) { + (Ok(a), Ok((b, _))) => assert_eq!(a, &b, "{what}, {per_call} rows per call"), + (Err(_), Err(_)) => {} + (a, b) => panic!("{what}, {per_call} rows per call: decode {:?}, steps {:?}", a.as_ref().map(|_| ()), b.map(|_| ())), + } + } +} + +#[test] +fn png_rows_equal_decode_in_every_layout_and_depth() { + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + for sample in [SampleType::U8, SampleType::U16] { + let mut img = synth(41, 29, layout, sample, 77, 0.3).with_icc(Some(sample_icc(400))); + img.meta.text.push(("Comment".into(), "stepped".into())); + img.meta.dpi = Some((300.0, 300.0)); + for interlaced in [false, true] { + let bytes = encode(&img, Format::Png, &EncodeOptions { png_interlaced: interlaced, ..Default::default() }).unwrap(); + assert_steps_equal_decode(&bytes, &format!("{layout:?} {sample:?} interlaced={interlaced}")); + } + } + } +} + +#[test] +fn png_rows_equal_decode_for_palette_and_transparency() { + let pal = [[255, 0, 0], [0, 255, 0], [0, 0, 255], [9, 9, 9]]; + let idx: Vec = (0..50 * 20).map(|i| (i % 4) as u8).collect(); + let bytes = encode_png_indexed(50, 20, &idx, &pal, Some(3), None, None).unwrap(); + assert_steps_equal_decode(&bytes, "palette with a transparent entry"); +} + +#[test] +fn png_rows_take_one_call_per_row_and_report_progress() { + let bytes = encode(&synth(16, 40, ChannelLayout::Rgb, SampleType::U8, 5, 0.2), Format::Png, &EncodeOptions::default()).unwrap(); + let mut rows = PngDecodeRows::new(bytes.as_slice(), &Limits::default()).unwrap(); + assert_eq!(rows.size(), (16, 40)); + assert_eq!(rows.progress(), 0.0); + let mut seen = Vec::new(); + let img = loop { + if let Some(img) = rows.step(&mut || false).unwrap() { + break img; + } + seen.push(rows.progress()); + }; + assert_eq!(seen.len(), 39, "one call per row but the last, which also finishes the file"); + assert!(seen.windows(2).all(|w| w[0] < w[1])); + assert_eq!(rows.progress(), 1.0); + assert_eq!(img, decode(&bytes).unwrap()); + // A decoder that has finished says so instead of decoding again. + assert!(rows.step(&mut || true).is_err()); +} + +#[test] +fn png_rows_share_their_bytes_without_copying_them() { + let bytes = + std::sync::Arc::new(encode(&synth(20, 20, ChannelLayout::Rgba, SampleType::U8, 3, 0.2), Format::Png, &EncodeOptions::default()).unwrap()); + let mut rows = PngDecodeRows::new(SharedBytes(bytes.clone()), &Limits::default()).unwrap(); + let img = rows.step(&mut || true).unwrap().unwrap(); + assert_eq!(img, decode(&bytes).unwrap()); + drop(rows); + assert_eq!(std::sync::Arc::strong_count(&bytes), 1, "the decoder let go of the bytes"); +} + +/// A CRC-valid PNG chunk. +fn png_chunk(kind: &[u8; 4], data: &[u8]) -> Vec { + let mut crc = flate2::Crc::new(); + crc.update(kind); + crc.update(data); + let mut out = u32::try_from(data.len()).unwrap().to_be_bytes().to_vec(); + out.extend_from_slice(kind); + out.extend_from_slice(data); + out.extend_from_slice(&crc.sum().to_be_bytes()); + out +} + +fn zlib(text: &[u8]) -> Vec { + use std::io::Write; + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(text).unwrap(); + encoder.finish().unwrap() +} + +/// A `zTXt` chunk (`deflated`) or a compressed `iTXt` one (`!deflated`) with `text` as its value. +fn png_text_chunk(kind: &[u8; 4], keyword: &[u8], text: &[u8]) -> Vec { + let mut data = keyword.to_vec(); + data.push(0); + if kind == b"zTXt" { + data.push(0); + } else { + data.extend_from_slice(&[1, 0, 0, 0]); + } + data.extend_from_slice(&zlib(text)); + png_chunk(kind, &data) +} + +/// `png` with `chunks` between the image data and `IEND` (the text-after-IDAT case). +fn with_chunks_before_iend(png: &[u8], chunks: &[Vec]) -> Vec { + let mut at = 8; + loop { + let len = u32::from_be_bytes(png[at..at + 4].try_into().unwrap()) as usize; + if &png[at + 4..at + 8] == b"IEND" { + let mut out = png[..at].to_vec(); + for c in chunks { + out.extend_from_slice(c); + } + out.extend_from_slice(&png[at..]); + return out; + } + at += len + 12; + } +} + +#[test] +fn png_rows_read_compressed_text_in_bounded_batches() { + let img = synth(19, 23, ChannelLayout::Rgba, SampleType::U8, 9, 0.4); + let base = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let text = "A".repeat(150_000); + for kind in [b"zTXt", b"iTXt"] { + let bytes = with_chunks_before_iend(&base, &[png_text_chunk(kind, b"Comment", text.as_bytes())]); + let mut rows = PngDecodeRows::new(bytes.as_slice(), &Limits::default()).unwrap(); + let mut calls = 0; + // An empty budget: every step reads one bounded batch and stops. + let stepped = loop { + calls += 1; + if let Some(img) = rows.step(&mut || false).unwrap() { + break img; + } + assert!(calls < 100_000, "the decoder never finished"); + }; + assert_eq!(stepped, decode(&bytes).unwrap(), "{kind:?}"); + assert_eq!(stepped.meta.text, vec![("Comment".into(), text.clone())], "{kind:?}: the whole value is kept"); + assert!(calls > 20, "{kind:?}: {calls} bounded calls, not one blocking pass"); + assert_steps_equal_decode(&bytes, &format!("compressed {kind:?} text")); + } +} + +/// An Adam7 image whose compressed text needs many metadata yields: the pixels are decoded +/// once, before the first yield, and later steps only read text. +#[test] +fn png_rows_finish_adam7_text_in_bounded_batches() { + let img = synth(19, 23, ChannelLayout::Rgba, SampleType::U8, 9, 0.4); + let base = encode(&img, Format::Png, &EncodeOptions { png_interlaced: true, ..Default::default() }).unwrap(); + let text = "A".repeat(150_000); + for kind in [b"zTXt", b"iTXt"] { + let bytes = with_chunks_before_iend(&base, &[png_text_chunk(kind, b"Comment", text.as_bytes())]); + let mut rows = PngDecodeRows::new(bytes.as_slice(), &Limits::default()).unwrap(); + let mut calls = 0; + let stepped = loop { + calls += 1; + if let Some(img) = rows.step(&mut || false).unwrap() { + break img; + } + assert!(calls < 100_000, "the decoder never finished"); + }; + assert_eq!(stepped, decode(&bytes).unwrap(), "{kind:?}"); + assert_eq!(stepped.meta.text, vec![("Comment".into(), text.clone())], "{kind:?}: the whole value is kept"); + assert!(calls > 20, "{kind:?}: {calls} bounded calls after the one Adam7 pass"); + } +} + +/// An uncompressed `iTXt` chunk (`value` is its text, so its CRC covers the whole value). +fn png_itxt_uncompressed(keyword: &[u8], value: &[u8]) -> Vec { + let mut data = keyword.to_vec(); + data.push(0); + data.extend_from_slice(&[0, 0, 0, 0]); + data.extend_from_slice(value); + png_chunk(b"iTXt", &data) +} + +/// The CRC of a large text chunk takes bounded batches too, before any of its value is read. +#[test] +fn png_rows_read_a_large_text_crc_in_bounded_batches() { + let img = synth(5, 5, ChannelLayout::Rgb, SampleType::U8, 3, 0.2); + let base = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let text: String = (0..300_000u32).map(|i| char::from(b'a' + (i % 26) as u8)).collect(); + let bytes = with_chunks_before_iend(&base, &[png_itxt_uncompressed(b"Comment", text.as_bytes())]); + let mut rows = PngDecodeRows::new(bytes.as_slice(), &Limits::default()).unwrap(); + let mut calls = 0; + let stepped = loop { + calls += 1; + if let Some(img) = rows.step(&mut || false).unwrap() { + break img; + } + assert!(calls < 100_000, "the decoder never finished"); + }; + assert_eq!(stepped.meta.text, vec![("Comment".into(), text)]); + assert!(calls > 60, "{calls} calls: the CRC and the value each take bounded batches"); +} + +/// A text chunk whose CRC does not match is still dropped after its bounded CRC batches. +#[test] +fn png_rows_drop_a_corrupt_text_crc() { + let img = synth(5, 5, ChannelLayout::Rgb, SampleType::U8, 3, 0.2); + let base = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let mut chunk = png_itxt_uncompressed(b"Comment", b"not accepted"); + let at = chunk.len() - 5; + chunk[at] ^= 0xFF; + let bytes = with_chunks_before_iend(&base, &[chunk]); + let stepped = decode_in_steps(&bytes, 0).unwrap().0; + assert!(stepped.meta.text.is_empty(), "a broken CRC drops the text"); + assert_eq!(stepped, decode(&bytes).unwrap(), "the one-pass decode drops it too"); +} + +/// An uncompressed iTXt value that is not UTF-8 is dropped without charging its bytes against +/// the budget, even when it is larger than all of it: the previous reader checked UTF-8 first. +#[test] +fn png_rows_drop_invalid_uncompressed_text_before_charging_the_budget() { + let img = synth(4, 4, ChannelLayout::Rgb, SampleType::U8, 3, 0.2); + let base = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let mut value = vec![b'A'; 12_000]; + value[11_999] = 0xFF; + let bytes = with_chunks_before_iend(&base, &[png_itxt_uncompressed(b"Comment", &value)]); + let opts = DecodeOptions { limits: Limits { max_alloc: 200, ..Default::default() }, ..Default::default() }; + let mut rows = PngDecodeRows::new(bytes.as_slice(), &opts.limits).unwrap(); + let stepped = rows.step(&mut || true).unwrap().unwrap(); + let stepped = finish_png_decode(stepped, &opts).unwrap(); + assert!(stepped.meta.text.is_empty(), "invalid UTF-8 is skipped, not a budget failure"); + let whole = decode_with(&bytes, &opts).unwrap(); + assert!(whole.meta.text.is_empty(), "the one-pass decode skips it too"); +} + +#[test] +fn png_rows_fail_like_decode_on_damaged_files() { + let img = synth(64, 64, ChannelLayout::Rgb, SampleType::U8, 11, 1.0); + for interlaced in [false, true] { + let bytes = encode(&img, Format::Png, &EncodeOptions { png_interlaced: interlaced, ..Default::default() }).unwrap(); + // Cuts inside the signature, the IHDR chunk or the next chunk header leave no complete image. + for cut in [8, 20, 40] { + let truncated = bytes.get(..cut).unwrap(); + assert!(decode(truncated).is_err(), "cut at {cut} interlaced={interlaced}"); + assert!(decode_in_steps(truncated, 1).is_err(), "cut at {cut} interlaced={interlaced}"); + } + for cut in [bytes.len() / 3, bytes.len() / 2, bytes.len() - 20] { + assert_steps_equal_decode(bytes.get(..cut).unwrap(), &format!("cut at {cut} interlaced={interlaced}")); + } + // A flipped byte in the middle of the compressed data. + let mut bad = bytes.clone(); + let mid = bad.len() / 2; + bad[mid] ^= 0xFF; + assert_steps_equal_decode(&bad, "a damaged byte"); + } + assert!(decode_in_steps(&[], 1).is_err()); + assert!(decode_in_steps(b"not a png at all, just some text bytes", 1).is_err()); +} + +// --------------------------------------------------------------------------- +// PngSuite corpus (repo-root corpus/pngsuite/*.png; feature `corpus`, fetched by +// `cargo xtask corpus --all`; missing = failure) +// --------------------------------------------------------------------------- + +#[cfg(feature = "corpus")] +#[test] +fn pngsuite_corpus_matches_oracle() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pngsuite"); + let Ok(entries) = std::fs::read_dir(&dir) else { + panic!("{} is missing: run `cargo xtask corpus --all`", dir.display()); + }; + let mut checked = 0; + for e in entries.flatten() { + let p = e.path(); + if p.extension().and_then(|s| s.to_str()) != Some("png") { + continue; + } + let bytes = std::fs::read(&p).unwrap(); + let oracle = image::load_from_memory_with_format(&bytes, image::ImageFormat::Png); + let ours = decode(&bytes); + match (ours, oracle) { + (Ok(a), Ok(b)) => { + assert_eq!(a.to_rgba16(), b.into_rgba16().into_raw(), "{}", p.display()); + checked += 1; + } + (Err(e), Ok(_)) => panic!("{}: we failed but oracle decoded: {e}", p.display()), + _ => {} + } + } + eprintln!("pngsuite: {checked} files compared"); + assert!(checked > 100, "{}: only {checked} PNGs compared: run `cargo xtask corpus --all`", dir.display()); +} + +/// The preflight charges the decoded layout's bytes per pixel: a 1000×1000 Gray8 PNG (1 MB) fits a +/// 1.5 MB budget, which the old worst-case RGBA16 preflight (8 MB) refused. +#[test] +fn png_preflight_uses_the_output_layout() { + let img = synth(1000, 1000, ChannelLayout::Gray, SampleType::U8, 7, 0.5); + let bytes = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let opts = DecodeOptions { limits: Limits { max_alloc: 1_500_000, ..Limits::none() }, ..Default::default() }; + let got = decode_with(&bytes, &opts).expect("a Gray8 image is charged 1 byte per pixel"); + assert_eq!(got.data(), img.data()); + let tight = DecodeOptions { limits: Limits { max_alloc: 900_000, ..Limits::none() }, ..Default::default() }; + assert!(decode_with(&bytes, &tight).is_err(), "the actual pixel budget still applies"); +} + +#[cfg(feature = "corpus")] +#[test] +fn png_rows_equal_decode_on_the_pngsuite_corpus() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pngsuite"); + let Ok(entries) = std::fs::read_dir(&dir) else { + panic!("{} is missing: run `cargo xtask corpus --all`", dir.display()); + }; + let mut checked = 0; + for e in entries.flatten() { + let p = e.path(); + if p.extension().and_then(|s| s.to_str()) != Some("png") { + continue; + } + let bytes = std::fs::read(&p).unwrap(); + assert_steps_equal_decode(&bytes, &p.display().to_string()); + checked += 1; + } + assert!(checked > 100, "{}: only {checked} PNGs compared: run `cargo xtask corpus --all`", dir.display()); +} + +/// Row-by-row decoding gives `decode`'s pixels and metadata for every PngSuite file it takes (all +/// but the interlaced ones), and fails where `decode` fails. +#[cfg(feature = "corpus")] +#[test] +fn pngsuite_corpus_rows_match_the_whole_decode() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pngsuite"); + let Ok(entries) = std::fs::read_dir(&dir) else { + panic!("{} is missing: run `cargo xtask corpus --all`", dir.display()); + }; + let opts = DecodeOptions { keep_orientation: true, ..Default::default() }; + let (mut streamed, mut refused, mut skipped) = (0, 0, 0); + for e in entries.flatten() { + let p = e.path(); + if p.extension().and_then(|s| s.to_str()) != Some("png") { + continue; + } + let bytes = std::fs::read(&p).unwrap(); + // The importer only streams files `detect` takes for PNG. + if detect(&bytes) != Some(Format::Png) { + skipped += 1; + continue; + } + let whole = decode_with(&bytes, &opts); + let (mut data, mut header) = (Vec::new(), None); + let rows = decode_png_rows::( + &bytes, + &opts, + 256, + &mut |h| { + header = Some(*h); + Ok(true) + }, + &mut |_, r| { + data.extend_from_slice(r); + Ok(()) + }, + ); + match (whole, rows) { + (Ok(img), Ok(Some(x))) => { + let h = header.unwrap(); + assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (h.width, h.height, h.layout, h.sample), "{}", p.display()); + assert!(img.data() == data, "{}: pixels", p.display()); + assert_eq!( + (&img.icc, &img.meta.exif, &img.meta.text, &img.meta.xmp, &img.warnings), + (&x.icc, &x.meta.exif, &x.meta.text, &x.meta.xmp, &x.warnings), + "{}", + p.display() + ); + assert_eq!(img.meta.dpi, x.meta.dpi, "{}", p.display()); + streamed += 1; + } + (Ok(_), Ok(None)) => refused += 1, + (Err(a), Err(b)) => assert_eq!(a.to_string(), b.to_string(), "{}", p.display()), + (a, b) => panic!("{}: whole {:?}, rows {:?}", p.display(), a.map(|_| ()), b.map(|r| r.map(|_| ()))), + } + } + eprintln!("pngsuite: {streamed} streamed, {refused} refused (interlaced), {skipped} skipped (not detected as PNG)"); + assert!(streamed > 100, "only {streamed} PNGs streamed: run `cargo xtask corpus --all`"); +} + +/// Streaming a PNG's rows a group at a time gives the pixels and metadata `decode` gives, holds +/// only a group at a time, and stops between rows when asked. +#[test] +fn step_stream_gives_the_pixels_and_extras_of_decode() { + for (layout, sample) in [(ChannelLayout::Gray, SampleType::U8), (ChannelLayout::Rgb, SampleType::U16), (ChannelLayout::Rgba, SampleType::U8)] { + let (w, h) = (23u32, 19u32); + let len = w as usize * h as usize * layout.channels() * sample.bytes(); + let mut img = Image::from_raw(w, h, layout, sample, (0..len).map(|i| (i.wrapping_mul(2_654_435_761) >> 11) as u8).collect()).unwrap(); + img.meta.dpi = Some((144.0, 144.0)); + img.meta.text.push(("Comment".into(), "streamed".into())); + img.icc = Some(vec![7; 40]); + let bytes = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + let want = decode(&bytes).unwrap(); + for group in [0u32, 1, 4, 19, 64] { + let mut dec = PngDecodeRows::new_streaming(bytes.as_slice(), &Limits::default()).unwrap(); + assert!(dec.streamable()); + assert_eq!(dec.header(), PngHeader { width: w, height: h, layout, sample }); + let (mut data, mut next, mut calls) = (Vec::new(), 0u32, 0); + let extras = loop { + calls += 1; + let mut sink = |y0: u32, rows: &[u8]| -> Result<(), CodecError> { + assert_eq!(y0, next, "{layout:?} group {group}"); + assert!(rows.len() <= group.max(1) as usize * dec_line(w, layout, sample), "a group at most"); + next += (rows.len() / dec_line(w, layout, sample)) as u32; + data.extend_from_slice(rows); + Ok(()) + }; + if let Some(extras) = dec.step_stream(group, &mut || false, &mut sink).unwrap() { + break extras; + } + assert!(calls < 10_000); + }; + assert!(calls > 1, "{layout:?}: stopped between rows"); + assert_eq!(data, want.data(), "{layout:?} group {group}: pixels"); + assert_eq!((extras.icc.as_deref(), extras.meta.dpi, &extras.meta.text), (want.icc.as_deref(), want.meta.dpi, &want.meta.text)); + assert_eq!(extras.orientation, 1); + assert!(dec.step_stream::(group, &mut || true, &mut |_, _| Ok(())).is_err(), "spent"); + } + } + // An interlaced file doesn't stream. + let img = Image::from_raw(9, 9, ChannelLayout::Gray, SampleType::U8, vec![1; 81]).unwrap(); + let bytes = encode(&img, Format::Png, &EncodeOptions { png_interlaced: true, ..Default::default() }).unwrap(); + assert!(!PngDecodeRows::new_streaming(bytes.as_slice(), &Limits::default()).unwrap().streamable()); +} + +/// Bytes per row of `w` pixels. +fn dec_line(w: u32, layout: ChannelLayout, sample: SampleType) -> usize { + w as usize * layout.channels() * sample.bytes() +} diff --git a/crates/image-codecs/tests/it/pnm_headers.rs b/crates/image-codecs/tests/it/pnm_headers.rs new file mode 100644 index 000000000..363e941ad --- /dev/null +++ b/crates/image-codecs/tests/it/pnm_headers.rs @@ -0,0 +1,82 @@ +use craft_image_codecs::{ChannelLayout, CodecError, Format, SampleType, decode, decode_as}; + +const PAM_GRAY: &[u8] = b"P7\nWIDTH 1\nHEIGHT 1\nDEPTH 1\nMAXVAL 255\nENDHDR"; +const ENDINGS: &[&[u8]] = &[b"\n", b" \n", b"\t \n", b"\r\n", b" \t\r\n"]; + +#[test] +fn pam_consumes_the_complete_endhdr_line() { + for ending in ENDINGS { + let mut bytes = PAM_GRAY.to_vec(); + bytes.extend_from_slice(ending); + bytes.push(0x80); + let img = decode(&bytes).unwrap(); + assert_eq!(img.dimensions(), (1, 1)); + assert_eq!(img.layout(), ChannelLayout::Gray); + assert_eq!(img.data(), &[0x80], "ENDHDR ending {ending:?}"); + } +} + +#[test] +fn pam_preserves_every_possible_first_raster_byte() { + // In particular, LF, CR, space, tab and '#' must not be skipped as header syntax. + for ending in ENDINGS { + for first in 0..=255u8 { + let mut bytes = PAM_GRAY.to_vec(); + bytes.extend_from_slice(ending); + bytes.push(first); + assert_eq!(decode(&bytes).unwrap().data(), &[first], "ENDHDR ending {ending:?}, raster byte {first}"); + } + } +} + +#[test] +fn pam_crlf_header_preserves_u16_samples_and_alpha() { + for ending in ENDINGS { + let mut bytes = b"P7\r\nWIDTH 1\r\nHEIGHT 1\r\nDEPTH 2\r\nMAXVAL 65535\r\nTUPLTYPE GRAYSCALE_ALPHA\r\nENDHDR".to_vec(); + bytes.extend_from_slice(ending); + bytes.extend_from_slice(b" \n#\r"); + let img = decode(&bytes).unwrap(); + assert_eq!(img.layout(), ChannelLayout::GrayA); + assert_eq!(img.sample_type(), SampleType::U16); + assert_eq!(img.to_u16_samples().unwrap(), vec![0x200a, 0x230d], "ENDHDR ending {ending:?}"); + } +} + +#[test] +fn pam_rejects_unterminated_or_invalid_endhdr_lines() { + for suffix in [b"".as_slice(), b" ", b"\r", b" \t\r", b" \x80", b" \x80\n\x7f", b"#comment\n\x7f"] { + let mut bytes = PAM_GRAY.to_vec(); + bytes.extend_from_slice(suffix); + assert!(matches!(decode_as(Format::Pnm, &bytes), Err(CodecError::Malformed { .. })), "ENDHDR suffix {suffix:?}"); + } +} + +#[test] +fn pam_truncation_never_uses_header_whitespace_as_pixels() { + for ending in ENDINGS { + let mut bytes = PAM_GRAY.to_vec(); + bytes.extend_from_slice(ending); + bytes.push(0x80); + for cut in 0..bytes.len() { + assert!(decode_as(Format::Pnm, &bytes[..cut]).is_err(), "ENDHDR ending {ending:?}, truncated at {cut}"); + } + } +} + +#[test] +fn binary_pnm_keeps_its_single_whitespace_separator() { + for separator in *b" \t\r\n" { + for first in *b" \t\r\n#" { + for (header, channels) in [(b"P5\n1 1\n255".as_slice(), 1), (b"P6\n1 1\n255", 3)] { + let mut bytes = header.to_vec(); + bytes.push(separator); + bytes.extend(std::iter::repeat_n(first, channels)); + assert_eq!(decode(&bytes).unwrap().data(), vec![first; channels]); + } + let mut bytes = b"P4\n8 1".to_vec(); + bytes.extend_from_slice(&[separator, first]); + let expected: Vec = (0..8).map(|bit| if first & (0x80 >> bit) == 0 { 255 } else { 0 }).collect(); + assert_eq!(decode(&bytes).unwrap().data(), expected); + } + } +} diff --git a/crates/image-codecs/tests/it/resolution.rs b/crates/image-codecs/tests/it/resolution.rs new file mode 100644 index 000000000..14c682f8d --- /dev/null +++ b/crates/image-codecs/tests/it/resolution.rs @@ -0,0 +1,554 @@ +//! Resolution recorded in metadata (#1691): a camera JPEG keeps its resolution in EXIF only, and +//! Photoshop opens it at that resolution; Photoshop's own APP13 ResolutionInfo and the XMP +//! `tiff:` properties outrank EXIF, and the JFIF density comes last. Exports write the image's +//! resolution into JFIF, EXIF and XMP alike. Every file here is built by hand. + +use std::borrow::Cow; + +use craft_image_codecs::resolution::{exif_with_resolution, xmp_with_resolution}; +use craft_image_codecs::*; +use proptest::prelude::{any, proptest}; + +const X: u16 = 282; +const Y: u16 = 283; +const UNIT: u16 = 296; +const SHORT: u16 = 3; +const LONG: u16 = 4; +const RATIONAL: u16 = 5; +const ASCII: u16 = 2; + +/// One IFD0 entry: tag, type, count and the value bytes (in the block's byte order). +struct E(u16, u16, u32, Vec); + +struct Exif { + big: bool, +} + +impl Exif { + fn u16(&self, v: u16) -> [u8; 2] { + if self.big { v.to_be_bytes() } else { v.to_le_bytes() } + } + fn u32(&self, v: u32) -> [u8; 4] { + if self.big { v.to_be_bytes() } else { v.to_le_bytes() } + } + fn rational(&self, n: u32, d: u32) -> Vec { + [self.u32(n), self.u32(d)].concat() + } + fn short(&self, v: u16) -> Vec { + self.u16(v).to_vec() + } + fn x(&self, n: u32, d: u32) -> E { + E(X, RATIONAL, 1, self.rational(n, d)) + } + fn y(&self, n: u32, d: u32) -> E { + E(Y, RATIONAL, 1, self.rational(n, d)) + } + fn unit(&self, u: u16) -> E { + E(UNIT, SHORT, 1, self.short(u)) + } + fn make(&self) -> E { + E(0x010F, ASCII, 6, b"NIKON\0".to_vec()) + } + fn orientation(&self, o: u16) -> E { + E(0x0112, SHORT, 1, self.short(o)) + } + + /// A TIFF-structured EXIF block: header, IFD0 at 8, then the out-of-line values. + fn build(&self, entries: &[E]) -> Vec { + let mut v = if self.big { b"MM\0*".to_vec() } else { b"II*\0".to_vec() }; + v.extend_from_slice(&self.u32(8)); + v.extend_from_slice(&self.u16(entries.len() as u16)); + let mut data_at = 8 + 2 + entries.len() * 12 + 4; + let mut data = Vec::new(); + for E(tag, ty, count, value) in entries { + v.extend_from_slice(&self.u16(*tag)); + v.extend_from_slice(&self.u16(*ty)); + v.extend_from_slice(&self.u32(*count)); + if value.len() <= 4 { + let mut inline = value.clone(); + inline.resize(4, 0); + v.extend_from_slice(&inline); + } else { + v.extend_from_slice(&self.u32(data_at as u32)); + data.extend_from_slice(value); + data_at += value.len(); + } + } + v.extend_from_slice(&self.u32(0)); + v.extend_from_slice(&data); + v + } +} + +const LE: Exif = Exif { big: false }; +const BE: Exif = Exif { big: true }; + +/// A D3200-style block: Make, XResolution 300/1, YResolution 300/1, ResolutionUnit 2. +fn camera_exif(e: &Exif) -> Vec { + e.build(&[e.make(), e.x(300, 1), e.y(300, 1), e.unit(2)]) +} + +fn segment(marker: u8, payload: &[u8]) -> Vec { + let mut s = vec![0xFF, marker]; + s.extend_from_slice(&((payload.len() + 2) as u16).to_be_bytes()); + s.extend_from_slice(payload); + s +} + +fn jfif(unit: u8, x: u16, y: u16) -> Vec { + let mut p = b"JFIF\0\x01\x02".to_vec(); + p.push(unit); + p.extend_from_slice(&x.to_be_bytes()); + p.extend_from_slice(&y.to_be_bytes()); + p.extend_from_slice(&[0, 0]); + segment(0xE0, &p) +} + +fn exif_app1(exif: &[u8]) -> Vec { + segment(0xE1, &[b"Exif\0\0".as_slice(), exif].concat()) +} + +fn xmp_app1(xmp: &str) -> Vec { + segment(0xE1, &[b"http://ns.adobe.com/xap/1.0/\0".as_slice(), xmp.as_bytes()].concat()) +} + +/// An 8BIM resource block holding ResolutionInfo at `ppi` (display unit inch), after an +/// unrelated resource with an odd-length name and an odd-sized body. +fn irb(ppi: f64) -> Vec { + let mut b = b"8BIM".to_vec(); + b.extend_from_slice(&0x0404u16.to_be_bytes()); + b.extend_from_slice(&[3, b'a', b'b', b'c']); + b.extend_from_slice(&3u32.to_be_bytes()); + b.extend_from_slice(&[1, 2, 3, 0]); + b.extend_from_slice(b"8BIM"); + b.extend_from_slice(&0x03EDu16.to_be_bytes()); + b.extend_from_slice(&[0, 0]); + b.extend_from_slice(&16u32.to_be_bytes()); + let fixed = ((ppi * 65536.0).round() as u32).to_be_bytes(); + for _ in 0..2 { + b.extend_from_slice(&fixed); + b.extend_from_slice(&1u16.to_be_bytes()); + b.extend_from_slice(&2u16.to_be_bytes()); + } + b +} + +fn app13(resources: &[u8]) -> Vec { + segment(0xED, &[b"Photoshop 3.0\0".as_slice(), resources].concat()) +} + +/// A 16×8 baseline JPEG whose APPn segments are exactly `segments` (the encoder's own JFIF +/// header is taken out). +fn jpeg_with(segments: &[Vec]) -> Vec { + let img = Image::from_raw(16, 8, ChannelLayout::Gray, SampleType::U8, vec![128; 128]).unwrap(); + let base = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + let mut i = 2; + while (0xE0..=0xEF).contains(&base[i + 1]) { + i += 2 + u16::from_be_bytes([base[i + 2], base[i + 3]]) as usize; + } + let mut out = base[..2].to_vec(); + for s in segments { + out.extend_from_slice(s); + } + out.extend_from_slice(&base[i..]); + out +} + +fn dpi_of(bytes: &[u8]) -> Option<(f32, f32)> { + decode(bytes).unwrap().meta.dpi +} + +fn near(got: Option<(f32, f32)>, want: f32) { + let (x, y) = got.unwrap_or_else(|| panic!("no resolution, wanted {want}")); + assert!((x - want).abs() < 0.01 && (y - want).abs() < 0.01, "got {x}×{y}, wanted {want}"); +} + +/// The APPn payloads of a JPEG: (marker, payload). +fn app_segments(b: &[u8]) -> Vec<(u8, Vec)> { + let mut out = Vec::new(); + let mut i = 2; + while i + 4 <= b.len() && b[i] == 0xFF && b[i + 1] != 0xDA { + let len = u16::from_be_bytes([b[i + 2], b[i + 3]]) as usize; + if (0xE0..=0xEF).contains(&b[i + 1]) { + out.push((b[i + 1], b[i + 4..i + 2 + len].to_vec())); + } + i += 2 + len; + } + out +} + +/// (unit, x, y) of a JPEG's JFIF header. +fn jfif_density(b: &[u8]) -> (u8, u16, u16) { + let (_, p) = app_segments(b).into_iter().find(|(m, p)| *m == 0xE0 && p.starts_with(b"JFIF\0")).unwrap(); + (p[7], u16::from_be_bytes([p[8], p[9]]), u16::from_be_bytes([p[10], p[11]])) +} + +fn exif_of(b: &[u8]) -> Vec { + let (_, p) = app_segments(b).into_iter().find(|(m, p)| *m == 0xE1 && p.starts_with(b"Exif\0\0")).unwrap(); + p[6..].to_vec() +} + +#[test] +fn exif_only_camera_jpeg_opens_at_its_exif_resolution() { + for e in [&LE, &BE] { + near(dpi_of(&jpeg_with(&[exif_app1(&camera_exif(e))])), 300.0); + // ResolutionUnit absent: inch, the EXIF default. + near(dpi_of(&jpeg_with(&[exif_app1(&e.build(&[e.x(300, 1), e.y(300, 1)]))])), 300.0); + // Non-unit denominators. + near(dpi_of(&jpeg_with(&[exif_app1(&e.build(&[e.x(3_000_000, 10_000), e.y(1200, 4), e.unit(2)]))])), 300.0); + } +} + +#[test] +fn exif_resolution_in_centimetres_is_converted() { + for e in [&LE, &BE] { + let exif = e.build(&[e.x(11811, 100), e.y(11811, 100), e.unit(3)]); + near(exif_resolution(&exif), 299.9994); + near(dpi_of(&jpeg_with(&[exif_app1(&exif)])), 299.9994); + } +} + +#[test] +fn jfif_aspect_ratio_only_with_exif_uses_exif() { + near(dpi_of(&jpeg_with(&[jfif(0, 1, 1), exif_app1(&camera_exif(&LE))])), 300.0); +} + +#[test] +fn exif_outranks_a_default_jfif_density() { + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif_app1(&camera_exif(&LE))])), 300.0); + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif_app1(&camera_exif(&BE))])), 300.0); + // JFIF alone still counts, in either unit. + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72)])), 72.0); + near(dpi_of(&jpeg_with(&[jfif(2, 100, 100)])), 254.0); + assert_eq!(dpi_of(&jpeg_with(&[jfif(0, 1, 1)])), None); +} + +#[test] +fn photoshop_resolution_info_wins_then_xmp_then_exif() { + let xmp = r#""#; + let exif = exif_app1(&camera_exif(&LE)); + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif.clone(), xmp_app1(xmp), app13(&irb(240.0))])), 240.0); + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif.clone(), xmp_app1(xmp)])), 200.0); + // A fractional Photoshop resolution survives the 16.16 fixed point. + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif, app13(&irb(299.5))])), 299.5); + near(photoshop_resolution(&irb(150.0)), 150.0); + // Element form, centimetres. + let elem = "100/11003"; + near(xmp_resolution(elem), 254.0); +} + +#[test] +fn unusable_exif_falls_back_to_jfif() { + let e = &LE; + let bad = [ + // No absolute unit. + e.build(&[e.x(300, 1), e.y(300, 1), e.unit(1)]), + // Unknown unit, malformed unit entry. + e.build(&[e.x(300, 1), e.y(300, 1), e.unit(9)]), + e.build(&[e.x(300, 1), e.y(300, 1), E(UNIT, LONG, 1, e.u32(2).to_vec())]), + // Zero denominator, zero value, wrong type, wrong count. + e.build(&[e.x(300, 0), e.y(300, 1)]), + e.build(&[e.x(0, 1), e.y(300, 1)]), + e.build(&[E(X, SHORT, 1, e.short(300)), e.y(300, 1)]), + e.build(&[E(X, RATIONAL, 2, [e.rational(300, 1), e.rational(300, 1)].concat()), e.y(300, 1)]), + // Only one axis. + e.build(&[e.x(300, 1)]), + ]; + for exif in &bad { + assert_eq!(exif_resolution(exif), None); + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif_app1(exif)])), 72.0); + } + // A value offset pointing past the end. + let mut far = camera_exif(e); + let at = 8 + 2 + 12 + 8; // XResolution entry's value field + far[at..at + 4].copy_from_slice(&e.u32(60_000)); + assert_eq!(exif_resolution(&far), None); + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif_app1(&far)])), 72.0); +} + +#[test] +fn truncated_metadata_never_panics_and_falls_back() { + let exif = camera_exif(&BE); + for n in 0..exif.len() { + let _ = exif_resolution(&exif[..n]); + let _ = exif_with_resolution(&exif[..n], (240.0, 240.0)); + let dpi = dpi_of(&jpeg_with(&[jfif(1, 72, 72), exif_app1(&exif[..n])])); + assert!(dpi == Some((72.0, 72.0)) || dpi == Some((300.0, 300.0)), "{n}: {dpi:?}"); + } + let r = irb(300.0); + for n in 0..r.len() { + assert_eq!(photoshop_resolution(&r[..n]), None, "{n}"); + near(dpi_of(&jpeg_with(&[jfif(1, 72, 72), app13(&r[..n])])), 72.0); + } + for x in + ["tiff:XResolution=", "tiff:XResolution=\"", "tiff:XResolution=\"\" tiff:YResolution=\"5/0\"", "", "9<"] + { + assert_eq!(xmp_resolution(x), None, "{x}"); + let _ = xmp_with_resolution(x, (1.0, 1.0)); + } + + // Property-like text repeated inside the packet must not trigger a full-suffix rescan + // per occurrence (it used to be quadratic). With the cursor advancing past each located + // value, this completes well under a second even with many decoys; a timeout here would + // indicate the quadratic scan returned. A clean packet still extracts correctly. + let start = std::time::Instant::now(); + let decoys = " tiff:ResolutionUnit>".repeat(50_000); + let _ = xmp_resolution(&decoys); + assert!(start.elapsed() < std::time::Duration::from_secs(2), "xmp scan took too long: {:?}", start.elapsed()); + let clean = "300/1300/12"; + near(xmp_resolution(clean), 300.0); +} + +proptest! { + #[test] + fn hostile_bytes_never_panic(bytes in proptest::collection::vec(any::(), 0..256), ppi in 0.0f32..2e6) { + let _ = exif_resolution(&bytes); + let _ = exif_with_resolution(&bytes, (ppi, ppi)); + let _ = photoshop_resolution(&bytes); + let s = String::from_utf8_lossy(&bytes); + let _ = xmp_resolution(&s); + let _ = xmp_with_resolution(&s, (ppi, ppi)); + } + + #[test] + fn mutated_exif_never_panics(i in 0usize..80, v in any::(), ppi in 1.0f32..10_000.0) { + let mut exif = camera_exif(&LE); + if let Some(b) = exif.get_mut(i) { *b = v; } + let _ = exif_resolution(&exif); + let fixed = exif_with_resolution(&exif, (ppi, ppi)); + let _ = exif_resolution(&fixed); + let _ = exif_orientation(&fixed); + } +} + +#[test] +fn rewriting_keeps_every_other_byte() { + for e in [&LE, &BE] { + let src = e.build(&[e.make(), e.orientation(6), e.x(300, 1), e.y(300, 1), e.unit(3)]); + let out = exif_with_resolution(&src, (240.0, 240.0)).into_owned(); + assert_eq!(out.len(), src.len()); + near(exif_resolution(&out), 240.0); + let want = e.build(&[e.make(), e.orientation(6), e.x(240, 1), e.y(240, 1), e.unit(2)]); + assert_eq!(out, want); + // Fractions keep three decimals. + near(exif_resolution(&exif_with_resolution(&src, (299.5, 72.25))).map(|(x, _)| (x, x)), 299.5); + // Nothing to change: borrowed. + assert!(matches!(exif_with_resolution(&camera_exif(e), (300.0, 300.0)), Cow::Borrowed(_))); + let no_res = e.build(&[e.make()]); + assert!(matches!(exif_with_resolution(&no_res, (240.0, 240.0)), Cow::Borrowed(_))); + // With the JPEG prefix. + let prefixed = [b"Exif\0\0".as_slice(), &src].concat(); + assert_eq!(exif_with_resolution(&prefixed, (240.0, 240.0)).as_ref(), [b"Exif\0\0".as_slice(), &want].concat()); + } +} + +#[test] +fn entries_that_cannot_be_rewritten_are_removed() { + for e in [&LE, &BE] { + // XResolution as a SHORT and a LONG unit: both go, the rest stays readable. + let src = e.build(&[e.make(), E(X, SHORT, 1, e.short(300)), e.y(300, 1), E(UNIT, LONG, 1, e.u32(2).to_vec()), e.orientation(6)]); + let out = exif_with_resolution(&src, (240.0, 240.0)).into_owned(); + assert_eq!(out.len(), src.len()); + let count = if e.big { u16::from_be_bytes([out[8], out[9]]) } else { u16::from_le_bytes([out[8], out[9]]) }; + assert_eq!(count, 3); + assert_eq!(exif_orientation(&out), 6); + // Only YResolution is left (240): no X, so no resolution at all, and no stale 300. + assert_eq!(exif_resolution(&out), None); + let tags: Vec = (0..3) + .map(|k| { + let at = 10 + k * 12; + if e.big { u16::from_be_bytes([out[at], out[at + 1]]) } else { u16::from_le_bytes([out[at], out[at + 1]]) } + }) + .collect(); + assert_eq!(tags, vec![0x010F, Y, 0x0112]); + // The next-IFD pointer followed the table, the freed entries are zero. + assert_eq!(&out[10 + 36..10 + 40], &[0, 0, 0, 0]); + assert!(out[10 + 40..10 + 64].iter().all(|&b| b == 0)); + // The Make string (out of line) did not move. + assert!(out.windows(6).any(|w| w == b"NIKON\0")); + } +} + +#[test] +fn xmp_resolution_is_rewritten() { + let attr = r#""#; + assert_eq!( + xmp_with_resolution(attr, (240.0, 240.0)), + r#""# + ); + let elem = "300/1300/1"; + assert_eq!( + xmp_with_resolution(elem, (72.5, 72.5)), + "72500/100072500/1000" + ); + for same in [r#"tiff:XResolution="240" tiff:YResolution="240/1""#, ""] { + assert!(matches!(xmp_with_resolution(same, (240.0, 240.0)), Cow::Borrowed(_)), "{same}"); + } +} + +#[test] +fn jpeg_export_writes_one_resolution_everywhere() { + for e in [&LE, &BE] { + let xmp = r#""#; + let src = jpeg_with(&[jfif(1, 72, 72), exif_app1(&camera_exif(e)), xmp_app1(xmp)]); + let mut img = decode(&src).unwrap(); + near(img.meta.dpi, 300.0); + img.meta.dpi = Some((240.0, 240.0)); + let out = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + assert_eq!(jfif_density(&out), (1, 240, 240)); + near(exif_resolution(&exif_of(&out)), 240.0); + let (_, x) = app_segments(&out).into_iter().find(|(m, p)| *m == 0xE1 && p.starts_with(b"http://ns.adobe.com/xap/1.0/\0")).unwrap(); + near(xmp_resolution(std::str::from_utf8(&x).unwrap()), 240.0); + near(dpi_of(&out), 240.0); + // The rest of the EXIF came along. + assert!(exif_of(&out).windows(6).any(|w| w == b"NIKON\0")); + } +} + +#[test] +fn other_formats_take_the_exif_resolution_when_they_have_none() { + let mut img = Image::from_raw(4, 4, ChannelLayout::Rgb, SampleType::U8, vec![90; 48]).unwrap(); + img.meta.exif = Some(camera_exif(&LE)); + for fmt in [Format::WebP, Format::Png] { + // Written at 240 with the 300 ppi EXIF: the EXIF follows the image. + img.meta.dpi = Some((240.0, 240.0)); + let out = encode(&img, fmt, &EncodeOptions::default()).unwrap(); + let back = decode(&out).unwrap(); + near(back.meta.dpi, 240.0); + near(back.meta.exif.as_deref().and_then(exif_resolution), 240.0); + // Written without a resolution: the EXIF one is read back. + img.meta.dpi = None; + let out = encode(&img, fmt, &EncodeOptions::default()).unwrap(); + near(decode(&out).unwrap().meta.dpi, 300.0); + } +} + +fn crc32(data: &[u8]) -> u32 { + let mut c = !0u32; + for &b in data { + c ^= u32::from(b); + for _ in 0..8 { + c = if c & 1 != 0 { 0xEDB8_8320 ^ (c >> 1) } else { c >> 1 }; + } + } + !c +} + +#[test] +fn png_phys_outranks_its_exif() { + // eXIf says 300 ppi, then a pHYs chunk of 150 ppi (5906 px/m) goes in after IHDR. + let mut img = Image::from_raw(4, 4, ChannelLayout::Rgb, SampleType::U8, vec![90; 48]).unwrap(); + img.meta.exif = Some(camera_exif(&LE)); + let png = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + near(decode(&png).unwrap().meta.dpi, 300.0); + near(decode(&with_phys(&png, 5906)).unwrap().meta.dpi, 150.0); +} + +/// `png` with a `pHYs` chunk of `ppm` pixels per metre after IHDR. +fn with_phys(png: &[u8], ppm: u32) -> Vec { + let mut body = b"pHYs".to_vec(); + body.extend_from_slice(&ppm.to_be_bytes()); + body.extend_from_slice(&ppm.to_be_bytes()); + body.push(1); + let mut chunk = 9u32.to_be_bytes().to_vec(); + chunk.extend_from_slice(&body); + chunk.extend_from_slice(&crc32(&body).to_be_bytes()); + let ihdr_end = 8 + 8 + 13 + 4; + [&png[..ihdr_end], &chunk, &png[ihdr_end..]].concat() +} + +#[test] +fn png_phys_of_a_whole_ppi_reads_back_whole() { + // pHYs holds whole pixels per metre: 300 ppi is stored as 11811 (299.9994 ppi), 72 as 2835 + // (72.009). The whole number that rounds to the stored value is read, as Photoshop does; a + // value no whole ppi rounds to stays as it is. + let img = Image::from_raw(4, 4, ChannelLayout::Rgb, SampleType::U8, vec![90; 48]).unwrap(); + let png = encode(&img, Format::Png, &EncodeOptions { embed_metadata: false, ..Default::default() }).unwrap(); + for (ppm, want) in [(11811, 300.0), (2835, 72.0), (3937, 100.0), (11812, 300.0248)] { + let (x, y) = decode(&with_phys(&png, ppm)).unwrap().meta.dpi.unwrap(); + assert!((x - want).abs() < 1e-4 && (y - want).abs() < 1e-4, "{ppm} px/m: {x} × {y}, wanted {want}"); + } + // And a 300 ppi PNG written by the encoder opens at exactly 300. + let mut img = img; + img.meta.dpi = Some((300.0, 300.0)); + let png = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + assert_eq!(decode(&png).unwrap().meta.dpi, Some((300.0, 300.0))); +} + +/// `camera_exif` plus IFD1, the embedded thumbnail a camera (or the last editor) wrote: +/// JPEGInterchangeFormat / JPEGInterchangeFormatLength naming a fake JPEG after the IFD. +fn exif_with_thumbnail(e: &Exif) -> Vec { + let mut v = camera_exif(e); + let ifd1 = v.len() as u32; + // IFD0 (4 entries) ends with its next-IFD pointer. + v[8 + 2 + 4 * 12..8 + 2 + 4 * 12 + 4].copy_from_slice(&e.u32(ifd1)); + let thumb = b"\xFF\xD8stale thumbnail\xFF\xD9"; + let thumb_at = ifd1 + 2 + 2 * 12 + 4; + v.extend_from_slice(&e.u16(2)); + for (tag, value) in [(0x0201u16, thumb_at), (0x0202, thumb.len() as u32)] { + v.extend_from_slice(&e.u16(tag)); + v.extend_from_slice(&e.u16(LONG)); + v.extend_from_slice(&e.u32(1)); + v.extend_from_slice(&e.u32(value)); + } + v.extend_from_slice(&e.u32(0)); + v.extend_from_slice(thumb); + v +} + +fn next_ifd(e: &Exif, exif: &[u8]) -> u32 { + let at = 8 + 2 + 4 * 12; + let b: [u8; 4] = exif[at..at + 4].try_into().unwrap(); + if e.big { u32::from_be_bytes(b) } else { u32::from_le_bytes(b) } +} + +/// #2147 / #2373: an export must not carry the source's EXIF thumbnail (IFD1). Finder, Android +/// and upload pickers show it instead of the image, so an edited photo kept its old thumbnail. +#[test] +fn exports_drop_the_source_exif_thumbnail() { + for e in [&LE, &BE] { + let exif = exif_with_thumbnail(e); + assert_ne!(next_ifd(e, &exif), 0); + for ppi in [None, Some((300.0, 300.0)), Some((240.0, 240.0))] { + let out = export_exif(&exif, ppi); + assert_eq!(next_ifd(e, &out), 0, "IFD1 is unlinked ({ppi:?})"); + assert_eq!(out.len(), exif.len(), "nothing moves"); + assert!(out.windows(6).any(|w| w == b"NIKON\0"), "the rest of the EXIF is kept"); + near(exif_resolution(&out), ppi.map_or(300.0, |p| p.0)); + } + // Only the IFD1 pointer and thumbnail payload change: the pointer is unlinked and the + // stale JPEG bytes are blanked by the shared metadata editor. + let out = export_exif(&exif, None); + let thumb_len = b"\xFF\xD8stale thumbnail\xFF\xD9".len(); + let differing: Vec = (0..exif.len()).filter(|&i| exif[i] != out[i]).collect(); + assert!(differing.iter().all(|&i| (8 + 2 + 4 * 12..8 + 2 + 4 * 12 + 4).contains(&i) || i >= exif.len() - thumb_len), "{differing:?}"); + assert!(!out.windows(5).any(|w| w == b"stale")); + // With a JPEG-style prefix too. + let prefixed = [b"Exif\0\0".as_slice(), &exif].concat(); + assert_eq!(next_ifd(e, &export_exif(&prefixed, None)[6..]), 0); + // An EXIF without a thumbnail comes through untouched. + assert!(matches!(export_exif(&camera_exif(e), None), Cow::Borrowed(_))); + // Through every encoder that writes EXIF. + let mut img = Image::from_raw(4, 4, ChannelLayout::Rgb, SampleType::U8, vec![90; 48]).unwrap(); + img.meta.exif = Some(exif.clone()); + let jpeg = encode(&img, Format::Jpeg, &EncodeOptions::default()).unwrap(); + assert_eq!(next_ifd(e, &exif_of(&jpeg)), 0, "JPEG"); + for format in [Format::Png, Format::WebP] { + let back = decode(&encode(&img, format, &EncodeOptions::default()).unwrap()).unwrap(); + let got = back.meta.exif.unwrap_or_else(|| panic!("{format:?} kept no EXIF")); + let body = got.strip_prefix(b"Exif\0\0").unwrap_or(&got); + assert_eq!(next_ifd(e, body), 0, "{format:?}"); + } + } +} + +#[test] +fn thumbnail_removal_never_panics_on_truncated_exif() { + for e in [&LE, &BE] { + let exif = exif_with_thumbnail(e); + for n in 0..exif.len() { + let _ = export_exif(&exif[..n], None); + let _ = export_exif(&exif[..n], Some((300.0, 300.0))); + } + } +} diff --git a/crates/image-codecs/tests/it/roundtrip.rs b/crates/image-codecs/tests/it/roundtrip.rs new file mode 100644 index 000000000..6d5d38bca --- /dev/null +++ b/crates/image-codecs/tests/it/roundtrip.rs @@ -0,0 +1,350 @@ +//! Round-trip every writable format × every sample type × every layout. +//! Lossless formats must reproduce the (planned) conversion of the input +//! exactly; lossy formats must stay above a PSNR threshold. + +use crate::common; +use common::*; +use craft_image_codecs::*; + +/// Minimum PSNR (dB) for lossy formats at default options. +fn psnr_threshold(format: Format) -> f64 { + match format { + Format::Jpeg => 30.0, + Format::Gif => 20.0, + Format::Hdr => 40.0, + Format::Avif => 28.0, + _ => f64::INFINITY, + } +} + +fn check_roundtrip(format: Format, layout: ChannelLayout, sample: SampleType) { + let c = caps(format); + if !c.write { + return; + } + // GIF quantizes; keep its input smooth so the threshold is meaningful. + let img = if format == Format::Gif { + synth(37, 23, layout, sample, 7, 0.0) + } else if c.lossy { + synth(37, 23, layout, sample, 7, 0.02) + } else { + test_image(layout, sample) + }; + let opts = EncodeOptions::default(); + let bytes = encode(&img, format, &opts).unwrap_or_else(|e| panic!("{format:?} {layout:?} {sample:?}: {e}")); + assert_eq!(detect(&bytes), Some(format), "detect after encode"); + if !c.read { + return; + } + let back = decode(&bytes).unwrap_or_else(|e| panic!("{format:?} {layout:?} {sample:?} decode: {e}")); + assert_eq!(back.dimensions(), img.dimensions()); + + // Native storage must be preserved when the format declares support. + let float_pnm_special = format == Format::Pnm && sample.is_float(); + if c.layouts.contains(&layout) && c.depths.contains(&sample) && !float_pnm_special { + assert_eq!(back.layout(), layout, "{format:?} must keep layout"); + assert_eq!(back.sample_type(), sample, "{format:?} must keep sample type"); + } + if c.depths.contains(&sample) && !float_pnm_special { + assert_eq!(back.sample_type(), sample, "{format:?} must keep supported depth"); + } + // Integer sources never lose precision in formats offering that depth. + if sample == SampleType::U16 && c.depths.contains(&SampleType::U16) { + assert!(matches!(back.sample_type(), SampleType::U16 | SampleType::F32)); + } + + let expected = img.convert(back.layout(), back.sample_type()); + let mut expected_cmp = expected.clone(); + if format == Format::Gif && layout.has_alpha() { + // GIF alpha is binary: compare colour only on opaque pixels. + let ch = back.layout().channels(); + let mut e = expected.to_normalized(); + let b = back.to_normalized(); + for (pe, pb) in e.chunks_mut(ch).zip(b.chunks(ch)) { + if pb[ch - 1] == 0.0 { + pe.copy_from_slice(pb); + } + pe[ch - 1] = pb[ch - 1]; + } + expected_cmp = Image::from_normalized(img.width(), img.height(), back.layout(), back.sample_type(), &e).unwrap(); + } + + if c.lossy { + let p = psnr(&expected_cmp, &back); + assert!(p >= psnr_threshold(format), "{format:?} {layout:?} {sample:?}: PSNR {p:.2} dB"); + } else { + assert_eq!( + expected_cmp.data(), + back.data(), + "{format:?} {layout:?} {sample:?}: lossless round-trip mismatch (max diff {})", + max_abs_diff(&expected_cmp, &back) + ); + } +} + +macro_rules! cases { + ($fmt:expr) => { + use super::*; + cases!(@ $fmt; + gray_u8: Gray, U8; gray_u16: Gray, U16; gray_f16: Gray, F16; gray_f32: Gray, F32; + graya_u8: GrayA, U8; graya_u16: GrayA, U16; graya_f16: GrayA, F16; graya_f32: GrayA, F32; + rgb_u8: Rgb, U8; rgb_u16: Rgb, U16; rgb_f16: Rgb, F16; rgb_f32: Rgb, F32; + rgba_u8: Rgba, U8; rgba_u16: Rgba, U16; rgba_f16: Rgba, F16; rgba_f32: Rgba, F32; + cmyk_u8: Cmyk, U8; cmyk_u16: Cmyk, U16; cmyk_f16: Cmyk, F16; cmyk_f32: Cmyk, F32; + cmyka_u8: CmykA, U8; cmyka_u16: CmykA, U16; cmyka_f16: CmykA, F16; cmyka_f32: CmykA, F32 + ); + }; + (@ $fmt:expr; $($name:ident : $l:ident, $s:ident);*) => { + $( + #[test] + fn $name() { + check_roundtrip($fmt, ChannelLayout::$l, SampleType::$s); + } + )* + }; +} + +mod png { + cases!(Format::Png); +} +mod jpeg { + cases!(Format::Jpeg); +} +mod tiff { + cases!(Format::Tiff); +} +mod webp { + cases!(Format::WebP); +} +mod gif { + cases!(Format::Gif); +} +mod bmp { + cases!(Format::Bmp); +} +mod tga { + cases!(Format::Tga); +} +mod ico { + cases!(Format::Ico); +} +mod pnm { + cases!(Format::Pnm); +} +mod fits { + cases!(Format::Fits); +} +mod dpx { + cases!(Format::Dpx); +} +mod dds { + cases!(Format::Dds); +} +mod qoi { + cases!(Format::Qoi); +} +mod exr { + cases!(Format::OpenExr); +} +mod hdr { + cases!(Format::Hdr); +} + +// --------------------------------------------------------------------------- +// Targeted round-trips beyond the matrix +// --------------------------------------------------------------------------- + +/// Float formats must keep HDR values (> 1, and negative where allowed). +#[test] +fn hdr_values_survive_float_formats() { + let vals: Vec = (0..16 * 8 * 3).map(|i| (i as f32 - 40.0) * 0.37).collect(); + let img = Image::from_f32(16, 8, ChannelLayout::Rgb, &vals).unwrap(); + for f in [Format::Tiff, Format::OpenExr, Format::Pnm] { + let back = decode(&encode(&img, f, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.to_f32_samples().unwrap(), vals, "{f:?}"); + } +} + +#[test] +fn hdr_format_keeps_bright_values_approximately() { + let vals: Vec = (0..16 * 8 * 3).map(|i| 0.01 + i as f32 * 0.25).collect(); + let img = Image::from_f32(16, 8, ChannelLayout::Rgb, &vals).unwrap(); + let back = decode(&encode(&img, Format::Hdr, &EncodeOptions::default()).unwrap()).unwrap(); + // RGBE shares one exponent per pixel: error is relative to the brightest channel. + let got = back.to_f32_samples().unwrap(); + for (pa, pb) in vals.chunks(3).zip(got.chunks(3)) { + let m = pa.iter().copied().fold(0.0, f32::max); + for (a, b) in pa.iter().zip(pb) { + assert!((a - b).abs() <= m * 0.01, "{a} vs {b}"); + } + } +} + +#[test] +fn f16_exr_is_bit_exact() { + let vals: Vec = (0..20 * 10 * 4).map(|i| f16::from_f32(i as f32 * 0.013 - 1.0)).collect(); + let img = Image::from_f16(20, 10, ChannelLayout::Rgba, &vals).unwrap(); + for comp in [ExrCompression::None, ExrCompression::Rle, ExrCompression::Zip1, ExrCompression::Zip16, ExrCompression::Piz] { + let opts = EncodeOptions { exr_compression: comp, ..Default::default() }; + let back = decode(&encode(&img, Format::OpenExr, &opts).unwrap()).unwrap(); + assert_eq!(back.to_f16_samples().unwrap(), vals, "{comp:?}"); + } +} + +#[test] +fn tiff_all_compressions_lossless() { + for layout in ChannelLayout::ALL { + for sample in [SampleType::U8, SampleType::U16, SampleType::F32] { + let img = test_image(layout, sample); + for comp in [TiffCompression::None, TiffCompression::Lzw, TiffCompression::Deflate, TiffCompression::PackBits] { + let opts = EncodeOptions { tiff_compression: comp, ..Default::default() }; + let back = decode(&encode(&img, Format::Tiff, &opts).unwrap()).unwrap(); + if layout.is_lab() && sample != SampleType::U8 { + // CIELab holds a* and b* as 256·a at 16 bits and has no float form: 16-bit and + // float Lab come back within one 16-bit step (the PSD's 16-bit Lab does the same). + let (a, b) = (img.to_normalized(), back.convert(layout, sample).to_normalized()); + let worst = a.iter().zip(&b).map(|(x, y)| (x - y).abs()).fold(0.0f32, f32::max); + assert!(worst <= 1.5 / 65535.0, "{layout:?} {sample:?} {comp:?}: {worst}"); + continue; + } + assert_eq!(back, img.clone().with_meta(back.meta.clone()), "{layout:?} {sample:?} {comp:?}"); + } + } + } +} + +#[test] +fn compressed_tiff_does_not_keep_the_uncompressed_reservation() { + // The encoder reserves the uncompressed size up front for an uncompressed file (#6607). A + // smooth image compresses to a fraction of it, and no returned file holds much spare room. + let (w, h) = (512u32, 512u32); + let px: Vec = (0..h).flat_map(|y| (0..w).flat_map(move |x| [(x / 2) as u8, (y / 2) as u8, 128])).collect(); + let img = Image::from_u8(w, h, ChannelLayout::Rgb, px).unwrap(); + for comp in [TiffCompression::None, TiffCompression::Lzw, TiffCompression::Deflate] { + let v = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: comp, ..Default::default() }).unwrap(); + if !matches!(comp, TiffCompression::None) { + assert!(v.len() < img.data().len() / 2, "{comp:?}: the image should compress well ({} bytes)", v.len()); + } + assert!(v.capacity() - v.len() <= v.len() / 8, "{comp:?}: {} bytes kept for a {} byte file", v.capacity(), v.len()); + assert_eq!(decode(&v).unwrap().data(), img.data(), "{comp:?}"); + } +} + +#[test] +fn png_all_compressions_lossless() { + let img = test_image(ChannelLayout::Rgba, SampleType::U16); + for comp in [PngCompression::None, PngCompression::Fast, PngCompression::Default, PngCompression::Best] { + let opts = EncodeOptions { png_compression: comp, ..Default::default() }; + let back = decode(&encode(&img, Format::Png, &opts).unwrap()).unwrap(); + assert_eq!(back.data(), img.data(), "{comp:?}"); + } +} + +#[test] +fn jpeg_quality_affects_size_and_error() { + let img = test_image(ChannelLayout::Rgb, SampleType::U8); + let lo = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_quality: 20, ..Default::default() }).unwrap(); + let hi = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_quality: 98, ..Default::default() }).unwrap(); + assert!(lo.len() < hi.len()); + let plo = psnr(&img, &decode(&lo).unwrap()); + let phi = psnr(&img, &decode(&hi).unwrap()); + assert!(phi > plo, "{phi} <= {plo}"); +} + +#[test] +fn jpeg_444_beats_420_on_chroma_detail() { + let img = synth(64, 64, ChannelLayout::Rgb, SampleType::U8, 3, 0.5); + let a = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_chroma_subsampling: false, jpeg_quality: 95, ..Default::default() }).unwrap(); + let b = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_chroma_subsampling: true, jpeg_quality: 95, ..Default::default() }).unwrap(); + assert!(psnr(&img, &decode(&a).unwrap()) > psnr(&img, &decode(&b).unwrap())); +} + +/// JPEG Options › Format Options: each scan layout decodes back to the same image, Progressive +/// writes a progressive frame (SOF2), and Baseline Optimized fits its Huffman tables (smaller at +/// 4:4:4; at 4:2:0 it keeps the standard tables, see `JpegFormat::Optimized`). +#[test] +fn jpeg_format_options_round_trip() { + let img = synth(64, 48, ChannelLayout::Rgb, SampleType::U8, 3, 0.5); + let sof = |b: &[u8], m: u8| b.windows(2).any(|w| w == [0xFF, m]); + for subsampled in [false, true] { + let enc = |f| { + encode(&img, Format::Jpeg, &EncodeOptions { jpeg_format: f, jpeg_chroma_subsampling: subsampled, jpeg_quality: 90, ..Default::default() }) + .unwrap() + }; + let (base, opt, prog) = (enc(JpegFormat::Baseline), enc(JpegFormat::Optimized), enc(JpegFormat::Progressive)); + // The scan layout only changes how the same quantized coefficients are stored, so every + // format decodes as faithfully as Baseline. + let reference = psnr(&img, &decode(&base).unwrap()); + for (name, b) in [("optimized", &opt), ("progressive", &prog)] { + let back = decode(b).unwrap(); + assert_eq!(back.dimensions(), img.dimensions(), "{name}"); + let p = psnr(&img, &back); + assert!((p - reference).abs() < 0.5, "{name}, subsampled {subsampled}: {p} vs baseline {reference}"); + } + assert!(sof(&base, 0xC0) && !sof(&base, 0xC2)); + assert!(sof(&prog, 0xC2), "progressive frame"); + if subsampled { + assert_eq!(opt, base, "4:2:0 keeps the standard tables"); + } else { + assert!(opt.len() < base.len(), "fitted tables are smaller: {} vs {}", opt.len(), base.len()); + } + } + // Gray is 4:4:4 too, so Baseline Optimized shrinks it. + let gray = synth(64, 48, ChannelLayout::Gray, SampleType::U8, 3, 0.5); + let enc = |f| encode(&gray, Format::Jpeg, &EncodeOptions { jpeg_format: f, ..Default::default() }).unwrap(); + let (base, opt) = (enc(JpegFormat::Baseline), enc(JpegFormat::Optimized)); + assert!(opt.len() < base.len(), "gray: {} vs {}", opt.len(), base.len()); + assert!((psnr(&gray, &decode(&opt).unwrap()) - psnr(&gray, &decode(&base).unwrap())).abs() < 0.5); + assert_eq!(JpegFormat::parse("progressive"), Some(JpegFormat::Progressive)); + assert_eq!(JpegFormat::parse(JpegFormat::Optimized.name()), Some(JpegFormat::Optimized)); + for bad in ["Progressive", "", "standard"] { + assert_eq!(JpegFormat::parse(bad), None, "{bad}"); + } +} + +#[test] +fn webp_lossy_request_is_a_lossy_file() { + let img = test_image(ChannelLayout::Rgb, SampleType::U8); + let bytes = encode(&img, Format::WebP, &EncodeOptions { webp_lossless: false, ..Default::default() }).unwrap(); + assert_eq!(&bytes[12..16], b"VP8 ", "a bare lossy frame"); + let back = decode(&bytes).unwrap(); + assert_eq!(back.dimensions(), img.dimensions()); + assert!( + fidelity_warnings_with(&img, Format::WebP, &EncodeOptions { webp_lossless: false, ..Default::default() }) + .contains(&FidelityWarning::LossyCompression) + ); +} + +#[test] +fn one_pixel_images_roundtrip() { + for f in rw_formats() { + let img = synth(1, 1, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + let back = decode(&encode(&img, f, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.dimensions(), (1, 1), "{f:?}"); + } +} + +#[test] +fn ico_rejects_oversize() { + let img = Image::new(300, 10, ChannelLayout::Rgba, SampleType::U8).unwrap(); + assert!(matches!(encode(&img, Format::Ico, &EncodeOptions::default()), Err(CodecError::Encode { .. }))); + let img = Image::new(256, 256, ChannelLayout::Rgba, SampleType::U8).unwrap(); + assert!(encode(&img, Format::Ico, &EncodeOptions::default()).is_ok()); +} + +#[test] +fn empty_image_rejected() { + let img = Image::new(0, 5, ChannelLayout::Rgb, SampleType::U8).unwrap(); + for f in writable_formats() { + assert!(encode(&img, f, &EncodeOptions::default()).is_err(), "{f:?}"); + } +} + +#[test] +fn larger_image_png_tiff_exact() { + let img = synth(513, 257, ChannelLayout::Rgba, SampleType::U16, 99, 1.0); + for f in [Format::Png, Format::Tiff, Format::Pnm] { + let back = decode(&encode(&img, f, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.data(), img.data(), "{f:?}"); + } +} diff --git a/crates/image-codecs/tests/it/source_settings.rs b/crates/image-codecs/tests/it/source_settings.rs new file mode 100644 index 000000000..5923d562d --- /dev/null +++ b/crates/image-codecs/tests/it/source_settings.rs @@ -0,0 +1,111 @@ +//! Reading back the settings a lossy file was saved with: a JPEG's quality from its +//! quantisation tables, a WebP's lossy or lossless encoding (and its quality). + +use crate::common; +use common::*; +use craft_image_codecs::*; + +fn photo(w: u32, h: u32, layout: ChannelLayout) -> Image { + synth(w, h, layout, SampleType::U8, 5, 0.03) +} + +#[test] +fn jpeg_quality_is_estimated_within_two() { + for layout in [ChannelLayout::Rgb, ChannelLayout::Gray] { + let img = photo(40, 24, layout); + for q in [1u8, 5, 10, 25, 40, 50, 60, 70, 75, 80, 85, 90, 95, 98, 100] { + let bytes = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_quality: q, ..Default::default() }).unwrap(); + let est = jpeg_quality(&bytes).unwrap_or_else(|| panic!("{layout:?} q{q}: no estimate")); + assert!(est.abs_diff(q) <= 2, "{layout:?}: saved at {q}, estimated {est}"); + } + } +} + +#[test] +fn jpeg_quality_of_hand_built_tables() { + // A minimal JPEG prefix: SOI, one DQT holding the Annex K luminance table (quality 50) in + // zig-zag order, then SOS. + let luma: [u8; 64] = [ + 16, 11, 12, 14, 12, 10, 16, 14, 13, 14, 18, 17, 16, 19, 24, 40, 26, 24, 22, 22, 24, 49, 35, 37, 29, 40, 58, 51, 61, 60, 57, 51, 56, 55, 64, + 72, 92, 78, 64, 68, 87, 69, 55, 56, 80, 109, 81, 87, 95, 98, 103, 104, 103, 62, 77, 113, 121, 112, 100, 120, 92, 101, 103, 99, + ]; + let mut b = vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 67, 0x00]; + b.extend_from_slice(&luma); + b.extend_from_slice(&[0xFF, 0xDA, 0, 2]); + assert_eq!(jpeg_quality(&b), Some(50)); +} + +#[test] +fn hostile_jpeg_tables_give_no_estimate_and_never_panic() { + let img = photo(16, 16, ChannelLayout::Rgb); + let good = encode(&img, Format::Jpeg, &EncodeOptions { jpeg_quality: 70, ..Default::default() }).unwrap(); + // Every truncation of a real file. + for n in 0..good.len().min(800) { + let _ = jpeg_quality(&good[..n]); + } + let cases: Vec> = vec![ + vec![], + vec![0xFF], + vec![0xFF, 0xD8], + // A DQT whose length runs past the end. + vec![0xFF, 0xD8, 0xFF, 0xDB, 0xFF, 0xFF, 0x00, 1, 2, 3], + // A DQT of length 0 and 1 (less than its own length field). + vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 0x00], + vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 0x01], + // A table cut short, a bad precision, a slot past 3, a 16-bit table cut mid-entry. + [vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 12, 0x00], vec![9; 9]].concat(), + [vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 67, 0x70], vec![9; 64]].concat(), + [vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 67, 0x0F], vec![9; 64]].concat(), + [vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 68, 0x10], vec![0xFF; 65]].concat(), + // All-zero and all-0xFF tables. + [vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 67, 0x00], vec![0; 64]].concat(), + [vec![0xFF, 0xD8, 0xFF, 0xDB, 0x00, 131, 0x10], vec![0xFF; 128]].concat(), + vec![0xFF; 64], + ]; + for c in &cases { + let _ = jpeg_quality(c); + } + assert_eq!(jpeg_quality(&[0xFF, 0xD8, 0xFF, 0xDB, 0x00, 0x00]), None); + assert_eq!(jpeg_quality(b"not a jpeg at all"), None); + let png = encode(&img, Format::Png, &EncodeOptions::default()).unwrap(); + assert_eq!(jpeg_quality(&png), None); +} + +#[test] +fn webp_lossless_and_lossy_are_told_apart() { + let img = photo(33, 21, ChannelLayout::Rgb); + let lossless = encode(&img, Format::WebP, &EncodeOptions::default()).unwrap(); + assert_eq!(webp_encoding(&lossless), Some(WebpEncoding::Lossless)); + for q in [10u8, 30, 50, 75, 80, 90, 100] { + let bytes = encode(&img, Format::WebP, &EncodeOptions { webp_lossless: false, webp_quality: q, ..Default::default() }).unwrap(); + let Some(WebpEncoding::Lossy { quality: Some(est) }) = webp_encoding(&bytes) else { panic!("q{q}: {:?}", webp_encoding(&bytes)) }; + assert!(est.abs_diff(q) <= 2, "lossy WebP saved at {q}, estimated {est}"); + } + // With alpha and metadata (the extended container). + let rgba = photo(20, 20, ChannelLayout::Rgba); + let bytes = encode(&rgba, Format::WebP, &EncodeOptions { webp_lossless: false, webp_quality: 60, ..Default::default() }).unwrap(); + assert!(matches!(webp_encoding(&bytes), Some(WebpEncoding::Lossy { quality: Some(58..=62) })), "{:?}", webp_encoding(&bytes)); +} + +#[test] +fn hostile_webp_never_panics() { + let img = photo(24, 24, ChannelLayout::Rgb); + for opts in [EncodeOptions::default(), EncodeOptions { webp_lossless: false, ..Default::default() }] { + let good = encode(&img, Format::WebP, &opts).unwrap(); + for n in 0..good.len().min(400) { + let _ = webp_encoding(&good[..n]); + } + let mut flipped = good.clone(); + for i in 12..flipped.len().min(64) { + flipped[i] ^= 0xA5; + let _ = webp_encoding(&flipped); + } + } + let mut huge = b"RIFF\0\0\0\0WEBPVP8 ".to_vec(); + huge.extend_from_slice(&u32::MAX.to_le_bytes()); + let _ = webp_encoding(&huge); + let mut anmf = b"RIFF\0\0\0\0WEBPANMF".to_vec(); + anmf.extend_from_slice(&3u32.to_le_bytes()); + let _ = webp_encoding(&anmf); + assert_eq!(webp_encoding(b"RIFF\0\0\0\0WAVE"), None); +} diff --git a/crates/image-codecs/tests/it/tga.rs b/crates/image-codecs/tests/it/tga.rs new file mode 100644 index 000000000..489e2eed1 --- /dev/null +++ b/crates/image-codecs/tests/it/tga.rs @@ -0,0 +1,91 @@ +//! Targa writing (#6382): run-length packets end with their scanline, rows are stored bottom-up, +//! and the file reads back exactly. + +use craft_image_codecs::{ChannelLayout, EncodeOptions, Format, Image, decode_as, encode}; + +/// The packets of a run-length encoded Targa body, as (pixels, is a run), from just after the +/// 18-byte header (no image ID or colour map) up to `pixels` pixels. +fn packets(file: &[u8], bpp: usize, pixels: usize) -> Vec<(usize, bool)> { + let mut at = 18; + let mut seen = 0; + let mut out = Vec::new(); + while seen < pixels { + let head = file[at]; + let count = usize::from(head & 0x7f) + 1; + let run = head & 0x80 != 0; + at += 1 + if run { bpp } else { count * bpp }; + seen += count; + out.push((count, run)); + } + out +} + +/// A `w`×`h` image whose rows mix long runs (wider than a packet), short ones and noise, and +/// whose last pixel equals the next row's first, so a packet would run on if it could. +fn sample(w: u32, h: u32, layout: ChannelLayout) -> Image { + let n = layout.channels(); + let mut data = Vec::new(); + for y in 0..h { + for x in 0..w { + let v = match x { + _ if x < 150 => 40, + _ if x % 3 == 0 => (x * 7 + y * 13) as u8, + _ => 200, + }; + let px = [v, v.wrapping_add(y as u8), v ^ 0x5a, 255 - v]; + data.extend_from_slice(&px[..n]); + } + } + Image::from_u8(w, h, layout, data).unwrap() +} + +#[test] +fn rle_packets_end_with_their_scanline() { + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + let (w, h) = (300u32, 4u32); + let file = encode(&sample(w, h, layout), Format::Tga, &EncodeOptions::default()).unwrap(); + let bpp = layout.channels(); + let mut col = 0; + for (count, _) in packets(&file, bpp, (w * h) as usize) { + assert!(count <= 128, "{layout:?}: a packet holds at most 128 pixels"); + assert!(col + count <= w as usize, "{layout:?}: a packet runs from column {col} past the row's end"); + col = (col + count) % w as usize; + } + // A solid image is one run per row (three packets of 128, 128 and 44), not one long run. + let solid = Image::from_u8(w, h, layout, vec![90; (w * h) as usize * bpp]).unwrap(); + let file = encode(&solid, Format::Tga, &EncodeOptions::default()).unwrap(); + let p = packets(&file, bpp, (w * h) as usize); + assert_eq!(p.len(), 3 * h as usize, "{layout:?}"); + assert!(p.iter().all(|&(_, run)| run)); + } +} + +#[test] +fn the_file_is_bottom_up_rle_with_its_alpha_bits_and_the_tga2_footer() { + let mut data = vec![0u8; 2 * 2 * 4]; + // Top row red, bottom row blue, opaque. + for (i, px) in data.as_chunks_mut::<4>().0.iter_mut().enumerate() { + *px = if i < 2 { [255, 0, 0, 255] } else { [0, 0, 255, 255] }; + } + let file = encode(&Image::from_u8(2, 2, ChannelLayout::Rgba, data).unwrap(), Format::Tga, &EncodeOptions::default()).unwrap(); + assert_eq!(file[2], 10, "run-length true colour"); + assert_eq!((file[16], file[17]), (32, 8), "32 bits per pixel, 8 alpha bits, bottom-left origin"); + // The first stored row is the bottom one: a run of two blue pixels, in BGRA. + assert_eq!(&file[18..23], &[0x81, 255, 0, 0, 255]); + assert!(file.ends_with(b"TRUEVISION-XFILE.\0")); + let gray = encode(&Image::from_u8(2, 1, ChannelLayout::Gray, vec![1, 2]).unwrap(), Format::Tga, &EncodeOptions::default()).unwrap(); + assert_eq!((gray[2], gray[16], gray[17]), (11, 8, 0), "run-length grayscale, no alpha bits"); +} + +#[test] +fn every_layout_reads_back_exactly() { + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + for (w, h) in [(1, 1), (300, 4), (129, 3), (5, 130)] { + let img = sample(w, h, layout); + let file = encode(&img, Format::Tga, &EncodeOptions::default()).unwrap(); + let back = decode_as(Format::Tga, &file).unwrap(); + assert_eq!((back.dimensions(), back.layout()), ((w, h), layout), "{layout:?} {w}x{h}"); + assert_eq!(back.data(), img.data(), "{layout:?} {w}x{h}"); + } + } +} diff --git a/crates/image-codecs/tests/it/tiff_bands.rs b/crates/image-codecs/tests/it/tiff_bands.rs new file mode 100644 index 000000000..fe61178dd --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_bands.rs @@ -0,0 +1,370 @@ +//! `TiffBands` (#8169): decoding a TIFF a group of bands at a time gives exactly the image the +//! whole-image decoder does, for every layout the decoder reads, every orientation, and groups of +//! any size; and it opens images whose single buffer the whole-image decoder refuses. + +use crate::common::tiffgen::*; +use craft_image_codecs::*; + +/// A deterministic byte pattern. +fn noise(n: usize, seed: u32) -> Vec { + let mut x = seed.wrapping_mul(2_654_435_761).max(1); + (0..n) + .map(|_| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + (x >> 24) as u8 + }) + .collect() +} + +/// The image `TiffBands` decodes, assembled from its groups. +fn banded(bytes: &[u8], page: Option, limits: &Limits, upright: bool) -> Result { + let mut t = TiffBands::open(bytes, page, limits)?; + if upright { + t = t.upright(); + } + let (w, h) = t.dimensions(); + let mut out = Image::new(w, h, t.layout(), t.sample_type())?; + let bpp = t.layout().channels() * t.sample_type().bytes(); + let mut groups = 0; + t.decode(|x, y, g: &Image| -> Result<(), CodecError> { + groups += 1; + assert_eq!((g.layout(), g.sample_type()), (out.layout(), out.sample_type())); + let (gw, gh) = g.dimensions(); + assert!(x + gw <= w && y + gh <= h, "group {gw}x{gh} at {x},{y} outside {w}x{h}"); + let row = gw as usize * bpp; + for r in 0..gh as usize { + let dst = ((y as usize + r) * w as usize + x as usize) * bpp; + out.data_mut()[dst..dst + row].copy_from_slice(&g.data()[r * row..(r + 1) * row]); + } + Ok(()) + })?; + assert!(groups >= 1); + out.icc = t.icc.clone(); + out.meta = t.meta.clone(); + out.warnings = t.warnings.clone(); + Ok(out) +} + +/// The whole-image decode of the same page. +fn whole(bytes: &[u8], page: Option, upright: bool) -> Image { + let opts = DecodeOptions { keep_orientation: !upright, ..Default::default() }; + match page { + Some(p) => decode_tiff_page(bytes, p, &opts).unwrap(), + None => decode_with(bytes, &opts).unwrap(), + } +} + +/// Limits whose `max_alloc` makes each group `bands` bands of `band_bytes`. +fn grouped(band_bytes: usize, bands: usize) -> Limits { + Limits { max_alloc: (band_bytes * bands) as u64, ..Limits::banded() } +} + +fn assert_same(bytes: &[u8], band_bytes: usize, what: &str) { + for upright in [false, true] { + let want = whole(bytes, None, upright); + for limits in [Limits::banded(), grouped(band_bytes, 1), grouped(band_bytes, 2), grouped(band_bytes, 3)] { + let got = banded(bytes, None, &limits, upright).unwrap_or_else(|e| panic!("{what}: {e}")); + assert_eq!(got.dimensions(), want.dimensions(), "{what}"); + assert_eq!((got.layout(), got.sample_type()), (want.layout(), want.sample_type()), "{what}"); + assert!(got.data() == want.data(), "{what}: pixels differ (max_alloc {})", limits.max_alloc); + assert_eq!(got.icc, want.icc, "{what}"); + assert_eq!(got.meta, want.meta, "{what}"); + assert_eq!(got.warnings, want.warnings, "{what}"); + } + } +} + +/// `n / 1` as a little-endian RATIONAL value. +fn rational(n: u64) -> u64 { + n | 1 << 32 +} + +const W: u32 = 37; +const H: u32 = 29; + +#[test] +fn every_layout_matches_the_whole_image_decoder() { + let n = (W * H) as usize; + let rows = 4u32; + let row = |spp: usize, bps: usize| W as usize * spp * bps; + let cases: Vec<(&str, Dir, usize)> = vec![ + ("rgb8 strips", strips(W, H, 8, 3, 2, rows, &noise(n * 3, 1)), row(3, 1) * rows as usize), + ("gray8 WhiteIsZero", strips(W, H, 8, 1, 0, rows, &noise(n, 2)), row(1, 1) * rows as usize), + ("gray16 big-endian", strips(W, H, 16, 1, 1, rows, &noise(n * 2, 3)), row(1, 2) * rows as usize), + ("cmyk8", strips(W, H, 8, 4, 5, rows, &noise(n * 4, 4)), row(4, 1) * rows as usize), + ("cielab8", strips(W, H, 8, 3, 8, rows, &noise(n * 3, 5)), row(3, 1) * rows as usize), + ("rgba16 associated alpha", strips(W, H, 16, 4, 2, rows, &noise(n * 8, 6)).tag(338, SHORT, &[1]), row(4, 2) * rows as usize), + ( + "gray f32", + strips(W, H, 32, 1, 1, rows, &f32s(&noise(n, 7).iter().map(|&b| f32::from(b) / 255.0).collect::>(), true)).tag(339, SHORT, &[3]), + row(1, 4) * rows as usize, + ), + ( + "palette 4-bit", + strips(W, H, 4, 1, 3, rows, &noise((W as usize).div_ceil(2) * H as usize, 8)).tag( + 320, + SHORT, + &(0..48u64).map(|i| i * 1365).collect::>(), + ), + row(3, 1) * rows as usize, + ), + ("gray 1-bit", strips(W, H, 1, 1, 1, rows, &noise((W as usize).div_ceil(8) * H as usize, 9)), row(1, 1) * rows as usize), + ]; + for (what, dir, band) in cases { + // Both byte orders: the generator writes multi-byte samples as given, so the + // big-endian files hold different values; each is compared against its own decode. + for little in [true, false] { + assert_same(&build(little, false, std::slice::from_ref(&dir), &[0]).bytes, band, what); + } + } +} + +#[test] +fn tiles_and_planar_data_match() { + let (w, h) = (W as usize, H as usize); + let px = noise(w * h * 3 * 2, 11); + let tiled = |planar: bool| { + let mut d = Dir { entries: base(W, H, 16, 3, 2), chunks: tile_split(&px, w, h, 3, 2, 16, 16, planar), tiled: true } + .tag(322, SHORT, &[16]) + .tag(323, SHORT, &[16]); + if planar { + d = d.tag(284, SHORT, &[2]); + } + d + }; + let band = w * 3 * 2 * 16; + assert_same(&build(true, false, &[tiled(false)], &[0]).bytes, band, "chunky tiles"); + assert_same(&build(true, true, &[tiled(true)], &[0]).bytes, band, "planar tiles, BigTIFF"); + let planar = + Dir { entries: base(W, H, 16, 3, 2), chunks: planar_strips(&px, w, h, 3, 2, 5), tiled: false }.tag(278, LONG, &[5]).tag(284, SHORT, &[2]); + assert_same(&build(true, false, &[planar], &[0]).bytes, w * 3 * 2 * 5, "planar strips"); +} + +#[test] +fn every_orientation_matches() { + let n = (W * H) as usize; + for o in 1..=8u64 { + let dir = strips(W, H, 8, 3, 2, 3, &noise(n * 3, 20 + o as u32)) + .tag(274, SHORT, &[o]) + .tag(282, RATIONAL, &[rational(300)]) + .tag(283, RATIONAL, &[rational(150)]) + .tag(296, SHORT, &[2]); + let bytes = build(true, false, &[dir], &[0]).bytes; + assert_same(&bytes, W as usize * 3 * 3, &format!("orientation {o}")); + let t = TiffBands::open(&bytes, None, &Limits::banded()).unwrap(); + assert_eq!(t.orientation(), o as u16); + let up = t.upright(); + let swapped = o >= 5; + assert_eq!(up.dimensions(), if swapped { (H, W) } else { (W, H) }, "orientation {o}"); + assert_eq!(up.meta.dpi, Some(if swapped { (150.0, 300.0) } else { (300.0, 150.0) }), "orientation {o}"); + } +} + +#[test] +fn cut_off_data_matches_and_warns() { + let n = (W * H) as usize; + let built = build(true, false, &[strips(W, H, 8, 3, 2, 4, &noise(n * 3, 30))], &[0]); + // The last four of eight strips point past the end of the file: their data is missing. + let mut built = built; + let past = built.bytes.len() as u64 + 4096; + for i in 4..8 { + patch_value(&mut built, true, false, 0, 273, i, past); + } + assert_same(&built.bytes, W as usize * 3 * 4, "cut off"); + let img = banded(&built.bytes, None, &Limits::banded(), true).unwrap(); + assert!(img.warnings.contains(&DecodeWarning::Truncated { format: Format::Tiff })); +} + +#[test] +fn a_strip_cut_off_partway_decodes_as_zeros() { + // An uncompressed strip read straight into its output rows, whose data starts in the file + // and ends past it: the bytes read before the end must not survive (#8536 review). + let n = (W * H) as usize; + let mut built = build(true, false, &[strips(W, H, 8, 3, 2, 4, &noise(n * 3, 31))], &[0]); + let strip = W as usize * 3 * 4; + let near_end = built.bytes.len() as u64 - 100; + patch_value(&mut built, true, false, 0, 273, 6, near_end); + let img = banded(&built.bytes, None, &Limits::banded(), false).unwrap(); + assert!(img.warnings.contains(&DecodeWarning::Truncated { format: Format::Tiff })); + assert!(img.data()[6 * strip..7 * strip].iter().all(|&v| v == 0), "partial strip bytes kept"); + assert!(img.data()[..6 * strip] == whole(&built.bytes, None, false).data()[..6 * strip]); +} + +#[test] +fn other_pages_and_the_more_pages_warning_match() { + let n = (W * H) as usize; + let a = strips(W, H, 8, 1, 1, 5, &noise(n, 40)); + let b = strips(W, H, 8, 3, 2, 5, &noise(n * 3, 41)); + let bytes = build(true, false, &[a, b], &[0, 1]).bytes; + for page in [None, Some(0), Some(1)] { + let want = whole(&bytes, page, false); + let got = banded(&bytes, page, &Limits::banded(), false).unwrap(); + assert!(got.data() == want.data(), "page {page:?}"); + assert_eq!(got.warnings, want.warnings, "page {page:?}"); + } + assert!(matches!(TiffBands::open(&bytes, Some(9), &Limits::banded()), Err(CodecError::InvalidImage(_)))); +} + +#[test] +fn a_cut_off_page_of_several_warns_in_the_whole_image_order() { + // Truncated before MorePages, as the whole-image decode reports them (#8536 review). + let n = (W * H) as usize; + let a = strips(W, H, 8, 3, 2, 4, &noise(n * 3, 42)); + let b = strips(W, H, 8, 1, 1, 5, &noise(n, 43)); + let mut built = build(true, false, &[a, b], &[0, 1]); + let past = built.bytes.len() as u64 + 4096; + for i in 4..8 { + patch_value(&mut built, true, false, 0, 273, i, past); + } + let want = whole(&built.bytes, None, false); + assert!(matches!(want.warnings[..], [DecodeWarning::Truncated { .. }, DecodeWarning::MorePages { .. }]), "{:?}", want.warnings); + let got = banded(&built.bytes, None, &Limits::banded(), false).unwrap(); + assert_eq!(got.warnings, want.warnings); +} + +#[test] +fn group_bytes_is_the_largest_group_handed_over() { + let n = (W * H) as usize; + // Eight strips of four rows, two to a group; one strip holding the whole image. + let bytes = build(true, false, &[strips(W, H, 8, 3, 2, 4, &noise(n * 3, 44))], &[0]).bytes; + let band = W as usize * 3 * 4; + let mut t = TiffBands::open(&bytes, None, &grouped(band, 2)).unwrap(); + assert_eq!(t.group_bytes().unwrap(), 2 * band); + let mut largest = 0; + t.decode(|_, _, g: &Image| -> Result<(), CodecError> { + largest = largest.max(g.data().len()); + Ok(()) + }) + .unwrap(); + assert_eq!(largest, 2 * band); + let single = build(true, false, &[strips(W, H, 8, 3, 2, H, &noise(n * 3, 45))], &[0]).bytes; + let t = TiffBands::open(&single, None, &Limits::banded()).unwrap(); + assert_eq!(t.group_bytes().unwrap(), n * 3, "a single strip is one group: the whole image"); +} + +#[test] +#[cfg(target_pointer_width = "64")] +fn opens_an_image_over_the_whole_image_budget() { + // The whole-image decoder needs the image in one buffer; the banded one only a band (on + // 64-bit targets: 32-bit ones still check the whole image). + let n = (W * H) as usize; + let bytes = build(true, false, &[strips(W, H, 8, 3, 2, 2, &noise(n * 3, 50))], &[0]).bytes; + let band = W as u64 * 3 * 2; + let tight = Limits { max_alloc: band * 3, ..Limits::banded() }; + let refused = decode_with(&bytes, &DecodeOptions { limits: tight, keep_orientation: false }); + assert!(matches!(refused, Err(CodecError::LimitExceeded(_))), "{refused:?}"); + let got = banded(&bytes, None, &tight, true).unwrap(); + assert!(got.data() == whole(&bytes, None, true).data()); +} + +#[test] +fn size_limits_still_apply() { + let n = (W * H) as usize; + let bytes = build(true, false, &[strips(W, H, 8, 3, 2, 4, &noise(n * 3, 60))], &[0]).bytes; + let few_pixels = Limits { max_pixels: u64::from(W * H) - 1, ..Limits::banded() }; + assert!(matches!(TiffBands::open(&bytes, None, &few_pixels), Err(CodecError::LimitExceeded(_)))); + let narrow = Limits { max_width: W - 1, ..Limits::banded() }; + assert!(matches!(TiffBands::open(&bytes, None, &narrow), Err(CodecError::LimitExceeded(_)))); + // One strip is more than `max_alloc`: refused before decoding, with the image size. + let tiny = Limits { max_alloc: 10, ..Limits::banded() }; + match TiffBands::open(&bytes, None, &tiny) { + Err(CodecError::LimitExceeded(m)) => assert!(m.contains(&format!("{W}x{H}")), "{m}"), + other => panic!("{:?}", other.map(|t| t.dimensions())), + } +} + +#[test] +fn banded_limits_allow_the_largest_document_size() { + let l = Limits::banded(); + if cfg!(target_pointer_width = "64") { + assert_eq!((l.max_width, l.max_height), (300_000, 300_000)); + assert!(l.max_pixels >= 300_000 * 300_000); + // The banner from #8456 / #8169: 136 ft × 26 ft at 72 ppi. + assert!(117_504u64 * 22_464 > Limits::default().max_pixels); + assert!(117_504u64 * 22_464 <= l.max_pixels); + } else { + assert_eq!(l, Limits::default()); + } + assert_eq!(l.max_alloc, Limits::default().max_alloc); +} + +#[test] +fn a_hostile_header_is_refused_before_allocating() { + // 300000² declared in a file of a few kilobytes: one 270 GB strip, which neither the + // decompression-bomb guard nor the per-group budget lets through. + let mut built = build(true, true, &[strips(1000, 4, 8, 3, 2, 4, &noise(12_000, 70))], &[0]); + for tag in [256, 257, 278] { + patch_value(&mut built, true, true, 0, tag, 0, 300_000); + } + assert!(matches!(TiffBands::open(&built.bytes, None, &Limits::banded()), Err(CodecError::Malformed { .. } | CodecError::LimitExceeded(_)))); + // Past the 300000 px a side documents allow. + let mut wide = build(true, true, &[strips(1000, 4, 8, 3, 2, 4, &noise(12_000, 71))], &[0]); + patch_value(&mut wide, true, true, 0, 256, 0, 300_001); + assert!(matches!(TiffBands::open(&wide.bytes, None, &Limits::banded()), Err(CodecError::LimitExceeded(_)))); +} + +#[test] +fn padding_does_not_let_missing_chunks_past_the_bomb_guard() { + // 20000² RGB declared as Deflate (1.2 GB of samples), every strip past the end of the file, + // and 400 KB of padding: the file is large enough for that size at Deflate's ratio, but its + // strips hold nothing. Refused before anything is allocated, by both decoders (#8536 review). + let (w, h) = (20_000, 20_000); + let mut dir = Dir { entries: base(w, h, 8, 3, 2), chunks: vec![vec![0]; 20], tiled: false }.tag(278, LONG, &[1000]); + dir.entries.retain(|e| e.0 != 259); + let dir = dir.tag(259, SHORT, &[8]).bytes(65_000, UNDEFINED, &vec![0; 400_000]); + let mut built = build(true, false, &[dir], &[0]); + let past = built.bytes.len() as u64 + 4096; + for i in 0..20 { + patch_value(&mut built, true, false, 0, 273, i, past); + } + match TiffBands::open(&built.bytes, None, &Limits::banded()) { + Err(CodecError::Malformed { message, .. }) => assert!(message.contains("hold 0 bytes"), "{message}"), + other => panic!("{:?}", other.map(|t| t.dimensions())), + } + assert!(matches!(decode(&built.bytes), Err(CodecError::Malformed { .. }))); +} + +/// An all-white CCITT Group 4 strip: one V0 code (a 1 bit) per row, then the EOFB. +fn blank_g4(h: u32) -> Vec { + let mut bits = vec![true; h as usize]; + for _ in 0..2 { + bits.extend([false; 11]); + bits.push(true); + } + bits.chunks(8).map(|c| c.iter().enumerate().fold(0u8, |a, (i, &b)| a | (u8::from(b) << (7 - i)))).collect() +} + +#[test] +fn a_blank_fax_page_wider_than_4096_px_passes_the_bomb_guard() { + // A blank 600 dpi page (A4 landscape, A3) codes each row in one bit, past 4096:1: it must + // open, with or without the metadata and other pages that used to raise the bound. + for (w, h) in [(7016u32, 4961u32), (7016, 9921)] { + let mut d = Dir { entries: base(w, h, 1, 1, 0), chunks: vec![blank_g4(h)], tiled: false }.tag(278, LONG, &[u64::from(h)]); + d.entries.retain(|e| e.0 != 259); + let d = d.tag(259, SHORT, &[4]); + let pages: Vec

= (0..3).map(|_| d.clone()).collect(); + for (what, bytes) in + [("one page", build(true, false, std::slice::from_ref(&d), &[0]).bytes), ("three pages", build(true, false, &pages, &[0, 1, 2]).bytes)] + { + let img = decode(&bytes).unwrap_or_else(|e| panic!("{w}x{h} {what}: {e}")); + assert_eq!(img.dimensions(), (w, h), "{what}"); + assert!(img.data().iter().all(|&v| v == 255), "{w}x{h} {what}: not white"); + } + } + let mut d = Dir { entries: base(7016, 4961, 1, 1, 0), chunks: vec![blank_g4(4961)], tiled: false }.tag(278, LONG, &[4961]); + d.entries.retain(|e| e.0 != 259); + let bytes = build(true, false, &[d.tag(259, SHORT, &[4])], &[0]).bytes; + assert!(banded(&bytes, None, &Limits::banded(), true).unwrap().data() == decode(&bytes).unwrap().data()); + // Still a bound: 100 bytes of Group 4 can't hold that page (4096:1 for Group 3, which is not decoded). + for compression in [4, 3] { + let mut d = Dir { entries: base(7016, 4961, 1, 1, 0), chunks: vec![vec![0x80; 100]], tiled: false }.tag(278, LONG, &[4961]); + d.entries.retain(|e| e.0 != 259); + let bytes = build(true, false, &[d.tag(259, SHORT, &[compression])], &[0]).bytes; + match decode(&bytes) { + Err(CodecError::Malformed { message, .. }) => assert!(message.contains("hold 100 bytes"), "{message}"), + other => panic!("compression {compression}: {:?}", other.map(|i| i.dimensions())), + } + } +} diff --git a/crates/image-codecs/tests/it/tiff_bench.rs b/crates/image-codecs/tests/it/tiff_bench.rs new file mode 100644 index 000000000..99ed4f5d6 --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_bench.rs @@ -0,0 +1,126 @@ +//! TIFF decode timings on a synthetic ~30 MP image (6336×4752). Ignored by default; run in +//! release: +//! +//! ```sh +//! cargo test --release -p craft-image-codecs --test it -- tiff_bench:: --ignored --nocapture +//! ``` + +use crate::common; +use common::tiffgen::*; +use craft_image_codecs::*; +use std::time::Instant; + +const W: u32 = 6336; +const H: u32 = 4752; + +fn pixels(spp: usize) -> Vec { + // A smooth gradient with a little texture: compresses like a photo, not like noise. + let mut v = Vec::with_capacity(W as usize * H as usize * spp); + let mut s = 0x1234_5678u32; + for y in 0..H { + for x in 0..W { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + for c in 0..spp { + let g = (x * (c as u32 + 1) / 25 + y / 19) as u8; + v.push(g.wrapping_add((s >> (8 * c)) as u8 & 3)); + } + } + } + v +} + +/// The decoder before FILE-215-8, for a same-run comparison: the `tiff` crate decodes the whole +/// image into one buffer (`read_image`, first plane only), which is then copied into an `Image`. +fn legacy(bytes: &[u8]) -> Result { + use tiff::decoder::{Decoder, DecodingResult, Limits as TLimits}; + let mut l = TLimits::default(); + l.decoding_buffer_size = 8 << 30; + l.intermediate_buffer_size = 1 << 30; + let mut dec = Decoder::new(std::io::Cursor::new(bytes)).map_err(|e| e.to_string())?.with_limits(l); + let (w, h) = dec.dimensions().map_err(|e| e.to_string())?; + let layout = match dec.colortype().map_err(|e| e.to_string())? { + tiff::ColorType::RGB(_) => ChannelLayout::Rgb, + tiff::ColorType::Gray(_) => ChannelLayout::Gray, + c => return Err(format!("{c:?}")), + }; + match dec.read_image().map_err(|e| e.to_string())? { + DecodingResult::U8(v) => Image::from_u8(w, h, layout, v), + DecodingResult::U16(v) => Image::from_u16(w, h, layout, &v), + _ => return Err("sample type".into()), + } + .map_err(|e| e.to_string()) +} + +fn best_of(f: impl Fn() -> Result) -> Result { + let mut best = f64::MAX; + for _ in 0..7 { + let t = Instant::now(); + f()?; + best = best.min(t.elapsed().as_secs_f64() * 1000.0); + } + Ok(best) +} + +fn time(name: &str, bytes: &[u8]) { + let new = best_of(|| decode(bytes).map_err(|e| e.to_string())); + let old = best_of(|| legacy(bytes)); + let show = |r: &Result| match r { + Ok(ms) => format!("{ms:>7.1} ms"), + Err(e) => format!("error ({})", e.chars().take(48).collect::()), + }; + println!("{name:<34} new {} before {} ({:.1} MB file)", show(&new), show(&old), bytes.len() as f64 / 1e6); +} + +#[test] +#[ignore = "benchmark: run in release with --ignored --nocapture"] +fn decode_30mp() { + let rgb = pixels(3); + let img = Image::from_u8(W, H, ChannelLayout::Rgb, rgb.clone()).unwrap(); + for (name, c) in [ + ("strips, none", TiffCompression::None), + ("strips, LZW + predictor", TiffCompression::Lzw), + ("strips, Deflate + predictor", TiffCompression::Deflate), + ] { + let bytes = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: c, ..Default::default() }).unwrap(); + time(&format!("RGB 8 {name}"), &bytes); + } + let img16 = img.convert(ChannelLayout::Rgb, SampleType::U16); + let bytes = encode(&img16, Format::Tiff, &EncodeOptions { tiff_compression: TiffCompression::Deflate, ..Default::default() }).unwrap(); + time("RGB 16 strips, Deflate + predictor", &bytes); + let bytes = encode(&img16, Format::Tiff, &EncodeOptions { tiff_compression: TiffCompression::None, ..Default::default() }).unwrap(); + time("RGB 16 strips, none", &bytes); + + // Hand-built layouts the encoder does not write: one big strip, tiles, planar. + let le = cfg!(target_endian = "little"); + let one = build(le, false, &[strips(W, H, 8, 3, 2, H, &rgb)], &[0]); + time("RGB 8 one strip, none", &one.bytes); + let tiles = tile_split(&rgb, W as usize, H as usize, 3, 1, 256, 256, false); + let mut d = Dir { entries: base(W, H, 8, 3, 2), chunks: tiles, tiled: true }; + d = d.tag(322, LONG, &[256]).tag(323, LONG, &[256]); + time("RGB 8 tiles 256, none", &build(le, false, &[d], &[0]).bytes); + let planes = planar_strips(&rgb, W as usize, H as usize, 3, 1, 64); + let mut d = Dir { entries: base(W, H, 8, 3, 2), chunks: planes, tiled: false }; + d = d.tag(278, LONG, &[64]).tag(284, SHORT, &[2]); + time("RGB 8 planar strips, none", &build(le, false, &[d], &[0]).bytes); + let d = strips(W, H, 8, 3, 2, 64, &rgb); + time("RGB 8 BigTIFF strips, none", &build(le, true, &[d], &[0]).bytes); +} + +/// Save Image Pyramid's cost on top of a plain encode (#8827): RGB/8 and RGBA/16. +#[test] +#[ignore = "benchmark: run in release with --ignored --nocapture"] +fn encode_with_pyramid() { + for (layout, sample, spp) in [(ChannelLayout::Rgb, SampleType::U8, 3), (ChannelLayout::Rgba, SampleType::U16, 4)] { + let raw = pixels(spp); + let data = if sample == SampleType::U16 { raw.iter().flat_map(|&v| (u16::from(v) * 257).to_ne_bytes()).collect() } else { raw }; + let img = Image::from_raw(W, H, layout, sample, data).unwrap(); + for pyramid in [false, true] { + let opts = EncodeOptions { tiff_pyramid: pyramid, ..Default::default() }; + let t = Instant::now(); + let bytes = encode(&img, Format::Tiff, &opts).unwrap(); + println!("{layout:?} {sample:?} pyramid {pyramid}: {:.0} ms, {} MB", t.elapsed().as_secs_f64() * 1000.0, bytes.len() >> 20); + } + } +} diff --git a/crates/image-codecs/tests/it/tiff_exif.rs b/crates/image-codecs/tests/it/tiff_exif.rs new file mode 100644 index 000000000..6c08c091f --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_exif.rs @@ -0,0 +1,271 @@ +//! EXIF in TIFF: an EXIF block's Exif and GPS directories and IFD0's descriptive tags are +//! written as the TIFF's own directories (IFD0 pointing to them, as cameras and Photoshop write +//! them) and read back into a block. + +use craft_image_codecs::*; + +/// A classic TIFF stream builder for EXIF blocks: directories of (tag, type, count, value bytes). +struct Block { + le: bool, +} + +type Entry = (u16, u16, u32, Vec); + +impl Block { + fn u16(&self, v: u16) -> [u8; 2] { + if self.le { v.to_le_bytes() } else { v.to_be_bytes() } + } + fn u32(&self, v: u32) -> [u8; 4] { + if self.le { v.to_le_bytes() } else { v.to_be_bytes() } + } + fn rational(&self, n: u32, d: u32) -> Vec { + [self.u32(n), self.u32(d)].concat() + } + /// IFD0 (with pointers to `exif` and `gps` appended) followed by the two directories. + fn build(&self, mut ifd0: Vec, exif: Vec, gps: Vec) -> Vec { + let size = |d: &[Entry]| 2 + 12 * d.len() + 4 + d.iter().filter(|e| e.3.len() > 4).map(|e| e.3.len().next_multiple_of(2)).sum::(); + ifd0.push((34665, 4, 1, vec![0; 4])); + ifd0.push((34853, 4, 1, vec![0; 4])); + let at_exif = 8 + size(&ifd0); + let at_gps = at_exif + size(&exif); + let n = ifd0.len(); + ifd0[n - 2].3 = self.u32(at_exif as u32).to_vec(); + ifd0[n - 1].3 = self.u32(at_gps as u32).to_vec(); + let mut out = if self.le { b"II".to_vec() } else { b"MM".to_vec() }; + out.extend(self.u16(42)); + out.extend(self.u32(8)); + for d in [ifd0, exif, gps] { + let start = out.len(); + let mut extra = start + 2 + 12 * d.len() + 4; + let mut values = Vec::new(); + out.extend(self.u16(d.len() as u16)); + for (tag, ty, count, v) in &d { + out.extend(self.u16(*tag)); + out.extend(self.u16(*ty)); + out.extend(self.u32(*count)); + if v.len() <= 4 { + let mut inline = v.clone(); + inline.resize(4, 0); + out.extend(inline); + } else { + out.extend(self.u32(extra as u32)); + values.extend(v); + if v.len() % 2 == 1 { + values.push(0); + } + extra += v.len().next_multiple_of(2); + } + } + out.extend(self.u32(0)); + out.extend(values); + } + out + } +} + +/// The entries (tag, type, count, value bytes in the block's order) of IFD0 and of the +/// directories its tags `34665` and `34853` point to. +fn dirs(b: &[u8]) -> (bool, Vec, Vec, Vec) { + let le = &b[..2] == b"II"; + let u16 = |at: usize| if le { u16::from_le_bytes([b[at], b[at + 1]]) } else { u16::from_be_bytes([b[at], b[at + 1]]) }; + let u32 = |at: usize| { + let s = [b[at], b[at + 1], b[at + 2], b[at + 3]]; + if le { u32::from_le_bytes(s) } else { u32::from_be_bytes(s) } + }; + let size = |ty: u16| match ty { + 3 | 8 => 2, + 4 | 9 | 11 | 13 => 4, + 5 | 10 | 12 => 8, + _ => 1, + }; + let read = |at: usize| -> Vec { + (0..usize::from(u16(at))) + .map(|i| { + let e = at + 2 + 12 * i; + let (tag, ty, count) = (u16(e), u16(e + 2), u32(e + 4)); + let len = count as usize * size(ty); + let at = if len <= 4 { e + 8 } else { u32(e + 8) as usize }; + (tag, ty, count, b[at..at + len].to_vec()) + }) + .collect() + }; + let ifd0 = read(u32(4) as usize); + let sub = |tag: u16| ifd0.iter().find(|e| e.0 == tag).map(|e| read(u32_of(&e.3, le) as usize)).unwrap_or_default(); + let (exif, gps) = (sub(34665), sub(34853)); + (le, ifd0.into_iter().filter(|e| !matches!(e.0, 34665 | 34853)).collect(), exif, gps) +} + +fn u32_of(v: &[u8], le: bool) -> u32 { + let s = [v[0], v[1], v[2], v[3]]; + if le { u32::from_le_bytes(s) } else { u32::from_be_bytes(s) } +} + +/// The entries of `d` with their values in little-endian order, sorted by tag. +fn normalized(d: &[Entry], le: bool) -> Vec<(u16, u16, u32, Vec)> { + let swap = |ty: u16, v: &[u8]| -> Vec { + let n = match ty { + 3 | 8 => 2, + 4 | 5 | 9 | 10 | 11 | 13 => 4, + 12 => 8, + _ => 1, + }; + if le || n == 1 { v.to_vec() } else { v.chunks(n).flat_map(|c| c.iter().rev().copied()).collect() } + }; + let mut v: Vec<_> = d.iter().map(|e| (e.0, e.1, e.2, swap(e.1, &e.3))).collect(); + v.sort_by_key(|e| e.0); + v +} + +fn camera_block(le: bool) -> Vec { + let b = Block { le }; + let ascii = |s: &str| [s.as_bytes(), b"\0"].concat(); + b.build( + vec![(271, 2, 4, ascii("Cam")), (272, 2, 9, ascii("Model X1")), (315, 2, 16, ascii("A. Photographer")), (274, 3, 1, b.u16(1).to_vec())], + vec![ + (33434, 5, 1, b.rational(1, 125)), + (34855, 3, 1, b.u16(200).to_vec()), + (36867, 2, 20, ascii("2026:10:13 09:30:00")), + (37386, 5, 1, b.rational(50, 1)), + (37500, 7, 6, b"maker!".to_vec()), + ], + vec![(1, 2, 2, ascii("N")), (2, 5, 3, [b.rational(51, 1), b.rational(30, 1), b.rational(0, 1)].concat())], + ) +} + +#[test] +fn exif_round_trips_through_a_tiff_in_either_byte_order() { + for le in [true, false] { + let block = camera_block(le); + let mut img = Image::from_raw(4, 3, ChannelLayout::Rgb, SampleType::U8, vec![90; 36]).unwrap(); + img.meta.exif = Some(block.clone()); + let tiff = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert!(fidelity_warnings(&img, Format::Tiff).iter().all(|w| *w != FidelityWarning::ExifDropped)); + let back = decode(&tiff).unwrap().meta.exif.expect("EXIF read back"); + let (ble, b0, bexif, bgps) = dirs(&back); + let (ole, o0, oexif, ogps) = dirs(&block); + assert_eq!(normalized(&bexif, ble), normalized(&oexif, ole), "Exif directory (le {le})"); + assert_eq!(normalized(&bgps, ble), normalized(&ogps, ole), "GPS directory (le {le})"); + // IFD0's descriptive tags come along; its orientation is the image's own. + let descriptive: Vec<_> = o0.into_iter().filter(|e| e.0 != 274).collect(); + assert_eq!(normalized(&b0, ble), normalized(&descriptive, ole), "IFD0 (le {le})"); + } +} + +#[test] +fn a_tiff_without_exif_directories_reads_no_exif() { + let mut img = Image::from_raw(2, 2, ChannelLayout::Gray, SampleType::U8, vec![0; 4]).unwrap(); + img.meta.text = vec![("Artist".into(), "A. Photographer".into())]; + let tiff = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert_eq!(decode(&tiff).unwrap().meta.exif, None); +} + +#[test] +fn no_exif_is_written_without_embed_metadata() { + let mut img = Image::from_raw(2, 2, ChannelLayout::Gray, SampleType::U8, vec![0; 4]).unwrap(); + img.meta.exif = Some(camera_block(true)); + let tiff = encode(&img, Format::Tiff, &EncodeOptions { embed_metadata: false, ..Default::default() }).unwrap(); + assert_eq!(decode(&tiff).unwrap().meta.exif, None); +} + +// --------------------------------------------------------------------------- +// Bounded copying: an entry's value only has to lie inside the file, so a hostile file can have +// many entries point at the same bytes. +// --------------------------------------------------------------------------- + +/// A valid 1x1 gray TIFF whose Exif directory has `entries` UNDEFINED entries of `count` bytes +/// that all point at the same `count` bytes: the file is about `count` bytes long and asks to +/// copy `entries * count`. +fn overlapping_tiff(entries: usize, count: u32) -> Vec { + let entry = |tag: u16, ty: u16, count: u32, value: u32| { + [tag.to_le_bytes().to_vec(), ty.to_le_bytes().to_vec(), count.to_le_bytes().to_vec(), value.to_le_bytes().to_vec()].concat() + }; + let at_exif = 8 + 2 + 12 * 10 + 4; + let data_at = at_exif + 2 + 12 * entries + 4; + let mut out = b"II".to_vec(); + out.extend(42u16.to_le_bytes()); + out.extend(8u32.to_le_bytes()); + out.extend(10u16.to_le_bytes()); + for (tag, ty, value) in [ + (256, 3, 1), + (257, 3, 1), + (258, 3, 8), + (259, 3, 1), + (262, 3, 1), + (273, 4, 0), + (277, 3, 1), + (278, 3, 1), + (279, 4, 1), + (34665, 4, at_exif as u32), + ] { + out.extend(entry(tag, ty, 1, value)); + } + out.extend(0u32.to_le_bytes()); + out.extend((entries as u16).to_le_bytes()); + for i in 0..entries { + out.extend(entry(i as u16 + 1, 7, count, data_at as u32)); + } + out.extend(0u32.to_le_bytes()); + out.extend(vec![0x5A; count as usize]); + out +} + +#[test] +fn overlapping_field_values_are_bounded() { + // About 62 KB on disk; copied entry by entry it would be 1,000 x 50,000 bytes. + let tiff = overlapping_tiff(1000, 50_000); + assert!(tiff.len() < 64 * 1024); + let img = decode(&tiff).expect("the image itself is fine"); + assert_eq!(img.dimensions(), (1, 1)); + let exif = img.meta.exif.expect("the first entries are kept"); + assert!(exif.len() <= 2 * tiff.len(), "{} bytes of EXIF from a {} byte file", exif.len(), tiff.len()); +} + +#[test] +fn overlapping_field_values_in_an_exif_block_are_bounded_on_export() { + // The same block as an image's EXIF: the writer reads it with the same code. + let block = overlapping_tiff(1000, 50_000); + let mut img = Image::from_raw(2, 2, ChannelLayout::Gray, SampleType::U8, vec![0; 4]).unwrap(); + img.meta.exif = Some(block.clone()); + let tiff = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert!(tiff.len() <= 2 * block.len(), "a {} byte block wrote a {} byte file", block.len(), tiff.len()); +} + +/// An EXIF block with a maker note of `len` bytes, an exposure time and a GPS latitude reference. +fn block_with_maker_note(le: bool, len: usize) -> Vec { + let b = Block { le }; + b.build( + vec![(271, 2, 4, b"Cam\0".to_vec())], + vec![(33434, 5, 1, b.rational(1, 125)), (37500, 7, len as u32, (0..len).map(|i| (i * 7) as u8).collect())], + vec![(1, 2, 2, b"N\0".to_vec())], + ) +} + +#[test] +fn large_maker_notes_still_survive() { + // Nikon maker notes run from 100 KB to nearly 1 MB. + for len in [200 * 1024, 1000 * 1024] { + let block = block_with_maker_note(true, len); + let mut img = Image::from_raw(4, 3, ChannelLayout::Rgb, SampleType::U8, vec![90; 36]).unwrap(); + img.meta.exif = Some(block.clone()); + let tiff = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + let back = decode(&tiff).unwrap().meta.exif.expect("EXIF read back"); + let (ble, _, bexif, bgps) = dirs(&back); + let (ole, _, oexif, ogps) = dirs(&block); + assert_eq!(normalized(&bexif, ble), normalized(&oexif, ole), "{len} byte maker note"); + assert_eq!(normalized(&bgps, ble), normalized(&ogps, ole)); + } +} + +#[test] +fn a_field_beyond_the_cap_is_dropped_and_the_rest_kept() { + let block = block_with_maker_note(false, 6 * 1024 * 1024); + let mut img = Image::from_raw(2, 2, ChannelLayout::Gray, SampleType::U8, vec![0; 4]).unwrap(); + img.meta.exif = Some(block); + let tiff = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert!(tiff.len() < 1024 * 1024, "{} bytes", tiff.len()); + let back = decode(&tiff).unwrap().meta.exif.expect("EXIF read back"); + let (_, ifd0, exif, gps) = dirs(&back); + assert_eq!(exif.iter().map(|e| e.0).collect::>(), [33434], "the exposure time stays, the 6 MiB maker note goes"); + assert_eq!(gps.iter().map(|e| e.0).collect::>(), [1]); + assert_eq!(ifd0.iter().map(|e| e.0).collect::>(), [271]); +} diff --git a/crates/image-codecs/tests/it/tiff_lab.rs b/crates/image-codecs/tests/it/tiff_lab.rs new file mode 100644 index 000000000..b36435ef4 --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_lab.rs @@ -0,0 +1,109 @@ +//! CIE L*a*b* TIFFs: CIELab (PhotometricInterpretation 8) as Photoshop writes it, with a* and b* +//! signed, and ICCLab (9) with them offset, both read into the Lab layout (a* and b* as +//! `(a + 128) / 255`). + +use craft_image_codecs::*; + +/// One Lab pixel in the codec's normalized form. +fn lab(l: f32, a: f32, b: f32) -> [f32; 3] { + [l / 100.0, (a + 128.0) / 255.0, (b + 128.0) / 255.0] +} + +fn image(sample: SampleType) -> Image { + let px: Vec = [lab(50.0, 50.0, -30.0), lab(100.0, 0.0, 0.0)].concat(); + Image::from_normalized(2, 1, ChannelLayout::Lab, sample, &px).unwrap() +} + +fn uncompressed(img: &Image) -> Vec { + encode(img, Format::Tiff, &EncodeOptions { tiff_compression: TiffCompression::None, ..Default::default() }).unwrap() +} + +fn find(hay: &[u8], needle: &[u8]) -> Option { + hay.windows(needle.len()).position(|w| w == needle) +} + +#[test] +fn eight_bit_cielab_holds_a_and_b_signed() { + let file = uncompressed(&image(SampleType::U8)); + // L* 50 → 128, a* 50, b* −30 as a signed byte; then white with neutral a* and b*. + assert!(find(&file, &[128, 50, (-30i8) as u8, 255, 0, 0]).is_some(), "CIELab samples"); + let back = decode(&file).unwrap(); + assert_eq!((back.layout(), back.sample_type()), (ChannelLayout::Lab, SampleType::U8)); + assert_eq!(back.data(), image(SampleType::U8).data()); +} + +#[test] +fn sixteen_bit_cielab_holds_256_a() { + let file = uncompressed(&image(SampleType::U16)); + let le = &file[..2] == b"II"; + let u16b = |v: u16| if le { v.to_le_bytes() } else { v.to_be_bytes() }; + // a* 50 → 12800, b* −30 → −7680, white's a* and b* 0. + let want: Vec = [32768u16, 12800, (-7680i16) as u16, 65535, 0, 0].iter().flat_map(|v| u16b(*v)).collect(); + assert!(find(&file, &want).is_some(), "CIELab samples"); + let back = decode(&file).unwrap(); + assert_eq!((back.layout(), back.sample_type()), (ChannelLayout::Lab, SampleType::U16)); + let (a, b) = (image(SampleType::U16).to_normalized(), back.to_normalized()); + assert!(a.iter().zip(&b).all(|(x, y)| (x - y).abs() <= 1.5 / 65535.0), "{a:?} {b:?}"); +} + +#[test] +fn icclab_reads_with_offset_a_and_b() { + // The CIELab file turned into ICCLab: PhotometricInterpretation 9, a* and b* + 128. + let mut file = uncompressed(&image(SampleType::U8)); + let at = find(&file, &[128, 50, (-30i8) as u8, 255, 0, 0]).unwrap(); + for i in [1, 2, 4, 5] { + file[at + i] ^= 0x80; + } + let le = &file[..2] == b"II"; + let entry: Vec = if le { [262u16.to_le_bytes(), 3u16.to_le_bytes()].concat() } else { [262u16.to_be_bytes(), 3u16.to_be_bytes()].concat() }; + let e = find(&file, &entry).expect("PhotometricInterpretation entry"); + let nine = if le { 9u16.to_le_bytes() } else { 9u16.to_be_bytes() }; + file[e + 8..e + 10].copy_from_slice(&nine); + let back = decode(&file).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Lab); + assert_eq!(back.data(), image(SampleType::U8).data()); +} + +#[test] +fn lab_converts_to_rgb_for_other_formats() { + // White and a saturated orange-red; PNG has no Lab, so the encoder plan converts to RGB. + let img = image(SampleType::U8); + let png = decode(&encode(&img, Format::Png, &EncodeOptions::default()).unwrap()).unwrap(); + assert!(png.layout().is_rgb()); + let p = png.to_rgba8(); + assert_eq!(&p[4..7], &[255, 255, 255], "white"); + assert!(p[0] > 180 && p[1] < 120 && p[2] > 140, "L 50, a 50, b −30 is a magenta-ish purple: {:?}", &p[..3]); +} + +#[test] +fn lab_keeps_extra_samples() { + // A spot plate next to Lab: written as CIELab plus one unspecified extra sample, and read + // back as Lab with the spot (not as LabA). + for (layout, sample) in [(ChannelLayout::Lab, SampleType::U8), (ChannelLayout::LabA, SampleType::U16)] { + let px: Vec = + [lab(50.0, 50.0, -30.0), lab(100.0, 0.0, 0.0)].iter().flat_map(|p| p.iter().copied().chain(layout.has_alpha().then_some(0.5))).collect(); + let mut img = Image::from_normalized(2, 1, layout, sample, &px).unwrap(); + let spot: Vec = (0..2 * sample.bytes()).map(|i| (i * 37 + 5) as u8).collect(); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: spot.clone() }]; + let file = uncompressed(&img); + if layout == ChannelLayout::Lab { + assert!(find(&file, &[128, 50, (-30i8) as u8, spot[0], 255, 0, 0, spot[1]]).is_some(), "CIELab samples with the spot"); + } + let back = decode(&file).unwrap(); + assert_eq!((back.layout(), back.sample_type()), (layout, sample)); + let (a, b) = (img.to_normalized(), back.to_normalized()); + assert!(a.iter().zip(&b).all(|(x, y)| (x - y).abs() <= 1.5 / 65535.0), "{layout:?}: {a:?} {b:?}"); + assert_eq!(back.meta.extra_samples, vec![ExtraSample { kind: 0, data: spot }], "{layout:?}"); + } +} + +#[test] +fn float_lab_writes_its_extra_samples_with_16_bits() { + let px: Vec = [lab(50.0, 50.0, -30.0), lab(100.0, 0.0, 0.0)].concat(); + let mut img = Image::from_normalized(2, 1, ChannelLayout::Lab, SampleType::F32, &px).unwrap(); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: [0.25f32, 1.0].iter().flat_map(|v| v.to_ne_bytes()).collect() }]; + let back = decode(&uncompressed(&img)).unwrap(); + assert_eq!((back.layout(), back.sample_type()), (ChannelLayout::Lab, SampleType::U16)); + let spot: Vec = [16384u16, 65535].iter().flat_map(|v| v.to_ne_bytes()).collect(); + assert_eq!(back.meta.extra_samples, vec![ExtraSample { kind: 0, data: spot }]); +} diff --git a/crates/image-codecs/tests/it/tiff_pages.rs b/crates/image-codecs/tests/it/tiff_pages.rs new file mode 100644 index 000000000..b94509598 --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_pages.rs @@ -0,0 +1,864 @@ +//! Large-file TIFF support (FILE-215-8): BigTIFF in both byte orders, every IFD of a multi-page +//! file and its SubIFDs, banded decoding of strips and tiles in either planar configuration, the +//! sample formats and interpretations around them, and hostile structures (cycles, counts and +//! offsets past the end, absurd sizes) that must give `Ok` or a clean `Err`, never a panic. +//! Every file here is hand-built by `common::tiffgen`. + +use crate::common; +use common::Rng; +use common::tiffgen::*; +use craft_image_codecs::*; + +const ORDERS: [(bool, bool); 4] = [(true, false), (false, false), (true, true), (false, true)]; + +fn ne16(v: &[u16]) -> Vec { + v.iter().flat_map(|x| x.to_ne_bytes()).collect() +} +fn ne32f(v: &[f32]) -> Vec { + v.iter().flat_map(|x| x.to_ne_bytes()).collect() +} + +/// A test image's samples in each storage: 8-bit, 16-bit and 32-bit float. +struct Truth { + u8s: Vec, + u16s: Vec, + f32s: Vec, +} + +fn truth(n: usize, seed: u64) -> Truth { + let mut r = Rng::new(seed); + let u16s: Vec = (0..n).map(|_| r.next_u64() as u16).collect(); + Truth { u8s: u16s.iter().map(|v| (v >> 8) as u8).collect(), f32s: u16s.iter().map(|&v| f32::from(v) / 65535.0).collect(), u16s } +} + +/// `(bits, sample format, file bytes in this order, expected native bytes, sample type)`. +type Depth = (u16, u16, Vec, Vec, SampleType); + +fn depths(t: &Truth, little: bool) -> Vec { + vec![ + (8, 1, t.u8s.clone(), t.u8s.clone(), SampleType::U8), + (16, 1, u16s(&t.u16s, little), ne16(&t.u16s), SampleType::U16), + (32, 3, f32s(&t.f32s, little), ne32f(&t.f32s), SampleType::F32), + ] +} + +fn decode_ok(bytes: &[u8]) -> Image { + decode(bytes).unwrap_or_else(|e| panic!("decode failed: {e}")) +} + +#[test] +fn bigtiff_and_classic_in_both_byte_orders_at_every_depth() { + let (w, h) = (7u32, 5u32); + for spp in [1u16, 3, 4] { + let t = truth((w * h) as usize * spp as usize, u64::from(spp)); + for (little, big) in ORDERS { + for (bits, format, file, want, sample) in depths(&t, little) { + let photometric = if spp == 1 { 1 } else { 2 }; + let mut d = strips(w, h, bits, spp, photometric, 2, &file).tag(339, SHORT, &vec![u64::from(format); spp as usize]); + if spp == 4 { + d = d.tag(338, SHORT, &[2]); + } + let b = build(little, big, &[d], &[0]); + let img = decode_ok(&b.bytes); + assert_eq!((img.dimensions(), img.sample_type()), ((w, h), sample), "little {little} big {big} bits {bits} spp {spp}"); + assert_eq!(img.layout().channels(), spp as usize); + assert_eq!(img.data(), &want[..], "little {little} big {big} bits {bits} spp {spp}"); + assert_eq!(img.warnings, []); + let info = tiff_info(&b.bytes).unwrap(); + assert_eq!((info.big_tiff, info.little_endian, info.pages.len(), info.complete), (big, little, 1, true)); + } + } + } +} + +#[test] +fn tiles_with_padding_in_both_planar_configurations() { + let (w, h) = (37usize, 29usize); + for spp in [1usize, 3, 4] { + let t = truth(w * h * spp, 9 + spp as u64); + for (little, big) in ORDERS { + for (bits, format, file, want, _) in depths(&t, little) { + let bps = bits as usize / 8; + for planar in [false, true] { + let mut d = Dir { + entries: base(w as u32, h as u32, bits, spp as u16, if spp == 1 { 1 } else { 2 }), + chunks: tile_split(&file, w, h, spp, bps, 16, 16, planar), + tiled: true, + }; + d = d.tag(322, SHORT, &[16]).tag(323, SHORT, &[16]).tag(339, SHORT, &[u64::from(format)]); + if planar { + d = d.tag(284, SHORT, &[2]); + } + if spp == 4 { + d = d.tag(338, SHORT, &[2]); + } + let b = build(little, big, &[d], &[0]); + let img = decode_ok(&b.bytes); + assert_eq!(img.data(), &want[..], "tiles: little {little} big {big} bits {bits} spp {spp} planar {planar}"); + let info = tiff_info(&b.bytes).unwrap(); + assert!(info.pages[0].tiled && info.pages[0].planar == planar); + } + } + } + } +} + +#[test] +fn planar_strips_at_every_depth() { + let (w, h) = (11usize, 9usize); + for spp in [3usize, 4] { + let t = truth(w * h * spp, 3); + for (little, big) in ORDERS { + for (bits, format, file, want, _) in depths(&t, little) { + for rows in [1usize, 4, 9] { + let mut d = Dir { + entries: base(w as u32, h as u32, bits, spp as u16, 2), + chunks: planar_strips(&file, w, h, spp, bits as usize / 8, rows), + tiled: false, + }; + d = d.tag(278, LONG, &[rows as u64]).tag(284, SHORT, &[2]).tag(339, SHORT, &[u64::from(format)]); + if spp == 4 { + d = d.tag(338, SHORT, &[2]); + } + let img = decode_ok(&build(little, big, &[d], &[0]).bytes); + assert_eq!(img.data(), &want[..], "planar strips: little {little} big {big} bits {bits} spp {spp} rows {rows}"); + } + } + } + } +} + +/// Extra samples that are not the alpha (ExtraSamples 0: a spot colour or an extra channel) are +/// not part of the image: they come back in `meta.extra_samples`, one plane each. +#[test] +fn extra_samples_that_are_not_alpha_are_kept_apart() { + let (w, h) = (5usize, 3usize); + let t = truth(w * h * 5, 4); + let rgb_of = |spp: usize, keep: usize| -> Vec { t.u8s[..w * h * spp].chunks(spp).flat_map(|p| p[..keep].to_vec()).collect() }; + let extra = |spp: usize, at: usize| vec![ExtraSample { kind: 0, data: t.u8s[..w * h * spp].chunks(spp).map(|p| p[at]).collect() }]; + for planar in [false, true] { + // RGB + one unspecified extra sample: RGB, and the sample as an extra plane. + let px = &t.u8s[..w * h * 4]; + let chunks = if planar { planar_strips(px, w, h, 4, 1, 2) } else { px.chunks(w * 4 * 2).map(<[u8]>::to_vec).collect() }; + let mut d = Dir { entries: base(w as u32, h as u32, 8, 4, 2), chunks, tiled: false }.tag(278, LONG, &[2]).tag(338, SHORT, &[0]); + if planar { + d = d.tag(284, SHORT, &[2]); + } + let img = decode_ok(&build(true, false, &[d], &[0]).bytes); + assert_eq!(img.layout(), ChannelLayout::Rgb); + assert_eq!(img.data(), &rgb_of(4, 3)[..], "planar {planar}"); + assert_eq!(img.meta.extra_samples, extra(4, 3), "planar {planar}"); + // RGB + alpha + an unspecified extra sample: RGBA and one extra plane. + let px = &t.u8s[..w * h * 5]; + let chunks = if planar { planar_strips(px, w, h, 5, 1, 2) } else { px.chunks(w * 5 * 2).map(<[u8]>::to_vec).collect() }; + let mut d = Dir { entries: base(w as u32, h as u32, 8, 5, 2), chunks, tiled: false }.tag(278, LONG, &[2]).tag(338, SHORT, &[2, 0]); + if planar { + d = d.tag(284, SHORT, &[2]); + } + let img = decode_ok(&build(false, true, &[d], &[0]).bytes); + assert_eq!(img.layout(), ChannelLayout::Rgba); + assert_eq!(img.data(), &rgb_of(5, 4)[..], "planar {planar}"); + assert_eq!(img.meta.extra_samples, extra(5, 4), "planar {planar}"); + } + // Gray + alpha + extra: gray and alpha; gray + an unspecified extra: gray. Each keeps the + // extra sample as a plane. + let px = &t.u8s[..w * h * 3]; + let d = strips(w as u32, h as u32, 8, 3, 1, 3, px).tag(338, SHORT, &[2, 0]); + let img = decode_ok(&build(true, false, &[d], &[0]).bytes); + assert_eq!(img.layout(), ChannelLayout::GrayA); + assert_eq!(img.data(), &rgb_of(3, 2)[..]); + assert_eq!(img.meta.extra_samples, extra(3, 2)); + let px = &t.u8s[..w * h * 2]; + let d = strips(w as u32, h as u32, 8, 2, 1, 3, px).tag(338, SHORT, &[0]); + let img = decode_ok(&build(true, false, &[d], &[0]).bytes); + assert_eq!(img.layout(), ChannelLayout::Gray); + assert_eq!(img.data(), &rgb_of(2, 1)[..]); + assert_eq!(img.meta.extra_samples, extra(2, 1)); +} + +/// A file may hold more extra samples than the `MAX_EXTRA_SAMPLES` a decode keeps (up to 64 +/// samples a pixel open): the first ones open, and a warning counts those left out, so nothing +/// is lost without notice. +#[test] +fn extra_samples_past_the_cap_are_reported() { + let (w, h) = (3usize, 2usize); + const EXTRAS: usize = MAX_EXTRA_SAMPLES + 4; + let spp = 3 + EXTRAS; + let px: Vec = (0..w * h * spp).map(|i| (i % spp) as u8).collect(); + let d = strips(w as u32, h as u32, 8, spp as u16, 2, h as u32, &px).tag(338, SHORT, &[0; EXTRAS]); + let bytes = build(true, false, &[d], &[0]).bytes; + let img = decode_ok(&bytes); + assert_eq!(img.layout(), ChannelLayout::Rgb); + assert_eq!(img.meta.extra_samples.len(), MAX_EXTRA_SAMPLES); + assert_eq!(img.meta.extra_samples.last().map(|e| e.data[0]), Some((3 + MAX_EXTRA_SAMPLES - 1) as u8)); + assert_eq!(img.warnings, vec![DecodeWarning::ExtraSamplesOverLimit { format: Format::Tiff, count: 4 }]); + assert!(has_unretained_source_data(&bytes).unwrap()); + let text = img.warnings[0].to_string(); + assert!(text.contains("4 extra TIFF file channels") && text.contains("past the first 16"), "{text}"); + // At the cap exactly, nothing is left out. + let spp = 3 + MAX_EXTRA_SAMPLES; + let px: Vec = (0..w * h * spp).map(|i| (i % spp) as u8).collect(); + let d = strips(w as u32, h as u32, 8, spp as u16, 2, h as u32, &px).tag(338, SHORT, &[0; MAX_EXTRA_SAMPLES]); + let bytes = build(true, false, &[d], &[0]).bytes; + let img = decode_ok(&bytes); + assert_eq!(img.meta.extra_samples.len(), MAX_EXTRA_SAMPLES); + assert!(img.warnings.is_empty(), "{:?}", img.warnings); + assert!(!has_unretained_source_data(&bytes).unwrap()); +} + +/// The guard follows decoder classification, including non-first, repeated or missing alpha tags. +#[test] +fn source_sample_cap_matches_the_decoder_at_layout_and_alpha_boundaries() { + for (photo, color) in [(1u16, 1usize), (2, 3), (5, 4), (8, 3), (9, 3)] { + for extras in [MAX_EXTRA_SAMPLES - 1, MAX_EXTRA_SAMPLES, MAX_EXTRA_SAMPLES + 1] { + for kind in 0..7 { + let alpha = matches!(kind, 1..=3); + let spp = color + extras + usize::from(alpha); + let px: Vec<_> = (0..spp).map(|i| i as u8 + 1).collect(); + let mut kinds = vec![0; spp - color]; + if alpha { + kinds[if kind == 2 { 1 } else { 0 }] = 2; + } + if kind == 3 { + kinds[1] = 1; + } + if kind == 4 { + kinds.push(2); + } + let mut dir = strips(1, 1, 8, spp as u16, photo, 1, &px); + if kind != 5 { + dir = dir.tag(338, SHORT, &kinds); + } + if kind == 6 { + dir = dir.tag(338, SHORT, &vec![2; spp - color]); + } + let bytes = build(true, false, &[dir], &[0]).bytes; + let image = decode(&bytes); + let guarded = has_unretained_source_data(&bytes); + match image { + Ok(image) => { + let dropped = image.warnings.iter().any(|w| matches!(w, DecodeWarning::ExtraSamplesOverLimit { .. })); + assert_eq!(guarded.unwrap(), dropped, "photo={photo} extras={extras} alpha tags={kind}"); + if kind <= 3 { + assert_eq!(dropped, extras > MAX_EXTRA_SAMPLES); + } + } + Err(_) if kind >= 4 => assert!(guarded.is_err(), "unsupported tags remain refused"), + Err(error) => panic!("supported photo={photo} extras={extras} tags={kind}: {error}"), + } + } + } + } +} + +/// Independent compressed zero rows satisfy the decoder's file-size safety bound. Inspection +/// follows import's default budget without allocating or decoding the declared whole image. +#[cfg(target_pointer_width = "64")] +#[test] +fn source_sample_guard_follows_whole_aux_and_banded_plain_budget_routes() { + for (size, bits, extras, guarded) in [(16384u32, 8u16, MAX_EXTRA_SAMPLES, true), (32768, 32, 0, false)] { + let samples = 3 + extras; + let entries = if extras == 0 { 11u16 } else { 12 }; + let bits_at = 8 + 2 + usize::from(entries) * 12 + 4; + let kinds_at = bits_at + samples * 2; + let offsets_at = kinds_at + extras * 2; + let counts_at = offsets_at + size as usize * 4; + let data_at = counts_at + size as usize * 4; + let sample = if bits == 32 { SampleType::F32 } else { SampleType::U8 }; + let mut image = Image::from_raw(size, 1, ChannelLayout::Rgb, sample, vec![0; size as usize * 3 * sample.bytes()]).unwrap(); + image.meta.extra_samples = (0..extras).map(|_| ExtraSample { kind: 0, data: vec![0; size as usize * sample.bytes()] }).collect(); + let encoded = encode(&image, Format::Tiff, &EncodeOptions { tiff_compression: TiffCompression::Deflate, ..Default::default() }).unwrap(); + decode_ok(&encoded); + assert_eq!(&encoded[..2], b"II"); + let first = u32::from_le_bytes(encoded[4..8].try_into().unwrap()) as usize; + let n = u16::from_le_bytes(encoded[first..first + 2].try_into().unwrap()) as usize; + let field = |tag: u16| { + let at = (0..n).map(|i| first + 2 + i * 12).find(|&at| u16::from_le_bytes(encoded[at..at + 2].try_into().unwrap()) == tag).unwrap(); + assert_eq!(u32::from_le_bytes(encoded[at + 4..at + 8].try_into().unwrap()), 1); + u32::from_le_bytes(encoded[at + 8..at + 12].try_into().unwrap()) as usize + }; + let row = &encoded[field(273)..field(273) + field(279)]; + let mut tags = vec![ + (256u16, 4u16, 1u32, size), + (257, 4, 1, size), + (258, 3, samples as u32, bits_at as u32), + (259, 3, 1, field(259) as u32), + (262, 3, 1, 2), + (273, 4, size, offsets_at as u32), + (277, 3, 1, samples as u32), + (278, 4, 1, 1), + (279, 4, size, counts_at as u32), + (284, 3, 1, 1), + (339, 3, 1, if bits == 32 { 3 } else { 1 }), + ]; + if extras > 0 { + tags.push((338, 3, extras as u32, kinds_at as u32)); + } + tags.sort_unstable_by_key(|t| t.0); + let mut bytes = b"II\x2a\0\x08\0\0\0".to_vec(); + bytes.extend_from_slice(&entries.to_le_bytes()); + for (tag, ty, count, value) in tags { + bytes.extend_from_slice(&tag.to_le_bytes()); + bytes.extend_from_slice(&ty.to_le_bytes()); + bytes.extend_from_slice(&count.to_le_bytes()); + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes.extend_from_slice(&0u32.to_le_bytes()); + for _ in 0..samples { + bytes.extend_from_slice(&bits.to_le_bytes()); + } + for _ in 0..extras { + bytes.extend_from_slice(&0u16.to_le_bytes()); + } + for index in 0..size { + bytes.extend_from_slice(&((data_at + index as usize * row.len()) as u32).to_le_bytes()); + } + for _ in 0..size { + bytes.extend_from_slice(&(row.len() as u32).to_le_bytes()); + } + assert_eq!(bytes.len(), data_at); + for _ in 0..size { + bytes.extend_from_slice(row); + } + assert!(bytes.len() < 32 << 20); + assert_eq!(has_unretained_source_data(&bytes).unwrap(), guarded); + } +} + +#[test] +fn alpha_after_an_unspecified_extra_sample_is_kept() { + // #1349: TIFF 6.0 lets alpha be any extra sample; ExtraSamples [0, 2] puts it second. + let (w, h) = (5usize, 3usize); + for (photometric, color, layout) in [(2u16, 3usize, ChannelLayout::Rgba), (5, 4, ChannelLayout::CmykA), (1, 1, ChannelLayout::GrayA)] { + let spp = color + 2; + let t = truth(w * h * spp, 13 + spp as u64); + for (little, big) in ORDERS { + for (bits, format, file, want_all, sample) in depths(&t, little) { + let bps = bits as usize / 8; + // The colour samples, then the second extra sample (the alpha). + let want: Vec = want_all.chunks(spp * bps).flat_map(|p| [&p[..color * bps], &p[(color + 1) * bps..]].concat()).collect(); + for planar in [false, true] { + let chunks = + if planar { planar_strips(&file, w, h, spp, bps, 2) } else { file.chunks(w * spp * bps * 2).map(<[u8]>::to_vec).collect() }; + let mut d = Dir { entries: base(w as u32, h as u32, bits, spp as u16, photometric), chunks, tiled: false } + .tag(278, LONG, &[2]) + .tag(339, SHORT, &vec![u64::from(format); spp]) + .tag(338, SHORT, &[0, 2]); + if planar { + d = d.tag(284, SHORT, &[2]); + } + let img = decode_ok(&build(little, big, &[d], &[0]).bytes); + let what = format!("photometric {photometric} little {little} big {big} bits {bits} planar {planar}"); + assert_eq!((img.layout(), img.sample_type()), (layout, sample), "{what}"); + assert_eq!(img.data(), &want[..], "{what}"); + assert_eq!(img.warnings, [], "{what}"); + } + } + } + } + // The issue's file: 2x2 RGB, every pixel 200 100 50, then 17 (unspecified), then alpha 128. + let px: Vec = [200u8, 100, 50, 17, 128].repeat(4); + let d = strips(2, 2, 8, 5, 2, 2, &px).tag(338, SHORT, &[0, 2]); + let img = decode_ok(&build(true, false, &[d], &[0]).bytes); + assert_eq!(img.data(), &[200u8, 100, 50, 128].repeat(4)[..]); + // Associated alpha in the same position is made straight. + let px: Vec = [100u8, 50, 25, 17, 128].repeat(4); + let d = strips(2, 2, 8, 5, 2, 2, &px).tag(338, SHORT, &[0, 1]); + let img = decode_ok(&build(true, false, &[d], &[0]).bytes); + assert_eq!(img.data(), &[199u8, 100, 50, 128].repeat(4)[..]); +} + +#[test] +fn associated_alpha_is_made_straight() { + // Premultiplied: half-transparent mid gray is stored as 64 with alpha 128. + let px = [64u8, 128, 200, 255, 0, 0]; + let d = strips(3, 1, 8, 2, 1, 1, &px).tag(338, SHORT, &[1]); + let img = decode_ok(&build(true, false, &[d], &[0]).bytes); + assert_eq!(img.layout(), ChannelLayout::GrayA); + assert_eq!(img.data(), &[128, 128, 200, 255, 0, 0]); + let px16 = u16s(&[16384, 32768], false); + let d = strips(1, 1, 16, 2, 1, 1, &px16).tag(338, SHORT, &[1]); + let img = decode_ok(&build(false, false, &[d], &[0]).bytes); + assert_eq!(img.to_u16_samples().unwrap(), [32768, 32768]); +} + +#[test] +fn ink_set_and_extra_samples_are_listed() { + let px = [10u8, 20, 30, 40, 17, 128]; + let d = strips(1, 1, 8, 6, 5, 1, &px).tag(332, SHORT, &[2]).tag(338, SHORT, &[0, 1]); + let info = tiff_info(&build(true, false, &[d], &[0]).bytes).unwrap(); + use ExtraSampleKind::*; + assert_eq!((info.pages[0].ink_set, &info.pages[0].extra_samples[..]), (Some(InkSet::InkNames), &[Unspecified, AssociatedAlpha][..])); + let d = strips(1, 1, 8, 6, 5, 1, &px).tag(332, SHORT, &[7]).tag(338, SHORT, &[2, 5]); + let info = tiff_info(&build(true, false, &[d], &[0]).bytes).unwrap(); + assert_eq!((info.pages[0].ink_set, &info.pages[0].extra_samples[..]), (Some(InkSet::Invalid(7)), &[UnassociatedAlpha, Invalid(5)][..])); + let d = strips(1, 1, 8, 4, 5, 1, &px[..4]); + let info = tiff_info(&build(true, false, &[d], &[0]).bytes).unwrap(); + assert_eq!((info.pages[0].ink_set, &info.pages[0].extra_samples[..]), (None, &[][..])); +} + +#[test] +fn white_is_zero_opens_with_the_right_tones() { + // #691: 00 FF 00 in a WhiteIsZero file is white, black, white. + let b = build(true, false, &[strips(3, 1, 8, 1, 0, 1, &[0x00, 0xff, 0x00])], &[0]); + let ours = decode_ok(&b.bytes); + let oracle = image::load_from_memory_with_format(&b.bytes, image::ImageFormat::Tiff).unwrap().into_luma8(); + assert_eq!(oracle.as_raw(), &[255u8, 0, 255]); + assert_eq!(ours.data(), &oracle.as_raw()[..]); + // The BlackIsZero control keeps the stored bytes. + let b = build(true, false, &[strips(3, 1, 8, 1, 1, 1, &[0x00, 0xff, 0x00])], &[0]); + assert_eq!(decode_ok(&b.bytes).data(), &[0, 255, 0]); + // 16-bit, in both byte orders and BigTIFF. + for (little, big) in ORDERS { + let b = build(little, big, &[strips(2, 1, 16, 1, 0, 1, &u16s(&[0, 65535], little))], &[0]); + assert_eq!(decode_ok(&b.bytes).to_u16_samples().unwrap(), [65535, 0]); + } + // 1-bit (a fax-style scan): bit set = black. + let b = build(true, false, &[strips(8, 1, 1, 1, 0, 1, &[0b1010_0000])], &[0]); + assert_eq!(decode_ok(&b.bytes).data(), &[0, 255, 0, 255, 255, 255, 255, 255]); + // With alpha: the gray inverts, the alpha does not. + let b = build(true, false, &[strips(2, 1, 8, 2, 0, 1, &[0, 255, 255, 10]).tag(338, SHORT, &[2])], &[0]); + let img = decode_ok(&b.bytes); + assert_eq!(img.layout(), ChannelLayout::GrayA); + assert_eq!(img.data(), &[255, 255, 0, 10]); + // 32-bit float. + let b = build(true, false, &[strips(2, 1, 32, 1, 0, 1, &f32s(&[0.0, 0.25], true)).tag(339, SHORT, &[3])], &[0]); + assert_eq!(decode_ok(&b.bytes).to_f32_samples().unwrap(), [1.0, 0.75]); +} + +#[test] +fn bigtiff_orientation_is_applied_like_classic_tiff() { + // #693: the same 2×1 gray image with Orientation 6, classic and BigTIFF, both byte orders. + for (little, big) in ORDERS { + let d = strips(2, 1, 8, 1, 1, 1, &[10, 200]).tag(274, SHORT, &[6]); + let b = build(little, big, &[d], &[0]); + assert_eq!(exif_orientation(&b.bytes), 6, "little {little} big {big}"); + assert_eq!(tiff_orientation(&b.bytes), 6); + let img = decode_ok(&b.bytes); + assert_eq!(img.dimensions(), (1, 2), "little {little} big {big}"); + assert_eq!(img.data(), &[10, 200]); + let kept = decode_with(&b.bytes, &DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap(); + assert_eq!(kept.dimensions(), (2, 1)); + } +} + +#[test] +fn palette_images_open_as_rgb() { + // Two 8-bit entries used: 0 = red, 1 = (16, 32, 48); the map has 256 entries per channel. + let mut map = vec![0u64; 768]; + map[0] = 0xffff; + map[1] = 16 * 257; + map[256 + 1] = 32 * 257; + map[512 + 1] = 48 * 257; + for (little, big) in ORDERS { + let d = strips(3, 1, 8, 1, 3, 1, &[0, 1, 0]).tag(320, SHORT, &map); + let img = decode_ok(&build(little, big, &[d], &[0]).bytes); + assert_eq!((img.layout(), img.sample_type()), (ChannelLayout::Rgb, SampleType::U8)); + assert_eq!(img.data(), &[255, 0, 0, 16, 32, 48, 255, 0, 0]); + } + // 4-bit indices. + let mut map = vec![0u64; 48]; + map[15] = 0xffff; + map[16 + 15] = 0x8000; + let d = strips(2, 1, 4, 1, 3, 1, &[0xf0]).tag(320, SHORT, &map); + assert_eq!(decode_ok(&build(true, false, &[d], &[0]).bytes).data(), &[255, 128, 0, 0, 0, 0]); + // A missing or short colour map is an error. + let d = strips(2, 1, 8, 1, 3, 1, &[0, 1]); + assert!(decode(&build(true, false, &[d], &[0]).bytes).is_err()); + let d = strips(2, 1, 8, 1, 3, 1, &[0, 1]).tag(320, SHORT, &[1, 2, 3]); + assert!(decode(&build(true, false, &[d], &[0]).bytes).is_err()); +} + +#[test] +fn wide_and_odd_sample_formats() { + for (little, big) in ORDERS { + let e = |v: u32| if little { v.to_le_bytes() } else { v.to_be_bytes() }; + // 32-bit unsigned keeps its top 16 bits. + let px: Vec = [0x1234_5678u32, 0xffff_0000].iter().flat_map(|&v| e(v)).collect(); + let img = decode_ok(&build(little, big, &[strips(2, 1, 32, 1, 1, 1, &px)], &[0]).bytes); + assert_eq!(img.to_u16_samples().unwrap(), [0x1234, 0xffff]); + // 64-bit float becomes 32-bit float. + let px: Vec = [0.5f64, 2.0].iter().flat_map(|v| if little { v.to_le_bytes() } else { v.to_be_bytes() }).collect(); + let img = decode_ok(&build(little, big, &[strips(2, 1, 64, 1, 1, 1, &px).tag(339, SHORT, &[3])], &[0]).bytes); + assert_eq!(img.to_f32_samples().unwrap(), [0.5, 2.0]); + // 16-bit float stays half. + let px: Vec = [f16::from_f32(0.25), f16::ONE].iter().flat_map(|v| if little { v.to_le_bytes() } else { v.to_be_bytes() }).collect(); + let img = decode_ok(&build(little, big, &[strips(2, 1, 16, 1, 1, 1, &px).tag(339, SHORT, &[3])], &[0]).bytes); + assert_eq!(img.sample_type(), SampleType::F16); + assert_eq!(img.to_normalized(), [0.25, 1.0]); + } + // 2- and 4-bit gray, rows padded to a byte, in several strips. + let img = decode_ok(&build(true, false, &[strips(3, 2, 2, 1, 1, 1, &[0b0001_1011, 0b1110_0100])], &[0]).bytes); + assert_eq!(img.data(), &[0, 85, 170, 255, 170, 85]); + let img = decode_ok(&build(false, true, &[strips(3, 2, 4, 1, 1, 1, &[0x0f, 0x80, 0x12, 0x30])], &[0]).bytes); + assert_eq!(img.data(), &[0, 255, 136, 17, 34, 51]); + // Signed integers are unsupported, not a crash. + let d = strips(2, 1, 16, 1, 1, 1, &[0, 0, 0, 0]).tag(339, SHORT, &[2]); + assert!(matches!(decode(&build(true, false, &[d], &[0]).bytes), Err(CodecError::Unsupported { .. }))); +} + +/// A flat gray page of `w`×`h` at `level`, with an optional NewSubfileType. +fn gray_page(w: u32, h: u32, level: u8, kind: Option) -> Dir { + let d = strips(w, h, 8, 1, 1, h, &vec![level; (w * h) as usize]); + match kind { + Some(k) => d.tag(254, LONG, &[k]), + None => d, + } +} + +#[test] +fn every_ifd_and_subifd_is_listed_and_decodable() { + for (little, big) in ORDERS { + let sub_ty = if big { IFD8 } else { IFD }; + let dirs = vec![ + gray_page(4, 3, 1, Some(1)), // 0: thumbnail first + gray_page(9, 7, 2, None).sub_ifds(sub_ty, &[4, 5]), // 1: page A, two SubIFDs + gray_page(9, 7, 3, Some(4)), // 2: its transparency mask + gray_page(5, 5, 4, Some(2)), // 3: page B (multi-page bit) + gray_page(3, 2, 5, Some(1)), // 4: SubIFD: reduced copy of A + gray_page(2, 1, 6, Some(1)), // 5: SubIFD: smaller copy + ]; + let b = build(little, big, &dirs, &[0, 1, 2, 3]); + let info = tiff_info(&b.bytes).unwrap(); + assert!(info.complete); + let summary: Vec<(u32, Option, TiffPageKind)> = info.pages.iter().map(|p| (p.width, p.parent, p.kind)).collect(); + use TiffPageKind::*; + assert_eq!( + summary, + [ + (4, None, ReducedResolution), + (9, None, Page), + (3, Some(1), ReducedResolution), + (2, Some(1), ReducedResolution), + (9, None, Mask), + (5, None, Page) + ] + ); + assert_eq!(info.pages[1].ifd_offset, b.ifd_at[1]); + assert_eq!((info.default_page(), info.page_count()), (Some(1), 2)); + // Like Photoshop: the first full-resolution page opens; the thumbnail is skipped. + let img = decode_ok(&b.bytes); + assert_eq!((img.dimensions(), img.data()[0]), ((9, 7), 2)); + assert_eq!(img.warnings, [DecodeWarning::MorePages { total: Some(2) }]); + // Every directory decodes on request. + for (i, (w, level)) in [(4, 1u8), (9, 2), (3, 5), (2, 6), (9, 3), (5, 4)].into_iter().enumerate() { + let img = decode_tiff_page(&b.bytes, i, &DecodeOptions::default()).unwrap(); + assert_eq!((img.width(), img.data()[0]), (w, level), "page {i} little {little} big {big}"); + } + assert!(matches!(decode_tiff_page(&b.bytes, 6, &DecodeOptions::default()), Err(CodecError::InvalidImage(_)))); + } +} + +#[test] +fn a_file_of_only_thumbnails_still_opens() { + let b = build(true, false, &[gray_page(4, 3, 7, Some(1))], &[0]); + assert_eq!(tiff_info(&b.bytes).unwrap().default_page(), Some(0)); + assert_eq!(decode_ok(&b.bytes).data()[0], 7); +} + +#[test] +fn cyclic_chains_and_subifds_terminate() { + for (little, big) in ORDERS { + // IFD1 points back to IFD0. + let mut b = build(little, big, &[gray_page(2, 2, 1, None), gray_page(2, 2, 2, None)], &[0, 1]); + let first = b.ifd_at[0]; + set_next(&mut b, little, big, 1, first); + let info = tiff_info(&b.bytes).unwrap(); + assert_eq!((info.pages.len(), info.complete), (2, false)); + let img = decode_ok(&b.bytes); + assert_eq!(img.warnings, [DecodeWarning::MorePages { total: None }]); + // A directory pointing at itself. + let mut b = build(little, big, &[gray_page(2, 2, 1, None)], &[0]); + let first = b.ifd_at[0]; + set_next(&mut b, little, big, 0, first); + assert_eq!(tiff_info(&b.bytes).unwrap().pages.len(), 1); + assert_eq!(decode_ok(&b.bytes).warnings, []); + // A SubIFD listing its own parent, and one listing itself. + let sub_ty = if big { IFD8 } else { IFD }; + let b = build(little, big, &[gray_page(2, 2, 1, None).sub_ifds(sub_ty, &[1]), gray_page(1, 1, 2, Some(1)).sub_ifds(sub_ty, &[0, 1])], &[0]); + let info = tiff_info(&b.bytes).unwrap(); + assert_eq!((info.pages.len(), info.complete), (2, false)); + decode_ok(&b.bytes); + // A next pointer past the end, or into the header. + for to in [b.bytes.len() as u64 + 100, 3, u64::MAX] { + let mut b = build(little, big, &[gray_page(2, 2, 1, None), gray_page(2, 2, 2, None)], &[0, 1]); + set_next(&mut b, little, big, 1, to); + let info = tiff_info(&b.bytes).unwrap(); + assert_eq!((info.pages.len(), info.complete), (2, false)); + decode_ok(&b.bytes); + } + } +} + +#[test] +fn the_chain_walk_is_capped() { + // 10 001 tiny directories: the walk stops at 10 000 and says the list is incomplete. + let dirs: Vec = (0..10_001).map(|_| gray_page(1, 1, 9, None)).collect(); + let chain: Vec = (0..dirs.len()).collect(); + let b = build(true, false, &dirs, &chain); + let info = tiff_info(&b.bytes).unwrap(); + assert_eq!((info.pages.len(), info.complete), (10_000, false)); + assert_eq!(decode_ok(&b.bytes).warnings, [DecodeWarning::MorePages { total: None }]); +} + +/// A partial child walk remains usable for decoding, but cannot authorize destructive saves. +#[test] +fn incomplete_subifd_lists_and_depth_are_reported() { + for (little, big) in ORDERS { + let ty = if big { IFD8 } else { IFD }; + for depth in [4usize, 5] { + let dirs: Vec<_> = (0..=depth) + .map(|i| { + let dir = gray_page( + 2, + 2, + i as u8, + Some(if i == 0 { + 0 + } else if i == depth { + 4 + } else { + 1 + }), + ); + if i < depth { dir.sub_ifds(ty, &[i + 1]) } else { dir } + }) + .collect(); + let b = build(little, big, &dirs, &[0]); + assert_eq!(tiff_info(&b.bytes).unwrap().complete, depth == 4); + decode_ok(&b.bytes); + if depth == 4 { + assert!(has_unretained_source_data(&b.bytes).unwrap()); + } else { + assert!(has_unretained_source_data(&b.bytes).is_err()); + } + } + for count in [0usize, 1, 256, 257] { + let b = build(little, big, &[gray_page(2, 2, 9, None).tag(330, ty, &vec![0; count])], &[0]); + assert_eq!(tiff_info(&b.bytes).unwrap().complete, count <= 256, "null child slots {count}"); + decode_ok(&b.bytes); + } + for invalid in [SHORT, u16::MAX] { + let b = build(little, big, &[gray_page(2, 2, 9, None).tag(330, invalid, &[0])], &[0]); + assert!(!tiff_info(&b.bytes).unwrap().complete); + assert!(has_unretained_source_data(&b.bytes).is_err()); + decode_ok(&b.bytes); + } + let b = build(little, big, &[gray_page(2, 2, 9, None).tag(330, ty, &[0]).tag(330, ty, &[0])], &[0]); + assert!(!tiff_info(&b.bytes).unwrap().complete, "duplicate child declarations"); + let mut b = build(little, big, &[gray_page(2, 2, 9, None).tag(330, ty, &[0, 0])], &[0]); + let (entry, ..) = find_entry(&b, little, big, 0, 330).unwrap(); + let offset = b.bytes.len() as u64 - 1; + let value = if little { offset.to_le_bytes() } else { offset.to_be_bytes() }; + let (field, width) = if big { (entry + 12, 8) } else { (entry + 8, 4) }; + b.bytes[field..field + width].copy_from_slice(if little { &value[..width] } else { &value[8 - width..] }); + assert!(!tiff_info(&b.bytes).unwrap().complete, "truncated child list"); + assert!(has_unretained_source_data(&b.bytes).is_err()); + decode_ok(&b.bytes); + } +} + +#[test] +fn image_data_cut_off_decodes_with_a_warning() { + let (w, h) = (4u32, 8u32); + let px: Vec = (0..32).collect(); + for (little, big) in ORDERS { + // The last strip starts past the end of the file: its rows stay empty. + let mut b = build(little, big, &[strips(w, h, 8, 1, 1, 2, &px)], &[0]); + let len = b.bytes.len() as u64; + patch_value(&mut b, little, big, 0, 273, 3, len + 1000); + let img = decode_ok(&b.bytes); + assert_eq!(img.warnings, [DecodeWarning::Truncated { format: Format::Tiff }]); + assert_eq!(&img.data()[..24], &px[..24]); + assert_eq!(&img.data()[24..], &[0; 8]); + // A strip running into the end of the file. + let mut b = build(little, big, &[strips(w, h, 8, 1, 1, 2, &px)], &[0]); + let len = b.bytes.len() as u64; + patch_value(&mut b, little, big, 0, 273, 1, len - 3); + assert_eq!(decode_ok(&b.bytes).warnings, [DecodeWarning::Truncated { format: Format::Tiff }]); + // Every strip missing: nothing to show, an error. + let mut b = build(little, big, &[strips(w, h, 8, 1, 1, 2, &px)], &[0]); + let len = b.bytes.len() as u64; + for i in 0..4 { + patch_value(&mut b, little, big, 0, 273, i, len + 7); + } + assert!(decode(&b.bytes).is_err()); + } +} + +#[test] +fn hostile_sizes_and_counts_are_refused_without_allocating() { + let tight = DecodeOptions { limits: Limits { max_alloc: 64 << 20, ..Limits::default() }, ..Default::default() }; + for (little, big) in ORDERS { + let px = vec![7u8; 16]; + let fresh = || build(little, big, &[strips(4, 4, 8, 1, 1, 1, &px)], &[0]); + // Huge declared dimensions: over the limits, or far more than the file can hold. + for (wv, hv) in [(1u64 << 31, 4u64), (4, 1 << 31), (u32::MAX as u64, u32::MAX as u64), (200_000, 200_000), (30_000, 30_000)] { + let mut b = fresh(); + patch_value(&mut b, little, big, 0, 256, 0, wv); + patch_value(&mut b, little, big, 0, 257, 0, hv); + assert!(decode(&b.bytes).is_err(), "{wv}x{hv}"); + assert!(decode_with(&b.bytes, &DecodeOptions { limits: Limits::none(), ..Default::default() }).is_err(), "{wv}x{hv} without limits"); + } + // A strip-offset count far past the end of the file. + let mut b = fresh(); + patch_count(&mut b, little, big, 0, 273, 1 << 30); + assert!(matches!(decode(&b.bytes), Err(CodecError::Malformed { .. }))); + let mut b = fresh(); + patch_count(&mut b, little, big, 0, 258, u32::MAX as u64); + assert!(decode(&b.bytes).is_err()); + // Zero rows per strip, zero samples, absurd samples per pixel. + for (tag, v) in [(278u16, 0u64), (277, 0), (277, 60_000)] { + let mut b = fresh(); + patch_value(&mut b, little, big, 0, tag, 0, v); + assert!(decode(&b.bytes).is_err(), "tag {tag} = {v}"); + } + // Absurd tiles over a tiny image, chunky and planar. + for planar in [false, true] { + for (tw, th) in [(1u64 << 30, 16u64), (16, 1 << 30), (u32::MAX as u64, u32::MAX as u64)] { + let spp = if planar { 3 } else { 1 }; + let mut d = Dir { entries: base(4, 4, 8, spp, if planar { 2 } else { 1 }), chunks: vec![vec![1u8; 64]; spp as usize], tiled: true }; + d = d.tag(322, LONG, &[tw]).tag(323, LONG, &[th]); + if planar { + d = d.tag(284, SHORT, &[2]); + } + let b = build(little, big, &[d], &[0]); + let _ = decode_with(&b.bytes, &tight); + let _ = decode(&b.bytes); + } + } + } + // A BigTIFF directory claiming 2^60 entries. + let mut b = build(true, true, &[gray_page(2, 2, 1, None)], &[0]); + let at = b.ifd_at[0] as usize; + b.bytes[at..at + 8].copy_from_slice(&(1u64 << 60).to_le_bytes()); + assert!(tiff_info(&b.bytes).is_err()); + assert!(decode(&b.bytes).is_err()); +} + +#[test] +fn banded_decode_matches_the_encoder_on_larger_images() { + // Many strips decoded in parallel bands, at every depth and compression. + for (layout, sample) in [ + (ChannelLayout::Rgb, SampleType::U8), + (ChannelLayout::Rgba, SampleType::U16), + (ChannelLayout::Gray, SampleType::F32), + (ChannelLayout::Cmyk, SampleType::U8), + ] { + let img = common::synth(613, 411, layout, sample, 3, 0.2); + for c in [TiffCompression::None, TiffCompression::Lzw, TiffCompression::Deflate, TiffCompression::PackBits] { + let bytes = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: c, ..Default::default() }).unwrap(); + let back = decode_ok(&bytes); + assert_eq!(back.data(), img.data(), "{layout:?} {sample:?} {c:?}"); + } + } + // Tiles across many bands. + let t = truth(300 * 260 * 3, 77); + let d = Dir { entries: base(300, 260, 16, 3, 2), chunks: tile_split(&u16s(&t.u16s, false), 300, 260, 3, 2, 64, 32, false), tiled: true } + .tag(322, SHORT, &[64]) + .tag(323, SHORT, &[32]); + assert_eq!(decode_ok(&build(false, true, &[d], &[0]).bytes).data(), &ne16(&t.u16s)[..]); +} + +#[test] +fn bigtiff_is_written_on_request_and_reads_back() { + let mut img = common::synth(23, 17, ChannelLayout::Rgba, SampleType::U16, 2, 0.3); + img.meta.photoshop_layers = Some(b"Adobe Photoshop Document Data Block\0MIB8ryaL".to_vec()); + img.meta.photoshop_resources = Some(b"8BIM\x04\x04\0\0\0\0\0\0".to_vec()); + for c in [TiffCompression::None, TiffCompression::Deflate, TiffCompression::Lzw] { + let classic = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: c, ..Default::default() }).unwrap(); + let big = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: c, tiff_bigtiff: true, ..Default::default() }).unwrap(); + let version = |b: &[u8]| if b.starts_with(b"II") { u16::from_le_bytes([b[2], b[3]]) } else { u16::from_be_bytes([b[2], b[3]]) }; + assert_eq!((version(&classic), version(&big)), (42, 43)); + assert!(tiff_info(&big).unwrap().big_tiff); + let back = decode_ok(&big); + assert_eq!(back.data(), img.data(), "{c:?}"); + assert_eq!(back.meta.photoshop_layers, img.meta.photoshop_layers); + assert_eq!(back.meta.photoshop_resources, img.meta.photoshop_resources); + assert_eq!(tiff_photoshop_tags(&big), (img.meta.photoshop_resources.as_deref(), img.meta.photoshop_layers.as_deref())); + } +} + +/// Small files covering every structure: multi-IFD, SubIFDs, BigTIFF both orders, tiles, planar. +fn zoo() -> Vec> { + let mut v = Vec::new(); + let t = truth(6 * 5 * 4, 5); + for (little, big) in ORDERS { + let sub_ty = if big { IFD8 } else { IFD }; + let tiles = Dir { entries: base(6, 5, 8, 3, 2), chunks: tile_split(&t.u8s, 6, 5, 3, 1, 16, 16, true), tiled: true } + .tag(322, SHORT, &[16]) + .tag(323, SHORT, &[16]) + .tag(284, SHORT, &[2]); + let planar = Dir { entries: base(6, 5, 16, 4, 2), chunks: planar_strips(&u16s(&t.u16s, little), 6, 5, 4, 2, 2), tiled: false } + .tag(278, SHORT, &[2]) + .tag(284, SHORT, &[2]) + .tag(338, SHORT, &[1]); + let dirs = vec![gray_page(3, 2, 1, Some(1)), tiles.sub_ifds(sub_ty, &[3]), planar, gray_page(2, 2, 4, Some(1)).tag(274, SHORT, &[6])]; + v.push(build(little, big, &dirs, &[0, 1, 2]).bytes); + v.push(build(little, big, &[strips(5, 3, 1, 1, 0, 2, &[0xa5; 3])], &[0]).bytes); + } + v +} + +fn poke(bytes: &[u8]) { + let _ = decode(bytes); + let _ = decode_with(bytes, &DecodeOptions { limits: Limits { max_alloc: 16 << 20, ..Limits::default() }, ..Default::default() }); + let _ = exif_orientation(bytes); + let _ = tiff_orientation(bytes); + if let Ok(info) = tiff_info(bytes) { + for i in 0..info.pages.len().min(8) { + let _ = decode_tiff_page(bytes, i, &DecodeOptions::default()); + } + } +} + +#[test] +fn zoo_decodes_and_never_panics_when_cut_or_corrupted() { + for f in zoo() { + let info = tiff_info(&f).unwrap(); + for i in 0..info.pages.len() { + decode_tiff_page(&f, i, &DecodeOptions::default()).unwrap(); + } + for cut in 0..f.len() { + poke(&f[..cut]); + } + for i in 0..f.len() { + for flip in [0xffu8, 0x80, 0x01] { + let mut g = f.clone(); + g[i] ^= flip; + poke(&g); + } + } + let mut r = Rng::new(f.len() as u64); + for _ in 0..300 { + let mut g = f.clone(); + for _ in 0..4 { + let i = (r.next_u64() as usize) % g.len(); + g[i] = r.next_u64() as u8; + } + poke(&g); + } + } +} + +/// A longer random-mutation run over the zoo (ignored by default; `-- --ignored`, with +/// `TIFF_FUZZ_ITERS` to change the count). +#[test] +#[ignore = "long fuzz run"] +fn zoo_long_random_mutations() { + let iters: usize = std::env::var("TIFF_FUZZ_ITERS").ok().and_then(|v| v.parse().ok()).unwrap_or(20_000); + let mut r = Rng::new(0x5eed); + let zoo = zoo(); + for k in 0..iters { + let mut g = zoo[k % zoo.len()].clone(); + for _ in 0..1 + r.next_u64() % 8 { + let i = (r.next_u64() as usize) % g.len(); + g[i] = match r.next_u64() % 4 { + 0 => 0, + 1 => 0xff, + _ => r.next_u64() as u8, + }; + } + poke(&g); + } +} diff --git a/crates/image-codecs/tests/it/tiff_photoshop.rs b/crates/image-codecs/tests/it/tiff_photoshop.rs new file mode 100644 index 000000000..d413ccc34 --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_photoshop.rs @@ -0,0 +1,437 @@ +//! Photoshop's private TIFF tags (34377 image resources, 37724 ImageSourceData) travel through +//! `Metadata` as opaque bytes: written when present, read from either byte order and from +//! BigTIFF, and never a panic or an allocation on malformed input. + +use crate::common; +use common::*; +use craft_image_codecs::*; + +fn with_tags(layout: ChannelLayout, sample: SampleType) -> Image { + let mut img = synth(9, 7, layout, sample, 5, 0.3); + img.meta.photoshop_resources = + Some(b"8BIM\x03\xed\x00\x00\x00\x00\x00\x10\x00\x48\x00\x00\x00\x01\x00\x01\x00\x48\x00\x00\x00\x01\x00\x01".to_vec()); + let mut layers = b"Adobe Photoshop Document Data Block\0".to_vec(); + layers.extend_from_slice(b"MIB8ryaL"); + layers.extend_from_slice(&4u32.to_le_bytes()); + layers.extend_from_slice(&[0, 0, 0, 0]); + img.meta.photoshop_layers = Some(layers); + img +} + +#[test] +fn tags_round_trip_through_the_encoder() { + for (layout, sample) in [ + (ChannelLayout::Rgb, SampleType::U8), + (ChannelLayout::Rgba, SampleType::U16), + (ChannelLayout::Gray, SampleType::F32), + (ChannelLayout::CmykA, SampleType::U8), + ] { + for compression in [TiffCompression::None, TiffCompression::Deflate, TiffCompression::Lzw, TiffCompression::PackBits] { + let img = with_tags(layout, sample); + let bytes = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: compression, ..Default::default() }).unwrap(); + // The encoder writes the running target's byte order. + assert_eq!(bytes.starts_with(b"II"), tiff_writes_little_endian()); + let (res, layers) = tiff_photoshop_tags(&bytes); + assert_eq!(res, img.meta.photoshop_resources.as_deref()); + assert_eq!(layers, img.meta.photoshop_layers.as_deref()); + let back = decode(&bytes).unwrap(); + assert_eq!(back.meta.photoshop_resources, img.meta.photoshop_resources, "{layout:?} {sample:?} {compression:?}"); + assert_eq!(back.meta.photoshop_layers, img.meta.photoshop_layers); + assert_eq!(back.data(), img.data()); + } + } +} + +#[test] +fn extra_samples_round_trip() { + for (layout, sample) in [ + (ChannelLayout::Rgb, SampleType::U8), + (ChannelLayout::Rgba, SampleType::U8), + (ChannelLayout::Gray, SampleType::U16), + (ChannelLayout::Cmyk, SampleType::F32), + ] { + let mut img = synth(4, 3, layout, sample, 2, 0.4); + let n = 4 * 3; + let b = sample.bytes(); + let data: Vec = (0..n * b).map(|i| (i * 17) as u8).collect(); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: data.clone() }]; + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + let back = decode(&bytes).unwrap(); + assert_eq!(back.layout(), layout, "{layout:?} {sample:?}"); + assert_eq!(back.data(), img.data(), "{layout:?} {sample:?}"); + assert_eq!(back.meta.extra_samples.len(), 1, "{layout:?} {sample:?}"); + assert_eq!(back.meta.extra_samples[0].kind, 0, "{layout:?} {sample:?}"); + assert_eq!(back.meta.extra_samples[0].data, data, "{layout:?} {sample:?}"); + } +} + +/// Tall enough that the default strip size (about 1 MB) cannot hold every row, so a writer +/// that only fills the first strip, or predicts across the wrong sample count, cannot hide. +/// The spot plane is checked with the `tiff` crate, not PhotoCraft's decoder. +#[test] +fn multi_strip_spot_round_trip_and_dump() { + let w = 800u32; + let h = 400u32; + let n = (w as usize) * (h as usize); + for (layout, channels) in [(ChannelLayout::Rgb, 3usize), (ChannelLayout::Rgba, 4usize)] { + let mut px = vec![0u8; n * channels]; + let mut spot = vec![0u8; n]; + for y in 0..h as usize { + for x in 0..w as usize { + let i = y * w as usize + x; + px[i * channels] = (x % 251) as u8; + px[i * channels + 1] = (y % 251) as u8; + px[i * channels + 2] = 40; + if channels == 4 { + px[i * channels + 3] = if x > 20 && x < 780 && y > 10 && y + 10 < h as usize { 255 } else { 0 }; + } + spot[i] = ((x * 3 + y * 17) % 251) as u8; + } + } + let mut img = Image::from_u8(w, h, layout, px).unwrap(); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: spot.clone() }]; + for compression in [TiffCompression::None, TiffCompression::Deflate, TiffCompression::Lzw, TiffCompression::PackBits] { + let bytes = encode(&img, Format::Tiff, &EncodeOptions { tiff_compression: compression, ..Default::default() }).unwrap(); + let label = format!("{layout:?} {compression:?}"); + let back = decode(&bytes).unwrap(); + assert_eq!(back.meta.extra_samples.len(), 1, "{label}"); + assert_eq!(back.meta.extra_samples[0].data, spot, "own decoder {label}"); + assert_eq!(back.data(), img.data(), "own decoder rgb {label}"); + let mut dec = tiff::decoder::Decoder::new(std::io::Cursor::new(bytes.as_slice())).unwrap(); + let spp: u16 = dec.get_tag_unsigned(tiff::tags::Tag::SamplesPerPixel).unwrap(); + let bps = dec.get_tag_u16_vec(tiff::tags::Tag::BitsPerSample).unwrap(); + let extra = dec.get_tag_u16_vec(tiff::tags::Tag::ExtraSamples).unwrap(); + let rows = dec.find_tag_unsigned::(tiff::tags::Tag::RowsPerStrip).unwrap(); + assert_eq!(bps.len(), spp as usize, "{label}"); + assert_eq!(spp as usize, channels + 1, "{label}"); + assert_eq!(*extra.last().unwrap(), 0, "{label} spot ExtraSamples {extra:?}"); + assert!(rows.is_some_and(|r| r < h) || compression == TiffCompression::PackBits, "{label} rows {rows:?}"); + let patched = min_is_black(&bytes); + let mut indep = tiff::decoder::Decoder::new(std::io::Cursor::new(patched.as_slice())).unwrap(); + let tiff::decoder::DecodingResult::U8(samples) = indep.read_image().unwrap() else { panic!("{label} not u8") }; + let spp = spp as usize; + assert_eq!(samples.len(), n * spp, "{label}"); + let mut bad = 0usize; + let mut first = None; + for (i, want) in spot.iter().enumerate() { + let got = samples.get(i * spp + spp - 1).copied(); + if got != Some(*want) { + bad += 1; + first.get_or_insert((i, got)); + } + } + assert_eq!(bad, 0, "{label} independent reader, first {first:?}"); + } + } +} + +/// Retag photometric interpretation 262 as MinIsBlack so `tiff`'s decoder returns every sample. +fn min_is_black(src: &[u8]) -> Vec { + let mut bytes = src.to_vec(); + let le = bytes.first().copied() == Some(b'I'); + let ru16 = |b: &[u8], i: usize| -> Option { + let s = b.get(i..i + 2)?; + Some(if le { u16::from_le_bytes([s[0], s[1]]) } else { u16::from_be_bytes([s[0], s[1]]) }) + }; + let ru32 = |b: &[u8], i: usize| -> Option { + let s = b.get(i..i + 4)?; + Some(if le { u32::from_le_bytes(s.try_into().ok()?) } else { u32::from_be_bytes(s.try_into().ok()?) }) + }; + let Some(ifd) = ru32(&bytes, 4) else { return bytes }; + let Some(n) = ru16(&bytes, ifd as usize) else { return bytes }; + for e in 0..n as usize { + let at = ifd as usize + 2 + e * 12; + if ru16(&bytes, at) == Some(262) { + let value = if le { 1u16.to_le_bytes() } else { 1u16.to_be_bytes() }; + if let Some(slot) = bytes.get_mut(at + 8..at + 10) { + slot.copy_from_slice(&value); + } + break; + } + } + bytes +} + +#[test] +fn more_than_sixteen_extra_samples_is_refused() { + let mut img = synth(2, 2, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + img.meta.extra_samples = (0..MAX_EXTRA_SAMPLES + 1).map(|_| ExtraSample { kind: 0, data: vec![0; 4] }).collect(); + assert!(encode(&img, Format::Tiff, &EncodeOptions::default()).is_err()); +} + +#[test] +fn sixteen_extra_samples_round_trip() { + for (layout, sample, base) in [ + (ChannelLayout::Rgb, SampleType::U8, 3usize), + (ChannelLayout::Rgba, SampleType::U8, 4), + (ChannelLayout::Gray, SampleType::U16, 1), + (ChannelLayout::Cmyk, SampleType::F32, 4), + ] { + let mut img = synth(2, 2, layout, sample, 1, 0.0); + let n = 2 * 2 * sample.bytes(); + img.meta.extra_samples = (0..MAX_EXTRA_SAMPLES).map(|k| ExtraSample { kind: 0, data: vec![k as u8; n] }).collect(); + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + let back = decode(&bytes).unwrap(); + assert_eq!(back.layout(), layout, "{layout:?} {sample:?} base {base}"); + assert_eq!(back.data(), img.data(), "{layout:?}"); + assert_eq!(back.meta.extra_samples.len(), MAX_EXTRA_SAMPLES, "{layout:?}"); + assert_eq!(back.meta.extra_samples[7].kind, 0, "{layout:?}"); + assert_eq!(back.meta.extra_samples[15].data, vec![15; n], "{layout:?}"); + } +} + +/// An extra plane holds exactly one sample per pixel of the image's sample type. A plane of any +/// other length is refused rather than written: the writer copies one sample per pixel, so a +/// longer plane would lose its tail and a shorter one would run out mid-image. +#[test] +fn wrong_sized_extra_planes_are_refused() { + let (w, h) = (2u32, 1u32); + for (label, plane) in [("too long", vec![10u8, 20, 30]), ("too short", vec![10u8]), ("empty", Vec::new())] { + let mut img = synth(w, h, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: plane.clone() }]; + let err = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap_err(); + assert!(matches!(err, CodecError::Encode { .. }), "{label}: {err}"); + assert!(err.to_string().contains("extra sample 0") && err.to_string().contains("exactly 2"), "{label}: {err}"); + } + // The exact length still writes, and comes back unchanged. + let plane = vec![10u8, 20]; + let mut img = synth(w, h, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: plane.clone() }]; + let back = decode(&encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap()).unwrap(); + assert_eq!(back.meta.extra_samples, vec![ExtraSample { kind: 0, data: plane }]); +} + +/// The sample-type conversion an encode performs must not launder a wrong-sized plane: float Lab +/// is stored as CIELab at 16 bits, and `Image::convert` leaves a plane whose length does not match +/// the image alone, so a 7- or 9-byte plane for two pixels would otherwise reach the writer +/// reinterpreted at the new width. +#[test] +fn a_wrong_sized_plane_is_refused_before_a_sample_type_conversion() { + let (w, h) = (2u32, 1u32); + // Two pixels at 32-bit float are 8 bytes; the file's 16-bit samples are 4. + for (label, len) in [("short", 7usize), ("long", 9)] { + let mut img = synth(w, h, ChannelLayout::Lab, SampleType::F32, 1, 0.0); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: vec![0; len] }]; + let err = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap_err(); + assert!(matches!(err, CodecError::Encode { .. }), "{label}: {err}"); + assert!(err.to_string().contains("exactly 8"), "{label}: {err}"); + } +} + +/// Extra samples count against `max_alloc` with the image (their interleaved buffer and the +/// planes split from it): a budget that holds the RGB image but not sixteen extra planes opens +/// the image without them, with a warning, instead of allocating past it. +#[test] +fn extra_samples_count_against_the_allocation_limit() { + // Big enough for several strips (about 1 MB each): one strip of all 19 samples fits the budget + // of the image alone, as in real files. + let (w, h) = (1024u32, 1024u32); + let n = (w * h) as usize; + let mut img = synth(w, h, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + img.meta.extra_samples = (0..MAX_EXTRA_SAMPLES).map(|k| ExtraSample { kind: 0, data: vec![k as u8; n] }).collect(); + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + let with = |max_alloc: u64| decode_with(&bytes, &DecodeOptions { limits: Limits { max_alloc, ..Limits::default() }, ..DecodeOptions::default() }); + // RGB is 3 bytes a pixel; the extra samples add 2 x 16 more. + let image_only = 3 * n as u64; + let with_extras = (3 + 2 * MAX_EXTRA_SAMPLES as u64) * n as u64; + let back = with(with_extras).expect("everything fits"); + assert_eq!(back.meta.extra_samples.len(), MAX_EXTRA_SAMPLES); + assert!(back.warnings.is_empty()); + // The image fits, its extra samples don't: it opens as before they were read, with a warning. + for max_alloc in [image_only, with_extras - 1] { + let only = with(max_alloc).expect("the image alone fits"); + assert!(only.meta.extra_samples.is_empty()); + assert_eq!(only.warnings, vec![DecodeWarning::ExtraSamplesDropped { format: Format::Tiff, count: MAX_EXTRA_SAMPLES as u32 }]); + assert_eq!(only.data(), back.data()); + } + assert!(matches!(with(image_only - 1), Err(CodecError::LimitExceeded(_))), "the image itself must still fit"); +} + +/// The value field of a little-endian classic TIFF's ExtraSamples entry (tag 338, SHORT) holding +/// two values, so they sit in the entry itself. +fn extra_samples_values(bytes: &mut [u8]) -> &mut [u8] { + assert_eq!(&bytes[..4], b"II*\0"); + let entry = [0x52, 0x01, 3, 0, 2, 0, 0, 0]; + let at = bytes.windows(8).position(|w| w == entry).expect("an ExtraSamples entry with two values"); + &mut bytes[at + 8..at + 12] +} + +/// ExtraSamples values above 2 are reserved (TIFF 6.0, p. 31): they read as 0 (unspecified) and +/// are written as 0, so a save never carries a reserved value. +#[test] +fn reserved_extra_sample_kinds_read_as_unspecified() { + let n = 4 * 4; + let mut img = synth(4, 4, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: vec![7; n] }, ExtraSample { kind: 0, data: vec![9; n] }]; + let mut bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + // A file from another writer with reserved values 3 and 65535. + let values = extra_samples_values(&mut bytes); + assert_eq!(values, [0, 0, 0, 0]); + values.copy_from_slice(&[3, 0, 0xFF, 0xFF]); + let back = decode(&bytes).unwrap(); + assert_eq!(back.meta.extra_samples.iter().map(|e| e.kind).collect::>(), [0, 0]); + assert_eq!(back.layout(), ChannelLayout::Rgb, "a reserved value is no alpha"); + assert_eq!(back.meta.extra_samples[0].data, vec![7; n]); + assert_eq!(back.meta.extra_samples[1].data, vec![9; n]); + + // An image that carries a reserved kind saves it as 0. + img.meta.extra_samples[0].kind = 3; + img.meta.extra_samples[1].kind = u16::MAX; + let mut bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert_eq!(extra_samples_values(&mut bytes), [0, 0, 0, 0]); +} + +/// Converting an image's sample type converts its extra samples with it, so they keep matching +/// the image: a 16-bit float TIFF's spot planes survive the 32-bit float a layered import uses. +#[test] +fn extra_samples_follow_a_sample_type_conversion() { + let (w, h) = (3u32, 2u32); + let n = (w * h) as usize; + let values = [0.0f32, 0.25, 0.5, 0.75, 1.0, 0.125]; + let half = |v: f32| f16::from_f32(v).to_bits().to_ne_bytes(); + let rgb: Vec = (0..n * 3).flat_map(|_| half(0.5)).collect(); + let mut img = Image::from_raw(w, h, ChannelLayout::Rgb, SampleType::F16, rgb).unwrap(); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: values.iter().flat_map(|v| half(*v)).collect() }]; + + let wide = img.convert(ChannelLayout::Rgb, SampleType::F32); + let plane: Vec = wide.meta.extra_samples[0].data.as_chunks::<4>().0.iter().map(|c| f32::from_ne_bytes(*c)).collect(); + assert_eq!(plane, values); + + // To 8 bits, the plane quantizes exactly as a gray image of the same values does. + let narrow = wide.convert(ChannelLayout::Rgb, SampleType::U8); + let gray = Image::from_raw(w, h, ChannelLayout::Gray, SampleType::F32, wide.meta.extra_samples[0].data.clone()).unwrap(); + assert_eq!(narrow.meta.extra_samples[0].data, gray.convert(ChannelLayout::Gray, SampleType::U8).data()); + assert_eq!(narrow.meta.extra_samples[0].data.len(), n); +} + +#[test] +fn absent_and_empty_tags_are_not_written() { + let mut img = synth(4, 4, ChannelLayout::Rgb, SampleType::U8, 1, 0.0); + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert_eq!(tiff_photoshop_tags(&bytes), (None, None)); + assert!(decode(&bytes).unwrap().meta.photoshop_layers.is_none()); + img.meta.photoshop_layers = Some(Vec::new()); + img.meta.photoshop_resources = Some(Vec::new()); + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap(); + assert_eq!(tiff_photoshop_tags(&bytes), (None, None)); +} + +/// A minimal uncompressed 2×1 gray TIFF with a 37724 tag, in either byte order, classic or +/// BigTIFF: `payload` is stored inline when it fits the value field, at an offset otherwise. +fn hand_made(little: bool, big: bool, payload: &[u8]) -> Vec { + let w16 = |v: u16| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() }; + let w32 = |v: u32| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() }; + let w64 = |v: u64| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() }; + let mut f = Vec::new(); + f.extend_from_slice(if little { b"II" } else { b"MM" }); + let header_len = if big { 16 } else { 8 }; + let pixels_at = header_len; + let ifd_at = pixels_at + 2; + f.extend(w16(if big { 43 } else { 42 })); + if big { + f.extend(w16(8)); + f.extend(w16(0)); + f.extend(w64(ifd_at as u64)); + } else { + f.extend(w32(ifd_at as u32)); + } + f.extend_from_slice(&[0x40, 0xc0]); + // Entries: width, length, bits, compression, photometric, strip offsets, samples, rows per + // strip, strip byte counts, 37724. + let tags: [(u16, u16, u64, u64); 10] = [ + (256, 3, 1, 2), + (257, 3, 1, 1), + (258, 3, 1, 8), + (259, 3, 1, 1), + (262, 3, 1, 1), + (273, 4, 1, pixels_at as u64), + (277, 3, 1, 1), + (278, 3, 1, 1), + (279, 4, 1, 2), + (37724, 7, payload.len() as u64, 0), + ]; + let entry_len = if big { 20 } else { 12 }; + let inline = if big { 8 } else { 4 }; + let count_len = if big { 8 } else { 2 }; + let next_len = if big { 8 } else { 4 }; + let data_at = ifd_at + count_len + tags.len() * entry_len + next_len; + let mut entries = Vec::new(); + for (tag, ty, count, value) in tags { + entries.extend(w16(tag)); + entries.extend(w16(ty)); + entries.extend(if big { w64(count) } else { w32(count as u32) }); + let mut v = Vec::new(); + if tag == 37724 { + if payload.len() <= inline { + v.extend_from_slice(payload); + } else { + v.extend(if big { w64(data_at as u64) } else { w32(data_at as u32) }); + } + } else if ty == 3 { + v.extend(w16(value as u16)); + } else { + v.extend(if big { w64(value) } else { w32(value as u32) }); + } + v.resize(inline, 0); + entries.extend(v); + } + f.extend(if big { w64(tags.len() as u64) } else { w16(tags.len() as u16) }); + f.extend(entries); + f.extend(if big { w64(0) } else { w32(0) }); + assert_eq!(f.len(), data_at); + if payload.len() > inline { + f.extend_from_slice(payload); + } + f +} + +#[test] +fn hand_made_files_in_every_layout() { + let long = b"Adobe Photoshop Document Data Block\0MIB8".to_vec(); + for little in [false, true] { + for big in [false, true] { + for payload in [&b"ab"[..], &b"abcd"[..], &b"abcdefgh"[..], &long[..]] { + let f = hand_made(little, big, payload); + let (res, layers) = tiff_photoshop_tags(&f); + assert_eq!(res, None); + assert_eq!(layers, Some(payload), "little {little} big {big} len {}", payload.len()); + let img = decode(&f).unwrap(); + assert_eq!(img.dimensions(), (2, 1)); + assert_eq!(img.meta.photoshop_layers.as_deref(), Some(payload)); + } + } + } +} + +#[test] +fn malformed_tags_read_as_absent_without_panicking() { + let long = vec![7u8; 40]; + for little in [false, true] { + for big in [false, true] { + let f = hand_made(little, big, &long); + // Truncated anywhere: never a panic; the payload is absent once it is cut. + for cut in 0..f.len() { + let (_, layers) = tiff_photoshop_tags(&f[..cut]); + if cut < f.len() { + assert!(layers.is_none_or(|l| l.len() == 40)); + } + let _ = decode(&f[..cut]); + } + // Bit flips in the directory. + for i in 0..f.len() - 40 { + let mut g = f.clone(); + g[i] ^= 0xff; + let _ = tiff_photoshop_tags(&g); + let _ = decode(&g); + } + } + } + assert_eq!(tiff_photoshop_tags(b""), (None, None)); + assert_eq!(tiff_photoshop_tags(b"II*\0"), (None, None)); + assert_eq!(tiff_photoshop_tags(b"MM\0*\xff\xff\xff\xff"), (None, None)); + // A count that overflows the file. + let mut f = hand_made(true, false, &long); + let at = f.windows(2).position(|w| w == 37724u16.to_le_bytes()).unwrap(); + f[at + 4..at + 8].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(tiff_photoshop_tags(&f).1, None); +} diff --git a/crates/image-codecs/tests/it/tiff_pyramid.rs b/crates/image-codecs/tests/it/tiff_pyramid.rs new file mode 100644 index 000000000..adecd3fc7 --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_pyramid.rs @@ -0,0 +1,194 @@ +//! TIFF "Save Image Pyramid" (#8827): reduced-resolution levels after the full image, each half +//! the size of the one before, that every reader skips when it opens the file. + +use craft_image_codecs::*; + +fn pyramid() -> EncodeOptions { + EncodeOptions { tiff_pyramid: true, ..Default::default() } +} + +/// An image of `layout`/`sample` where every sample is `v` (on the 0–1 scale). +fn solid(w: u32, h: u32, layout: ChannelLayout, sample: SampleType, v: f32) -> Image { + let n = w as usize * h as usize * layout.channels(); + let data: Vec = match sample { + SampleType::U8 => vec![(v * 255.0).round() as u8; n], + SampleType::U16 => (0..n).flat_map(|_| ((v * 65535.0).round() as u16).to_ne_bytes()).collect(), + SampleType::F32 => (0..n).flat_map(|_| v.to_ne_bytes()).collect(), + SampleType::F16 => (0..n).flat_map(|_| f16::from_f32(v).to_bits().to_ne_bytes()).collect(), + }; + Image::from_raw(w, h, layout, sample, data).unwrap() +} + +fn sizes(bytes: &[u8]) -> Vec<(TiffPageKind, u32, u32)> { + tiff_info(bytes).unwrap().pages.iter().map(|p| (p.kind, p.width, p.height)).collect() +} + +#[test] +fn levels_halve_down_to_the_minimum() { + use TiffPageKind::*; + let img = solid(300, 200, ChannelLayout::Rgb, SampleType::U8, 0.5); + let bytes = encode(&img, Format::Tiff, &pyramid()).unwrap(); + // 300 → 150 → 75 → 38: the first level no longer than 64 px is the last. + assert_eq!(sizes(&bytes), [(Page, 300, 200), (ReducedResolution, 150, 100), (ReducedResolution, 75, 50), (ReducedResolution, 38, 25)]); + // Opening the file opens the full image, and the levels are not extra pages. + let back = decode(&bytes).unwrap(); + assert_eq!(back.dimensions(), (300, 200)); + assert_eq!(back.data(), img.data()); + assert!(!back.warnings.iter().any(|w| matches!(w, DecodeWarning::MorePages { .. })), "{:?}", back.warnings); +} + +#[test] +fn off_by_default_and_none_for_small_images() { + let img = solid(300, 200, ChannelLayout::Rgb, SampleType::U8, 0.5); + assert_eq!(sizes(&encode(&img, Format::Tiff, &EncodeOptions::default()).unwrap()).len(), 1); + let small = solid(64, 10, ChannelLayout::Rgb, SampleType::U8, 0.5); + assert_eq!(sizes(&encode(&small, Format::Tiff, &pyramid()).unwrap()).len(), 1); + // A 1-px-wide strip still halves its long side. + let strip = solid(1, 200, ChannelLayout::Gray, SampleType::U8, 0.5); + let s = sizes(&encode(&strip, Format::Tiff, &pyramid()).unwrap()); + assert_eq!(s.iter().map(|&(_, w, h)| (w, h)).collect::>(), [(1, 200), (1, 100), (1, 50)]); +} + +#[test] +fn every_depth_layout_and_compression() { + let layouts = [ + ChannelLayout::Gray, + ChannelLayout::GrayA, + ChannelLayout::Rgb, + ChannelLayout::Rgba, + ChannelLayout::Cmyk, + ChannelLayout::CmykA, + ChannelLayout::Lab, + ]; + for layout in layouts { + for sample in [SampleType::U8, SampleType::U16, SampleType::F32] { + for compression in [TiffCompression::None, TiffCompression::Lzw, TiffCompression::Deflate, TiffCompression::PackBits] { + let img = solid(120, 70, layout, sample, 0.6); + let opts = EncodeOptions { tiff_compression: compression, ..pyramid() }; + let bytes = encode(&img, Format::Tiff, &opts).unwrap_or_else(|e| panic!("{layout:?} {sample:?} {compression:?}: {e}")); + let info = tiff_info(&bytes).unwrap(); + assert_eq!(info.pages.len(), 2, "{layout:?} {sample:?} {compression:?}"); + let level = decode_tiff_page(&bytes, 1, &DecodeOptions::default()).unwrap(); + assert_eq!(level.dimensions(), (60, 35)); + assert_eq!(level.layout(), decode(&bytes).unwrap().layout(), "{layout:?} {sample:?}"); + // A solid image stays the same colour at every level. + let full = decode(&bytes).unwrap().convert(level.layout(), SampleType::F32); + let level = level.convert(level.layout(), SampleType::F32); + let (a, b) = (full.to_f32_samples().unwrap(), level.to_f32_samples().unwrap()); + for (c, v) in b.iter().take(level.layout().channels()).enumerate() { + assert!((v - a[c]).abs() < 1e-3, "{layout:?} {sample:?} channel {c}: {v} vs {}", a[c]); + } + } + } + } +} + +/// A 128×128 image of 2×2 `tile`s (top left, top right, bottom left, bottom right; alpha last), +/// and the first pixel of its first reduced level. +fn first_level_pixel(layout: ChannelLayout, tile: [&[u8]; 4]) -> Vec { + let mut data = Vec::new(); + for _ in 0..64 { + for x in 0..128 { + data.extend_from_slice(tile[x % 2]); + } + for x in 0..128 { + data.extend_from_slice(tile[2 + x % 2]); + } + } + let img = Image::from_raw(128, 128, layout, SampleType::U8, data).unwrap(); + let bytes = encode(&img, Format::Tiff, &pyramid()).unwrap(); + let level = decode_tiff_page(&bytes, 1, &DecodeOptions::default()).unwrap(); + assert_eq!(level.dimensions(), (64, 64), "{layout:?}"); + assert_eq!(level.layout(), layout); + level.data()[..layout.channels()].to_vec() +} + +#[test] +fn colour_is_weighted_by_alpha() { + // One opaque pixel and three transparent ones of another colour: the level keeps the opaque + // colour at a quarter alpha. Unweighted, the transparent colour would bleed in (pink, not red). + assert_eq!( + first_level_pixel(ChannelLayout::Rgba, [&[255, 0, 0, 255], &[255, 255, 255, 0], &[255, 255, 255, 0], &[255, 255, 255, 0]]), + [255, 0, 0, 64] + ); + // Gray and CMYK reach the same path with other channel counts. + assert_eq!(first_level_pixel(ChannelLayout::GrayA, [&[40, 255], &[255, 0], &[255, 0], &[255, 0]]), [40, 64]); + assert_eq!(first_level_pixel(ChannelLayout::CmykA, [&[200, 10, 30, 90, 255], &[0; 5], &[0; 5], &[0; 5]]), [200, 10, 30, 90, 64]); +} + +#[test] +fn colour_is_weighted_between_translucent_pixels() { + // Two half-transparent pixels of different colours (red, blue) and two clear ones: the level + // averages the two by their alpha (purple), where copying either pixel would give red or blue. + let tile: [&[u8]; 4] = [&[255, 0, 0, 128], &[0, 0, 255, 128], &[0; 4], &[0; 4]]; + assert_eq!(first_level_pixel(ChannelLayout::Rgba, tile), [128, 0, 128, 64]); +} + +#[test] +fn layered_and_tagged_data_stay_on_the_full_image() { + let mut img = solid(200, 100, ChannelLayout::Rgb, SampleType::U8, 0.25); + img.icc = Some(photocraft_cms_free_profile()); + img.meta.photoshop_layers = Some(vec![1, 2, 3, 4]); + img.meta.extra_samples = vec![ExtraSample { kind: 0, data: vec![9; 200 * 100] }]; + let bytes = encode(&img, Format::Tiff, &pyramid()).unwrap(); + let info = tiff_info(&bytes).unwrap(); + assert_eq!(info.pages.len(), 3); + // Only the full image carries the layers and the extra samples. + assert_eq!(info.pages[0].samples_per_pixel, 4); + assert!(info.pages[1..].iter().all(|p| p.samples_per_pixel == 3)); + assert_eq!(tiff_photoshop_tags(&bytes).1, Some(&[1u8, 2, 3, 4][..])); + let level = decode_tiff_page(&bytes, 1, &DecodeOptions::default()).unwrap(); + assert_eq!(level.icc, img.icc, "levels keep the profile so they show the same colours"); + assert!(level.meta.extra_samples.is_empty()); +} + +/// Any bytes a decoder keeps as the ICC profile: the codec stores it without parsing. +fn photocraft_cms_free_profile() -> Vec { + (0..=255u8).collect() +} + +#[test] +fn levels_average_exactly_at_every_depth() { + // One 2×2 block of gray values 0, 0.25, 0.5, 1 averages to 0.4375; a 2×2 block with half + // transparent pixels keeps only the opaque colour and averages the alpha. + for sample in [SampleType::U16, SampleType::F32] { + let vals = [0.0f32, 0.25, 0.5, 1.0]; + let mut img = solid(130, 2, ChannelLayout::Gray, sample, 0.0); + let w = 130; + for (i, v) in vals.iter().enumerate() { + let (x, y) = (i % 2, i / 2); + let at = (y * w + x) * sample.bytes(); + let bytes: Vec = match sample { + SampleType::U16 => ((v * 65535.0).round() as u16).to_ne_bytes().to_vec(), + _ => v.to_ne_bytes().to_vec(), + }; + img.data_mut()[at..at + bytes.len()].copy_from_slice(&bytes); + } + let bytes = encode(&img, Format::Tiff, &pyramid()).unwrap(); + let level = decode_tiff_page(&bytes, 1, &DecodeOptions::default()).unwrap().convert(ChannelLayout::Gray, SampleType::F32); + let v = level.to_f32_samples().unwrap()[0]; + assert!((v - 0.4375).abs() < 1e-4, "{sample:?}: {v}"); + } + let mut data = Vec::new(); + for _ in 0..2 { + for x in 0..130u32 { + // RGBA16: opaque blue on even columns, transparent red on odd. + let px: [u16; 4] = if x % 2 == 0 { [0, 0, 65535, 65535] } else { [65535, 0, 0, 0] }; + data.extend(px.iter().flat_map(|v| v.to_ne_bytes())); + } + } + let img = Image::from_raw(130, 2, ChannelLayout::Rgba, SampleType::U16, data).unwrap(); + let bytes = encode(&img, Format::Tiff, &pyramid()).unwrap(); + let level = decode_tiff_page(&bytes, 1, &DecodeOptions::default()).unwrap(); + assert_eq!(level.to_u16_samples().unwrap()[..4], [0, 0, 65535, 32768]); +} + +#[test] +fn bigtiff_files_carry_the_pyramid_too() { + let img = solid(300, 200, ChannelLayout::Rgba, SampleType::U8, 0.5); + let bytes = encode(&img, Format::Tiff, &EncodeOptions { tiff_bigtiff: true, ..pyramid() }).unwrap(); + let info = tiff_info(&bytes).unwrap(); + assert!(info.big_tiff); + assert_eq!(info.pages.iter().map(|p| (p.kind, p.width)).collect::>().len(), 4); + assert_eq!(decode_tiff_page(&bytes, 3, &DecodeOptions::default()).unwrap().dimensions(), (38, 25)); +} diff --git a/crates/image-codecs/tests/it/tiff_regressions.rs b/crates/image-codecs/tests/it/tiff_regressions.rs new file mode 100644 index 000000000..2df89fbbb --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_regressions.rs @@ -0,0 +1,176 @@ +use crate::common; + +use common::tiffgen::*; +use craft_image_codecs::{Image, decode, exif_orientation, upright_exif}; + +fn classic_white_is_zero() -> Vec { + let width = 4u16; + let height = 1u16; + let pixels = [0u8, 64, 192, 255]; + let entries: [(u16, u16, u32); 9] = [ + (256, 3, u32::from(width)), + (257, 3, u32::from(height)), + (258, 3, 8), + (259, 3, 1), + (262, 3, 0), // WhiteIsZero + (273, 4, 122), // StripOffsets: immediately after IFD0. + (277, 3, 1), + (278, 3, u32::from(height)), + (279, 4, pixels.len() as u32), + ]; + let mut bytes = b"II*\0\x08\0\0\0".to_vec(); + bytes.extend_from_slice(&(entries.len() as u16).to_le_bytes()); + for (tag, field_type, value) in entries { + bytes.extend_from_slice(&tag.to_le_bytes()); + bytes.extend_from_slice(&field_type.to_le_bytes()); + bytes.extend_from_slice(&1u32.to_le_bytes()); + if field_type == 3 { + bytes.extend_from_slice(&(value as u16).to_le_bytes()); + bytes.extend_from_slice(&[0; 2]); + } else { + bytes.extend_from_slice(&value.to_le_bytes()); + } + } + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&pixels); + bytes +} + +fn big_tiff_gray(stored: &[u8], width: u16, height: u16, orientation: u16) -> Vec { + let mut entries = vec![ + (256u16, 3u16, 1u64, u64::from(width)), + (257, 3, 1, u64::from(height)), + (258, 3, 1, 8), + (259, 3, 1, 1), + (262, 3, 1, 1), // BlackIsZero + (273, 4, 1, 0), // StripOffsets, patched below. + (274, 3, 1, u64::from(orientation)), + (277, 3, 1, 1), + (278, 3, 1, u64::from(height)), + (279, 4, 1, stored.len() as u64), + ]; + entries.sort_by_key(|entry| entry.0); + let pixel_offset = 16u64 + 8 + entries.len() as u64 * 20 + 8; + if let Some(strip_offset) = entries.iter_mut().find(|entry| entry.0 == 273) { + strip_offset.3 = pixel_offset; + } + + let mut bytes = b"II+\0\x08\0\0\0\x10\0\0\0\0\0\0\0".to_vec(); + bytes.extend_from_slice(&(entries.len() as u64).to_le_bytes()); + for (tag, field_type, count, value) in entries { + bytes.extend_from_slice(&tag.to_le_bytes()); + bytes.extend_from_slice(&field_type.to_le_bytes()); + bytes.extend_from_slice(&count.to_le_bytes()); + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes.extend_from_slice(&0u64.to_le_bytes()); + bytes.extend_from_slice(stored); + bytes +} + +fn stored_for_orientation_6_or_8(orientation: u16) -> Vec { + match orientation { + 6 => vec![3, 6, 2, 5, 1, 4], + 8 => vec![4, 1, 5, 2, 6, 3], + _ => unreachable!(), + } +} + +#[test] +fn white_is_zero_is_normalized_once_by_the_tiff_decoder() { + let image = decode(&classic_white_is_zero()).expect("synthetic WhiteIsZero TIFF should decode"); + + assert_eq!(image.dimensions(), (4, 1)); + assert_eq!(image.data(), &[255, 191, 63, 0]); +} + +#[test] +fn big_tiff_orientation_6_and_8_are_applied_to_decoded_pixels() { + let upright = [1u8, 2, 3, 4, 5, 6]; + + for orientation in [6, 8] { + let bytes = big_tiff_gray(&stored_for_orientation_6_or_8(orientation), 2, 3, orientation); + let image: Image = decode(&bytes).expect("synthetic BigTIFF should decode"); + + assert_eq!(exif_orientation(&bytes), orientation); + assert_eq!(exif_orientation(&upright_exif(&bytes)), 1); + assert_eq!(image.dimensions(), (3, 2), "orientation {orientation}"); + assert_eq!(image.data(), &upright, "orientation {orientation}"); + } +} + +fn with_compression(mut dir: Dir, code: u64) -> Dir { + for entry in &mut dir.entries { + if entry.0 == 259 { + entry.2 = vec![V::N(code)]; + } + } + dir +} + +#[test] +fn jpeg_compressed_tiff_decodes() { + // A flat mid-grey 16x16 image as a baseline JPEG stored in one strip (Compression = 7). + let mut jpeg = Vec::new(); + let gray = vec![100u8; 16 * 16]; + jpeg_encoder::Encoder::new(&mut jpeg, 95).encode(&gray, 16, 16, jpeg_encoder::ColorType::Luma).expect("encode the JPEG strip"); + let mut dir = with_compression(strips(16, 16, 8, 1, 1, 16, &[0; 256]), 7); + dir.chunks = vec![jpeg]; + let bytes = build(true, false, &[dir], &[0]).bytes; + + let image = decode(&bytes).expect("a JPEG-compressed TIFF should decode"); + assert_eq!(image.dimensions(), (16, 16)); + assert!(image.data().iter().all(|&v| v.abs_diff(100) <= 4), "{:?}", image.data().get(..4)); +} + +#[test] +fn ccitt_group4_tiff_decodes() { + // 8x3 bilevel: a white row, then two black rows. Bits: V0 | H(white 0, black 8) | V0 V0 | EOFB. + let bits = "1".to_string() + "001" + "00110101" + "000101" + "11" + "000000000001" + "000000000001"; + let mut data = Vec::new(); + for chunk in bits.as_bytes().chunks(8) { + let mut byte = 0u8; + for (i, c) in chunk.iter().enumerate() { + if *c == b'1' { + byte |= 0x80 >> i; + } + } + data.push(byte); + } + let mut dir = with_compression(strips(8, 3, 1, 1, 0, 3, &[0; 3]), 4); + dir.chunks = vec![data]; + let bytes = build(true, false, &[dir], &[0]).bytes; + + let image = decode(&bytes).expect("a CCITT G4 TIFF should decode"); + assert_eq!(image.dimensions(), (8, 3)); + let d = image.data(); + assert_eq!(d.len(), 24, "{d:?}"); + assert!(d[..8].iter().all(|&v| v == d[0]), "{d:?}"); + assert!(d[8..].iter().all(|&v| v == d[8]), "{d:?}"); + assert_ne!(d[0], d[8], "{d:?}"); + assert_eq!(d[0], 255, "white row of a WhiteIsZero image"); +} + +/// #4484: a TIFF saved with the default options must open in software that only reads +/// baseline TIFF (embroidery and plotter tools, older readers). Like Photoshop's default, that +/// means uncompressed strips with no predictor, classic offsets and unassociated alpha. +#[test] +fn default_tiff_is_baseline() { + use craft_image_codecs::{ChannelLayout, EncodeOptions, Format, SampleType, encode}; + use std::io::Cursor; + use tiff::tags::Tag; + + for layout in [ChannelLayout::Rgb, ChannelLayout::Rgba] { + let img = common::synth(33, 21, layout, SampleType::U8, 7, 0.4); + let bytes = encode(&img, Format::Tiff, &EncodeOptions::default()).expect("encode"); + assert!(bytes.starts_with(b"II*\0") || bytes.starts_with(b"MM\0*"), "{layout:?}: not a classic TIFF"); + let mut dec = tiff::decoder::Decoder::new(Cursor::new(&bytes[..])).expect("open"); + assert_eq!(dec.get_tag_u32(Tag::Compression).ok(), Some(1), "{layout:?}: Compression"); + assert_eq!(dec.find_tag(Tag::Predictor).ok().flatten().and_then(|v| v.into_u16().ok()).unwrap_or(1), 1, "{layout:?}: Predictor"); + assert!(dec.find_tag(Tag::TileWidth).ok().flatten().is_none(), "{layout:?}: tiled"); + assert_eq!(dec.get_tag_u32(Tag::PlanarConfiguration).ok().unwrap_or(1), 1, "{layout:?}: planar"); + let extra = dec.find_tag(Tag::ExtraSamples).ok().flatten().and_then(|v| v.into_u16_vec().ok()); + assert_eq!(extra, if layout == ChannelLayout::Rgba { Some(vec![2]) } else { None }, "{layout:?}: ExtraSamples"); + assert_eq!(decode(&bytes).expect("decode").data(), img.data(), "{layout:?}: round trip"); + } +} diff --git a/crates/image-codecs/tests/it/tiff_ycbcr.rs b/crates/image-codecs/tests/it/tiff_ycbcr.rs new file mode 100644 index 000000000..0f456bf84 --- /dev/null +++ b/crates/image-codecs/tests/it/tiff_ycbcr.rs @@ -0,0 +1,258 @@ +//! YCbCr TIFFs (PhotometricInterpretation 6), as scanners write them: JPEG-compressed with 2×2 +//! chroma subsampling, and uncompressed with ReferenceBlackWhite and YCbCrCoefficients (#10316). +//! Every file here is hand-built by `common::tiffgen`. + +use crate::common::sample_icc; +use crate::common::tiffgen::*; +use craft_image_codecs::*; + +const COMPRESSION: u16 = 259; +const SUBSAMPLING: u16 = 530; +const YCBCR_COEFFICIENTS: u16 = 529; +const REFERENCE_BLACK_WHITE: u16 = 532; +const ICC_PROFILE: u16 = 34675; + +/// A directory whose Compression is `c`. +fn compressed(mut d: Dir, c: u64) -> Dir { + d.entries.retain(|e| e.0 != COMPRESSION); + d.tag(COMPRESSION, SHORT, &[c]) +} + +/// A little-endian RATIONAL value for `tiffgen`. +fn rational(num: u32, den: u32) -> u64 { + u64::from(num) | (u64::from(den) << 32) +} + +/// The image's pixels as RGB bytes. +fn rgb(img: &Image) -> Vec { + img.to_rgba8().as_chunks::<4>().0.iter().flat_map(|[r, g, b, _]| [*r, *g, *b]).collect() +} + +/// A smooth colour test card, `w`×`h` RGB. +fn card(w: u32, h: u32) -> Vec { + let mut v = Vec::new(); + for y in 0..h { + for x in 0..w { + v.extend([(x * 255 / (w - 1)) as u8, (y * 255 / (h - 1)) as u8, (255 - x * 255 / (w - 1)) as u8]); + } + } + v +} + +/// A 4:2:0 baseline JPEG stream of `rgb`. +fn jpeg(rgb: &[u8], w: u32, h: u32) -> Vec { + let img = Image::from_raw(w, h, ChannelLayout::Rgb, SampleType::U8, rgb.to_vec()).unwrap(); + encode(&img, Format::Jpeg, &EncodeOptions { jpeg_chroma_subsampling: true, ..Default::default() }).unwrap() +} + +#[test] +fn jpeg_compressed_ycbcr_with_subsampled_chroma_reads_as_rgb() { + let (w, h) = (32, 24); + let stream = jpeg(&card(w, h), w, h); + // The stream really is subsampled: the luma component samples 2×2 in the frame header. + let sof = stream.windows(2).position(|m| m == [0xFF, 0xC0]).expect("baseline frame header"); + assert_eq!(stream[sof + 11], 0x22, "luma sampling factors"); + let dir = compressed(strips(w, h, 8, 3, 6, h, &stream), 7).tag(SUBSAMPLING, SHORT, &[2, 2]); + let file = build(true, false, &[dir], &[0]).bytes; + + let img = decode(&file).unwrap(); + assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (w, h, ChannelLayout::Rgb, SampleType::U8)); + // The oracle: the same JPEG stream decoded to RGB by an independent decoder. + let want = image::load_from_memory_with_format(&stream, image::ImageFormat::Jpeg).unwrap().to_rgb8().into_raw(); + let worst = rgb(&img).iter().zip(&want).map(|(g, o)| g.abs_diff(*o)).max().unwrap(); + assert!(worst <= 2, "largest difference from the oracle: {worst}"); +} + +#[test] +fn uncompressed_ycbcr_uses_the_default_conversion() { + // Black, white, mid gray and JFIF's pure red, without subsampling. + let px = [0, 128, 128, 255, 128, 128, 128, 128, 128, 76, 85, 255]; + let dir = strips(4, 1, 8, 3, 6, 1, &px).tag(SUBSAMPLING, SHORT, &[1, 1]); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!(img.layout(), ChannelLayout::Rgb); + assert_eq!(rgb(&img), [0, 0, 0, 255, 255, 255, 128, 128, 128, 254, 0, 0]); +} + +#[test] +fn reference_black_white_sets_the_code_range() { + // Studio range: Y 16–235, chroma 128 ± 112. + let rbw = [rational(16, 1), rational(235, 1), rational(128, 1), rational(240, 1), rational(128, 1), rational(240, 1)]; + let dir = strips(2, 1, 8, 3, 6, 1, &[16, 128, 128, 235, 128, 128]).tag(SUBSAMPLING, SHORT, &[1, 1]).tag(REFERENCE_BLACK_WHITE, RATIONAL, &rbw); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!(rgb(&img), [0, 0, 0, 255, 255, 255]); +} + +#[test] +fn weights_that_do_not_sum_to_one_keep_grays_neutral() { + // 0.25 each (sum 0.75): TIFF 6.0's full inverse would scale green's Y by (1 − Lr − Lb) / Lg + // = 2 and give mid gray G 255 (clamped). Like libtiff, green keeps Y, so gray stays gray, and + // the weights still set the chroma terms: R = Y + 1.5·Cr, B = Y + 1.5·Cb. + let weights = [rational(1, 4), rational(1, 4), rational(1, 4)]; + let dir = strips(2, 1, 8, 3, 6, 1, &[128, 128, 128, 128, 64, 192]).tag(SUBSAMPLING, SHORT, &[1, 1]).tag(YCBCR_COEFFICIENTS, RATIONAL, &weights); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + // Green for the second pixel: 128 − (Lb·1.5/Lg)·Cb − (Lr·1.5/Lg)·Cr = 128 + 96 − 96. + assert_eq!(rgb(&img), [128, 128, 128, 224, 128, 32]); +} + +#[test] +fn ycbcr_coefficients_set_the_luma_weights() { + // Weights 0.25, 0.5, 0.25 with Cb −64 and Cr +64: R = Y + 1.5·Cr, B = Y + 1.5·Cb, and G keeps Y + // (BT.601 would give R 218 and B 15). + let weights = [rational(1, 4), rational(1, 2), rational(1, 4)]; + let dir = strips(1, 1, 8, 3, 6, 1, &[128, 64, 192]).tag(SUBSAMPLING, SHORT, &[1, 1]).tag(YCBCR_COEFFICIENTS, RATIONAL, &weights); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!(rgb(&img), [224, 128, 32]); +} + +#[test] +fn ycbcr_tags_with_the_wrong_count_use_the_defaults() { + // Four weights and seven black/white codes: both tags are malformed, so BT.601 and full range + // apply (the weights alone would give 224, 128, 32). + let weights = [rational(1, 4), rational(1, 2), rational(1, 4), rational(1, 4)]; + let rbw = [rational(16, 1), rational(235, 1), rational(128, 1), rational(240, 1), rational(128, 1), rational(240, 1), rational(0, 1)]; + let dir = strips(1, 1, 8, 3, 6, 1, &[128, 64, 192]).tag(SUBSAMPLING, SHORT, &[1, 1]).tag(YCBCR_COEFFICIENTS, RATIONAL, &weights).tag( + REFERENCE_BLACK_WHITE, + RATIONAL, + &rbw, + ); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!(rgb(&img), [218, 104, 15]); +} + +#[test] +fn sixteen_bit_ycbcr_reads_as_rgb() { + let px: Vec = [65535u16, 32768, 32768, 0, 32768, 32768].iter().flat_map(|v| v.to_le_bytes()).collect(); + let dir = strips(2, 1, 16, 3, 6, 1, &px).tag(SUBSAMPLING, SHORT, &[1, 1]); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!((img.layout(), img.sample_type()), (ChannelLayout::Rgb, SampleType::U16)); + let n = img.to_normalized(); + assert!(n[..3].iter().all(|&v| v == 1.0) && n[3..].iter().all(|&v| v == 0.0), "{n:?}"); +} + +#[test] +#[cfg(target_pointer_width = "64")] +fn sixteen_bit_ycbcr_bands_match_whole_decode_with_a_missing_last_strip() { + // On 32-bit targets max_alloc also bounds the whole image, so a small allocation cap + // cannot force these tiny strips into separate output groups there. + let (w, h, rows) = (4u32, 5u32, 2u32); + let pixels = [[0u16, 32768, 32768], [65535, 32768, 32768], [19595, 21710, 65535], [32768, 40960, 24576]]; + let samples: Vec = (0..w * h).flat_map(|i| pixels[i as usize % pixels.len()]).collect(); + let weights = [rational(1, 4), rational(1, 2), rational(1, 4)]; + let icc = sample_icc(512); + for little in [true, false] { + for missing in [false, true] { + let dir = strips(w, h, 16, 3, 6, rows, &u16s(&samples, little)) + .tag(SUBSAMPLING, SHORT, &[1, 1]) + .tag(YCBCR_COEFFICIENTS, RATIONAL, &weights) + .bytes(ICC_PROFILE, UNDEFINED, &icc); + let mut file = build(little, false, &[dir], &[0]); + if missing { + let past = file.bytes.len() as u64 + 4096; + patch_value(&mut file, little, false, 0, 273, 2, past); + } + let want = decode_with(&file.bytes, &DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap(); + let limits = Limits { max_alloc: u64::from(w * rows * 3 * 2), ..Limits::banded() }; + let mut bands = TiffBands::open(&file.bytes, None, &limits).unwrap(); + assert_eq!((bands.dimensions(), bands.layout(), bands.sample_type()), ((w, h), ChannelLayout::Rgb, SampleType::U16)); + let mut got = Image::new(w, h, bands.layout(), bands.sample_type()).unwrap(); + let mut positions = Vec::new(); + bands + .decode(|x, y, group: &Image| -> Result<(), CodecError> { + positions.push((x, y, group.height())); + assert_eq!((x, group.width(), group.layout(), group.sample_type()), (0, w, ChannelLayout::Rgb, SampleType::U16)); + let at = y as usize * w as usize * 3 * 2; + got.data_mut()[at..at + group.data().len()].copy_from_slice(group.data()); + Ok(()) + }) + .unwrap(); + assert_eq!(positions, [(0, 0, 2), (0, 2, 2), (0, 4, 1)], "one output group per strip, including the short last strip"); + assert_eq!(got.data(), want.data(), "little endian: {little}, missing last strip: {missing}"); + assert_eq!(bands.icc, want.icc); + assert_eq!(bands.meta, want.meta); + assert_eq!(bands.warnings, want.warnings); + assert_eq!(bands.warnings.iter().any(|w| matches!(w, DecodeWarning::Truncated { .. })), missing); + } + } +} + +#[test] +fn unsupported_ycbcr_files_fail_cleanly() { + // Four samples (an extra sample next to YCbCr). + let four = strips(1, 1, 8, 4, 6, 1, &[128, 128, 128, 255]).tag(SUBSAMPLING, SHORT, &[1, 1]); + assert!(decode(&build(true, false, &[four], &[0]).bytes).is_err()); + // Subsampled chroma without JPEG compression (the default subsampling is 2×2). + let sub = strips(2, 2, 8, 3, 6, 2, &[128; 12]); + assert!(decode(&build(true, false, &[sub], &[0]).bytes).is_err()); + // A JPEG strip that is not JPEG data. + let junk = compressed(strips(16, 16, 8, 3, 6, 16, &[0x5A; 64]), 7).tag(SUBSAMPLING, SHORT, &[2, 2]); + assert!(decode(&build(true, false, &[junk], &[0]).bytes).is_err()); + // Coefficients and ReferenceBlackWhite that would divide by zero fall back to the defaults. + let bad = strips(1, 1, 8, 3, 6, 1, &[255, 128, 128]) + .tag(SUBSAMPLING, SHORT, &[1, 1]) + .tag(YCBCR_COEFFICIENTS, RATIONAL, &[rational(0, 1), rational(0, 0), rational(5, 1)]) + .tag(REFERENCE_BLACK_WHITE, RATIONAL, &[rational(7, 1); 6]); + let img = decode(&build(true, false, &[bad], &[0]).bytes).unwrap(); + assert_eq!(&img.to_rgba8()[..3], &[255, 255, 255]); +} + +#[test] +fn tiled_jpeg_ycbcr_keeps_its_icc_profile() { + // Two 16×16 tiles, each its own 4:2:0 JPEG stream, and an embedded profile, like a scanner's + // tiled output. + let (w, h, t) = (32u32, 16u32, 16u32); + let px = card(w, h); + let tile = |tx: u32| -> Vec { + (0..t) + .flat_map(|y| { + let at = ((y * w + tx * t) * 3) as usize; + px[at..at + (t * 3) as usize].to_vec() + }) + .collect() + }; + let streams = [jpeg(&tile(0), t, t), jpeg(&tile(1), t, t)]; + let icc = sample_icc(512); + let dir = Dir { entries: base(w, h, 8, 3, 6), chunks: streams.to_vec(), tiled: true }; + let dir = compressed(dir, 7).tag(322, SHORT, &[u64::from(t)]).tag(323, SHORT, &[u64::from(t)]).tag(SUBSAMPLING, SHORT, &[2, 2]).bytes( + ICC_PROFILE, + UNDEFINED, + &icc, + ); + let img = decode(&build(true, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!((img.width(), img.height(), img.layout()), (w, h, ChannelLayout::Rgb)); + assert_eq!(img.icc.as_deref(), Some(icc.as_slice()), "the embedded profile is kept"); + // The oracle: each tile's stream decoded by an independent decoder, placed side by side. + let tiles: Vec> = + streams.iter().map(|s| image::load_from_memory_with_format(s, image::ImageFormat::Jpeg).unwrap().to_rgb8().into_raw()).collect(); + let row = (t * 3) as usize; + let want: Vec = (0..t as usize).flat_map(|y| [&tiles[0][y * row..(y + 1) * row], &tiles[1][y * row..(y + 1) * row]].concat()).collect(); + let worst = rgb(&img).iter().zip(&want).map(|(g, o)| g.abs_diff(*o)).max().unwrap(); + assert!(worst <= 2, "largest difference from the oracle: {worst}"); +} + +#[test] +fn big_endian_sixteen_bit_ycbcr_reads_as_rgb() { + // White, black, and red as full-range 16-bit YCbCr (Y 0.299·65535, Cb 32768 − 0.1687·65535, Cr max). + let px = [65535u16, 32768, 32768, 0, 32768, 32768, 19595, 21710, 65535]; + let dir = strips(3, 1, 16, 3, 6, 1, &u16s(&px, false)).tag(SUBSAMPLING, SHORT, &[1, 1]); + let file = build(false, false, &[dir], &[0]).bytes; + assert_eq!(&file[..2], b"MM"); + let img = decode(&file).unwrap(); + assert_eq!((img.layout(), img.sample_type()), (ChannelLayout::Rgb, SampleType::U16)); + let n = img.to_normalized(); + let want = [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + assert!(n.iter().zip(want).all(|(g, w)| (g - w).abs() < 2.0 / 255.0), "{n:?}"); +} + +#[test] +fn planar_ycbcr_reads_as_rgb_in_both_byte_orders() { + // The pixels of the default-conversion test, one plane per component. + let px = [0u8, 128, 128, 255, 128, 128, 128, 128, 128, 76, 85, 255]; + for little in [true, false] { + let dir = Dir { entries: base(4, 1, 8, 3, 6), chunks: planar_strips(&px, 4, 1, 3, 1, 1), tiled: false } + .tag(278, LONG, &[1]) + .tag(284, SHORT, &[2]) + .tag(SUBSAMPLING, SHORT, &[1, 1]); + let img = decode(&build(little, false, &[dir], &[0]).bytes).unwrap(); + assert_eq!(rgb(&img), [0, 0, 0, 255, 255, 255, 128, 128, 128, 254, 0, 0], "little endian: {little}"); + } +} diff --git a/crates/image-codecs/tests/it/webp_frames.rs b/crates/image-codecs/tests/it/webp_frames.rs new file mode 100644 index 000000000..2c8e54e98 --- /dev/null +++ b/crates/image-codecs/tests/it/webp_frames.rs @@ -0,0 +1,166 @@ +//! `decode_webp_frames`: every frame of an animated WebP, composited, with its duration (#1198). + +use craft_image_codecs::{ + AnimationFrame, ChannelLayout, CodecError, DecodeOptions, DecodeWarning, EncodeOptions, Format, Image, Limits, decode, decode_webp_frames, + encode, webp_is_animated, +}; + +const RED: [u8; 4] = [255, 0, 0, 255]; +const GREEN: [u8; 4] = [0, 255, 0, 255]; +const BLUE: [u8; 4] = [0, 0, 255, 255]; + +fn frames(bytes: &[u8]) -> Vec { + decode_webp_frames(bytes, &DecodeOptions::default()).unwrap() +} + +fn solid(c: [u8; 4], w: u32, h: u32) -> Image { + Image::from_u8(w, h, ChannelLayout::Rgba, c.repeat((w * h) as usize)).unwrap() +} + +fn chunk(id: &[u8; 4], data: &[u8]) -> Vec { + let mut c = id.to_vec(); + c.extend_from_slice(&(data.len() as u32).to_le_bytes()); + c.extend_from_slice(data); + if data.len() % 2 == 1 { + c.push(0); + } + c +} + +fn u24(v: u32) -> [u8; 3] { + let b = v.to_le_bytes(); + [b[0], b[1], b[2]] +} + +/// One frame of [`animated`]: the picture, where it sits on the canvas (even offsets, as WebP +/// stores them halved), its duration in milliseconds, and whether it is alpha-blended over the +/// canvas rather than replacing it. +struct Part { + image: Image, + at: [u32; 2], + duration_ms: u32, + blend: bool, +} + +fn part(c: [u8; 4], w: u32, h: u32, duration_ms: u32) -> Part { + Part { image: solid(c, w, h), at: [0, 0], duration_ms, blend: false } +} + +/// An animated WebP with a `w`x`h` canvas (transparent background) and one ANMF chunk per part, +/// each holding a lossless still from our own encoder. +fn animated(w: u32, h: u32, parts: &[Part]) -> Vec { + let mut body = b"WEBP".to_vec(); + let mut vp8x = vec![0x02 | 0x10, 0, 0, 0]; // animation and alpha flags + vp8x.extend_from_slice(&u24(w - 1)); + vp8x.extend_from_slice(&u24(h - 1)); + body.extend(chunk(b"VP8X", &vp8x)); + body.extend(chunk(b"ANIM", &[0, 0, 0, 0, 0, 0])); // background 0, loop forever + for p in parts { + let still = encode(&p.image, Format::WebP, &EncodeOptions::default()).unwrap(); + let at = still.windows(4).position(|w| w == b"VP8L").unwrap(); + let len = u32::from_le_bytes(still[at + 4..at + 8].try_into().unwrap()) as usize; + let mut anmf = Vec::new(); + anmf.extend_from_slice(&u24(p.at[0] / 2)); + anmf.extend_from_slice(&u24(p.at[1] / 2)); + anmf.extend_from_slice(&u24(p.image.width() - 1)); + anmf.extend_from_slice(&u24(p.image.height() - 1)); + anmf.extend_from_slice(&u24(p.duration_ms)); + anmf.push(if p.blend { 0 } else { 0x02 }); // bit 1: do not blend + anmf.extend(chunk(b"VP8L", &still[at + 8..at + 8 + len])); + body.extend(chunk(b"ANMF", &anmf)); + } + let mut out = b"RIFF".to_vec(); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend(body); + out +} + +fn pixel(f: &AnimationFrame, x: u32, y: u32) -> [u8; 4] { + let rgba = f.image.to_rgba8(); + let i = ((y * f.image.width() + x) * 4) as usize; + rgba[i..i + 4].try_into().unwrap() +} + +#[test] +fn a_still_webp_gives_one_frame_equal_to_decode() { + let still = encode(&solid(RED, 6, 4), Format::WebP, &EncodeOptions::default()).unwrap(); + assert!(!webp_is_animated(&still)); + let f = frames(&still); + assert_eq!(f.len(), 1); + assert_eq!(f[0].delay_ms, 0); + assert_eq!(f[0].image, decode(&still).unwrap()); +} + +#[test] +fn a_one_frame_animation_is_not_animated() { + let a = animated(6, 4, &[part(RED, 6, 4, 70)]); + assert!(!webp_is_animated(&a)); + assert_eq!(frames(&a).len(), 1); +} + +#[test] +fn every_frame_comes_back_in_order_with_its_duration() { + let a = animated(6, 4, &[part(RED, 6, 4, 40), part(GREEN, 6, 4, 120), part(BLUE, 6, 4, 7)]); + assert!(webp_is_animated(&a)); + let f = frames(&a); + assert_eq!(f.iter().map(|f| f.delay_ms).collect::>(), [40, 120, 7], "WebP durations are milliseconds"); + for (f, c) in f.iter().zip([RED, GREEN, BLUE]) { + assert_eq!((f.image.width(), f.image.height()), (6, 4)); + assert_eq!(pixel(f, 0, 0), c); + assert_eq!(pixel(f, 5, 3), c); + assert_eq!(f.image.warnings, []); + } +} + +#[test] +fn a_smaller_frame_is_drawn_over_the_previous_one() { + let small = Part { image: solid(BLUE, 2, 2), at: [2, 2], duration_ms: 50, blend: true }; + let f = frames(&animated(6, 4, &[part(RED, 6, 4, 50), small])); + assert_eq!(f.len(), 2); + // Alpha blending rounds: within one level. + let near = |a: [u8; 4], b: [u8; 4]| a.iter().zip(b).all(|(x, y)| x.abs_diff(y) <= 1); + assert!(near(pixel(&f[1], 0, 0), RED), "outside the second frame the first shows through"); + assert!(near(pixel(&f[1], 2, 2), BLUE)); + assert!(near(pixel(&f[1], 3, 3), BLUE)); + assert!(near(pixel(&f[1], 4, 2), RED)); +} + +#[test] +fn a_damaged_later_frame_keeps_the_frames_before_it() { + let mut a = animated(6, 4, &[part(RED, 6, 4, 50), part(GREEN, 6, 4, 50), part(BLUE, 6, 4, 50)]); + // Break the last frame's VP8L signature byte; the container around it stays whole. + let last = a.windows(4).rposition(|w| w == b"VP8L").unwrap(); + a[last + 8] = 0; + let f = frames(&a); + assert_eq!(f.len(), 2); + assert_eq!(pixel(&f[0], 0, 0), RED); + assert_eq!(pixel(&f[1], 0, 0), GREEN); + assert_eq!(f[0].image.warnings, [DecodeWarning::Truncated { format: Format::WebP }]); +} + +#[test] +fn every_prefix_of_an_animated_webp_returns() { + let a = animated(5, 4, &[part(RED, 5, 4, 50), part(GREEN, 5, 4, 50), part(BLUE, 5, 4, 50)]); + for n in 0..a.len() { + let _ = decode_webp_frames(&a[..n], &DecodeOptions::default()); + } +} + +#[test] +fn dimension_limits_apply() { + let a = animated(4, 4, &[part(RED, 4, 4, 50), part(GREEN, 4, 4, 50)]); + let opts = DecodeOptions { limits: Limits { max_width: 3, ..Limits::default() }, ..Default::default() }; + assert!(matches!(decode_webp_frames(&a, &opts), Err(CodecError::LimitExceeded(_)))); +} + +#[test] +fn the_byte_budget_covers_all_frames_together() { + // One 4x4 RGBA frame is 64 bytes. The decoder's own canvas counts against the budget too, so + // two frames need more than 128; ten frames of the same size must not fit where two do. + let two = animated(4, 4, &[part(RED, 4, 4, 50), part(GREEN, 4, 4, 50)]); + let ten: Vec = (0..10).map(|i| part(if i % 2 == 0 { RED } else { BLUE }, 4, 4, 50)).collect(); + let ten = animated(4, 4, &ten); + let opts = DecodeOptions { limits: Limits { max_alloc: 256, ..Limits::default() }, ..Default::default() }; + assert_eq!(decode_webp_frames(&two, &opts).unwrap().len(), 2); + assert!(matches!(decode_webp_frames(&ten, &opts), Err(CodecError::LimitExceeded(_)))); +} diff --git a/crates/image-codecs/tests/it/webp_lossy.rs b/crates/image-codecs/tests/it/webp_lossy.rs new file mode 100644 index 000000000..2d0c4861c --- /dev/null +++ b/crates/image-codecs/tests/it/webp_lossy.rs @@ -0,0 +1,140 @@ +//! Lossy WebP (our VP8 encoder) checked against an independent decoder (`image-webp`): the +//! files decode, look like the input (PSNR by quality), get smaller as quality drops and are far +//! smaller than lossless on photographic content; alpha and metadata travel in the container; +//! odd sizes and the limits behave. + +use crate::common; +use common::*; +use craft_image_codecs::*; + +fn lossy(q: u8) -> EncodeOptions { + EncodeOptions { webp_lossless: false, webp_quality: q, ..Default::default() } +} + +/// A smooth photograph-like image: gradients with a little grain. +fn photo(w: u32, h: u32, seed: u64) -> Image { + synth(w, h, ChannelLayout::Rgb, SampleType::U8, seed, 0.03) +} + +#[test] +fn decodes_with_the_right_size_and_looks_like_the_input() { + let img = photo(97, 61, 3); + for (q, min_psnr) in [(100u8, 40.0), (90, 36.0), (75, 33.0), (50, 30.0), (20, 26.0)] { + let bytes = encode(&img, Format::WebP, &lossy(q)).unwrap(); + let back = decode(&bytes).unwrap_or_else(|e| panic!("q{q}: {e}")); + assert_eq!(back.dimensions(), (97, 61)); + assert_eq!(back.layout(), ChannelLayout::Rgb); + let p = psnr(&img, &back); + assert!(p >= min_psnr, "quality {q}: PSNR {p:.1} dB below {min_psnr}"); + } +} + +#[test] +fn smaller_at_lower_quality_and_much_smaller_than_lossless_on_photos() { + let img = synth(256, 192, ChannelLayout::Rgb, SampleType::U8, 9, 0.08); + let lossless = encode(&img, Format::WebP, &EncodeOptions::default()).unwrap().len(); + let sizes: Vec = [95u8, 80, 60, 40, 20].iter().map(|&q| encode(&img, Format::WebP, &lossy(q)).unwrap().len()).collect(); + for w in sizes.windows(2) { + assert!(w[1] <= w[0], "sizes must not grow as quality drops: {sizes:?}"); + } + assert!(sizes[1] * 4 < lossless, "q80 {} vs lossless {lossless}", sizes[1]); + assert!(sizes[4] * 10 < lossless, "q20 {} vs lossless {lossless}", sizes[4]); +} + +#[test] +fn flat_areas_cost_almost_nothing() { + // A single colour: every macroblock is skipped; the file is dominated by the header. + let img = Image::from_u8(640, 480, ChannelLayout::Rgb, vec![200u8; 640 * 480 * 3]).unwrap(); + let bytes = encode(&img, Format::WebP, &lossy(80)).unwrap(); + assert!(bytes.len() < 1200, "{} bytes", bytes.len()); + let back = decode(&bytes).unwrap(); + assert!(max_abs_diff(&img, &back) <= 3.0 / 255.0, "flat colour reproduced within rounding"); +} + +#[test] +fn alpha_is_kept_exactly_and_metadata_travels() { + let mut img = synth(40, 30, ChannelLayout::Rgba, SampleType::U8, 5, 0.05); + img.icc = Some(sample_icc(400)); + img.meta.exif = Some(sample_exif()); + img.meta.xmp = Some(SAMPLE_XMP.to_owned()); + let bytes = encode(&img, Format::WebP, &lossy(85)).unwrap(); + assert_eq!(&bytes[12..16], b"VP8X", "extended container"); + let back = decode(&bytes).unwrap(); + assert_eq!(back.layout(), ChannelLayout::Rgba); + for (a, b) in img.data().as_chunks::<4>().0.iter().zip(back.data().as_chunks::<4>().0) { + assert_eq!(a[3], b[3], "alpha is stored losslessly"); + } + assert!(psnr(&img, &back) > 30.0); + assert_eq!(back.icc, img.icc); + assert!(back.meta.xmp.is_some()); + assert!(back.meta.exif.is_some()); + // Fully opaque alpha is dropped rather than stored. + let mut data = synth(40, 30, ChannelLayout::Rgba, SampleType::U8, 5, 0.0).data().to_vec(); + for px in data.as_chunks_mut::<4>().0 { + px[3] = 255; + } + let opaque = Image::from_u8(40, 30, ChannelLayout::Rgba, data).unwrap(); + let bytes = encode(&opaque, Format::WebP, &EncodeOptions { embed_icc: false, embed_metadata: false, ..lossy(85) }).unwrap(); + assert_eq!(&bytes[12..16], b"VP8 "); + assert_eq!(decode(&bytes).unwrap().layout(), ChannelLayout::Rgb); +} + +#[test] +fn gray_and_odd_sizes_and_tiny_images() { + for (w, h) in [(1, 1), (2, 3), (17, 9), (33, 1), (1, 47), (16, 16), (31, 15)] { + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + // Little noise: at these sizes the 2 × 2 chroma average alone would otherwise dominate. + let img = synth(w, h, layout, SampleType::U8, u64::from(w * 100 + h), 0.02); + let bytes = encode(&img, Format::WebP, &lossy(80)).unwrap_or_else(|e| panic!("{w}x{h} {layout:?}: {e}")); + let back = decode(&bytes).unwrap_or_else(|e| panic!("{w}x{h} {layout:?}: {e}")); + assert_eq!(back.dimensions(), (w, h)); + // Below 8 px the 2 × 2 chroma average of random saturated pixels is the whole error. + if w >= 8 && h >= 8 { + let p = psnr(&img.convert(back.layout(), SampleType::U8), &back); + assert!(p > 24.0, "{w}x{h} {layout:?}: {p:.1} dB"); + } + } + } +} + +#[test] +fn sixteen_bit_input_is_quantised_to_eight() { + let img = synth(24, 24, ChannelLayout::Rgb, SampleType::U16, 2, 0.02); + let bytes = encode(&img, Format::WebP, &lossy(90)).unwrap(); + let back = decode(&bytes).unwrap(); + assert!(psnr(&img.convert(ChannelLayout::Rgb, SampleType::U8), &back) > 34.0); +} + +#[test] +fn dimension_limit_is_an_error_not_a_panic() { + let img = Image::from_u8(16384, 1, ChannelLayout::Rgb, vec![0u8; 16384 * 3]).unwrap(); + let e = encode(&img, Format::WebP, &lossy(50)).unwrap_err(); + assert!(matches!(e, CodecError::Encode { .. }), "{e}"); + let img = Image::from_u8(16383, 1, ChannelLayout::Rgb, vec![0u8; 16383 * 3]).unwrap(); + let bytes = encode(&img, Format::WebP, &lossy(50)).unwrap(); + assert_eq!(decode(&bytes).unwrap().dimensions(), (16383, 1)); +} + +#[test] +fn hard_edges_and_text_like_content_keep_their_structure() { + // Black text-like strokes on white: B_PRED sub-block modes keep edges sharp enough that the + // thresholded result matches the source almost everywhere. + let (w, h) = (128u32, 64u32); + let mut data = vec![255u8; (w * h * 3) as usize]; + for y in 0..h { + for x in 0..w { + let stroke = (x % 11 < 2 && y > 8 && y < 56) || (y % 13 < 2 && x > 4 && x < 124); + if stroke { + let i = ((y * w + x) * 3) as usize; + data[i..i + 3].fill(0); + } + } + } + let img = Image::from_u8(w, h, ChannelLayout::Rgb, data).unwrap(); + let bytes = encode(&img, Format::WebP, &lossy(90)).unwrap(); + let back = decode(&bytes).unwrap(); + let wrong = img.data().as_chunks::<3>().0.iter().zip(back.data().as_chunks::<3>().0).filter(|(a, b)| (a[0] < 128) != (b[0] < 128)).count(); + let total = (w * h) as usize; + assert!(wrong * 100 < total * 2, "{wrong} of {total} pixels flipped side of mid-gray"); + assert!(psnr(&img, &back) > 30.0, "{:.1}", psnr(&img, &back)); +} diff --git a/crates/image-codecs/tests/it/xpm.rs b/crates/image-codecs/tests/it/xpm.rs new file mode 100644 index 000000000..b499cea33 --- /dev/null +++ b/crates/image-codecs/tests/it/xpm.rs @@ -0,0 +1,123 @@ +use craft_image_codecs::{ChannelLayout, CodecError, Format, Image, caps, decode, decode_as, detect, encode, from_extension}; + +const SAMPLE: &[u8] = br#"/* XPM */ +static char *icon[] = { +"2 2 3 1", +". c #ff0000", +" c None", +"+ c #00ff00", +".+", +" +" +}; +"#; + +#[test] +fn xpm3_decodes_palette_transparency_and_spaces_in_keys() { + assert_eq!(detect(SAMPLE), Some(Format::Xpm)); + assert_eq!(from_extension("icon.XPM"), Some(Format::Xpm)); + let img = decode(SAMPLE).unwrap(); + assert_eq!(img.layout(), ChannelLayout::Rgba); + assert_eq!(img.dimensions(), (2, 2)); + assert_eq!(img.data(), &[255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 0, 0, 0, 255, 0, 255]); +} + +#[test] +fn rgb_precisions_and_x11_name() { + let xpm = br#"/* XPM */ +static char *i[] = { +"3 1 3 1", +"a c #f00", +"b c #ffff00000000", +"c c gray70", +"abc" +};"#; + let img = decode(xpm).unwrap(); + assert_eq!(img.data(), &[255, 0, 0, 255, 255, 0, 0, 255, 179, 179, 179, 255]); +} + +#[test] +fn rejects_truncation_unsupported_colours_and_unknown_keys() { + for bytes in [ + &b"/* XPM */\nstatic char *i[] = { \"1 1 1 1\", \"a c None\" };"[..], + &b"/* XPM */\nstatic char *i[] = { \"1 1 1 1\", \"a c NotAColor\", \"a\" };"[..], + &b"/* XPM */\nstatic char *i[] = { \"1 1 1 1\", \"a c #a\xC3\xA9\", \"a\" };"[..], + &b"/* XPM */\nstatic char *i[] = { \"1 1 1 1\", \"a c #+f0000\", \"a\" };"[..], + &b"/* XPM */\nstatic char *i[] = { \"1 1 1 1\", \"a c black\", \"z\" };"[..], + &b"/* XPM */\nstatic char *i[] = { \"1 1 1 1\", \"a c black\", \"aa\" };"[..], + &b"/* XPM */\nstatic char *i[] = { \"1 1 2 1\", \"a c black\", \"a c white\", \"a\" };"[..], + ] { + assert!(decode_as(Format::Xpm, bytes).is_err(), "{:?}", bytes); + } + assert!(decode_as(Format::Xpm, b"/* XPM */\n\"4294967295 4294967295 1 1\"").is_err()); + assert!(decode_as(Format::Xpm, b"/* XPM */\n\"2 1 1 1\" \"a c black\" \"aa\" /* bad").is_err()); +} + +#[test] +fn x11_names_and_field_priority() { + let xpm = br#"/* XPM */ +static char *i[] = { +"4 1 4 1", +"a c LightGray", +"b c DarkBlue", +"c m black g white c light blue", +"d m black g4 red g green", +"abcd" +};"#; + let image = decode(xpm).unwrap(); + assert_eq!(image.data(), &[211, 211, 211, 255, 0, 0, 139, 255, 173, 216, 230, 255, 0, 255, 0, 255]); +} + +#[test] +fn named_x11_colors_use_x11_not_css_values() { + let bytes = br#"/* XPM */ +static char *icon[] = { +"6 1 6 1", +"a c gray", +"b c maroon", +"c c purple", +"d c LightGoldenrodYellow", +"e c DarkSlateGray4", +"f c Ghost White", +"abcdef" +};"#; + let image = decode(bytes).unwrap(); + assert_eq!(image.data(), &[190, 190, 190, 255, 176, 48, 96, 255, 160, 32, 240, 255, 250, 250, 210, 255, 82, 139, 139, 255, 248, 248, 255, 255,]); +} + +#[test] +fn xpm2_decodes_crlf_comments_spaces_and_multichar_keys() { + let bytes = b"! XPM2\r\n! exported by an icon tool\r\n2 2 3 2\r\n.a c #f00\r\n!_ c None\r\nzz c DarkBlue\r\n.a!_\r\nzz.a\r\n"; + assert_eq!(detect(bytes), Some(Format::Xpm)); + let image = decode(bytes).unwrap(); + assert_eq!(image.dimensions(), (2, 2)); + assert_eq!(image.data(), &[255, 0, 0, 255, 0, 0, 0, 0, 0, 0, 139, 255, 255, 0, 0, 255]); +} + +#[test] +fn xpm2_allows_exclamation_as_a_pixel_key_and_preserves_space_keys() { + let bytes = b"! XPM2\n2 1 2 1\n! c red\n c None\n! \n"; + let image = decode(bytes).unwrap(); + assert_eq!(image.data(), &[255, 0, 0, 255, 0, 0, 0, 0]); +} + +#[test] +fn xpm2_malformed_inputs_return_errors() { + for bytes in [ + &b"! XPM2\n1 1 1 1\na c black\n"[..], + &b"! XPM2\n1 1 1 1\na c black\nb\n"[..], + &b"! XPM2\n1 1 1 1\na c #a\xc3\xa9\na\n"[..], + &b"! XPM2 invalid\n1 1 1 1\na c black\na\n"[..], + &b"! XPM2\n100000 100000 1 1\na c red\na\n"[..], + ] { + assert!(decode_as(Format::Xpm, bytes).is_err(), "{bytes:?}"); + } + assert_ne!(detect(b"! XPM23\n1 1 1 1\na c black\na\n"), Some(Format::Xpm)); +} + +#[test] +fn xpm_is_import_only() { + assert!(caps(Format::Xpm).read); + assert!(!caps(Format::Xpm).write); + let image = Image::from_u8(1, 1, ChannelLayout::Rgba, vec![0, 0, 0, 0]).unwrap(); + assert!(matches!(encode(&image, Format::Xpm, &Default::default()), Err(CodecError::Unsupported { .. }))); +} diff --git a/crates/image-codecs/tests/jxl_export.rs b/crates/image-codecs/tests/jxl_export.rs new file mode 100644 index 000000000..57a91143c --- /dev/null +++ b/crates/image-codecs/tests/jxl_export.rs @@ -0,0 +1,111 @@ +//! Lossless JPEG XL export and honest export-only capability reporting. + +use craft_image_codecs::{ + ASYMMETRIC_EXCEPTIONS, ChannelLayout, CodecError, EncodeOptions, Format, Image, SampleType, caps, decode, detect, encode, from_extension, +}; + +#[test] +fn jpeg_xl_is_export_only_and_detects_both_signatures() { + let c = caps(Format::Jxl); + assert!(c.write && !c.read); + assert_eq!(c.depths, &[SampleType::U8, SampleType::U16]); + assert_eq!(c.layouts, &[ChannelLayout::Gray, ChannelLayout::Rgb, ChannelLayout::Rgba]); + assert!(c.alpha && !c.icc && !c.exif && !c.xmp && !c.dpi && !c.lossy); + assert_eq!(Format::Jxl.mime_type(), "image/jxl"); + assert_eq!(from_extension("image.JXL"), Some(Format::Jxl)); + assert_eq!(detect(&[0xff, 0x0a, 0, 0]), Some(Format::Jxl)); + assert_eq!(detect(b"\0\0\0\x0cJXL \r\n\x87\n"), Some(Format::Jxl)); + assert!(ASYMMETRIC_EXCEPTIONS.iter().any(|(format, reason)| *format == Format::Jxl && !reason.is_empty())); + let err = decode(&[0xff, 0x0a]); + assert!(matches!(err, Err(CodecError::Unsupported { format: Format::Jxl, .. })), "{err:?}"); +} + +#[test] +fn jpeg_xl_encoding_is_deterministic_and_converts_float_to_integer() { + let input = + Image::from_u8(3, 2, ChannelLayout::Rgba, vec![0, 255, 17, 127, 20, 40, 80, 255, 255, 0, 2, 0, 10, 20, 30, 64, 0, 0, 0, 1, 90, 80, 70, 128]) + .unwrap(); + let opts = EncodeOptions::default(); + let one = encode(&input, Format::Jxl, &opts).unwrap(); + let two = encode(&input, Format::Jxl, &opts).unwrap(); + assert_eq!(one, two); + assert_eq!(detect(&one), Some(Format::Jxl)); + let float = Image::from_f32(2, 2, ChannelLayout::Rgb, &[0.0, 0.5, 1.0, 0.25, 0.5, 0.75, 0.75, 0.5, 0.25, 1.0, 0.0, 0.5]).unwrap(); + let bytes = encode(&float, Format::Jxl, &opts).unwrap(); + assert_eq!(detect(&bytes), Some(Format::Jxl)); +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn jpeg_xl_decodes_losslessly_with_independent_decoder() { + use jxl_oxide::JxlImage; + + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + let channels = layout.channels(); + for sample in [SampleType::U8, SampleType::U16] { + let (w, h) = (7, 5); + let count = w as usize * h as usize * channels; + let expected = match sample { + SampleType::U8 => (0..count).map(|i| (((i * 37 + 19) % 256) as u16) * 257).collect::>(), + SampleType::U16 => (0..count).map(|i| ((i * 9973 + 27) % 65536) as u16).collect::>(), + _ => unreachable!(), + }; + let input = match sample { + SampleType::U8 => { + let data = expected.iter().map(|v| (v / 257) as u8).collect(); + Image::from_u8(w, h, layout, data).unwrap() + } + SampleType::U16 => Image::from_u16(w, h, layout, &expected).unwrap(), + _ => unreachable!(), + }; + // GrayA is losslessly widened to RGBA. Keep the oracle comparison + // against original gray and alpha samples, not only against a converted image. + let (output_channels, expected_output) = if layout == ChannelLayout::GrayA { + let expanded: Vec = expected.as_chunks::<2>().0.iter().flat_map(|[gray, alpha]| [*gray, *gray, *gray, *alpha]).collect(); + (4, expanded) + } else { + (channels, expected.clone()) + }; + let encoded = encode(&input, Format::Jxl, &EncodeOptions::default()).unwrap(); + let decoder = JxlImage::builder().read(&encoded[..]).unwrap(); + assert_eq!((decoder.width(), decoder.height()), (w, h)); + let render = decoder.render_frame(0).unwrap(); + let mut stream = render.stream(); + assert_eq!(stream.channels() as usize, output_channels, "{layout:?} {sample:?}"); + let mut output = vec![0u16; expected_output.len()]; + assert_eq!(stream.write_to_buffer(&mut output), expected_output.len()); + assert_eq!(output, expected_output, "{layout:?} {sample:?}"); + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn jpeg_xl_preserves_sixteen_bit_alpha_across_dimension_header_sizes() { + use jxl_oxide::JxlImage; + + // Exercise each size-header coding tier, and non-opaque alpha extremes. + for width in [2u32, 513, 8193] { + let height = 2u32; + let mut samples = Vec::with_capacity(width as usize * height as usize * 4); + for i in 0..(width as usize * height as usize) { + samples.extend_from_slice(&[(i as u16).wrapping_mul(127), 65535, (i as u16).wrapping_mul(9973), [0u16, 1, 32768, 65535][i % 4]]); + } + let input = Image::from_u16(width, height, ChannelLayout::Rgba, &samples).unwrap(); + let encoded = encode(&input, Format::Jxl, &EncodeOptions::default()).unwrap(); + let decoder = JxlImage::builder().read(&encoded[..]).unwrap(); + let render = decoder.render_frame(0).unwrap(); + let mut stream = render.stream(); + assert_eq!(stream.channels(), 4, "width={width}"); + let mut output = vec![0u16; samples.len()]; + assert_eq!(stream.write_to_buffer(&mut output), samples.len()); + assert_eq!(output, samples, "width={width}"); + } +} + +#[test] +fn jpeg_xl_rejects_one_pixel_images_without_panicking() { + // zune-jpegxl currently requires at least two pixels on each axis. + let image = Image::from_u8(1, 1, ChannelLayout::Rgb, vec![42, 100, 255]).unwrap(); + assert!(matches!(encode(&image, Format::Jxl, &EncodeOptions::default()), Err(CodecError::Encode { format: Format::Jxl, .. }))); +} diff --git a/crates/image-codecs/tests/pnm_peak_memory.rs b/crates/image-codecs/tests/pnm_peak_memory.rs new file mode 100644 index 000000000..33bf39b3c --- /dev/null +++ b/crates/image-codecs/tests/pnm_peak_memory.rs @@ -0,0 +1,36 @@ +//! #2938: `Limits::max_alloc` budgets the decoded pixel buffer, so the PNM decoder must not build a +//! wider scratch copy (it used to collect one `u32` per sample, four times an 8-bit image) next to it. +//! +//! This file holds a single test so it runs in its own process: it reads the process's peak RSS +//! (`VmHWM`, Linux only) and nothing else may allocate meanwhile. + +#[cfg(target_os = "linux")] +fn peak_rss_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|l| l.starts_with("VmHWM:"))?; + let kib: u64 = line.trim_start_matches("VmHWM:").trim().trim_end_matches("kB").trim().parse().ok()?; + Some(kib * 1024) +} + +#[cfg(target_os = "linux")] +#[test] +fn pnm_decode_peak_memory_stays_near_the_output_size() { + use craft_image_codecs::*; + + const SIDE: u32 = 4096; // 16 MiB of 8-bit gray output + let out_bytes = (SIDE * SIDE) as u64; + let mut file = format!("P5\n{SIDE} {SIDE}\n255\n").into_bytes(); + file.extend((0..out_bytes).map(|i| (i % 251) as u8)); + // Writing 5 resets the peak RSS to the current RSS; skip where the kernel doesn't allow it. + if std::fs::write("/proc/self/clear_refs", "5").is_err() { + return; + } + let Some(before) = peak_rss_bytes() else { return }; + let img = decode(&file).unwrap(); + let after = peak_rss_bytes().unwrap(); + assert_eq!(img.data().len() as u64, out_bytes); + assert_eq!(img.data()[300], (300 % 251) as u8); + // The output buffer plus slack; a `u32`-per-sample scratch copy alone would be 4× it. + let grown = after.saturating_sub(before); + assert!(grown < 2 * out_bytes, "decoding a {out_bytes} byte image raised peak RSS by {grown} bytes"); +} diff --git a/crates/indd/Cargo.toml b/crates/indd/Cargo.toml new file mode 100644 index 000000000..874ccb0d6 --- /dev/null +++ b/crates/indd/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-indd" +description = "Shared InDesign document to IDML conversion facade for Craft applications" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +indd = { workspace = true } + +[lints] +workspace = true diff --git a/crates/indd/src/lib.rs b/crates/indd/src/lib.rs new file mode 100644 index 000000000..657b417fc --- /dev/null +++ b/crates/indd/src/lib.rs @@ -0,0 +1,17 @@ +//! Shared InDesign document conversion boundary. +//! +//! InDesign's binary document model is converted to IDML here. Applications can import the +//! resulting package through [`craft_idml`] while retaining only their own model and UI mapping. + +pub use indd::header::SIGNATURE; +pub use indd::{Conversion, Error, Preview, Warning}; + +/// Whether `bytes` begins with the fixed signature of an InDesign document. +pub fn is_indd(bytes: &[u8]) -> bool { + bytes.starts_with(&SIGNATURE) +} + +/// Convert an InDesign document to IDML. `name` is the file name stored in the package. +pub fn convert(bytes: &[u8], name: &str) -> Result { + indd::convert(bytes, name) +} diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml new file mode 100644 index 000000000..5b597ba13 --- /dev/null +++ b/crates/jmap/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-jmap" +description = "JMAP (RFC 8620) protocol core without sockets: session, requests of method calls with result references, responses, ids, address templates and the judgement of HTTP answers, over any HTTP implementation." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde_json = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/jmap/README.md b/crates/jmap/README.md new file mode 100644 index 000000000..5da2a8c7b --- /dev/null +++ b/crates/jmap/README.md @@ -0,0 +1,37 @@ +# craft-jmap + +The JMAP protocol core (RFC 8620) without sockets: a program brings an HTTP implementation +(`HttpExchange`), this crate describes each request and judges each answer. + +- **Session**: `Client`, `fetch_session` / `read_session`, `Session`, `Account`, `CoreLimits`, + the capabilities (`CAPABILITY_CORE`, `CAPABILITY_MAIL`). +- **Requests**: `ApiRequest` of method calls with `CallId`s and `ResultReference`s + (back-references), checked against the session's limits. +- **Responses**: `ApiResponse`, `MethodResponse`, `MethodError` (RFC 8620 §3.6.2), per call. +- **Addresses**: `Address` (absolute `http(s)` URLs, RFC 3986 and the WHATWG URL Standard), the + transport rule (`https`, or plain `http` to loopback only), `session_address` for what a person + types, `AddressTemplate` (RFC 6570 level 1, the download and upload templates). +- **HTTP**: `HttpRequest`, `HttpResponse`, `judge_response` (status codes, problem details, + JSON), `RetryAfter`, `Credentials` (Basic, Bearer), `Redactor` (excerpts of what a server says, + with the credentials and addresses taken out). +- `Id`: the JMAP id alphabet. + +Method calls' arguments and results are `serde_json::Value`s: what a program asks +(`Mailbox/get`, `Email/query`…) and what it makes of the answers stays in the program. + +## Hostile input + +Every size, count, nesting depth and string length a server sends is bounded; ids are held to +the JMAP alphabet; nothing indexes into what came over the wire, and the protocol modules are +checked with `clippy::indexing_slicing` and `clippy::arithmetic_side_effects`. Credentials and +the addresses of the session and the account never reach error text or `Debug` output. + +## Origin + +MailCraft `crates/jmap` (the protocol core: `address`, `classify`, `client`, `credentials`, +`discovery`, `error`, `host`, `http`, `id`, `percent`, `redact`, `request`, `response`, `retry`, +`session`, `template`) at +[a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5), +by Martin Ball and the MailCraft contributors (MIT OR Apache-2.0), with all of its tests. The mail +client that sits on it (the mailbox and email reads into MailCraft's sync model, the fetch of +whole messages, the in-process test server) stays in MailCraft. diff --git a/crates/jmap/src/address.rs b/crates/jmap/src/address.rs new file mode 100644 index 000000000..b9547b7f6 --- /dev/null +++ b/crates/jmap/src/address.rs @@ -0,0 +1,533 @@ +//! Absolute `http` and `https` addresses (RFC 3986, RFC 9110 §4.2), reference resolution (RFC 3986 §5.2), +//! origins, and the transport rule that decides whether an address may carry credentials. +//! +//! An [`Address`] is always absolute, always `http` or `https`, always has a host, never has user information +//! or a fragment, and is kept in a normal form (RFC 3986 §6.2.2 and §6.2.3, RFC 9110 §4.2.3): lower-case scheme +//! and host, no default port, percent-escapes in upper case with escaped unreserved characters decoded, no +//! dot-segments, and `/` for an empty path. +//! +//! Its `Debug` form shows the origin only, so that a path or a query holding something private never reaches +//! a log by accident. [`Address::as_str`] gives the full form, for the request itself. + +use std::fmt; + +use crate::host::Host; +use crate::percent::{self, is_sub_delim, is_unreserved}; + +/// The longest address accepted, in octets. +pub const MAX_ADDRESS_LEN: usize = 8 * 1024; + +/// An address scheme this crate can use. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Scheme { + /// `http` (RFC 9110 §4.2.1); only usable towards a loopback host. + Http, + /// `https` (RFC 9110 §4.2.2). + Https, +} + +impl Scheme { + /// The scheme's name, lower case. + pub fn as_str(self) -> &'static str { + match self { + Self::Http => "http", + Self::Https => "https", + } + } + + /// The TCP port used when an address names none (RFC 9110 §4.2.1 and §4.2.2). + pub fn default_port(self) -> u16 { + match self { + Self::Http => 80, + Self::Https => 443, + } + } +} + +/// Why a text is not a usable address. +/// +/// No variant carries the text that was refused: it may hold a password or a token. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum AddressError { + /// Nothing was given. + Empty, + /// Longer than [`MAX_ADDRESS_LEN`]. + TooLong, + /// A space, a control character, a character outside ASCII, or a character no URI may contain. + ForbiddenCharacter, + /// A backslash, which browsers read as a slash and RFC 3986 does not allow. + Backslash, + /// No scheme where an absolute address was needed. + MissingScheme, + /// A scheme other than `http` or `https`. + UnsupportedScheme, + /// No `//` and authority after the scheme. + MissingAuthority, + /// The host is empty (RFC 9110 §4.2.1). + EmptyHost, + /// User information (`name@` or `name:password@`) before the host (RFC 9110 §4.2.4). + UserInfo, + /// The host is not a usable name. + InvalidHost, + /// The host is outside ASCII; it must be given in its `xn--` form. + InternationalHost, + /// A numeric host that is not a plain dotted IPv4 address (`127.1`, `0x7f.0.0.1`, …). + AmbiguousNumericHost, + /// A bracketed IPv6 literal that does not parse. + InvalidIpv6, + /// The port is not a number from 1 to 65535. + InvalidPort, + /// The path holds a character or an escape that RFC 3986 §3.3 does not allow. + InvalidPath, + /// The query holds a character or an escape that RFC 3986 §3.4 does not allow. + InvalidQuery, + /// The fragment holds a character or an escape that RFC 3986 §3.5 does not allow. + InvalidFragment, + /// Plain `http` to a host that is not loopback. + InsecureTransport, + /// An address on this computer or its local network, given by a host farther away (see + /// [`HostScope`](crate::host::HostScope)). + LocalFromRemote, +} + +impl fmt::Display for AddressError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Empty => "no address was entered; type the JMAP server's address, for example example.com", + Self::TooLong => "the address is longer than 8 KiB (8192 characters), more than any server address needs; check that nothing else was pasted with it", + Self::ForbiddenCharacter => "the address contains a space, a control character, a non-ASCII character or one of \" < > ^ ` { | }; remove it or percent-encode it", + Self::Backslash => "the address contains a backslash (\\); addresses use forward slashes (/)", + Self::MissingScheme => "the address does not say how to connect; start it with https://", + Self::UnsupportedScheme => "the address uses a scheme other than https; enter an https:// address (http is only for this computer)", + Self::MissingAuthority => "the scheme is not followed by // and a server name; write it as https://server.example/path", + Self::EmptyHost => "the address has no server name after https://; add the server's name", + Self::UserInfo => "the address contains a user name followed by @ before the server name; remove it and enter the user name in the sign-in fields", + Self::InvalidHost => "the server name contains characters a DNS name cannot hold, or an empty or overlong label; check its spelling", + Self::InternationalHost => "the server name has non-ASCII letters; enter its ASCII form, which starts with xn--", + Self::AmbiguousNumericHost => "the server name looks numeric but is not a plain IPv4 address; write four numbers such as 192.0.2.1, without leading zeros, or use the server's name", + Self::InvalidIpv6 => "the IPv6 address in brackets cannot be read; zone suffixes such as %eth0 and IPvFuture forms are not supported", + Self::InvalidPort => "the port after the server name is not a number from 1 to 65535", + Self::InvalidPath => "the path contains a character or a % sign that addresses do not allow there; percent-encode it", + Self::InvalidQuery => "the part after ? contains a character or a % sign that addresses do not allow there; percent-encode it", + Self::InvalidFragment => "the part after # contains a character or a % sign that addresses do not allow there", + Self::InsecureTransport => "this address would send the password unencrypted over plain http to another computer; use https:// (plain http is only accepted for localhost, 127.0.0.0/8 and [::1])", + Self::LocalFromRemote => "the server tried to send this program to this computer or its local network, which a remote server may not do; its configuration may be wrong or tampered with", + }) + } +} + +impl std::error::Error for AddressError {} + +/// An absolute `http` or `https` address in normal form. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Address { + text: String, + scheme: Scheme, + host: Host, + port: Option, + path_start: usize, + query_start: Option, +} + +/// The origin of an address: scheme, host and port (the tuple origin of the WHATWG URL Standard §4.7 for +/// `http` and `https`). Its `Display` form is `scheme://host` with `:port` only when it is not the default. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Origin { + scheme: Scheme, + host: Host, + port: Option, +} + +impl Origin { + /// The scheme. + pub fn scheme(&self) -> Scheme { + self.scheme + } + + /// The host. + pub fn host(&self) -> &Host { + &self.host + } + + /// The port, the scheme's default when none was given. + pub fn port(&self) -> u16 { + self.port.unwrap_or_else(|| self.scheme.default_port()) + } +} + +impl fmt::Display for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}://{}", self.scheme.as_str(), self.host)?; + if let Some(port) = self.port { + write!(f, ":{port}")?; + } + Ok(()) + } +} + +impl fmt::Debug for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// The parts of a URI reference (RFC 3986 Appendix B), the fragment already set aside. +struct Parts<'a> { + scheme: Option<&'a str>, + authority: Option<&'a str>, + path: &'a str, + query: Option<&'a str>, +} + +fn refuse_unusable_characters(input: &str) -> Result<(), AddressError> { + if input.is_empty() { + return Err(AddressError::Empty); + } + if input.len() > MAX_ADDRESS_LEN { + return Err(AddressError::TooLong); + } + for b in input.bytes() { + if b == b'\\' { + return Err(AddressError::Backslash); + } + if !(0x21..=0x7E).contains(&b) || matches!(b, b'"' | b'<' | b'>' | b'^' | b'`' | b'{' | b'|' | b'}') { + return Err(AddressError::ForbiddenCharacter); + } + } + Ok(()) +} + +fn looks_like_scheme(text: &str) -> bool { + let mut bytes = text.bytes(); + matches!(bytes.next(), Some(b) if b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')) +} + +/// Splits a reference into its components (RFC 3986 Appendix B); the fragment is dropped, as RFC 9110 §7.1 +/// leaves it out of every target. +fn reference_parts(reference: &str) -> Result, AddressError> { + let (without_fragment, fragment) = match reference.split_once('#') { + Some((head, fragment)) => (head, Some(fragment)), + None => (reference, None), + }; + if let Some(fragment) = fragment + && !chars_allowed(fragment, true) + { + return Err(AddressError::InvalidFragment); + } + let (scheme, rest) = match without_fragment.find([':', '/', '?']) { + Some(index) if without_fragment.get(index..).is_some_and(|tail| tail.starts_with(':')) && index > 0 => { + let (scheme, rest) = without_fragment.split_at_checked(index).ok_or(AddressError::MissingScheme)?; + (Some(scheme), rest.get(1..).unwrap_or("")) + } + _ => (None, without_fragment), + }; + let (authority, rest) = match rest.strip_prefix("//") { + Some(after) => { + let end = after.find(['/', '?']).unwrap_or(after.len()); + let (authority, rest) = after.split_at_checked(end).ok_or(AddressError::MissingAuthority)?; + (Some(authority), rest) + } + None => (None, rest), + }; + let (path, query) = match rest.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (rest, None), + }; + Ok(Parts { scheme, authority, path, query }) +} + +/// Whether every character of a path (`allow_question` false) or a query/fragment (`allow_question` true) is +/// allowed by RFC 3986 §3.3–§3.5, escapes included. +fn chars_allowed(text: &str, allow_question: bool) -> bool { + let mut bytes = text.bytes(); + while let Some(b) = bytes.next() { + let ok = match b { + b'%' => bytes.next().and_then(percent::hex_value).is_some() && bytes.next().and_then(percent::hex_value).is_some(), + b':' | b'@' | b'/' => true, + b'?' => allow_question, + _ => is_unreserved(b) || is_sub_delim(b), + }; + if !ok { + return false; + } + } + true +} + +fn port_number(text: &str, scheme: Scheme) -> Result, AddressError> { + if text.is_empty() { + return Ok(None); + } + let mut value: u32 = 0; + for b in text.bytes() { + let digit = match b { + b'0'..=b'9' => u32::from(b.saturating_sub(b'0')), + _ => return Err(AddressError::InvalidPort), + }; + value = value.checked_mul(10).and_then(|v| v.checked_add(digit)).filter(|v| *v <= u32::from(u16::MAX)).ok_or(AddressError::InvalidPort)?; + } + let port = u16::try_from(value).map_err(|_| AddressError::InvalidPort)?; + if port == 0 { + return Err(AddressError::InvalidPort); + } + Ok(if port == scheme.default_port() { None } else { Some(port) }) +} + +/// Separates an authority's host text (brackets kept for IPv6) from its port text, empty when absent. +fn separate_port(authority: &str) -> Result<(&str, &str), AddressError> { + if authority.starts_with('[') { + let close = authority.find(']').ok_or(AddressError::InvalidIpv6)?; + let end = close.checked_add(1).ok_or(AddressError::InvalidIpv6)?; + let (host, rest) = authority.split_at_checked(end).ok_or(AddressError::InvalidIpv6)?; + if rest.is_empty() { + return Ok((host, "")); + } + let port = rest.strip_prefix(':').ok_or(AddressError::InvalidHost)?; + return Ok((host, port)); + } + if authority.contains(['[', ']']) { + return Err(AddressError::InvalidHost); + } + Ok(authority.split_once(':').unwrap_or((authority, ""))) +} + +/// Removes the `.` and `..` segments of a path (RFC 3986 §5.2.4). +pub(crate) fn collapse_dot_segments(path: &str) -> String { + let mut input = path; + let mut output = String::with_capacity(path.len()); + while !input.is_empty() { + if let Some(rest) = input.strip_prefix("../") { + input = rest; + } else if let Some(rest) = input.strip_prefix("./") { + input = rest; + } else if input.starts_with("/./") { + input = input.get(2..).unwrap_or(""); + } else if input == "/." { + input = "/"; + } else if input.starts_with("/../") { + input = input.get(3..).unwrap_or(""); + drop_last_segment(&mut output); + } else if input == "/.." { + input = "/"; + drop_last_segment(&mut output); + } else if input == "." || input == ".." { + input = ""; + } else { + let end = input.get(1..).and_then(|after_first| after_first.find('/')).and_then(|index| index.checked_add(1)).unwrap_or(input.len()); + let (segment, rest) = input.split_at_checked(end).unwrap_or((input, "")); + output.push_str(segment); + input = rest; + } + } + output +} + +fn drop_last_segment(output: &mut String) { + match output.rfind('/') { + Some(index) => output.truncate(index), + None => output.clear(), + } +} + +/// Merges a relative path with the path of a base that has an authority (RFC 3986 §5.2.3). +fn base_directory_plus(base_path: &str, reference_path: &str) -> String { + let directory = match base_path.rfind('/') { + Some(index) => base_path.get(..=index).unwrap_or("/"), + None => "/", + }; + let mut merged = String::with_capacity(directory.len().saturating_add(reference_path.len())); + merged.push_str(directory); + merged.push_str(reference_path); + merged +} + +impl Address { + /// Parses an absolute `http` or `https` address. + /// + /// This checks the syntax only; [`Address::parse_secure`] also applies the transport rule. A fragment is + /// accepted and dropped (RFC 9110 §7.1). + pub fn parse(input: &str) -> Result { + refuse_unusable_characters(input)?; + let parts = reference_parts(input)?; + let scheme_text = parts.scheme.ok_or(AddressError::MissingScheme)?; + if !looks_like_scheme(scheme_text) { + return Err(AddressError::MissingScheme); + } + let scheme = match scheme_text.to_ascii_lowercase().as_str() { + "http" => Scheme::Http, + "https" => Scheme::Https, + _ => return Err(AddressError::UnsupportedScheme), + }; + let authority = parts.authority.ok_or(AddressError::MissingAuthority)?; + if authority.contains('@') { + return Err(AddressError::UserInfo); + } + let (host_text, port_text) = separate_port(authority)?; + let host = Host::parse(host_text)?; + let port = port_number(port_text, scheme)?; + if !chars_allowed(parts.path, false) { + return Err(AddressError::InvalidPath); + } + let path = percent::normalise_escapes(parts.path).ok_or(AddressError::InvalidPath)?; + let mut path = collapse_dot_segments(&path); + if path.is_empty() { + path.push('/'); + } + let query = match parts.query { + Some(query) => { + if !chars_allowed(query, true) { + return Err(AddressError::InvalidQuery); + } + Some(percent::normalise_escapes(query).ok_or(AddressError::InvalidQuery)?) + } + None => None, + }; + Self::assemble(scheme, host, port, &path, query.as_deref()) + } + + /// Parses an absolute address and applies the transport rule ([`Address::require_secure_transport`]). + pub fn parse_secure(input: &str) -> Result { + let address = Self::parse(input)?; + address.require_secure_transport()?; + Ok(address) + } + + fn assemble(scheme: Scheme, host: Host, port: Option, path: &str, query: Option<&str>) -> Result { + let origin = Origin { scheme, host, port }; + let mut text = origin.to_string(); + let path_start = text.len(); + text.push_str(path); + let query_start = match query { + Some(query) => { + let start = text.len(); + text.push('?'); + text.push_str(query); + Some(start) + } + None => None, + }; + if text.len() > MAX_ADDRESS_LEN { + return Err(AddressError::TooLong); + } + let Origin { scheme, host, port } = origin; + Ok(Self { text, scheme, host, port, path_start, query_start }) + } + + /// Resolves a reference, relative or absolute, against this address (RFC 3986 §5.2.2, strict). + /// + /// The result is checked like [`Address::parse`]; the transport rule is not applied. + pub fn resolve(&self, reference: &str) -> Result { + if reference.is_empty() { + return Ok(self.clone()); + } + refuse_unusable_characters(reference)?; + let parts = reference_parts(reference)?; + if parts.scheme.is_some() { + return Self::parse(reference); + } + let without_fragment = reference.split_once('#').map_or(reference, |(head, _)| head); + if parts.authority.is_some() { + return Self::parse(&format!("{}:{}", self.scheme.as_str(), without_fragment)); + } + let (path, query) = if parts.path.is_empty() { + (self.path().to_owned(), parts.query.or(self.query())) + } else if parts.path.starts_with('/') { + (parts.path.to_owned(), parts.query) + } else { + (base_directory_plus(self.path(), parts.path), parts.query) + }; + // Dot-segment removal (§5.2.2, §5.2.4) runs in `parse`, after escapes are normalised (§6.2.2.2), so that + // `%2e` segments are treated the same way whether the reference was relative or absolute. + let mut target = self.origin().to_string(); + target.push_str(&path); + if let Some(query) = query { + target.push('?'); + target.push_str(query); + } + Self::parse(&target) + } + + /// The full address, for the request line. Do not log it: use [`Address::origin`] for diagnostics. + pub fn as_str(&self) -> &str { + &self.text + } + + /// The scheme. + pub fn scheme(&self) -> Scheme { + self.scheme + } + + /// The host. + pub fn host(&self) -> &Host { + &self.host + } + + /// The port, the scheme's default when the address names none. + pub fn port(&self) -> u16 { + self.port.unwrap_or_else(|| self.scheme.default_port()) + } + + /// The path, starting with `/`. + pub fn path(&self) -> &str { + let end = self.query_start.unwrap_or(self.text.len()); + self.text.get(self.path_start..end).unwrap_or("/") + } + + /// The query without its `?`, if there is one. + pub fn query(&self) -> Option<&str> { + let start = self.query_start?.checked_add(1)?; + self.text.get(start..) + } + + /// The origin: scheme, host and port. + pub fn origin(&self) -> Origin { + Origin { scheme: self.scheme, host: self.host.clone(), port: self.port } + } + + /// Whether both addresses have the same origin. + pub fn same_origin(&self, other: &Self) -> bool { + self.scheme == other.scheme && self.host == other.host && self.port == other.port + } + + /// Whether this address may carry credentials: `https`, or `http` to a loopback host + /// (see [`Host::is_loopback`]). + pub fn uses_secure_transport(&self) -> bool { + self.scheme == Scheme::Https || self.host.is_loopback() + } + + /// The transport rule as a `Result`. + pub fn require_secure_transport(&self) -> Result<(), AddressError> { + if self.uses_secure_transport() { Ok(()) } else { Err(AddressError::InsecureTransport) } + } + + /// Checks an address that `self` handed out (a session's endpoint, a template's expansion, a redirect's + /// target): it must satisfy the transport rule, and it may not be closer to this computer than `self` + /// ([`Host::scope`]), so that a remote server cannot point the program, credentials and all, at a service + /// on this computer or its local network. Only the address text is judged: a name that resolves to such an + /// address can only be caught by the program performing the request. + pub fn check_link(&self, target: &Self) -> Result<(), AddressError> { + target.require_secure_transport()?; + if target.host.scope() > self.host.scope() { + return Err(AddressError::LocalFromRemote); + } + Ok(()) + } + + /// The same origin with another path (and no query). + pub(crate) fn with_path(&self, path: &str) -> Result { + let mut text = self.origin().to_string(); + text.push_str(path); + Self::parse(&text) + } +} + +impl fmt::Debug for Address { + /// Shows the origin only (requirement 1.8): a path or query may hold something private. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Address({})", self.origin()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/address/tests.rs b/crates/jmap/src/address/tests.rs new file mode 100644 index 000000000..392f64641 --- /dev/null +++ b/crates/jmap/src/address/tests.rs @@ -0,0 +1,243 @@ +use super::*; + +fn parsed(input: &str) -> String { + Address::parse(input).unwrap().as_str().to_owned() +} + +// RFC 3986 §3 and RFC 9110 §4.2.1–§4.2.2: the components of http and https addresses. +#[test] +fn parses_components() { + let address = Address::parse("https://jmap.example.com:8443/api/v1?x=1&y=2").unwrap(); + assert_eq!(address.scheme(), Scheme::Https); + assert_eq!(address.host(), &Host::Name("jmap.example.com".into())); + assert_eq!(address.port(), 8443); + assert_eq!(address.path(), "/api/v1"); + assert_eq!(address.query(), Some("x=1&y=2")); + assert_eq!(address.as_str(), "https://jmap.example.com:8443/api/v1?x=1&y=2"); +} + +// RFC 3986 §6.2.2 and §6.2.3, RFC 9110 §4.2.3: the three equivalent forms normalise to one. +#[test] +fn normal_form() { + assert_eq!(parsed("http://example.com:80/~smith/home.html"), "http://example.com/~smith/home.html"); + assert_eq!(parsed("http://EXAMPLE.com/%7Esmith/home.html"), "http://example.com/~smith/home.html"); + assert_eq!(parsed("http://EXAMPLE.com:/%7esmith/home.html"), "http://example.com/~smith/home.html"); + assert_eq!(parsed("HTTPS://Example.ORG"), "https://example.org/"); + assert_eq!(parsed("https://example.org:443/a/./b/../c"), "https://example.org/a/c"); + assert_eq!(parsed("https://example.org/a%2fb?q=%2f%41"), "https://example.org/a%2Fb?q=%2FA"); + assert_eq!(parsed("https://example.org/a/%2e%2E/b"), "https://example.org/b"); +} + +// RFC 9110 §7.1: the fragment is not part of the target. +#[test] +fn drops_fragment() { + assert_eq!(parsed("https://example.com/a?b#c"), "https://example.com/a?b"); + assert_eq!(Address::parse("https://example.com/#a b"), Err(AddressError::ForbiddenCharacter)); + assert_eq!(Address::parse("https://example.com/#a#b"), Err(AddressError::InvalidFragment)); +} + +// Requirement 1.1: what is not a usable absolute http/https address is refused. +#[test] +fn refuses_unusable_addresses() { + let cases: &[(&str, AddressError)] = &[ + ("", AddressError::Empty), + ("https://exa mple.com/", AddressError::ForbiddenCharacter), + ("https://example.com/\u{7f}", AddressError::ForbiddenCharacter), + ("https://example.com/\n", AddressError::ForbiddenCharacter), + ("https://example.com/é", AddressError::ForbiddenCharacter), + ("https://example.com/{x}", AddressError::ForbiddenCharacter), + ("https://example.com\\@example.org/", AddressError::Backslash), + ("https:\\\\example.com/", AddressError::Backslash), + ("//example.com/", AddressError::MissingScheme), + ("example.com/path", AddressError::MissingScheme), + ("ftp://example.com/", AddressError::UnsupportedScheme), + ("javascript:alert(1)", AddressError::UnsupportedScheme), + ("https:example.com", AddressError::MissingAuthority), + ("https:///path", AddressError::EmptyHost), + ("https://:443/", AddressError::EmptyHost), + ("https://user@example.com/", AddressError::UserInfo), + ("https://user:secret@example.com/", AddressError::UserInfo), + ("https://[::1/", AddressError::InvalidIpv6), + ("https://[::g]/", AddressError::InvalidIpv6), + ("https://[::1]x/", AddressError::InvalidHost), + ("https://example.com:65536/", AddressError::InvalidPort), + ("https://example.com:0/", AddressError::InvalidPort), + ("https://example.com:-1/", AddressError::InvalidPort), + ("https://example.com:12a/", AddressError::InvalidPort), + ("https://example.com:1:2/", AddressError::InvalidPort), + ("https://example.com/%zz", AddressError::InvalidPath), + ("https://example.com/[x]", AddressError::InvalidPath), + ("https://example.com/?%", AddressError::InvalidQuery), + ("https://127.1/", AddressError::AmbiguousNumericHost), + ]; + for (input, expected) in cases { + assert_eq!(Address::parse(input).as_ref().map(Address::as_str), Err(expected), "{input:?}"); + } + let long = format!("https://example.com/{}", "a".repeat(MAX_ADDRESS_LEN)); + assert_eq!(Address::parse(&long), Err(AddressError::TooLong)); +} + +// RFC 3986 §3.2.3: an empty port means the default one; leading zeros are still the same number. +#[test] +fn ports() { + assert_eq!(parsed("https://example.com:/"), "https://example.com/"); + assert_eq!(parsed("https://example.com:00443/"), "https://example.com/"); + assert_eq!(parsed("http://localhost:8080"), "http://localhost:8080/"); + assert_eq!(Address::parse("https://example.com:65535/").unwrap().port(), 65535); + assert_eq!(parsed("https://[::1]:8443/x"), "https://[::1]:8443/x"); +} + +// Requirement 1.2: https always; http only to localhost, 127.0.0.0/8 or [::1]. +#[test] +fn transport_rule() { + for ok in ["https://example.com/", "http://localhost/", "http://LocalHost:8080/", "http://127.0.0.1/", "http://127.9.8.7:1/", "http://[::1]/"] { + assert!(Address::parse_secure(ok).is_ok(), "{ok}"); + } + for refused in [ + "http://example.com/", + "http://sub.localhost/", + "http://localhost./", + "http://192.0.2.1/", + "http://[2001:db8::2]/", + "http://[::ffff:127.0.0.1]/", + ] { + assert_eq!(Address::parse_secure(refused), Err(AddressError::InsecureTransport), "{refused}"); + } +} + +// Requirement 1.2 hardened, RFC 9110 §4.2.1 and RFC 8620 §8.3: a remote host cannot hand out a loopback address. +#[test] +fn links_from_remote_to_loopback_are_refused() { + let remote = Address::parse("https://jmap.example.com/").unwrap(); + let local = Address::parse("http://127.0.0.1:8080/").unwrap(); + let other = Address::parse("https://api.example.net/").unwrap(); + let plain = Address::parse("http://example.org/").unwrap(); + assert_eq!(remote.check_link(&local), Err(AddressError::LocalFromRemote)); + for sneaky in [ + "https://localhost./", + "https://x.localhost/", + "https://0.0.0.0/", + "https://[::]/", + "https://[::ffff:7f00:1]/", + "https://192.168.1.10/", + "https://[fe80::1]/", + ] { + let target = Address::parse(sneaky).unwrap(); + assert_eq!(remote.check_link(&target), Err(AddressError::LocalFromRemote), "{sneaky}"); + } + let lan = Address::parse("https://192.168.1.10/jmap").unwrap(); + assert_eq!(lan.check_link(&Address::parse("https://192.168.1.11/api").unwrap()), Ok(())); + assert_eq!(lan.check_link(&Address::parse("https://localhost/api").unwrap()), Err(AddressError::LocalFromRemote)); + assert_eq!(remote.check_link(&plain), Err(AddressError::InsecureTransport)); + assert_eq!(remote.check_link(&other), Ok(())); + assert_eq!(local.check_link(&local), Ok(())); + assert_eq!(local.check_link(&remote), Ok(())); +} + +// RFC 3986 §5.4.1: the normal examples, hosts moved under example domains (the non-http "g:h" is +// refused, as this crate only knows http and https). +#[test] +fn resolves_normal_examples() { + let base = Address::parse("http://a.example.com/b/c/d;p?q").unwrap(); + let cases = [ + ("g", "http://a.example.com/b/c/g"), + ("./g", "http://a.example.com/b/c/g"), + ("g/", "http://a.example.com/b/c/g/"), + ("/g", "http://a.example.com/g"), + ("//g.example.org", "http://g.example.org/"), + ("?y", "http://a.example.com/b/c/d;p?y"), + ("g?y", "http://a.example.com/b/c/g?y"), + ("#s", "http://a.example.com/b/c/d;p?q"), + ("g#s", "http://a.example.com/b/c/g"), + ("g?y#s", "http://a.example.com/b/c/g?y"), + (";x", "http://a.example.com/b/c/;x"), + ("g;x", "http://a.example.com/b/c/g;x"), + ("g;x?y#s", "http://a.example.com/b/c/g;x?y"), + ("", "http://a.example.com/b/c/d;p?q"), + (".", "http://a.example.com/b/c/"), + ("./", "http://a.example.com/b/c/"), + ("..", "http://a.example.com/b/"), + ("../", "http://a.example.com/b/"), + ("../g", "http://a.example.com/b/g"), + ("../..", "http://a.example.com/"), + ("../../", "http://a.example.com/"), + ("../../g", "http://a.example.com/g"), + ]; + for (reference, expected) in cases { + assert_eq!(base.resolve(reference).unwrap().as_str(), expected, "{reference:?}"); + } + assert_eq!(base.resolve("g:h"), Err(AddressError::UnsupportedScheme)); +} + +// RFC 3986 §5.4.2: the abnormal examples (hosts under example domains), strict for "http:g". +#[test] +fn resolves_abnormal_examples() { + let base = Address::parse("http://a.example.com/b/c/d;p?q").unwrap(); + let cases = [ + ("../../../g", "http://a.example.com/g"), + ("../../../../g", "http://a.example.com/g"), + ("/./g", "http://a.example.com/g"), + ("/../g", "http://a.example.com/g"), + ("g.", "http://a.example.com/b/c/g."), + (".g", "http://a.example.com/b/c/.g"), + ("g..", "http://a.example.com/b/c/g.."), + ("..g", "http://a.example.com/b/c/..g"), + ("./../g", "http://a.example.com/b/g"), + ("./g/.", "http://a.example.com/b/c/g/"), + ("g/./h", "http://a.example.com/b/c/g/h"), + ("g/../h", "http://a.example.com/b/c/h"), + ("g;x=1/./y", "http://a.example.com/b/c/g;x=1/y"), + ("g;x=1/../y", "http://a.example.com/b/c/y"), + ("g?y/./x", "http://a.example.com/b/c/g?y/./x"), + ("g?y/../x", "http://a.example.com/b/c/g?y/../x"), + ("g#s/./x", "http://a.example.com/b/c/g"), + ("g#s/../x", "http://a.example.com/b/c/g"), + ]; + for (reference, expected) in cases { + assert_eq!(base.resolve(reference).unwrap().as_str(), expected, "{reference:?}"); + } + assert_eq!(base.resolve("http:g"), Err(AddressError::MissingAuthority)); + // RFC 3986 §6.2.2.2: an escaped dot is a dot, whether the reference is relative or absolute. + assert_eq!(base.resolve("/a/%2e%2e/../b"), Address::parse("http://a.example.com/a/%2e%2e/../b")); + assert_eq!(base.resolve("/a/%2e%2e/../b").unwrap().path(), "/b"); +} + +// RFC 3986 §5.2.4: the two worked examples of dot-segment removal. +#[test] +fn removes_dot_segments() { + assert_eq!(collapse_dot_segments("/a/b/c/./../../g"), "/a/g"); + assert_eq!(collapse_dot_segments("mid/content=5/../6"), "mid/6"); + assert_eq!(collapse_dot_segments("/.."), "/"); + assert_eq!(collapse_dot_segments("//x/../y"), "//y"); +} + +// Requirement 1.4: the origin elides the default port; the full form keeps everything else. +#[test] +fn origin_and_full_form() { + let address = Address::parse("https://example.com:443/session?token=abc").unwrap(); + assert_eq!(address.origin().to_string(), "https://example.com"); + assert_eq!(address.origin().port(), 443); + let address = Address::parse("http://[::1]:8080/x").unwrap(); + assert_eq!(address.origin().to_string(), "http://[::1]:8080"); + assert!(address.same_origin(&Address::parse("http://[0::1]:8080/y").unwrap())); + assert!(!address.same_origin(&Address::parse("http://[::1]:8081/x").unwrap())); +} + +// Requirement 1.8: diagnostics never show more than the origin. +#[test] +fn debug_shows_origin_only() { + let address = Address::parse("https://example.com/private/path?token=s3cr3t").unwrap(); + let shown = format!("{address:?}"); + assert_eq!(shown, "Address(https://example.com)"); + assert!(!shown.contains("s3cr3t") && !shown.contains("private")); +} + +// RFC 3986 §6.2.2: normalisation is idempotent, so parsing the full form gives the same address. +#[test] +fn reparse_is_stable() { + for input in ["https://EXAMPLE.com:443/a/../b/%7e?Q=%2f", "http://[0:0::1]:80//x/./y", "https://example.org"] { + let first = Address::parse(input).unwrap(); + let second = Address::parse(first.as_str()).unwrap(); + assert_eq!(first, second); + } +} diff --git a/crates/jmap/src/classify.rs b/crates/jmap/src/classify.rs new file mode 100644 index 000000000..ad674e59e --- /dev/null +++ b/crates/jmap/src/classify.rs @@ -0,0 +1,199 @@ +//! Judging a response: status codes (RFC 9110 §15), problem details (RFC 9457, which obsoletes RFC 7807 and +//! which JMAP's request-level errors use, RFC 8620 §3.6.1), content types (RFC 9110 §8.3) and JSON (RFC 8259). + +use serde_json::{Map, Value}; + +use crate::error::{Error, LimitName, RequestProblem, Unusable}; +use crate::http::{HttpRequest, HttpResponse, PROBLEM_BODY_LIMIT}; +use crate::redact::{Excerpt, Redactor}; +use crate::retry::RetryAfter; + +/// The URN prefix of JMAP's request-level problem types (RFC 8620 §3.6.1, §9.3). +const JMAP_PROBLEM_PREFIX: &str = "urn:ietf:params:jmap:error:"; +/// The longest JMAP problem name kept. +const MAX_PROBLEM_NAME_LEN: usize = 64; + +/// Whether a `Content-Type` value names `expected` (RFC 9110 §8.3.1: type and subtype are case-insensitive; +/// parameters such as `charset` are ignored, JSON being UTF-8 by RFC 8259 §8.1). +pub(crate) fn media_type_is(content_type: Option<&str>, expected: &str) -> bool { + content_type.is_some_and(|value| { + let essence = value.split(';').next().unwrap_or("").trim_matches([' ', '\t']); + essence.eq_ignore_ascii_case(expected) + }) +} + +/// The most JSON values a body may hold before it is parsed into memory. +pub const MAX_JSON_VALUES: usize = 1_000_000; +/// The most JSON objects a body may hold before it is parsed into memory. +pub const MAX_JSON_OBJECTS: usize = 100_000; + +/// Counts, without building anything, an upper bound of the values and the objects of a JSON text, so that a +/// small text cannot expand into a huge tree (RFC 8259 §9 lets a parser limit what it accepts; RFC 8620 §8.4). +/// Every value but the first follows `[`, `,` or `:`; strings are skipped with their escapes. +fn within_budget(body: &[u8]) -> bool { + let mut in_string = false; + let mut escaped = false; + let mut values = 1usize; + let mut objects = 0usize; + for &b in body { + if in_string { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_string = false; + } + continue; + } + match b { + b'"' => in_string = true, + b'{' => objects = objects.saturating_add(1), + b'[' | b',' | b':' => values = values.saturating_add(1), + _ => {} + } + if values > MAX_JSON_VALUES || objects > MAX_JSON_OBJECTS { + return false; + } + } + true +} + +/// Parses a JSON text (RFC 8259), skipping a leading byte order mark (RFC 8259 §8.1 allows ignoring one). +/// Nesting is bounded by `serde_json`'s recursion limit (128); the number of values and objects by +/// [`MAX_JSON_VALUES`] and [`MAX_JSON_OBJECTS`]. +pub(crate) fn parse_json(body: &[u8]) -> Result { + let body = body.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(body); + if !within_budget(body) { + return Err(Unusable::TooComplex); + } + serde_json::from_slice(body).map_err(|_| Unusable::InvalidJson) +} + +/// What was understood of a problem details object. +#[derive(Debug, Default)] +struct Problem { + jmap: Option, + status: Option, + limit: Option, + text: Option, + names_bad_token: bool, +} + +fn jmap_problem(kind: &str) -> Option { + let prefix = kind.get(..JMAP_PROBLEM_PREFIX.len())?; + if !prefix.eq_ignore_ascii_case(JMAP_PROBLEM_PREFIX) { + return None; + } + let name = kind.get(JMAP_PROBLEM_PREFIX.len()..)?; + Some(match name { + "unknownCapability" => RequestProblem::UnknownCapability, + "notJSON" => RequestProblem::NotJson, + "notRequest" => RequestProblem::NotRequest, + "limit" => RequestProblem::Limit, + other if !other.is_empty() && other.len() <= MAX_PROBLEM_NAME_LEN && other.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') => { + RequestProblem::Other(other.to_owned()) + } + _ => return None, + }) +} + +/// Reads a problem details object (RFC 9457 §3.1). Members of the wrong type are ignored, as §3.1 requires. +fn read_problem(response: &HttpResponse, limit: usize) -> Option { + let body = response.body(); + if body.is_empty() || body.len() > PROBLEM_BODY_LIMIT.min(limit) { + return None; + } + let content_type = response.content_type(); + if content_type.is_some() && !media_type_is(content_type, "application/problem+json") && !media_type_is(content_type, "application/json") { + return None; + } + let value = parse_json(body).ok()?; + let object: &Map = value.as_object()?; + let mut problem = Problem::default(); + if let Some(kind) = object.get("type").and_then(Value::as_str) { + problem.jmap = jmap_problem(kind); + let last = kind.rsplit([':', '/', '#']).next().unwrap_or(""); + problem.names_bad_token = last.eq_ignore_ascii_case("invalid_token") || last.eq_ignore_ascii_case("insufficient_scope"); + } + problem.status = object.get("status").and_then(Value::as_u64).and_then(|status| u16::try_from(status).ok()); + problem.limit = object.get("limit").and_then(Value::as_str).and_then(LimitName::new); + problem.text = object.get("detail").and_then(Value::as_str).or_else(|| object.get("title").and_then(Value::as_str)).map(str::to_owned); + Some(problem) +} + +fn excerpt(problem: Option<&Problem>, redactor: &Redactor) -> Option { + let text = problem?.text.as_deref()?; + let excerpt = redactor.excerpt(text); + if excerpt.as_str().is_empty() { None } else { Some(excerpt) } +} + +/// Judges a response to `request`. +/// +/// `Ok(())` means the body may be read as the request's kind expects (for session and API requests: a 2xx +/// status, `application/json`, and a body within the size limit). Everything else becomes the [`Error`] +/// variant the mail client can act on. Server text appears only as an [`Excerpt`] cleaned by `redactor`. +pub fn judge_response(request: &HttpRequest, response: &HttpResponse, redactor: &Redactor) -> Result<(), Error> { + let status = response.status(); + if (200..=299).contains(&status) { + if response.body().len() > request.max_response_bytes() { + return Err(Error::Unusable(Unusable::BodyTooLarge)); + } + if request.kind().expects_json() && !media_type_is(response.content_type(), "application/json") { + return Err(Error::Unusable(Unusable::WrongContentType)); + } + return Ok(()); + } + if (300..=399).contains(&status) { + return Err(judge_redirect(request, response)); + } + let problem = read_problem(response, request.max_response_bytes()); + let detail = excerpt(problem.as_ref(), redactor); + let retry_after = || response.retry_after().and_then(RetryAfter::parse); + // A server-chosen identifier is shown in messages: drop it when it echoes a secret. + let jmap = problem.as_ref().and_then(|problem| problem.jmap.clone()).map(|jmap| match jmap { + RequestProblem::Other(name) if redactor.mentions_secret(&name) => RequestProblem::Other(String::new()), + other => other, + }); + let limit = problem.as_ref().and_then(|problem| problem.limit.clone()).filter(|limit| !redactor.mentions_secret(limit.as_str())); + // RFC 9457 §3.1.2: the status member tells what the origin server meant when an intermediary changed it. + let says_unauthorised = problem.as_ref().is_some_and(|problem| problem.status == Some(401) || problem.names_bad_token); + Err(match status { + 401 => Error::CredentialsRefused, + 403 if says_unauthorised => Error::CredentialsRefused, + 429 | 503 => Error::Busy { retry_after: retry_after() }, + 413 => Error::TooLarge { limit }, + _ if jmap == Some(RequestProblem::Limit) => match limit { + Some(limit) if limit.is_concurrency() => Error::Busy { retry_after: retry_after() }, + limit => Error::TooLarge { limit }, + }, + 403 => Error::Forbidden { detail }, + 404 | 410 => Error::NotFound, + 400..=499 if jmap.is_some() => Error::Refused { problem: jmap, detail }, + 400 | 405 | 406 | 415 | 422 => Error::Refused { problem: None, detail }, + _ => Error::Http { status, detail }, + }) +} + +/// A redirect (RFC 9110 §15.4) is never followed here: its target is checked and handed to the caller. +fn judge_redirect(request: &HttpRequest, response: &HttpResponse) -> Error { + let status = response.status(); + let Some(location) = response.location() else { + return Error::Http { status, detail: None }; + }; + if !matches!(status, 301 | 302 | 303 | 307 | 308) { + return Error::Http { status, detail: None }; + } + let from = request.address(); + let target = match from.resolve(location) { + Ok(target) => target, + Err(error) => return Error::RedirectRefused(error), + }; + if let Err(error) = from.check_link(&target) { + return Error::RedirectRefused(error); + } + Error::Redirected { same_origin: from.same_origin(&target), target } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/classify/tests.rs b/crates/jmap/src/classify/tests.rs new file mode 100644 index 000000000..9a179bc0b --- /dev/null +++ b/crates/jmap/src/classify/tests.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::address::{Address, AddressError}; +use crate::credentials::Credentials; +use crate::http::{API_RESPONSE_LIMIT, HttpRequest, Method, RequestKind}; +use std::time::Duration; + +fn request(kind: RequestKind) -> HttpRequest { + let credentials = Credentials::bearer("token-abcdef").unwrap(); + let address = Address::parse("https://jmap.example.com/api/").unwrap(); + HttpRequest::new(Method::Post, address, kind, credentials.authorization()) +} + +fn judge(response: HttpResponse) -> Result<(), Error> { + let mut redactor = Redactor::new(); + redactor.add("token-abcdef"); + judge_response(&request(RequestKind::Api), &response, &redactor) +} + +fn problem(status: u16, body: &str) -> HttpResponse { + HttpResponse::new(status, body.as_bytes().to_vec()).with_field("Content-Type", "application/problem+json") +} + +// RFC 8620 §3.1: a successful API response is application/json. +#[test] +fn success_needs_json() { + let ok = HttpResponse::new(200, b"{}".to_vec()).with_field("Content-Type", "Application/JSON; charset=utf-8"); + assert_eq!(judge(ok), Ok(())); + let html = HttpResponse::new(200, b"".to_vec()).with_field("Content-Type", "text/html"); + assert_eq!(judge(html), Err(Error::Unusable(Unusable::WrongContentType))); + assert_eq!(judge(HttpResponse::new(200, b"{}".to_vec())), Err(Error::Unusable(Unusable::WrongContentType))); + let download = HttpResponse::new(200, vec![1, 2, 3]).with_field("Content-Type", "application/pdf"); + assert_eq!(judge_response(&request(RequestKind::Download), &download, &Redactor::new()), Ok(())); +} + +// Requirement 2.2: a limit the caller set for one request is enforced too, at the octet. +#[test] +fn per_request_body_limit() { + let limited = request(RequestKind::Download).with_response_limit(10); + let exact = HttpResponse::new(200, vec![0; 10]); + assert_eq!(judge_response(&limited, &exact, &Redactor::new()), Ok(())); + let over = HttpResponse::new(200, vec![0; 11]); + assert_eq!(judge_response(&limited, &over, &Redactor::new()), Err(Error::Unusable(Unusable::BodyTooLarge))); +} + +// Requirement 2.2: the crate refuses a body above the limit it states for the kind of request. +#[test] +fn success_body_limit() { + let big = HttpResponse::new(200, vec![b' '; API_RESPONSE_LIMIT + 1]).with_field("Content-Type", "application/json"); + assert_eq!(judge(big), Err(Error::Unusable(Unusable::BodyTooLarge))); +} + +// RFC 9110 §15.5.2 and §15.5.4, RFC 9457 §3.1.2: 401, and 403 when the problem says so. +#[test] +fn credentials_refused() { + assert_eq!(judge(HttpResponse::new(401, Vec::new())), Err(Error::CredentialsRefused)); + assert_eq!(judge(problem(403, r#"{"status":401,"title":"Unauthorized"}"#)), Err(Error::CredentialsRefused)); + assert_eq!(judge(problem(403, r#"{"type":"https://example.com/probs/invalid_token"}"#)), Err(Error::CredentialsRefused)); + assert!(matches!(judge(problem(403, r#"{"status":403,"detail":"no access to this account"}"#)), Err(Error::Forbidden { detail: Some(_) }))); + assert!(matches!(judge(HttpResponse::new(403, Vec::new())), Err(Error::Forbidden { detail: None }))); +} + +// Requirement 2.4 (429 and 503), RFC 9110 §15.6.4 and §10.2.3: busy, with the delay asked for. +#[test] +fn busy() { + let response = HttpResponse::new(503, Vec::new()).with_field("Retry-After", "120"); + assert_eq!(judge(response), Err(Error::Busy { retry_after: Some(RetryAfter::Delay(Duration::from_secs(120))) })); + let response = HttpResponse::new(429, Vec::new()).with_field("Retry-After", "soon"); + assert_eq!(judge(response), Err(Error::Busy { retry_after: None })); + let concurrency = problem(400, r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxConcurrentRequests","status":400}"#); + assert_eq!(judge(concurrency), Err(Error::Busy { retry_after: None })); +} + +// RFC 9110 §15.5.14 and RFC 8620 §3.6.1: 413 and the JMAP limit problem with its limit name. +#[test] +fn too_large() { + assert_eq!(judge(HttpResponse::new(413, Vec::new())), Err(Error::TooLarge { limit: None })); + let limit = problem(400, r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxSizeRequest","status":400,"detail":"too big"}"#); + assert_eq!(judge(limit), Err(Error::TooLarge { limit: LimitName::new("maxSizeRequest") })); + let odd = problem(400, r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"max size\n"}"#); + assert_eq!(judge(odd), Err(Error::TooLarge { limit: None })); +} + +// RFC 8620 §3.6.1: the other request-level problems are recognised; RFC 9110 §15.5.5 and §15.5.11. +#[test] +fn refused_and_not_found() { + let unknown = problem( + 400, + r#"{"type":"urn:ietf:params:jmap:error:unknownCapability","status":400,"detail":"The Request object used capability 'https://example.com/apis/foobar', which is not supported by this server."}"#, + ); + match judge(unknown) { + Err(Error::Refused { problem: Some(RequestProblem::UnknownCapability), detail: Some(detail) }) => { + assert!(detail.as_str().starts_with("The Request object used capability")); + } + other => panic!("{other:?}"), + } + assert!(matches!( + judge(problem(400, r#"{"type":"urn:ietf:params:jmap:error:notJSON"}"#)), + Err(Error::Refused { problem: Some(RequestProblem::NotJson), .. }) + )); + assert!(matches!( + judge(problem(400, r#"{"type":"urn:ietf:params:jmap:error:notRequest"}"#)), + Err(Error::Refused { problem: Some(RequestProblem::NotRequest), .. }) + )); + assert!(matches!( + judge(problem(400, r#"{"type":"URN:IETF:params:jmap:error:somethingNew"}"#)), + Err(Error::Refused { problem: Some(RequestProblem::Other(_)), .. }) + )); + assert!(matches!(judge(HttpResponse::new(400, b"not json".to_vec())), Err(Error::Refused { problem: None, detail: None }))); + assert_eq!(judge(HttpResponse::new(404, Vec::new())), Err(Error::NotFound)); + assert_eq!(judge(HttpResponse::new(410, Vec::new())), Err(Error::NotFound)); +} + +// Requirement 3.5: a server-chosen identifier that echoes a secret is not shown. +#[test] +fn identifiers_echoing_secrets_are_dropped() { + let mut redactor = Redactor::new(); + redactor.add("hunter22"); + let judge = |body: &str| judge_response(&request(RequestKind::Api), &problem(400, body), &redactor); + assert_eq!( + judge(r#"{"type":"urn:ietf:params:jmap:error:hunter22"}"#), + Err(Error::Refused { problem: Some(RequestProblem::Other(String::new())), detail: None }) + ); + assert_eq!(judge(r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"xhunter22"}"#), Err(Error::TooLarge { limit: None })); +} + +// RFC 9457 §3.1: a member of the wrong type is ignored. +#[test] +fn wrong_member_types_are_ignored() { + let response = problem(400, r#"{"type":42,"status":"401","limit":[],"detail":{"x":1},"title":"Bad"}"#); + match judge(response) { + Err(Error::Refused { problem: None, detail: Some(detail) }) => assert_eq!(detail.as_str(), "Bad"), + other => panic!("{other:?}"), + } +} + +// Requirement 2.4: server text is a short, clean excerpt without secrets. +#[test] +fn detail_is_cleaned() { + let body = format!(r#"{{"detail":"bad token token-abcdef\n\u001b[2J{}"}}"#, "x".repeat(500)); + match judge(problem(500, &body)) { + Err(Error::Http { status: 500, detail: Some(detail) }) => { + assert!(detail.as_str().starts_with("bad token [redacted] [2J")); + assert!(!detail.as_str().contains('\n') && !detail.as_str().contains('\u{1b}')); + assert!(detail.as_str().chars().count() <= crate::redact::EXCERPT_MAX_CHARS + 1); + } + other => panic!("{other:?}"), + } + let html = HttpResponse::new(502, b"

Bad gateway

".to_vec()).with_field("Content-Type", "text/html"); + assert_eq!(judge(html), Err(Error::Http { status: 502, detail: None })); +} + +// RFC 9110 §10.2.2 and §15.4: redirects are judged, not followed; http and loopback targets are refused. +#[test] +fn redirects() { + let response = HttpResponse::new(307, Vec::new()).with_field("Location", "/api/v2/"); + match judge(response) { + Err(Error::Redirected { target, same_origin: true }) => assert_eq!(target.as_str(), "https://jmap.example.com/api/v2/"), + other => panic!("{other:?}"), + } + let response = HttpResponse::new(301, Vec::new()).with_field("Location", "https://jmap.example.net/api/"); + assert!(matches!(judge(response), Err(Error::Redirected { same_origin: false, .. }))); + let response = HttpResponse::new(302, Vec::new()).with_field("Location", "http://jmap.example.com/api/"); + assert_eq!(judge(response), Err(Error::RedirectRefused(AddressError::InsecureTransport))); + let response = HttpResponse::new(308, Vec::new()).with_field("Location", "http://127.0.0.1:8080/"); + assert_eq!(judge(response), Err(Error::RedirectRefused(AddressError::LocalFromRemote))); + let response = HttpResponse::new(302, Vec::new()).with_field("Location", "https://user@example.com/"); + assert_eq!(judge(response), Err(Error::RedirectRefused(AddressError::UserInfo))); + assert_eq!(judge(HttpResponse::new(302, Vec::new())), Err(Error::Http { status: 302, detail: None })); + assert_eq!(judge(HttpResponse::new(304, Vec::new())), Err(Error::Http { status: 304, detail: None })); +} + +// RFC 9110 §15: anything else is an HTTP error with its status. +#[test] +fn other_statuses() { + assert_eq!(judge(HttpResponse::new(500, Vec::new())), Err(Error::Http { status: 500, detail: None })); + assert_eq!(judge(HttpResponse::new(418, Vec::new())), Err(Error::Http { status: 418, detail: None })); + assert_eq!(judge(HttpResponse::new(100, Vec::new())), Err(Error::Http { status: 100, detail: None })); + assert_eq!(judge(HttpResponse::new(0, Vec::new())), Err(Error::Http { status: 0, detail: None })); +} + +// RFC 8259 §8.1: a byte order mark may be ignored. +#[test] +fn json_with_bom() { + assert!(parse_json(b"\xEF\xBB\xBF{\"a\":1}").is_ok()); + assert_eq!(parse_json(b"{\"a\":"), Err(Unusable::InvalidJson)); + let deep = "[".repeat(10_000); + assert_eq!(parse_json(deep.as_bytes()), Err(Unusable::InvalidJson)); +} + +// RFC 8259 §9 and RFC 8620 §8.4: a text that would expand into too many values is refused before parsing. +#[test] +fn json_budget() { + let many_objects = format!("[{}{{}}]", "{},".repeat(MAX_JSON_OBJECTS)); + assert_eq!(parse_json(many_objects.as_bytes()), Err(Unusable::TooComplex)); + let many_values = format!("[{}0]", "0,".repeat(MAX_JSON_VALUES)); + assert_eq!(parse_json(many_values.as_bytes()), Err(Unusable::TooComplex)); + let in_strings = format!("[\"{}\"]", "{},:[\\\"".repeat(200_000)); + assert!(parse_json(in_strings.as_bytes()).is_ok()); +} diff --git a/crates/jmap/src/client.rs b/crates/jmap/src/client.rs new file mode 100644 index 000000000..43ee983af --- /dev/null +++ b/crates/jmap/src/client.rs @@ -0,0 +1,245 @@ +//! Fetching the session and talking to the API through the program's HTTP exchange. +//! +//! Each operation exists in two forms: a synchronous pair (build the [`HttpRequest`], then read the +//! [`HttpResponse`]) for programs that drive the exchange themselves, and an `async` helper that does both +//! through an [`HttpExchange`]. Neither follows redirects: a redirect comes back as +//! [`Error::Redirected`] with a checked target, and the caller decides. + +use std::fmt; + +use crate::address::Address; +use crate::classify::judge_response; +use crate::credentials::Credentials; +use crate::error::{BuildError, Error}; +use crate::http::{HttpExchange, HttpRequest, HttpResponse, Method, RequestKind}; +use crate::id::Id; +use crate::redact::{Excerpt, Redactor}; +use crate::request::ApiRequest; +use crate::response::ApiResponse; +use crate::session::{Account, CAPABILITY_MAIL, CoreLimits, Session}; + +/// A redactor knowing the credentials and the session address. +fn base_redactor(address: &Address, credentials: &Credentials) -> Redactor { + let mut redactor = Redactor::new(); + credentials.register_secrets(&mut redactor); + redactor.add_personal(address.as_str()); + if let Some(query) = address.query() { + redactor.add_personal(query); + // A query parameter may carry a token: each value, as written and decoded. + for pair in query.split('&') { + let value = pair.split_once('=').map_or(pair, |(_, value)| value); + redactor.add_personal(value); + if let Ok(decoded) = crate::percent::decode(value) { + let decoded = crate::credentials::Secret::new(decoded); + redactor.add_personal(decoded.expose()); + } + } + } + redactor +} + +/// Builds the request that fetches the session object (RFC 8620 §2). The address must satisfy the transport +/// rule, since the credentials travel with the request. +pub fn session_request(address: &Address, credentials: &Credentials) -> Result { + address.require_secure_transport()?; + Ok(HttpRequest::new(Method::Get, address.clone(), RequestKind::Session, credentials.authorization())) +} + +/// Reads the response to a [`session_request`]. +pub fn read_session(request: &HttpRequest, response: &HttpResponse, credentials: &Credentials) -> Result { + let redactor = base_redactor(request.address(), credentials); + judge_response(request, response, &redactor)?; + Session::parse(response.body(), request.address()) +} + +/// Fetches and reads the session object through `exchange`. +pub async fn fetch_session(exchange: &E, address: &Address, credentials: &Credentials) -> Result { + let request = session_request(address, credentials)?; + let response = exchange.exchange(&request).await?; + read_session(&request, &response, credentials) +} + +/// A JMAP client for one session and one mail account. +#[derive(Clone)] +pub struct Client { + session: Session, + credentials: Credentials, + account: Id, + redactor: Redactor, +} + +impl Client { + /// A client for the session's mail account ([`Session::mail_account`]). + pub fn new(session: Session, credentials: Credentials) -> Result { + let account = session.mail_account().cloned().ok_or(Error::NoMailAccount)?; + Self::for_account(session, credentials, account) + } + + /// A client for a chosen account of the session. + pub fn for_account(session: Session, credentials: Credentials, account: Id) -> Result { + let account_name = session.account(&account).ok_or(Error::NoMailAccount)?.name().to_owned(); + let mut redactor = base_redactor(session.address(), &credentials); + redactor.add_personal(session.username()); + redactor.add_personal(&account_name); + Ok(Self { session, credentials, account, redactor }) + } + + /// The session. + pub fn session(&self) -> &Session { + &self.session + } + + /// The account id in use. + pub fn account_id(&self) -> &Id { + &self.account + } + + /// The account in use. + pub fn account(&self) -> Option<&Account> { + self.session.account(&self.account) + } + + /// The core limits, defaulted and capped. + pub fn limits(&self) -> &CoreLimits { + self.session.core_limits() + } + + /// Whether the server advertises `capability`. + pub fn has_capability(&self, capability: &str) -> bool { + self.session.has_capability(capability) + } + + /// Whether the account in use offers `capability`. + pub fn account_has_capability(&self, capability: &str) -> bool { + self.account().is_some_and(|account| account.has_capability(capability)) + } + + /// Whether the account offers mail. + pub fn has_mail(&self) -> bool { + self.account_has_capability(CAPABILITY_MAIL) + } + + /// Cleans a server's text for an error message, with every secret this client knows replaced. + pub fn excerpt(&self, text: &str) -> Excerpt { + self.redactor.excerpt(text) + } + + /// The redactor holding this client's secrets. + pub fn redactor(&self) -> &Redactor { + &self.redactor + } + + /// Whether `response` reports a session state other than the one this client holds (RFC 8620 §3.4): the + /// session should then be fetched again. + pub fn is_session_stale(&self, response: &ApiResponse) -> bool { + response.session_state() != self.session.state() + } + + /// The address to download a blob (RFC 8620 §6.2), with `media_type` as `type` and `name` as `name`. + pub fn download_address(&self, blob: &Id, media_type: &str, name: &str) -> Result { + let values = [("accountId", self.account.as_str()), ("blobId", blob.as_str()), ("type", media_type), ("name", name)]; + Ok(self.session.download_template().expand(&values)?) + } + + /// The address to upload a blob to the account (RFC 8620 §6.1). + pub fn upload_address(&self) -> Result { + Ok(self.session.upload_template().expand(&[("accountId", self.account.as_str())])?) + } + + /// The request that fetches the session again. + pub fn session_request(&self) -> Result { + session_request(self.session.address(), &self.credentials) + } + + /// The HTTP request for an API request (RFC 8620 §3.1). Checked first: the capabilities used must be + /// advertised, the number of calls must be within `maxCallsInRequest`, and the body within + /// `maxSizeRequest`. + pub fn api_request(&self, request: &ApiRequest) -> Result { + if request.capabilities().any(|capability| !self.session.has_capability(capability)) { + return Err(BuildError::UnsupportedCapability.into()); + } + let calls = u64::try_from(request.len()).unwrap_or(u64::MAX); + let max_calls = self.limits().max_calls_in_request(); + if calls > max_calls { + return Err(BuildError::TooManyCalls { limit: max_calls }.into()); + } + let body = request.to_json()?; + let max_size = self.limits().max_size_request(); + if u64::try_from(body.len()).unwrap_or(u64::MAX) > max_size { + return Err(BuildError::TooLarge { limit: max_size }.into()); + } + let address = self.session.api_address().clone(); + let http = HttpRequest::new(Method::Post, address, RequestKind::Api, self.credentials.authorization()).with_json_body(body); + Ok(match request.response_limit() { + Some(limit) => http.with_response_limit(limit), + None => http, + }) + } + + /// Reads the response to an [`Client::api_request`]. + pub fn read_api_response(&self, request: &HttpRequest, response: &HttpResponse) -> Result { + judge_response(request, response, &self.redactor)?; + ApiResponse::parse(response.body(), &self.redactor) + } + + /// The HTTP request that downloads a blob (RFC 8620 §6.2). + pub fn download_request(&self, blob: &Id, media_type: &str, name: &str) -> Result { + let address = self.download_address(blob, media_type, name)?; + Ok(HttpRequest::new(Method::Get, address, RequestKind::Download, self.credentials.authorization())) + } + + /// Reads the response to a [`Client::download_request`] and hands back the blob's bytes. + pub fn read_download(&self, request: &HttpRequest, response: HttpResponse) -> Result, Error> { + judge_response(request, &response, &self.redactor)?; + Ok(response.into_body()) + } + + /// Sends an API request through `exchange` and reads the response. + pub async fn call(&self, exchange: &E, request: &ApiRequest) -> Result { + let http = self.api_request(request)?; + let response = exchange.exchange(&http).await?; + self.read_api_response(&http, &response) + } + + /// Downloads a blob through `exchange`. + /// + /// `max_bytes`, when given, lowers the largest body accepted for this download below + /// [`DOWNLOAD_RESPONSE_LIMIT`](crate::http::DOWNLOAD_RESPONSE_LIMIT); the exchange sees it before + /// reading the body. + pub async fn download( + &self, + exchange: &E, + blob: &Id, + media_type: &str, + name: &str, + max_bytes: Option, + ) -> Result, Error> { + let mut http = self.download_request(blob, media_type, name)?; + if let Some(limit) = max_bytes { + http = http.with_response_limit(limit); + } + let response = exchange.exchange(&http).await?; + self.read_download(&http, response) + } + + /// Fetches the session again through `exchange`, for when [`Client::is_session_stale`] says so. + pub async fn refetch_session(&self, exchange: &E) -> Result { + fetch_session(exchange, self.session.address(), &self.credentials).await + } +} + +impl fmt::Debug for Client { + /// Only what says nothing about the person: the authentication scheme, how many capabilities the + /// server offers, and the core limits. No account id, no address or origin (a personal domain names its + /// owner), no user name, no account name, no secret. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Client") + .field("scheme", &self.credentials.scheme_name()) + .field("capabilities", &self.session.capabilities().count()) + .field("limits", self.session.core_limits()) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/client/tests.rs b/crates/jmap/src/client/tests.rs new file mode 100644 index 000000000..b398dbaf2 --- /dev/null +++ b/crates/jmap/src/client/tests.rs @@ -0,0 +1,245 @@ +use super::*; +use crate::address::AddressError; +use crate::error::Unusable; +use crate::http::TransportError; +use crate::retry::RetryAfter; +use crate::session::tests::{EXAMPLE, session_address}; +use crate::test_support::{FakeExchange, block_on, made_secret}; +use serde_json::{Map, Value, json}; +use std::time::Duration; + +fn credentials() -> Credentials { + Credentials::basic("john@example.com", &made_secret("open-sesame-42")).unwrap() +} + +fn json_response(status: u16, body: &Value) -> HttpResponse { + HttpResponse::new(status, body.to_string().into_bytes()).with_field("Content-Type", "application/json") +} + +fn client() -> Client { + let session = Session::parse(EXAMPLE.as_bytes(), &session_address()).unwrap(); + Client::new(session, credentials()).unwrap() +} + +// RFC 8620 §2: the session is fetched with an authenticated GET and parsed. +#[test] +fn fetches_session() { + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(200, EXAMPLE.as_bytes().to_vec()).with_field("Content-Type", "application/json"))]); + let session = block_on(fetch_session(&exchange, &session_address(), &credentials())).unwrap(); + assert_eq!(session.state(), "75128aab4b1b"); + let sent = exchange.sent(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].method(), Method::Get); + assert_eq!(sent[0].url(), "https://jmap.example.com/.well-known/jmap"); + assert!(sent[0].fields().any(|(name, value)| name == "Authorization" && value == "Basic am9obkBleGFtcGxlLmNvbTpvcGVuLXNlc2FtZS00Mg==")); +} + +// Requirement 1.2: credentials are never sent over plain http to a remote host. +#[test] +fn refuses_insecure_session_address() { + let exchange = FakeExchange::new(Vec::new()); + let address = Address::parse("http://jmap.example.com/").unwrap(); + assert_eq!(block_on(fetch_session(&exchange, &address, &credentials())), Err(Error::Address(AddressError::InsecureTransport))); + assert!(exchange.sent().is_empty()); +} + +// RFC 8620 §2.2 ("following any redirects") with requirement 3.1: the redirect comes back to the caller. +#[test] +fn session_redirect_is_returned() { + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(301, Vec::new()).with_field("Location", "https://mail.example.net/jmap/session"))]); + match block_on(fetch_session(&exchange, &session_address(), &credentials())) { + Err(Error::Redirected { target, same_origin: false }) => assert_eq!(target.as_str(), "https://mail.example.net/jmap/session"), + other => panic!("{other:?}"), + } + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(302, Vec::new()).with_field("Location", "http://mail.example.net/"))]); + assert_eq!(block_on(fetch_session(&exchange, &session_address(), &credentials())), Err(Error::RedirectRefused(AddressError::InsecureTransport))); +} + +// Requirement 3.2: the account, limits, capabilities, download and upload addresses. +#[test] +fn client_exposes_what_mail_code_needs() { + let client = client(); + assert_eq!(client.account_id().as_str(), "A13824"); + assert_eq!(client.limits().max_calls_in_request(), 32); + assert!(client.has_capability(CAPABILITY_MAIL)); + assert!(client.has_mail()); + assert!(!client.has_capability("urn:ietf:params:jmap:calendars")); + let blob = Id::new("Gb1").unwrap(); + assert_eq!( + client.download_address(&blob, "image/png", "photo 1.png").unwrap().as_str(), + "https://jmap.example.com/download/A13824/Gb1/photo%201.png?accept=image%2Fpng" + ); + assert_eq!(client.upload_address().unwrap().as_str(), "https://jmap.example.com/upload/A13824/"); + assert!(matches!(client.download_address(&blob, "image/png", ".."), Err(Error::Template(_)))); +} + +// RFC 8620 §3.1, §3.3, §3.4 and §3.7: a request with a result reference, and its per-call results. +#[test] +fn calls_the_api() { + let client = client(); + let mut request = ApiRequest::new(); + request.using(CAPABILITY_MAIL).unwrap(); + let query = request.add_call("Email/query", Map::from_iter([("accountId".to_owned(), json!("A13824"))])).unwrap(); + let ids = request.reference(&query, "Email/query", "/ids").unwrap(); + let get = request.add_call_with_references("Email/get", Map::from_iter([("accountId".to_owned(), json!("A13824"))]), vec![("ids", ids)]).unwrap(); + let answer = json!({ + "methodResponses": [ + ["Email/query", {"accountId": "A13824", "ids": ["M1"], "position": 0}, "c0"], + ["error", {"type": "invalidResultReference"}, "c1"] + ], + "sessionState": "changed" + }); + let exchange = FakeExchange::new(vec![Ok(json_response(200, &answer))]); + let mut response = block_on(client.call(&exchange, &request)).unwrap(); + assert!(client.is_session_stale(&response)); + assert_eq!(response.take_named(&query, "Email/query").unwrap()["ids"], json!(["M1"])); + assert!(matches!(response.take(&get), Err(crate::response::CallFailure::Method(_)))); + let sent = &exchange.sent()[0]; + assert_eq!(sent.method(), Method::Post); + assert_eq!(sent.url(), "https://jmap.example.com/api/"); + let body: Value = serde_json::from_slice(sent.body()).unwrap(); + assert_eq!(body["using"], json!(["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"])); +} + +// RFC 8620 §2 limits: maxCallsInRequest and maxSizeRequest are enforced before sending. +#[test] +fn enforces_limits_before_sending() { + let client = client(); + let mut request = ApiRequest::new(); + for _ in 0..33 { + request.add_call("Core/echo", Map::new()).unwrap(); + } + assert_eq!(client.api_request(&request).unwrap_err(), Error::Build(BuildError::TooManyCalls { limit: 32 })); + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::from_iter([("blob".to_owned(), json!("x".repeat(10_000_001)))])).unwrap(); + assert_eq!(client.api_request(&request).unwrap_err(), Error::Build(BuildError::TooLarge { limit: 10_000_000 })); + let mut request = ApiRequest::new(); + request.using("urn:ietf:params:jmap:calendars").unwrap(); + request.add_call("Core/echo", Map::new()).unwrap(); + assert_eq!(client.api_request(&request).unwrap_err(), Error::Build(BuildError::UnsupportedCapability)); +} + +// Requirement 2.4 through the client: busy, credentials, transport overload, unusable answers. +#[test] +fn classifies_api_failures() { + let client = client(); + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::new()).unwrap(); + let cases: Vec<(Result, Error)> = vec![ + ( + Ok(HttpResponse::new(429, Vec::new()).with_field("Retry-After", "7")), + Error::Busy { retry_after: Some(RetryAfter::Delay(Duration::from_secs(7))) }, + ), + (Ok(HttpResponse::new(401, Vec::new())), Error::CredentialsRefused), + (Err(TransportError::Overloaded), Error::Busy { retry_after: None }), + (Err(TransportError::TimedOut), Error::Transport(TransportError::TimedOut)), + (Ok(HttpResponse::new(200, b"".to_vec()).with_field("Content-Type", "text/html")), Error::Unusable(Unusable::WrongContentType)), + (Ok(HttpResponse::new(200, b"{\"x\":".to_vec()).with_field("Content-Type", "application/json")), Error::Unusable(Unusable::InvalidJson)), + ]; + for (answer, expected) in cases { + let exchange = FakeExchange::new(vec![answer]); + assert_eq!(block_on(client.call(&exchange, &request)).unwrap_err(), expected); + } +} + +// Requirement 3.5 through the client: the user name, the account name and the password never show. +#[test] +fn errors_hide_secrets() { + let client = client(); + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::new()).unwrap(); + let problem = json!({"status": 500, "detail": "user john@example.com with password open-sesame-42 at https://jmap.example.com/.well-known/jmap"}); + let response = HttpResponse::new(500, problem.to_string().into_bytes()).with_field("Content-Type", "application/problem+json"); + let exchange = FakeExchange::new(vec![Ok(response)]); + let error = block_on(client.call(&exchange, &request)).unwrap_err(); + let shown = error.to_string(); + assert!(!shown.contains("john") && !shown.contains("open-sesame") && !shown.contains("well-known")); +} + +// Requirement 2.3: the client's debug form shows no secret, no user name, no account id and no address. +#[test] +fn debug_shows_nothing_personal() { + let client = client(); + let debug = format!("{client:?}"); + for hidden in ["john", "jane", "open-sesame", "A13824", "jmap.example.com", "75128aab4b1b", "am9obk"] { + assert!(!debug.contains(hidden), "{hidden} shows in the client's debug form"); + } + assert!(debug.starts_with("Client { scheme: \"Basic\"")); +} + +// Requirement 3.5: a token in the session address's query is redacted on its own, and a short server-chosen +// user name does not blank every message. +#[test] +fn session_address_parts_are_secrets() { + let address = Address::parse("https://jmap.example.com/.well-known/jmap?apikey=SECRETKEY123").unwrap(); + let text = EXAMPLE.replace(r#""username": "john@example.com""#, r#""username": "jo""#); + let session = Session::parse(text.as_bytes(), &address).unwrap(); + let client = Client::new(session, Credentials::bearer("token-0123456789").unwrap()).unwrap(); + assert_eq!(client.excerpt("invalid key SECRETKEY123 for jo").as_str(), "invalid key [redacted] for jo"); + assert_eq!(client.excerpt("bad token-0123456789").as_str(), "bad [redacted]"); +} + +// RFC 8620 §6.2: a download is an authenticated GET of the expanded template; the body comes back as is. +#[test] +fn downloads() { + let client = client(); + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(200, vec![0, 159, 146, 150]).with_field("Content-Type", "application/octet-stream"))]); + let bytes = block_on(client.download(&exchange, &Id::new("B1").unwrap(), "application/octet-stream", "a.bin", None)).unwrap(); + assert_eq!(bytes, vec![0, 159, 146, 150]); + assert_eq!(exchange.sent()[0].url(), "https://jmap.example.com/download/A13824/B1/a.bin?accept=application%2Foctet-stream"); + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(404, Vec::new()))]); + assert_eq!(block_on(client.download(&exchange, &Id::new("B1").unwrap(), "", "a.bin", None)), Err(Error::NotFound)); +} + +// Requirement 2.2: a limit the caller sets for one request reaches the exchange before the body is read, +// and a body above it is refused, whether the exchange stops early or hands the whole body over. +#[test] +fn per_request_limits() { + let client = client(); + let blob = Id::new("B1").unwrap(); + let body = || Ok(HttpResponse::new(200, vec![7; 5])); + let careful = FakeExchange::capping(vec![body()]); + assert_eq!(block_on(client.download(&careful, &blob, "", "a.bin", Some(4))), Err(Error::Unusable(Unusable::BodyTooLarge))); + assert_eq!(careful.sent()[0].max_response_bytes(), 4); + let careless = FakeExchange::new(vec![body()]); + assert_eq!(block_on(client.download(&careless, &blob, "", "a.bin", Some(4))), Err(Error::Unusable(Unusable::BodyTooLarge))); + let fits = FakeExchange::capping(vec![body()]); + assert_eq!(block_on(client.download(&fits, &blob, "", "a.bin", Some(5))).unwrap(), vec![7; 5]); + let unlimited = FakeExchange::new(vec![body()]); + block_on(client.download(&unlimited, &blob, "", "a.bin", None)).unwrap(); + assert_eq!(unlimited.sent()[0].max_response_bytes(), crate::http::DOWNLOAD_RESPONSE_LIMIT); + + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::new()).unwrap(); + request.set_response_limit(64); + assert_eq!(client.api_request(&request).unwrap().max_response_bytes(), 64); + let answer = json!({"methodResponses": [["Core/echo", {"pad": "x".repeat(100)}, "c0"]], "sessionState": "s"}); + let exchange = FakeExchange::capping(vec![Ok(json_response(200, &answer))]); + assert_eq!(block_on(client.call(&exchange, &request)).unwrap_err(), Error::Unusable(Unusable::BodyTooLarge)); + request.set_response_limit(usize::MAX); + assert_eq!(client.api_request(&request).unwrap().max_response_bytes(), crate::http::API_RESPONSE_LIMIT); +} + +// Requirement 3.2: no mail account, no client. +#[test] +fn needs_a_mail_account() { + let text = EXAMPLE.replace("urn:ietf:params:jmap:mail\": {", "urn:example:other\": {"); + let session = Session::parse(text.as_bytes(), &session_address()).unwrap(); + assert_eq!(session.mail_account(), None); + assert_eq!(Client::new(session, credentials()).unwrap_err(), Error::NoMailAccount); +} + +// Requirement 2.6 context (RFC 8620 §3.1 calls made through the caller's exchange): the helpers' futures are +// Send when the exchange's are, so native programs can spawn them. +#[test] +fn futures_are_send() { + fn assert_send(_: &T) {} + let client = client(); + let exchange = FakeExchange::new(Vec::new()); + let request = ApiRequest::new(); + let future = client.call(&exchange, &request); + assert_send(&future); + let address = session_address(); + let credentials = credentials(); + assert_send(&fetch_session(&exchange, &address, &credentials)); +} diff --git a/crates/jmap/src/credentials.rs b/crates/jmap/src/credentials.rs new file mode 100644 index 000000000..5585602e4 --- /dev/null +++ b/crates/jmap/src/credentials.rs @@ -0,0 +1,245 @@ +//! Credentials: HTTP Basic (RFC 7617) and Bearer tokens (RFC 6750). +//! +//! Credentials live in memory only. Their `Debug` form names the scheme and nothing else: no secret and no +//! user name, which is usually an e-mail address and so personal data. The memory holding a secret is +//! overwritten when it is dropped; this is best effort (the allocator, the operating system or an earlier +//! copy made by the caller are out of reach without `unsafe`), not a guarantee. + +use std::fmt; + +use crate::redact::Redactor; + +/// The longest user name, password or token accepted, in octets. +pub const MAX_CREDENTIAL_LEN: usize = 8 * 1024; + +/// A piece of text that must not be shown, and that is overwritten when dropped. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct Secret(String); + +impl Secret { + pub(crate) fn new(text: String) -> Self { + Self(text) + } + + pub(crate) fn expose(&self) -> &str { + &self.0 + } +} + +impl Drop for Secret { + fn drop(&mut self) { + let mut bytes = std::mem::take(&mut self.0).into_bytes(); + bytes.iter_mut().for_each(|b| *b = 0); + std::hint::black_box(&bytes); + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } +} + +/// Why credentials were refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum CredentialsError { + /// The user name is empty. + EmptyUser, + /// The token is empty. + EmptyToken, + /// The user name holds a colon, which Basic cannot carry (RFC 7617 §2). + ColonInUser, + /// A control character in the user name or password (RFC 7617 §2). + ControlCharacter, + /// The token holds a character RFC 6750 §2.1 does not allow. + InvalidToken, + /// Longer than [`MAX_CREDENTIAL_LEN`]. + TooLong, +} + +impl fmt::Display for CredentialsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::EmptyUser => "the user name is empty; enter the user name of the mail account", + Self::EmptyToken => "the access token is empty; sign in again to obtain one", + Self::ColonInUser => { + "the user name contains a colon (:), which password sign-in cannot carry; use a token instead, or the user name without it" + } + Self::ControlCharacter => "the user name or password contains an invisible control character, often from a paste; type it again", + Self::InvalidToken => { + "the access token contains a space, a comma or another character a token cannot hold; copy it again without extra characters" + } + Self::TooLong => "the user name, password or token is longer than 8 KiB, more than this program accepts; check what was pasted", + }) + } +} + +impl std::error::Error for CredentialsError {} + +#[derive(Clone)] +enum Kind { + Basic { user: Secret, password: Secret }, + Bearer { token: Secret }, +} + +/// Credentials for the JMAP server. +#[derive(Clone)] +pub struct Credentials { + kind: Kind, + authorization: Secret, +} + +const BASE64_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +fn base64_char(index: u8) -> char { + char::from(BASE64_ALPHABET.get(usize::from(index & 0x3F)).copied().unwrap_or(b'A')) +} + +/// Base64 with padding (RFC 4648 §4), written into `out`. +fn base64_into(input: &[u8], out: &mut String) { + for chunk in input.chunks(3) { + let (a, b, c) = match *chunk { + [a, b, c] => (a, Some(b), Some(c)), + [a, b] => (a, Some(b), None), + [a] => (a, None, None), + _ => continue, + }; + let b0 = b.unwrap_or(0); + let c0 = c.unwrap_or(0); + out.push(base64_char(a >> 2)); + out.push(base64_char(((a & 0x03) << 4) | (b0 >> 4))); + out.push(if b.is_some() { base64_char(((b0 & 0x0F) << 2) | (c0 >> 6)) } else { '=' }); + out.push(if c.is_some() { base64_char(c0 & 0x3F) } else { '=' }); + } +} + +/// The part an octet may play in a bearer token (RFC 6750 §2.1, `b64token`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TokenOctet { + /// A letter, a digit, or one of `-` `.` `_` `~` `+` `/`: allowed anywhere before the padding. + Body, + /// `=`: allowed only at the end, any number of times. + Padding, + /// Anything else, spaces, commas, quotes, controls and non-ASCII octets among them. + Refused, +} + +/// Sorts one octet into its [`TokenOctet`] part. +const fn token_octet(octet: u8) -> TokenOctet { + match octet { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' => TokenOctet::Body, + b'-' | b'.' | b'_' | b'~' | b'+' | b'/' => TokenOctet::Body, + b'=' => TokenOctet::Padding, + _ => TokenOctet::Refused, + } +} + +/// Reads a bearer token in one pass: at least one body octet, then nothing but padding once padding began. +fn is_bearer_token(token: &str) -> bool { + let mut body_octets = 0usize; + let mut padding_started = false; + for octet in token.bytes() { + match (token_octet(octet), padding_started) { + (TokenOctet::Body, false) => body_octets = body_octets.saturating_add(1), + (TokenOctet::Padding, _) => padding_started = true, + (TokenOctet::Body, true) | (TokenOctet::Refused, _) => return false, + } + } + body_octets > 0 +} + +impl Credentials { + /// Basic credentials (RFC 7617 §2): `user:password` in UTF-8, base64-encoded. + /// + /// The user name may not be empty or hold a colon; neither part may hold a control character. The text is + /// sent as given: Unicode normalisation (RFC 7617 §2.1 mentions NFC) is left to the caller. + pub fn basic(user: &str, password: &str) -> Result { + if user.is_empty() { + return Err(CredentialsError::EmptyUser); + } + if user.len() > MAX_CREDENTIAL_LEN || password.len() > MAX_CREDENTIAL_LEN { + return Err(CredentialsError::TooLong); + } + if user.contains(':') { + return Err(CredentialsError::ColonInUser); + } + if user.chars().chain(password.chars()).any(char::is_control) { + return Err(CredentialsError::ControlCharacter); + } + let mut pair = Vec::with_capacity(user.len().saturating_add(password.len()).saturating_add(1)); + pair.extend_from_slice(user.as_bytes()); + pair.push(b':'); + pair.extend_from_slice(password.as_bytes()); + let encoded_len = pair.len().div_ceil(3).saturating_mul(4); + let mut header = String::with_capacity(encoded_len.saturating_add(6)); + header.push_str("Basic "); + base64_into(&pair, &mut header); + pair.iter_mut().for_each(|b| *b = 0); + std::hint::black_box(&pair); + Ok(Self { + kind: Kind::Basic { user: Secret::new(user.to_owned()), password: Secret::new(password.to_owned()) }, + authorization: Secret::new(header), + }) + } + + /// Bearer credentials (RFC 6750 §2.1). The token must match `b64token`. + pub fn bearer(token: &str) -> Result { + if token.is_empty() { + return Err(CredentialsError::EmptyToken); + } + if token.len() > MAX_CREDENTIAL_LEN { + return Err(CredentialsError::TooLong); + } + if !is_bearer_token(token) { + return Err(CredentialsError::InvalidToken); + } + let mut header = String::with_capacity(token.len().saturating_add(7)); + header.push_str("Bearer "); + header.push_str(token); + Ok(Self { kind: Kind::Bearer { token: Secret::new(token.to_owned()) }, authorization: Secret::new(header) }) + } + + /// The authentication scheme's name: `Basic` or `Bearer`. + pub fn scheme_name(&self) -> &'static str { + match self.kind { + Kind::Basic { .. } => "Basic", + Kind::Bearer { .. } => "Bearer", + } + } + + /// The value of the `Authorization` header field. + pub(crate) fn authorization(&self) -> &Secret { + &self.authorization + } + + /// Teaches a redactor every secret these credentials hold. + pub fn register_secrets(&self, redactor: &mut Redactor) { + match &self.kind { + Kind::Basic { user, password } => { + redactor.add_personal(user.expose()); + redactor.add(password.expose()); + } + Kind::Bearer { token } => redactor.add(token.expose()), + } + let header = self.authorization.expose(); + let value = header.split_once(' ').map_or(header, |(_, value)| value); + redactor.add(value); + if let Kind::Basic { .. } = self.kind { + // The base64 forms a server might echo: without padding, and in the URL-safe alphabet (RFC 4648 §5). + let unpadded = value.trim_end_matches('='); + redactor.add(unpadded); + let url_safe = Secret::new(unpadded.replace('+', "-").replace('/', "_")); + redactor.add(url_safe.expose()); + } + } +} + +impl fmt::Debug for Credentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Credentials({}, )", self.scheme_name()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/credentials/tests.rs b/crates/jmap/src/credentials/tests.rs new file mode 100644 index 000000000..2b2973a2a --- /dev/null +++ b/crates/jmap/src/credentials/tests.rs @@ -0,0 +1,107 @@ +use super::*; +use crate::test_support::made_secret; + +// RFC 7617 §2: the "Aladdin" / "open sesame" example. +#[test] +fn basic_example() { + let credentials = Credentials::basic("Aladdin", &made_secret("open sesame")).unwrap(); + assert_eq!(credentials.authorization().expose(), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="); +} + +// RFC 7617 §2.1: the UTF-8 example ("test" / "123£"). +#[test] +fn basic_utf8_example() { + let credentials = Credentials::basic("test", &made_secret("123\u{A3}")).unwrap(); + assert_eq!(credentials.authorization().expose(), "Basic dGVzdDoxMjPCow=="); +} + +// RFC 4648 §10: the base64 test vectors. +#[test] +fn base64_vectors() { + for (input, expected) in + [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] + { + let mut out = String::new(); + base64_into(input.as_bytes(), &mut out); + assert_eq!(out, expected); + } +} + +// RFC 7617 §2: no colon in the user name, no control characters anywhere. +#[test] +fn basic_refusals() { + assert_eq!(Credentials::basic("", &made_secret("x")).unwrap_err(), CredentialsError::EmptyUser); + assert_eq!(Credentials::basic("a:b", &made_secret("x")).unwrap_err(), CredentialsError::ColonInUser); + assert_eq!(Credentials::basic("user", &made_secret("pa\nss")).unwrap_err(), CredentialsError::ControlCharacter); + assert_eq!(Credentials::basic("us\u{7f}er", &made_secret("pass")).unwrap_err(), CredentialsError::ControlCharacter); + assert!(Credentials::basic("user", &made_secret("with:colon")).is_ok()); +} + +// RFC 6750 §2.1: the specification's example token, and an empty token. +#[test] +fn bearer_tokens() { + let credentials = Credentials::bearer("mF_9.B5f-4.1JqM").unwrap(); + assert_eq!(credentials.authorization().expose(), "Bearer mF_9.B5f-4.1JqM"); + assert_eq!(Credentials::bearer("").unwrap_err(), CredentialsError::EmptyToken); +} + +// RFC 6750 §2.1, rule 1: the body is made of ALPHA, DIGIT and the six marks - . _ ~ + /. +#[test] +fn token_body_octets() { + for token in ["AZaz09", "a-b", "a.b", "a_b", "a~b", "a+b", "a/b", "-._~+/"] { + assert!(is_bearer_token(token), "{token}"); + } + for octet in 0u8..=255 { + let expected = octet.is_ascii_alphanumeric() || b"-._~+/".contains(&octet); + assert_eq!(token_octet(octet) == TokenOctet::Body, expected, "{octet}"); + } +} + +// RFC 6750 §2.1, rule 2: at least one body octet ("1*"). +#[test] +fn token_needs_a_body() { + for token in ["", "=", "==="] { + assert!(!is_bearer_token(token), "{token:?}"); + } + assert_eq!(Credentials::bearer("==").unwrap_err(), CredentialsError::InvalidToken); +} + +// RFC 6750 §2.1, rule 3: "=" only at the end, any number of times ("*\"=\""). +#[test] +fn token_padding_only_at_the_end() { + for token in ["abc=", "abc==", "abc===="] { + assert!(is_bearer_token(token), "{token}"); + } + for token in ["=abc", "a=b", "ab=c=", "ab==c"] { + assert!(!is_bearer_token(token), "{token}"); + } +} + +// RFC 6750 §2.1, rule 4: nothing else, which also rules out header injection (RFC 9110 §5.5). +#[test] +fn token_refused_octets() { + for token in ["a b", "a\r\nX-Injected: 1", "a,b", "a\"b", "a:b", "tok\u{e9}n", "a\tb", "a;b"] { + assert!(!is_bearer_token(token), "{token:?}"); + assert_eq!(Credentials::bearer(token).unwrap_err(), CredentialsError::InvalidToken, "{token:?}"); + } +} + +// Requirement 2.3: no secret and no user name in the debug form. +#[test] +fn debug_shows_no_secret() { + let basic = Credentials::basic("john@example.com", &made_secret("hunter2-long")).unwrap(); + let shown = format!("{basic:?}"); + assert_eq!(shown, "Credentials(Basic, )"); + let bearer = Credentials::bearer("tok-123456").unwrap(); + assert!(!format!("{bearer:?}").contains("tok-123456")); +} + +// Requirement 3.5: the redactor learns the user name, the password and the encoded header value. +#[test] +fn registers_secrets() { + let credentials = Credentials::basic("john@example.com", &made_secret("open sesame")).unwrap(); + let mut redactor = Redactor::new(); + credentials.register_secrets(&mut redactor); + let excerpt = redactor.excerpt("john@example.com open sesame am9obkBleGFtcGxlLmNvbTpvcGVuIHNlc2FtZQ=="); + assert_eq!(excerpt.as_str(), "[redacted] [redacted] [redacted]"); +} diff --git a/crates/jmap/src/discovery.rs b/crates/jmap/src/discovery.rs new file mode 100644 index 000000000..65ca76041 --- /dev/null +++ b/crates/jmap/src/discovery.rs @@ -0,0 +1,60 @@ +//! Turning what a person types into the JMAP session address to fetch (requirement 1.6, RFC 8620 §2.2). +//! +//! A person may type `example.com`, `example.com/jmap/session`, `example.com:8443`, `https://example.com` or a +//! full address. The result defaults to `https`, and to [`WELL_KNOWN_PATH`] when only a host was given. Plain +//! `http` is kept only when it was typed and the host is loopback (requirement 1.2); it is never chosen by +//! this function, and `http` typed for a remote host is refused rather than silently upgraded, so that the +//! person sees that what they typed cannot be used. + +use crate::address::{Address, AddressError, MAX_ADDRESS_LEN}; + +/// The well-known path of the JMAP session resource (RFC 8620 §2.2, registered by §9.2). +pub const WELL_KNOWN_PATH: &str = "/.well-known/jmap"; + +/// Whether `text` begins with `scheme://`, the scheme being well formed (RFC 3986 §3.1). +fn has_explicit_scheme(text: &str) -> bool { + match text.split_once("://") { + Some((scheme, _)) => { + let mut bytes = scheme.bytes(); + matches!(bytes.next(), Some(b) if b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')) + } + None => false, + } +} + +/// Whether `text` starts with `http:` or `https:` without the `//` that must follow (`https:example.com`). +fn has_scheme_without_slashes(text: &str) -> bool { + let lower = text.get(..6).unwrap_or(text).to_ascii_lowercase(); + (lower.starts_with("http:") || lower.starts_with("https:")) && !text.contains("://") +} + +/// Turns a session address typed by a person into the address to fetch. +/// +/// Surrounding white space is ignored; anything else that is not a valid address is refused with the same +/// errors as [`Address::parse`]. The result always satisfies the transport rule. +pub fn session_address(input: &str) -> Result { + let trimmed = input.trim_matches(|c: char| c.is_ascii_whitespace()); + if trimmed.is_empty() { + return Err(AddressError::Empty); + } + if trimmed.len() > MAX_ADDRESS_LEN { + return Err(AddressError::TooLong); + } + if has_scheme_without_slashes(trimmed) { + return Err(AddressError::MissingAuthority); + } + let address = if has_explicit_scheme(trimmed) { + Address::parse(trimmed)? + } else { + let mut with_scheme = String::with_capacity(trimmed.len().saturating_add(8)); + with_scheme.push_str("https://"); + with_scheme.push_str(trimmed); + Address::parse(&with_scheme)? + }; + let address = if address.path() == "/" && address.query().is_none() { address.with_path(WELL_KNOWN_PATH)? } else { address }; + address.require_secure_transport()?; + Ok(address) +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/discovery/tests.rs b/crates/jmap/src/discovery/tests.rs new file mode 100644 index 000000000..f38cdeaa6 --- /dev/null +++ b/crates/jmap/src/discovery/tests.rs @@ -0,0 +1,53 @@ +use super::*; + +fn fetched(input: &str) -> String { + session_address(input).unwrap().as_str().to_owned() +} + +// RFC 8620 §2.2: a bare host means https and the well-known path. +#[test] +fn bare_host_gets_well_known_path() { + assert_eq!(fetched("example.com"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched(" Example.COM \n"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched("example.com/"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched("example.com:8443"), "https://example.com:8443/.well-known/jmap"); + assert_eq!(fetched("https://example.com"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched("[::1]:8080"), "https://[::1]:8080/.well-known/jmap"); +} + +// Requirement 1.6: a host with a path keeps the path; https is the default scheme. +#[test] +fn host_with_path_keeps_it() { + assert_eq!(fetched("example.com/jmap/session"), "https://example.com/jmap/session"); + assert_eq!(fetched("jmap.example.org:8443/api/session?x=1"), "https://jmap.example.org:8443/api/session?x=1"); + assert_eq!(fetched("HTTPS://example.net/jmap/"), "https://example.net/jmap/"); + assert_eq!(fetched("example.com/?a=b"), "https://example.com/?a=b"); +} + +// Requirement 1.2 and 1.6: never a downgrade; http only as typed, and only to loopback. +#[test] +fn never_downgrades() { + assert_eq!(fetched("localhost:8080"), "https://localhost:8080/.well-known/jmap"); + assert_eq!(fetched("http://localhost:8080"), "http://localhost:8080/.well-known/jmap"); + assert_eq!(fetched("http://127.0.0.1:8080/jmap"), "http://127.0.0.1:8080/jmap"); + assert_eq!(session_address("http://example.com"), Err(AddressError::InsecureTransport)); + assert_eq!(session_address("http://sub.localhost/"), Err(AddressError::InsecureTransport)); +} + +// Requirement 1.1 applied to typed input: what is not an address is refused. +#[test] +fn refuses_junk() { + assert_eq!(session_address(""), Err(AddressError::Empty)); + assert_eq!(session_address(" "), Err(AddressError::Empty)); + assert_eq!(session_address("https:example.com"), Err(AddressError::MissingAuthority)); + assert_eq!(session_address("http:localhost"), Err(AddressError::MissingAuthority)); + assert_eq!(session_address("ftp://example.com"), Err(AddressError::UnsupportedScheme)); + assert_eq!(session_address("user@example.com"), Err(AddressError::UserInfo)); + assert_eq!(session_address("exa mple.com"), Err(AddressError::ForbiddenCharacter)); + assert_eq!(session_address("example.com\\jmap"), Err(AddressError::Backslash)); + assert_eq!(session_address("bücher.example.com"), Err(AddressError::ForbiddenCharacter)); + assert_eq!( + session_address("example.com/path?u=http://example.org"), + Ok(Address::parse("https://example.com/path?u=http://example.org").unwrap()) + ); +} diff --git a/crates/jmap/src/error.rs b/crates/jmap/src/error.rs new file mode 100644 index 000000000..4db0f26c6 --- /dev/null +++ b/crates/jmap/src/error.rs @@ -0,0 +1,412 @@ +//! The crate's error type, arranged by what the mail client can do about each case. + +use std::fmt; + +use crate::address::{Address, AddressError}; +use crate::credentials::CredentialsError; +use crate::http::TransportError; +use crate::id::IdError; +use crate::redact::Excerpt; +use crate::retry::RetryAfter; +use crate::template::TemplateError; + +/// The longest limit name kept from a server. +const MAX_LIMIT_NAME_LEN: usize = 64; + +/// The name of a JMAP limit a server says was exceeded (`maxSizeRequest`, `maxCallsInRequest`, …), checked to +/// be a short identifier (letters, digits and `_`). +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct LimitName(String); + +impl LimitName { + /// Checks a limit name received from a server. + pub fn new(text: &str) -> Option { + if text.is_empty() || text.len() > MAX_LIMIT_NAME_LEN || !text.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return None; + } + Some(Self(text.to_owned())) + } + + /// The name. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Whether the limit is about concurrency (`maxConcurrentRequests`, `maxConcurrentUpload`) rather than size. + pub fn is_concurrency(&self) -> bool { + self.0.starts_with("maxConcurrent") + } +} + +impl fmt::Display for LimitName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl fmt::Debug for LimitName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "LimitName({})", self.0) + } +} + +/// A request-level JMAP problem (RFC 8620 §3.6.1), the `urn:ietf:params:jmap:error:` prefix removed. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RequestProblem { + /// `unknownCapability`: the request used a capability the server does not support. + UnknownCapability, + /// `notJSON`: the request was not `application/json` or not I-JSON. + NotJson, + /// `notRequest`: the request did not have the shape of a Request object. + NotRequest, + /// `limit`: a request limit was exceeded. + Limit, + /// Another JMAP problem type, checked to be a short identifier. + Other(String), +} + +impl fmt::Display for RequestProblem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnknownCapability => f.write_str("it uses a capability this server does not support"), + Self::NotJson => f.write_str("the server could not read it as JSON"), + Self::NotRequest => f.write_str("its JSON is not shaped like a JMAP request"), + Self::Limit => f.write_str("it goes beyond one of the server's request limits"), + Self::Other(name) if name.is_empty() => f.write_str("the server reported an unnamed JMAP problem"), + Self::Other(name) => write!(f, "the server reported the JMAP problem {name}"), + } + } +} + +/// Why a server's successful-looking answer cannot be used. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Unusable { + /// The content type is not the one expected (`application/json`). + WrongContentType, + /// The body is larger than the request's limit ([`HttpRequest::max_response_bytes`](crate::HttpRequest::max_response_bytes): + /// the kind's, or the lower one the caller set), whether the crate or the transport found it. + BodyTooLarge, + /// The body is not valid JSON (RFC 8259), or nests too deeply. + InvalidJson, + /// The body holds more JSON values or objects than the crate accepts. + TooComplex, + /// The JSON does not have the expected shape; the text says which part, in the crate's own words. + Shape(&'static str), + /// An address the server gave cannot be used; the text names the field. + Address(&'static str, AddressError), + /// An address template the server gave cannot be used; the text names the field. + Template(&'static str, TemplateError), +} + +impl fmt::Display for Unusable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongContentType => f.write_str("its content type is not application/json"), + Self::BodyTooLarge => f.write_str("it is larger than the size limit set for this request"), + Self::InvalidJson => f.write_str("it is not valid JSON, or nests more than 128 levels deep"), + Self::TooComplex => f.write_str("it holds more JSON values or objects than this program reads at once"), + Self::Shape(what) => write!(f, "{what}"), + Self::Address(field, error) => write!(f, "its {field} cannot be used: {error}"), + Self::Template(field, error) => write!(f, "its {field} cannot be used: {error}"), + } + } +} + +/// A mistake in a request being built, found before anything is sent. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum BuildError { + /// A method name that is empty, too long, or not printable ASCII. + InvalidMethodName, + /// A capability identifier that is empty, too long, or not printable ASCII. + InvalidCapability, + /// A capability the session does not advertise. + UnsupportedCapability, + /// An argument given both plainly and as a result reference (RFC 8620 §3.7). + DuplicateArgument, + /// An argument name that is empty or starts with `#`. + InvalidArgumentName, + /// A result reference to a call that is not earlier in this request (RFC 8620 §3.7). + UnknownCall, + /// A result reference path that is not a JSON Pointer (RFC 6901, with JMAP's `*`). + InvalidPath, + /// More calls than the server accepts (`maxCallsInRequest`) or than the crate builds. + TooManyCalls { + /// The limit that applies. + limit: u64, + }, + /// A request body larger than the server accepts (`maxSizeRequest`). + TooLarge { + /// The limit that applies, in octets. + limit: u64, + }, + /// A request without any method call. + NoCalls, + /// The request could not be written as JSON. + Serialisation, +} + +impl fmt::Display for BuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidMethodName => f.write_str("the method name is empty, longer than 255 characters, or holds a space or non-ASCII character"), + Self::InvalidCapability => { + f.write_str("the capability identifier is empty, longer than 256 characters, or holds a space or non-ASCII character") + } + Self::UnsupportedCapability => { + f.write_str("the request uses a capability this server's session does not offer; remove it or check the server") + } + Self::DuplicateArgument => f.write_str("an argument is given both directly and as a result reference (#name); keep only one"), + Self::InvalidArgumentName => { + f.write_str("a result reference argument name is empty or starts with #; give the plain name, the # is added") + } + Self::UnknownCall => f.write_str("a result reference names a call that is not earlier in this same request"), + Self::InvalidPath => { + f.write_str("a result reference path is not a JSON pointer: it must be empty or start with /, and use ~ only as ~0 or ~1") + } + Self::TooManyCalls { limit } => { + write!(f, "the request holds more than {limit} method calls, the server's limit; send it in several requests") + } + Self::TooLarge { limit } => { + write!(f, "the request body is larger than the server's limit of {limit} octets; send it in several requests") + } + Self::NoCalls => f.write_str("the request holds no method call, so there is nothing to send"), + Self::Serialisation => f.write_str("the request could not be written out as JSON"), + } + } +} + +/// Everything that can go wrong, arranged by what the mail client can do about it. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// The program's HTTP exchange failed before a response came (offline, timeout, TLS, …). + Transport(TransportError), + /// The server is busy or limiting the rate (429, 503, connection-level overload, a concurrency limit): + /// try again later, after at least the delay it asked for when there is one. + Busy { + /// What the server asked for, if anything usable. + retry_after: Option, + }, + /// The credentials were refused (401, or 403 when the problem details say so): ask for them again. + CredentialsRefused, + /// The request was too large for the server (413, or a JMAP `limit` problem): split it. + TooLarge { + /// The limit the server named, if any. + limit: Option, + }, + /// The server refuses access to this resource (403). + Forbidden { + /// What the server said, cleaned. + detail: Option, + }, + /// Nothing at that address (404, 410). + NotFound, + /// The server rejected the request as malformed or unsupported (400 and other client errors, or a JMAP + /// request-level problem). + Refused { + /// The JMAP problem, if one was recognised. + problem: Option, + /// What the server said, cleaned. + detail: Option, + }, + /// The server redirected; the crate follows nothing. The target has passed the transport rule and the + /// loopback check; whether to follow (and whether to send the credentials to another origin) is the + /// caller's decision. + Redirected { + /// Where the server points. + target: Address, + /// Whether the target has the same origin as the request. + same_origin: bool, + }, + /// The server redirected to an address that may not be used (plain http, loopback from a remote host, + /// malformed). + RedirectRefused(AddressError), + /// The server answered something that cannot be used. + Unusable(Unusable), + /// Any other HTTP status. + Http { + /// The status code. + status: u16, + /// What the server said, cleaned. + detail: Option, + }, + /// An address given to the crate is not usable. + Address(AddressError), + /// An address template could not be expanded. + Template(TemplateError), + /// Credentials given to the crate are not usable. + Credentials(CredentialsError), + /// An id given to the crate is not valid. + Id(IdError), + /// A request being built is wrong. + Build(BuildError), + /// The session offers no account for mail (RFC 8621 `urn:ietf:params:jmap:mail`). + NoMailAccount, +} + +impl Error { + /// The delay before trying again, for [`Error::Busy`]. + pub fn retry_after(&self) -> Option<&RetryAfter> { + match self { + Self::Busy { retry_after } => retry_after.as_ref(), + _ => None, + } + } +} + +fn with_detail(f: &mut fmt::Formatter<'_>, text: &str, detail: Option<&Excerpt>) -> fmt::Result { + f.write_str(text)?; + if let Some(detail) = detail { + write!(f, " (server said: \"{detail}\")")?; + } + Ok(()) +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Transport(error) => write!(f, "the request did not complete: {error}"), + Self::Busy { + retry_after: Some(RetryAfter::Delay(delay)), + } => write!( + f, + "the server is busy and asks to wait {} s before the next attempt", + delay.as_secs() + ), + Self::Busy { + retry_after: Some(RetryAfter::At(_)), + } => f.write_str("the server is busy and named a time before which not to try again"), + Self::Busy { retry_after: None } => { + f.write_str("the server is busy or limiting requests; wait a little before trying again") + } + Self::CredentialsRefused => f.write_str( + "the server rejected the user name and password (or the access token) for this account; check them, or sign in again if they were changed or have expired", + ), + Self::TooLarge { limit: Some(limit) } => write!( + f, + "the request went beyond the server's limit {limit}; send it in smaller parts" + ), + Self::TooLarge { limit: None } => { + f.write_str("the request was larger than the server accepts; send it in smaller parts") + } + Self::Forbidden { detail } => with_detail( + f, + "the server does not let this account reach that resource", + detail.as_ref(), + ), + Self::NotFound => f.write_str( + "the server has nothing at that address; it may have been moved or deleted, so refresh and try again", + ), + Self::Refused { + problem: Some(problem), + detail, + } => with_detail( + f, + &format!("the server rejected the request: {problem}"), + detail.as_ref(), + ), + Self::Refused { + problem: None, + detail, + } => with_detail(f, "the server rejected the request as invalid", detail.as_ref()), + Self::Redirected { + target, + same_origin: true, + } => write!( + f, + "the server moved this resource to another path on {}; it was not followed automatically", + target.origin() + ), + Self::Redirected { target, .. } => write!( + f, + "the server sent this request on to {}; confirm before following, since the credentials would go there too", + target.origin() + ), + Self::RedirectRefused(error) => write!( + f, + "the server redirected to an address that cannot be used: {error}" + ), + Self::Unusable(why) => write!(f, "the server's answer cannot be used: {why}"), + Self::Http { status, detail } => with_detail( + f, + &format!("the server answered with HTTP status {status}, which this program cannot act on"), + detail.as_ref(), + ), + Self::Address(error) => fmt::Display::fmt(error, f), + Self::Template(error) => fmt::Display::fmt(error, f), + Self::Credentials(error) => fmt::Display::fmt(error, f), + Self::Id(error) => fmt::Display::fmt(error, f), + Self::Build(error) => fmt::Display::fmt(error, f), + Self::NoMailAccount => f.write_str( + "the server offers no mail account to these credentials; check that mail is enabled for this account", + ), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Transport(error) => Some(error), + Self::RedirectRefused(error) | Self::Address(error) => Some(error), + Self::Template(error) => Some(error), + Self::Credentials(error) => Some(error), + Self::Id(error) => Some(error), + _ => None, + } + } +} + +impl From for Error { + /// A connection-level overload is the server being busy; a body cut at the request's limit is the same + /// failure as one the crate refuses after reading; anything else stays a transport failure. + fn from(error: TransportError) -> Self { + match error { + TransportError::Overloaded => Self::Busy { retry_after: None }, + TransportError::BodyTooLarge => Self::Unusable(Unusable::BodyTooLarge), + other => Self::Transport(other), + } + } +} + +impl From for Error { + fn from(error: AddressError) -> Self { + Self::Address(error) + } +} + +impl From for Error { + fn from(error: TemplateError) -> Self { + Self::Template(error) + } +} + +impl From for Error { + fn from(error: CredentialsError) -> Self { + Self::Credentials(error) + } +} + +impl From for Error { + fn from(error: IdError) -> Self { + Self::Id(error) + } +} + +impl From for Error { + fn from(error: BuildError) -> Self { + Self::Build(error) + } +} + +impl From for Error { + fn from(why: Unusable) -> Self { + Self::Unusable(why) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/error/tests.rs b/crates/jmap/src/error/tests.rs new file mode 100644 index 000000000..3a451c91e --- /dev/null +++ b/crates/jmap/src/error/tests.rs @@ -0,0 +1,39 @@ +use super::*; +use std::time::Duration; + +// RFC 8620 §3.6.1: the limit property names the limit; only short identifiers are kept. +#[test] +fn limit_names() { + assert_eq!(LimitName::new("maxSizeRequest").unwrap().as_str(), "maxSizeRequest"); + assert!(LimitName::new("maxConcurrentRequests").unwrap().is_concurrency()); + assert!(!LimitName::new("maxCallsInRequest").unwrap().is_concurrency()); + for bad in ["", "max size", "x\ny", &"a".repeat(65), "", + "x\0\r\n X-Evil: 3", + "x\"\r\nX-Evil: 4", + "x>\r\nX-Evil: 5\r\n", craft_mime::clean_id(evil)); + for subject in [evil.to_string(), craft_mime::encode_unstructured("Subject", evil), craft_mime::encode_words(evil)] { + let note = craft_mime::seven_bit_value("X-Note", evil); + let headers = [("Subject", subject.as_str()), ("X-Note", note.as_str()), ("To", to.as_str()), ("Message-ID", id.as_str())]; + let out = craft_mime::write_message(&headers, &Entity::text("plain", "body")); + assert_no_injection(&out, ALLOWED); + let m = craft_mime::parse(&out, &Limits::default()); + assert_eq!(m.root.data(), b"body\r\n", "the body starts where the writer put it"); + assert_eq!(craft_mime::parse_address_list(m.root.headers.get("to").unwrap_or_default()).len(), 1, "{to}"); + } + } +} + +#[test] +fn header_names_cannot_inject_fields() { + for name in ["X-A\r\nBcc", "X-A: 1\r\nBcc", "X A", "", "X-A\0", "Bcc:", "X-É"] { + let out = craft_mime::write_message(&[(name, "eve@example.org")], &Entity::text("plain", "body")); + assert_no_injection(&out, &["MIME-Version", "Content-Type", "Content-Transfer-Encoding"]); + let mut entity = Entity::text("plain", "body"); + entity.fields.push((name.to_string(), "eve@example.org".to_string())); + assert_no_injection(&craft_mime::write_message(&[], &entity), &["MIME-Version", "Content-Type", "Content-Transfer-Encoding"]); + } +} + +#[test] +fn part_values_cannot_inject_fields_or_parts() { + const ALLOWED: &[&str] = &["MIME-Version", "Content-Type", "Content-Transfer-Encoding", "Content-Disposition", "Content-ID", "Content-Location"]; + for evil in EVIL { + let mut file = Entity::file(&File { media_type: evil, name: evil, inline: false, content_id: evil, data: b"data" }); + file.fields.push(("Content-Location".into(), evil.to_string())); + assert_no_injection(&craft_mime::write_message(&[], &file), ALLOWED); + assert_no_injection(&craft_mime::write_message(&[], &Entity::text(evil, "x")), ALLOWED); + + // In a multipart: the subtype, a parameter's name and value, the boundary prefix, and + // a body that holds what looks like a delimiter. + let mut boundaries = Boundaries::new(evil); + let text = Entity::text("plain", &format!("--{evil}\n--=_craft_\n{evil}")); + let root = Entity::multipart(evil, &[("type", evil), (evil, "x"), ("boundary", "mine")], &[text, file], &mut boundaries); + let out = craft_mime::write_message(&[], &root); + assert_no_injection(&out, ALLOWED); + let m = craft_mime::parse(&out, &Limits::default()); + assert_eq!(m.root.parts().len(), 2, "{}", String::from_utf8_lossy(&out)); + assert_eq!(m.root.parts()[1].data(), b"data"); + for part in m.root.parts() { + assert!(part.headers.0.iter().all(|f| ALLOWED.iter().any(|a| a.eq_ignore_ascii_case(&f.name))), "{:?}", part.headers); + } + } +} + +#[test] +fn mbox_values_cannot_inject_messages() { + let date = craft_mime::parse_date("Tue, 1 Jul 2003 08:52:37 +0000").unwrap_or_default(); + for evil in EVIL { + let mut out = Vec::new(); + let sender = format!("a@example.org{evil}\nFrom eve@example.org Tue Jul 1 08:52:37 2003"); + let message = format!("Subject: x\r\n\r\nFrom the top\r\n\r\nFrom eve@example.org Tue Jul 1 08:52:37 2003\r\n{evil}\r\n"); + craft_mime::write_mbox_message(&mut out, &sender, &date, message.as_bytes()); + let back: Vec> = craft_mime::mbox_messages(&out).collect(); + assert_eq!(back, [message.into_bytes()]); + } +} diff --git a/crates/odf/Cargo.toml b/crates/odf/Cargo.toml new file mode 100644 index 000000000..8b5702c46 --- /dev/null +++ b/crates/odf/Cargo.toml @@ -0,0 +1,71 @@ +[package] +name = "craft-odf" +description = "OpenDocument packages: the ZIP container with its mimetype and manifest, password protection (AES-256-CBC, PBKDF2) and flat single-file documents, read and written." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-xml = { workspace = true } +thiserror = { workspace = true } +# `test-support`: `zip_guard::zip64_entry_count_sample` is public here. +craft-zip = { workspace = true, features = ["test-support"] } +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +base64 = "0.23" +# Password-protected packages (ODF 1.3 Part 2 section 3.4): AES-256-CBC, SHA-1/SHA-256, PBKDF2 +# (HMAC-SHA-1). +aes = "0.9" +sha1 = "0.11" +sha2 = "0.11" +hmac = "0.13" +getrandom = "0.3" +zeroize = "1" + +[features] +# ODT documents: read into and written from the craft-flow model and a craft-wordmodel Document. +text = ["dep:craft-flow", "dep:craft-wordmodel", "dep:craft-math", "dep:craft-metafile", "dep:image", "dep:chrono", "dep:log", "dep:js-sys"] + +[dependencies.craft-flow] +workspace = true +optional = true + +[dependencies.craft-wordmodel] +workspace = true +optional = true + +[dependencies.craft-math] +workspace = true +optional = true + +[dependencies.craft-metafile] +workspace = true +optional = true + +[dependencies.image] +version = "0.25" +default-features = false +features = ["png"] +optional = true + +[dependencies.chrono] +workspace = true +optional = true + +[dependencies.log] +version = "0.4" +optional = true + +[dev-dependencies] +# The tests build archives, hostile ones included, by hand. +zip = { version = "9", default-features = false, features = ["deflate-flate2"] } + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +# The browser clock for tracked-change dates (`text` feature; `SystemTime::now()` panics there). +js-sys = { version = "0.3", optional = true } +# The browser's `crypto.getRandomValues` for the salts and vectors of encrypted saves. +getrandom = { version = "0.3", features = ["wasm_js"] } diff --git a/crates/odf/README.md b/crates/odf/README.md new file mode 100644 index 000000000..9825fc47a --- /dev/null +++ b/crates/odf/README.md @@ -0,0 +1,80 @@ +# craft-odf + +OpenDocument packages (ODF 1.3 Part 2): what every OpenDocument file has in common, whatever it +holds. Text documents, spreadsheets, presentations and drawings all sit in the same container; +this crate reads and writes that container and nothing inside it. + +| Module | What it does | +| --- | --- | +| `Package`, `PackageWriter` | The ZIP container: the `mimetype` file, the manifest, the files, within `Limits`. | +| `Manifest` | `META-INF/manifest.xml`: each file's media type, and how an encrypted file is encrypted. Read by namespace, whatever prefix the file uses; `parse_strict` / `Package::manifest_strict` refuse a damaged one instead of reading what they can. | +| `crypt` | Password-protected packages: AES-256-CBC, SHA-256 start key, PBKDF2 (HMAC-SHA-1), SHA-256 or SHA-1 checksums. A package in, a package out. | +| `flat` | Flat single-file documents (`.fodt`, `.fods`, `.fodp`, `.fodg`): one `office:document` taken apart into a package's files, and a package put together into one. | +| `zip_guard` | Refuses a ZIP whose end record declares too many entries, before `zip` reserves room for them. | + +```rust +use craft_odf::{Limits, Package, PackageWriter, crypt, flat, mime}; + +fn main() -> Result<(), Box> { + let mut w = PackageWriter::new(mime::SPREADSHEET); + w.file("content.xml", "text/xml", b""); + w.stored("Pictures/chart.png", "image/png", b"..."); + let plain = w.finish()?; + + let locked = crypt::encrypt(&plain, "password")?; + assert!(crypt::is_encrypted(&locked)); + let opened = crypt::decrypt(&locked, "password")?; + + let mut p = Package::open(&opened, &Limits::DEFAULT)?; + assert_eq!(p.mimetype().as_deref(), Some(mime::SPREADSHEET)); + assert_eq!(p.read("content.xml")?.as_deref(), Some(b"".as_slice())); + + let one_file = flat::from_package(&opened, &Limits::DEFAULT)?; + let again = flat::to_package(&one_file, mime::SPREADSHEET, &Limits::DEFAULT)?; + assert_eq!(Package::open(&again, &Limits::DEFAULT)?.mimetype().as_deref(), Some(mime::SPREADSHEET)); + Ok(()) +} +``` + +## Text documents (feature `text`) + +`craft_odf::text` reads and writes OpenDocument Text (`.odt`, `.ott`, `.fodt`) through the +[craft-flow](../flow/README.md) model and a craft-wordmodel `Document`: paragraphs and headings +with their formatting and styles, lists and outline numbering, tables, pictures, shapes, text +boxes and groups, charts, equations (as formula objects, through craft-math's MathML), embedded +objects, fields, notes, comments, bookmarks, indexes, forms, sections and page setup with headers +and footers, tracked changes, metadata and settings. What a document holds that the model does not +is kept as read and written back (`craft_wordmodel::kept`). Packages are read and written with +this crate's `Package` and `PackageWriter`, password protection is `crypt`, flat files are `flat`. + +## Limits + +Input is hostile. Nothing here panics on any bytes. + +| Limit | Default | Where | +| --- | --- | --- | +| Entries a ZIP may declare | 20,000 | `Limits::max_entries`, checked on the raw bytes before `zip` parses the directory | +| Bytes one file inflates to | 256 MiB | `Limits::max_entry`; the size a file declares is not trusted | +| Bytes read from one package | 2 GiB | `Limits::max_total` | +| `mimetype` file | 255 bytes | longer is no media type | +| Manifest entries | 20,000 | the rest are dropped | +| PBKDF2 iterations for one file | 10,000,000 | more is refused | +| Key-derivation work for a package | 200,000,000 iteration-blocks | a thousand files as LibreOffice writes them | +| Files decrypted | 10,000 | | +| Pictures and objects taken out of a flat document | 10,000 | the rest are dropped | +| Nesting of one element name in a flat document | 512 | deeper is treated as unclosed | + +## Not supported + +- Blowfish CFB (the default before ODF 1.2) and OpenPGP encryption: such packages are refused with + a message that names the algorithm. +- Digital signatures (`META-INF/documentsignatures.xml`) are not checked or written; the file is + kept like any other. +- The meaning of `content.xml`, `styles.xml`, `meta.xml` and `settings.xml`. + +## Provenance + +Moved from WordCraft (`crates/formats/src/odt/crypt.rs`, `odt/flat.rs`, the package reading and +writing in `odt/import.rs` and `odt/package.rs`, and `crates/doc/src/zip_guard.rs`), with +GridCraft's namespace-aware manifest reading. Written from the OASIS OpenDocument 1.3 +specification, W3C XML Encryption (padding), RFC 2898 (PBKDF2) and PKWARE's APPNOTE.TXT. diff --git a/crates/odf/src/crypt.rs b/crates/odf/src/crypt.rs new file mode 100644 index 000000000..f90eb9ab9 --- /dev/null +++ b/crates/odf/src/crypt.rs @@ -0,0 +1,395 @@ +//! Password-protected packages (ODF 1.3 Part 2 §3.4, §4.4–4.16): each encrypted file is +//! deflated, then encrypted with a key derived from the password, and described in the +//! manifest (`manifest:encryption-data`). Read: the package decrypted into a plain one, which +//! the reader opens as any other. Written as LibreOffice writes it: AES-256-CBC, a SHA-256 +//! start key, PBKDF2 with HMAC-SHA-1 and a salt of its own for each file, a SHA-256 checksum of +//! the first 1024 bytes of each deflated file. +//! +//! Blowfish (the old default, §3.4.2) isn't supported: such files say so. +//! +//! [`decrypt`] gives a plain package, which a reader opens as any other; [`encrypt`] takes a +//! plain package. Neither looks at what the files hold. + +use std::io::{Read, Write}; + +use aes::cipher::{BlockCipherDecrypt, BlockCipherEncrypt, KeyInit}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use hmac::Mac; +use sha2::Digest; +use zeroize::Zeroizing; + +use crate::manifest::{Encryption as Params, MANIFEST_PATH, Manifest}; +use crate::{Limits, zip_guard}; + +/// Most bytes one file of a package decrypts or inflates to. +const MAX_ENTRY: u64 = Limits::DEFAULT.max_entry; +/// Most entries a package may declare. +const MAX_ZIP_ENTRIES: u64 = Limits::DEFAULT.max_entries; + +/// Why an encrypted package can't be opened, or a package can't be encrypted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CryptError { + /// The password doesn't decrypt it (the checksums don't match). + WrongPassword, + /// An encryption that isn't supported, a broken package, or (saving) no random numbers. + Unsupported(String), +} + +impl std::error::Error for CryptError {} + +impl std::fmt::Display for CryptError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CryptError::WrongPassword => write!(f, "the password is incorrect"), + CryptError::Unsupported(why) => write!(f, "{why}"), + } + } +} + +const AES256_CBC: &str = "http://www.w3.org/2001/04/xmlenc#aes256-cbc"; +const SHA256: &str = "http://www.w3.org/2000/09/xmldsig#sha256"; +const SHA256_1K: &str = "urn:oasis:names:tc:opendocument:xmlns:manifest:1.0#sha256-1k"; +/// PBKDF2 iterations written (LibreOffice's), and most read (a hostile count would hang). +const ITERATIONS: u32 = 100_000; +const MAX_ITERATIONS: u32 = 10_000_000; +/// Most files in a package decrypted. +const MAX_FILES: usize = 10_000; +/// Most PBKDF2 work (iterations times the key's 20-byte blocks) for a whole package: a thousand +/// files as LibreOffice writes them (100,000 iterations, a 32-byte key), so many +/// files at the per-file cap can't add up to a hang. +const MAX_WORK: u64 = 200_000_000; + +/// `p`'s key derivation added to `spent`, an error once the package's work passes [`MAX_WORK`]. +fn charge(spent: &mut u64, p: &Params) -> Result<(), CryptError> { + let blocks = p.key_size.unwrap_or(16).clamp(16, 32).div_ceil(20) as u64; + *spent = spent.saturating_add(u64::from(p.iterations.min(MAX_ITERATIONS)).saturating_mul(blocks)); + if *spent > MAX_WORK { + return Err(CryptError::Unsupported("the package's key derivations would take too long".into())); + } + Ok(()) +} + +/// Each encrypted file's parameters and declared size, by path, from a manifest. +fn read_manifest(xml: &[u8]) -> Vec<(String, Params, Option)> { + Manifest::parse(xml) + .entries + .into_iter() + .filter_map(|e| Some((e.path, e.encryption?, e.size.and_then(|s| usize::try_from(s).ok())))) + .take(MAX_FILES) + .collect() +} + +/// The package, its declared entry count checked before it is indexed (with +/// [`zip_guard`]'s wording). +fn open_zip(bytes: &[u8]) -> Result, CryptError> { + zip_guard::check(bytes, MAX_ZIP_ENTRIES).map_err(CryptError::Unsupported)?; + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: MAX_ZIP_ENTRIES, max_total_bytes: u64::MAX }) + .map_err(|e| CryptError::Unsupported(format!("not a valid package ({e})"))) +} + +/// `name` read from `z`, at most [`MAX_ENTRY`] bytes of it. `None` when it is missing, damaged +/// or larger. +fn read_entry(z: &mut craft_zip::Archive<'_>, name: &str) -> Option> { + z.read(name, MAX_ENTRY).ok().flatten() +} + +/// The files of a package in its order, folders left out, each at most [`MAX_ENTRY`] bytes. +fn files_of(z: &craft_zip::Archive<'_>) -> Vec { + z.names().filter(|n| !n.ends_with('/')).map(|n| n.into_owned()).collect() +} + +/// Does this package hold encrypted files? +pub fn is_encrypted(bytes: &[u8]) -> bool { + // A package with too many entries isn't parsed here: opening it refuses it (`zip_guard`). + let Ok(mut z) = open_zip(bytes) else { return false }; + read_entry(&mut z, MANIFEST_PATH).is_some_and(|m| !read_manifest(&m).is_empty()) +} + +/// The start key: the password's UTF-8 digest (§4.16.6), `size` bytes of it. +fn start_key(name: &str, size: Option, password: &str) -> Result>, CryptError> { + let mut k = match name { + SHA256 | "SHA256" => sha2::Sha256::digest(password.as_bytes()).to_vec(), + "SHA1" | "http://www.w3.org/2000/09/xmldsig#sha1" => sha1::Sha1::digest(password.as_bytes()).to_vec(), + other => return Err(CryptError::Unsupported(format!("a start key made with {other} isn't supported"))), + }; + if let Some(s) = size.filter(|s| *s > 0 && *s < k.len()) { + k.truncate(s); + } + Ok(Zeroizing::new(k)) +} + +/// PBKDF2 with HMAC-SHA-1 (RFC 2898 §5.2): `len` bytes. +fn pbkdf2(password: &[u8], salt: &[u8], iterations: u32, len: usize) -> Result>, CryptError> { + let mac = hmac::Hmac::::new_from_slice(password).map_err(|_| CryptError::Unsupported("bad key".into()))?; + let mut out = Zeroizing::new(Vec::with_capacity(len + 20)); + let mut block = 1u32; + while out.len() < len { + let mut m = mac.clone(); + m.update(salt); + m.update(&block.to_be_bytes()); + let mut u = m.finalize().into_bytes(); + let mut t = u; + for _ in 1..iterations { + let mut m = mac.clone(); + m.update(&u); + u = m.finalize().into_bytes(); + t.iter_mut().zip(u.iter()).for_each(|(a, b)| *a ^= b); + } + out.extend_from_slice(&t); + block = block.saturating_add(1); + } + out.truncate(len); + Ok(out) +} + +/// The key for a file with parameters `p`. +fn file_key(p: &Params, password: &str) -> Result>, CryptError> { + let start = start_key(&p.start_key, p.start_key_size, password)?; + if !(p.derivation == "PBKDF2" || p.derivation.ends_with("#pbkdf2")) { + return Err(CryptError::Unsupported(format!("key derivation {:?} isn't supported", p.derivation))); + } + if p.iterations == 0 || p.iterations > MAX_ITERATIONS { + return Err(CryptError::Unsupported("the key derivation's iteration count is out of range".into())); + } + pbkdf2(&start, &p.salt, p.iterations, p.key_size.unwrap_or(16).clamp(16, 32)) +} + +/// AES-256-CBC, `data` whole blocks. +fn cbc(key: &[u8], iv: &[u8], data: &[u8], encrypt: bool) -> Result, CryptError> { + let aes = aes::Aes256::new_from_slice(key).map_err(|_| CryptError::Unsupported("the key isn't 256 bits".into()))?; + let mut prev: [u8; 16] = iv.try_into().map_err(|_| CryptError::Unsupported("the initialisation vector isn't 16 bytes".into()))?; + let (blocks, rest) = data.as_chunks::<16>(); + if !rest.is_empty() { + return Err(CryptError::Unsupported("encrypted data isn't a whole number of blocks".into())); + } + let mut out = Vec::with_capacity(data.len()); + for b in blocks { + if encrypt { + let mut x = aes::Block::from(*b); + x.iter_mut().zip(prev.iter()).for_each(|(a, p)| *a ^= p); + aes.encrypt_block(&mut x); + prev = x.into(); + out.extend_from_slice(&prev); + } else { + let mut x = aes::Block::from(*b); + aes.decrypt_block(&mut x); + x.iter_mut().zip(prev.iter()).for_each(|(a, p)| *a ^= p); + out.extend_from_slice(&x); + prev = *b; + } + } + Ok(out) +} + +/// The checksum of a deflated file (§4.16.3): of its first 1024 bytes. +fn checksum(kind: &str, deflated: &[u8]) -> Result, CryptError> { + let head = deflated.get(..deflated.len().min(1024)).unwrap_or_default(); + match kind { + SHA256_1K => Ok(sha2::Sha256::digest(head).to_vec()), + "SHA1/1K" | "urn:oasis:names:tc:opendocument:xmlns:manifest:1.0#sha1-1k" => Ok(sha1::Sha1::digest(head).to_vec()), + other => Err(CryptError::Unsupported(format!("checksum {other} isn't supported"))), + } +} + +/// One encrypted file decrypted and inflated. +fn decrypt_file(data: &[u8], p: &Params, size: Option, password: &str) -> Result, CryptError> { + if p.algorithm != AES256_CBC { + let what = if p.algorithm.to_ascii_lowercase().contains("blowfish") { "Blowfish".to_string() } else { format!("{:?}", p.algorithm) }; + return Err(CryptError::Unsupported(format!("this file is encrypted with {what}, which isn't supported"))); + } + let key = file_key(p, password)?; + let plain = Zeroizing::new(cbc(&key, &p.iv, data, false)?); + // W3C XML Encryption padding: the last byte says how many bytes were added. + let pad = usize::from(plain.last().copied().unwrap_or(0)); + let deflated = (1..=16).contains(&pad).then(|| plain.get(..plain.len().saturating_sub(pad))).flatten().ok_or(CryptError::WrongPassword)?; + if !p.checksum.is_empty() && checksum(&p.checksum_type, deflated)? != p.checksum { + return Err(CryptError::WrongPassword); + } + let limit = size.unwrap_or(MAX_ENTRY as usize).min(MAX_ENTRY as usize); + inflate(deflated, limit).ok_or(CryptError::WrongPassword) +} + +/// Raw deflate data inflated, `None` when it is damaged or longer than `limit`. +fn inflate(deflated: &[u8], limit: usize) -> Option> { + let mut out = Vec::new(); + flate2::read::DeflateDecoder::new(deflated).take(limit as u64 + 1).read_to_end(&mut out).ok()?; + (out.len() <= limit).then_some(out) +} + +fn deflate(data: &[u8]) -> Result, std::io::Error> { + let mut e = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::new(6)); + e.write_all(data)?; + e.finish() +} + +/// The manifest without its encryption data (each entry keeps its `manifest:size`). +fn plain_manifest(xml: &str) -> String { + let mut out = String::with_capacity(xml.len()); + let mut rest = xml; + while let Some(i) = rest.find("").map(|e| e + "".len()).or_else(|| after.find("/>").map(|e| e + 2)); + rest = end.and_then(|e| after.get(e..)).unwrap_or(""); + } + out.push_str(rest); + out +} + +/// A password-protected package decrypted into a plain one. +pub fn decrypt(bytes: &[u8], password: &str) -> Result, CryptError> { + let broken = |e: craft_zip::Error| CryptError::Unsupported(format!("not a valid package ({e})")); + let mut z = open_zip(bytes)?; + let manifest = read_entry(&mut z, MANIFEST_PATH).unwrap_or_default(); + let encrypted = read_manifest(&manifest); + let mut w = craft_zip::Writer::new(); + let mut checked = false; + let mut work = 0u64; + for name in files_of(&z).into_iter().take(MAX_FILES) { + // A file past the limit is refused, not cut short. + let Some(data) = z.read(&name, MAX_ENTRY).map_err(broken)? else { continue }; + let data = match encrypted.iter().find(|(p, ..)| *p == name) { + Some((_, p, size)) => { + charge(&mut work, p)?; + let plain = decrypt_file(&data, p, *size, password)?; + checked = true; + plain + } + None if name == MANIFEST_PATH => plain_manifest(&String::from_utf8_lossy(&data)).into_bytes(), + None => data, + }; + let compression = if name == "mimetype" { craft_zip::Compression::Stored } else { craft_zip::Compression::Deflated }; + w.add(&name, &data, compression).map_err(broken)?; + } + if !checked { + return Err(CryptError::Unsupported("no encrypted file could be read".into())); + } + w.finish().map_err(broken) +} + +/// An attribute value's text (the manifest's own entity escapes). +fn unescape(s: &str) -> String { + s.replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">").replace("&", "&") +} + +fn random(n: usize) -> Result, CryptError> { + let mut v = vec![0u8; n]; + getrandom::fill(&mut v).map_err(|e| CryptError::Unsupported(format!("no random numbers for encryption ({e})")))?; + Ok(v) +} + +/// A package encrypted with `password`: every file but the `mimetype` and the manifest, the +/// manifest saying how. +pub fn encrypt(bytes: &[u8], password: &str) -> Result, CryptError> { + let err = |e: craft_zip::Error| CryptError::Unsupported(e.to_string()); + let io = |e: std::io::Error| CryptError::Unsupported(e.to_string()); + let mut z = open_zip(bytes)?; + let start = start_key(SHA256, Some(32), password)?; + let mut files: Vec<(String, Vec, bool)> = Vec::new(); + let mut manifest = String::new(); + // name → its encryption data and size, for the manifest. + let mut data_of: Vec<(String, String)> = Vec::new(); + for name in files_of(&z) { + // At most `MAX_ENTRY` a file: it was read without a limit. + let Some(data) = z.read(&name, MAX_ENTRY).map_err(err)? else { continue }; + if name == MANIFEST_PATH { + manifest = String::from_utf8_lossy(&data).into_owned(); + continue; + } + if name == "mimetype" { + files.push((name, data, false)); + continue; + } + let size = data.len(); + let mut deflated = deflate(&data).map_err(io)?; + let sum = checksum(SHA256_1K, &deflated)?; + // W3C XML Encryption padding: random bytes, the last one their count. + let pad = 16 - deflated.len() % 16; + let mut tail = random(pad)?; + if let Some(last) = tail.last_mut() { + *last = pad as u8; + } + deflated.extend_from_slice(&tail); + let (salt, iv) = (random(16)?, random(16)?); + let key = pbkdf2(&start, &salt, ITERATIONS, 32)?; + let enc = cbc(&key, &iv, &deflated, true)?; + data_of.push(( + name.clone(), + format!( + " manifest:size=\"{size}\">", + STANDARD.encode(&sum), + STANDARD.encode(&iv), + STANDARD.encode(&salt) + ), + )); + files.push((name, enc, true)); + } + // Each encrypted file's entry gets its size and encryption data; a file the manifest + // doesn't list gets an entry of its own. + let mut listed = std::collections::HashSet::new(); + let mut out = String::with_capacity(manifest.len() + data_of.len() * 600); + let mut rest = manifest.as_str(); + while let Some(i) = rest.find("") else { break }; + let tag = after.get(..end).unwrap_or(""); + let path = tag.split("manifest:full-path=\"").nth(1).and_then(|r| r.split('"').next()).map(unescape).unwrap_or_default(); + match data_of.iter().find(|(n, _)| *n == path) { + Some((n, enc)) => { + listed.insert(n.clone()); + out.push_str(tag); + out.push_str(enc); + } + None => out.push_str(after.get(..end + 2).unwrap_or("")), + } + rest = after.get(end + 2..).unwrap_or(""); + } + let unlisted: String = data_of + .iter() + .filter(|(n, _)| !listed.contains(n)) + .map(|(n, enc)| format!("") { + Some(e) => { + out.push_str(rest.get(..e).unwrap_or("")); + out.push_str(&unlisted); + out.push_str(rest.get(e..).unwrap_or("")); + } + None => out.push_str(rest), + } + use craft_zip::Compression::{Deflated, Stored}; + let mut w = craft_zip::Writer::new(); + for (name, data, _) in files.iter().filter(|(n, _, _)| n == "mimetype") { + w.add(name, data, Stored).map_err(err)?; + } + w.add(MANIFEST_PATH, out.as_bytes(), Deflated).map_err(err)?; + // Encrypted files are stored: they're deflated already (§3.4.1). + for (name, data, _) in files.iter().filter(|(n, _, _)| n != "mimetype") { + w.add(name, data, Stored).map_err(err)?; + } + w.finish().map_err(err) +} + +#[cfg(test)] +mod work_tests { + use super::*; + + /// Key derivations are budgeted over the whole package, not only per file: a thousand of + /// LibreOffice's fit, twenty-one at the per-file cap don't. + #[test] + fn a_packages_key_derivations_share_one_budget() { + let ours = Params { iterations: ITERATIONS, key_size: Some(32), ..Default::default() }; + let mut spent = 0; + for _ in 0..1000 { + assert!(charge(&mut spent, &ours).is_ok()); + } + assert!(charge(&mut spent, &ours).is_err()); + let hostile = Params { iterations: MAX_ITERATIONS, key_size: Some(16), ..Default::default() }; + let mut spent = 0; + let ok = (0..30).take_while(|_| charge(&mut spent, &hostile).is_ok()).count(); + assert_eq!(ok, 20); + } +} diff --git a/crates/odf/src/flat.rs b/crates/odf/src/flat.rs new file mode 100644 index 000000000..d71fbffe9 --- /dev/null +++ b/crates/odf/src/flat.rs @@ -0,0 +1,446 @@ +//! Flat OpenDocument files (`.fodt`, `.fods`, `.fodp`, `.fodg`; ODF 1.3 Part 3 §2.1.2 and +//! §3.1.2): the whole document in one `office:document` element, its pictures inline +//! (`office:binary-data`, base64) and its embedded objects inline (`math:math` or another +//! `office:document` inside `draw:object`). [`to_package`] takes one apart into a package's +//! files; [`from_package`] puts a package's files together. Neither looks at what the document +//! says. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; + +use crate::{Error, Limits, Package, PackageWriter}; + +/// Deepest nesting of one element name followed (deeper is malformed). +const MAX_XML_DEPTH: usize = 512; + +/// The top-level parts of a flat document, in the order a package's files take them. +const META: &str = "office:meta"; +const SETTINGS: &str = "office:settings"; +const FONTS: &str = "office:font-face-decls"; +const STYLES: &str = "office:styles"; +const AUTOMATIC: &str = "office:automatic-styles"; +const MASTERS: &str = "office:master-styles"; +const BODY: &str = "office:body"; + +/// Most pictures and formulas taken out of a flat document (hostile files). +const MAX_INLINE: usize = 10_000; + +/// Element `name` in `xml` (its first one, start to end, or self-closed); "" when there is none. +fn element<'a>(xml: &'a str, name: &str) -> &'a str { + let open = format!("<{name}"); + let Some(start) = xml + .match_indices(&open) + .map(|(i, _)| i) + .find(|&i| xml.get(i + open.len()..).and_then(|r| r.chars().next()).is_some_and(|c| c == '>' || c == '/' || c.is_whitespace())) + else { + return ""; + }; + let Some(tag_end) = xml.get(start..).and_then(|r| r.find('>')).map(|e| start + e) else { return "" }; + if xml.get(..tag_end).is_some_and(|t| t.ends_with('/')) { + return xml.get(start..=tag_end).unwrap_or(""); + } + // Its end: the close tag at its own depth (a chart object inside holds an `office:body`, + // an `office:document` of its own). + let close = format!(""); + let mut depth = 0usize; + let mut at = tag_end; + // Every step moves past a tag, so this ends; nesting is bounded like any XML here (deeper is + // malformed), however many elements of the name sit side by side within. + loop { + let rest = xml.get(at..).unwrap_or(""); + let next_close = rest.find(&close); + // Only opens before that close count (and only they are looked for). + let before = next_close.and_then(|c| rest.get(..c)).unwrap_or(rest); + let next_open = before.match_indices(&open).map(|(i, _)| i).find(|&i| { + let after = rest.get(i + open.len()..).unwrap_or(""); + after.chars().next().is_some_and(|c| c == '>' || c == '/' || c.is_whitespace()) + // Not self-closed. + && after.find('>').and_then(|e| after.get(..e)).is_some_and(|t| !t.ends_with('/')) + }); + match (next_open, next_close) { + (Some(o), Some(c)) if o < c => { + depth += 1; + if depth > MAX_XML_DEPTH { + return xml.get(start..).unwrap_or(""); + } + at += o + open.len(); + } + (_, Some(c)) if depth > 0 => { + depth -= 1; + at += c + close.len(); + } + (_, Some(c)) => return xml.get(start..at + c + close.len()).unwrap_or(""), + // Unclosed (malformed): to the end. + (_, None) => return xml.get(start..).unwrap_or(""), + } + } +} + +/// What is inside element `el` (an element string as [`element`] gives). +fn inner(el: &str) -> &str { + let Some(tag_end) = el.find('>') else { return "" }; + if el.get(..tag_end).is_some_and(|t| t.ends_with('/')) { + return ""; + } + let end = el.rfind(" &'static str { + if data.starts_with(b"\x89PNG") { + "png" + } else if data.starts_with(b"\xff\xd8") { + "jpg" + } else if data.starts_with(b"GIF8") { + "gif" + } else if data.starts_with(b" Option { + let mut from = 0; + while let Some(i) = xml.get(from..).and_then(|r| r.find("').and_then(|e| r.get(..e))).unwrap_or(""); + let next = tag.get(")>, folders: &mut Vec<(String, String)>, max_entry: u64) -> String { + let mut out = String::with_capacity(xml.len()); + let mut rest = xml; + loop { + let pic = rest.find(""); + let obj = inline_object(rest); + let (at, picture) = match (pic, obj) { + (Some(p), Some(o)) if o < p => (o, false), + (Some(p), _) => (p, true), + (None, Some(o)) => (o, false), + (None, None) => break, + }; + let (before, from) = rest.split_at(at); + let close = if picture { "" } else { "" }; + let Some(end) = from.find(close) else { + out.push_str(before); + rest = from; + break; + }; + let open_len = if picture { "".len() } else { from.find('>').map_or(0, |e| e + 1) }; + let body = from.get(open_len..end).unwrap_or(""); + let after = from.get(end + close.len()..).unwrap_or(""); + let n = files.len() + 1; + if files.len() >= MAX_INLINE { + out.push_str(before); + rest = after; + continue; + } + if picture { + // The picture's `draw:image` start tag (just before) gets the package's reference. + let b64: String = body.chars().filter(|c| !c.is_whitespace()).collect(); + let data = STANDARD.decode(b64.as_bytes()).ok().filter(|d| d.len() as u64 <= max_entry).unwrap_or_default(); + // The element holding the data: the start tag just before it. + let holder = before.rfind('<').filter(|_| !data.is_empty()).and_then(|i| { + let tag = before.get(i..).unwrap_or(""); + ["', '\n', '\t', '\r']))) + .map(|el| (i, el)) + }); + match holder { + Some((at, el)) => { + let name = if el == " out.push_str(before), + } + } else { + let dir = format!("Object flat{n}"); + out.push_str(before); + out.push_str(&format!("")); + // A sub-document (a chart, a spreadsheet) goes back into its files; a formula is + // its `content.xml`. + match package_parts(body.trim()) { + Some(parts) => { + if let Some(mime) = mimetype_of(body.trim()) { + folders.push((format!("{dir}/"), mime)); + } + for (name, xml) in ["content.xml", "styles.xml", "meta.xml", "settings.xml"].into_iter().zip(parts) { + files.push((format!("{dir}/{name}"), xml.into_bytes())); + } + } + None => files.push((format!("{dir}/content.xml"), body.as_bytes().to_vec())), + } + } + rest = after; + } + out.push_str(rest); + out +} + +/// A flat document taken apart. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FlatDocument { + /// The root's `office:mimetype`. + pub mimetype: Option, + /// `content.xml`: fonts, automatic styles and the body. + pub content: String, + /// `styles.xml`: fonts, styles, automatic styles and master styles. + pub styles: String, + /// `meta.xml`. + pub meta: String, + /// `settings.xml`. + pub settings: String, + /// The pictures and embedded objects that were inline, named as a package names them + /// (`Pictures/flat1.png`, `Object flat2/content.xml`); `content` and `styles` point at them. + pub files: Vec<(String, Vec)>, + /// The folders of embedded sub-documents and their media types (`Object flat2/`). + pub folders: Vec<(String, String)>, +} + +/// Takes a flat document apart. At most 10,000 pictures and objects are taken out (the rest +/// are dropped), and a picture larger than [`Limits::max_entry`] is left out. +pub fn split(bytes: &[u8], limits: &Limits) -> Result { + let xml = std::str::from_utf8(bytes).map_err(|_| Error::Flat("not UTF-8 text"))?; + let root = element(xml, "office:document"); + let [content, styles, meta, settings] = package_parts(root).ok_or(Error::Flat("no office:document"))?; + let (mut files, mut folders) = (Vec::new(), Vec::new()); + let content = take_out(&content, &mut files, &mut folders, limits.max_entry); + let styles = take_out(&styles, &mut files, &mut folders, limits.max_entry); + Ok(FlatDocument { mimetype: mimetype_of(root), content, styles, meta, settings, files, folders }) +} + +/// A flat document as a package, of the media type its root declares, else `default_mime`. +/// The files are stored, not deflated: the package is for a reader to open, not to keep. +pub fn to_package(bytes: &[u8], default_mime: &str, limits: &Limits) -> Result, Error> { + let doc = split(bytes, limits)?; + let mut w = PackageWriter::new(doc.mimetype.as_deref().unwrap_or(default_mime)); + for (name, xml) in [("content.xml", &doc.content), ("styles.xml", &doc.styles), ("meta.xml", &doc.meta), ("settings.xml", &doc.settings)] { + w.stored(name, "text/xml", xml.as_bytes()); + } + for (path, mime) in &doc.folders { + w.folder(path, mime); + } + for (name, data) in &doc.files { + w.stored(name, if name.ends_with(".xml") { "text/xml" } else { "" }, data); + } + w.finish() +} + +/// A package as a flat document: its files in one `office:document`, the automatic styles of +/// its content and its styles together, pictures and embedded objects inline. +pub fn from_package(package: &[u8], limits: &Limits) -> Result, Error> { + let mut z = Package::open(package, limits)?; + let manifest = z.manifest(); + let mime = z + .mimetype() + .filter(|m| !m.is_empty()) + .or_else(|| manifest.media_type("/").map(str::to_string)) + .ok_or(Error::Flat("the package has no media type"))?; + let mut bytes_of = |name: &str| -> Vec { z.read(name).ok().flatten().unwrap_or_default() }; + let mut read = |name: &str| -> String { String::from_utf8_lossy(&bytes_of(name)).into_owned() }; + let (content, styles, meta, settings) = (read("content.xml"), read("styles.xml"), read("meta.xml"), read("settings.xml")); + let body = flat_parts(&content, &styles, &meta, &settings); + // Pictures and formulas in the document, not beside it. + let mut flat = String::with_capacity(body.len()); + let mut rest = body.as_str(); + while let Some(at) = rest.find("xlink:href=\"") { + let (before, from) = rest.split_at(at); + let target_start = "xlink:href=\"".len(); + let Some(len) = from.get(target_start..).and_then(|r| r.find('"')) else { break }; + let target = from.get(target_start..target_start + len).unwrap_or(""); + let after = from.get(target_start + len + 1..).unwrap_or(""); + let tag = before.rfind('<').and_then(|i| before.get(i..)).unwrap_or(""); + // A picture, or an embedded object's replacement picture. + let picture = target.starts_with("Pictures/") || target.trim_start_matches("./").starts_with("ObjectReplacements/"); + // The rest of the linking element's start tag, whether it closes itself, and what follows + // the tag (markup kept as read may have ``). + let gt = after.find('>').unwrap_or(after.len()); + let closed = after.get(..gt).is_some_and(|t| t.ends_with('/')); + let tag_rest = after.get(..gt).unwrap_or("").trim_end_matches('/'); + let past_tag = after.get(gt + 1..).unwrap_or(""); + // What follows the element: past its tag, or past its end tag. + let past = |el: &str| { + if closed { past_tag } else { past_tag.find(&format!("")).and_then(|e| past_tag.get(e + el.len() + 3..)).unwrap_or("") } + }; + // The link's other attributes go with it. + let attrs: String = tag_rest.split_whitespace().filter(|a| !a.starts_with("xlink:")).map(|a| format!(" {a}")).collect(); + if tag.starts_with("` → `…`, before + // anything else it holds. + let data = bytes_of(target.trim_start_matches("./")); + flat.push_str(before.trim_end()); + flat.push_str(&attrs); + flat.push_str(&format!(">{}", STANDARD.encode(&data))); + if closed { + flat.push_str(""); + } + rest = past_tag; + } else if tag.starts_with("{}", STANDARD.encode(&data))); + rest = past("draw:object-ole"); + } else if tag.starts_with("").map_or(content.as_str(), |(_, m)| m).trim().to_string() + }; + // A plain `draw:object` around it. + flat.push_str(before.rfind("{math}")); + rest = past("draw:object"); + } else { + flat.push_str(before); + flat.push_str(from.get(..target_start + len + 1).unwrap_or("")); + rest = after; + } + } + flat.push_str(rest); + Ok(format!( + "\n{flat}\n", + merged_namespaces(&[&content, &styles, &meta, &settings]), + craft_xml::esc_attr(&mime) + ) + .into_bytes()) +} + +/// Every namespace the parts declare (the settings' `config`, say), each prefix once. +fn merged_namespaces(parts: &[&str]) -> String { + let mut ns: Vec<&str> = Vec::new(); + for part in parts { + for decl in namespaces(part) { + let prefix = decl.split('=').next().unwrap_or(""); + if !ns.iter().any(|d| d.split('=').next() == Some(prefix)) { + ns.push(decl); + } + } + } + ns.join(" ") +} + +/// What a flat document holds, from a package's parts in the order §3.1.2 gives them: one +/// font declaration list and one automatic styles list for its content and its styles together. +fn flat_parts(content: &str, styles: &str, meta: &str, settings: &str) -> String { + let both = |name: &str| { + let (a, b) = (element(styles, name), element(content, name)); + if a.is_empty() && b.is_empty() { String::new() } else { format!("<{name}>{}{}", inner(a), inner(b)) } + }; + [ + element(meta, META).to_string(), + element(settings, SETTINGS).to_string(), + both(FONTS), + element(styles, STYLES).to_string(), + both(AUTOMATIC), + element(styles, MASTERS).to_string(), + element(content, BODY).to_string(), + ] + .concat() +} + +/// A sub-document's parts as one `office:document` of media type `mime`. +fn flat_document(mime: &str, content: &str, styles: &str, meta: &str, settings: &str) -> String { + format!( + "{}", + merged_namespaces(&[content, styles, meta, settings]), + craft_xml::esc_attr(mime), + flat_parts(content, styles, meta, settings) + ) +} + +/// An `office:document` element taken apart into a package's `content.xml`, `styles.xml`, +/// `meta.xml` and `settings.xml` (each part's root with the document's attributes but its +/// media type). `None` when `root` is no `office:document`. +fn package_parts(root: &str) -> Option<[String; 4]> { + if !root.starts_with("').and_then(|e| root.get(" { + let from = i + " office:mimetype=\"".len(); + let end = attrs.get(from..).and_then(|r| r.find('"')).map_or(attrs.len(), |e| from + e + 1); + format!("{}{}", attrs.get(..i).unwrap_or(""), attrs.get(end..).unwrap_or("")) + } + None => attrs.to_string(), + }; + let doc = |name: &str, parts: &[&str]| -> String { + let body: String = parts.iter().map(|p| element(root, p)).collect(); + format!("\n{body}") + }; + Some([ + doc("document-content", &[FONTS, AUTOMATIC, BODY]), + doc("document-styles", &[FONTS, STYLES, AUTOMATIC, MASTERS]), + doc("document-meta", &[META]), + doc("document-settings", &[SETTINGS]), + ]) +} + +/// The `office:mimetype` of an `office:document` element. +fn mimetype_of(root: &str) -> Option { + let tag = root.find('>').and_then(|e| root.get(..e))?; + let from = tag.find(" office:mimetype=\"")? + " office:mimetype=\"".len(); + let value = tag.get(from..)?; + Some(craft_xml::unescape(value.get(..value.find('"')?)?)) +} + +/// The namespace declarations (`xmlns:p="…"`) on the root element of `xml`. +fn namespaces(xml: &str) -> Vec<&str> { + let start = xml.find("').and_then(|e| r.get(..e))).unwrap_or(""); + let mut out = Vec::new(); + let mut rest = tag; + while let Some(i) = rest.find("xmlns:") { + let from = rest.get(i..).unwrap_or(""); + // `xmlns:p="…"`: up to the closing quote. + let Some(q1) = from.find('"') else { break }; + let Some(q2) = from.get(q1 + 1..).and_then(|r| r.find('"')) else { break }; + out.push(from.get(..q1 + 1 + q2 + 1).unwrap_or("")); + rest = from.get(q1 + 1 + q2 + 1..).unwrap_or(""); + } + out +} + +#[cfg(test)] +mod element_tests { + use super::element; + + /// An element ends at its own close tag however many elements of its name it holds side by + /// side (inline objects' own `office:body`s), not after a count of tags. + #[test] + fn an_element_holding_many_of_its_name_ends_at_its_own_close() { + let inner = "x".repeat(1200); + let outer = format!("{inner}"); + let xml = format!("{outer}"); + assert_eq!(element(&xml, "office:body"), outer); + // Nested deeper than any XML here: malformed, to the end, as an unclosed one. + let deep = format!("{}{}", "".repeat(600), "".repeat(600)); + assert_eq!(element(&deep, "office:body"), deep); + assert_eq!(element("", "office:body"), ""); + } +} diff --git a/crates/odf/src/lib.rs b/crates/odf/src/lib.rs new file mode 100644 index 000000000..9730d1e94 --- /dev/null +++ b/crates/odf/src/lib.rs @@ -0,0 +1,95 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! OpenDocument packages (ODF 1.3 Part 2): what every OpenDocument file has in common, whatever +//! it holds (text, a spreadsheet, a presentation, a drawing, a chart, a formula). +//! +//! - [`Package`] opens the ZIP container within [`Limits`] and reads its `mimetype`, its +//! [`Manifest`] and its files. [`PackageWriter`] writes one: the `mimetype` first and stored, +//! the manifest, then the files. +//! - [`crypt`] opens and saves password-protected packages (AES-256-CBC with PBKDF2, as +//! LibreOffice writes them). +//! - [`flat`] turns a flat single-file document (`.fodt`, `.fods`, `.fodp`, `.fodg`) into a +//! package and back. +//! - [`zip_guard`] refuses a ZIP that declares too many entries before `zip` parses them. +//! +//! What the XML inside means (paragraphs, cells, styles) is not here. +//! +//! Input is hostile: entries, sizes, key-derivation work and nesting are bounded, and nothing +//! here panics on any bytes. +//! +//! ``` +//! use craft_odf::{Limits, Package, PackageWriter, mime}; +//! +//! let mut w = PackageWriter::new(mime::TEXT); +//! w.file("content.xml", "text/xml", b""); +//! let bytes = w.finish()?; +//! +//! let mut p = Package::open(&bytes, &Limits::DEFAULT)?; +//! assert_eq!(p.mimetype().as_deref(), Some(mime::TEXT)); +//! assert_eq!(p.read("content.xml")?.as_deref(), Some(b"".as_slice())); +//! assert_eq!(p.manifest().media_type("content.xml"), Some("text/xml")); +//! # Ok::<(), craft_odf::Error>(()) +//! ``` + +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeExamples; + +pub mod crypt; +pub mod flat; +mod manifest; +mod package; +#[cfg(feature = "text")] +pub mod text; +pub mod zip_guard; + +pub use manifest::{Encryption, FileEntry, MANIFEST_NS, MANIFEST_PATH, Manifest}; +pub use package::{Limits, Package, PackageWriter}; + +/// The media types of OpenDocument files (ODF 1.3 Part 3 section 3.3, and the `mimetype` file of +/// Part 2 section 3.3). +pub mod mime { + pub const TEXT: &str = "application/vnd.oasis.opendocument.text"; + pub const TEXT_TEMPLATE: &str = "application/vnd.oasis.opendocument.text-template"; + pub const SPREADSHEET: &str = "application/vnd.oasis.opendocument.spreadsheet"; + pub const SPREADSHEET_TEMPLATE: &str = "application/vnd.oasis.opendocument.spreadsheet-template"; + pub const PRESENTATION: &str = "application/vnd.oasis.opendocument.presentation"; + pub const PRESENTATION_TEMPLATE: &str = "application/vnd.oasis.opendocument.presentation-template"; + pub const GRAPHICS: &str = "application/vnd.oasis.opendocument.graphics"; + pub const GRAPHICS_TEMPLATE: &str = "application/vnd.oasis.opendocument.graphics-template"; + pub const CHART: &str = "application/vnd.oasis.opendocument.chart"; + pub const FORMULA: &str = "application/vnd.oasis.opendocument.formula"; +} + +/// Why a package was not read or written. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The ZIP declares more entries than [`Limits::max_entries`], or has too many end records. + #[error("{0}")] + TooManyEntries(String), + /// A file is larger than [`Limits::max_entry`]. + #[error("the file {0} in the package is larger than {1} bytes")] + EntryTooLarge(String, u64), + /// The files read add up to more than [`Limits::max_total`]. + #[error("the package's files are larger than {0} bytes together")] + TotalTooLarge(u64), + /// The bytes are no ZIP package, or one that cannot be read or written. + #[error("not a valid package ({0})")] + Zip(String), + /// A flat document that is no `office:document`, or not UTF-8. + #[error("not a flat OpenDocument file ({0})")] + Flat(&'static str), + /// A manifest that [`Manifest::parse_strict`] refuses: malformed XML, a document type + /// declaration, a root that is no `manifest:manifest`, or more entries than it reads. + #[error("the package's manifest is not valid ({0})")] + Manifest(String), +} + +impl Error { + /// Was a limit exceeded (as opposed to the package being wrong)? + pub fn is_limit(&self) -> bool { + matches!(self, Error::TooManyEntries(_) | Error::EntryTooLarge(..) | Error::TotalTooLarge(_)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/odf/src/manifest.rs b/crates/odf/src/manifest.rs new file mode 100644 index 000000000..89d4f0631 --- /dev/null +++ b/crates/odf/src/manifest.rs @@ -0,0 +1,218 @@ +//! The manifest (`META-INF/manifest.xml`, ODF 1.3 Part 2 section 4): each file's path and +//! media type, and how an encrypted file is encrypted. + +use std::fmt::Write as _; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use craft_xml::{Canonical, Doctype, El, Options, esc_attr}; + +use crate::Error; + +/// The manifest's path in a package. +pub const MANIFEST_PATH: &str = "META-INF/manifest.xml"; +/// The manifest's XML namespace. +pub const MANIFEST_NS: &str = "urn:oasis:names:tc:opendocument:xmlns:manifest:1.0"; + +/// Most file entries read from a manifest (as many as a package may hold by default). +pub const MAX_ENTRIES: usize = 20_000; + +/// How one file is encrypted (a `manifest:encryption-data`, section 4.4 to 4.7). +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct Encryption { + /// `manifest:checksum-type` (`SHA1/1K` when absent). + pub checksum_type: String, + /// `manifest:checksum`, decoded. + pub checksum: Vec, + /// `manifest:algorithm-name`. + pub algorithm: String, + /// `manifest:initialisation-vector`, decoded. + pub iv: Vec, + /// `manifest:start-key-generation-name` (`SHA1` when absent). + pub start_key: String, + /// The start key's `manifest:key-size`. + pub start_key_size: Option, + /// `manifest:key-derivation-name`. + pub derivation: String, + /// `manifest:iteration-count` (1024 when absent). + pub iterations: u32, + /// `manifest:salt`, decoded. + pub salt: Vec, + /// The derived key's `manifest:key-size`. + pub key_size: Option, +} + +/// One `manifest:file-entry`. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct FileEntry { + /// `manifest:full-path`: `/` is the package itself, a path ending in `/` a folder (a + /// sub-document). + pub path: String, + /// `manifest:media-type` (may be empty). + pub media_type: String, + /// `manifest:version`. + pub version: Option, + /// `manifest:size`: the file's size before encryption. + pub size: Option, + /// Present when the file is encrypted. + pub encryption: Option, +} + +/// A package's manifest. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct Manifest { + /// `manifest:version` of the root element. + pub version: Option, + pub entries: Vec, +} + +impl Manifest { + /// Reads a manifest. Elements are found by namespace, whatever prefix the file uses. What + /// can't be read is left out: damaged XML gives the entries before the damage, and entries + /// past [`MAX_ENTRIES`] are dropped. + pub fn parse(xml: &[u8]) -> Manifest { + let canonical = Canonical::new(&[("manifest", MANIFEST_NS)]); + let (root, _) = Options::canonical(&canonical).parse_streaming_partial(xml, &[], |_| Ok::<(), craft_xml::Error>(())); + let Some(root) = root.filter(|r| named(r, "manifest")) else { return Manifest::default() }; + let entries = root.elements().filter(|e| named(e, "file-entry")).take(MAX_ENTRIES).filter_map(file_entry).collect(); + Manifest { version: attr(&root, "version").map(str::to_string), entries } + } + + /// Reads a manifest, refusing one a lenient read would repair: malformed or unbalanced + /// XML, a document type declaration (of any kind), an undefined entity, text or a second + /// element outside the root, a root that is not `manifest` in the manifest namespace, a + /// `file-entry` without a `full-path`, or more than [`MAX_ENTRIES`] entries. For a reader + /// that refuses a damaged package rather than open what it can. + pub fn parse_strict(xml: &[u8]) -> Result { + let bad = |why: String| Error::Manifest(why); + let canonical = Canonical::new(&[("manifest", MANIFEST_NS)]); + let opts = Options { doctype: Doctype::Reject, ..Options::canonical(&canonical).strict() }; + let root = opts.parse(xml).map_err(|e| bad(e.to_string()))?; + if root.name != "manifest:manifest" { + return Err(bad(format!("the root element is {}, not manifest:manifest", root.name))); + } + let mut entries = Vec::new(); + for e in root.elements().filter(|e| e.name == "manifest:file-entry") { + if entries.len() == MAX_ENTRIES { + return Err(bad(format!("more than {MAX_ENTRIES} file entries"))); + } + entries.push(file_entry(e).ok_or_else(|| bad("a file entry has no full-path".into()))?); + } + Ok(Manifest { version: attr(&root, "version").map(str::to_string), entries }) + } + + /// The entry for `path`. + pub fn entry(&self, path: &str) -> Option<&FileEntry> { + self.entries.iter().find(|e| e.path == path) + } + + /// The media type the manifest gives `path` (`/` for the package, `Object 1/` for a + /// sub-document). + pub fn media_type(&self, path: &str) -> Option<&str> { + self.entry(path).map(|e| e.media_type.as_str()) + } + + /// Does any file say it is encrypted? + pub fn is_encrypted(&self) -> bool { + self.entries.iter().any(|e| e.encryption.is_some()) + } + + /// The manifest as XML, as LibreOffice lays it out: one entry a line, encryption data + /// included. + pub fn to_xml(&self) -> String { + let mut o = format!("{}\n"); + for e in &self.entries { + let _ = write!(o, " { + o.push('>'); + encryption_xml(&mut o, c); + o.push_str("\n"); + } + None => o.push_str("/>\n"), + } + } + o.push_str("\n"); + o + } +} + +fn encryption_xml(o: &mut String, c: &Encryption) { + let size = |s: Option| s.map(|s| format!(" manifest:key-size=\"{s}\"")).unwrap_or_default(); + let _ = write!( + o, + "", + esc_attr(&c.checksum_type), + STANDARD.encode(&c.checksum), + esc_attr(&c.algorithm), + STANDARD.encode(&c.iv), + esc_attr(&c.start_key), + size(c.start_key_size), + esc_attr(&c.derivation), + c.iterations, + STANDARD.encode(&c.salt), + size(c.key_size) + ); +} + +/// Is `e` the manifest element `local`: in the manifest namespace, or (a prefix nothing declares, +/// a file with no namespace at all) in none? +fn named(e: &El, local: &str) -> bool { + e.name.strip_prefix("manifest:").unwrap_or(&e.name) == local +} + +fn child<'a>(e: &'a El, local: &str) -> Option<&'a El> { + e.elements().find(|c| named(c, local)) +} + +/// A manifest attribute: in the manifest namespace, or (as some writers leave it) in none. +fn attr<'a>(e: &'a El, name: &str) -> Option<&'a str> { + e.attr(&format!("manifest:{name}")).or_else(|| e.attr(name)) +} + +fn file_entry(e: &El) -> Option { + fn num(e: &El, name: &str) -> Option { + attr(e, name).and_then(|s| s.trim().parse().ok()) + } + let b64 = |e: &El, name: &str| attr(e, name).and_then(|s| STANDARD.decode(s.trim()).ok()).unwrap_or_default(); + let encryption = child(e, "encryption-data").map(|d| { + let mut c = Encryption { + checksum_type: attr(d, "checksum-type").unwrap_or("SHA1/1K").to_string(), + checksum: b64(d, "checksum"), + ..Default::default() + }; + if let Some(a) = child(d, "algorithm") { + c.algorithm = attr(a, "algorithm-name").unwrap_or_default().to_string(); + c.iv = b64(a, "initialisation-vector"); + } + if let Some(k) = child(d, "start-key-generation") { + c.start_key = attr(k, "start-key-generation-name").unwrap_or("SHA1").to_string(); + c.start_key_size = num(k, "key-size"); + } + if let Some(k) = child(d, "key-derivation") { + c.derivation = attr(k, "key-derivation-name").unwrap_or_default().to_string(); + c.iterations = num(k, "iteration-count").unwrap_or(1024); + c.salt = b64(k, "salt"); + c.key_size = num(k, "key-size"); + } + c + }); + Some(FileEntry { + path: attr(e, "full-path")?.to_string(), + media_type: attr(e, "media-type").unwrap_or_default().to_string(), + version: attr(e, "version").map(str::to_string), + size: num(e, "size"), + encryption, + }) +} diff --git a/crates/odf/src/package.rs b/crates/odf/src/package.rs new file mode 100644 index 000000000..4249e7e15 --- /dev/null +++ b/crates/odf/src/package.rs @@ -0,0 +1,230 @@ +//! The ZIP container (ODF 1.3 Part 2 section 3): a `mimetype` file first and stored, a +//! manifest, and the document's files. + +use crate::manifest::{FileEntry, MANIFEST_PATH, Manifest}; +use crate::{Error, zip_guard}; + +/// What a [`Package`] refuses to exceed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Most entries the ZIP may declare, checked before `zip` parses them. Default 20,000. + pub max_entries: u64, + /// Most bytes one file may inflate to. Default 256 MiB. + pub max_entry: u64, + /// Most bytes all the files read from one [`Package`] may inflate to together (a file read + /// twice counts twice). Default 2 GiB. + pub max_total: u64, +} + +impl Limits { + pub const DEFAULT: Limits = Limits { max_entries: 20_000, max_entry: 256 << 20, max_total: 2 << 30 }; +} + +impl Default for Limits { + fn default() -> Limits { + Limits::DEFAULT + } +} + +/// The longest `mimetype` file read: media types are short, and a longer one is no media type. +const MAX_MIMETYPE: u64 = 255; + +pub(crate) fn zip_err(e: craft_zip::Error) -> Error { + match e { + craft_zip::Error::NotZip(m) | craft_zip::Error::Write(m) => Error::Zip(m), + e => Error::Zip(e.to_string()), + } +} + +/// An OpenDocument package open for reading. +pub struct Package<'a> { + zip: craft_zip::Archive<'a>, + /// The files' names (folders and names that are not text left out), in the ZIP's order. + names: Vec, + limits: Limits, + /// Bytes inflated so far, against [`Limits::max_total`]. + spent: u64, +} + +impl<'a> Package<'a> { + /// Opens `bytes` as a ZIP package. The declared entry count is checked against + /// [`Limits::max_entries`] before anything is parsed. No file is inflated yet. + pub fn open(bytes: &'a [u8], limits: &Limits) -> Result, Error> { + zip_guard::check(bytes, limits.max_entries).map_err(Error::TooManyEntries)?; + // This reader keeps its own count (a damaged file is charged what it cost), so the + // archive's total is not used. + let zip = + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries, max_total_bytes: u64::MAX }).map_err( + |e| match e { + craft_zip::Error::TooManyEntries { declared, limit } => { + Error::TooManyEntries(format!("the package has {declared} entries, more than {limit}")) + } + e => zip_err(e), + }, + )?; + // `zip` keeps one file of a repeated name; which one the writer meant is not known. + if zip.has_duplicate_names() { + return Err(Error::Zip("two files of the package share a name".into())); + } + let names = zip.names().filter(|n| !n.ends_with('/')).map(|n| n.into_owned()).collect(); + Ok(Package { zip, names, limits: *limits, spent: 0 }) + } + + /// The names of the package's files, folders left out, in the order the ZIP lists them. + pub fn names(&self) -> impl Iterator { + self.names.iter().map(String::as_str) + } + + /// Is there a file `name`? + pub fn contains(&self, name: &str) -> bool { + self.zip.contains(name) + } + + /// The size the ZIP declares for `name` (what it claims to inflate to; [`Package::read`] + /// does not trust it). + pub fn declared_size(&mut self, name: &str) -> Option { + self.zip.declared_size(name) + } + + /// File `name` inflated. `Ok(None)` when the package has no such file, or it cannot be + /// inflated (damaged data); an error when it is larger than [`Limits::max_entry`] or takes + /// the package past [`Limits::max_total`]. A file that inflates to more than it declares is + /// held to the same limits. + pub fn read(&mut self, name: &str) -> Result>, Error> { + let (max_entry, room) = (self.limits.max_entry, self.limits.max_total.saturating_sub(self.spent)); + let Some(size) = self.zip.declared_size(name) else { return Ok(None) }; + if size > max_entry { + return Err(Error::EntryTooLarge(name.to_string(), max_entry)); + } + if size > room { + return Err(Error::TotalTooLarge(self.limits.max_total)); + } + match self.zip.read(name, max_entry.min(room)) { + Ok(Some(out)) => { + self.spent = self.spent.saturating_add(out.len() as u64); + Ok(Some(out)) + } + Ok(None) => Ok(None), + // Damaged, or inflating past what it declares (the read stops one byte past that): + // unreadable, and charged its declared size. + Err(_) => { + self.spent = self.spent.saturating_add(size); + Ok(None) + } + } + } + + /// Bytes inflated from this package so far. + pub fn bytes_read(&self) -> u64 { + self.spent + } + + /// The package's media type: what its `mimetype` file holds, surrounding whitespace + /// dropped. `None` when it has none, or one longer than a media type can be. + pub fn mimetype(&mut self) -> Option { + let out = self.zip.read("mimetype", MAX_MIMETYPE).ok()??; + self.spent = self.spent.saturating_add(out.len() as u64); + Some(String::from_utf8_lossy(&out).trim().to_string()) + } + + /// The package's manifest; empty when it has none or it cannot be read. + pub fn manifest(&mut self) -> Manifest { + self.read(MANIFEST_PATH).ok().flatten().map(|xml| Manifest::parse(&xml)).unwrap_or_default() + } + + /// The package's manifest read with [`Manifest::parse_strict`]: an error when the package + /// has none, or one a strict read refuses, or it is over a limit. + pub fn manifest_strict(&mut self) -> Result { + let xml = self.read(MANIFEST_PATH)?.ok_or_else(|| Error::Manifest("the package has no manifest".into()))?; + Manifest::parse_strict(&xml) + } + + /// Does the manifest say any file is encrypted ([`crate::crypt`])? + pub fn is_encrypted(&mut self) -> bool { + self.manifest().is_encrypted() + } +} + +struct Entry { + path: String, + data: Vec, + stored: bool, +} + +/// Writes an OpenDocument package: the `mimetype` file first and stored (so the media type can +/// be read at a fixed offset, section 3.3), then the manifest, then the files in the order they +/// were added. +pub struct PackageWriter { + mime: String, + manifest: Manifest, + manifest_xml: Option, + entries: Vec, +} + +impl PackageWriter { + /// A package of media type `mime` ([`crate::mime`]), ODF version 1.3. + pub fn new(mime: &str) -> PackageWriter { + let root = FileEntry { path: "/".into(), media_type: mime.into(), version: Some("1.3".into()), ..Default::default() }; + PackageWriter { + mime: mime.into(), + manifest: Manifest { version: Some("1.3".into()), entries: vec![root] }, + manifest_xml: None, + entries: Vec::new(), + } + } + + fn add(&mut self, path: &str, media_type: &str, data: &[u8], stored: bool) { + self.entries.push(Entry { path: path.into(), data: data.to_vec(), stored }); + if self.manifest.entry(path).is_none() { + self.manifest.entries.push(FileEntry { path: path.into(), media_type: media_type.into(), ..Default::default() }); + } + } + + /// Adds file `path`, deflated, listed in the manifest with `media_type`. + pub fn file(&mut self, path: &str, media_type: &str, data: &[u8]) { + self.add(path, media_type, data, false); + } + + /// Adds file `path` stored as it is: for data that is compressed already (PNG, JPEG). + pub fn stored(&mut self, path: &str, media_type: &str, data: &[u8]) { + self.add(path, media_type, data, true); + } + + /// Lists a folder in the manifest (`Object 1/`, a sub-document with its own media type and + /// version 1.3). It has no ZIP entry of its own. + pub fn folder(&mut self, path: &str, media_type: &str) { + if self.manifest.entry(path).is_none() { + self.manifest.entries.push(FileEntry { + path: path.into(), + media_type: media_type.into(), + version: Some("1.3".into()), + ..Default::default() + }); + } + } + + /// The manifest that will be written, to adjust (an entry's version, entries for files + /// written some other way). + pub fn manifest_mut(&mut self) -> &mut Manifest { + &mut self.manifest + } + + /// Writes `xml` as the manifest instead of the one built from the files added: for a + /// writer that keeps a manifest as it was read. + pub fn manifest_xml(&mut self, xml: String) { + self.manifest_xml = Some(xml); + } + + /// The package's bytes. + pub fn finish(self) -> Result, Error> { + use craft_zip::Compression::{Deflated, Stored}; + let mut z = craft_zip::Writer::new(); + z.add("mimetype", self.mime.as_bytes(), Stored).map_err(zip_err)?; + let manifest = self.manifest_xml.unwrap_or_else(|| self.manifest.to_xml()); + z.add(MANIFEST_PATH, manifest.as_bytes(), Deflated).map_err(zip_err)?; + for e in &self.entries { + z.add(&e.path, &e.data, if e.stored { Stored } else { Deflated }).map_err(zip_err)?; + } + z.finish().map_err(zip_err) + } +} diff --git a/crates/odf/src/tests.rs b/crates/odf/src/tests.rs new file mode 100644 index 000000000..dd144ea61 --- /dev/null +++ b/crates/odf/src/tests.rs @@ -0,0 +1,396 @@ +//! Packages, manifests, limits, password protection and flat documents. + +use std::io::Write; + +use crate::crypt::{CryptError, decrypt, encrypt, is_encrypted}; +use crate::{Error, Limits, Manifest, Package, PackageWriter, flat, mime, zip_guard}; + +/// A ZIP of `files`, stored. +fn zip_of(files: &[(&str, &[u8])]) -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + for (name, data) in files { + z.start_file(*name, stored).unwrap(); + z.write_all(data).unwrap(); + } + z.finish().unwrap().into_inner() +} + +fn secret() -> Vec { + let mut w = PackageWriter::new(mime::TEXT); + w.file("content.xml", "text/xml", b"Top secret words."); + w.file("styles.xml", "text/xml", b""); + w.stored("Pictures/a.png", "image/png", b"\x89PNG not really"); + w.finish().unwrap() +} + +#[test] +fn a_written_package_reads_back() { + let bytes = secret(); + // The media type first and stored, at the offset `file` looks for it (section 3.3). + assert_eq!(bytes.get(30..38), Some(b"mimetype".as_slice())); + assert_eq!(bytes.get(38..38 + mime::TEXT.len()), Some(mime::TEXT.as_bytes())); + let mut p = Package::open(&bytes, &Limits::DEFAULT).unwrap(); + assert_eq!(p.mimetype().as_deref(), Some(mime::TEXT)); + assert_eq!(p.names().collect::>(), ["mimetype", "META-INF/manifest.xml", "content.xml", "styles.xml", "Pictures/a.png"]); + assert!(p.contains("styles.xml") && !p.contains("nothing.xml")); + assert_eq!(p.read("nothing.xml"), Ok(None)); + assert_eq!(p.read("Pictures/a.png").unwrap().as_deref(), Some(b"\x89PNG not really".as_slice())); + let m = p.manifest(); + assert_eq!(m.version.as_deref(), Some("1.3")); + assert_eq!(m.media_type("/"), Some(mime::TEXT)); + assert_eq!(m.media_type("Pictures/a.png"), Some("image/png")); + assert!(!m.is_encrypted() && !p.is_encrypted()); + assert_eq!(Manifest::parse(m.to_xml().as_bytes()), m); +} + +#[test] +fn a_manifest_is_read_by_namespace_and_survives_damage() { + // Another prefix, a default namespace with plain attributes, and a prefix nothing declares. + for xml in [ + r#""#, + r#""#, + r#""#, + ] { + let m = Manifest::parse(xml.as_bytes()); + assert_eq!(m.media_type("/"), Some("a/b"), "{xml}"); + assert_eq!(m.entry("x & y.xml").and_then(|e| e.size), Some(12), "{xml}"); + } + // Cut short: the entries before the cut. + let cut = r#""[..], b"\xff\xfe\x00", b"", b"]>"] { + assert!(Manifest::parse(junk).entries.is_empty()); + } + // More entries than a package may hold are dropped. + let many = format!("{}", r#""#.repeat(crate::manifest::MAX_ENTRIES + 5)); + assert_eq!(Manifest::parse(many.as_bytes()).entries.len(), crate::manifest::MAX_ENTRIES); +} + +#[test] +fn a_strict_manifest_refuses_what_a_lenient_one_repairs() { + const NS: &str = r#"xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0""#; + let good = format!( + r#""# + ); + let m = Manifest::parse_strict(good.as_bytes()).unwrap(); + assert_eq!((m.version.as_deref(), m.media_type("/")), (Some("1.3"), Some("a/b"))); + // Another prefix for the same namespace is the same manifest. + let other = r#""#; + assert_eq!(Manifest::parse_strict(other.as_bytes()), Ok(m.clone())); + for bad in [ + format!(r#""#), + format!(r#""#), + format!(r#"]>"#), + format!(r#""#), + format!(r#"trailing"#), + format!(r#""#), + format!(r#""#), + format!(r#""#), + r#""#.to_string(), + r#""#.to_string(), + String::new(), + ] { + assert!(matches!(Manifest::parse_strict(bad.as_bytes()), Err(Error::Manifest(_))), "{bad}"); + let _ = Manifest::parse(bad.as_bytes()); + } + // The lenient read takes what it can of the first. + assert_eq!(Manifest::parse(format!(r#""#).as_bytes()).entries.len(), 1); + // Entries: up to the cap, not past it. + let many = |n: usize| format!(r#"{}"#, r#""#.repeat(n)); + assert_eq!(Manifest::parse_strict(many(crate::manifest::MAX_ENTRIES).as_bytes()).map(|m| m.entries.len()), Ok(crate::manifest::MAX_ENTRIES)); + assert!(Manifest::parse_strict(many(crate::manifest::MAX_ENTRIES + 1).as_bytes()).is_err()); + // Through a package: missing and present. + let mut p_bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes())]); + assert!(matches!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().manifest_strict(), Err(Error::Manifest(_)))); + p_bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes()), ("META-INF/manifest.xml", good.as_bytes())]); + assert_eq!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().manifest_strict(), Ok(m)); + // A written package passes. + assert!(Package::open(&secret(), &Limits::DEFAULT).unwrap().manifest_strict().is_ok_and(|m| m.entries.len() == 4)); +} + +#[test] +fn a_package_is_held_to_its_limits() { + let bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes()), ("a.xml", &[b'a'; 100]), ("b.xml", &[b'b'; 100])]); + // Entries. + assert!(matches!(Package::open(&bytes, &Limits { max_entries: 2, ..Limits::DEFAULT }), Err(Error::TooManyEntries(_)))); + let bomb = zip_guard::zip64_entry_count_sample(Limits::DEFAULT.max_entries + 1); + assert!(matches!(Package::open(&bomb, &Limits::DEFAULT), Err(e) if e.is_limit())); + // One file. + let mut p = Package::open(&bytes, &Limits { max_entry: 99, ..Limits::DEFAULT }).unwrap(); + assert_eq!(p.read("a.xml"), Err(Error::EntryTooLarge("a.xml".into(), 99))); + let mut p = Package::open(&bytes, &Limits { max_entry: 100, ..Limits::DEFAULT }).unwrap(); + assert_eq!(p.read("a.xml").map(|d| d.map(|d| d.len())), Ok(Some(100))); + // All files together; a file read twice counts twice. + let mut p = Package::open(&bytes, &Limits { max_total: 250, ..Limits::DEFAULT }).unwrap(); + assert!(p.read("a.xml").is_ok() && p.read("b.xml").is_ok()); + assert_eq!(p.bytes_read(), 200); + assert_eq!(p.read("a.xml"), Err(Error::TotalTooLarge(250))); + // Not a ZIP. + assert!(matches!(Package::open(b"not a zip", &Limits::DEFAULT), Err(Error::Zip(_)))); +} + +/// A deflated entry whose headers declare `declared` bytes but which inflates to `real.len()`. +fn lying_zip(real: &[u8], declared: u32) -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + z.start_file("a.xml", zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated)).unwrap(); + z.write_all(real).unwrap(); + let mut bytes = z.finish().unwrap().into_inner(); + // The uncompressed size, in the central directory header (offset 24) and the local one (22). + let want = (real.len() as u32).to_le_bytes(); + let central = bytes.windows(4).rposition(|w| w == b"PK\x01\x02").unwrap(); + assert_eq!(&bytes[central + 24..central + 28], &want); + bytes[central + 24..central + 28].copy_from_slice(&declared.to_le_bytes()); + assert_eq!(&bytes[22..26], &want); + bytes[22..26].copy_from_slice(&declared.to_le_bytes()); + bytes +} + +#[test] +fn a_file_that_inflates_past_what_it_declares_is_not_read_past_the_limit() { + let bytes = lying_zip(&vec![b'a'; 100_000], 10); + let mut p = Package::open(&bytes, &Limits { max_entry: 1000, ..Limits::DEFAULT }).unwrap(); + assert_eq!(p.declared_size("a.xml"), Some(10)); + // Refused or unreadable, and never more than the limit (and the byte that shows it) inflated. + assert!(!matches!(p.read("a.xml"), Ok(Some(_)))); + assert!(p.bytes_read() <= 1001, "{}", p.bytes_read()); +} + +#[test] +fn a_mimetype_is_short_or_nothing() { + let long = "x".repeat(300); + let mut p_bytes = zip_of(&[("mimetype", long.as_bytes())]); + assert_eq!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().mimetype(), None); + p_bytes = zip_of(&[("mimetype", b" application/vnd.oasis.opendocument.spreadsheet\n")]); + assert_eq!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().mimetype().as_deref(), Some(mime::SPREADSHEET)); + p_bytes = zip_of(&[("content.xml", b"
")]); + assert_eq!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().mimetype(), None); +} + +#[test] +fn an_encrypted_package_opens_with_its_password_only() { + let plain = secret(); + assert!(!is_encrypted(&plain)); + let enc = encrypt(&plain, "pässword").unwrap(); + assert!(is_encrypted(&enc)); + // Nothing readable is left: the content is encrypted, the manifest says how. + let mut p = Package::open(&enc, &Limits::DEFAULT).unwrap(); + assert_eq!(p.names().next(), Some("mimetype")); + assert!(!enc.windows(10).any(|w| w == b"Top secret")); + let manifest = String::from_utf8(p.read("META-INF/manifest.xml").unwrap().unwrap()).unwrap(); + for want in [ + r#""# + ) + }; + let pkg = |m: String, content: &[u8]| { + zip_of(&[("mimetype", b"application/vnd.oasis.opendocument.text"), ("META-INF/manifest.xml", m.as_bytes()), ("content.xml", content)]) + }; + // Blowfish: said, not guessed at. + let blowfish = pkg(manifest("Blowfish CFB", r#"manifest:iteration-count="1024""#), &[0u8; 64]); + assert!(is_encrypted(&blowfish)); + assert!(matches!(decrypt(&blowfish, "x"), Err(CryptError::Unsupported(w)) if w.contains("Blowfish"))); + // An iteration count that would hang, a vector of the wrong size, ragged data: refused. + for (extra, data) in [ + (r#"manifest:iteration-count="4000000000""#, vec![0u8; 32]), + (r#"manifest:iteration-count="1""#, vec![0u8; 33]), + (r#"manifest:iteration-count="1""#, vec![]), + ] { + assert!(decrypt(&pkg(manifest("http://www.w3.org/2001/04/xmlenc#aes256-cbc", extra), &data), "x").is_err(), "{extra}"); + } + // Junk that isn't a package. + assert!(!is_encrypted(b"not a zip")); + assert!(decrypt(b"not a zip", "x").is_err()); + // A plain package has nothing to decrypt. + assert!(matches!(decrypt(&secret(), "x"), Err(CryptError::Unsupported(_)))); +} + +#[test] +fn a_package_declaring_too_many_entries_is_refused_before_it_is_decrypted() { + let bomb = zip_guard::zip64_entry_count_sample(Limits::DEFAULT.max_entries + 1); + assert!(!is_encrypted(&bomb)); + assert!(matches!(decrypt(&bomb, "x"), Err(CryptError::Unsupported(e)) if e.contains("declares"))); + assert!(matches!(encrypt(&bomb, "x"), Err(CryptError::Unsupported(e)) if e.contains("declares"))); +} + +#[test] +fn files_past_the_most_decrypted_are_not_reached() { + // The one encrypted file sits after 10,000 others: it is never derived a key for. + let manifest = r#""#; + let names: Vec = (0..10_000).map(|i| format!("f{i}")).collect(); + let mut files: Vec<(&str, &[u8])> = vec![("META-INF/manifest.xml", manifest.as_bytes())]; + files.extend(names.iter().map(|n| (n.as_str(), &b""[..]))); + files.push(("z.xml", &[0u8; 32])); + let last = zip_of(&files); + assert_eq!(decrypt(&last, "x"), Err(CryptError::Unsupported("no encrypted file could be read".into()))); + // Within reach, the same file is tried, and refused for what it is. + let first = zip_of(&[("META-INF/manifest.xml", manifest.as_bytes()), ("z.xml", &[0u8; 32])]); + assert_eq!(decrypt(&first, "x"), Err(CryptError::Unsupported("the key isn't 256 bits".into()))); +} + +/// A 1×1 PNG, base64. +const PNG_B64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + +fn flat_doc(mimetype: &str) -> String { + format!( + r#" + + Flat title + + + + + Some words. + +{PNG_B64} + + x + + +"# + ) +} + +#[test] +fn a_flat_document_becomes_a_package_and_back() { + let png = { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.decode(PNG_B64).unwrap() + }; + let doc = flat::split(flat_doc(mime::TEXT).as_bytes(), &Limits::DEFAULT).unwrap(); + assert_eq!(doc.mimetype.as_deref(), Some(mime::TEXT)); + // Each part has what a package's file has; the root's media type stays with the package. + assert!(doc.content.contains(""#), "{}", doc.content); + assert!(doc.content.contains(r#""#)); + let names: Vec<&str> = doc.files.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + names, + [ + "Pictures/flat1.png", + "Object flat2/content.xml", + "Object flat3/content.xml", + "Object flat3/styles.xml", + "Object flat3/meta.xml", + "Object flat3/settings.xml" + ] + ); + assert_eq!(doc.files.first().map(|(_, d)| d.as_slice()), Some(png.as_slice())); + assert_eq!(doc.folders, [("Object flat3/".to_string(), mime::CHART.to_string())]); + + let package = flat::to_package(flat_doc(mime::TEXT).as_bytes(), mime::GRAPHICS, &Limits::DEFAULT).unwrap(); + let mut p = Package::open(&package, &Limits::DEFAULT).unwrap(); + assert_eq!(p.mimetype().as_deref(), Some(mime::TEXT)); + assert_eq!(p.read("Pictures/flat1.png").unwrap().as_deref(), Some(png.as_slice())); + assert_eq!(p.manifest().media_type("Object flat3/"), Some(mime::CHART)); + + // And back: one document of the same media type, the picture, formula and chart inline. + let again = String::from_utf8(flat::from_package(&package, &Limits::DEFAULT).unwrap()).unwrap(); + assert!(again.contains(&format!("office:mimetype=\"{}\">", mime::TEXT)), "{again}"); + assert!(again.contains(&format!("{PNG_B64}")), "{again}"); + assert!(again.contains("x"), "{again}"); + assert!( + again.contains( + r#"office:mimetype="application/vnd.oasis.opendocument.chart">"# + ), + "{again}" + ); + assert!(!again.contains("xlink:href"), "{again}"); + // Taking that apart gives the same files. + let twice = flat::split(again.as_bytes(), &Limits::DEFAULT).unwrap(); + assert_eq!((twice.files.len(), &twice.folders), (doc.files.len(), &doc.folders)); +} + +#[test] +fn a_flat_document_of_another_kind_keeps_its_media_type() { + let package = flat::to_package(flat_doc(mime::SPREADSHEET).as_bytes(), mime::TEXT, &Limits::DEFAULT).unwrap(); + assert_eq!(Package::open(&package, &Limits::DEFAULT).unwrap().mimetype().as_deref(), Some(mime::SPREADSHEET)); + // One that declares none takes the caller's. + let bare = ""; + let package = flat::to_package(bare.as_bytes(), mime::PRESENTATION, &Limits::DEFAULT).unwrap(); + assert_eq!(Package::open(&package, &Limits::DEFAULT).unwrap().mimetype().as_deref(), Some(mime::PRESENTATION)); +} + +#[test] +fn hostile_flat_documents_do_not_panic() { + assert_eq!(flat::split(b"\xff\xfe", &Limits::DEFAULT), Err(Error::Flat("not UTF-8 text"))); + assert_eq!(flat::split(b"", &Limits::DEFAULT), Err(Error::Flat("no office:document"))); + assert!(flat::split(b"", &Limits::DEFAULT).is_err()); + for xml in [ + "!!!not base64", + "!!!", + "AAAA", + ] { + let _ = flat::split(xml.as_bytes(), &Limits::DEFAULT); + let _ = flat::to_package(xml.as_bytes(), mime::TEXT, &Limits::DEFAULT); + } + // A picture over the per-file limit is left out; the document still opens. + let doc = flat::split(flat_doc(mime::TEXT).as_bytes(), &Limits { max_entry: 10, ..Limits::DEFAULT }).unwrap(); + assert!(!doc.files.iter().any(|(n, _)| n.starts_with("Pictures/"))); + // More inline objects than are taken out: the first 10,000 become files, the rest go. + let many = format!("{}", "".repeat(10_050)); + assert_eq!(flat::split(many.as_bytes(), &Limits::DEFAULT).unwrap().files.len(), 10_000); + // A package with no media type has no flat form. + assert_eq!(flat::from_package(&zip_of(&[("content.xml", b"")]), &Limits::DEFAULT), Err(Error::Flat("the package has no media type"))); +} + +/// Two files of one name: `zip` keeps one, and which one the writer meant is not known. +#[test] +fn two_files_of_one_name_are_refused() { + let bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes()), ("dupA.xml", b""), ("dupB.xml", b"")]); + assert!(Package::open(&bytes, &Limits::DEFAULT).is_ok()); + let mut renamed = bytes.clone(); + let mut i = 0; + while let Some(at) = renamed[i..].windows(4).position(|w| w == b"dupB") { + renamed[i + at + 3] = b'A'; + i += at + 4; + } + assert_eq!(Package::open(&renamed, &Limits::DEFAULT).err(), Some(Error::Zip("two files of the package share a name".into()))); +} + +/// Encrypting and decrypting read each file within the per-file limit: a file declaring more is +/// refused before it is inflated (encrypting read every file whole; decrypting cut it short). +#[test] +fn encrypting_and_decrypting_refuse_a_file_past_the_limit() { + let over = u32::try_from(Limits::DEFAULT.max_entry + 1).unwrap(); + let bytes = lying_zip(b"", over); + assert!(crate::crypt::encrypt(&bytes, "pw").is_err()); + assert!(crate::crypt::decrypt(&bytes, "pw").is_err()); + // The same file declaring its real size encrypts. + assert!(crate::crypt::encrypt(&lying_zip(b"", 4), "pw").is_ok()); +} diff --git a/crates/odf/src/text.rs b/crates/odf/src/text.rs new file mode 100644 index 000000000..f9d039508 --- /dev/null +++ b/crates/odf/src/text.rs @@ -0,0 +1,86 @@ +// The code came from an app whose lints allow this. +#![allow(clippy::type_complexity)] +//! OpenDocument Text (`.odt`, ODF 1.3), feature `text`: documents read into and written from the +//! craft-flow model and a craft-wordmodel `Document`. +//! +//! Both ways, through the flow model ([`craft_flow`]) with what ODT keeps besides +//! ([`model::FlowOpts`]): paragraphs and headings with their spacing, indents, line spacing, +//! contextual spacing, alignment, borders, background and tab stops; character formatting (the +//! extras in [`model::Fmt::extra`] included); custom paragraph and character styles by name, and +//! the look of the heading, title, quotation and preformatted styles; lists with their identity, +//! number formats, labels and positions, and heading numbering (`text:outline-style`); tables with +//! column widths, spans, header rows, cell borders, padding, shading and vertical alignment; +//! pictures (in line or floating), equations as formula objects, text boxes; links, bookmarks, +//! text fields, footnotes and endnotes, comments with ranges and replies; sections with their own +//! master pages, page setup, columns and headers and footers; document defaults, metadata and +//! LibreOffice's layout settings (`settings.xml`, kept from the file or Word-compatible). +//! +//! Export writes `mimetype` (stored, first), `META-INF/manifest.xml`, `content.xml`, +//! `styles.xml`, `meta.xml`, `settings.xml`, pictures under `Pictures/` and formula +//! sub-documents. Import reads `content.xml`, `styles.xml` (style inheritance resolved, automatic +//! and common styles), `meta.xml`, `settings.xml` and the parts frames point at; drawing shapes +//! are still skipped. +//! +//! Tracked changes (ODF 1.3 Part 3 §5.5) go both ways: `text:tracked-changes` holds one +//! `text:changed-region` per change; an insertion is the body text between +//! `text:change-start`/`text:change-end`, a deletion keeps the deleted paragraphs in its region +//! and leaves a `text:change` point in the body. They map to `Document::revisions` and the +//! `ins`/`del` marks of runs and paragraph marks; a deleted paragraph mark is a region of +//! several paragraphs. Format changes are read as no change. + +use std::collections::HashMap; +use std::sync::Arc; + +use craft_wordmodel::props::{Borders, VAlign}; +use craft_wordmodel::section::{Columns, SectionStart}; +use craft_wordmodel::{Align, Document, RevisionKind, Rgb, SectionProps}; + +use craft_flow::{self as model, Cell, FBlock, FComment, FTable, Flow, Fmt, Inline, Kind, ListInfo, Meta, Para, RevMark, make_img, mime_of}; + +mod body; +mod cell_formula; +mod chart; +mod datastyle; +mod fields; +mod flat_doc; +mod forms; +mod formula_size; +mod frames; +mod geometry; +mod import; +mod kept; +mod limits; +mod objects; +mod package; +mod parts; +mod path; +mod shapes; +mod strokes; +mod styles; +mod writer; +mod xml; + +pub use crate::crypt::{CryptError, decrypt, encrypt, is_encrypted}; +use body::Body; +pub use flat_doc::{export_flat, import_flat, import_flat_on}; +use import::Pages; +#[cfg(test)] +use import::{Pictures, apply_page_layout, read_entry, read_meta}; +pub use import::{import, import_on, parse}; +#[cfg(test)] +use limits::{MAX_EXTRA_SPACES, MAX_XML_DEPTH}; +pub use package::{TEMPLATE_MIME, TEXT_MIME, export, export_template}; +use styles::Styles; +#[cfg(test)] +use writer::x; +#[cfg(test)] +use xml::length; +use xml::{Ev, events, get}; + +mod page_parts; + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod tests_more; diff --git a/crates/odf/src/text/body.rs b/crates/odf/src/text/body.rs new file mode 100644 index 000000000..f07595676 --- /dev/null +++ b/crates/odf/src/text/body.rs @@ -0,0 +1,2178 @@ +use super::import::Pages; +use super::limits::{ + MAX_COMMENTS, MAX_EXTRA_SPACES, MAX_FIELD_RESULT, MAX_INFO_CHARS, MAX_OPEN_INSERTIONS, MAX_REGIONS, MAX_SECTIONS, MAX_XML_DEPTH, +}; +use super::styles::Styles; +use super::xml::{Attrs, Ev, events, get, length, local}; +use super::{ + Borders, Cell, FBlock, FComment, FTable, Fmt, HashMap, Inline, Kind, ListInfo, Para, RevMark, RevisionKind, SectionProps, SectionStart, make_img, + model, +}; + +/// Most drawings waiting outside a paragraph. +const MAX_LOOSE: usize = 10_000; + +pub(in crate::text) mod drawing; +mod indexes; +mod notes; +mod tables; +mod tracked; + +use drawing::{OpenGroup, read_formula}; +use indexes::{OpenIndex, is_index}; +use notes::{OpenField, OpenNote, balance_comments}; +use tables::TableB; +use tracked::{DelState, Region, change_mark}; + +/// Most bibliography sources read. +const MAX_SOURCES: usize = 10_000; + +/// Most index entries over text open at once in a paragraph. +const MAX_OPEN_MARKS: usize = 64; + +/// An open frame: its size, picture, alt text, equation, chart and other embedded object. +type Frame = + (Option, Option, Option>, String, Option<(String, bool)>, Option, Option); + +pub(super) struct Body<'a> { + styles: &'a Styles, + zip: &'a mut dyn FnMut(&str) -> Option>, + /// The package's entry names and its manifest's media types, for embedded objects. + pub(super) entries: &'a [String], + /// The bibliography sources the citations carry. + pub(super) sources: Vec, + /// The paragraph before had a page break after it: the next one starts a page. + break_next: bool, + /// Shapes' ids (`xml:id`, `draw:id`) as WordCraft drawing ids, and where the shapes with one + /// sit (kind, frame relative to its anchor, turn, placement, the paragraph they are in), for + /// connectors glued to them. + pub(super) conn_ids: HashMap, + pub(super) shape_places: HashMap, + /// Paragraphs started so far: which paragraph a shape is in. + pub(super) para_seq: usize, + /// Index entries over text whose end hasn't come (in the paragraph being read): their id, + /// keys and where they start: an inline, and a byte offset in it when it is text. + index_marks: Vec<(String, (Option, Option), (usize, usize))>, + /// A ruby being read: its base text, ruby text, which of them the text goes to (`true`: + /// the ruby), and the base's formatting. + ruby: Option<(String, String, Option, Fmt)>, + /// A placeholder being read (§7.7.2 `text:placeholder`): its text, its run's formatting. + placeholder: Option<(String, Fmt, &'static str, String)>, + /// The form controls of `office:forms`, placed where a `draw:control` names them. + forms: super::forms::FormsReader, + /// A watermark found (in a header): see [`Self::start_shape`]. + pub(super) watermark: Option, + /// The shape being read is a watermark's, if its geometry follows its text: its size, turn, + /// colour and see-through, and whether its geometry is fontwork. + watermark_shape: Option<((f32, f32), f32, Option, bool, bool)>, + pub(super) manifest: Option<&'a HashMap>, + containers: Vec>, + /// Drawings anchored in the text but outside any paragraph (LibreOffice's page-anchored + /// frames before the first one): the next paragraph's, so they don't make a paragraph of + /// their own (which would start the document on another page style). + loose: Vec, + /// The markup of each drawing in `loose`, as read, while each one's is (`loose_whole`): the + /// paragraph they join keeps it before its own (see `feed_at`). + loose_raw: Vec>, + loose_whole: bool, + /// The drawing outside a paragraph being read: where it starts, its depth, and how many + /// drawings were waiting before it. + loose_open: Option<(usize, usize, usize)>, + fmt: Vec, + /// For each paragraph being read: the size its heading, title or code style gives its text. + para_sizes: Vec<(Option, bool)>, + /// That size for the paragraph being finished. + para_size: Option, + para: Option, + /// The text read last ended in a space collapsed from XML whitespace (not a `text:s`). + xml_space_end: bool, + /// What is left of [`MAX_EXTRA_SPACES`]. + pub(super) spaces_left: usize, + para_cont: bool, + /// Open lists: (style name, item open, item used). + lists: Vec<(Option, bool)>, + item_used: Vec, + /// Lists outside the table cells being read, and the list number there. + lists_saved: Vec<(Vec<(Option, bool)>, Vec, u32)>, + tables: Vec, + cells: Vec, + /// How many containers each open cell's content makes open (its top level, where sections + /// are kept around the blocks as in the body's). + cell_tops: Vec, + /// Each open cell's formula, as a Word formula field code (§19.646 `table:formula`). + formulas: Vec>, + pub(super) in_text: bool, + skip: usize, + depth: usize, + /// The markup being read (`content.xml`), for keeping paragraphs' own (empty: none kept). + pub(super) raw: &'a [u8], + /// The paragraph whose markup is being kept (see [`KeptOpen`]). + kept_open: Option, + /// The body table whose markup is being kept (its `done`: the container's length before it). + kept_table: Option, + /// The body index (a table of contents…) whose markup is being kept, likewise. + kept_index: Option, + /// Sections hidden (§19.818 `text:display="none"`) open: their depth, how many containers were + /// open, and the length of the one holding them, at their start. + hidden_sections: Vec<(usize, usize, usize)>, + /// A hidden section just read: the containers open and where its blocks start, hidden + /// before the next event (when what ends with it has ended). + hide_pending: Option<(usize, usize)>, + /// The top-level list whose markup is being kept, likewise (its items' paragraphs all kept). + kept_list: Option, + /// Sections whose wrapper is kept as read (in the body's or a text box's top level): start + /// tag, depth, the length of what holds it before it and how many containers are open; + /// `None` for one read as a columned section (kept as a section break) or anywhere else. + kept_sections: Vec, usize, usize, usize)>>, + /// Each section's master page, as the paragraph or table starting it named it (`None`: as + /// the file's default, or a columned section), in order; and the one just named. + pub(super) section_masters: Vec>, + pending_master: Option, + /// Paragraphs finished so far (a kept paragraph must come out of its element whole). + paras_done: usize, + /// `(width, height, picture bytes, alt text, equation read from an embedded `draw:object`)`. + frame: Option, + /// The first picture the open frame offers that WordCraft can't show: an object coming after it + /// keeps it as its replacement. + frame_unread: Option>, + in_desc: bool, + /// The master pages' page setups. + pages: &'a Pages, + /// The current master page's setup (a column section's page, without its columns). + page: SectionProps, + /// The open section; where its blocks start in the body; how many sections have ended. + pub(super) sec: SectionProps, + pub(super) sec_from: usize, + sections: usize, + /// Element depth of the open columned `text:section`. + cols_at: Option, + /// Inside `text:tracked-changes`. + in_tc: bool, + /// `text:track-changes` of the document's tracked changes (default on), when it has them. + pub(super) track: Option, + /// Changed regions read, by id. + regions: HashMap, + /// Regions seen (capped at [`MAX_REGIONS`]). + region_count: usize, + /// The region being read. + region: Option<(String, Region)>, + /// Depth of the `office:change-info` being read, and the field its text goes to. + info_at: Option, + info_field: Option, + /// Reading a deletion's content. + del: Option, + /// Insertions open in the body: (region id, revision). + active_ins: Vec<(String, RevMark)>, + /// Format changes open where text is being read (ODF records no old formatting). + active_fmt: Vec<(String, RevMark)>, + /// The text field being read. + field: Option, + /// Where the frame being read floats. + frame_float: Option, + /// The content of the text box in the frame being read. + frame_box: Option>, + /// The look of the frame being read, should it hold a text box. + frame_look: model::BoxLook, + /// The open frame's picture crop (`fo:clip`: top, right, bottom, left, points of its original + /// size). + frame_clip: Option<[f32; 4]>, + /// The open frame's name (`draw:name`), which a linked text box names. + pub(super) frame_name: Option, + /// The kind of the drawing shape being read (its text is read as a text box's). + shape_kind: Option, + /// A custom shape's mirroring (left to right, top to bottom), from its geometry. + shape_flip: (bool, bool), + /// A freeform shape's paths (a polygon's, polyline's, path's or regular polygon's). + shape_path: Option>, + /// A custom shape's geometry being read, when its type has no WordCraft kind: drawn from its + /// own path and equations. + geometry: Option>, + /// That geometry's markup as read (see `ShapeExtra::odf_geometry`): being read, then + /// finished for the shape it ends in. + geometry_xml: Option, + pub(in crate::text) shape_geometry: Option>, + /// The turn of a group's member being read, from its `draw:transform`. + member_rot: Option, + /// The depth of the `draw:g` group being read: its shapes are placed by it, not anchored. + group_at: Option, + /// The table of contents or other index being read. + index: Option, + /// The group being read (its anchor, style and members so far), and where its member being + /// read sits (unturned, from where the group is anchored). + group: Option, + member_at: Option<(f32, f32)>, + /// The note or comment being read. + note: Option, + /// The list being read (its flow number, [`ListInfo::list`]), the last one, the next number, + /// lists by `xml:id`, and a start value an item set for the items from it on. + list_id: u32, + last_list: Option, + next_list: u32, + list_xml_ids: HashMap, + item_start: Option, + /// Comments by `office:name`: their flow ids; and the next flow id. + comment_names: HashMap, + next_comment: u32, +} + +impl<'a> Body<'a> { + pub(super) fn new(styles: &'a Styles, pages: &'a Pages, zip: &'a mut dyn FnMut(&str) -> Option>) -> Self { + let page = pages.first.clone().unwrap_or_default(); + Body { + styles, + zip, + entries: &[], + forms: Default::default(), + watermark: None, + watermark_shape: None, + break_next: false, + sources: Vec::new(), + ruby: None, + placeholder: None, + index_marks: Vec::new(), + conn_ids: HashMap::new(), + shape_places: HashMap::new(), + para_seq: 0, + manifest: None, + containers: vec![Vec::new()], + loose: Vec::new(), + loose_raw: Vec::new(), + loose_whole: true, + loose_open: None, + fmt: Vec::new(), + para: None, + xml_space_end: false, + spaces_left: MAX_EXTRA_SPACES, + para_cont: false, + para_sizes: Vec::new(), + para_size: None, + lists: Vec::new(), + lists_saved: Vec::new(), + item_used: Vec::new(), + tables: Vec::new(), + cells: Vec::new(), + cell_tops: Vec::new(), + formulas: Vec::new(), + in_text: false, + skip: 0, + depth: 0, + raw: &[], + kept_open: None, + kept_table: None, + kept_index: None, + kept_list: None, + hidden_sections: Vec::new(), + hide_pending: None, + kept_sections: Vec::new(), + section_masters: vec![None], + pending_master: None, + paras_done: 0, + frame: None, + frame_unread: None, + in_desc: false, + pages, + sec: page.clone(), + page, + sec_from: 0, + sections: 0, + cols_at: None, + in_tc: false, + track: None, + regions: HashMap::new(), + region_count: 0, + region: None, + info_at: None, + info_field: None, + del: None, + active_ins: Vec::new(), + active_fmt: Vec::new(), + field: None, + frame_float: None, + frame_box: None, + frame_look: model::BoxLook::default(), + frame_clip: None, + frame_name: None, + shape_kind: None, + shape_flip: (false, false), + shape_path: None, + geometry: None, + geometry_xml: None, + shape_geometry: None, + member_rot: None, + group_at: None, + index: None, + group: None, + member_at: None, + note: None, + comment_names: HashMap::new(), + next_comment: 0, + list_id: 0, + last_list: None, + next_list: 1, + list_xml_ids: HashMap::new(), + item_start: None, + } + } + + /// A new list number. + fn new_list_id(&mut self) -> u32 { + let id = self.next_list; + self.next_list = self.next_list.saturating_add(1); + id + } + + /// A top-level `text:list` starting (ODF 1.3 §5.3.1): a list of its own, unless it continues + /// the one before it (`text:continue-numbering`) or the one it names (`text:continue-list`). + fn start_list_id(&mut self, a: &Attrs) { + let named = get(a, "text:continue-list").and_then(|n| self.list_xml_ids.get(n)).copied(); + let before = (get(a, "text:continue-numbering").is_some_and(|v| v.trim() == "true")).then_some(self.last_list).flatten(); + let id = match named.or(before) { + Some(id) => id, + None => self.new_list_id(), + }; + if let Some(x) = get(a, "xml:id").filter(|x| x.len() <= MAX_INFO_CHARS) + && self.list_xml_ids.len() < MAX_SECTIONS + { + self.list_xml_ids.entry(x.to_string()).or_insert(id); + } + self.list_id = id; + self.last_list = Some(id); + self.item_start = None; + } + + /// [`Body::feed_at`] for a section hidden (§19.818 `text:display="none"`): what it holds is + /// hidden text, so it stays hidden written without the section (see [`Body::hide_read`]). + fn hidden_section(&mut self, ev: &Ev) { + match ev { + Ev::Start(q, a) if q == "text:section" && get(a, "text:display").map(str::trim) == Some("none") && self.hidden_sections.len() < 64 => { + self.end_para(); + let at = (self.depth + 1, self.containers.len(), self.container().len()); + self.hidden_sections.push(at); + } + Ev::End(q) if q == "text:section" && self.hidden_sections.last().is_some_and(|h| h.0 == self.depth) => { + if let Some((_, open, from)) = self.hidden_sections.pop() { + self.hide_pending = Some((open, from)); + } + } + _ => {} + } + } + + /// The hidden section just read made hidden text. Its blocks are written from the model, + /// hidden: markup kept as read doesn't say so itself (its section might not be written + /// around it). + pub(super) fn hide_read(&mut self) { + if let Some((open, from)) = self.hide_pending.take() + && self.containers.len() == open + { + hide(self.container().get_mut(from..).unwrap_or_default(), 0); + } + } + + /// [`Body::feed_at`] for a top-level list (§5.3.1): kept whole, as read, when each of its + /// paragraphs is (a paragraph's own markup can't say how its list numbers and draws its + /// labels: a label followed by a space, a picture bullet, an unnumbered header). Not one + /// continuing another by its `xml:id`, which WordCraft may write anew (one without its own + /// is given one, for a list continuing it to name). `None` when it took the event. + fn whole_list(&mut self, ev: Ev, at: (usize, usize)) -> Option { + match &ev { + Ev::Start(q, a) + if q == "text:list" + && self.kept_list.is_none() + && self.lists.is_empty() + && self.para.is_none() + && get(a, "text:continue-list").is_none() + && !self.raw.is_empty() + && self.in_text + && self.skip == 0 + && self.del.is_none() + && self.loose.is_empty() + && self.at_body_level() => + { + self.end_para(); + let done = self.container().len(); + self.kept_list = + Some(KeptOpen { from: at.0, depth: self.depth + 1, clean: true, done, comments: KeptComments::default(), prefix: String::new() }); + Some(ev) + } + Ev::End(q) if q == "text:list" && self.kept_list.as_ref().is_some_and(|k| k.depth == self.depth) => { + let k = self.kept_list.take(); + self.feed(ev); + if let Some(k) = k + && self.containers.len() == 1 + && let Some(xml) = self.raw.get(k.from..at.1).and_then(|b| std::str::from_utf8(b).ok()) + { + let blocks = self.container(); + let n = blocks.len().saturating_sub(k.done); + let whole = |b: &FBlock| matches!(b, FBlock::Para(p) if p.kept.is_some() && p.index.is_none() && p.wrap.is_none()); + if n > 0 + && blocks.get(k.done..).is_some_and(|r| r.iter().all(whole)) + && let Some(FBlock::Para(p)) = blocks.get_mut(k.done) + { + p.index = Some(Box::new((std::sync::Arc::from(xml), n))); + } + } + None + } + _ => Some(ev), + } + } + + /// [`Body::feed_at`] for a drawing outside a paragraph (a page-anchored frame in the text): + /// its markup kept, for the paragraph it joins. `None` when it took the event. + fn loose_drawing(&mut self, ev: Ev, at: (usize, usize)) -> Option { + match &ev { + Ev::Start(q, _) + if self.loose_open.is_none() + && q.starts_with("draw:") + && is_drawing(q) + && self.para.is_none() + && self.kept_open.is_none() + && !self.raw.is_empty() + && self.in_text + && self.skip == 0 + && self.at_body_level() => + { + self.loose_open = Some((at.0, self.depth + 1, self.loose.len())); + } + Ev::End(_) if self.loose_open.is_some_and(|(_, d, _)| d == self.depth) => { + self.feed(ev); + if let Some((from, _, before)) = self.loose_open.take() { + match self.raw.get(from..at.1).and_then(|b| std::str::from_utf8(b).ok()) { + Some(xml) if self.loose.len() == before + 1 => self.loose_raw.push(std::sync::Arc::from(xml)), + _ if self.loose.len() == before => {} + _ => self.loose_whole = false, + } + } + return None; + } + _ => {} + } + Some(ev) + } + + /// [`Body::feed`], `at` the event's markup in [`Body::raw`]: a body paragraph that comes out + /// whole and refers to nothing outside itself keeps its markup (`Para::kept`), so a save + /// that doesn't change it writes it as it was (see `craft_wordmodel::kept`). + pub(super) fn feed_at(&mut self, ev: Ev, at: (usize, usize)) { + self.hide_read(); + self.hidden_section(&ev); + // A body index, kept whole: its settings and templates, which WordCraft writes its own. + match &ev { + Ev::Start(q, _) => match &mut self.kept_index { + Some(k) => k.clean &= !(q.starts_with("draw:") || q.starts_with("office:annotation") || q.starts_with("text:change")), + None if q.starts_with("text:") + && is_index(local(q)) + && !self.raw.is_empty() + && self.in_text + && self.skip == 0 + && self.containers.len() == 1 + && self.tables.is_empty() + && self.frame.is_none() + && self.note.is_none() + && self.del.is_none() => + { + self.end_para(); + self.kept_index = Some(KeptOpen { + from: at.0, + depth: self.depth + 1, + clean: true, + done: self.container().len(), + comments: KeptComments::default(), + prefix: String::new(), + }); + } + None => {} + }, + Ev::End(q) if q.starts_with("text:") && is_index(local(q)) && self.kept_index.as_ref().is_some_and(|k| k.depth == self.depth) => { + let k = self.kept_index.take(); + self.feed(ev); + if let Some(k) = k + && k.clean + && self.containers.len() == 1 + && let Some(xml) = self.raw.get(k.from..at.1).and_then(|b| std::str::from_utf8(b).ok()) + { + let blocks = self.container(); + let n = blocks.len().saturating_sub(k.done); + if n > 0 + && blocks.get(k.done..).is_some_and(|r| r.iter().all(|b| matches!(b, FBlock::Para(_)))) + && let Some(FBlock::Para(p)) = blocks.get_mut(k.done) + { + p.index = Some(Box::new((std::sync::Arc::from(xml), n))); + } + } + return; + } + _ => {} + } + // A body section: its start tag kept, put around its paragraphs at its end. + match &ev { + Ev::Start(q, _) if q == "text:section" && self.kept_sections.len() < 64 => { + let body = !self.raw.is_empty() + && self.in_text + && self.skip == 0 + && self.frame.is_none() + && self.del.is_none() + && self.lists.is_empty() + && self.at_story_top(); + let tag = self.raw.get(at.0..at.1).and_then(|b| std::str::from_utf8(b).ok()).filter(|t| !t.ends_with("/>")).map(std::sync::Arc::from); + let depth = self.depth + 1; + let cols = self.cols_at; + self.feed(ev); + // Read as a columned section (a section break): no wrapper besides. + let columned = self.cols_at != cols; + let len = self.container().len(); + let open = self.containers.len(); + self.kept_sections.push(tag.filter(|_| body && !columned).map(|t| (t, depth, len, open))); + return; + } + Ev::End(q) if q == "text:section" && !self.kept_sections.is_empty() => { + let depth = self.depth; + self.feed(ev); + if let Some(Some((tag, _, from, open))) = self.kept_sections.pop().filter(|s| s.as_ref().is_none_or(|s| s.1 == depth)) + && self.containers.len() == open + && self.at_story_top() + { + // Around top-level paragraphs and tables: the first opens it, the last closes it. + let blocks = self.container(); + // (A list item too: the list its items are written as opens and closes them, + // a section boundary ending it.) + if blocks.len() > from { + if let Some(b) = blocks.get_mut(from) { + block_wrap(b).0.insert(0, tag); + } + if let Some(b) = blocks.last_mut() { + let w = block_wrap(b); + w.1 = w.1.saturating_add(1); + } + } + } + return; + } + _ => {} + } + // A body table, kept whole as the paragraphs are. + match &ev { + Ev::Start(q, _) => match &mut self.kept_table { + Some(k) => k.clean &= keepable(q), + None if q == "table:table" + && self.tables.is_empty() + && !self.raw.is_empty() + && self.in_text + && self.skip == 0 + && self.note.is_none() + && self.frame.is_none() + && self.del.is_none() + && self.field.is_none() + && self.loose.is_empty() => + { + // Before it starts: what ends the paragraph open before it. + self.end_para(); + self.kept_table = Some(KeptOpen { + from: at.0, + depth: self.depth + 1, + clean: true, + done: self.container().len(), + comments: KeptComments::default(), + prefix: String::new(), + }); + } + None => {} + }, + Ev::End(q) if q == "table:table" && self.kept_table.as_ref().is_some_and(|k| k.depth == self.depth) => { + let k = self.kept_table.take(); + self.feed(ev); + if let Some(k) = k + && k.clean + && let Some(xml) = self.raw.get(k.from..at.1).and_then(|b| std::str::from_utf8(b).ok()) + && self.container().len() == k.done + 1 + && let Some(FBlock::Table(t)) = self.container().last_mut() + { + t.kept = Some(std::sync::Arc::from(xml)); + } + return; + } + _ => {} + } + let Some(ev) = self.loose_drawing(ev, at) else { return }; + let Some(ev) = self.whole_list(ev, at) else { return }; + match &ev { + Ev::Start(q, a) => match &mut self.kept_open { + Some(k) => { + // Comments and tracked changes paired in it. + let paired = q.starts_with("office:annotation") + || matches!(q.as_str(), "text:change" | "text:change-start" | "text:change-end") + || mark_pair(q).is_some(); + k.clean &= (paired || keepable(q)) && k.comments.start(q, a, self.depth + 1, self.paras_done); + } + None if (q == "text:p" || q == "text:h") + && !self.raw.is_empty() + && self.in_text + && self.skip == 0 + && self.para.is_none() + && self.note.is_none() + && self.frame.is_none() + && self.del.is_none() + && self.field.is_none() + && !self.in_desc => + { + // Drawings read before it that join it: their markup before its own, when + // each one's was kept. + let clean = self.loose.is_empty() || (self.loose_whole && self.loose_raw.len() == self.loose.len()); + let prefix = if self.loose.is_empty() { String::new() } else { self.loose_raw.concat() }; + self.kept_open = + Some(KeptOpen { from: at.0, depth: self.depth + 1, clean, done: self.paras_done, comments: KeptComments::default(), prefix }); + } + None => {} + }, + // A comment in it ending: its paragraphs counted (once read, its last included). + Ev::End(_) if self.kept_open.as_ref().is_some_and(|k| k.comments.ends_at(self.depth)) => { + self.feed(ev); + let done = self.paras_done; + if let Some(k) = &mut self.kept_open { + k.comments.end(done); + } + return; + } + Ev::End(q) if (q == "text:p" || q == "text:h") && self.kept_open.as_ref().is_some_and(|k| k.depth == self.depth) => { + if let Some(k) = self.kept_open.take() + && k.clean + && k.comments.closed() + && k.done + k.comments.paras == self.paras_done + && let Some(p) = &mut self.para + && let Some(xml) = self.raw.get(k.from..at.1).and_then(|b| std::str::from_utf8(b).ok()) + { + p.kept = Some(std::sync::Arc::from(format!("{}{xml}", k.prefix))); + p.group = k.comments.group(); + } + } + _ => {} + } + self.feed(ev); + } + + pub(super) fn feed(&mut self, ev: Ev) { + match ev { + // `office:forms` is read on its own; a `draw:control` (always empty) is its control. + Ev::Start(q, a) if self.forms.start(&q, &a) => {} + Ev::End(q) if self.forms.end(&q) => {} + Ev::Start(q, a) if q == "draw:control" => self.place_control(&a), + Ev::End(q) if q == "draw:control" => {} + // Ruby (ODF 1.3 §6.4): its base and ruby text, read on their own. + Ev::Start(q, _) if q == "text:ruby" && self.para.is_some() && self.field.is_none() => { + self.ruby = Some((String::new(), String::new(), None, self.cur_fmt())); + } + Ev::Start(q, _) if self.ruby.is_some() && matches!(q.as_str(), "text:ruby-base" | "text:ruby-text") => { + if let Some(r) = &mut self.ruby { + r.2 = Some(q == "text:ruby-text"); + } + } + Ev::End(q) if self.ruby.is_some() && matches!(q.as_str(), "text:ruby-base" | "text:ruby-text") => { + if let Some(r) = &mut self.ruby { + r.2 = None; + } + } + Ev::End(q) if q == "text:ruby" && self.ruby.is_some() => { + if let Some((base, ruby, _, fmt)) = self.ruby.take() { + let (base, ruby) = (base.trim().to_string(), ruby.trim().to_string()); + match (base.is_empty(), ruby.is_empty()) { + (false, false) => self.push_inline(Inline::Ruby { base, ruby, fmt }), + // A guide over nothing, or reading nothing: its text. + (false, true) | (true, false) => self.push_inline(Inline::Text(if base.is_empty() { ruby } else { base }, fmt)), + (true, true) => {} + } + } + } + // A placeholder (§7.7.2): a plain text content control showing its text until + // something is typed in its place. + Ev::Start(q, a) if q == "text:placeholder" && self.para.is_some() && self.field.is_none() && self.placeholder.is_none() => { + // Its description (§19.800 `text:description`): the control's title. + let about = get(&a, "text:description").map(|d| d.chars().take(MAX_FIELD_RESULT).collect()).unwrap_or_default(); + self.placeholder = Some((String::new(), self.cur_fmt(), super::forms::placeholder_type(get(&a, "text:placeholder-type")), about)); + } + Ev::Text(t) if self.placeholder.is_some() => { + if let Some((s, ..)) = &mut self.placeholder { + let room = MAX_FIELD_RESULT.saturating_sub(s.len()); + s.extend(t.chars().take(room)); + } + } + Ev::End(q) if q == "text:placeholder" && self.placeholder.is_some() => { + if let Some((text, fmt, kind, about)) = self.placeholder.take() { + let mut control = + craft_wordmodel::control::ContentControl::new(craft_wordmodel::control::ControlKind::Text { multi_line: false }); + control.placeholder_text.clone_from(&text); + control.showing_placeholder = true; + // Its element, so it is written back a placeholder of its type. + control.form_xml = super::forms::placeholder_start(kind); + control.title = about; + let shown = control.placeholder_shown().to_string(); + self.push_inline(Inline::Control { control: Box::new(control), text: shown, fmt }); + } + } + Ev::Text(t) if self.ruby.is_some() => { + if let Some((base, ruby, Some(in_ruby), _)) = &mut self.ruby { + let s = if *in_ruby { ruby } else { base }; + let room = MAX_FIELD_RESULT.saturating_sub(s.len()); + s.extend(t.chars().take(room)); + } + } + Ev::Start(q, a) => { + if self.depth < MAX_XML_DEPTH { + self.start(&q, &a); + } else { + self.depth += 1; + } + } + Ev::End(q) => { + if self.depth <= MAX_XML_DEPTH { + self.end(&q); + } else { + self.depth -= 1; + } + } + Ev::Text(t) => self.text(&t), + } + } + + /// The blocks read, with anything still open closed. + /// The hidden form controls read (see [`super::forms::FormsReader::hidden`]). + pub(super) fn take_hidden_controls(&mut self) -> Vec { + std::mem::take(&mut self.forms.hidden) + } + + /// Whether the reader is in the text itself: not in a table, note, comment or frame. + pub(in crate::text) fn at_body_level(&self) -> bool { + self.tables.is_empty() && self.cells.is_empty() && self.note.is_none() && self.frame.is_none() && self.containers.len() == 1 + } + + pub(super) fn take_blocks(&mut self) -> Vec { + // What truncated XML left open: a field, a note or comment (its paragraph goes on). + self.end_field(); + if self.note.is_some() { + self.end_note(); + } + self.end_para(); + self.hide_read(); + // Drawings after the last paragraph: a paragraph of their own. + if !self.loose.is_empty() { + self.start_para(None, None); + self.end_para(); + } + while self.containers.len() > 1 { + let c = self.containers.pop().unwrap_or_default(); + self.container().extend(c); + } + let mut blocks = self.containers.pop().unwrap_or_default(); + for t in self.tables.drain(..) { + if !t.rows.is_empty() { + blocks.push(FBlock::Table(t.finish())); + } + } + balance_comments(&mut blocks); + blocks + } +} + +impl Body<'_> { + /// Whether what is read goes in the body's top level, a text box's or a table cell's (not a + /// note's or a comment's): where sections are kept around the blocks. + fn at_story_top(&self) -> bool { + // A table cell's (the innermost one's). + if let Some(&top) = self.cell_tops.last() + && self.containers.len() == top + { + return true; + } + match &self.note { + None => self.containers.len() == 1 && self.tables.is_empty(), + Some(n) => n.text_box.is_some() && self.containers.len() == n.containers + 1 && self.tables.len() == n.tables, + } + } + + fn container(&mut self) -> &mut Vec { + if self.containers.is_empty() { + self.containers.push(Vec::new()); + } + let n = self.containers.len() - 1; + &mut self.containers[n] + } + + fn cur_fmt(&self) -> Fmt { + self.fmt.last().cloned().unwrap_or_default() + } + + /// The size of the text being read, points (its own, else the document's). + pub(in crate::text) fn text_size(&self) -> f32 { + self.cur_fmt().size.or_else(|| self.styles.doc_size()).unwrap_or(12.0) + } + + /// Reading the body itself, not a table cell or a frame. + fn in_body(&self) -> bool { + self.containers.len() == 1 && self.tables.is_empty() && self.frame.is_none() + } + + /// End the open section before what comes next, which starts section `next`. An open section + /// with nothing in it yet takes `next`'s place instead. Only in the body, and at most + /// [`MAX_SECTIONS`]. True when what comes next follows a section break. + fn break_section(&mut self, next: SectionProps) -> bool { + if !self.in_body() { + return false; + } + let Some(body) = self.containers.first_mut() else { return false }; + if body.len() <= self.sec_from { + // The open section, empty, takes its place: its master page too. + if let Some(m) = self.pending_master.take() + && let Some(last) = self.section_masters.last_mut() + { + *last = Some(m); + } + let restart = self.sec.page_num_start; + self.sec = next; + self.sec.page_num_start = self.sec.page_num_start.or(restart); + return self.sections > 0; + } + if self.sections + 1 >= MAX_SECTIONS { + return false; + } + // A section ends on a paragraph: after a table, an empty one (as Word writes it). + if !matches!(body.last(), Some(FBlock::Para(p)) if p.section.is_none()) { + body.push(FBlock::Para(Para::default())); + } + if let Some(FBlock::Para(p)) = body.last_mut() { + p.section = Some(Box::new(std::mem::replace(&mut self.sec, next))); + } + self.sec_from = body.len(); + self.sections += 1; + self.section_masters.push(self.pending_master.take()); + true + } + + /// A body paragraph or table of style `name` that names a master page starts a new page with + /// that page's setup and, when the style says so, page numbering restarted (ODF 1.3 applies + /// `style:page-number` only with a master page). The section that starts there, if any. + /// + /// Rare cases left simple: a master-page paragraph first inside a columned `text:section` + /// replaces that (still empty) column section, so the columns are lost; a columned section + /// nested in another is read as part of the outer one. + fn page_section(&mut self, name: &str) -> Option { + if !self.in_body() { + return None; + } + let (master, number) = self.styles.page_start(name); + let restart = number.flatten(); + self.pending_master = master.map(str::to_string); + let page = master.and_then(|m| self.pages.masters.get(m))?.clone(); + self.page = page.clone(); + Some(SectionProps { start: SectionStart::NextPage, page_num_start: restart, ..page }) + } + + fn start_para(&mut self, style: Option<&str>, heading: Option) { + self.end_para(); + // Paragraphs of the text (not of a shape's, a note's or a comment's) say where shapes are. + if self.note.is_none() { + self.para_seq = self.para_seq.wrapping_add(1); + } + let look = style.map(|s| self.styles.para(s)).unwrap_or_default(); + let (mut kind, align, mut pb, rule, keep_next, base) = (look.kind, look.align, look.page_break, look.rule, look.keep_next, look.fmt); + // The section break starts the new page; no page break besides. + let mut new_page = false; + if let Some(next) = style.and_then(|s| self.page_section(s)) + && self.break_section(next) + { + (pb, new_page) = (false, true); + } + // A heading looks as its style says (a Heading 2 style without a level attribute, which + // makes it level 1, still looks as Heading 2); its outline level is the attribute's. + let mut level = None; + if let Some(h) = heading { + match kind { + Kind::Heading(k) if k != h.clamp(1, 6) => level = Some(h.clamp(1, 10) - 1), + _ => kind = Kind::Heading(h.clamp(1, 6)), + } + } + // The paragraph before ended with a page break after it (`fo:break-after`): this one + // starts the page. + if std::mem::take(&mut self.break_next) && !new_page { + pb = true; + } + let mut p = Para { kind, align, page_break: pb, keep_next, spacing: look.spacing, ..Default::default() }; + // A paragraph starting a column (`fo:break-before="column"`): Word's column break first. + if look.column_break && !pb { + p.inlines.push(Inline::Text(craft_wordmodel::para::COLUMN_BREAK.to_string(), model::Fmt::default())); + } + p.borders = look.borders; + if let Some(l) = level { + p.extra.get_or_insert_with(Default::default).outline_level = Some(l); + } + if let Some(n) = look.named { + // The ODF name for now; import makes it the document style's id. + p.extra.get_or_insert_with(Default::default).style = Some(n); + } + if let Some(bg) = look.shading { + p.extra.get_or_insert_with(Default::default).shading = Some(bg); + } + if let Some(c) = look.contextual { + p.extra.get_or_insert_with(Default::default).contextual_spacing = Some(c); + } + if look.rtl == Some(true) { + p.extra.get_or_insert_with(Default::default).bidi = Some(true); + } + if let Some((lines, distance)) = look.drop_cap { + let x = p.extra.get_or_insert_with(Default::default); + (x.drop_cap, x.drop_cap_distance) = (Some(lines), (distance > 0.0).then_some(distance)); + } + if look.keep_lines == Some(true) { + p.extra.get_or_insert_with(Default::default).keep_lines = Some(true); + } + if look.widow_control == Some(false) { + p.extra.get_or_insert_with(Default::default).widow_control = Some(false); + } + // Hyphenated: `Some(false)` (not suppressed); the document then hyphenates (see + // `import::hyphenation`). + if look.hyphenate == Some(true) { + p.extra.get_or_insert_with(Default::default).suppress_hyphens = Some(false); + } + if look.break_after { + self.break_next = true; + } + if let Some(mut tabs) = look.tabs { + // LibreOffice measures tabs from the indent (its `TabsRelativeToIndent`); Word from + // the margin. + if self.styles.tabs_from_indent { + let indent = look.tab_origin; + for t in &mut tabs { + t.pos = (t.pos + indent).clamp(-1584.0, 1584.0); + } + } + p.extra.get_or_insert_with(Default::default).tabs = Some(tabs); + } + if rule && kind == Kind::Normal { + p.kind = Kind::Rule; + } + if let Some(lv) = self.lists.len().checked_sub(1) + && self.lists.last().is_some_and(|l| l.1) + { + let style = self.lists.iter().rev().find_map(|l| l.0.clone()); + let level = style.and_then(|s| self.styles.lists.get(&s)).and_then(|v| v.get(lv.min(9)).copied()).unwrap_or_default(); + let start = self.item_start.take().unwrap_or(level.start); + p.list = Some(ListInfo { ordered: level.ordered, level: lv.min(8) as u8, start, list: self.list_id, label: level.label }); + self.para_cont = self.item_used.last().copied().unwrap_or(false); + if let Some(u) = self.item_used.last_mut() { + *u = true; + } + } else { + self.para_cont = false; + } + let kind_size = matches!(kind, Kind::Code | Kind::Heading(_) | Kind::Title).then(|| style.and_then(|s| self.styles.named_size(s))).flatten(); + self.para_sizes.push((kind_size, style.is_some_and(|s| self.styles.states_size(s)))); + self.fmt.push(base); + // Drawings waiting outside a paragraph come first in it, when it is the text's own (not + // a text box's, a note's or a cell's). + if !self.loose.is_empty() && self.at_body_level() { + let mut loose = std::mem::take(&mut self.loose); + loose.append(&mut p.inlines); + p.inlines = loose; + self.loose_raw.clear(); + self.loose_whole = true; + } + self.para = Some(p); + } + + fn end_para(&mut self) { + // An index entry is marked within one paragraph. + self.index_marks.clear(); + let Some(mut p) = self.para.take() else { return }; + self.glue_connectors(&mut p); + self.para_size = self.para_sizes.pop().and_then(|(size, stated)| size.filter(|_| stated)); + // A paragraph without text has its own text formatting as its mark's: its size is its + // height, and its drawings' runs have it. + if let Some(base) = self.fmt.pop() + && p.textless() + && base != model::Fmt::default() + { + p.mark = Some(Box::new(craft_flow::char_props(&base))); + } + // A paragraph ending inside an open insertion has an inserted mark. (Paragraphs that + // `place_deletion` splits off keep only the marks read inside their region.) + if p.mark_ins.is_none() { + p.mark_ins = self.open_insertion().map(Box::new); + } + self.finish_para(p); + } + + /// The innermost insertion open where text is being read (the body, or deleted content). + fn open_insertion(&self) -> Option { + let open = match &self.del { + Some(d) => &d.ins, + None => &self.active_ins, + }; + open.last().map(|(_, m)| m.clone()) + } + + /// Add a finished paragraph to the current container. + fn finish_para(&mut self, mut p: Para) { + // Spaces next to tracked text are part of a change. + let tracked = self.del.is_some() + || p.mark_ins.is_some() + || p.mark_del.is_some() + || p.inlines.iter().any(|i| matches!(i, Inline::Text(_, f) if f.ins.is_some() || f.del.is_some())); + // Leading XML whitespace is never read (see `text`); a trailing one is trimmed here. + // Explicit `text:s` spaces and tabs stay. + let xml_space_end = std::mem::take(&mut self.xml_space_end); + if p.kind != Kind::Code + && !tracked + && xml_space_end + && let Some(Inline::Text(t, _)) = p.inlines.last_mut() + && t.ends_with(' ') + { + t.pop(); + if t.is_empty() { + p.inlines.pop(); + } + } + if p.kind == Kind::Rule && !p.is_empty() { + p.kind = Kind::Normal; + } + if matches!(p.kind, Kind::Code | Kind::Heading(_) | Kind::Title) { + // The style says the size of its text, stated again or not: text of another size keeps + // its own. + let style_size = self.para_size; + for i in &mut p.inlines { + if let Inline::Text(_, f) = i { + if p.kind == Kind::Code { + f.code = false; + f.font = None; + } + if f.size == style_size { + f.size = None; + } + } + } + } + self.paras_done += 1; + let cont = std::mem::take(&mut self.para_cont); + let level = p.list.map(|l| l.level); + let c = self.container(); + if cont + && let Some(FBlock::Para(prev)) = c.last_mut() + && prev.list.is_some() + && prev.list.map(|l| l.level) == level + { + prev.push_text("\n", &Fmt::default()); + prev.inlines.extend(p.inlines); + // Two elements made one paragraph: neither's markup is it. + prev.kept = None; + return; + } + c.push(FBlock::Para(p)); + } + + fn push_text(&mut self, t: &str) { + let mut f = self.cur_fmt(); + f.ins = self.open_insertion(); + // In a format change: a formatting revision whose old formatting is the text's own. + if let Some((_, mark)) = self.active_fmt.last().filter(|_| self.del.is_none()) { + let old = Fmt { ins: None, del: None, fmt_change: None, ..f.clone() }; + f.fmt_change = Some(Box::new(model::FmtChange { mark: mark.clone(), old })); + } + if self.para.is_none() { + if t.trim().is_empty() { + return; + } + self.start_para(None, None); + } + self.xml_space_end = false; + if let Some(p) = &mut self.para { + p.push_text(t, &f); + } + } + + fn text(&mut self, t: &str) { + if self.skip > 0 || !self.in_text { + return; + } + if self.info_at.is_some() { + if let (Some(field), Some((_, r))) = (self.info_field, &mut self.region) { + let s = if field { &mut r.author } else { &mut r.date }; + let room = MAX_INFO_CHARS.saturating_sub(s.chars().count()); + s.extend(t.chars().take(room)); + } + return; + } + if self.in_tc && self.del.is_none() { + return; + } + if self.in_desc { + if let Some(fr) = &mut self.frame { + fr.3.push_str(t); + } + return; + } + if let Some(c) = self.note.as_mut().and_then(|n| n.comment.as_mut()) + && let Some(k) = c.field + { + let s = match k { + 0 => &mut c.author, + 1 => &mut c.date, + _ => &mut c.initials, + }; + let room = MAX_INFO_CHARS.saturating_sub(s.chars().count()); + s.extend(t.chars().take(room)); + return; + } + if let Some(OpenField { result, .. }) = &mut self.field { + // A field's shown text, its XML whitespace collapsed; bounded like any one text run. + for c in t.chars() { + if result.len() >= MAX_FIELD_RESULT { + break; + } + if matches!(c, ' ' | '\t' | '\n' | '\r') { + if !result.ends_with(' ') { + result.push(' '); + } + } else { + result.push(c); + } + } + return; + } + let Some(p) = &self.para else { + return; + }; + // Collapse XML whitespace. + let mut prev_space = match p.inlines.last() { + Some(Inline::Text(x, _)) => x.ends_with([' ', '\n', '\t']), + Some(_) => false, + None => true, + }; + let mut s = String::with_capacity(t.len()); + for c in t.chars() { + if matches!(c, ' ' | '\t' | '\n' | '\r') { + if !prev_space { + s.push(' '); + prev_space = true; + } + } else { + s.push(c); + prev_space = false; + } + } + if !s.is_empty() { + self.push_text(&s); + self.xml_space_end = s.ends_with(' '); + } + } + + fn start(&mut self, q: &str, a: &Attrs) { + self.depth += 1; + if q == "office:text" { + self.in_text = true; + return; + } + if self.skip > 0 { + if self.skip == 1 + && q == "draw:equation" + && let Some(g) = &mut self.geometry + { + g.equation(a); + } + if self.skip == 1 + && (q == "draw:equation" || q == "draw:handle") + && let Some(xml) = &mut self.geometry_xml + { + xml.push_str(&element_xml(q, a, true)); + } + self.skip += 1; + return; + } + if !self.in_text { + return; + } + if self.info_at.is_some() { + self.info_field = match local(q) { + "creator" => Some(true), + "date" => Some(false), + _ => None, + }; + return; + } + if self.tracked_context(q, a) { + return; + } + // A custom shape's geometry: its kind, or, for a type WordCraft has no kind for, its + // path, worked out from its equations when the element ends. Its handles aren't read. + if q == "draw:enhanced-geometry" && self.shape_kind.is_some() { + let t = get(a, "draw:type").unwrap_or_default(); + if let Some(wm) = &mut self.watermark_shape { + wm.4 = get(a, "draw:text-path").is_some_and(|v| v.trim() == "true") || t.starts_with("fontwork"); + } + self.shape_kind = Some(super::shapes::kind_of(t)); + self.geometry = super::geometry::Pending::of(t, a); + // Kept as read, its mirroring aside (written from the shape's own). + let kept: Attrs = a.iter().filter(|(k, _)| !k.starts_with("draw:mirror-")).cloned().collect(); + self.geometry_xml = (self.geometry.is_some() || super::shapes::unknown_preset(t)).then(|| element_xml(q, &kept, false)); + self.shape_geometry = None; + // Mirrored (§20.112 `draw:mirror-horizontal`, §20.113 `draw:mirror-vertical`). + let on = |k: &str| get(a, k).is_some_and(|v| v.trim() == "true"); + self.shape_flip = (on("draw:mirror-horizontal"), on("draw:mirror-vertical")); + self.skip = 1; + return; + } + if self.start_inline(q, a) || self.start_note_part(q, a) || self.start_list_part(q, a) || self.start_index_part(q, a) { + return; + } + match local(q) { + "tracked-changes" => { + if self.para.is_none() && self.tables.is_empty() && self.cells.is_empty() && self.frame.is_none() { + self.in_tc = true; + self.track = Some(get(a, "text:track-changes") != Some("false")); + } else { + self.skip = 1; + } + } + "change" => { + if let Some(id) = get(a, "text:change-id") + && self.regions.get(id).is_some_and(|r| r.kind == Some(RevisionKind::Delete)) + && let Some(r) = self.regions.remove(id) + { + let mark = r.mark(); + self.place_deletion(r.blocks, mark); + } + } + "change-start" | "change-end" => { + let start = local(q) == "change-start"; + change_mark(&self.regions, &mut self.active_ins, RevisionKind::Insert, start, a); + change_mark(&self.regions, &mut self.active_fmt, RevisionKind::Format, start, a); + } + "note" | "annotation" | "text-box" | "sequence-decls" | "forms" | "variable-decls" | "user-field-decls" | "index-title-template" => { + self.skip = 1; + } + // Drawing shapes (ODF 1.3 §10.3) anchored in the text, with their text. + "rect" | "ellipse" | "circle" | "line" | "custom-shape" | "polygon" | "polyline" | "path" | "regular-polygon" | "connector" + if q.starts_with("draw:") && self.frame.is_none() && self.note.is_none() && self.del.is_none() && !self.in_tc => + { + self.start_shape(q, a); + } + // A group of shapes and pictures (§10.3.15): read as one object. Groups in it are walked, + // their members its own. + "g" if q == "draw:g" && self.group_at.is_none() => { + self.group_at = Some(self.depth); + if self.frame.is_none() && self.note.is_none() && self.del.is_none() && !self.in_tc { + self.group = Some(OpenGroup { + anchor: get(a, "text:anchor-type").map(String::from), + page: get(a, "text:anchor-page-number").map(String::from), + style: get(a, "draw:style-name").map(String::from), + members: Vec::new(), + }); + } + } + // Connectors, captions and measures, and shapes where none is read (in a note, a + // comment, a shape's text or deleted text), are left out with their text. + "rect" | "line" | "polyline" | "polygon" | "regular-polygon" | "path" | "circle" | "ellipse" | "connector" | "caption" | "measure" + | "custom-shape" + if q.starts_with("draw:") => + { + self.skip = 1; + } + "p" | "h" if !self.in_desc => { + let h = + if local(q) == "h" { Some(get(a, "text:outline-level").and_then(|v| v.trim().parse::().ok()).unwrap_or(1)) } else { None }; + self.start_para(get(a, "text:style-name"), h); + } + "table" if q.starts_with("table:") => { + // Drawings waiting for a paragraph get their own before a table. + if !self.loose.is_empty() && self.tables.is_empty() { + self.start_para(None, None); + } + self.end_para(); + // A body table whose style names a master page starts that page (ODF 1.3 §19.509). + if let Some(next) = get(a, "table:style-name").and_then(|s| self.page_section(s)) { + self.break_section(next); + } + if self.tables.len() >= model::MAX_DEPTH { + self.skip = 1; + return; + } + // A table starting a page: its first paragraph starts it, as Word moves a table + // whose first row's paragraph has a page break before it. + if get(a, "table:style-name").is_some_and(|s| self.styles.table_breaks(s)) { + self.break_next = true; + } + self.start_table(a); + } + "table-column" => { + let w = get(a, "table:style-name").and_then(|n| self.styles.column_width(n)).unwrap_or(0.0); + let share = self.styles.column_share(get(a, "table:style-name")).unwrap_or(0.0); + let cells = get(a, "table:default-cell-style-name").and_then(|s| self.styles.cell_background(s)); + let n = get(a, "table:number-columns-repeated").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).max(1); + if let Some(t) = self.tables.last_mut() { + let room = craft_wordmodel::table::MAX_COLS.saturating_sub(t.widths.len()); + t.widths.extend(std::iter::repeat_n(w, n.min(room))); + t.shares.extend(std::iter::repeat_n(share, n.min(room))); + t.column_backgrounds.extend(std::iter::repeat_n(cells, n.min(room))); + } + } + "table-row" => self.start_row(a), + "table-cell" => { + self.end_para(); + let span = get(a, "table:number-columns-spanned").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(1, 63); + let rows = get(a, "table:number-rows-spanned").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(1, 1000); + // WordCraft's tables have their backgrounds on their cells: a row's or table's on + // those with none of their own, as it shows behind them. A cell with no style has + // its column's default cell style; LibreOffice doesn't apply a row's. + let behind = self.tables.last().and_then(|t| t.row_background.or(t.background)); + let in_column = self.tables.last().and_then(|t| { + let at: usize = t.row.iter().map(|c| c.colspan.max(1) as usize).sum(); + t.column_backgrounds.get(at).copied().flatten() + }); + let own = get(a, "table:style-name").map_or(in_column, |s| self.styles.cell_background(s)); + let shading = own.or(behind); + let valign = get(a, "table:style-name").and_then(|s| self.styles.cell_valign(s)); + let borders = get(a, "table:style-name").and_then(|s| self.styles.cell_borders(s)); + let margins = get(a, "table:style-name").and_then(|s| self.styles.cell_padding(s)); + let direction = get(a, "table:style-name").map(|s| self.styles.cell_direction(s)).unwrap_or_default(); + let picture = get(a, "table:style-name").and_then(|s| self.styles.cell_picture(s)).and_then(|p| self.cell_picture(p)); + if let Some(t) = self.tables.last_mut() { + t.hcover = span as usize - 1; + } + self.cells.push(Cell { colspan: span, rowspan: rows, shading, valign, borders, margins, direction, picture, ..Default::default() }); + self.formulas.push(get(a, "table:formula").and_then(super::cell_formula::word_of)); + self.containers.push(Vec::new()); + self.cell_tops.push(self.containers.len()); + self.lists_saved.push((std::mem::take(&mut self.lists), std::mem::take(&mut self.item_used), self.list_id)); + } + "covered-table-cell" => { + if let Some(t) = self.tables.last_mut() { + if t.hcover > 0 { + t.hcover -= 1; + } else if t.row.len() < 63 { + t.row.push(Cell { covered: true, ..Default::default() }); + } + } + // What a covered cell holds isn't shown (§9.1.5): skipped (it went before the table). + self.skip = 1; + } + "frame" => { + let w = get(a, "svg:width").and_then(length); + let h = get(a, "svg:height").and_then(length); + self.frame = Some((w, h, None, String::new(), None, None, None)); + self.frame_unread = None; + let g = get(a, "draw:style-name").map(|s| self.styles.graphic(s)).unwrap_or_default(); + self.frame_float = Some(super::frames::place(a, w.unwrap_or(0.0), h.unwrap_or(0.0), &g, self.text_size())); + self.member_start(a, w.unwrap_or(0.0), h.unwrap_or(0.0), None); + // Without a height, a text box in it grows to its text (`fo:min-height` below). + self.frame_look = g.look(h.is_none(), &self.styles.defs); + self.frame_name = get(a, "draw:name").map(|n| n.chars().take(256).collect()); + self.frame_clip = g.clip; + } + "image" | "object" | "object-ole" | "desc" | "title" | "contour-polygon" => self.frame_part(q, a), + // A columned section (ODF 1.3 §5.4) is a continuous section with its columns. + "section" if q.starts_with("text:") => { + self.end_para(); + if self.cols_at.is_none() + && self.in_body() + && let Some(columns) = get(a, "text:style-name") + .and_then(|s| self.styles.columns(s, self.page.text_width())) + // A section whose columns match the open section's (a plain one-column + // section in a one-column page) is no new section. + .filter(|c| c.count != self.sec.columns.count || (c.count > 1 && *c != self.sec.columns)) + { + let balance = get(a, "text:style-name").and_then(|s| self.styles.section_balance(s)); + let next = SectionProps { + start: SectionStart::Continuous, + page_num_start: None, + columns, + balance_columns: balance, + ..self.page.clone() + }; + self.break_section(next); + self.cols_at = Some(self.depth); + } + } + _ => {} + } + } + + /// A table (`table:table`) begins: its alignment and width, a relative width or a fixed + /// one when aligned (a table filling the margins, LibreOffice's default, takes the text width). + fn start_table(&mut self, a: &Attrs) { + let (align, width, rel) = get(a, "table:style-name").map(|s| self.styles.table_look(s)).unwrap_or_default(); + // Filling the margins (§20.300 `margins`, the default), its columns are fitted to the + // text width whatever widths they state, as LibreOffice fits them. + let width_pct = match align { + None => Some(rel.unwrap_or(100.0)), + Some(_) => rel.or_else(|| width.map(|w| (w / self.sec.text_width().max(1.0) * 100.0).min(100.0))), + }; + // LibreOffice's tables keep their columns' widths, whatever their content (and however + // far past the margins): Word's fixed layout. + let space = get(a, "table:style-name").map(|s| self.styles.table_space(s)).unwrap_or_default(); + // What the shares divide: an aligned table's stated width, however wide, as LibreOffice + // keeps it; else its part of the text width. + let room = width.filter(|_| align.is_some() && rel.is_none()).or_else(|| width_pct.map(|p| self.sec.text_width() * p / 100.0)); + let contained = !self.tables.is_empty() || self.containers.len() > 1 || self.sec.columns.count > 1; + let extra = + model::TableExtra { align, width: None, width_pct, heights: Vec::new(), keep_rows: Vec::new(), fixed: true, space, header_rows: 0 }; + let background = get(a, "table:style-name").and_then(|s| self.styles.table_background(s)); + self.tables.push(TableB { + rows: Vec::new(), + row: Vec::new(), + hcover: 0, + widths: Vec::new(), + shares: Vec::new(), + room, + contained, + column_backgrounds: Vec::new(), + extra, + background, + row_background: None, + }); + } + + /// A table row (`table:table-row`) begins: its height, kept as rows are (see the row's end). + fn start_row(&mut self, a: &Attrs) { + let height = get(a, "table:style-name").and_then(|s| self.styles.row_height(s)); + let background = get(a, "table:style-name").and_then(|s| self.styles.table_background(s)); + let keep = self.styles.row_keep(get(a, "table:style-name")); + if let Some(t) = self.tables.last_mut() { + t.row_background = background; + t.row.clear(); + t.hcover = 0; + if t.extra.heights.len() < craft_wordmodel::table::MAX_ROWS { + t.extra.heights.truncate(t.rows.len()); + t.extra.heights.push(height); + t.extra.keep_rows.truncate(t.rows.len()); + t.extra.keep_rows.push(keep); + } + } + } + + /// A form control (`draw:control`, ODF 1.3 §10.3.13) in the paragraph: the control it names, + /// with its text, as a content control. Inside a field or a frame's text it is nothing. + fn place_control(&mut self, a: &Attrs) { + if self.field.is_some() { + return; + } + if let Some((mut control, text)) = get(a, "draw:control").and_then(|id| self.forms.controls.get(id)).cloned() { + // Its size (§19.539, §19.571). + let len = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite() && *v > 0.0).map(|v| v.min(1584.0)); + if let (Some(w), Some(h)) = (len("svg:width"), len("svg:height")) { + control.form_size = Some([w, h]); + // Where it stands on the line (§20.387), as an in-line frame does. + let g = self.styles.shape_graphic(get(a, "draw:style-name")); + let raise = super::frames::place(a, w, h, &g, self.text_size()).y; + control.form_raise = (raise.is_finite() && raise != 0.0).then_some(raise); + } + let fmt = self.cur_fmt(); + self.push_inline(Inline::Control { control: Box::new(control), text, fmt }); + } + } + + /// Part of a frame: its picture, embedded object, description or contour. + fn frame_part(&mut self, q: &str, a: &Attrs) { + match local(q) { + "image" => { + // A chart's replacement picture (what readers that can't draw it show): kept with + // the chart as read, written back with it while it is unchanged. + if let Some(href) = get(a, "xlink:href") + && let Some(fr) = &mut self.frame + && fr.5.is_some() + && let Some(o) = fr.6.as_mut().filter(|o| o.replacement.is_none()) + { + o.replacement = (self.zip)(href.trim_start_matches("./")).map(std::sync::Arc::new); + return; + } + if let Some(href) = get(a, "xlink:href") + && let Some(fr) = &mut self.frame + && fr.2.is_none() + && fr.4.is_none() + && fr.5.is_none() + { + let path = href.trim_start_matches("./"); + // A frame may offer one picture in several formats, in order of preference + // (LibreOffice writes its own `.svm` metafile before a PNG): the first + // WordCraft can read. An object's replacement is the first, kept whatever it + // is, also when the object is still to come. + if let Some(d) = (self.zip)(path) { + let readable = drawing::readable_picture(&d); + match &mut fr.6 { + Some(o) if o.replacement.is_none() => o.replacement = Some(std::sync::Arc::new(d.clone())), + Some(_) => {} + None if !readable => { + self.frame_unread.get_or_insert_with(|| d.clone()); + } + None => {} + } + if readable { + fr.2 = Some(d); + } + } + } + } + // An embedded object (ODF 1.3 §10.4.3): a formula sub-document's `content.xml` holds + // a `math:math` tree (#3099), a chart's an `office:chart`. Other object kinds (and + // charts WordCraft can't edit) fall back to the frame's `draw:image` replacement + // preview, read at `end` below. + "object" | "object-ole" => { + if let Some(href) = get(a, "xlink:href") { + self.read_object(href, local(q) == "object-ole"); + } + } + "desc" | "title" if self.frame.is_some() => self.in_desc = true, + // The outline a frame's tight or through wrapping follows. + "contour-polygon" if self.frame.is_some() => { + if let Some(f) = &mut self.frame_float { + f.wrap_polygon = super::frames::contour_polygon(a); + } + } + _ => {} + } + } + + /// The open frame's object at `href`: its formula, or else its chart, or else the object + /// itself, kept whole (an OLE object's file, when `ole`). + fn read_object(&mut self, href: &str, ole: bool) { + if let Some(fr) = &mut self.frame + && fr.4.is_none() + && fr.5.is_none() + && fr.6.is_none() + { + if !ole { + let path = format!("{}/content.xml", href.trim_start_matches("./").trim_end_matches('/')); + if let Some(xml) = (self.zip)(&path) { + fr.4 = read_formula(&xml); + if fr.4.is_none() { + fr.5 = super::chart::read_chart(&xml); + } + } + } + // A chart is kept whole too, to be written back as it was while it is unchanged. + if fr.4.is_none() { + fr.6 = super::objects::gather(href, ole, self.entries, self.manifest, &mut *self.zip); + // A replacement picture read before the object: the first the frame offered. + if let Some(o) = &mut fr.6 { + o.replacement = self.frame_unread.take().or_else(|| fr.2.clone()).map(std::sync::Arc::new); + } + } + } + } + + fn start_inline(&mut self, q: &str, a: &Attrs) -> bool { + // Inside a field everything is its shown text. + if self.field.is_some() { + return true; + } + // A citation carries its source: the document's (the first by its identifier). + if q == "text:bibliography-mark" + && let Some(s) = super::fields::bib_source(a) + && self.sources.len() < MAX_SOURCES + && !self.sources.iter().any(|o| o.tag == s.tag) + { + self.sources.push(s); + } + // A variable given a value (§7.4.5 `text:variable-set`): a SET field, then, unless it is + // hidden, a REF showing the value. + if q == "text:variable-set" + && self.frame.is_none() + && let Some(set) = super::fields::set_instr(a) + { + let fmt = self.cur_fmt(); + self.push_inline(Inline::Field { instr: set, result: String::new(), fmt, locked: false }); + if get(a, "text:display").map(str::trim) == Some("none") { + self.skip = 1; + return true; + } + } + if q.starts_with("text:") + && self.frame.is_none() + && let Some(instr) = super::fields::instr_of(local(q), a, |n| self.styles.date_picture(n).map(String::from)) + { + let anchor = (local(q) == "sequence") + .then(|| get(a, "text:ref-name").map(str::trim).filter(|n| !n.is_empty() && n.len() <= 1024).map(String::from)) + .flatten(); + let fixed = get(a, "text:fixed").is_some_and(|v| v.trim() == "true"); + self.field = Some(OpenField { instr, result: String::new(), name: q.to_string(), depth: self.depth, fmt: self.cur_fmt(), anchor, fixed }); + return true; + } + match local(q) { + "span" => { + let base = self.cur_fmt(); + // A size in percent is of the size it is in: its heading's, where nothing nearer + // states one. + let sized = Fmt { size: base.size.or(self.para_sizes.last().and_then(|(size, _)| *size)), ..base.clone() }; + let fmt = get(a, "text:style-name").map_or_else( + || base.clone(), + |s| { + let mut fmt = self.styles.span_fmt(s, &sized); + // That size is only for working out a percent: saying nothing of size, it has none. + if base.size.is_none() && self.styles.span_fmt(s, &base).size.is_none() { + fmt.size = None; + } + fmt + }, + ); + self.fmt.push(fmt); + } + "a" => { + let mut fmt = self.cur_fmt(); + if let Some(href) = get(a, "xlink:href").filter(|href| !href.is_empty()) { + fmt.link = model::kept_link(href); + } + if let Some(style) = get(a, "text:style-name") { + fmt = self.styles.text_fmt(style, &fmt); + } + self.fmt.push(fmt); + } + // Outside a paragraph `push_text` drops spaces, so they cost no budget. + "s" if self.para.is_some() => { + let count = get(a, "text:c").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(1, 1000); + let want = count.saturating_sub(1); + let extra = want.min(self.spaces_left); + if extra < want { + model::dropped(model::Dropped::Spaces); + } + self.spaces_left -= extra; + self.push_text(&" ".repeat(1 + extra)); + } + "tab" => self.push_text("\t"), + "line-break" => self.push_text("\n"), + // A bookmark (§6.2.1): a point, or the start or end of the text it covers. + // A reference mark (§7.6) is a bookmark too: references to it are Word's to bookmarks. + // An index entry over text (§8.1.11–8.1.12): an `XE` field at its start, the entry the + // marked text (in one paragraph). + "alphabetical-index-mark-start" => { + if let (Some(id), Some(p)) = (get(a, "text:id"), &self.para) + && self.index_marks.len() < MAX_OPEN_MARKS + { + let keys = (get(a, "text:key1").map(String::from), get(a, "text:key2").map(String::from)); + // Where it starts: after the last inline, or inside the text run the marked + // text will join. + let at = match p.inlines.last() { + Some(Inline::Text(t, _)) => (p.inlines.len() - 1, t.len()), + _ => (p.inlines.len(), 0), + }; + self.index_marks.push((id.to_string(), keys, at)); + } + } + "alphabetical-index-mark-end" => { + if let Some(at) = get(a, "text:id").and_then(|id| self.index_marks.iter().position(|(i, ..)| i == id)) { + let (_, (k1, k2), (from, off)) = self.index_marks.remove(at); + if let Some(p) = &mut self.para { + // The marked text: from the offset in the first run on. + let text: String = p + .inlines + .iter() + .enumerate() + .skip(from) + .filter_map(|(k, i)| match i { + Inline::Text(t, _) if k == from => t.get(off..), + Inline::Text(t, _) => Some(t.as_str()), + _ => None, + }) + .collect(); + if let Some(instr) = super::fields::xe_instr(k1.as_deref(), k2.as_deref(), &text) { + let field = Inline::Field { instr, result: String::new(), fmt: Fmt::default(), locked: false }; + // Inside a run: split it there. + let split = match p.inlines.get_mut(from) { + Some(Inline::Text(t, f)) if off > 0 && t.is_char_boundary(off) && off < t.len() => { + let rest = t.split_off(off); + Some(Inline::Text(rest, f.clone())) + } + _ => None, + }; + match split { + Some(rest) => { + p.inlines.insert(from + 1, rest); + p.inlines.insert(from + 1, field); + } + None if off > 0 => p.inlines.insert((from + 1).min(p.inlines.len()), field), + None => p.inlines.insert(from.min(p.inlines.len()), field), + } + } + } + } + } + "bookmark" | "bookmark-start" | "bookmark-end" | "reference-mark" | "reference-mark-start" | "reference-mark-end" => { + if let Some(name) = get(a, "text:name").filter(|n| !n.is_empty()) { + let name = name.to_string(); + self.push_inline(if local(q).ends_with("-end") { Inline::AnchorEnd(name) } else { Inline::Anchor(name) }); + } + } + _ => return false, + } + true + } + + /// A list, list item or list header starting: true when handled. + fn start_list_part(&mut self, q: &str, a: &Attrs) -> bool { + if !q.starts_with("text:") { + return false; + } + match local(q) { + "list" => { + self.end_para(); + if self.lists.is_empty() { + self.start_list_id(a); + } + if self.lists.len() < 64 { + self.lists.push((get(a, "text:style-name").map(String::from), false)); + self.item_used.push(false); + } else { + self.skip = 1; + } + } + "list-item" | "list-header" => { + // An item restarting the numbering begins a list of its own from there. + if let Some(v) = get(a, "text:start-value").and_then(|v| v.trim().parse::().ok()) { + self.list_id = self.new_list_id(); + self.last_list = Some(self.list_id); + self.item_start = Some(v.min(32_767)); + } + if let Some(l) = self.lists.last_mut() { + l.1 = true; + } + if let Some(u) = self.item_used.last_mut() { + *u = false; + } + } + _ => return false, + } + true + } + + /// A cell's background picture from its package path, `style:repeat` (§20.349: no-repeat, + /// repeat, stretch; unstated, repeat) and `style:position` (§20.335: two of left, center, + /// right, top, bottom; unstated, center). + fn cell_picture(&mut self, (href, repeat, position): (String, String, String)) -> Option> { + let data = (self.zip)(href.trim_start_matches("./"))?; + let img = model::make_img(data, None, None, "")?; + let repeat = match repeat.as_str() { + "no-repeat" => craft_wordmodel::props::PictureRepeat::None, + "stretch" => craft_wordmodel::props::PictureRepeat::Stretch, + _ => craft_wordmodel::props::PictureRepeat::Tile, + }; + let mut align = [0.5, 0.5]; + for word in position.split_whitespace() { + match word { + "left" => align[0] = 0.0, + "right" => align[0] = 1.0, + "top" => align[1] = 0.0, + "bottom" => align[1] = 1.0, + _ => {} + } + } + Some(Box::new(model::CellPic { img, repeat, align })) + } + + /// A skipped element ends; the end of a custom shape's geometry works out its path and + /// keeps the geometry as read. + fn end_skipped(&mut self) { + self.skip -= 1; + if self.skip > 0 { + return; + } + // Not a custom shape's geometry (a covered cell, say): nothing more. + if self.geometry.is_none() && self.geometry_xml.is_none() { + return; + } + let xml = self.geometry_xml.take(); + let keep = |x: Option| x.filter(|x| x.len() < MAX_GEOMETRY_XML).map(|x| std::sync::Arc::new(format!("{x}"))); + if let Some(g) = self.geometry.take() + && let Some(f) = g.freeform() + { + self.shape_kind = Some(craft_wordmodel::para::ShapeKind::Freeform); + self.shape_path = Some(std::sync::Arc::new(f)); + self.shape_geometry = keep(xml); + } else if self.shape_kind == Some(craft_wordmodel::para::ShapeKind::Rectangle) { + // A preset WordCraft draws as a rectangle: its geometry kept as read. + self.shape_geometry = keep(xml); + } + } + + fn end(&mut self, q: &str) { + self.depth = self.depth.saturating_sub(1); + if self.end_note_or_field(q) { + return; + } + if q == "office:text" { + self.end_field(); + if self.note.is_some() { + self.end_note(); + } + if self.del.is_some() { + // Unclosed deleted content (hostile): drop it. + self.close_deletion(); + } + self.in_tc = false; + self.region = None; + self.info_at = None; + self.end_para(); + self.in_text = false; + return; + } + if self.skip > 0 { + self.end_skipped(); + return; + } + if !self.in_text { + return; + } + if let Some(d) = self.info_at { + if self.depth < d { + self.info_at = None; + } + self.info_field = None; + return; + } + if self.in_tc { + match local(q) { + "deletion" if self.del.is_some() => { + self.close_deletion(); + return; + } + "changed-region" => { + // A deletion left open (hostile) ends with its region. + self.close_deletion(); + if let Some((id, r)) = self.region.take() + && r.kind.is_some() + { + // The first region with an id wins. + self.regions.entry(id).or_insert(r); + } + return; + } + "tracked-changes" => { + self.close_deletion(); + self.in_tc = false; + self.region = None; + return; + } + _ if self.del.is_some() => {} + _ => return, + } + } + match local(q) { + "p" | "h" if !self.in_desc => self.end_para(), + "span" | "a" if self.fmt.len() > usize::from(self.para.is_some()) => { + self.fmt.pop(); + } + "list" => { + self.end_para(); + self.lists.pop(); + self.item_used.pop(); + } + "list-item" | "list-header" => { + self.end_para(); + if let Some(l) = self.lists.last_mut() { + l.1 = false; + } + } + "table-cell" => { + self.end_para(); + self.cell_tops.pop(); + let mut blocks = if self.containers.len() > 1 { self.containers.pop().unwrap_or_default() } else { Vec::new() }; + if let Some(f) = self.formulas.pop().flatten() { + super::cell_formula::into_field(&mut blocks, f); + } + (self.lists, self.item_used, self.list_id) = self.lists_saved.pop().unwrap_or_default(); + let mut cell = self.cells.pop().unwrap_or_default(); + cell.blocks = blocks; + if let Some(t) = self.tables.last_mut() + && t.row.len() < 63 + { + t.row.push(cell); + } + } + "table-row" => { + if let Some(t) = self.tables.last_mut() { + let r = std::mem::take(&mut t.row); + if !r.is_empty() && t.rows.len() < craft_wordmodel::table::MAX_ROWS { + t.rows.push(r); + } + } + } + "table-header-rows" => { + if let Some(t) = self.tables.last_mut() { + for r in &mut t.rows { + for c in r { + c.header = true; + } + } + } + } + "table" if q.starts_with("table:") => { + self.end_para(); + if let Some(t) = self.tables.pop() + && !t.rows.is_empty() + { + self.container().push(FBlock::Table(t.finish())); + } + } + "desc" | "title" => self.in_desc = false, + "section" if q.starts_with("text:") && self.cols_at == Some(self.depth + 1) => { + self.end_para(); + self.cols_at = None; + let next = SectionProps { start: SectionStart::Continuous, page_num_start: None, ..self.page.clone() }; + self.break_section(next); + } + "frame" => self.end_frame(), + name if q.starts_with("text:") && is_index(name) && self.index.as_ref().is_some_and(|ix| ix.depth == self.depth + 1) => self.end_index(), + "index-title" if q.starts_with("text:") => self.index_title(true), + "g" if self.group_at == Some(self.depth + 1) => { + self.group_at = None; + self.end_group(); + } + _ => {} + } + } +} + +/// The most markup a custom shape's kept geometry may take (see `ShapeExtra::odf_geometry`). +const MAX_GEOMETRY_XML: usize = 256 * 1024; + +/// Element `q` with attributes `a` as markup: empty (``) or an opening tag. +fn element_xml(q: &str, a: &Attrs, empty: bool) -> String { + let attrs: String = a.iter().map(|(k, v)| format!(" {k}=\"{}\"", super::writer::x(v))).collect(); + format!("<{q}{attrs}{}>", if empty { "/" } else { "" }) +} + +/// A body paragraph whose markup is being kept: where its element starts in the markup, its +/// depth, whether everything in it so far can be kept, and how many paragraphs were finished +/// before it. +struct KeptOpen { + from: usize, + depth: usize, + clean: bool, + done: usize, + /// The comments in a paragraph being kept (see [`KeptComments`]). + comments: KeptComments, + /// The markup of drawings read before it that join it, kept before its own. + prefix: String, +} + +/// The comments (§14.1 `office:annotation`), tracked changes (§5.5.7), index and reference +/// marks over text in a paragraph being kept, and the text boxes it shows. Begun and ended in +/// it, they are its own; one reaching into another paragraph (a comment ending there, a +/// change, a box continuing in another's) ties it to that one, as a link of its [`Group`] +/// (`model::Group`): the paragraphs are kept as read together, or not at all. +#[derive(Default)] +struct KeptComments { + /// The keys of those begun, and of those ended. + opened: Vec, + ended: Vec, + /// What ties the paragraph to others: `(key, needed)` (see `model::Group::links`). + links: Vec<(String, bool)>, + /// The comment or drawing being read: its depth and how many paragraphs were finished + /// before it. + open: Option<(usize, usize)>, + /// The comments' and drawings' own paragraphs, which aren't the paragraph's. + paras: usize, +} + +/// Most pairs and links a paragraph kept as read has (hostile input). +const MAX_PAIRS: usize = 64; + +impl KeptComments { + /// Element `q` (`a` its attributes, at `depth`) starting in the paragraph, `done` + /// paragraphs finished so far: false when the paragraph can't be kept for it. + fn start(&mut self, q: &str, a: &Attrs, depth: usize, done: usize) -> bool { + if self.opened.len() + self.links.len() >= MAX_PAIRS { + return false; + } + let comment = |n: &str| format!("\u{3}{n}"); + match q { + // A comment in a drawing's text (its own story): not kept. + "office:annotation" if self.open.is_some() => false, + "office:annotation" => { + // A reply to one in another paragraph needs it. + if let Some(p) = get(a, "loext:parent-name").map(comment) + && !self.opened.contains(&p) + { + self.links.push((p, true)); + } + if let Some(n) = get(a, "office:name").map(comment) { + self.opened.push(n); + } + self.open = Some((depth, done)); + true + } + "office:annotation-end" => match get(a, "office:name").map(comment) { + Some(n) => { + self.end_or_need(n); + true + } + None => false, + }, + "text:change-start" => match get(a, "text:change-id") { + Some(i) => { + self.opened.push(format!("\u{0}{i}")); + true + } + None => false, + }, + "text:change-end" => match get(a, "text:change-id") { + Some(i) => { + self.end_or_need(format!("\u{0}{i}")); + true + } + None => false, + }, + "text:change" => get(a, "text:change-id").is_some(), + // An index or reference mark over text (§6.6.2, §8.1.10). + q if mark_pair(q).is_some() => { + let Some((kind, attr, start)) = mark_pair(q) else { return false }; + let Some(key) = get(a, attr).map(|k| format!("\u{2}{kind}\u{2}{k}")) else { return false }; + if start { + self.opened.push(key); + } else { + self.end_or_need(key); + } + true + } + // A text box continuing in another (§19.112 `draw:chain-next-name`) needs it. + "draw:text-box" => { + if let Some(next) = get(a, "draw:chain-next-name").map(str::trim).filter(|n| !n.is_empty()) { + self.links.push((format!("\u{4}{next}"), true)); + } + true + } + // A drawing or a note: the paragraphs of its text (a text box's, a shape's, the + // note's) are its story's. A named frame is there for a box continuing in it. + q if self.open.is_none() && (q == "text:note" || q.starts_with("draw:") && is_drawing(q)) => { + if q == "draw:frame" + && let Some(n) = get(a, "draw:name").map(str::trim).filter(|n| !n.is_empty()) + { + self.links.push((format!("\u{4}{n}"), false)); + } + self.open = Some((depth, done)); + true + } + _ => true, + } + } + + /// `key` ended: one begun in the paragraph, or one begun in another, needed. + fn end_or_need(&mut self, key: String) { + if self.opened.contains(&key) { + self.ended.push(key); + } else { + self.links.push((key, true)); + } + } + + /// Whether the comment or drawing being read is the element ending at `depth`. + fn ends_at(&self, depth: usize) -> bool { + self.open.is_some_and(|(d, _)| d == depth) + } + + /// The comment being read ended, `done` paragraphs finished so far. + fn end(&mut self, done: usize) { + if let Some((_, before)) = self.open.take() { + self.paras += done.saturating_sub(before); + } + } + + /// Whether nothing is left open in it (a comment being read); and its links: what it begun + /// and didn't end needs the paragraph ending it, and what it begun is there for another + /// to need (a reply, an end). + fn closed(&self) -> bool { + self.open.is_none() + } + + /// The paragraph's [`Group`](model::Group) links, when it has any. + fn group(&self) -> Option> { + let mut links = self.links.clone(); + for k in &self.opened { + links.push((k.clone(), !self.ended.contains(k))); + } + (!links.is_empty()).then(|| Box::new(model::Group { links, id: None })) + } +} + +/// Whether markup holding element `q` can be kept as read. Pictures, shapes and text boxes can +/// (their text is a story of its own, kept while it is as read: see `KeptComments`), and +/// embedded objects (a formula's text and a chart's data are the paragraph's, an object's files +/// are kept with it), plugins and floating frames; form controls (named afresh in the forms) +/// can't, nor +/// marks that pair up with others elsewhere by ids the writer gives (comments and tracked +/// changes are paired apart). Bookmarks can: their names are kept, so either end may be +/// written as read and the other from the model. +fn keepable(q: &str) -> bool { + !(q == "draw:control" + || q.starts_with("office:annotation") + || q.starts_with("text:change") + || q.starts_with("form:") + || (q.ends_with("-start") || q.ends_with("-end")) && !q.starts_with("text:bookmark-")) +} + +/// `blocks` hidden: their text and paragraph marks, and those in their tables' cells. +fn hide(blocks: &mut [FBlock], depth: usize) { + let hidden = |f: &mut Fmt| f.extra.get_or_insert_with(Default::default).hidden = Some(true); + for b in blocks { + match b { + FBlock::Para(p) => { + for i in &mut p.inlines { + match i { + Inline::Text(_, f) + | Inline::Control { fmt: f, .. } + | Inline::Ruby { fmt: f, .. } + | Inline::Field { fmt: f, .. } + | Inline::Note { fmt: f, .. } => { + hidden(f); + } + _ => {} + } + } + p.mark.get_or_insert_with(Default::default).hidden = Some(true); + (p.kept, p.index) = (None, None); + } + FBlock::Table(t) if depth < model::MAX_DEPTH => { + t.kept = None; + for c in t.rows.iter_mut().flatten() { + hide(&mut c.blocks, depth + 1); + } + } + FBlock::Table(_) => {} + } + } +} + +/// For an index or reference mark's start or end (`q`): its kind, the attribute naming it, and +/// whether it starts. +fn mark_pair(q: &str) -> Option<(&str, &'static str, bool)> { + let (kind, start) = if let Some(k) = q.strip_suffix("-start") { (k, true) } else { (q.strip_suffix("-end")?, false) }; + match kind { + "text:toc-mark" | "text:user-index-mark" | "text:alphabetical-index-mark" => Some((kind, "text:id", start)), + "text:reference-mark" => Some((kind, "text:name", start)), + _ => None, + } +} + +/// Whether `q` is a drawing that can hold text: a frame, a group or a shape. +fn is_drawing(q: &str) -> bool { + matches!( + local(q), + "frame" + | "g" + | "rect" + | "ellipse" + | "circle" + | "line" + | "custom-shape" + | "polygon" + | "polyline" + | "path" + | "regular-polygon" + | "connector" + | "caption" + | "measure" + ) +} + +/// The section wrappers kept around top-level block `b` (a paragraph or a table), made empty +/// when it has none yet. +fn block_wrap(b: &mut FBlock) -> &mut (Vec>, u32) { + match b { + FBlock::Para(p) => p.wrap.get_or_insert_with(Default::default), + FBlock::Table(t) => t.wrap.get_or_insert_with(Default::default), + } +} diff --git a/crates/odf/src/text/body/drawing.rs b/crates/odf/src/text/body/drawing.rs new file mode 100644 index 000000000..28cb8cb12 --- /dev/null +++ b/crates/odf/src/text/body/drawing.rs @@ -0,0 +1,474 @@ +//! Frames, drawing shapes and groups in the text (ODF 1.3 §10.3–10.4): pictures, formula objects, +//! text boxes, shapes and groups, each read into the paragraph (or the group) it is in. + +use super::{Attrs, Body, Ev, FBlock, Inline, events, get, length, local, make_img, model}; + +/// An embedded ODF formula sub-document's `content.xml`, read for its `math:math` element +/// (its root, ODF 1.3 Part 3 §2.2.9 B; some writers wrap it in an `office:math`, which isn't +/// it): the first MathML element named `math` in document order, with its +/// subtree, passed to [`craft_math::mathml::read`]. `None` when there's no such element, it holds +/// nothing, or the XML is malformed. Nesting and element count are bounded like other XML here. +pub(in crate::text) fn read_formula(xml: &[u8]) -> Option<(String, bool)> { + use craft_math::mathml::{El, Item, MAX_NODES, local}; + let mut stack: Vec = Vec::new(); + let mut math: Option = None; + let mut over = 0usize; + let mut count = 0usize; + events(xml, |ev| { + if math.is_some() { + return; + } + count += 1; + if count > MAX_NODES * 4 { + math = stack.drain(..).next(); + return; + } + match ev { + Ev::Start(q, a) => { + if stack.is_empty() && (local(&q) != "math" || q.starts_with("office:")) { + return; + } + if stack.len() < craft_wordmodel::math::MAX_DEPTH { + stack.push(El { name: q, attrs: a, children: Vec::new() }); + } else { + over += 1; + } + } + Ev::End(_) => { + if over > 0 { + over -= 1; + } else if stack.len() > 1 { + if let Some(done) = stack.pop() + && let Some(parent) = stack.last_mut() + { + parent.children.push(Item::El(done)); + } + } else if stack.len() == 1 { + math = stack.pop(); + } + } + Ev::Text(t) => { + if over == 0 + && let Some(top) = stack.last_mut() + { + top.children.push(Item::Text(t)); + } + } + } + }); + craft_math::mathml::read(&math?) +} + +/// A frame being read: its width and height, picture bytes, alt text and the equation read from +/// an embedded formula object. +/// A group being read (ODF 1.3 §10.3.15 `draw:g`): its anchor and page, its style, and its members +/// so far. +pub(in crate::text) struct OpenGroup { + pub(in crate::text) anchor: Option, + /// Its `text:anchor-page-number`. + pub(in crate::text) page: Option, + pub(in crate::text) style: Option, + pub(in crate::text) members: Vec, +} + +/// A picture's crop as fractions of its size (left, top, right, bottom) from its `fo:clip` +/// (top, right, bottom, left, in lengths of its original size: its pixels at 96 per inch). No crop +/// when the picture's size can't be read or the crop would leave nothing. +pub(in crate::text) fn crop_fractions([t, r, b, l]: [f32; 4], data: &[u8]) -> [f32; 4] { + let Some((pw, ph)) = model::image_px(data).filter(|(w, h)| *w > 0 && *h > 0) else { return [0.0; 4] }; + let (w, h) = (pw as f32 * 0.75, ph as f32 * 0.75); + let c = [l / w, t / h, r / w, b / h].map(|v| if v.is_finite() { v.clamp(0.0, 0.99) } else { 0.0 }); + if c[0] + c[2] < 0.99 && c[1] + c[3] < 0.99 { c } else { [0.0; 4] } +} + +/// Largest size or offset read in a group, points (22 inches). +pub(in crate::text) const MAX_LEN: f32 = 1584.0; + +/// Most shape ids connectors can glue to. +const MAX_CONN_IDS: usize = 10_000; + +impl Body<'_> { + /// End a frame or drawing shape: its picture, equation, text box or shape goes into the + /// paragraph. + pub(in crate::text) fn end_frame(&mut self) { + if let Some((w, h, img, alt, eq, chart, object)) = self.frame.take() { + let boxed = self.frame_box.take(); + let mut float = self.frame_float.take(); + if boxed.is_some() + && self.shape_kind.is_some() + && let Some(f) = &mut float + { + let (h, v) = std::mem::take(&mut self.shape_flip); + f.flip_h |= h; + f.flip_v ^= v; + } + // Kept when it floats, or turns or mirrors the object in line. + let float = float + .filter(|f| f.wrap != craft_wordmodel::para::Wrap::Inline || crate::text::frames::turned(f) || f.flip_h || f.flip_v || f.y != 0.0); + if let Some(blocks) = &boxed + && self.take_watermark(blocks) + { + self.frame = None; + return; + } + // In line with no stated size, a picture comes at its own: fitted to the text, as + // other formats' pictures are. A stated size is the file's. + let room = self.sec.text_width(); + let says = |v: Option| v.is_some_and(|v| v.is_finite() && v > 0.0); + // (Still in line when raised, turned or mirrored.) + let in_line = float.as_ref().is_none_or(|f| f.wrap == craft_wordmodel::para::Wrap::Inline); + let fit = (!says(w) && !says(h) && in_line && room.is_finite() && room > 0.0).then_some(room); + let inline = match (boxed, eq) { + (Some(blocks), _) => { + let mut look = std::mem::take(&mut self.frame_look); + // Without a height it grows from nothing to its text. + let kind = self.shape_kind.take().unwrap_or(craft_wordmodel::para::ShapeKind::TextBox); + // A custom shape's geometry as read, kept with the path worked out from it. + let geometry = self.shape_geometry.take(); + if kind == craft_wordmodel::para::ShapeKind::Freeform || kind == craft_wordmodel::para::ShapeKind::Rectangle { + look.geometry = geometry; + } + // A shape whose paragraphs are all empty (LibreOffice writes `` in + // each) holds no text. + let empty = blocks.iter().all(|b| matches!(b, FBlock::Para(p) if p.inlines.is_empty())); + let blocks = if kind != craft_wordmodel::para::ShapeKind::TextBox && empty { Vec::new() } else { blocks }; + let path = self.shape_path.take().filter(|_| kind == craft_wordmodel::para::ShapeKind::Freeform); + let (w, h) = (w.unwrap_or(72.0), h.unwrap_or(1.0)); + self.note_place(kind, (w, h), &look, float.as_ref()); + Some(Inline::Shape { kind, w, h, float: float.map(Box::new), blocks, look, path }) + } + (None, Some((linear, display))) => Some(Inline::Equation { linear, display }), + (None, None) if let Some(spec) = chart => Some(Inline::Chart(model::Chart { + spec: std::sync::Arc::new(spec), + w: w.unwrap_or(360.0), + h: h.unwrap_or(216.0), + alt: alt.trim().to_string(), + float: float.map(Box::new), + source: object.map(std::sync::Arc::new), + })), + // An object kept whole: shown by its replacement picture when WordCraft can show + // it, else by a placeholder. + (None, None) if let Some(mut o) = object => { + if o.replacement.is_none() { + o.replacement = img.clone().map(std::sync::Arc::new); + } + let shown = img + .and_then(|data| picture(data, w, h, alt.trim())) + .or_else(|| crate::text::objects::placeholder(&o.mime).and_then(|p| make_img(p, w, h, alt.trim()))); + shown.map(|mut img| { + fit_to(&mut img, fit); + img.float = float.map(Box::new); + img.object = Some(std::sync::Arc::new(o)); + Inline::Image(img) + }) + } + (None, None) => img.and_then(|data| picture(data, w, h, alt.trim())).map(|mut img| { + fit_to(&mut img, fit); + img.float = float.map(Box::new); + if let Some(clip) = self.frame_clip.take() { + img.crop = crop_fractions(clip, &img.data); + } + Inline::Image(img) + }), + }; + if let Some(inline) = inline { + // A group's member goes into the group, at its place in it. + if let Some(((x, y), g)) = self.member_at.take().zip(self.group.as_mut().filter(|_| self.note.is_none())) { + if g.members.len() < craft_wordmodel::para::MAX_GROUP_CHILDREN { + g.members.push(model::Member { x, y, inline }); + } + } else { + self.push_inline(inline); + } + } + } + self.frame = None; + } + + /// The shape ending is a watermark (see [`Self::start_shape`]) holding `blocks`: the + /// document's watermark, from its text, font, colour, size and turn, and not a shape. + fn take_watermark(&mut self, blocks: &[FBlock]) -> bool { + let Some(((w, h), rot, color, see_through, true)) = self.watermark_shape.take() else { return false }; + let mut text = String::new(); + let mut font = None; + for b in blocks { + if let FBlock::Para(p) = b { + for i in &p.inlines { + if let Inline::Text(t, f) = i { + text.push_str(t); + font = font.or_else(|| f.font.as_deref().map(str::to_string)); + } + } + } + } + let text: String = text.trim().chars().take(200).collect(); + if text.is_empty() { + return false; + } + let turned = rot.rem_euclid(360.0); + let base = craft_wordmodel::Watermark::default(); + self.watermark = Some(craft_wordmodel::Watermark { + text, + font: font.unwrap_or(base.font), + color: color.unwrap_or(base.color), + diagonal: turned > 1.0 && turned < 359.0, + semitransparent: see_through, + size: Some([w, h]).filter(|[w, h]| w.is_finite() && h.is_finite() && *w > 0.0 && *h > 0.0), + }); + true + } + + /// A shape's drawing id (its `xml:id` or `draw:id`) and, for a connector, its glued ends (§19.228 + /// `draw:start-shape`, §19.229 `draw:start-glue-point`…), their glue points as written until + /// the paragraph ends (see [`Self::glue_connectors`]). `None` for a shape with neither. + fn conn_of(&mut self, a: &Attrs) -> Option { + let mut id_of = |name: &str| -> u32 { + let next = u32::try_from(self.conn_ids.len()).unwrap_or(u32::MAX).saturating_add(1); + match self.conn_ids.get(name) { + Some(id) => *id, + None if self.conn_ids.len() < MAX_CONN_IDS => { + self.conn_ids.insert(name.to_string(), next); + next + } + None => 0, + } + }; + let id = get(a, "xml:id").or_else(|| get(a, "draw:id")).map(str::trim).filter(|n| !n.is_empty()).map_or(0, &mut id_of); + let mut end = |shape: &str, glue: &str| { + let target = get(a, shape).map(str::trim).filter(|n| !n.is_empty()).map(&mut id_of).filter(|id| *id != 0)?; + let site = get(a, glue).and_then(|g| g.trim().parse::().ok()).filter(|g| *g <= craft_wordmodel::connector::MAX_SITE)?; + Some(craft_wordmodel::connector::ConnEnd { id: target, site }) + }; + let (start, finish) = (end("draw:start-shape", "draw:start-glue-point"), end("draw:end-shape", "draw:end-glue-point")); + (id != 0 || start.is_some() || finish.is_some()).then_some(model::Conn { id, start, end: finish }) + } + + /// Where a shape with a drawing id (`look`) sits, for connectors glued to it. + pub(in crate::text) fn note_place( + &mut self, + kind: craft_wordmodel::para::ShapeKind, + (w, h): (f32, f32), + look: &model::BoxLook, + float: Option<&craft_wordmodel::para::Float>, + ) { + let Some(c) = look.conn.as_deref().filter(|c| c.id != 0) else { return }; + if self.shape_places.len() >= MAX_CONN_IDS { + return; + } + // A group's member (placed in the group, not on the paragraph) or a shape in line: known + // by its kind only, never in the paragraph's coordinates. + let seq = if self.member_at.is_some() { usize::MAX } else { self.para_seq }; + let f = float.cloned().unwrap_or_default(); + self.shape_places.insert(c.id, (kind, [f.x, f.y, w, h], f, if float.is_some() { seq } else { usize::MAX })); + } + + /// The connectors of paragraph `p` glued to shapes (see [`Self::conn_of`]): each end's glue + /// point made WordCraft's site of the shape it is glued to (an end glued to a shape not read, + /// or to a glue point of the shape's own, is unglued); and a connector glued at both ends to + /// shapes anchored in this paragraph, placed alike, runs between their sites. (A connector's + /// own coordinates are in LibreOffice's drawing layer, not its anchor's.) + pub(in crate::text) fn glue_connectors(&self, p: &mut model::Para) { + for i in &mut p.inlines { + self.glue_connector(i, 0); + } + } + + /// Connector `i` of a paragraph or a group (`depth` deep in groups) glued to shapes (see + /// [`Self::glue_connectors`]). + fn glue_connector(&self, i: &mut Inline, depth: usize) { + use crate::text::shapes::site_of_glue; + if let Inline::Group { members, .. } = i { + if depth < model::MAX_GROUP_DEPTH { + for m in members { + self.glue_connector(&mut m.inline, depth + 1); + } + } + return; + } + let Inline::Shape { kind, w, h, float, look, .. } = i else { return }; + let Some(c) = look.conn.as_deref_mut().filter(|_| kind.is_connector()) else { return }; + let resolve = |e: Option| { + let e = e?; + let place = self.shape_places.get(&e.id)?; + site_of_glue(place.0, e.site).map(|site| (craft_wordmodel::connector::ConnEnd { id: e.id, site }, place)) + }; + let (start, end) = (resolve(c.start), resolve(c.end)); + (c.start, c.end) = (start.map(|(e, _)| e), end.map(|(e, _)| e)); + let (Some((s, sp)), Some((e, ep)), Some(f)) = (start, end, float.as_deref_mut()) else { return }; + let same = |pl: &(craft_wordmodel::para::ShapeKind, [f32; 4], craft_wordmodel::para::Float, usize)| { + pl.3 == self.para_seq && pl.2.h_rel == f.h_rel && pl.2.v_rel == f.v_rel + }; + if !(same(sp) && same(ep)) { + return; + } + let point = |pl: &(craft_wordmodel::para::ShapeKind, [f32; 4], craft_wordmodel::para::Float, usize), site| { + craft_wordmodel::connector::site_point(pl.0, pl.1, pl.2.spin(), site) + }; + let (Some((x1, y1)), Some((x2, y2))) = (point(sp, s.site), point(ep, e.site)) else { return }; + (f.x, f.y, f.flip_h, f.flip_v) = (x1.min(x2), y1.min(y2), x2 < x1, y2 < y1); + (*w, *h) = ((x2 - x1).abs().min(MAX_LEN), (y2 - y1).abs().min(MAX_LEN)); + } + + /// A frame or shape `w` × `h` (attributes `a`) begins: in a group (not in a shape's text), where + /// it sits in it, from its `svg:x`/`svg:y` or `draw:transform` (a line's from its ends, `line`). + pub(in crate::text) fn member_start(&mut self, a: &Attrs, w: f32, h: f32, line: Option<(Option, Option)>) { + self.member_rot = None; + if self.group.is_none() || self.note.is_some() { + return; + } + let len = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite()).map(|v| v.clamp(-MAX_LEN, MAX_LEN)); + let at = match (line, get(a, "draw:transform").and_then(|t| crate::text::frames::transform(t, w, h))) { + (Some((x, y)), _) => (x.unwrap_or(0.0), y.unwrap_or(0.0)), + (None, Some((x, y, rot))) => { + self.member_rot = Some(rot); + (x, y) + } + (None, None) => (len("svg:x").unwrap_or(0.0), len("svg:y").unwrap_or(0.0)), + }; + self.member_at = Some(at); + } + + /// The group being read ends: its members, placed from the corner of the box around them, + /// become one object anchored where the group is. + pub(in crate::text) fn end_group(&mut self) { + let Some(g) = self.group.take() else { return }; + let size = |i: &Inline| match i { + Inline::Shape { w, h, .. } => (*w, *h), + Inline::Image(img) => (img.w, img.h), + Inline::Group { w, h, .. } => (*w, *h), + _ => (0.0, 0.0), + }; + let fin = |v: f32| if v.is_finite() { v.clamp(-MAX_LEN, MAX_LEN) } else { 0.0 }; + let (mut x0, mut y0, mut x1, mut y1) = (f32::MAX, f32::MAX, f32::MIN, f32::MIN); + for m in &g.members { + let (w, h) = size(&m.inline); + (x0, y0) = (x0.min(fin(m.x)), y0.min(fin(m.y))); + (x1, y1) = (x1.max(fin(m.x + w)), y1.max(fin(m.y + h))); + } + if g.members.is_empty() { + return; + } + let members = g.members.into_iter().map(|m| model::Member { x: fin(m.x) - x0, y: fin(m.y) - y0, ..m }).collect(); + let graphic = self.styles.shape_graphic(g.style.as_deref()); + let mut float = + crate::text::frames::float_of(g.anchor.as_deref(), Some(&crate::text::writer::pt(x0)), Some(&crate::text::writer::pt(y0)), &graphic); + float.page = crate::text::frames::page_of(g.anchor.as_deref(), g.page.as_deref(), &graphic); + let float = (float.wrap != craft_wordmodel::para::Wrap::Inline).then(|| Box::new(float)); + let fit = |v: f32| if v.is_finite() { v.clamp(1.0, MAX_LEN) } else { 1.0 }; + self.push_inline(Inline::Group { w: fit(x1 - x0), h: fit(y1 - y0), float, members }); + } + + /// Begin a drawing shape (ODF 1.3 §10.3): its size, placement and look, and its text read + /// as a text box's. A custom shape's kind comes from its geometry, read inside it. + pub(in crate::text) fn start_shape(&mut self, q: &str, a: &Attrs) { + let mut kind = crate::text::shapes::element_kind(local(q)); + if kind.is_connector() { + kind = crate::text::shapes::connector_kind(get(a, "draw:type")); + } + let g = self.styles.shape_graphic(get(a, "draw:style-name")); + let (w, h, x, y, (flip_h, flip_v)) = crate::text::shapes::geometry(kind, a); + let mut float = if kind.is_open() { + // A line is placed by its ends. + let at = |v: Option| v.map(crate::text::writer::pt); + let mut f = crate::text::frames::float_of(get(a, "text:anchor-type"), at(x).as_deref(), at(y).as_deref(), &g); + f.z = crate::text::frames::z_of(a); + f.page = crate::text::frames::page_of(get(a, "text:anchor-type"), get(a, "text:anchor-page-number"), &g); + f + } else { + crate::text::frames::place(a, w, h, &g, self.text_size()) + }; + (float.flip_h, float.flip_v) = (flip_h, flip_v); + self.shape_flip = (false, false); + self.member_start(a, w, h, kind.is_open().then_some((x, y))); + if let Some(rot) = self.member_rot.take() { + float.rot = rot; + } + self.frame_look = g.shape_look(kind, &self.styles.defs); + self.frame_look.conn = self.conn_of(a).map(Box::new); + // Word's and LibreOffice's watermark: a shape of this name whose geometry follows its text. + self.watermark_shape = get(a, "draw:name").filter(|n| n.starts_with(crate::text::writer::WATERMARK_NAME)).map(|_| { + let (color, see_through) = g.watermark_look(); + ((w, h), float.rot, color, see_through, false) + }); + self.shape_path = crate::text::shapes::freeform(local(q), a, w, h).map(std::sync::Arc::new); + self.shape_kind = Some(kind); + let frame = (Some((Some(w), Some(h), None, String::new(), None, None, None)), Some(float), std::mem::take(&mut self.frame_look)); + self.start_note(a); + if let Some(n) = &mut self.note { + n.text_box = Some(frame); + } + } +} + +/// Whether WordCraft can show these bytes as a picture (see the function below). +pub(in crate::text) fn readable_picture(data: &[u8]) -> bool { + craft_metafile::is_metafile(data) || model::sniff_image(data).is_some() +} + +/// A picture from a package's bytes: a raster image, or a Windows metafile (WMF, EMF: LibreOffice +/// keeps clip art and logos so), sized `w` × `h` (points) when the frame says. +pub(in crate::text) fn picture(data: Vec, w: Option, h: Option, alt: &str) -> Option { + // What the frame states, if it could be a size (far past any page, kept from overflowing). + let stated = |v: Option| v.filter(|v| v.is_finite() && *v > 0.0).map(|v| v.min(1e6)); + let (w, h) = (stated(w), stated(h)); + let mut img = if craft_metafile::is_metafile(&data) { + // An EMF starts with its header record, type 1. + let ext = if data.get(..4) == Some(&[1, 0, 0, 0]) { "emf" } else { "wmf" }; + // Its own size, when the frame gives none: a placeable WMF's or an EMF's frame, points. + let natural = if w.is_some() && h.is_some() { None } else { metafile_size(&data) }; + model::sized_img(std::sync::Arc::new(data), ext, natural, w, h, alt) + } else { + make_img(data, w, h, alt)? + }; + // A size the frame states is the file's, however large (a picture across a whole page): only + // a picture with none is shrunk to a column. Stated on one side, the other follows the + // picture's proportions. + let (w, h) = match (w, h) { + (Some(w), Some(h)) => (w, h), + (Some(w), None) => (w, w * height_per_width(&img.data)), + (None, Some(h)) => (h / height_per_width(&img.data), h), + (None, None) => return Some(img), + }; + // Past what a document holds, it shrinks in proportion. + let (w, h) = match w.max(h) { + most if most <= 1584.0 => (w, h), + _ if w >= h => (1584.0, h * 1584.0 / w), + _ => (w * 1584.0 / h, 1584.0), + }; + (img.w, img.h) = (w.max(1.0), h.max(1.0)); + Some(img) +} + +/// `img` shrunk in proportion to `room` points wide, when it's given and the picture is wider. +fn fit_to(img: &mut model::Img, room: Option) { + if let Some(room) = room.filter(|room| img.w > *room) { + img.h = (img.h * room / img.w).max(1.0); + img.w = room.max(1.0); + } +} + +/// A picture's height for each point of its width, by its own size, before anything caps it (1 +/// when it doesn't say). +fn height_per_width(data: &[u8]) -> f32 { + let own = if craft_metafile::is_metafile(data) { metafile_size(data) } else { model::image_px(data).map(|(w, h)| (w as f32, h as f32)) }; + own.filter(|(w, h)| w.is_finite() && h.is_finite() && *w > 0.0 && *h > 0.0).map_or(1.0, |(w, h)| h / w) +} + +/// A metafile's own size, points, from its header: a placeable WMF's bounding box in its units +/// per inch, or an EMF's frame (`rclFrame`, 0.01 mm). `None` when the header doesn't say. +fn metafile_size(d: &[u8]) -> Option<(f32, f32)> { + let i16_at = |o: usize| d.get(o..o + 2).map(|b| f32::from(i16::from_le_bytes([b[0], b[1]]))); + let i32_at = |o: usize| d.get(o..o + 4).map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]]) as f32); + let (w, h) = if d.starts_with(&[0xd7, 0xcd, 0xc6, 0x9a]) { + let inch = d.get(14..16).map(|b| f32::from(u16::from_le_bytes([b[0], b[1]]))).filter(|v| *v > 0.0)?; + let (l, t, r, b) = (i16_at(6)?, i16_at(8)?, i16_at(10)?, i16_at(12)?); + ((r - l).abs() / inch * 72.0, (b - t).abs() / inch * 72.0) + } else if d.get(..4) == Some(&[1, 0, 0, 0]) { + let (l, t, r, b) = (i32_at(24)?, i32_at(28)?, i32_at(32)?, i32_at(36)?); + let pt = 72.0 / 2540.0; + ((r - l).abs() * pt, (b - t).abs() * pt) + } else { + return None; + }; + (w.is_finite() && h.is_finite() && w > 0.0 && h > 0.0).then_some((w.min(1584.0), h.min(1584.0))) +} diff --git a/crates/odf/src/text/body/indexes.rs b/crates/odf/src/text/body/indexes.rs new file mode 100644 index 000000000..a1578779f --- /dev/null +++ b/crates/odf/src/text/body/indexes.rs @@ -0,0 +1,120 @@ +//! Tables of contents and other generated indexes read (ODF 1.3 §8): their entries as last +//! generated, with the Word field they stand for — in the title paragraph when there is one, +//! as WordCraft inserts a table, or around the entries — so Update Table finds them. + +use super::{Attrs, Body, FBlock, Fmt, Inline, get}; +use crate::text::writer::{IndexKind, index_instr}; + +/// An index being read. +pub(in crate::text) struct OpenIndex { + pub(in crate::text) depth: usize, + /// What it stands for; `None` for indexes Word has no field for (user and object indexes), + /// read as their paragraphs. + pub(in crate::text) kind: Option, + /// Where its blocks start in the container it is in, and its title's (start, end) there. + pub(in crate::text) start: usize, + pub(in crate::text) title: (Option, Option), +} + +/// Whether `name` (local, in the text namespace) is an index element. +pub(in crate::text) fn is_index(name: &str) -> bool { + matches!(name, "table-of-content" | "illustration-index" | "table-index" | "alphabetical-index" | "bibliography" | "user-index" | "object-index") +} + +impl Body<'_> { + /// An index element, its source or its title (`q`, attributes `a`); false for anything else. + pub(in crate::text) fn start_index_part(&mut self, q: &str, a: &Attrs) -> bool { + let Some(name) = q.strip_prefix("text:") else { return false }; + if is_index(name) { + // A table of contents or another index (§8), read as its entries and the field it + // stands for. + if self.index.is_none() && self.note.is_none() && self.del.is_none() && !self.in_tc && self.frame.is_none() { + self.start_index(name); + } + } else if name.ends_with("-source") { + // What it is generated from: its levels or caption name; its templates aren't read. + self.index_source(a); + self.skip = 1; + } else if name == "index-title" { + self.index_title(false); + } else { + return false; + } + true + } + + /// An index element begins: its blocks are read into the container it is in. + pub(in crate::text) fn start_index(&mut self, name: &str) { + self.end_para(); + let kind = match name { + "table-of-content" => Some(IndexKind::Contents(10)), + "illustration-index" => Some(IndexKind::Captions("Figure".into())), + "table-index" => Some(IndexKind::Captions("Table".into())), + "alphabetical-index" => Some(IndexKind::Alphabetical), + "bibliography" => Some(IndexKind::Bibliography), + _ => None, + }; + let start = self.container().len(); + self.index = Some(OpenIndex { depth: self.depth, kind, start, title: (None, None) }); + } + + /// An index's source (what it is generated from): its outline levels or caption name. + pub(in crate::text) fn index_source(&mut self, a: &Attrs) { + let Some(ix) = &mut self.index else { return }; + match &mut ix.kind { + Some(IndexKind::Contents(levels)) => { + if let Some(n) = get(a, "text:outline-level").and_then(|v| v.trim().parse::().ok()) { + *levels = n.clamp(1, 10) as u8; + } + } + Some(IndexKind::Captions(name)) => { + if let Some(c) = get(a, "text:caption-sequence-name").map(str::trim).filter(|c| !c.is_empty()) { + *name = c.chars().take(64).collect(); + } + } + _ => {} + } + } + + /// An index title begins (`end` false) or ends. + pub(in crate::text) fn index_title(&mut self, end: bool) { + self.end_para(); + let len = self.container().len(); + if let Some(ix) = &mut self.index { + if end { + ix.title.1.get_or_insert(len); + } else { + ix.title.0.get_or_insert(len); + } + } + } + + /// The index being read ends: its field goes into its title's first paragraph, or around + /// its entries. + pub(in crate::text) fn end_index(&mut self) { + self.end_para(); + let Some(ix) = self.index.take() else { return }; + let Some(kind) = ix.kind else { return }; + let instr = index_instr(&kind); + let blocks = self.container(); + let start = ix.start.min(blocks.len()); + let titled = match ix.title { + (Some(s), Some(e)) if s == start && e > s => blocks.get_mut(s), + _ => None, + }; + if let Some(FBlock::Para(p)) = titled { + p.inlines.push(Inline::Field { instr, result: String::new(), fmt: Fmt::default(), locked: false }); + return; + } + let Some(entries) = blocks.get_mut(start..) else { return }; + let first = entries.iter().position(|b| matches!(b, FBlock::Para(_))); + let last = entries.iter().rposition(|b| matches!(b, FBlock::Para(_))); + let (Some(first), Some(last)) = (first, last) else { return }; + if let Some(FBlock::Para(p)) = entries.get_mut(first) { + p.inlines.insert(0, Inline::FieldStart(instr)); + } + if let Some(FBlock::Para(p)) = entries.get_mut(last) { + p.inlines.push(Inline::FieldEnd); + } + } +} diff --git a/crates/odf/src/text/body/notes.rs b/crates/odf/src/text/body/notes.rs new file mode 100644 index 000000000..b478fddcf --- /dev/null +++ b/crates/odf/src/text/body/notes.rs @@ -0,0 +1,381 @@ +//! Notes, comments and fields (ODF 1.3 §6.3, §14.1, §7): read as containers of their own while the +//! paragraph they are in is put aside. + +use super::{Attrs, Body, FBlock, FComment, Fmt, Frame, Inline, MAX_COMMENTS, MAX_INFO_CHARS, Para, get, length, model}; + +/// Make every comment range in `blocks` close: an end before its comment (or of no comment) is +/// dropped, and a comment whose end never comes covers nothing. Replies have no range. +pub(in crate::text) fn balance_comments(blocks: &mut [FBlock]) { + fn each_para(blocks: &mut [FBlock], depth: usize, f: &mut dyn FnMut(&mut Para)) { + for b in blocks { + match b { + FBlock::Para(p) => f(p), + FBlock::Table(t) if depth < model::MAX_DEPTH => { + for c in t.rows.iter_mut().flatten() { + each_para(&mut c.blocks, depth + 1, f); + } + } + FBlock::Table(_) => {} + } + } + } + let mut open = std::collections::HashSet::new(); + each_para(blocks, 0, &mut |p| { + p.inlines.retain(|i| match i { + Inline::Comment { id, comment } if comment.parent.is_none() => { + open.insert(*id); + true + } + Inline::CommentEnd(id) => open.remove(id), + _ => true, + }) + }); + if open.is_empty() { + return; + } + each_para(blocks, 0, &mut |p| { + let mut k = 0; + while k < p.inlines.len() { + if let Some(Inline::Comment { id, comment }) = p.inlines.get(k) + && comment.parent.is_none() + && open.remove(id) + { + let id = *id; + p.inlines.insert(k + 1, Inline::CommentEnd(id)); + k += 1; + } + k += 1; + } + }); +} + +/// A comment being read (ODF 1.3 §14.1): what `office:annotation` and its `dc:creator`, +/// `dc:date` and `meta:creator-initials` say. +#[derive(Default)] +pub(in crate::text) struct CommentInfo { + /// Its flow id. + pub(in crate::text) id: u32, + /// Its `office:name`, which its `office:annotation-end` names. + pub(in crate::text) named: bool, + /// The flow id of the comment it answers (`loext:parent-name`). + pub(in crate::text) parent: Option, + pub(in crate::text) resolved: bool, + pub(in crate::text) author: String, + pub(in crate::text) date: String, + pub(in crate::text) initials: String, + /// The child whose text is being read: 0 author, 1 date, 2 initials. + pub(in crate::text) field: Option, +} + +/// A footnote, endnote or comment being read (ODF 1.3 §6.3, §14.1): what it is, and the +/// reader state of the paragraph it sits in, put aside while its content is read into a +/// container of its own. +pub(in crate::text) struct OpenNote { + /// The `text:note` or `office:annotation` element's depth. + pub(in crate::text) depth: usize, + pub(in crate::text) endnote: bool, + /// The citation's `text:label` (a custom mark). + pub(in crate::text) custom: String, + /// Set for a comment. + pub(in crate::text) comment: Option, + /// Set for a text box: the frame it is in, put aside with its placement while its content is + /// read (a frame inside it is a frame of its own). + pub(in crate::text) text_box: Option<(Option, Option, model::BoxLook)>, + pub(in crate::text) para: Option, + pub(in crate::text) fmt: Vec, + pub(in crate::text) xml_space_end: bool, + pub(in crate::text) para_cont: bool, + pub(in crate::text) lists: Vec<(Option, bool)>, + pub(in crate::text) item_used: Vec, + pub(in crate::text) list_id: u32, + /// Stack heights when the note began: what the note opens and leaves open goes with it. + pub(in crate::text) containers: usize, + pub(in crate::text) tables: usize, + pub(in crate::text) cells: usize, + pub(in crate::text) lists_saved: usize, +} + +/// A text field being read ([`super::fields`]). +pub(in crate::text) struct OpenField { + /// Its Word field code. + pub(in crate::text) instr: String, + /// Its shown text so far. + pub(in crate::text) result: String, + /// Its element's name and depth. + pub(in crate::text) name: String, + pub(in crate::text) depth: usize, + /// The formatting it is in. + pub(in crate::text) fmt: Fmt, + /// A sequence's reference name: a bookmark over the caption's label and number, which + /// references to it show. + pub(in crate::text) anchor: Option, + /// Fixed without a value to be fixed at: a field while it shows nothing, else its text. + pub(in crate::text) fixed: bool, +} + +impl Body<'_> { + /// A note or its citation starting: true when handled. A note inside a note (allowed by the + /// schema, never written by Word or LibreOffice), or one in deleted text or a frame, is left + /// out with its content. + pub(in crate::text) fn start_note_part(&mut self, q: &str, a: &Attrs) -> bool { + let free = self.note.is_none() && self.del.is_none() && !self.in_tc && self.frame.is_none(); + if let Some(c) = self.note.as_mut().and_then(|n| n.comment.as_mut()) { + c.field = match q { + "dc:creator" => Some(0), + "dc:date" => Some(1), + "meta:creator-initials" => Some(2), + _ => None, + }; + if c.field.is_some() { + return true; + } + } + match q { + "office:annotation" if free && self.comment_names.len() < MAX_COMMENTS => self.start_comment(a), + // A text box: its content read as a note's, the frame it is in put aside meanwhile. + "draw:text-box" if self.frame.is_some() && self.note.is_none() && self.del.is_none() && !self.in_tc && self.frame_box.is_none() => { + // A least height (ODF 1.3 §20.207 `fo:min-height`): the box grows to its text. + if let Some(min) = get(a, "fo:min-height").and_then(length).filter(|v| v.is_finite() && *v >= 0.0) + && let Some(fr) = &mut self.frame + { + self.frame_look.auto_fit = true; + fr.1 = fr.1.or(Some(min)); + } + // Its name, and the box its text continues in (§19.112 `draw:chain-next-name`). + let next = get(a, "draw:chain-next-name").map(str::trim).filter(|n| !n.is_empty()).map(|n| n.chars().take(256).collect::()); + if let Some(name) = self.frame_name.take().filter(|n| !n.is_empty()) { + self.frame_look.chain = Some(Box::new(model::Chain { name, next })); + } + let frame = (self.frame.take(), self.frame_float.take(), std::mem::take(&mut self.frame_look)); + self.start_note(a); + if let Some(n) = &mut self.note { + n.text_box = Some(frame); + } + } + "office:annotation-end" => { + if let Some(id) = get(a, "office:name").and_then(|n| self.comment_names.get(n)).copied() { + self.push_inline(Inline::CommentEnd(id)); + } + } + "text:note" if free => self.start_note(a), + // The citation's text is LibreOffice's number: only a custom mark is kept. + "text:note-citation" => { + if let Some(n) = &mut self.note { + n.custom = get(a, "text:label").map(|l| l.chars().filter(|c| !c.is_control()).take(10).collect()).unwrap_or_default(); + } + self.skip = 1; + } + _ => return false, + } + true + } + + /// Begin reading a comment, as a note: its text in a container of its own. + pub(in crate::text) fn start_comment(&mut self, a: &Attrs) { + let id = self.next_comment; + self.next_comment = self.next_comment.saturating_add(1); + let name = get(a, "office:name").filter(|n| !n.is_empty() && n.len() <= MAX_INFO_CHARS); + if let Some(n) = name { + // The first comment of a name keeps it. + self.comment_names.entry(n.to_string()).or_insert(id); + } + let parent = get(a, "loext:parent-name").and_then(|n| self.comment_names.get(n)).copied(); + let resolved = get(a, "loext:resolved").is_some_and(|r| r.trim() == "true"); + let info = CommentInfo { id, named: name.is_some(), parent, resolved, ..Default::default() }; + self.start_note(a); + if let Some(n) = &mut self.note { + n.comment = Some(info); + } + } + + /// `inline` at the end of the paragraph being read (a new one if none is). + pub(in crate::text) fn push_inline(&mut self, inline: Inline) { + // A drawing in the text outside any paragraph waits for the next one. + let drawing = matches!(inline, Inline::Image(_) | Inline::Shape { .. } | Inline::Chart(_) | Inline::Group { .. }); + if self.para.is_none() && drawing && self.at_body_level() && self.loose.len() < super::MAX_LOOSE { + self.loose.push(inline); + return; + } + if self.para.is_none() { + self.start_para(None, None); + } + if let Some(p) = &mut self.para { + p.inlines.push(inline); + } + } + + /// Begin reading a note: put the paragraph it is in aside and read its content into a + /// container of its own, outside any list. + pub(in crate::text) fn start_note(&mut self, a: &Attrs) { + let note = OpenNote { + depth: self.depth, + endnote: get(a, "text:note-class").is_some_and(|c| c.trim() == "endnote"), + custom: String::new(), + comment: None, + text_box: None, + para: self.para.take(), + fmt: std::mem::take(&mut self.fmt), + xml_space_end: std::mem::take(&mut self.xml_space_end), + para_cont: std::mem::take(&mut self.para_cont), + lists: std::mem::take(&mut self.lists), + item_used: std::mem::take(&mut self.item_used), + list_id: self.list_id, + containers: self.containers.len(), + tables: self.tables.len(), + cells: self.cells.len(), + lists_saved: self.lists_saved.len(), + }; + self.containers.push(Vec::new()); + self.note = Some(note); + } + + /// End the note being read: its content becomes an [`Inline::Note`] in the paragraph put + /// aside, which goes on. Tables and cells the note left open (malformed) are dropped. + pub(in crate::text) fn end_note(&mut self) { + self.end_para(); + let Some(n) = self.note.take() else { return }; + self.tables.truncate(n.tables); + self.cells.truncate(n.cells); + self.formulas.truncate(n.cells); + self.cell_tops.truncate(n.cells); + self.lists_saved.truncate(n.lists_saved); + self.containers.truncate(n.containers + 1); + let blocks = if self.containers.len() > n.containers { self.containers.pop().unwrap_or_default() } else { Vec::new() }; + self.para = n.para; + self.fmt = n.fmt; + self.xml_space_end = n.xml_space_end; + self.para_cont = n.para_cont; + self.lists = n.lists; + self.item_used = n.item_used; + self.list_id = n.list_id; + if let Some((frame, float, look)) = n.text_box { + // Back in the frame, which holds the box's content until it ends. + (self.frame, self.frame_float, self.frame_look) = (frame, float, look); + self.frame_box = Some(blocks); + return; + } + match n.comment { + Some(c) => { + // LibreOffice writes an empty date as all zeroes; a date without a zone is + // read as UTC, as the document's own dates are. + let date = c.date.trim(); + let date = if date.is_empty() || date.starts_with("0000") { + String::new() + } else if date.ends_with(['Z', 'z']) || date.rsplit_once('T').is_some_and(|(_, t)| t.contains(['+', '-'])) { + date.to_string() + } else { + format!("{date}Z") + }; + let comment = FComment { + author: c.author.trim().to_string(), + initials: c.initials.trim().to_string(), + date, + resolved: c.resolved, + parent: c.parent, + blocks, + }; + self.push_inline(Inline::Comment { id: c.id, comment: Box::new(comment) }); + // A comment without a name has no end to wait for: it covers nothing. + if !c.named && c.parent.is_none() { + self.push_inline(Inline::CommentEnd(c.id)); + } + } + None => { + let fmt = self.cur_fmt(); + self.push_inline(Inline::Note { endnote: n.endnote, custom: n.custom, blocks, fmt }); + } + } + } + + /// An element ending at the current depth, for an open note or field: true when that was all + /// there was to it. + pub(in crate::text) fn end_note_or_field(&mut self, q: &str) -> bool { + if let Some(c) = self.note.as_mut().and_then(|n| n.comment.as_mut()) + && c.field.is_some() + { + c.field = None; + return true; + } + if self.note.as_ref().is_some_and(|n| self.depth < n.depth) { + // The note's own end, or (malformed) an outer element ending first: the note ends, + // then that element. + self.end_field(); + self.end_note(); + if q == "text:note" || q == "office:annotation" || q == "draw:text-box" { + return true; + } + if self.shape_kind.is_some() && crate::text::shapes::is_shape(q) { + // A drawing shape's own end: it goes into the paragraph, its text read. + self.end_frame(); + return true; + } + } + // Everything inside a field was its text: only the field's own end does anything. An + // outer element ending first (malformed) ends the field, then itself. + if let Some(f) = &self.field { + if self.depth >= f.depth { + return true; + } + let own = f.name == q; + self.end_field(); + return own; + } + false + } + + /// Put the field read so far into the paragraph. + pub(in crate::text) fn end_field(&mut self) { + let Some(OpenField { instr, result, mut fmt, anchor, fixed, .. }) = self.field.take() else { return }; + fmt.ins = self.open_insertion(); + if self.para.is_none() { + self.start_para(None, None); + } + if let Some(p) = &mut self.para { + if let Some(name) = &anchor { + // From the start of the paragraph when only the label is before the number (a + // caption, "Figure 1"), else from the number. + let label = crate::text::fields::sequence_name(&instr).unwrap_or(""); + let before: String = p.inlines.iter().map(|i| if let Inline::Text(t, _) = i { t.as_str() } else { "\u{1}" }).collect(); + let at = if before.trim() == label { 0 } else { p.inlines.len() }; + p.inlines.insert(at, Inline::Anchor(name.clone())); + } + // A fixed field keeps what it shows (§19.807 `text:fixed`): a locked field. + p.inlines.push(Inline::Field { instr, result: result.trim().to_string(), fmt, locked: fixed && !result.trim().is_empty() }); + if let Some(name) = anchor { + p.inlines.push(Inline::AnchorEnd(name)); + } + } + self.xml_space_end = false; + } +} + +#[cfg(test)] +mod tests { + use crate::text::{Body, Pages, Styles, events}; + + /// The cell stacks after reading `xml`: (cells, formulas, cell tops). + fn stacks_after(xml: &str) -> (usize, usize, usize) { + let (styles, pages) = (Styles::default(), Pages::default()); + let mut none = |_: &str| None; + let mut b = Body::new(&styles, &pages, &mut none); + b.in_text = true; + events(xml.as_bytes(), |ev| b.feed(ev)); + (b.cells.len(), b.formulas.len(), b.cell_tops.len()) + } + + /// A formula cell a note or a deletion leaves open (malformed) goes with it: nothing of it is + /// left for the cell round the note, or for the next cell read. + #[test] + fn a_cell_a_note_or_deletion_leaves_open_takes_its_formula_with_it() { + // The cell, its row and its table end with tags that are none of theirs. + let open_cell = r#"2"#; + let in_note = format!( + "a1{open_cell}" + ); + assert_eq!(stacks_after(&in_note), (0, 0, 0)); + let in_deletion = format!( + "A{open_cell}b" + ); + assert_eq!(stacks_after(&in_deletion), (0, 0, 0)); + } +} diff --git a/crates/odf/src/text/body/tables.rs b/crates/odf/src/text/body/tables.rs new file mode 100644 index 000000000..f23f31108 --- /dev/null +++ b/crates/odf/src/text/body/tables.rs @@ -0,0 +1,150 @@ +//! Tables being read (ODF 1.3 §9.1): rows, cells and their shared edges. + +use super::{Borders, Cell, FTable}; + +/// Give each edge two cells share the line either cell draws: LibreOffice gives a shared edge to +/// one of them only (the left cell's right side is `none`, the right cell's left side draws it), +/// while WordCraft's cells each draw their own four sides. The cell below is the one in the next +/// row starting at the same column. +pub(in crate::text) fn share_edges(rows: &mut [Vec]) { + let visible = |b: Option| b.filter(|b| b.is_visible()); + for r in 0..rows.len() { + let starts: Vec = rows + .get(r) + .map(|row| { + row.iter() + .scan(0usize, |g, c| { + let s = *g; + *g += c.colspan.max(1) as usize; + Some(s) + }) + .collect() + }) + .unwrap_or_default(); + for k in 0..starts.len() { + // The cell to the right. + if let Some(row) = rows.get_mut(r) + && let [Some(a), Some(b)] = [k, k + 1].map(|i| row.get(i).map(|c| c.borders)) + && (a.is_some() || b.is_some()) + { + let (a, b) = (a.unwrap_or_default(), b.unwrap_or_default()); + let line = visible(a.right).or(visible(b.left)); + if let Some(line) = line { + if let Some(c) = row.get_mut(k) { + c.borders = Some(Borders { right: Some(line), ..a }); + } + if let Some(c) = row.get_mut(k + 1) { + c.borders = Some(Borders { left: Some(line), ..b }); + } + } + } + // The cell below. + let start = starts.get(k).copied().unwrap_or(0); + let below = rows.get(r + 1).and_then(|row| { + let mut g = 0; + row.iter().position(|c| { + let s = g; + g += c.colspan.max(1) as usize; + s == start + }) + }); + let (Some(j), Some(a)) = (below, rows.get(r).and_then(|row| row.get(k)).map(|c| c.borders)) else { continue }; + let Some(b) = rows.get(r + 1).and_then(|row| row.get(j)).map(|c| c.borders) else { continue }; + if a.is_none() && b.is_none() { + continue; + } + let (a, b) = (a.unwrap_or_default(), b.unwrap_or_default()); + if let Some(line) = visible(a.bottom).or(visible(b.top)) { + if let Some(c) = rows.get_mut(r).and_then(|row| row.get_mut(k)) { + c.borders = Some(Borders { bottom: Some(line), ..a }); + } + if let Some(c) = rows.get_mut(r + 1).and_then(|row| row.get_mut(j)) { + c.borders = Some(Borders { top: Some(line), ..b }); + } + } + } + } +} + +pub(in crate::text) struct TableB { + pub(in crate::text) rows: Vec>, + pub(in crate::text) row: Vec, + /// Covered cells still owed to the previous cell's column span. + pub(in crate::text) hcover: usize, + /// Widths in points of the `table:table-column`s so far, 0 where the style gives none. + pub(in crate::text) widths: Vec, + /// Each column's share of the table (`style:rel-column-width`), 0 where the style gives none. + pub(in crate::text) shares: Vec, + /// The table's width, points: what the shares divide. + pub(in crate::text) room: Option, + /// It is in a cell, a frame or a column narrower than the page, whose width isn't known here. + pub(in crate::text) contained: bool, + /// Its alignment and width, and each row's height so far (see [`craft_flow::TableExtra`]). + pub(in crate::text) extra: craft_flow::TableExtra, + /// The background of each column's default cell style (`table:default-cell-style-name`). + pub(in crate::text) column_backgrounds: Vec>, + /// The table's background, and the row's being read. + pub(in crate::text) background: Option, + pub(in crate::text) row_background: Option, +} + +/// The narrowest column the document model keeps (a narrower one would lose the table its grid). +const MIN_COLUMN: f32 = 6.0; + +/// Each column's width out of `room` points, as LibreOffice shares it out: by its share if it has +/// one, else its stated width, which comes off the room first; none narrower than a column can be +/// (6 pt). Empty if a column has neither, or no room is left to share. +fn widths_of_shares(widths: &[f32], shares: &[f32], room: f32) -> Vec { + let columns = || widths.iter().zip(shares); + let usable = |v: f32| v.is_finite() && v > 0.0; + let left = room - columns().filter(|(_, s)| **s <= 0.0).map(|(w, _)| *w).sum::(); + let total: f32 = shares.iter().filter(|s| **s > 0.0).sum(); + if widths.len() != shares.len() || !usable(left) || !usable(total) || columns().any(|(w, s)| *w <= 0.0 && *s <= 0.0) { + return Vec::new(); + } + columns().map(|(w, s)| if *s > 0.0 { (s / total * left).max(MIN_COLUMN) } else { *w }).collect() +} + +impl TableB { + pub(in crate::text) fn finish(self) -> FTable { + // Columns stating their shares (LibreOffice's tables filling the margins, §20.334) divide + // the table's width by them; a column with neither leaves the whole grid to the equal split. + // An aligned table's shares go before the widths its columns state too, as LibreOffice + // reads them. (A table filling the margins is fitted to them whatever its widths say, and + // files give it both in proportion; in something narrower than the page, whose width isn't + // known here, stated widths are the best guess.) + let all_widths = self.widths.iter().all(|w| *w > 0.0); + let all_shares = !self.shares.is_empty() && self.shares.iter().all(|s| *s > 0.0); + let aligned = self.extra.align.is_some() && !self.contained; + let by_shares = self + .room + .filter(|_| all_shares && (aligned || !all_widths)) + .map(|room| widths_of_shares(&self.widths, &self.shares, room)) + .filter(|w| !w.is_empty()); + let widths = if let Some(w) = by_shares { + w + } else if all_widths { + self.widths + } else { + // With nothing to divide, or in something narrower than the page with widths stated + // too, the shares can't be worked out here: the equal split, as before. + let mixed = self.shares.iter().any(|s| *s <= 0.0); + match self.room.filter(|_| !(self.contained && mixed)) { + Some(room) => widths_of_shares(&self.widths, &self.shares, room), + None => Vec::new(), + } + }; + let mut rows = self.rows; + share_edges(&mut rows); + // A table whose cells all say "no line" draws none. + let stated = rows.iter().flatten().filter(|c| !c.covered).all(|c| c.borders.is_some()); + let borderless = stated && rows.iter().flatten().all(|c| !c.borders.is_some_and(|b| b.any_visible())); + let extra = self.extra; + let kept = extra.fixed + || extra.align.is_some() + || extra.width_pct.is_some() + || extra.heights.iter().any(Option::is_some) + || extra.keep_rows.contains(&true); + FTable { rows, widths, borderless, extra: kept.then(|| Box::new(extra)), kept: None, wrap: None } + } +} diff --git a/crates/odf/src/text/body/tracked.rs b/crates/odf/src/text/body/tracked.rs new file mode 100644 index 000000000..010929065 --- /dev/null +++ b/crates/odf/src/text/body/tracked.rs @@ -0,0 +1,211 @@ +//! Tracked changes (ODF 1.3 §5.5): the changed regions, deleted content and the insertions open +//! around the text. + +use super::{Attrs, Body, FBlock, HashMap, Inline, MAX_OPEN_INSERTIONS, MAX_REGIONS, Para, RevMark, RevisionKind, get, local}; + +/// A `text:changed-region` read from `text:tracked-changes`. +#[derive(Default)] +pub(in crate::text) struct Region { + pub(in crate::text) kind: Option, + /// Sequence number: tells regions by the same author at the same time apart. + pub(in crate::text) id: u32, + pub(in crate::text) author: String, + pub(in crate::text) date: String, + /// A deletion's deleted paragraphs. + pub(in crate::text) blocks: Vec, +} + +impl Region { + pub(in crate::text) fn mark(&self) -> RevMark { + RevMark { author: self.author.trim().to_string(), date: self.date.trim().to_string(), id: Some(self.id) } + } +} + +/// Reader state saved while a deletion's content is read into its own container. +pub(in crate::text) struct DelState { + pub(in crate::text) containers: usize, + pub(in crate::text) tables: usize, + pub(in crate::text) cells: usize, + pub(in crate::text) lists_saved: usize, + pub(in crate::text) fmt: usize, + pub(in crate::text) lists: Vec<(Option, bool)>, + pub(in crate::text) item_used: Vec, + /// Insertions open inside the deleted content (text inserted, then deleted). + pub(in crate::text) ins: Vec<(String, RevMark)>, +} + +/// A `text:change-start` (`start`) or `text:change-end` (attributes `a`) among the changes of +/// `kind` open (insertions, or format changes): unknown ids, other kinds' marks, ends without a +/// start and starts past [`MAX_OPEN_INSERTIONS`] are ignored. +pub(in crate::text) fn change_mark(regions: &HashMap, open: &mut Vec<(String, RevMark)>, kind: RevisionKind, start: bool, a: &Attrs) { + let Some(id) = get(a, "text:change-id") else { return }; + if !start { + if let Some(k) = open.iter().position(|(o, _)| o == id) { + open.remove(k); + } + return; + } + if let Some(r) = regions.get(id).filter(|r| r.kind == Some(kind)) + && open.len() < MAX_OPEN_INSERTIONS + && !open.iter().any(|(o, _)| o == id) + { + open.push((id.to_string(), r.mark())); + } +} + +impl Body<'_> { + /// An element inside `text:tracked-changes`, outside deleted content. + pub(in crate::text) fn tracked_start(&mut self, q: &str, a: &Attrs) { + match local(q) { + "changed-region" if self.region.is_none() && self.region_count < MAX_REGIONS => { + // ODF 1.3 names it by `xml:id`; `text:id` is the older name, kept beside it. + let id = get(a, "text:id").or_else(|| get(a, "xml:id")).unwrap_or_default().to_string(); + let r = Region { id: self.region_count as u32, ..Default::default() }; + self.region_count += 1; + self.region = Some((id, r)); + } + "insertion" | "deletion" | "format-change" if self.region.as_ref().is_some_and(|(_, r)| r.kind.is_none()) => { + let kind = match local(q) { + "insertion" => RevisionKind::Insert, + "deletion" => RevisionKind::Delete, + _ => RevisionKind::Format, + }; + if let Some((_, r)) = &mut self.region { + r.kind = Some(kind); + } + if kind == RevisionKind::Delete { + self.containers.push(Vec::new()); + self.del = Some(DelState { + containers: self.containers.len(), + tables: self.tables.len(), + cells: self.cells.len(), + lists_saved: self.lists_saved.len(), + fmt: self.fmt.len(), + lists: std::mem::take(&mut self.lists), + item_used: std::mem::take(&mut self.item_used), + ins: Vec::new(), + }); + } + } + "change-info" if self.region.is_some() => self.info_at = Some(self.depth), + _ => self.skip = 1, + } + } + + /// End a deletion's content: its paragraphs go to the region being read. + pub(in crate::text) fn close_deletion(&mut self) { + if self.del.is_none() { + return; + } + self.end_para(); + let Some(d) = self.del.take() else { return }; + let base = d.containers.saturating_sub(1).min(self.containers.len()); + let blocks: Vec = self.containers.split_off(base).into_iter().flatten().filter(|b| matches!(b, FBlock::Para(_))).collect(); + if self.containers.is_empty() { + self.containers.push(Vec::new()); + } + self.tables.truncate(d.tables); + self.cells.truncate(d.cells); + self.formulas.truncate(d.cells); + self.cell_tops.truncate(d.cells); + self.lists_saved.truncate(d.lists_saved); + self.fmt.truncate(d.fmt); + self.lists = d.lists; + self.item_used = d.item_used; + self.frame = None; + self.in_desc = false; + self.para_cont = false; + if let Some((_, r)) = &mut self.region { + r.blocks = blocks; + } + } + + /// Put a deletion's paragraphs back at a `text:change` point, as deleted text. Inside a + /// paragraph, each deleted paragraph mark splits it; between paragraphs, each deleted + /// paragraph stands alone with its mark deleted. + pub(in crate::text) fn place_deletion(&mut self, blocks: Vec, mark: RevMark) { + let mut paras: Vec = Vec::new(); + for b in blocks { + let FBlock::Para(mut p) = b else { continue }; + // Pictures and other objects carry no revision mark: leave them out. + p.inlines.retain(|i| matches!(i, Inline::Text(..) | Inline::Anchor(_))); + for i in &mut p.inlines { + if let Inline::Text(_, f) = i { + f.del = Some(mark.clone()); + } + } + paras.push(p); + } + let append = |dst: &mut Para, src: Vec| { + for i in src { + match i { + Inline::Text(t, f) => dst.push_text(&t, &f), + other => dst.inlines.push(other), + } + } + }; + let Some(mut cur) = self.para.take() else { + self.para_cont = false; + for mut p in paras { + p.mark_del = Some(Box::new(mark.clone())); + self.finish_para(p); + } + return; + }; + let n = paras.len(); + let mut it = paras.into_iter(); + let mut first_mark_ins = None; + if let Some(first) = it.next() { + first_mark_ins = first.mark_ins; + append(&mut cur, first.inlines); + } + if n < 2 { + self.para = Some(cur); + return; + } + let mut rest = Para { kind: cur.kind, list: cur.list, align: cur.align, ..Default::default() }; + // The host paragraph now ends at the first deleted paragraph's mark. + cur.mark_del = Some(Box::new(mark.clone())); + cur.mark_ins = first_mark_ins; + self.finish_para(cur); + for (k, mut p) in it.enumerate() { + if k + 2 == n { + // The last deleted paragraph runs on into the rest of the body paragraph. + rest.kind = p.kind; + rest.align = p.align.or(rest.align); + append(&mut rest, p.inlines); + } else { + p.mark_del = Some(Box::new(mark.clone())); + self.finish_para(p); + } + } + self.para = Some(rest); + } + + pub(in crate::text) fn tracked_context(&mut self, q: &str, a: &Attrs) -> bool { + if !self.in_tc { + return false; + } + if self.del.is_none() { + self.tracked_start(q, a); + return true; + } + // Inside deleted content: only insertions (text inserted, then deleted) nest. + if matches!(local(q), "change-start" | "change-end") { + let regions = &self.regions; + if let Some(d) = &mut self.del { + change_mark(regions, &mut d.ins, RevisionKind::Insert, local(q) == "change-start", a); + } + return true; + } + if matches!(local(q), "tracked-changes" | "changed-region" | "insertion" | "deletion" | "format-change" | "change") { + self.skip = 1; + return true; + } + if local(q) == "change-info" && self.region.is_some() { + self.info_at = Some(self.depth); + return true; + } + false + } +} diff --git a/crates/odf/src/text/cell_formula.rs b/crates/odf/src/text/cell_formula.rs new file mode 100644 index 000000000..21a91ee34 --- /dev/null +++ b/crates/odf/src/text/cell_formula.rs @@ -0,0 +1,364 @@ +//! Table cell formulas (ODF 1.3 §19.646 `table:formula`) and Word's formula fields. +//! +//! LibreOffice writes a text table's formulas in its own syntax (prefix `ooow:`): cells and +//! ranges in angle brackets (``, ``), functions written before their operand +//! (`sum `, a list joined by `|`) and some operators as words (`add`, `mul`…). WordCraft, +//! as Word, holds a formula as a field (`=SUM(A1:A3)`, `ABOVE`, `LEFT`). Formulas are turned one +//! into the other where both can say them; others are left as their value. + +use craft_flow::{FBlock, Inline}; + +/// Longest formula turned, in bytes; most tokens. +const MAX_LEN: usize = 4096; +const MAX_TOKENS: usize = 512; + +/// The functions both know: LibreOffice's name, Word's. +const FUNCTIONS: [(&str, &str); 8] = [ + ("sum", "SUM"), + ("mean", "AVERAGE"), + ("min", "MIN"), + ("max", "MAX"), + ("product", "PRODUCT"), + ("abs", "ABS"), + ("sign", "SIGN"), + ("count", "COUNT"), +]; + +/// Operators written as words: LibreOffice's, Word's. +const OPERATORS: [(&str, &str); 5] = [("add", "+"), ("sub", "-"), ("mul", "*"), ("div", "/"), ("pow", "^")]; + +#[derive(Clone, Debug, PartialEq)] +enum Tok { + /// A cell or range, upper case: `A1`, `A1:B3`. + Cell(String), + Num(String), + Word(String), + Op(char), +} + +/// Whether `s` is a cell (`A1`) or a range of cells (`A1:B3`). +fn is_cells(s: &str) -> bool { + let cell = |c: &str| { + let letters = c.chars().take_while(char::is_ascii_alphabetic).count(); + letters > 0 && letters <= 3 && c.len() > letters && c.get(letters..).is_some_and(|d| d.chars().all(|ch| ch.is_ascii_digit())) + }; + match s.split_once(':') { + Some((a, b)) => cell(a) && cell(b), + None => cell(s), + } +} + +/// A formula's tokens: angle-bracketed cells (LibreOffice), words, numbers and operators. +fn tokens(s: &str) -> Option> { + if s.len() > MAX_LEN { + return None; + } + let mut out = Vec::new(); + let mut it = s.char_indices().peekable(); + while let Some((i, c)) = it.next() { + if out.len() > MAX_TOKENS { + return None; + } + match c { + c if c.is_whitespace() => {} + '<' => { + let mut cell = String::new(); + loop { + match it.next() { + Some((_, '>')) => break, + Some((_, ch)) if cell.len() < 32 => cell.push(ch), + _ => return None, + } + } + let cell = cell.trim().to_ascii_uppercase(); + // A cell of another table (``) can't be said in Word. + if !is_cells(&cell) { + return None; + } + out.push(Tok::Cell(cell)); + } + '0'..='9' | '.' => { + let mut end = i + c.len_utf8(); + while let Some((j, ch)) = it.peek().copied() { + if ch.is_ascii_digit() || ch == '.' { + end = j + ch.len_utf8(); + it.next(); + } else { + break; + } + } + let n = s.get(i..end)?; + n.parse::().ok()?; + out.push(Tok::Num(n.to_owned())); + } + c if c.is_ascii_alphabetic() => { + let mut end = i + 1; + while let Some((j, ch)) = it.peek().copied() { + if ch.is_ascii_alphanumeric() || ch == ':' { + end = j + 1; + it.next(); + } else { + break; + } + } + out.push(Tok::Word(s.get(i..end)?.to_owned())); + } + '+' | '-' | '*' | '/' | '^' | '(' | ')' | '|' | ',' => out.push(Tok::Op(c)), + _ => return None, + } + } + Some(out) +} + +/// Whether `toks`' brackets match. +fn balanced(toks: &[Tok]) -> bool { + let mut depth = 0usize; + for t in toks { + match t { + Tok::Op('(') => depth += 1, + Tok::Op(')') => match depth.checked_sub(1) { + Some(d) => depth = d, + None => return false, + }, + _ => {} + } + } + depth == 0 +} + +/// Where the bracket opening `toks` closes. +fn closing(toks: &[Tok]) -> Option { + let mut depth = 0usize; + for (i, t) in toks.iter().enumerate() { + match t { + Tok::Op('(') => depth += 1, + Tok::Op(')') => { + depth = depth.checked_sub(1)?; + if depth == 0 { + return Some(i); + } + } + _ => {} + } + } + None +} + +/// Word's formula field code for a LibreOffice cell formula (`ooow:sum ` → +/// `=SUM(A1:A3)`), when Word can say it. +pub(super) fn word_of(formula: &str) -> Option { + let f = formula.trim(); + let f = f.strip_prefix("ooow:").unwrap_or(f); + let f = f.strip_prefix('=').unwrap_or(f); + let mut toks = tokens(f).filter(|t| balanced(t))?; + // Brackets round the whole formula say nothing (they are written round a function). + while toks.len() >= 2 && toks.first() == Some(&Tok::Op('(')) && closing(&toks) == Some(toks.len() - 1) { + toks.pop(); + toks.remove(0); + } + let mut out = String::from("="); + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + match t { + Tok::Cell(c) => out.push_str(c), + Tok::Num(n) => out.push_str(n), + Tok::Op('|' | ',') => return None, + Tok::Op(c) => out.push(*c), + Tok::Word(w) => { + let w = w.to_ascii_lowercase(); + if let Some((_, op)) = OPERATORS.iter().find(|(n, _)| *n == w) { + out.push_str(op); + continue; + } + let (_, name) = FUNCTIONS.iter().find(|(n, _)| *n == w)?; + // Its operand: a cell, number or bracketed expression, or a list of them. + out.push_str(name); + out.push('('); + loop { + match toks.get(i)? { + Tok::Cell(c) => out.push_str(c), + Tok::Num(n) => out.push_str(n), + Tok::Op('(') => { + // A bracketed expression, its brackets matched. + let mut depth = 0usize; + let start = i; + loop { + match toks.get(i)? { + Tok::Op('(') => depth += 1, + Tok::Op(')') => { + depth -= 1; + if depth == 0 { + break; + } + } + _ => {} + } + i += 1; + } + let inner: Vec = toks.get(start + 1..i)?.to_vec(); + let inner = word_of_tokens(&inner)?; + out.push_str(&inner); + } + _ => return None, + } + i += 1; + if toks.get(i) == Some(&Tok::Op('|')) { + out.push(','); + i += 1; + } else { + break; + } + } + out.push(')'); + } + } + } + (out.len() > 1).then_some(out) +} + +/// Word's form of LibreOffice tokens (an expression in brackets), without the leading `=`. +fn word_of_tokens(toks: &[Tok]) -> Option { + let text: Vec = toks + .iter() + .map(|t| match t { + Tok::Cell(c) => format!("<{c}>"), + Tok::Num(n) | Tok::Word(n) => n.clone(), + Tok::Op(c) => c.to_string(), + }) + .collect(); + word_of(&text.join(" ")).map(|w| w.trim_start_matches('=').to_owned()) +} + +/// The column letters of column `c` (from 0): A…Z, AA… +fn column(c: usize) -> String { + let mut c = c + 1; + let mut s = Vec::new(); + while c > 0 && s.len() < 4 { + let d = (c - 1) % 26; + s.push(b'A' + d as u8); + c = (c - 1) / 26; + } + s.iter().rev().map(|b| char::from(*b)).collect() +} + +/// A LibreOffice cell formula for Word's formula field code `instr`, in the cell at row `r`, +/// column `c` (from 0) of a table `rows` × `cols` (`=SUM(ABOVE)` → `ooow:sum `), when it +/// can say it. Its `\#` picture and `\*` switches are dropped. +pub(super) fn odf_of(instr: &str, (r, c): (usize, usize), (rows, cols): (usize, usize)) -> Option { + let body = instr.trim().strip_prefix('=')?; + let body = body.split('\\').next().unwrap_or_default(); + let toks = tokens(body).filter(|t| balanced(t))?; + let col = column(c); + // The cells a direction names. + let dir = |d: &str| -> Option { + Some(match d { + "ABOVE" if r > 0 => format!("{col}1:{col}{r}"), + "BELOW" if r + 1 < rows => format!("{col}{}:{col}{rows}", r + 2), + "LEFT" if c > 0 => format!("A{0}:{1}{0}", r + 1, column(c - 1)), + "RIGHT" if c + 1 < cols => format!("{1}{0}:{2}{0}", r + 1, column(c + 1), column(cols - 1)), + _ => return None, + }) + }; + let operand = |t: &Tok| -> Option { + match t { + Tok::Num(n) => Some(n.clone()), + Tok::Word(w) => { + let w = w.to_ascii_uppercase(); + if is_cells(&w) { Some(format!("<{w}>")) } else { dir(&w).map(|d| format!("<{d}>")) } + } + _ => None, + } + }; + let mut parts: Vec = Vec::new(); + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + match t { + Tok::Op(',' | '|') => return None, + Tok::Op(c) => parts.push(c.to_string()), + Tok::Num(_) => parts.push(operand(t)?), + Tok::Word(w) => { + let upper = w.to_ascii_uppercase(); + let Some((name, _)) = FUNCTIONS.iter().find(|(_, n)| *n == upper) else { + parts.push(operand(t)?); + continue; + }; + // Its arguments: cells, ranges, directions or numbers, joined by `|`. + if toks.get(i) != Some(&Tok::Op('(')) { + return None; + } + i += 1; + let mut args = Vec::new(); + loop { + args.push(operand(toks.get(i)?)?); + i += 1; + match toks.get(i)? { + Tok::Op(',') => i += 1, + Tok::Op(')') => { + i += 1; + break; + } + _ => return None, + } + } + parts.push(format!("({name} {})", args.join("|"))); + } + Tok::Cell(_) => return None, + } + } + (!parts.is_empty()).then(|| format!("ooow:{}", parts.join(" "))) +} + +/// Turns a cell's content, its value as text, into formula field `instr` showing it, when the +/// cell is one paragraph of text. +pub(super) fn into_field(blocks: &mut [FBlock], instr: String) { + let [FBlock::Para(p)] = blocks else { return }; + if !p.inlines.iter().all(|i| matches!(i, Inline::Text(..))) { + return; + } + let fmt = p.inlines.iter().find_map(|i| if let Inline::Text(_, f) = i { Some(f.clone()) } else { None }).unwrap_or_default(); + let result: String = p.inlines.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect(); + p.inlines = vec![Inline::Field { instr, result: result.trim().to_owned(), fmt, locked: false }]; +} + +/// The formula field a cell is, when it is one: its field code and the result it shows (a cell +/// of one paragraph holding one `=` field, and perhaps spaces). +pub(super) fn of_cell(blocks: &[FBlock]) -> Option<(&str, &str)> { + let [FBlock::Para(p)] = blocks else { return None }; + let mut field = None; + for i in &p.inlines { + match i { + Inline::Field { instr, result, .. } if field.is_none() && instr.trim_start().starts_with('=') => { + field = Some((instr.as_str(), result.as_str())) + } + Inline::Text(t, _) if t.trim().is_empty() => {} + _ => return None, + } + } + field +} + +/// The number a cell holds, when it is one paragraph of a plain number (digits with perhaps a +/// sign and a decimal point, as LibreOffice reads a number cell's value): what formulas +/// calculate with. +pub(super) fn number_of(blocks: &[FBlock]) -> Option { + let [FBlock::Para(p)] = blocks else { return None }; + let mut text = String::new(); + for i in &p.inlines { + match i { + Inline::Text(t, _) if text.len() + t.len() <= 64 => text.push_str(t), + _ => return None, + } + } + let t = text.trim(); + let digits = t.strip_prefix('-').unwrap_or(t); + let plain = !digits.is_empty() + && digits.chars().all(|c| c.is_ascii_digit() || c == '.') + && digits.matches('.').count() <= 1 + && digits.chars().any(|c| c.is_ascii_digit()); + if !plain { + return None; + } + t.parse::().ok().filter(|v| v.is_finite()) +} diff --git a/crates/odf/src/text/chart.rs b/crates/odf/src/text/chart.rs new file mode 100644 index 000000000..9d073ecd2 --- /dev/null +++ b/crates/odf/src/text/chart.rs @@ -0,0 +1,321 @@ +//! Charts embedded in a document (ODF 1.3 §10.4.3 `draw:object` whose sub-document is an +//! `office:chart`, Part 3 §11): read into WordCraft's editable chart model when the chart is one +//! WordCraft can edit as it is (one plot type in two dimensions, its data in the chart's own +//! table). Anything else returns `None`, and the frame keeps its replacement picture. +//! +//! The data comes from the chart's `table:table` (§11.1): the series' and categories' cell range +//! addresses (§19.583, `local-table.$B$2:.$B$5`) point into it, counting its header rows and +//! columns like any other. + +use std::collections::HashMap; + +use craft_wordmodel::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_CELLS, MAX_POINTS, MAX_SERIES, MAX_TEXT}; + +use super::xml::Attrs; +use super::{Ev, events, get}; + +/// Most XML events read from a chart sub-document. +const MAX_EVENTS: usize = 2_000_000; +/// Most rows read from a chart's table: a header row and a row per category. +const MAX_ROWS: usize = MAX_POINTS + 1; +/// Most columns read from it: a label column and two per series (a scatter series' x values). +const MAX_COLS: usize = MAX_SERIES * 2 + 1; + +/// One cell of the chart's table: its text, and its number when it holds one (`Some(None)`: a +/// number cell without a usable value, LibreOffice's `NaN` gap). +#[derive(Clone, Default)] +struct DataCell { + text: String, + num: Option>, +} + +/// A `chart:series` element as read. +#[derive(Default)] +struct RawSeries { + values: Option, + label: Option, + class: Option, + style: Option, + domain: Option, + secondary: bool, +} + +#[derive(Default)] +struct Reader { + /// Chart styles' `style:chart-properties`, by style name. + props: HashMap, + style: Option, + class: Option, + plot_style: Option, + title: Option, + in_title: bool, + /// The legend's position, when there is a legend. + legend: Option, + categories: Option, + series: Vec, + in_series: bool, + /// The chart's table: its rows of cells, and how deep in tables the reader is. + rows: Vec>, + table_depth: usize, + /// How many times the open row and cell repeat. + row_repeat: usize, + cell_repeat: usize, + cell: Option, + cells: usize, + in_chart: bool, +} + +/// A count attribute (`table:number-rows-repeated`…): 1 when absent or unusable, at most `max`. +fn repeat(a: &Attrs, k: &str, max: usize) -> usize { + get(a, k).and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(1, max.max(1)) +} + +impl Reader { + fn start(&mut self, q: &str, a: &Attrs) { + match q { + "style:style" => self.style = get(a, "style:name").map(str::to_string), + "style:chart-properties" => { + if let Some(name) = self.style.clone() { + self.props.insert(name, a.clone()); + } + } + "chart:chart" if !self.in_chart => { + self.in_chart = true; + self.class = get(a, "chart:class").map(str::to_string); + } + "chart:title" => self.in_title = true, + "text:p" if self.in_title => { + let t = self.title.get_or_insert_with(String::new); + if !t.is_empty() { + t.push(' '); + } + } + "chart:legend" => self.legend = Some(get(a, "chart:legend-position").unwrap_or("end").to_string()), + "chart:plot-area" => self.plot_style = get(a, "chart:style-name").map(str::to_string), + "chart:categories" if self.categories.is_none() => self.categories = get(a, "table:cell-range-address").map(str::to_string), + "chart:series" => { + self.in_series = true; + if self.series.len() < MAX_SERIES { + self.series.push(RawSeries { + values: get(a, "chart:values-cell-range-address").map(str::to_string), + label: get(a, "chart:label-cell-address").map(str::to_string), + class: get(a, "chart:class").map(str::to_string), + style: get(a, "chart:style-name").map(str::to_string), + domain: None, + secondary: get(a, "chart:attached-axis").is_some_and(|v| v.starts_with("secondary")), + }); + } + } + "chart:domain" if self.in_series => { + if let Some(s) = self.series.last_mut().filter(|s| s.domain.is_none()) { + s.domain = get(a, "table:cell-range-address").map(str::to_string); + } + } + "table:table" => self.table_depth += 1, + "table:table-row" if self.table_depth == 1 => { + self.row_repeat = repeat(a, "table:number-rows-repeated", MAX_ROWS); + if self.rows.len() < MAX_ROWS { + self.rows.push(Vec::new()); + } + } + "table:table-cell" | "table:covered-table-cell" if self.table_depth == 1 => { + self.cell_repeat = repeat(a, "table:number-columns-repeated", MAX_COLS); + let num = (get(a, "office:value-type") == Some("float") || get(a, "office:value-type") == Some("percentage")) + .then(|| get(a, "office:value").and_then(|v| v.trim().parse::().ok()).filter(|v| v.is_finite())); + self.cell = Some(DataCell { text: String::new(), num }); + } + _ => {} + } + } + + fn end(&mut self, q: &str) { + match q { + "style:style" => self.style = None, + "chart:title" => self.in_title = false, + "chart:series" => self.in_series = false, + "table:table" => self.table_depth = self.table_depth.saturating_sub(1), + "table:table-cell" | "table:covered-table-cell" if self.table_depth == 1 => { + if let Some(cell) = self.cell.take() + && let Some(row) = self.rows.last_mut() + { + let n = self.cell_repeat.min(MAX_COLS.saturating_sub(row.len())).min(MAX_CELLS.saturating_sub(self.cells)); + self.cells += n; + row.extend(std::iter::repeat_n(cell, n)); + } + } + "table:table-row" if self.table_depth == 1 => { + // A repeated row: its copies, within the table's and the chart's limits. + if let Some(row) = self.rows.last().cloned() { + let room = MAX_ROWS.saturating_sub(self.rows.len()); + let by_cells = MAX_CELLS.saturating_sub(self.cells).checked_div(row.len()).unwrap_or(room); + let n = self.row_repeat.saturating_sub(1).min(room).min(by_cells); + self.cells += n.saturating_mul(row.len()); + self.rows.extend(std::iter::repeat_n(row, n)); + } + self.row_repeat = 1; + } + _ => {} + } + } + + fn text(&mut self, t: &str) { + if self.in_title + && let Some(title) = &mut self.title + && title.len() < MAX_TEXT * 4 + { + title.push_str(t); + } else if let Some(cell) = &mut self.cell + && cell.text.len() < MAX_TEXT * 4 + { + cell.text.push_str(t); + } + } + + /// The chart properties of style `name`, attribute `k`. + fn prop(&self, name: Option<&str>, k: &str) -> Option<&str> { + name.and_then(|n| self.props.get(n)).and_then(|a| get(a, k)) + } + + fn cell(&self, (col, row): (usize, usize)) -> Option<&DataCell> { + self.rows.get(row).and_then(|r| r.get(col)) + } + + /// The cells a range address covers, in order, at most [`MAX_POINTS`]. + fn range(&self, addr: &str) -> Vec> { + cells_of(addr).into_iter().map(|at| self.cell(at)).collect() + } + + fn chart(&self) -> Option { + let class = self.class.as_deref().map(|c| c.trim_start_matches("chart:"))?; + let plot = self.plot_style.as_deref(); + let on = |k: &str| self.prop(plot, k) == Some("true"); + if on("chart:three-dimensional") || on("chart:percentage") || on("chart:deep") { + return None; + } + let stacked = on("chart:stacked"); + let kind = match class { + "bar" => match (on("chart:vertical"), stacked) { + (false, false) => ChartType::Column, + (false, true) => ChartType::ColumnStacked, + (true, false) => ChartType::Bar, + (true, true) => ChartType::BarStacked, + }, + "line" if !stacked => ChartType::Line, + "area" if !stacked => ChartType::Area, + "circle" => ChartType::Pie, + "ring" => ChartType::Doughnut, + "scatter" => ChartType::Scatter, + _ => return None, + }; + // One plot type on one value axis: a series of another type, or on a second axis, is + // a combination chart WordCraft can't edit. + let own = |c: &str| { + let c = c.trim_start_matches("chart:"); + c == class || (kind == ChartType::Doughnut && c == "circle") + }; + if self.series.is_empty() || self.series.iter().any(|s| s.secondary || !s.class.as_deref().is_none_or(own)) { + return None; + } + let label = |c: Option<&DataCell>| c.map(|c| c.text.trim().to_string()).unwrap_or_default(); + let series = self + .series + .iter() + .enumerate() + .map(|(i, s)| { + // A series without its own range takes the table's next column. + let values = match &s.values { + Some(addr) => self.range(addr).into_iter().map(|c| c.and_then(value)).collect(), + None => self.rows.iter().skip(1).take(MAX_POINTS).map(|r| r.get(i + 1).and_then(value)).collect(), + }; + let name = s.label.as_deref().and_then(|a| cells_of(a).first().copied()).map(|at| label(self.cell(at))).unwrap_or_default(); + let name = if name.is_empty() { format!("Series {}", i + 1) } else { name }; + ChartSeries { name, values } + }) + .collect(); + // A scatter chart's x values: its first series' domain. + let domain = self.series.first().and_then(|s| s.domain.as_deref()).filter(|_| kind == ChartType::Scatter); + let categories = match domain.or(self.categories.as_deref()) { + Some(addr) => self.range(addr).into_iter().map(label).collect(), + None => Vec::new(), + }; + let data_labels = std::iter::once(plot) + .chain(self.series.iter().map(|s| s.style.as_deref())) + .any(|st| matches!(self.prop(st, "chart:data-label-number"), Some("value" | "percentage" | "value-and-percentage"))); + let legend = self.legend.as_deref().map(|p| match p { + "start" => LegendPos::Left, + p if p.starts_with("top") => LegendPos::Top, + p if p.starts_with("bottom") => LegendPos::Bottom, + _ => LegendPos::Right, + }); + let title = self.title.as_deref().map(str::trim).filter(|t| !t.is_empty()).map(str::to_string); + Some(ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized()) + } +} + +/// A cell's number: its value, or its text read as a number. +fn value(c: &DataCell) -> Option { + match c.num { + Some(v) => v, + None => c.text.trim().parse::().ok().filter(|v| v.is_finite()), + } +} + +/// A cell address (`$B$2`, `local-table.B2`, `'my table'.$B$2`): its column and row, from 0. +fn cell_at(s: &str) -> Option<(usize, usize)> { + // The table name, when there is one, ends at the last dot. + let s = s.rsplit('.').next().unwrap_or(s).trim().trim_start_matches('$'); + let letters = s.bytes().take_while(u8::is_ascii_alphabetic).count(); + if letters == 0 || letters > 3 { + return None; + } + let (col, rest) = s.split_at_checked(letters)?; + let col = col.bytes().try_fold(0usize, |n, b| n.checked_mul(26)?.checked_add(usize::from(b.to_ascii_uppercase() - b'A') + 1))?; + let row: usize = rest.trim_start_matches('$').parse().ok()?; + Some((col.checked_sub(1)?, row.checked_sub(1)?)) +} + +/// The cells of a range address's first range (`local-table.$B$2:.$B$5`), in reading order: down +/// a column, or along a row (a block: down its first column), at most [`MAX_POINTS`]. +fn cells_of(addr: &str) -> Vec<(usize, usize)> { + let first = addr.split_whitespace().next().unwrap_or_default(); + let mut ends = first.splitn(2, ':'); + let Some(a) = ends.next().and_then(cell_at) else { return Vec::new() }; + let b = match ends.next() { + Some(e) => match cell_at(e) { + Some(b) => b, + None => return Vec::new(), + }, + None => a, + }; + let (c0, c1) = (a.0.min(b.0), a.0.max(b.0)); + let (r0, r1) = (a.1.min(b.1), a.1.max(b.1)); + if r0 == r1 && c0 != c1 { (c0..=c1).take(MAX_POINTS).map(|c| (c, r0)).collect() } else { (r0..=r1).take(MAX_POINTS).map(|r| (c0, r)).collect() } +} + +/// The chart a chart sub-document's `content.xml` holds, when WordCraft can edit it. +pub(in crate::text) fn read_chart(xml: &[u8]) -> Option { + let mut r = Reader::default(); + let mut count = 0usize; + let mut done = false; + events(xml, |ev| { + count += 1; + if done || count > MAX_EVENTS { + return; + } + match ev { + Ev::Start(q, a) => r.start(&q, &a), + Ev::End(q) => { + if q == "chart:chart" && r.in_chart { + done = true; + } + r.end(&q); + } + Ev::Text(t) => r.text(&t), + } + }); + // A sub-document cut off before its chart ended is no chart. + if !done { + return None; + } + r.chart() +} diff --git a/crates/odf/src/text/datastyle.rs b/crates/odf/src/text/datastyle.rs new file mode 100644 index 000000000..324ad135d --- /dev/null +++ b/crates/odf/src/text/datastyle.rs @@ -0,0 +1,360 @@ +//! Date and time styles (ODF 1.3 §16.27 `number:date-style`, `number:time-style`) and Word's +//! date-time pictures (`DATE \@ "dddd, MMMM d, yyyy"`), both ways: the parts — day, month, +//! year, weekday, hours, minutes, seconds, AM/PM — and the text between them. + +use super::writer::x; + +/// Longest picture kept, characters. +const MAX_PICTURE: usize = 200; + +/// A part of a date or time. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum Part { + /// Day of the month (long: two digits). + Day(bool), + /// Month: long, and by name. + Month(bool, bool), + /// Year (long: four digits). + Year(bool), + /// Day of the week by name (long: in full). + Weekday(bool), + Hours(bool), + Minutes(bool), + Seconds(bool), + AmPm, + Text(String), +} + +/// The part element `name` (local, in the number namespace) stands for; its text, if any, +/// comes after. +pub(super) fn part(name: &str, long: bool, textual: bool) -> Option { + Some(match name { + "day" => Part::Day(long), + "month" => Part::Month(long, textual), + "year" => Part::Year(long), + "day-of-week" => Part::Weekday(long), + "hours" => Part::Hours(long), + "minutes" => Part::Minutes(long), + "seconds" => Part::Seconds(long), + "am-pm" => Part::AmPm, + _ => return None, + }) +} + +/// The Word picture for `parts`: 12-hour hours when an AM/PM shows, text in single quotes when +/// it holds letters (a quote doubled); quotes and backslashes, which would end the field's +/// argument, left out. +pub(super) fn picture(parts: &[Part]) -> String { + let ampm = parts.contains(&Part::AmPm); + let pick = |long: bool, a: &'static str, b: &'static str| if long { a } else { b }; + let mut s = String::new(); + for p in parts { + match p { + Part::Day(l) => s.push_str(pick(*l, "dd", "d")), + Part::Month(l, true) => s.push_str(pick(*l, "MMMM", "MMM")), + Part::Month(l, false) => s.push_str(pick(*l, "MM", "M")), + Part::Year(l) => s.push_str(pick(*l, "yyyy", "yy")), + Part::Weekday(l) => s.push_str(pick(*l, "dddd", "ddd")), + Part::Hours(l) if ampm => s.push_str(pick(*l, "hh", "h")), + Part::Hours(l) => s.push_str(pick(*l, "HH", "H")), + Part::Minutes(l) => s.push_str(pick(*l, "mm", "m")), + Part::Seconds(l) => s.push_str(pick(*l, "ss", "s")), + Part::AmPm => s.push_str("AM/PM"), + Part::Text(t) => { + let t: String = t.chars().filter(|c| !matches!(c, '"' | '\\')).collect(); + // Letters are quoted; an apostrophe is two, inside quoted text or out of it. + if t.chars().any(char::is_alphabetic) { + s.push('\''); + s.push_str(&t.replace('\'', "''")); + s.push('\''); + } else { + s.push_str(&t.replace('\'', "''")); + } + } + } + } + s.chars().take(MAX_PICTURE).collect() +} + +/// The parts of Word picture `pic` (the reverse of [`picture`]). +fn parts_of(pic: &str) -> Vec { + let chars: Vec = pic.chars().take(MAX_PICTURE).collect(); + let mut out: Vec = Vec::new(); + let mut text = String::new(); + let mut i = 0; + let flush = |text: &mut String, out: &mut Vec| { + if !text.is_empty() { + out.push(Part::Text(std::mem::take(text))); + } + }; + while let Some(&c) = chars.get(i) { + // Two quotes outside quoted text are an apostrophe, as inside it. + if c == '\'' && chars.get(i + 1) == Some(&'\'') { + text.push('\''); + i += 2; + continue; + } + if c == '\'' { + i += 1; + while let Some(&q) = chars.get(i) { + i += 1; + if q != '\'' { + text.push(q); + } else if chars.get(i) == Some(&'\'') { + text.push('\''); + i += 1; + } else { + break; + } + } + continue; + } + if chars.get(i..i + 5).is_some_and(|w| w.iter().collect::().eq_ignore_ascii_case("am/pm")) { + flush(&mut text, &mut out); + out.push(Part::AmPm); + i += 5; + continue; + } + let mut n = 1; + while chars.get(i + n) == Some(&c) { + n += 1; + } + let p = match (c, n) { + ('y' | 'Y', 4..) => Some(Part::Year(true)), + ('y' | 'Y', _) => Some(Part::Year(false)), + ('M', 4..) => Some(Part::Month(true, true)), + ('M', 3) => Some(Part::Month(false, true)), + ('M', 2) => Some(Part::Month(true, false)), + ('M', _) => Some(Part::Month(false, false)), + ('d' | 'D', 4..) => Some(Part::Weekday(true)), + ('d' | 'D', 3) => Some(Part::Weekday(false)), + ('d' | 'D', 2) => Some(Part::Day(true)), + ('d' | 'D', _) => Some(Part::Day(false)), + ('h' | 'H', n) => Some(Part::Hours(n >= 2)), + ('m', n) => Some(Part::Minutes(n >= 2)), + ('s', n) => Some(Part::Seconds(n >= 2)), + _ => None, + }; + match p { + Some(p) => { + flush(&mut text, &mut out); + out.push(p); + } + None => { + for _ in 0..n { + text.push(c); + } + } + } + i += n; + } + flush(&mut text, &mut out); + out +} + +/// A data style named `name` for Word picture `pic`: a date style when it shows any part of a +/// date, a time style otherwise. `None` for a picture with no part. +pub(super) fn style_xml(name: &str, pic: &str) -> Option { + let parts = parts_of(pic); + if !parts.iter().any(|p| !matches!(p, Part::Text(_))) { + return None; + } + let dated = parts.iter().any(|p| matches!(p, Part::Day(_) | Part::Month(..) | Part::Year(_) | Part::Weekday(_))); + let long = |l: bool| if l { " number:style=\"long\"" } else { "" }; + let body: String = parts + .iter() + .map(|p| match p { + Part::Day(l) => format!("", long(*l)), + Part::Month(l, t) => format!("", long(*l), if *t { " number:textual=\"true\"" } else { "" }), + Part::Year(l) => format!("", long(*l)), + Part::Weekday(l) => format!("", long(*l)), + Part::Hours(l) => format!("", long(*l)), + Part::Minutes(l) => format!("", long(*l)), + Part::Seconds(l) => format!("", long(*l)), + Part::AmPm => "".to_string(), + Part::Text(t) => format!("{}", x(t)), + }) + .collect(); + let el = if dated { "number:date-style" } else { "number:time-style" }; + Some(format!("<{el} style:name=\"{name}\">{body}")) +} + +/// The `\@` picture of Word field code `instr`, if it has one. +pub(super) fn picture_of(instr: &str) -> Option { + let at = instr.find("\\@")?; + let rest = instr.get(at + 2..)?.trim_start(); + let pic = match rest.strip_prefix('"') { + Some(q) => q.split('"').next()?, + None => rest.split_whitespace().next()?, + }; + (!pic.is_empty()).then(|| pic.chars().take(MAX_PICTURE).collect()) +} + +/// A date's parts in the order and with the separators the locale gives them, for a date style +/// with `number:automatic-order` (§19.341): its weekday, month, day and year as US English +/// orders them (LibreOffice's "November 2, 2026", or "11/2/2026" with a numeric month). Its own +/// text between them is left out; a time's parts stay as they are. +pub(super) fn auto_ordered(parts: &[Part]) -> Vec { + let find = |f: fn(&Part) -> bool| parts.iter().find(|p| f(p)).cloned(); + let weekday = find(|p| matches!(p, Part::Weekday(_))); + let month = find(|p| matches!(p, Part::Month(..))); + let day = find(|p| matches!(p, Part::Day(_))); + let year = find(|p| matches!(p, Part::Year(_))); + let Some(month) = month else { return parts.to_vec() }; + let textual = matches!(month, Part::Month(_, true)); + let text = |s: &str| Part::Text(s.to_string()); + let mut out = Vec::new(); + if let Some(w) = weekday { + out.extend([w, text(", ")]); + } + out.push(month); + if let Some(d) = day { + out.extend([text(if textual { " " } else { "/" }), d]); + } + if let Some(y) = year { + out.extend([text(if textual { ", " } else { "/" }), y]); + } + // A time with it keeps its own parts, after. + let time: Vec = parts.iter().filter(|p| matches!(p, Part::Hours(_) | Part::Minutes(_) | Part::Seconds(_) | Part::AmPm)).cloned().collect(); + if !time.is_empty() { + out.push(text(" ")); + for (i, p) in time.into_iter().enumerate() { + if i > 0 && !matches!(p, Part::AmPm) { + out.push(text(":")); + } else if matches!(p, Part::AmPm) { + out.push(text(" ")); + } + out.push(p); + } + } + out +} + +/// English month names, in order; a month shown is its name or the start of it (three letters +/// or more). +const MONTHS: [&str; 12] = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"]; + +/// The value a date or time shown as `shown` under Word picture `pic` stands for, as a fixed +/// field's `text:date-value` / `text:time-value` (§19.871, §19.880) state it: `YYYY-MM-DD`, with +/// `Thh:mm:ss` when it shows a time (on 1899-12-30, ODF's day 0, when it shows no date). `None` +/// when the text doesn't read under the picture (month names in another language, say). +pub(super) fn value_shown(pic: &str, shown: &str) -> Option { + let parts = parts_of(pic); + let s: Vec = shown.trim().chars().take(MAX_PICTURE).collect(); + let mut i = 0usize; + let (mut day, mut month, mut year) = (None, None, None); + let (mut hour, mut minute, mut second, mut pm) = (None, None, 0u32, None); + // A run of digits (at most `max`) or of letters at `i`. + let digits = |i: &mut usize, max: usize| -> Option { + let start = *i; + while *i - start < max && s.get(*i).is_some_and(char::is_ascii_digit) { + *i += 1; + } + s.get(start..*i).filter(|d| !d.is_empty())?.iter().collect::().parse().ok() + }; + let word = |i: &mut usize| -> String { + let start = *i; + while s.get(*i).is_some_and(|c| c.is_alphabetic()) { + *i += 1; + } + s.get(start..*i).map(|w| w.iter().collect::().to_lowercase()).unwrap_or_default() + }; + for p in &parts { + match p { + Part::Text(t) => { + for c in t.chars() { + match s.get(i) { + Some(&got) if got.eq_ignore_ascii_case(&c) => i += 1, + // Spaces are loose: one shown where the picture has none, or none where it has one. + Some(&got) if got.is_whitespace() => i += 1, + _ if c.is_whitespace() => {} + _ => return None, + } + } + } + Part::Day(_) => day = Some(digits(&mut i, 2)?), + Part::Month(_, false) => month = Some(digits(&mut i, 2)?), + Part::Month(_, true) => { + let w = word(&mut i); + let n = MONTHS.iter().position(|m| w.len() >= 3 && m.starts_with(w.as_str()))?; + month = Some(u32::try_from(n).ok()? + 1); + } + Part::Year(_) => { + let start = i; + let y = digits(&mut i, 4)?; + // Two digits: Word's window, 1930–2029. + year = Some(if i - start <= 2 { if y < 30 { 2000 + y } else { 1900 + y } } else { y }); + } + Part::Weekday(_) => { + if word(&mut i).is_empty() { + return None; + } + } + Part::Hours(_) => hour = Some(digits(&mut i, 2)?), + Part::Minutes(_) => minute = Some(digits(&mut i, 2)?), + Part::Seconds(_) => second = digits(&mut i, 2)?, + Part::AmPm => { + let w = word(&mut i); + pm = Some(match w.as_str() { + "am" | "a" => false, + "pm" | "p" => true, + _ => return None, + }); + } + } + } + if i < s.len() { + return None; + } + let time = match (hour, minute) { + (Some(h), m) => { + let h = match pm { + Some(true) if h < 12 => h + 12, + Some(false) if h == 12 => 0, + _ => h, + }; + let m = m.unwrap_or(0); + (h < 24 && m < 60 && second < 60).then(|| format!("T{h:02}:{m:02}:{second:02}"))? + } + (None, Some(_)) => return None, + (None, None) => String::new(), + }; + let date = match (year, month, day) { + (Some(y), Some(m), Some(d)) if (1..=12).contains(&m) && (1..=31).contains(&d) && y <= 9999 => format!("{y:04}-{m:02}-{d:02}"), + (None, None, None) if !time.is_empty() => "1899-12-30".to_string(), + _ => return None, + }; + Some(format!("{date}{time}")) +} + +#[cfg(test)] +mod value_tests { + use super::value_shown; + + #[test] + fn a_shown_date_or_time_reads_back_under_its_picture() { + assert_eq!(value_shown("dddd, MMMM dd, yyyy", "Sunday, November 01, 2026").as_deref(), Some("2026-11-01")); + assert_eq!(value_shown("dd/MM/yy", "03/11/26").as_deref(), Some("2026-11-03")); + assert_eq!(value_shown("hh:mm AM/PM", "04:04 PM").as_deref(), Some("1899-12-30T16:04:00")); + assert_eq!(value_shown("d MMM yyyy H:mm:ss", "5 Mar 1999 7:08:09").as_deref(), Some("1999-03-05T07:08:09")); + // Not readable: another language's month, extra text, an impossible day, junk. + assert_eq!(value_shown("d MMMM yyyy", "5 mars 2026"), None); + assert_eq!(value_shown("d MMMM yyyy", "5 March 2026 and more"), None); + assert_eq!(value_shown("dd/MM/yyyy", "45/11/2026"), None); + assert_eq!(value_shown("dd/MM/yyyy", ""), None); + assert_eq!(value_shown("", "anything"), None); + } + + /// An apostrophe in a date style's text is two quotes in the picture, quoted or not, and + /// reads back as one. + #[test] + fn an_apostrophe_in_a_date_styles_text_round_trips() { + use super::{Part, parts_of, picture}; + for text in ["'", "''", "o'clock", "'tis", " ' "] { + let parts = vec![Part::Text(text.to_string())]; + let pic = picture(&parts); + assert_eq!(parts_of(&pic), parts, "{text:?} as {pic:?}"); + } + assert_eq!(picture(&[Part::Text("'".into())]), "''"); + } +} diff --git a/crates/odf/src/text/fields.rs b/crates/odf/src/text/fields.rs new file mode 100644 index 000000000..becceccf8 --- /dev/null +++ b/crates/odf/src/text/fields.rs @@ -0,0 +1,543 @@ +//! Text fields (ODF 1.3 §7.3–7.7) and the Word field codes WordCraft keeps them as. +//! +//! Read: a field element becomes a field with that code and the element's text as its result: +//! a date or time with its date style as a `\@` picture, a sequence numbering figures or +//! tables as `SEQ`, a chapter field as `STYLEREF` to its heading style, references to +//! bookmarks, reference marks and sequences as `REF`/`PAGEREF`. A fixed date or time +//! (`text:fixed="true"`) is plain text, as is a field with no Word equivalent (variables, a +//! reference to a caption's text…), so nothing it shows is lost. Written: a field whose code +//! has an ODF element is that element around its result; any other field (or one with a code +//! ODF can't hold) is its result as text. + +use super::writer::x; +use super::xml::{Attrs, get}; + +/// Longest bookmark name a reference field keeps; a longer one leaves the field as text. +const MAX_REF_NAME: usize = 1024; + +/// ODF element (local name) ↔ Word field code, for the fields that take no argument. +const SIMPLE: &[(&str, &str)] = &[ + ("page-count", "NUMPAGES"), + ("date", "DATE"), + ("time", "TIME"), + ("title", "TITLE"), + ("subject", "SUBJECT"), + ("initial-creator", "AUTHOR"), + ("creator", "LASTSAVEDBY"), + ("author-name", "USERNAME"), + ("description", "COMMENTS"), + ("keywords", "KEYWORDS"), + ("file-name", "FILENAME"), + ("word-count", "NUMWORDS"), + ("character-count", "NUMCHARS"), + ("creation-date", "CREATEDATE"), + ("modification-date", "SAVEDATE"), + ("print-date", "PRINTDATE"), +]; + +/// The sender fields (§7.3.6 `text:sender-firstname` … §7.3.7 `text:sender-state-or-province`). +const SENDER_PARTS: [&str; 18] = [ + "firstname", + "lastname", + "initials", + "title", + "position", + "email", + "phone-private", + "fax", + "company", + "phone-work", + "street", + "city", + "postal-code", + "country", + "state-or-province", + "country-code", + "street-number", + "address", +]; + +/// `el`, a field element, fixed at what it shows (§19.807 `text:fixed`), for the fields that can +/// be. +/// Field element `el` for locked field `instr` showing `result`, fixed (see [`fixed`]). A date +/// or time is fixed at the value its text shows under its picture (§19.871, §19.880); `None` +/// when that can't be read, as a reader shows a fixed date without a value as today's (then it +/// is written as its text). +pub(super) fn locked(el: &str, instr: &str, result: &str) -> Option { + let code = instr.split_whitespace().next().unwrap_or("").to_ascii_uppercase(); + let attr = match code.as_str() { + "DATE" => "text:date-value", + "TIME" => "text:time-value", + _ => return Some(fixed(el)), + }; + let value = super::datastyle::picture_of(instr).and_then(|pic| super::datastyle::value_shown(&pic, result))?; + let el = fixed(el); + let (tag, rest) = el.find(['>', ' ']).and_then(|at| Some((el.get(..at)?, el.get(at..)?)))?; + Some(format!("{tag} {attr}=\"{value}\"{rest}")) +} + +pub(super) fn fixed(el: &str) -> String { + with_fixed(el, true) +} + +/// `el`, a field element, an author field said not to be fixed (§19.807's default, said: +/// LibreOffice reads an author field without `text:fixed` as fixed, showing what it holds, and +/// writes `false` itself). Other fields are written as LibreOffice writes them, without it. +pub(super) fn unfixed(el: &str) -> String { + if el.starts_with(" String { + const FIXABLE: [&str; 6] = ["', ' ']).and_then(|at| Some((el.get(..at)?, el.get(at..)?))) { + Some((tag, rest)) => format!("{tag} text:fixed=\"{fixed}\"{rest}"), + None => el.to_string(), + } +} + +/// The Word field code for the `text:` field element `local` with attributes `a`, or `None` +/// when it is no field WordCraft keeps (then its text is read as text). +pub(super) fn instr_of(local: &str, a: &Attrs, picture: impl Fn(&str) -> Option) -> Option { + // Fixed, unless it holds no value to be fixed at (LibreOffice's templates: a date that shows + // the day the document is made from them). + // A name a field code can hold as one word. + let name = |k: &str| { + get(a, k) + .map(str::trim) + .filter(|n| !n.is_empty() && n.len() <= MAX_REF_NAME && !n.contains(|c: char| c.is_whitespace() || c == '"' || c == '\\')) + }; + match local { + // A variable's value (§7.4.5, §7.4.6): what a SET gave its name. + "variable-get" | "variable-set" => Some(format!("REF {}", name("text:name")?)), + // A document property's value (§7.5.6 `text:user-defined`, a `meta:user-defined`) or a + // user field's (§7.4.9 `text:user-field-get`, declared once for the document): Word's + // `DOCPROPERTY`, the property kept as a custom document property. + "user-defined" | "user-field-get" => { + let n = clean_prop_name(get(a, "text:name")?)?; + Some(format!("DOCPROPERTY \"{n}\"")) + } + // A citation (§8.1.24 `text:bibliography-mark`): Word's `CITATION` of the source it + // carries (see [`bib_source`]). + "bibliography-mark" => bib_tag(get(a, "text:identifier")?).map(|t| format!("CITATION {t}")), + // An index entry at a point (§8.1.10): Word's `XE`, its keys the levels above it. + "alphabetical-index-mark" => xe_instr(get(a, "text:key1"), get(a, "text:key2"), get(a, "text:string-value")?), + // Numbering figures, tables and the like (§7.4.13): Word's `SEQ` and its number format. + "sequence" => { + let seq = name("text:name")?; + let fmt = match get(a, "style:num-format").map(str::trim) { + Some("i") => "roman", + Some("I") => "ROMAN", + Some("a") => "alphabetic", + Some("A") => "ALPHABETIC", + _ => "ARABIC", + }; + Some(format!("SEQ {seq} \\* {fmt}")) + } + // The chapter a page is in (§7.3.8): the nearest heading of its level, by name or number. + "chapter" => { + let level = get(a, "text:outline-level").and_then(|l| l.trim().parse::().ok()).unwrap_or(1).clamp(1, 9); + let number = matches!(get(a, "text:display").map(str::trim), Some("number" | "plain-number")); + Some(format!("STYLEREF \"Heading {level}\"{}", if number { " \\n" } else { "" })) + } + // References to reference marks (§7.7.5) and to sequences (§7.7.7): to the text or the + // page; a sequence's label and number is the bookmark over them. + "reference-ref" | "sequence-ref" => { + let target = name("text:ref-name")?; + match (local, get(a, "text:reference-format").map(str::trim)) { + (_, Some("page")) => Some(format!("PAGEREF {target} \\h")), + ("reference-ref", Some("text") | None) | ("sequence-ref", Some("category-and-value")) => Some(format!("REF {target} \\h")), + _ => None, + } + } + // Only the current page's number: "previous" and "next" have no Word equivalent. + "page-number" => (get(a, "text:select-page").is_none_or(|v| v.trim() == "current")).then(|| { + // Its own number format, over the page style's (Word's `\*` format switch). + match get(a, "style:num-format").map(str::trim) { + Some("1") => "PAGE \\* Arabic".to_string(), + Some("i") => "PAGE \\* roman".to_string(), + Some("I") => "PAGE \\* ROMAN".to_string(), + Some("a") => "PAGE \\* alphabetic".to_string(), + Some("A") => "PAGE \\* ALPHABETIC".to_string(), + _ => "PAGE".to_string(), + } + }), + "bookmark-ref" => { + let name = + get(a, "text:ref-name").map(str::trim).filter(|n| !n.is_empty() && n.len() <= MAX_REF_NAME && !n.contains(char::is_whitespace))?; + Some(match get(a, "text:reference-format").map(str::trim) { + Some("page") => format!("PAGEREF {name} \\h"), + Some("number" | "number-all-superior" | "number-no-superior") => format!("REF {name} \\r \\h"), + _ => format!("REF {name} \\h"), + }) + } + // The sender's details (§7.3.6–7.3.7), from the application's user data: WordCraft's + // own `SENDER` field code, shown as its last result. + _ if local.strip_prefix("sender-").is_some_and(|p| SENDER_PARTS.contains(&p)) => local.strip_prefix("sender-").map(|p| format!("SENDER {p}")), + _ => { + let code = SIMPLE.iter().find(|(e, _)| *e == local).map(|(_, code)| code.to_string())?; + // A date or time in its date style (§19.473 `style:data-style-name`). + let pic = get(a, "style:data-style-name").and_then(&picture).filter(|_| code.contains("DATE") || code == "TIME"); + Some(match pic { + Some(p) => format!("{code} \\@ \"{p}\""), + None => code, + }) + } + } +} + +/// The ODF field element for Word field code `instr` around `result` (already escaped), or +/// `None` when ODF has none for it. +pub(super) fn element(instr: &str, result: &str, data_style: Option<&str>) -> Option { + let mut words = instr.split_whitespace(); + let code = words.next()?.to_ascii_uppercase(); + match code.as_str() { + "SENDER" => { + let part = words.next().filter(|p| SENDER_PARTS.contains(p))?; + Some(format!("{result}")) + } + "SEQ" => { + let seq = words.next().filter(|n| !n.starts_with('\\') && n.len() <= MAX_REF_NAME)?; + let fmt = match switch_arg(instr, '*').as_deref() { + Some("roman") => "i", + Some("ROMAN") => "I", + Some("alphabetic") => "a", + Some("ALPHABETIC") => "A", + _ => "1", + }; + let seq = x(seq); + Some(format!("{result}")) + } + // `STYLEREF "Heading 2"` or `STYLEREF 2`: the chapter of that level. + "STYLEREF" => { + let target = instr.trim_start().get(code.len()..)?.trim_start(); + let target = target.strip_prefix('"').and_then(|t| t.split('"').next()).unwrap_or_else(|| target.split_whitespace().next().unwrap_or("")); + let level = target.trim().strip_prefix("Heading").unwrap_or(target).trim().parse::().ok().filter(|l| (1..=10).contains(l))?; + let number = [" \\n", " \\r", " \\w"].iter().any(|s| instr.contains(s)); + let display = if number { "number" } else { "name" }; + Some(format!("{result}")) + } + "PAGE" => { + let fmt = match switch_arg(instr, '*').as_deref() { + Some(f) if f.eq_ignore_ascii_case("arabic") => " style:num-format=\"1\"", + Some("roman") => " style:num-format=\"i\"", + Some("ROMAN") => " style:num-format=\"I\"", + Some("alphabetic") => " style:num-format=\"a\"", + Some("ALPHABETIC") => " style:num-format=\"A\"", + _ => "", + }; + Some(format!("{result}")) + } + // An index entry: a mark at its place, its first levels the keys (§8.1.10). + "XE" => { + let entry = instr.trim_start().get(code.len()..)?.trim_start(); + let entry = entry.strip_prefix('"').and_then(|t| t.split('"').next())?; + let mut levels: Vec<&str> = entry.split(':').map(str::trim).filter(|l| !l.is_empty()).collect(); + let value = levels.pop()?; + let keys: String = levels.iter().take(2).enumerate().map(|(i, k)| format!(" text:key{}=\"{}\"", i + 1, x(k))).collect(); + Some(format!("", x(value))) + } + "REF" | "PAGEREF" => { + let name = words.next().filter(|n| !n.starts_with('\\') && n.len() <= MAX_REF_NAME)?; + let rest: Vec<&str> = words.collect(); + let format = if code == "PAGEREF" { + "page" + } else if rest.iter().any(|s| s.eq_ignore_ascii_case("\\r") || s.eq_ignore_ascii_case("\\n") || s.eq_ignore_ascii_case("\\w")) { + "number" + } else { + "text" + }; + Some(format!("{result}", x(name))) + } + _ => { + let (el, _) = SIMPLE.iter().find(|(_, c)| *c == code)?; + let ds = data_style.map(|d| format!(" style:data-style-name=\"{}\"", x(d))).unwrap_or_default(); + Some(format!("{result}")) + } + } +} + +/// A custom property's name as a field code can quote it: no quotes or backslashes, no control +/// characters, at most 255 characters. `None` when nothing is left. +pub(super) fn clean_prop_name(n: &str) -> Option { + let n: String = n.chars().filter(|c| !matches!(c, '"' | '\\') && !c.is_control()).take(255).collect(); + let n = n.trim(); + (!n.is_empty()).then(|| n.to_string()) +} + +/// The property a `DOCPROPERTY` field code names, if it is one. +pub(super) fn docproperty_name(instr: &str) -> Option<&str> { + let rest = instr.trim_start().strip_prefix("DOCPROPERTY")?.trim_start(); + match rest.strip_prefix('"') { + Some(q) => q.split('"').next(), + None => rest.split_whitespace().next(), + } + .map(str::trim) + .filter(|n| !n.is_empty() && !n.starts_with('\\')) +} + +/// Most custom properties read from a document. +pub(super) const MAX_PROPS: usize = 1000; +/// Longest custom property value read, characters. +const MAX_PROP_VALUE: usize = 4096; + +/// A custom property of ODF value type `t` (`office:value-type`, `meta:value-type`) and value +/// `v`: as a document's custom property (OOXML variant type, value). +pub(super) fn custom_prop(name: &str, t: Option<&str>, v: &str) -> Option { + let name = clean_prop_name(name)?; + let kind = match t.map(str::trim) { + Some("float" | "percentage" | "currency") => "r8", + Some("boolean") => "bool", + Some("date") => "filetime", + _ => "lpwstr", + }; + let mut value: String = v.chars().filter(|c| !c.is_control() || *c == '\n').take(MAX_PROP_VALUE).collect(); + let mut kind = kind; + if kind == "filetime" { + // A date and time in UTC, as Word's `filetime` holds it: a bare date at midnight; one + // that isn't a date is kept as text. + let v = value.trim(); + let date_ok = v.len() >= 10 + && v.is_char_boundary(10) + && v.get(..10).is_some_and(|d| d.bytes().enumerate().all(|(i, b)| if i == 4 || i == 7 { b == b'-' } else { b.is_ascii_digit() })); + value = match (date_ok, v.get(10..)) { + (true, Some("")) => format!("{v}T00:00:00Z"), + (true, Some(rest)) if rest.ends_with('Z') || rest.contains('+') || rest.rfind('-').is_some() => v.to_string(), + (true, Some(_)) => format!("{v}Z"), + _ => { + kind = "lpwstr"; + v.to_string() + } + }; + } + Some(craft_wordmodel::CustomProp { name, kind: kind.into(), value, link_target: None, odf_user_field: false }) +} + +/// The custom properties of a package's `meta.xml` (§4.3.2.14 `meta:user-defined`). +pub(super) fn meta_props(xml: &[u8]) -> Vec { + let mut out: Vec = Vec::new(); + let mut open: Option<(String, Option, String)> = None; + super::events(xml, |ev| match ev { + super::Ev::Start(q, a) if q == "meta:user-defined" => { + open = get(&a, "meta:name").map(|n| (n.to_string(), get(&a, "meta:value-type").map(String::from), String::new())); + } + super::Ev::Text(t) => { + if let Some((_, _, v)) = &mut open + && v.len() < MAX_PROP_VALUE * 4 + { + v.push_str(&t); + } + } + super::Ev::End(q) if q == "meta:user-defined" => { + if let Some((n, t, v)) = open.take() + && out.len() < MAX_PROPS + && let Some(p) = custom_prop(&n, t.as_deref(), &v) + && !out.iter().any(|o| o.name == p.name) + { + out.push(p); + } + } + _ => {} + }); + out +} + +/// The user fields a package's `content.xml` declares (§7.4.7 `text:user-field-decl`), as custom +/// properties. +pub(super) fn user_fields(xml: &[u8]) -> Vec { + let mut out: Vec = Vec::new(); + super::events(xml, |ev| { + if let super::Ev::Start(q, a) = ev + && q == "text:user-field-decl" + && out.len() < MAX_PROPS + { + let t = get(&a, "office:value-type"); + let v = match t.map(str::trim) { + Some("float" | "percentage" | "currency") => get(&a, "office:value"), + Some("boolean") => get(&a, "office:boolean-value"), + Some("date") => get(&a, "office:date-value"), + _ => get(&a, "office:string-value"), + } + .unwrap_or(""); + if let Some(p) = get(&a, "text:name").and_then(|n| custom_prop(n, t, v)) + && !out.iter().any(|o| o.name == p.name) + { + out.push(craft_wordmodel::CustomProp { odf_user_field: true, ..p }); + } + } + }); + out +} + +/// A source's tag as a field code names it: one word (spaces made underscores), at most 64 +/// characters, no quotes or backslashes. `None` when nothing is left. +pub(super) fn bib_tag(id: &str) -> Option { + let t: String = + id.trim().chars().filter(|c| !matches!(c, '"' | '\\') && !c.is_control()).map(|c| if c.is_whitespace() { '_' } else { c }).take(64).collect(); + (!t.is_empty()).then_some(t) +} + +/// ODF bibliography types (§19.817 `text:bibliography-type`) and WordCraft's source kinds. +const BIB_KINDS: [(&str, &str); 10] = [ + ("book", "book"), + ("inbook", "bookSection"), + ("incollection", "bookSection"), + ("article", "article"), + ("journal", "periodical"), + ("inproceedings", "conference"), + ("proceedings", "conference"), + ("conference", "conference"), + ("techreport", "report"), + ("www", "website"), +]; + +/// The source a `text:bibliography-mark` carries: its identifier as the tag, its type as the +/// kind, and its author, title, year, publisher, address, journal, volume, pages, URL, editor, +/// edition, number (issue) and month. `None` without an identifier. +pub(super) fn bib_source(a: &Attrs) -> Option { + let tag = bib_tag(get(a, "text:identifier")?)?; + let v = |k: &str| get(a, k).map(|s| s.chars().filter(|c| !c.is_control()).take(1024).collect::()).unwrap_or_default(); + let t = get(a, "text:bibliography-type").map(str::trim).unwrap_or(""); + let kind = BIB_KINDS.iter().find(|(o, _)| *o == t).map_or("other", |(_, k)| k).to_string(); + Some(craft_wordmodel::Source { + tag, + kind, + author: v("text:author"), + title: v("text:title"), + year: v("text:year"), + publisher: v("text:publisher"), + city: v("text:address"), + journal: v("text:journal"), + volume: v("text:volume"), + pages: v("text:pages"), + url: v("text:url"), + editor: v("text:editor"), + edition: v("text:edition"), + issue: v("text:number"), + month: v("text:month"), + ..Default::default() + }) +} + +/// A source as a `text:bibliography-mark` around `result` (already escaped). +pub(super) fn bib_mark(s: &craft_wordmodel::Source, result: &str) -> String { + let t = BIB_KINDS.iter().find(|(_, k)| *k == s.kind).map_or("misc", |(o, _)| o); + let author = if s.corporate.is_empty() { &s.author } else { &s.corporate }; + let mut a = format!(" text:identifier=\"{}\" text:bibliography-type=\"{t}\"", x(&s.tag)); + for (k, v) in [ + ("author", author), + ("title", &s.title), + ("year", &s.year), + ("publisher", &s.publisher), + ("address", &s.city), + ("journal", &s.journal), + ("volume", &s.volume), + ("pages", &s.pages), + ("url", &s.url), + ("editor", &s.editor), + ("edition", &s.edition), + ("number", &s.issue), + ("month", &s.month), + ] { + if !v.is_empty() { + a.push_str(&format!(" text:{k}=\"{}\"", x(v))); + } + } + format!("{result}") +} + +/// Most characters of an index entry's level. +const MAX_ENTRY_LEVEL: usize = 255; + +/// The `XE` field code of an index entry `value` under keys `key1` and `key2` (its levels, top +/// first). Quotes, which end the entry, and colons, which split levels, are left out of each. +/// `None` when the entry is empty. +pub(super) fn xe_instr(key1: Option<&str>, key2: Option<&str>, value: &str) -> Option { + let clean = |s: &str| { + s.chars().filter(|c| !matches!(c, '"' | ':' | '\\') && !c.is_control()).take(MAX_ENTRY_LEVEL).collect::().trim().to_string() + }; + let value = clean(value); + if value.is_empty() { + return None; + } + let levels: Vec = [key1, key2].into_iter().flatten().map(clean).filter(|k| !k.is_empty()).chain([value]).collect(); + Some(format!("XE \"{}\"", levels.join(":"))) +} + +/// The argument of switch `\sw` in field code `instr` (quoted or one word). +fn switch_arg(instr: &str, sw: char) -> Option { + let at = instr.find(&format!("\\{sw}"))?; + let rest = instr.get(at + 2..)?.trim_start(); + let arg = match rest.strip_prefix('"') { + Some(q) => q.split('"').next()?, + None => rest.split_whitespace().next()?, + }; + Some(arg.chars().take(64).collect()) +} + +/// The sequence (`SEQ`) field code `instr` counts, if it is one. +pub(super) fn sequence_name(instr: &str) -> Option<&str> { + let mut words = instr.split_whitespace(); + words.next().filter(|c| c.eq_ignore_ascii_case("SEQ"))?; + words.next().filter(|n| !n.starts_with('\\') && n.len() <= MAX_REF_NAME) +} + +/// The SET field (ECMA-376 §17.16.5.58) a `text:variable-set` with attributes `a` is: its name and +/// value (§19.383 `office:string-value`, else the number, else the formula without its prefix). +pub(super) fn set_instr(a: &Attrs) -> Option { + let name = get(a, "text:name") + .map(str::trim) + .filter(|n| !n.is_empty() && n.len() <= MAX_REF_NAME && !n.contains(|c: char| c.is_whitespace() || c == '"' || c == '\\'))?; + let value = get(a, "office:string-value") + .or_else(|| get(a, "office:value")) + .or_else(|| get(a, "text:formula").map(|f| f.trim().strip_prefix("ooow:").unwrap_or(f.trim()))) + .unwrap_or_default(); + let value: String = value.chars().take(MAX_FIELD_VALUE).collect(); + Some(format!("SET {name} \"{}\"", value.replace('\\', "\\\\").replace('"', "\\\""))) +} + +/// Most characters of a variable's value kept. +const MAX_FIELD_VALUE: usize = 10_000; + +/// A SET field's name and value, when `instr` is one. +pub(super) fn set_of(instr: &str) -> Option<(&str, String)> { + let rest = instr.trim_start().strip_prefix("SET")?.strip_prefix(char::is_whitespace)?.trim_start(); + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + let (name, value) = (rest.get(..end)?, rest.get(end..)?.trim()); + if name.is_empty() { + return None; + } + // A quoted value, its \" and \\ undone; else the words as they are. + let value = match value.strip_prefix('"') { + Some(q) => { + let mut out = String::new(); + let mut it = q.chars(); + while let Some(c) = it.next() { + match c { + '"' => break, + '\\' => out.extend(it.next()), + c => out.push(c), + } + } + out + } + None => value.to_string(), + }; + Some((name, value)) +} + +#[cfg(test)] +mod tests { + /// Word writes field codes with a leading space (` STYLEREF 2 `); the argument is read after + /// the code all the same. + #[test] + fn a_field_code_with_leading_space_keeps_its_argument() { + let chapter = super::element(" STYLEREF \"Heading 2\" ", "Intro", None).unwrap_or_default(); + assert!(chapter.contains("text:outline-level=\"2\""), "{chapter}"); + assert_eq!(super::element("STYLEREF 2", "Intro", None), super::element(" STYLEREF 2", "Intro", None)); + assert_eq!(super::element(" XE \"Apple\" ", "", None), super::element("XE \"Apple\"", "", None)); + assert!(super::element(" XE \"Apple\" ", "", None).is_some_and(|e| e.contains("Apple"))); + } +} diff --git a/crates/odf/src/text/flat_doc.rs b/crates/odf/src/text/flat_doc.rs new file mode 100644 index 000000000..4bff8b183 --- /dev/null +++ b/crates/odf/src/text/flat_doc.rs @@ -0,0 +1,24 @@ +//! Flat OpenDocument text (`.fodt`): taken apart into a package ([`crate::flat`]) and opened as +//! one; saved by putting a package's files together. + +use craft_wordmodel::{Document, SectionProps}; + +use crate::{Limits, flat, mime}; + +/// Open a flat OpenDocument text document. +pub fn import_flat(bytes: &[u8]) -> Result { + import_flat_on(bytes, &SectionProps::default()) +} + +/// [`import_flat`] onto `page` (see [`super::import_on`]). +pub fn import_flat_on(bytes: &[u8], page: &SectionProps) -> Result { + let package = flat::to_package(bytes, mime::TEXT, &Limits::DEFAULT).map_err(|e| format!("FODT: {e}"))?; + super::import_on(&package, page) +} + +/// Save `doc` as flat OpenDocument text: its package's files in one `office:document`, pictures and +/// formulas inline. +pub fn export_flat(doc: &Document) -> Result, String> { + let package = super::export(doc)?; + flat::from_package(&package, &Limits::DEFAULT).map_err(|e| format!("FODT: {e}")) +} diff --git a/crates/odf/src/text/forms.rs b/crates/odf/src/text/forms.rs new file mode 100644 index 000000000..e97d52ee1 --- /dev/null +++ b/crates/odf/src/text/forms.rs @@ -0,0 +1,409 @@ +//! Form controls (ODF 1.3 §13: `office:forms` at the start of the text, each control placed by +//! a `draw:control` in a paragraph) read as WordCraft content controls and written back: +//! text fields (`form:text`, `form:textarea`) as plain text controls, check boxes and radio +//! buttons as check boxes (a radio button with round symbols), list boxes as drop-down lists, +//! combo boxes as combo boxes, date fields as date controls. A control's name is its tag, a +//! check box's label its title. Buttons, images and other controls aren't read. + +use std::collections::HashMap; + +use craft_wordmodel::control::{ContentControl, ControlKind, ListItem}; + +use super::writer::x; +use super::xml::{Attrs, get}; + +/// Most controls read, list entries in one, and characters of a value. +const MAX_CONTROLS: usize = 10_000; +const MAX_ITEMS: usize = 1000; +const MAX_VALUE: usize = 10_000; + +/// A radio button's symbols: a selected and an empty circle. +pub(in crate::text) const RADIO_ON: char = '\u{25C9}'; +pub(in crate::text) const RADIO_OFF: char = '\u{25CB}'; + +/// Form controls WordCraft has no kind for, kept verbatim (see [`ContentControl::form_xml`]): +/// every control of ODF 1.3 §13.5 but those it reads (text fields and areas, check boxes, option +/// buttons, list and combo boxes, date fields) and hidden ones (kept apart). +const KEPT: [&str; 12] = [ + "form:button", + "form:image", + "form:fixed-text", + "form:file", + "form:value-range", + "form:frame", + "form:image-frame", + "form:generic-control", + "form:password", + "form:number", + "form:time", + "form:grid", +]; +/// Most bytes of a kept control's XML. +const MAX_KEPT: usize = 64 * 1024; +/// The namespace prefixes a kept control may use: those the content WordCraft writes declares. +const PREFIXES: [&str; 19] = [ + "office", "style", "text", "table", "draw", "fo", "xlink", "dc", "meta", "svg", "number", "form", "ooo", "ooow", "oooc", "script", "dom", + "loext", "xml", +]; + +/// The `text:placeholder-type` (§19.851) of a placeholder: what it stands for. +pub(in crate::text) fn placeholder_type(t: Option<&str>) -> &'static str { + match t.map(str::trim) { + Some("table") => "table", + Some("text-box") => "text-box", + Some("image") => "image", + Some("object") => "object", + _ => "text", + } +} + +/// A placeholder's start tag, which a content control read from one keeps (in +/// [`ContentControl::form_xml`]) to be written back as one. +pub(in crate::text) fn placeholder_start(kind: &str) -> String { + format!("") +} + +/// Whether control `c` was read from a placeholder. +pub(in crate::text) fn is_placeholder(c: &ContentControl) -> bool { + c.form_xml.starts_with(" String { + s.chars().take(MAX_VALUE).collect() +} + +/// `office:forms`, read as it streams past. +#[derive(Default)] +pub(in crate::text) struct FormsReader { + /// How deep inside `office:forms` the reader is (0: outside). + depth: usize, + /// The control being read: its id, control, selected entries. + open: Option<(String, ContentControl, Vec)>, + /// The controls read, by `form:id` (or `xml:id`). + pub(in crate::text) controls: HashMap, + /// Hidden controls (§13.5.15 `form:hidden`: a name and a value the form sends, not shown), + /// kept verbatim without their ids. + pub(in crate::text) hidden: Vec, + /// A control WordCraft has no kind for being kept: its id, name, label, its XML so far (its + /// id left out) and how many of its elements are open. + kept: Option, +} + +struct Kept { + id: String, + name: String, + label: String, + xml: String, + open: usize, + /// It uses a prefix the content WordCraft writes doesn't declare: not kept. + foreign: bool, +} + +/// Whether name `q` has a prefix the content WordCraft writes declares. +fn known(q: &str) -> bool { + q.split_once(':').is_some_and(|(p, _)| PREFIXES.contains(&p)) +} + +/// Whether element `q` with attributes `a` uses only declared prefixes. +fn declared(q: &str, a: &Attrs) -> bool { + known(q) && a.iter().all(|(k, _)| known(k)) +} + +/// Element `q` with attributes `a` as XML, its ids (`form:id`, `xml:id`) left out. +fn element_xml(q: &str, a: &Attrs) -> String { + let mut s = format!("<{q}"); + for (k, v) in a.iter().filter(|(k, _)| k != "form:id" && k != "xml:id") { + s.push_str(&format!(" {k}=\"{}\"", x(v))); + } + s.push('>'); + s +} + +impl FormsReader { + /// Element `q` starts: whether it is part of `office:forms` (and so read here). + pub(in crate::text) fn start(&mut self, q: &str, a: &Attrs) -> bool { + if self.depth == 0 { + if q != "office:forms" { + return false; + } + self.depth = 1; + return true; + } + self.depth += 1; + if let Some(k) = &mut self.kept { + if k.xml.len() < MAX_KEPT { + k.xml.push_str(&element_xml(q, a)); + } + k.foreign |= !declared(q, a); + k.open += 1; + return true; + } + let id = get(a, "form:id").or_else(|| get(a, "xml:id")).unwrap_or_default().to_string(); + // A hidden control: kept as it is, its properties with it (its id is given anew). + if q == "form:hidden" && self.hidden.len() < MAX_CONTROLS { + self.kept = Some(Kept { + id: String::new(), + name: String::new(), + label: String::new(), + xml: element_xml(q, a), + open: 1, + foreign: !declared(q, a), + }); + return true; + } + if KEPT.contains(&q) && !id.is_empty() && self.controls.len() < MAX_CONTROLS { + let name = get(a, "form:name").map(capped).unwrap_or_default(); + let label = get(a, "form:label").map(capped).filter(|l| !l.is_empty()).unwrap_or_else(|| name.clone()); + self.kept = Some(Kept { id, name, label, xml: element_xml(q, a), open: 1, foreign: !declared(q, a) }); + return true; + } + let name = get(a, "form:name").map(capped).unwrap_or_default(); + let value = || get(a, "form:current-value").or_else(|| get(a, "form:value")).map(capped).unwrap_or_default(); + let kind = match q { + "form:text" | "form:formatted-text" => Some((ControlKind::Text { multi_line: false }, value())), + "form:textarea" => Some((ControlKind::Text { multi_line: true }, value())), + "form:checkbox" | "form:radio" => { + let radio = q == "form:radio"; + let on = if radio { + get(a, "form:current-selected").or_else(|| get(a, "form:selected")) == Some("true") + } else { + get(a, "form:current-state").or_else(|| get(a, "form:state")) == Some("checked") + }; + let (checked_char, unchecked_char) = + if radio { (RADIO_ON, RADIO_OFF) } else { (craft_wordmodel::control::CHECKED, craft_wordmodel::control::UNCHECKED) }; + let symbol = if on { checked_char } else { unchecked_char }; + let kind = + ControlKind::CheckBox { checked: on, checked_char, checked_font: String::new(), unchecked_char, unchecked_font: String::new() }; + Some((kind, symbol.to_string())) + } + "form:listbox" => Some((ControlKind::DropDown { items: Vec::new(), last_value: String::new() }, String::new())), + "form:combobox" => Some((ControlKind::ComboBox { items: Vec::new(), last_value: String::new() }, value())), + "form:date" => { + let v = value(); + // An xsd:date (`2026-11-01`). + let full_date = + if v.len() >= 10 && v.is_char_boundary(10) { format!("{}T00:00:00Z", v.get(..10).unwrap_or_default()) } else { String::new() }; + Some(( + ControlKind::Date { + full_date, + format: "yyyy-MM-dd".into(), + lid: String::new(), + calendar: String::new(), + store_as: String::new(), + }, + v, + )) + } + "form:option" | "form:item" => { + if let Some((_, c, selected)) = &mut self.open + && let ControlKind::DropDown { items, .. } | ControlKind::ComboBox { items, .. } = &mut c.kind + && items.len() < MAX_ITEMS + { + let display = get(a, "form:label").map(capped).unwrap_or_default(); + let value = get(a, "form:value").map(capped).filter(|v| *v != display).unwrap_or_default(); + if get(a, "form:current-selected").or_else(|| get(a, "form:selected")) == Some("true") { + selected.push(display.clone()); + } + items.push(ListItem { display, value }); + } + None + } + _ => None, + }; + if let Some((kind, text)) = kind + && !id.is_empty() + && self.controls.len() < MAX_CONTROLS + { + let mut c = ContentControl::new(kind); + c.tag = name; + c.title = get(a, "form:label").map(capped).unwrap_or_default(); + self.open = Some((id, c, vec![text])); + } + true + } + + /// Element `q` ends: whether it was part of `office:forms`. + pub(in crate::text) fn end(&mut self, q: &str) -> bool { + if self.depth == 0 { + return false; + } + self.depth -= 1; + if let Some(k) = &mut self.kept { + k.xml.push_str(&format!("")); + k.open = k.open.saturating_sub(1); + if k.open == 0 + && let Some(k) = self.kept.take() + { + // Too long to keep whole, or naming what WordCraft can't write: not kept. + if k.xml.len() <= MAX_KEPT && !k.foreign && k.id.is_empty() { + self.hidden.push(k.xml); + } else if k.xml.len() <= MAX_KEPT && !k.foreign { + let mut c = ContentControl::new(ControlKind::Text { multi_line: false }); + c.tag = k.name; + c.lock = craft_wordmodel::control::ControlLock::ContentLocked; + c.form_xml = k.xml; + self.controls.insert(k.id, (c, k.label)); + } + } + return true; + } + if matches!( + q, + "form:text" | "form:formatted-text" | "form:textarea" | "form:checkbox" | "form:radio" | "form:listbox" | "form:combobox" | "form:date" + ) && let Some((id, mut c, texts)) = self.open.take() + { + // The text: a list box's selected entry, else the value read with the control. + let mut text = match &mut c.kind { + ControlKind::DropDown { last_value, .. } => { + let sel = texts.get(1).cloned().unwrap_or_default(); + last_value.clone_from(&sel); + sel + } + ControlKind::ComboBox { last_value, .. } => { + let v = texts.first().cloned().unwrap_or_default(); + last_value.clone_from(&v); + v + } + _ => texts.first().cloned().unwrap_or_default(), + }; + // An empty text control or list shows its placeholder. + if text.is_empty() && !matches!(c.kind, ControlKind::CheckBox { .. }) { + c.showing_placeholder = true; + text = c.placeholder_shown().to_string(); + } + self.controls.insert(id, (c, text)); + } + true + } +} + +/// The text of a control's selected value, unless it shows its placeholder. +fn value(c: &ContentControl, text: &str) -> String { + if c.showing_placeholder { String::new() } else { text.to_string() } +} + +/// A content control as a form control in `office:forms`, with id `id`: the element, and the +/// size of its `draw:control` (points). `None` for kinds a form has no control for. +pub(in crate::text) fn form_xml(c: &ContentControl, text: &str, id: &str, n: usize) -> Option<(String, f32, f32)> { + let size = |(xml, w, h): (String, f32, f32)| { + let fit = |v: f32| if v.is_finite() { v.clamp(1.0, 1584.0) } else { 16.0 }; + match c.form_size { + Some([fw, fh]) => (xml, fit(fw), fit(fh)), + None => (xml, w, h), + } + }; + // A control kept verbatim, with its new id, while it is still the control it was read as + // (made another kind, it is written as that). + if !c.form_xml.is_empty() && c.kind == (ControlKind::Text { multi_line: false }) && !is_placeholder(c) { + let end = c.form_xml.find(|ch: char| ch.is_whitespace() || ch == '>')?; + let (head, rest) = (c.form_xml.get(..end)?, c.form_xml.get(end..)?); + return Some(size((format!("{head} xml:id=\"{id}\" form:id=\"{id}\"{rest}"), 72.0, 20.0))); + } + let name = if c.tag.is_empty() { format!("Control {n}") } else { c.tag.clone() }; + let common = format!(" form:name=\"{}\" xml:id=\"{id}\" form:id=\"{id}\"", x(&name)); + let label = if c.title.is_empty() { String::new() } else { format!(" form:label=\"{}\"", x(&c.title)) }; + let v = value(c, text); + // Wide enough for its text (at least a short word), one line high. + let width = |s: &str| (s.chars().count().clamp(8, 60) as f32 * 6.0 + 16.0).min(432.0); + // No `form:control-implementation`: the element says what the control is. + let impl_ = |_: &str| ""; + let out = match &c.kind { + ControlKind::Text { multi_line: false } => { + (format!("", impl_("TextField"), label, x(&v)), width(&v), 16.0) + } + ControlKind::Text { multi_line: true } => { + // Several lines (LibreOffice's `MultiLine` property): without it LibreOffice shows + // one line, centred in the box. + ( + format!( + "", + impl_("TextField"), + label, + x(&v) + ), + 216.0, + 48.0, + ) + } + ControlKind::CheckBox { checked, checked_char, .. } if *checked_char == RADIO_ON => ( + format!("", impl_("RadioButton")), + width(&c.title).min(18.0 + c.title.chars().count() as f32 * 6.0), + 16.0, + ), + ControlKind::CheckBox { checked, .. } => { + let state = if *checked { "checked" } else { "unchecked" }; + ( + format!("", impl_("CheckBox")), + 18.0 + c.title.chars().count() as f32 * 6.0, + 16.0, + ) + } + ControlKind::DropDown { items, last_value } => { + let chosen = if last_value.is_empty() { v.as_str() } else { last_value.as_str() }; + let options: String = items + .iter() + .map(|i| { + let value = if i.value.is_empty() { String::new() } else { format!(" form:value=\"{}\"", x(&i.value)) }; + let sel = if !chosen.is_empty() && (i.display == chosen || i.value() == chosen) { + " form:current-selected=\"true\" form:selected=\"true\"" + } else { + "" + }; + format!("", x(&i.display)) + }) + .collect(); + let widest = items.iter().map(|i| i.display.as_str()).max_by_key(|s| s.chars().count()).unwrap_or(""); + (format!("{options}", impl_("ListBox")), width(widest) + 14.0, 16.0) + } + ControlKind::ComboBox { items, .. } => { + let entries: String = items.iter().map(|i| format!("", x(&i.display))).collect(); + let widest = items.iter().map(|i| i.display.as_str()).chain([v.as_str()]).max_by_key(|s| s.chars().count()).unwrap_or(""); + ( + format!( + "{entries}", + impl_("ComboBox"), + x(&v) + ), + width(widest) + 14.0, + 16.0, + ) + } + ControlKind::Date { full_date, .. } => { + let date = full_date + .get(..10) + .filter(|d| d.len() == 10 && d.as_bytes().get(4) == Some(&b'-') && d.as_bytes().get(7) == Some(&b'-')) + .unwrap_or(""); + let current = if date.is_empty() || c.showing_placeholder { String::new() } else { format!(" form:current-value=\"{date}\"") }; + (format!("", impl_("DateField")), 96.0, 16.0) + } + _ => return None, + }; + Some(size(out)) +} + +/// The `office:forms` holding the controls `xml` (empty when there are none). +pub(in crate::text) fn forms_xml(xml: &[String], hidden: &[String]) -> String { + if xml.is_empty() && hidden.is_empty() { + return String::new(); + } + // Hidden controls, each with an id of its own. + let hidden: String = hidden + .iter() + .enumerate() + .filter_map(|(i, h)| { + let end = h.find(|ch: char| ch.is_whitespace() || ch == '>')?; + Some(format!("{} xml:id=\"hidden{i}\" form:id=\"hidden{i}\"{}", h.get(..end)?, h.get(end..)?)) + }) + .collect(); + format!( + "{}{hidden}", + xml.concat() + ) +} + +/// `Document::passthrough` key of a document's hidden form controls (their XML, one after +/// another), kept for the next save. +pub(in crate::text) const HIDDEN_PART: &str = "odf-hidden-controls"; diff --git a/crates/odf/src/text/formula_size.rs b/crates/odf/src/text/formula_size.rs new file mode 100644 index 000000000..23136f79f --- /dev/null +++ b/crates/odf/src/text/formula_size.rs @@ -0,0 +1,172 @@ +//! A rough size for an equation written as a formula object: LibreOffice scales a formula to its +//! frame, so the frame should be about the size the formula takes. A box per node (width, height +//! above and below the baseline, in ems of 12 pt text), sized by its structure. + +use craft_wordmodel::math::{MAX_DEPTH, MNode}; + +/// Width, ascent and descent, in ems. +#[derive(Clone, Copy, Default)] +struct Box3 { + w: f32, + asc: f32, + desc: f32, +} + +impl Box3 { + fn h(self) -> f32 { + self.asc + self.desc + } + fn scaled(self, k: f32) -> Box3 { + Box3 { w: self.w * k, asc: self.asc * k, desc: self.desc * k } + } +} + +/// Script and limit size relative to the base. +const SMALL: f32 = 0.7; + +fn row(nodes: &[MNode], depth: usize) -> Box3 { + if depth > MAX_DEPTH { + return Box3::default(); + } + nodes.iter().map(|n| node(n, depth + 1)).fold(Box3 { w: 0.0, asc: 0.75, desc: 0.25 }, |a, b| Box3 { + w: a.w + b.w, + asc: a.asc.max(b.asc), + desc: a.desc.max(b.desc), + }) +} + +fn text(s: &str) -> Box3 { + // Operators take room on both sides. + let w: f32 = s + .chars() + .map(|c| { + if "+-−=<>≤≥±×÷→←≠≈".contains(c) { + 1.0 + } else if c.is_whitespace() { + 0.25 + } else { + 0.55 + } + }) + .sum(); + Box3 { w, asc: 0.75, desc: 0.25 } +} + +fn node(n: &MNode, depth: usize) -> Box3 { + let r = |a: &[MNode]| row(a, depth); + match n { + MNode::Run(run) => text(&run.text), + MNode::Frac { num, den, .. } => { + let (a, b) = (r(num), r(den)); + Box3 { w: a.w.max(b.w) + 0.3, asc: a.h() + 0.35, desc: b.h() - 0.15 } + } + MNode::Script { base, sub, sup, .. } => { + let b = r(base); + let (sb, sp) = (r(sub).scaled(SMALL), r(sup).scaled(SMALL)); + let asc = if sup.is_empty() { b.asc } else { b.asc.max(sp.h() + 0.35) }; + let desc = if sub.is_empty() { b.desc } else { b.desc.max(sb.h() - 0.1) }; + Box3 { w: b.w + sb.w.max(sp.w) + 0.1, asc, desc } + } + MNode::Rad { deg, e, .. } => { + let b = r(e); + Box3 { w: b.w + 0.8 + r(deg).scaled(0.5).w, asc: b.asc + 0.2, desc: b.desc } + } + MNode::Nary { sub, sup, e, .. } => { + let b = r(e); + let (sb, sp) = (r(sub).scaled(SMALL), r(sup).scaled(SMALL)); + Box3 { + w: 1.0 + sb.w.max(sp.w) + b.w + 0.2, + asc: b.asc.max(1.0 + if sup.is_empty() { 0.0 } else { sp.h() }), + desc: b.desc.max(0.4 + if sub.is_empty() { 0.0 } else { sb.h() }), + } + } + MNode::Delim { elems, .. } => { + let inner = elems.iter().map(|a| r(a)).fold(Box3 { w: 0.0, asc: 0.75, desc: 0.25 }, |a, b| Box3 { + w: a.w + b.w + 0.3, + asc: a.asc.max(b.asc), + desc: a.desc.max(b.desc), + }); + Box3 { w: inner.w + 0.6, ..inner } + } + MNode::Func { name, e } => { + let (a, b) = (r(name), r(e)); + Box3 { w: a.w + b.w + 0.2, asc: a.asc.max(b.asc), desc: a.desc.max(b.desc) } + } + MNode::Lim { upper, e, lim } => { + let (b, l) = (r(e), r(lim).scaled(SMALL)); + let w = b.w.max(l.w); + if *upper { Box3 { w, asc: b.asc + l.h(), desc: b.desc } } else { Box3 { w, asc: b.asc, desc: b.desc + l.h() } } + } + MNode::Acc { e, .. } => { + let b = r(e); + Box3 { asc: b.asc + 0.3, ..b } + } + MNode::Bar { top, e } | MNode::GroupChr { top, e, .. } => { + let b = r(e); + if *top { Box3 { asc: b.asc + 0.3, ..b } } else { Box3 { desc: b.desc + 0.3, ..b } } + } + MNode::BorderBox { e, .. } | MNode::Boxed { e } => { + let b = r(e); + Box3 { w: b.w + 0.4, asc: b.asc + 0.2, desc: b.desc + 0.2 } + } + MNode::EqArr { rows } => stack(rows.iter().map(|a| r(a))), + MNode::Matrix { rows, .. } => { + let cols = rows.iter().map(Vec::len).max().unwrap_or(0); + let w: f32 = (0..cols).map(|c| rows.iter().filter_map(|row| row.get(c)).map(|a| r(a).w).fold(0.0, f32::max)).sum::() + + 0.6 * cols.saturating_sub(1) as f32; + let s = + stack(rows.iter().map(|row| { + row.iter().map(|a| r(a)).fold(Box3::default(), |a, b| Box3 { w: 0.0, asc: a.asc.max(b.asc), desc: a.desc.max(b.desc) }) + })); + Box3 { w, ..s } + } + MNode::Phant { show, zero_wid, zero_asc, zero_desc, e } => { + let b = r(e); + if !*show && *zero_wid && *zero_asc && *zero_desc { + return Box3::default(); + } + Box3 { w: if *zero_wid { 0.0 } else { b.w }, asc: if *zero_asc { 0.0 } else { b.asc }, desc: if *zero_desc { 0.0 } else { b.desc } } + } + } +} + +/// Rows one under another, centred on the math axis. +fn stack(rows: impl Iterator) -> Box3 { + let (mut w, mut h, mut n) = (0.0f32, 0.0f32, 0usize); + for b in rows { + w = w.max(b.w); + h += b.h(); + n += 1; + } + let h = h + 0.2 * n.saturating_sub(1) as f32; + Box3 { w, asc: h / 2.0 + 0.25, desc: h / 2.0 - 0.25 } +} + +/// Width and height in points of the equation `linear` set in 12 pt text. +pub(super) fn size(linear: &str) -> (f32, f32) { + let b = row(&craft_wordmodel::math::parse_linear(linear), 0); + let fit = |v: f32| if v.is_finite() { v } else { 1.0 }; + ((fit(b.w) * 12.0 + 2.0).clamp(4.0, 1584.0), (fit(b.h()) * 12.0 + 2.0).clamp(8.0, 1584.0)) +} + +#[cfg(test)] +mod tests { + use super::size; + + #[test] + fn structure_makes_room() { + let (xw, xh) = size("x"); + let (fw, fh) = size("(a+b)/c"); + assert!(fh > xh * 1.6, "a fraction is taller: {fh} vs {xh}"); + assert!(fw > xw * 2.0); + let (sw, sh) = size("x^2"); + assert!(sw > xw && sh > xh); + let (_, mh) = size("■(a&b@c&d)"); + assert!(mh > xh * 1.6, "{mh}"); + // Hostile: deep nesting and long text stay bounded. + let deep = "(".repeat(5000) + &")".repeat(5000); + let (dw, dh) = size(&deep); + assert!(dw.is_finite() && dh.is_finite() && dw <= 1584.0 && dh <= 1584.0); + assert!(size(&"x".repeat(100_000)).0 <= 1584.0); + } +} diff --git a/crates/odf/src/text/frames.rs b/crates/odf/src/text/frames.rs new file mode 100644 index 000000000..cb8766906 --- /dev/null +++ b/crates/odf/src/text/frames.rs @@ -0,0 +1,658 @@ +//! Where a picture's frame sits and how text wraps around it (ODF 1.3 §19.11 `text:anchor-type`, +//! §20.2xx `style:wrap`, `style:run-through`, `style:horizontal-pos`/`-rel`, +//! `style:vertical-pos`/`-rel`, `fo:margin-*` of `style:graphic-properties`), and WordCraft's +//! [`Float`] for it, both ways. + +use super::strokes::Defs; +use super::styles::parse_border; +use super::writer::pt; +use super::xml::{Attrs, get, length}; +use craft_flow::{BoxLook, parse_color}; +use craft_wordmodel::fill::Fill; +use craft_wordmodel::line_dash::LineDash; +use craft_wordmodel::para::{Anchor, Float, FloatAlign, ShapeKind, Wrap, WrapText}; +use craft_wordmodel::props::BorderStyle; +use craft_wordmodel::props::Rgb; + +/// Largest offset or distance read, points (22 inches). +const MAX_LEN: f32 = 1584.0; + +/// A drawing shape's fill colour when no style gives one (§20.119 `draw:fill-color` has no +/// default): WordCraft's shape blue. +const SHAPE_FILL: Rgb = Rgb(0x15, 0x60, 0x82); + +/// A drawing shape's line colour when no style gives one (§20.408 `svg:stroke-color` has no +/// default). +const SHAPE_LINE: Rgb = Rgb(0x0E, 0x40, 0x5A); + +/// A drawing shape's line width when no style gives one (§20.410 `svg:stroke-width` has no +/// default), and what a stated width of 0 (a hairline) draws at, points. +const HAIRLINE: f32 = 0.25; + +/// The placement a graphic style states; each part `None` where it says nothing (its parent's +/// then counts). +#[derive(Clone, Debug, Default)] +pub(super) struct Graphic { + wrap: Option, + run_through: Option, + hpos: Option, + hrel: Option, + vpos: Option, + vrel: Option, + /// `fo:margin-left`, `-right`, `-top`, `-bottom`. + margins: [Option; 4], + /// The frame's line (`fo:border`): colour and width, or `Some(None)` for none. + line: Option>, + /// `fo:padding-left`, `-top`, `-right`, `-bottom` (or `fo:padding` for each). + padding: [Option; 4], + /// Its fill style (`draw:fill`: none, solid, gradient, hatch, bitmap) and colour + /// (`draw:fill-color`), and its background (`fo:background-color`, `Some(None)` when + /// transparent). + fill: Option, + fill_color: Option, + /// `draw:opacity` (§20.130), percent. + opacity: Option, + background: Option>, + /// A drawing shape's line: its style (`draw:stroke`: none, solid, dash), colour and width. + stroke: Option, + stroke_color: Option, + stroke_width: Option, + /// A shape grows to fit its text (`draw:auto-grow-height`). + grow: Option, + /// The names of its dash (`draw:stroke-dash`), gradient (`draw:fill-gradient-name`) and + /// arrowheads (`draw:marker-start`, `draw:marker-end`; an empty name is none). + dash: Option, + gradient: Option, + markers: [Option; 2], + /// A picture's crop (`fo:clip`, §20.179): how much is cut off its top, right, bottom and + /// left, in lengths of its original size. + pub(super) clip: Option<[f32; 4]>, + /// Text follows the object's outline (`style:wrap-contour`, §20.401), into its hollows or + /// only along its outside (`style:wrap-contour-mode`, §20.402: `full` or `outside`). + wrap_contour: Option, + contour_mode: Option, + /// Which way a text box's text runs (§20.404 `style:writing-mode`; LibreOffice's upward + /// `bt-lr` is its own `loext:writing-mode`). + vert: Option, + /// The anchor and position a style gives frames that state none (§20.299 `text:anchor-type`, + /// `svg:x`, `svg:y` in `style:graphic-properties`: LibreOffice's label and Graphics styles). + anchor: Option, + x: Option, + y: Option, +} + +/// A `fo:clip` value, `rect(top, right, bottom, left)` (commas or spaces between; `auto` is 0): +/// each side's length, points. `None` when it isn't one. +fn clip_rect(v: &str) -> Option<[f32; 4]> { + let inner = v.trim().strip_prefix("rect(")?.strip_suffix(')')?; + let parts: Vec = inner + .split(|c: char| c == ',' || c.is_whitespace()) + .filter(|p| !p.is_empty()) + .map(|p| if p.trim() == "auto" { Some(0.0) } else { length(p).filter(|v| v.is_finite()) }) + .collect::>>()?; + let [t, r, b, l]: [f32; 4] = parts.try_into().ok()?; + Some([t, r, b, l].map(|v| v.clamp(0.0, MAX_LEN))) +} + +/// A frame's contour (§10.4.11 `draw:contour-polygon`: its `draw:points` in its `svg:viewBox`) +/// as a wrap polygon in WordCraft's wrap space (21600 across the object each way). `None` without +/// three usable points. +pub(super) fn contour_polygon(a: &Attrs) -> Option> { + let vb: Vec = get(a, "svg:viewBox")?.split(|c: char| c == ',' || c.is_whitespace()).filter_map(|v| v.parse().ok()).collect(); + let [x0, y0, vw, vh]: [f64; 4] = vb.try_into().ok()?; + if !(vw.is_finite() && vh.is_finite() && vw > 0.0 && vh > 0.0 && x0.is_finite() && y0.is_finite()) { + return None; + } + let space = f64::from(craft_wordmodel::wrap::WRAP_SPACE); + let nums: Vec = get(a, "draw:points")?.split(|c: char| c == ',' || c.is_whitespace()).filter_map(|v| v.parse().ok()).collect(); + // Points well outside the view box mean another coordinate system than the frame's (as + // LibreOffice writes for some pictures): the outline is then worked out from the object. + let outside = + nums.as_chunks::<2>().0.iter().any(|p| p[0] < x0 - vw * 0.1 || p[0] > x0 + vw * 1.1 || p[1] < y0 - vh * 0.1 || p[1] > y0 + vh * 1.1); + if outside { + return None; + } + let points = nums + .as_chunks::<2>() + .0 + .iter() + .take(craft_wordmodel::wrap::MAX_WRAP_POINTS) + .filter(|p| p.iter().all(|v| v.is_finite())) + .map(|p| (((p[0] - x0) / vw * space).round() as i64, ((p[1] - y0) / vh * space).round() as i64)); + craft_wordmodel::wrap::WrapPolygon::new(true, points).map(std::sync::Arc::new) +} + +/// A wrap polygon as a frame's `draw:contour-polygon`, the frame `w` × `h` points. +pub(super) fn contour_xml(p: &craft_wordmodel::wrap::WrapPolygon, w: f32, h: f32) -> String { + let points: Vec = p.points.iter().map(|[x, y]| format!("{x},{y}")).collect(); + let s = craft_wordmodel::wrap::WRAP_SPACE; + format!( + "", + pt(w), + pt(h), + points.join(" ") + ) +} + +impl Graphic { + /// A `style:graphic-properties` element. + pub(super) fn read(a: &Attrs) -> Graphic { + let s = |k: &str| get(a, k).map(|v| v.trim().to_string()); + let m = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite()).map(|v| v.clamp(0.0, MAX_LEN)); + Graphic { + wrap: s("style:wrap"), + run_through: s("style:run-through"), + hpos: s("style:horizontal-pos"), + hrel: s("style:horizontal-rel"), + vpos: s("style:vertical-pos"), + vrel: s("style:vertical-rel"), + margins: [m("fo:margin-left"), m("fo:margin-right"), m("fo:margin-top"), m("fo:margin-bottom")], + line: ["fo:border", "fo:border-top", "fo:border-left"] + .iter() + .find_map(|k| get(a, k)) + .and_then(parse_border) + .map(|b| (b.style != BorderStyle::None && b.width > 0.0).then(|| (b.color.unwrap_or(Rgb::BLACK), b.width.clamp(0.0, 12.0)))), + padding: ["left", "top", "right", "bottom"].map(|side| m(&format!("fo:padding-{side}")).or_else(|| m("fo:padding"))), + fill: s("draw:fill"), + fill_color: get(a, "draw:fill-color").and_then(parse_color), + opacity: get(a, "draw:opacity").and_then(|v| v.trim().strip_suffix('%')?.trim().parse::().ok()).filter(|v| v.is_finite()), + background: get(a, "fo:background-color").map(|v| parse_color(v).filter(|_| !v.trim().eq_ignore_ascii_case("transparent"))), + stroke: s("draw:stroke"), + stroke_color: get(a, "svg:stroke-color").and_then(parse_color), + grow: get(a, "draw:auto-grow-height").map(|v| v.trim() == "true"), + dash: s("draw:stroke-dash"), + gradient: s("draw:fill-gradient-name"), + markers: [s("draw:marker-start"), s("draw:marker-end")], + stroke_width: get(a, "svg:stroke-width").and_then(length).filter(|v| v.is_finite()).map(|v| v.clamp(0.0, 12.0)), + clip: get(a, "fo:clip").and_then(clip_rect), + wrap_contour: get(a, "style:wrap-contour").map(|v| v.trim() == "true"), + vert: get(a, "loext:writing-mode").or_else(|| get(a, "style:writing-mode")).map(|m| { + use craft_wordmodel::props::{TextDirection, TextVert}; + match super::styles::writing_direction(m) { + TextDirection::Down => TextVert::Vert, + TextDirection::Up => TextVert::Vert270, + TextDirection::Horizontal => TextVert::Horz, + } + }), + contour_mode: s("style:wrap-contour-mode"), + anchor: s("text:anchor-type"), + x: s("svg:x"), + y: s("svg:y"), + } + } + + /// A watermark's look from its shape's style: its fill colour, and whether it is see-through. + pub(super) fn watermark_look(&self) -> (Option, bool) { + (self.fill_color, self.opacity.is_some_and(|o| o < 100.0)) + } + + /// This placement with `parent`'s filling what it leaves unsaid. + pub(super) fn or(mut self, parent: &Graphic) -> Graphic { + let fill = |a: &mut Option, b: &Option| { + if a.is_none() { + a.clone_from(b); + } + }; + fill(&mut self.wrap, &parent.wrap); + fill(&mut self.run_through, &parent.run_through); + fill(&mut self.hpos, &parent.hpos); + fill(&mut self.hrel, &parent.hrel); + fill(&mut self.vpos, &parent.vpos); + fill(&mut self.vrel, &parent.vrel); + for (m, p) in self.margins.iter_mut().zip(parent.margins) { + *m = m.or(p); + } + for (m, p) in self.padding.iter_mut().zip(parent.padding) { + *m = m.or(p); + } + self.line = self.line.or(parent.line); + fill(&mut self.fill, &parent.fill); + self.fill_color = self.fill_color.or(parent.fill_color); + self.opacity = self.opacity.or(parent.opacity); + self.background = self.background.or(parent.background); + fill(&mut self.stroke, &parent.stroke); + self.stroke_color = self.stroke_color.or(parent.stroke_color); + self.stroke_width = self.stroke_width.or(parent.stroke_width); + self.clip = self.clip.or(parent.clip); + self.wrap_contour = self.wrap_contour.or(parent.wrap_contour); + self.vert = self.vert.or(parent.vert); + fill(&mut self.contour_mode, &parent.contour_mode); + fill(&mut self.anchor, &parent.anchor); + fill(&mut self.x, &parent.x); + fill(&mut self.y, &parent.y); + self.grow = self.grow.or(parent.grow); + fill(&mut self.dash, &parent.dash); + fill(&mut self.gradient, &parent.gradient); + for (m, p) in self.markers.iter_mut().zip(&parent.markers) { + fill(m, p); + } + self + } + + /// The fill a style of `draw:fill` `kind` gives, `default` when it states none: `None` is + /// no fill; a gradient, hatch or bitmap stands as its colour. + fn fill_with(&self, default: Option<&str>, defs: &Defs) -> Option { + let color = self.fill_color.or(self.background.flatten()); + let solid = |c: Option| c.map(Fill::solid); + match self.fill.as_deref().or(default) { + Some("none") => None, + Some("solid") => solid(color), + Some("gradient") => self.gradient.as_deref().and_then(|n| defs.gradient(n)).cloned().or_else(|| solid(color.or(Some(Rgb::WHITE)))), + Some(_) => solid(color.or(Some(Rgb::WHITE))), + None => solid(self.background.flatten()), + } + } + + /// The line's dash for a line `width` points wide, and its arrowheads (a named marker at + /// either end). + fn stroke_extra(&self, width: f32, defs: &Defs) -> (LineDash, (bool, bool)) { + let dash = match self.stroke.as_deref() { + Some("dash") => self.dash.as_deref().and_then(|n| defs.dash(n)).map_or(LineDash::Dash, |d| d.pattern(width)), + _ => LineDash::Solid, + }; + let marked = |m: &Option| m.as_deref().is_some_and(|m| !m.trim().is_empty()); + (dash, (marked(&self.markers[0]), marked(&self.markers[1]))) + } + + /// The look of drawing shape `kind` in this style: a solid fill and line where the styles + /// say nothing (the standard gives `draw:fill` and `draw:stroke` no default), in WordCraft's + /// colours and line width (nor do `draw:fill-color`, `svg:stroke-color` and + /// `svg:stroke-width` have defaults); a line has no fill. + pub(super) fn shape_look(&self, kind: ShapeKind, defs: &Defs) -> BoxLook { + let fill = if kind == ShapeKind::Line { + None + } else { + let mut g = self.clone(); + g.fill_color = g.fill_color.or(Some(SHAPE_FILL)); + g.fill_with(Some("solid"), defs) + }; + let line = match self.stroke.as_deref() { + Some("none") => None, + _ => Some((self.stroke_color.unwrap_or(SHAPE_LINE), self.stroke_width.filter(|w| *w > 0.0).unwrap_or(HAIRLINE))), + }; + // Only a shape that says it grows to its text does: WordCraft's fit shrinks it too. + let (dash, arrows) = self.stroke_extra(line.map_or(HAIRLINE, |l| l.1), defs); + BoxLook { + fill, + line, + dash, + arrows, + inset: Some(self.padding.map(|p| p.unwrap_or(0.0))), + auto_fit: self.grow == Some(true), + conn: None, + chain: None, + vert: Default::default(), + geometry: None, + } + } + + /// The look of a text box in a frame of this style, growing to its text when `auto_fit`. + /// What the styles leave unsaid is ODF's default: no line, no fill, no padding. + pub(super) fn look(&self, auto_fit: bool, defs: &Defs) -> BoxLook { + // A frame's line is its border (`fo:border`); without one stated, a drawn stroke + // (`draw:stroke`, as LibreOffice's letter logo box has), a 0 width a hairline. + let stroke = || { + matches!(self.stroke.as_deref(), Some("solid" | "dash")).then(|| { + let width = self.stroke_width.filter(|w| *w > 0.0).unwrap_or(HAIRLINE); + (self.stroke_color.unwrap_or(SHAPE_LINE), width) + }) + }; + BoxLook { + fill: self.fill_with(None, defs), + line: match self.line { + Some(line) => line, + None => stroke(), + }, + dash: LineDash::Solid, + arrows: (false, false), + inset: Some(self.padding.map(|p| p.unwrap_or(0.0))), + auto_fit, + conn: None, + chain: None, + vert: self.vert.unwrap_or_default(), + geometry: None, + } + } +} + +/// The [`Float`] of a frame anchored as `anchor` (`text:anchor-type`) at `x`, `y` (`svg:x`, +/// `svg:y`) with graphic style placement `g`. A frame in line with the text (`as-char`, or no +/// anchor stated) doesn't float. +pub(super) fn float_of(anchor: Option<&str>, x: Option<&str>, y: Option<&str>, g: &Graphic) -> Float { + // What the frame leaves unsaid, its style says. + let anchor = anchor.or(g.anchor.as_deref()).map(str::trim); + let (x, y) = (x.or(g.x.as_deref()), y.or(g.y.as_deref())); + if matches!(anchor, None | Some("as-char")) { + return Float::default(); + } + let page = anchor == Some("page"); + let (wrap, wrap_text) = match g.wrap.as_deref() { + Some("left") => (Wrap::Square, WrapText::Left), + Some("right") => (Wrap::Square, WrapText::Right), + Some("parallel") => (Wrap::Square, WrapText::BothSides), + Some("dynamic") | Some("biggest") => (Wrap::Square, WrapText::Largest), + Some("run-through") if g.run_through.as_deref() == Some("background") => (Wrap::BehindText, WrapText::BothSides), + Some("run-through") => (Wrap::InFrontOfText, WrapText::BothSides), + Some("none") => (Wrap::TopAndBottom, WrapText::BothSides), + // None stated (§20.400 gives no default): both sides, as LibreOffice wraps. + _ => (Wrap::Square, WrapText::BothSides), + }; + // Following its outline: only along its outside (Word's Tight), or into it as well (Through, + // `full` being the default). + let wrap = match (wrap, g.wrap_contour) { + (Wrap::Square, Some(true)) if g.contour_mode.as_deref() == Some("outside") => Wrap::Tight, + (Wrap::Square, Some(true)) => Wrap::Through, + (w, _) => w, + }; + let h_rel = match g.hrel.as_deref() { + Some("page") => Anchor::Page, + Some("page-start-margin") => Anchor::LeftMargin, + Some("page-end-margin") => Anchor::RightMargin, + Some("page-content") => Anchor::Margin, + // A page has no paragraph (§20.299: no such combination): relations to one are the + // page's text area. + Some("paragraph-content") if page => Anchor::Margin, + Some("paragraph-start-margin") if page => Anchor::LeftMargin, + Some("paragraph-end-margin") if page => Anchor::RightMargin, + Some("paragraph" | "char") if page => Anchor::Margin, + Some("paragraph-content") => Anchor::Column, + Some("char") if !page => Anchor::Character, + Some(_) | None if page => Anchor::Page, + Some(_) => Anchor::Paragraph, + None => Anchor::Paragraph, + }; + let v_rel = match g.vrel.as_deref() { + Some("page") => Anchor::Page, + Some("page-content") => Anchor::Margin, + Some("paragraph-content") if page => Anchor::Margin, + Some("paragraph" | "char" | "text" | "line" | "baseline") if page => Anchor::Margin, + Some("line") | Some("char") | Some("text") | Some("baseline") if !page => Anchor::Line, + Some(_) | None if page => Anchor::Page, + Some(_) => Anchor::Paragraph, + None => Anchor::Paragraph, + }; + let h_align = match g.hpos.as_deref() { + Some("left") => Some(FloatAlign::Start), + Some("center") => Some(FloatAlign::Center), + Some("right") => Some(FloatAlign::End), + Some("inside") => Some(FloatAlign::Inside), + Some("outside") => Some(FloatAlign::Outside), + _ => None, + }; + let v_align = match g.vpos.as_deref() { + Some("top") => Some(FloatAlign::Start), + Some("middle") => Some(FloatAlign::Center), + Some("bottom") => Some(FloatAlign::End), + _ => None, + }; + let off = |v: Option<&str>| v.and_then(length).filter(|v| v.is_finite()).map(|v| v.clamp(-MAX_LEN, MAX_LEN)).unwrap_or(0.0); + let [left, right, top, bottom] = g.margins; + Float { + wrap, + wrap_text, + h_rel, + v_rel, + x: if h_align.is_none() { off(x) } else { 0.0 }, + y: if v_align.is_none() { off(y) } else { 0.0 }, + h_align, + v_align, + dist: left.unwrap_or(0.0).max(right.unwrap_or(0.0)), + dist_top: top.unwrap_or(0.0), + dist_bottom: bottom.unwrap_or(0.0), + ..Float::default() + } +} + +/// Where a frame or shape `w` × `h` (points) sits: anchored as its `text:anchor-type` says, at +/// its `svg:x`/`svg:y`, or turned and moved by its `draw:transform`, in graphic style `g`, in +/// text `size` points tall. +pub(super) fn place(a: &Attrs, w: f32, h: f32, g: &Graphic, size: f32) -> Float { + let turned = get(a, "draw:transform").and_then(|t| transform(t, w, h)); + let (x, y) = match turned { + Some((x, y, _)) => (Some(pt(x)), Some(pt(y))), + None => (get(a, "svg:x").map(String::from), get(a, "svg:y").map(String::from)), + }; + let mut f = float_of(get(a, "text:anchor-type"), x.as_deref(), y.as_deref(), g); + if let Some((.., rot)) = turned { + f.rot = rot; + } + f.z = z_of(a); + f.page = page_of(get(a, "text:anchor-type"), get(a, "text:anchor-page-number"), g); + if f.wrap == Wrap::Inline { + f.y = inline_raise(g, y.as_deref().and_then(length), h, size); + } + f +} + +/// The page a drawing anchored as `anchor` (or as its style says) is on: `number`, its +/// `text:anchor-page-number` (§19.872). 0 when it states none, or isn't anchored to a page, where +/// LibreOffice ignores the number. +pub(super) fn page_of(anchor: Option<&str>, number: Option<&str>, g: &Graphic) -> u32 { + if anchor.or(g.anchor.as_deref()).map(str::trim) != Some("page") { + return 0; + } + number.and_then(|v| v.trim().parse::().ok()).unwrap_or(0).min(MAX_ANCHOR_PAGE) +} + +/// Highest anchor page kept. +const MAX_ANCHOR_PAGE: u32 = 1_000_000; + +/// A drawing's place in the stacking order (§19.231 `draw:z-index`, from 0 at the bottom) as +/// WordCraft's [`Float::z`] (higher on top, 0 for unknown): one more. +pub(super) fn z_of(a: &Attrs) -> u32 { + get(a, "draw:z-index").and_then(|v| v.trim().parse::().ok()).map_or(0, |z| z.saturating_add(1)) +} + +/// `float`'s place in the stacking order as `draw:z-index` (see [`z_of`]): 0 when unknown. +pub(crate) fn z_index(float: Option<&Float>) -> u32 { + float.map_or(0, |f| f.z.saturating_sub(1)) +} + +/// How far a frame `h` points tall in line with the text is raised from standing on the +/// baseline (points, negative lowered; WordCraft keeps it in an inline [`Float`]'s `y`): its +/// `style:vertical-pos` relative to the baseline (§20.387, §20.388; LibreOffice's placement +/// when no `style:vertical-rel` is given) — `top` stands on it, `middle` is centred on it, +/// `bottom` hangs from it, `from-top` has its top `svg:y` below it. Relative to the text +/// (`char`, `text`, in text `size` points tall), it is placed against the text's height, from +/// its descent to its ascent (a serif face's, as LibreOffice's templates use): `middle` centred +/// on it (measured in LibreOffice), `top` at its top, `bottom` at its bottom. Placement relative +/// to the line stays on the baseline. +fn inline_raise(g: &Graphic, y: Option, h: f32, size: f32) -> f32 { + if !h.is_finite() || g.vrel.as_deref() == Some("line") { + return 0.0; + } + if matches!(g.vrel.as_deref(), Some("char" | "text")) { + let size = if size.is_finite() { size.clamp(1.0, 1638.0) } else { 12.0 }; + let (ascent, descent) = (TEXT_ASCENT * size, TEXT_DESCENT * size); + let raise = match g.vpos.as_deref() { + Some("middle") => (ascent - descent) / 2.0 - h / 2.0, + Some("top") => ascent - h, + Some("bottom") => -descent, + _ => 0.0, + }; + return raise.clamp(-MAX_RAISE, MAX_RAISE); + } + // A relation an in-line frame can't have (§20.398 Table 23: a paragraph's or a page's, from + // a parent style for anchored frames): it stands on the baseline. + if !matches!(g.vrel.as_deref(), None | Some("baseline")) { + return 0.0; + } + let raise = match g.vpos.as_deref() { + Some("middle") => -h / 2.0, + Some("bottom") => -h, + Some("from-top") => -(h + y.filter(|v| v.is_finite()).unwrap_or(0.0)), + _ => 0.0, + }; + raise.clamp(-MAX_RAISE, MAX_RAISE) +} + +/// Most an inline frame is raised or lowered, points (22 inches). +const MAX_RAISE: f32 = 1584.0; + +/// The text's height above and below the baseline, of its size (Liberation Serif's ascent and +/// descent), for frames placed against the text. +const TEXT_ASCENT: f32 = 0.891; +const TEXT_DESCENT: f32 = 0.216; + +/// Most operations of a `draw:transform` read. +const MAX_TRANSFORMS: usize = 16; + +/// A `draw:transform` (ODF 1.3 §19.228) of an object `w` × `h`, as WordCraft places it: the +/// offset of its unrotated box and its clockwise turn about the box's centre, in degrees +/// (0–360). The file's operations apply in turn to the object's own coordinates, whose origin +/// is its top left corner: `rotate` turns it anticlockwise (radians), `translate` moves it and +/// `scale` scales it; `skewX`, `skewY` and `matrix` are left out. `None` when none reads. +pub(super) fn transform(v: &str, w: f32, h: f32) -> Option<(f32, f32, f32)> { + // x' = m0·x + m2·y + m4, y' = m1·x + m3·y + m5. + let mut m = [1.0f64, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut angle = 0.0f64; + let mut any = false; + let mut rest = v; + for _ in 0..MAX_TRANSFORMS { + let (Some(open), Some(close)) = (rest.find('('), rest.find(')')) else { break }; + let (Some(name), Some(args)) = (rest.get(..open), rest.get(open + 1..close)) else { break }; + let args: Vec<&str> = args.split(|c: char| c.is_whitespace() || c == ',').filter(|s| !s.is_empty()).collect(); + rest = rest.get(close + 1..).unwrap_or(""); + let num = |i: usize| args.get(i).and_then(|a| a.parse::().ok()).filter(|v| v.is_finite()); + let len = |i: usize| args.get(i).and_then(|a| length(a)).filter(|v| v.is_finite()).map(|v| f64::from(v.clamp(-MAX_LEN, MAX_LEN))); + match name.trim() { + "rotate" => { + let Some(a) = num(0) else { continue }; + let a = a % std::f64::consts::TAU; + let (sin, cos) = a.sin_cos(); + m = [ + cos * m[0] + sin * m[1], + -sin * m[0] + cos * m[1], + cos * m[2] + sin * m[3], + -sin * m[2] + cos * m[3], + cos * m[4] + sin * m[5], + -sin * m[4] + cos * m[5], + ]; + angle += a; + } + "translate" => { + let Some(tx) = len(0) else { continue }; + m[4] += tx; + m[5] += len(1).unwrap_or(0.0); + } + "scale" => { + let Some(sx) = num(0).map(|v| v.clamp(-100.0, 100.0)) else { continue }; + let sy = num(1).map_or(sx, |v| v.clamp(-100.0, 100.0)); + m = [m[0] * sx, m[1] * sy, m[2] * sx, m[3] * sy, m[4] * sx, m[5] * sy]; + } + _ => continue, + } + any = true; + } + if !any { + return None; + } + let (hw, hh) = (f64::from(w) / 2.0, f64::from(h) / 2.0); + let centre = (m[0] * hw + m[2] * hh + m[4], m[1] * hw + m[3] * hh + m[5]); + let fit = |v: f64| if v.is_finite() { v.clamp(-f64::from(MAX_LEN), f64::from(MAX_LEN)) as f32 } else { 0.0 }; + let rot = (-angle.to_degrees()).rem_euclid(360.0); + // Rounding can leave a whole turn. + let rot = if rot.is_finite() && rot < 359.9999 { rot as f32 } else { 0.0 }; + Some((fit(centre.0 - hw), fit(centre.1 - hh), rot)) +} + +/// The `draw:transform` of an object `w` × `h` whose unrotated box is at `x`, `y` and turned `rot` +/// degrees clockwise about its centre (the reverse of [`transform`]). +pub(super) fn transform_attr(x: f32, y: f32, w: f32, h: f32, rot: f32) -> String { + let fin = |v: f32| if v.is_finite() { f64::from(v.clamp(-MAX_LEN, MAX_LEN)) } else { 0.0 }; + let a = -fin(rot).to_radians(); + let (sin, cos) = a.sin_cos(); + let (hw, hh) = (fin(w) / 2.0, fin(h) / 2.0); + // The corner that ends up where the centre stays put. + let tx = fin(x) + hw - (cos * hw + sin * hh); + let ty = fin(y) + hh - (-sin * hw + cos * hh); + format!(" draw:transform=\"rotate ({a}) translate ({} {})\"", pt(tx as f32), pt(ty as f32)) +} + +/// Whether `f` turns its object. +pub(super) fn turned(f: &Float) -> bool { + f.rot.is_finite() && f.rot.rem_euclid(360.0) > 0.001 && f.rot.rem_euclid(360.0) < 359.999 +} + +/// How a floating picture is written: its `text:anchor-type`, its graphic style's +/// `style:graphic-properties` attributes, and the frame's `svg:x`/`svg:y` attributes, or its +/// `draw:transform` when it is turned (`w` × `h` its size). +pub(super) fn frame_of(f: &Float, w: f32, h: f32) -> (&'static str, String, String) { + // A page number holds until it is placed by its paragraph: moved in WordCraft. + let by_paragraph = + matches!(f.h_rel, Anchor::Column | Anchor::Character) || matches!(f.v_rel, Anchor::Paragraph | Anchor::Line | Anchor::Character); + let anchor = if (f.page > 0 && !by_paragraph) || (f.h_rel == Anchor::Page && f.v_rel == Anchor::Page) { + "page" + } else if f.v_rel == Anchor::Line { + "char" + } else { + "paragraph" + }; + let wrap = match (f.wrap, f.wrap_text) { + (Wrap::BehindText, _) => "run-through\" style:run-through=\"background", + (Wrap::InFrontOfText, _) => "run-through\" style:run-through=\"foreground", + (Wrap::TopAndBottom, _) => "none", + (_, WrapText::Left) => "left", + (_, WrapText::Right) => "right", + (_, WrapText::Largest) => "dynamic", + _ => "parallel", + }; + // Tight and Through: square wrapping along the outline (§20.401–20.402). + let contour = match f.wrap { + Wrap::Tight => "\" style:wrap-contour=\"true\" style:wrap-contour-mode=\"outside", + Wrap::Through => "\" style:wrap-contour=\"true\" style:wrap-contour-mode=\"full", + _ => "", + }; + let wrap = format!("{wrap}{contour}"); + let hrel = match f.h_rel { + Anchor::Page => "page", + Anchor::Margin | Anchor::InsideMargin | Anchor::OutsideMargin => "page-content", + Anchor::LeftMargin => "page-start-margin", + Anchor::RightMargin => "page-end-margin", + Anchor::Column => "paragraph-content", + Anchor::Character => "char", + _ => "paragraph", + }; + let vrel = match f.v_rel { + Anchor::Page | Anchor::TopMargin | Anchor::BottomMargin => "page", + Anchor::Margin => "page-content", + Anchor::Line => "line", + _ => "paragraph", + }; + let hpos = match f.h_align { + Some(FloatAlign::Start) => "left", + Some(FloatAlign::Center) => "center", + Some(FloatAlign::End) => "right", + Some(FloatAlign::Inside) => "inside", + Some(FloatAlign::Outside) => "outside", + None => "from-left", + }; + let vpos = match f.v_align { + Some(FloatAlign::Start) | Some(FloatAlign::Inside) => "top", + Some(FloatAlign::Center) => "middle", + Some(FloatAlign::End) | Some(FloatAlign::Outside) => "bottom", + None => "from-top", + }; + let d = |v: f32| pt(if v.is_finite() { v.clamp(0.0, MAX_LEN) } else { 0.0 }); + let props = format!( + " style:wrap=\"{wrap}\" style:horizontal-pos=\"{hpos}\" style:horizontal-rel=\"{hrel}\" style:vertical-pos=\"{vpos}\" style:vertical-rel=\"{vrel}\" fo:margin-left=\"{}\" fo:margin-right=\"{}\" fo:margin-top=\"{}\" fo:margin-bottom=\"{}\"", + d(f.dist), + d(f.dist), + d(f.dist_top), + d(f.dist_bottom) + ); + let mut xy = String::new(); + let p = |v: f32| pt(if v.is_finite() { v.clamp(-MAX_LEN, MAX_LEN) } else { 0.0 }); + if turned(f) { + let x = if f.h_align.is_none() { f.x } else { 0.0 }; + let y = if f.v_align.is_none() { f.y } else { 0.0 }; + return (anchor, props, transform_attr(x, y, w, h, f.rot)); + } + if f.h_align.is_none() { + xy.push_str(&format!(" svg:x=\"{}\"", p(f.x))); + } + if f.v_align.is_none() { + xy.push_str(&format!(" svg:y=\"{}\"", p(f.y))); + } + (anchor, props, xy) +} diff --git a/crates/odf/src/text/geometry.rs b/crates/odf/src/text/geometry.rs new file mode 100644 index 000000000..54edc974c --- /dev/null +++ b/crates/odf/src/text/geometry.rs @@ -0,0 +1,428 @@ +//! A custom shape's own geometry (ODF 1.3 §10.6.2 `draw:enhanced-geometry`): its path +//! (§19.145 `draw:enhanced-path`) in its view box, the path's values worked out from its +//! equations (§10.6.3 `draw:equation`, §19.170 `draw:formula`) and adjustment values +//! (`draw:modifiers`). Shapes whose type WordCraft has no kind for are drawn from it, as +//! freeforms. +//! +//! Arcs and ellipses are followed in short straight steps. Everything is bounded: the path's +//! length, the equations' count and nesting, the number of points. + +use std::collections::HashMap; + +use super::xml::{Attrs, get}; +use craft_wordmodel::freeform::Freeform; +use craft_wordmodel::para::ShapeKind; + +/// Most equations read, most levels one may refer down through, most path tokens and points. +const MAX_EQUATIONS: usize = 512; +const MAX_DEPTH: usize = 64; +const MAX_TOKENS: usize = 20_000; +const MAX_POINTS: usize = 20_000; +/// Farthest a point goes from the view box's origin, either way. +const MAX_COORD: f64 = 1e7; +/// Straight steps an arc of a full turn is followed in. +const ARC_STEPS: f64 = 64.0; + +/// Longest path or formula kept, in bytes. +const MAX_TEXT: usize = 64 * 1024; + +/// A custom shape's geometry being read: its path, view box and adjustment values, then its +/// equations, from the `draw:equation` children. +pub(super) struct Pending { + path: String, + vb: Option<[f32; 4]>, + modifiers: String, + equations: Vec<(String, String)>, +} + +/// `s` cut to at most `MAX_TEXT` bytes, at a character boundary. +fn capped(s: &str) -> String { + let mut end = s.len().min(MAX_TEXT); + while !s.is_char_boundary(end) { + end -= 1; + } + s.get(..end).unwrap_or_default().to_owned() +} + +impl Pending { + /// The geometry of a `draw:enhanced-geometry` of type `t` with attributes `a`, when it is + /// to be drawn from its own path: a type WordCraft has no kind for, with a path. + pub(super) fn of(t: &str, a: &Attrs) -> Option> { + let t = t.trim(); + let plain = matches!(t.strip_prefix("ooxml-").unwrap_or(t), "" | "rectangle" | "rect" | "flowchart-process" | "flowChartProcess"); + if super::shapes::kind_of(t) != ShapeKind::Rectangle || plain { + return None; + } + let path = get(a, "draw:enhanced-path")?; + Some(Box::new(Pending { + path: capped(path), + vb: super::path::view_box(get(a, "svg:viewBox")), + modifiers: capped(get(a, "draw:modifiers").unwrap_or_default()), + equations: Vec::new(), + })) + } + + /// Reads a `draw:equation` (§19.171 `draw:name`, §19.170 `draw:formula`). + pub(super) fn equation(&mut self, a: &Attrs) { + if self.equations.len() >= MAX_EQUATIONS { + return; + } + if let (Some(n), Some(f)) = (get(a, "draw:name"), get(a, "draw:formula")) { + self.equations.push((capped(n.trim()), capped(f))); + } + } + + /// The freeform the geometry draws. + pub(super) fn freeform(&self) -> Option { + freeform(&self.path, self.vb, &self.modifiers, &self.equations) + } +} + +/// What a geometry's values are worked out from. +struct Env<'a> { + vb: [f64; 4], + modifiers: Vec, + equations: &'a [(String, String)], + names: HashMap<&'a str, usize>, + /// Each equation worked out so far: its value, or `None` when it can't be (or is being worked + /// out: one referring to itself fails), so none is worked out twice. + memo: std::cell::RefCell>>, +} + +impl Env<'_> { + /// The value of a path or formula parameter: a number, `?name` (an equation), `$n` (an + /// adjustment value) or a keyword (§19.145). + fn param(&self, t: &str, depth: usize) -> Option { + if let Some(n) = t.strip_prefix('?') { + return self.equation(*self.names.get(n)?, depth); + } + if let Some(i) = t.strip_prefix('$') { + return self.modifiers.get(i.parse::().ok()?).copied(); + } + let [x, y, w, h] = self.vb; + Some(match t { + "left" => x, + "top" => y, + "right" => x + w, + "bottom" => y + h, + "width" | "logwidth" => w, + "height" | "logheight" => h, + "xstretch" | "ystretch" => 0.0, + "hasstroke" | "hasfill" => 1.0, + "pi" => std::f64::consts::PI, + _ => t.parse::().ok().filter(|v| v.is_finite())?, + }) + } + + fn equation(&self, i: usize, depth: usize) -> Option { + if depth > MAX_DEPTH { + return None; + } + if let Some(v) = self.memo.borrow().get(&i) { + return *v; + } + let f = &self.equations.get(i)?.1; + self.memo.borrow_mut().insert(i, None); + let mut p = Formula { s: f.as_bytes(), at: 0, env: self, depth: depth + 1 }; + let v = p.expr().map(|v| if v.is_finite() { v } else { 0.0 }); + self.memo.borrow_mut().insert(i, v); + v + } +} + +/// A `draw:formula` being read (recursive descent: sums of products of factors). +struct Formula<'a, 'b> { + s: &'a [u8], + at: usize, + env: &'b Env<'b>, + depth: usize, +} + +impl Formula<'_, '_> { + fn skip(&mut self) { + while self.s.get(self.at).is_some_and(u8::is_ascii_whitespace) { + self.at += 1; + } + } + + fn peek(&mut self) -> Option { + self.skip(); + self.s.get(self.at).copied() + } + + fn expr(&mut self) -> Option { + if self.depth > MAX_DEPTH { + return None; + } + let mut v = self.term()?; + while let Some(c @ (b'+' | b'-')) = self.peek() { + self.at += 1; + let r = self.term()?; + v = if c == b'+' { v + r } else { v - r }; + } + Some(v) + } + + fn term(&mut self) -> Option { + let mut v = self.factor()?; + while let Some(c @ (b'*' | b'/')) = self.peek() { + self.at += 1; + let r = self.factor()?; + v = if c == b'*' { + v * r + } else if r != 0.0 { + v / r + } else { + 0.0 + }; + } + Some(v) + } + + fn factor(&mut self) -> Option { + self.depth += 1; + if self.depth > MAX_DEPTH * 4 { + return None; + } + let v = match self.peek()? { + b'-' => { + self.at += 1; + -self.factor()? + } + b'+' => { + self.at += 1; + self.factor()? + } + b'(' => { + self.at += 1; + let v = self.expr()?; + if self.peek() == Some(b')') { + self.at += 1; + } + v + } + _ => { + let start = self.at; + while self.s.get(self.at).is_some_and(|c| c.is_ascii_alphanumeric() || matches!(c, b'?' | b'$' | b'.' | b'_')) { + self.at += 1; + } + let word = std::str::from_utf8(self.s.get(start..self.at)?).ok()?; + if word.is_empty() { + return None; + } + if self.peek() == Some(b'(') { + self.at += 1; + let mut args = Vec::new(); + if self.peek() != Some(b')') { + loop { + args.push(self.expr()?); + match self.peek() { + Some(b',') => self.at += 1, + _ => break, + } + if args.len() > 8 { + return None; + } + } + } + if self.peek() == Some(b')') { + self.at += 1; + } + function(word, &args)? + } else { + self.env.param(word, self.depth)? + } + } + }; + self.depth -= 1; + Some(v) + } +} + +/// A formula function (§19.170) of `args`. +fn function(name: &str, a: &[f64]) -> Option { + let x = |i: usize| a.get(i).copied(); + Some(match name { + "abs" => x(0)?.abs(), + "sqrt" => x(0)?.max(0.0).sqrt(), + "sin" => x(0)?.sin(), + "cos" => x(0)?.cos(), + "tan" => x(0)?.tan(), + "atan" => x(0)?.atan(), + "atan2" => x(0)?.atan2(x(1)?), + "min" => x(0)?.min(x(1)?), + "max" => x(0)?.max(x(1)?), + "if" => { + if x(0)? > 0.0 { + x(1)? + } else { + x(2)? + } + } + _ => return None, + }) +} + +/// A path being followed: its SVG path data, the pen's place, and how many points so far. +#[derive(Default)] +struct Pen { + d: String, + at: (f64, f64), + points: usize, +} + +impl Pen { + fn cmd(&mut self, c: char, pts: &[(f64, f64)]) { + if self.points >= MAX_POINTS { + return; + } + self.d.push(c); + // Far outside any view box is as far as it goes: points stay finite as `f32`. + let at = |v: f64| if v.is_finite() { v.clamp(-MAX_COORD, MAX_COORD) } else { 0.0 }; + let pts: Vec<(f64, f64)> = pts.iter().map(|(x, y)| (at(*x), at(*y))).collect(); + let pts = pts.as_slice(); + for (x, y) in pts { + self.d.push_str(&format!(" {x:.2} {y:.2}")); + } + self.d.push(' '); + self.points += pts.len().max(1); + if let Some(p) = pts.last() { + self.at = *p; + } + } + + /// An arc of the ellipse centred at `c` with radii `r`, from angle `a0` to `a1` (radians, + /// the y axis down): a line (or a move, `jump`) to its start, then steps along it. + fn arc(&mut self, c: (f64, f64), r: (f64, f64), a0: f64, a1: f64, jump: bool) { + let point = |a: f64| (c.0 + r.0 * a.cos(), c.1 + r.1 * a.sin()); + let start = point(a0); + // A line to the start, unless the pen is there already. + if jump || self.points == 0 || (start.0 - self.at.0).hypot(start.1 - self.at.1) > 1e-6 { + self.cmd(if jump || self.points == 0 { 'M' } else { 'L' }, &[start]); + } + let sweep = a1 - a0; + let steps = ((sweep.abs() / std::f64::consts::TAU) * ARC_STEPS).ceil().clamp(1.0, ARC_STEPS) as usize; + for k in 1..=steps { + self.cmd('L', &[point(a0 + sweep * k as f64 / steps as f64)]); + } + } +} + +/// The angle of `p` on the ellipse centred at `c` with radii `r` (radians). +fn angle_on(c: (f64, f64), r: (f64, f64), p: (f64, f64)) -> f64 { + let (dx, dy) = ((p.0 - c.0) / r.0.max(1e-9), (p.1 - c.1) / r.1.max(1e-9)); + dy.atan2(dx) +} + +/// The freeform an enhanced geometry draws: its `path` with adjustment values `modifiers` and +/// `equations` (name, formula), in view box `vb` (§19.570; the default 21600 square). `None` +/// when the path draws nothing. +pub(super) fn freeform(path: &str, vb: Option<[f32; 4]>, modifiers: &str, equations: &[(String, String)]) -> Option { + let vb = vb.map_or([0.0, 0.0, 21600.0, 21600.0], |v| v.map(f64::from)); + if !(vb[2] > 0.0 && vb[3] > 0.0) { + return None; + } + let equations = equations.get(..equations.len().min(MAX_EQUATIONS)).unwrap_or_default(); + let names = equations.iter().enumerate().map(|(i, (n, _))| (n.as_str(), i)).collect(); + let modifiers = modifiers.split_whitespace().take(64).map(|m| m.parse::().ok().filter(|v| v.is_finite()).unwrap_or(0.0)).collect(); + let env = Env { vb, modifiers, equations, names, memo: Default::default() }; + // The path's tokens: command letters, and parameters (numbers, ?names, $n, keywords). + let tokens: Vec<&str> = path.split(|c: char| c.is_whitespace() || c == ',').filter(|t| !t.is_empty()).take(MAX_TOKENS).collect(); + let mut pen = Pen::default(); + let mut i = 0; + let mut cmd = 'M'; + let is_cmd = |t: &str| t.len() == 1 && t.chars().all(|c| "MLCQZNFSTUABWVXY".contains(c)); + while i < tokens.len() { + let t = tokens.get(i).copied().unwrap_or_default(); + if is_cmd(t) { + cmd = t.chars().next().unwrap_or('M'); + i += 1; + match cmd { + 'Z' => pen.cmd('Z', &[]), + 'N' | 'F' | 'S' => {} + _ => continue, + } + continue; + } + // The command's parameters, as many as it takes each time. + let n = match cmd { + 'M' | 'L' | 'X' | 'Y' => 2, + 'Q' => 4, + 'C' | 'T' | 'U' => 6, + 'A' | 'B' | 'W' | 'V' => 8, + _ => { + i += 1; + continue; + } + }; + let Some(args) = tokens.get(i..i + n) else { break }; + i += n; + let v: Option> = args.iter().map(|a| env.param(a, 0)).collect(); + let Some(v) = v else { continue }; + let p = |k: usize| (v.get(k).copied().unwrap_or(0.0), v.get(k + 1).copied().unwrap_or(0.0)); + match cmd { + 'M' => { + pen.cmd('M', &[p(0)]); + cmd = 'L'; + } + 'L' => pen.cmd('L', &[p(0)]), + 'Q' => pen.cmd('Q', &[p(0), p(2)]), + 'C' => pen.cmd('C', &[p(0), p(2), p(4)]), + // An ellipse from angle to angle, in degrees (§19.145 T, U: U starts a new path). + 'T' | 'U' => { + let (c, r) = (p(0), p(2)); + let (a0, a1) = (p(4).0.to_radians(), p(4).1.to_radians()); + let a1 = if a1 <= a0 { a1 + std::f64::consts::TAU } else { a1 }; + pen.arc(c, r, a0, a1, cmd == 'U'); + } + // Arcs of the ellipse in a box, from where a ray to one point meets it to where a + // ray to another does: anticlockwise (A, B) or clockwise (W, V); B and V start a new + // path. + 'A' | 'B' | 'W' | 'V' => { + let ((x1, y1), (x2, y2)) = (p(0), p(2)); + let c = ((x1 + x2) / 2.0, (y1 + y2) / 2.0); + let r = (((x2 - x1) / 2.0).abs(), ((y2 - y1) / 2.0).abs()); + let (a0, a1) = (angle_on(c, r, p(4)), angle_on(c, r, p(6))); + let clockwise = matches!(cmd, 'W' | 'V'); + // The y axis points down: clockwise on the page is increasing angles. + let a1 = if clockwise { + if a1 <= a0 { a1 + std::f64::consts::TAU } else { a1 } + } else if a1 >= a0 { + a1 - std::f64::consts::TAU + } else { + a1 + }; + pen.arc(c, r, a0, a1, matches!(cmd, 'B' | 'V')); + } + // A quarter ellipse to a point, setting out along the x axis (X) or the y axis (Y), + // the next one the other way. + 'X' | 'Y' => { + let (from, to) = (pen.at, p(0)); + let (c, r, a0, a1) = if cmd == 'X' { + let c = (from.0, to.1); + let r = ((to.0 - from.0).abs(), (to.1 - from.1).abs()); + (c, r, angle_on(c, r, from), angle_on(c, r, to)) + } else { + let c = (to.0, from.1); + let r = ((to.0 - from.0).abs(), (to.1 - from.1).abs()); + (c, r, angle_on(c, r, from), angle_on(c, r, to)) + }; + // The short way round: a quarter. + let mut a1 = a1; + while a1 - a0 > std::f64::consts::PI { + a1 -= std::f64::consts::TAU; + } + while a0 - a1 > std::f64::consts::PI { + a1 += std::f64::consts::TAU; + } + pen.arc(c, r, a0, a1, false); + cmd = if cmd == 'X' { 'Y' } else { 'X' }; + } + _ => {} + } + } + if pen.points < 2 { + return None; + } + super::path::path(&pen.d, Some(vb.map(|v| v as f32))) +} diff --git a/crates/odf/src/text/import.rs b/crates/odf/src/text/import.rs new file mode 100644 index 000000000..486e67a26 --- /dev/null +++ b/crates/odf/src/text/import.rs @@ -0,0 +1,1513 @@ +use super::body::Body; +use super::limits::{MAX_ENTRY, MAX_EXTRA_SPACES, MAX_SECTIONS}; +use super::styles::{Styles, column_sep, columns_of}; +use super::xml::{Attrs, Ev, events, get, length, local}; +use super::{Columns, Document, FBlock, Flow, HashMap, Meta, Para, SectionProps, model, page_parts}; +use std::sync::Arc; + +pub(super) fn read_entry(z: &mut crate::Package<'_>, name: &str) -> Option> { + z.read(name).ok().flatten() +} + +/// How the reader opens a package: as [`crate::Limits::DEFAULT`], each file at most [`MAX_ENTRY`]. +pub(super) const LIMITS: crate::Limits = crate::Limits { max_entry: MAX_ENTRY, ..crate::Limits::DEFAULT }; + +/// The pictures a package's frames point at, within one byte budget. Each entry is inflated +/// at most once, and only when the size it declares fits in what is left; the bytes inflated +/// (even from an entry bigger than it declares) and every later copy are charged, so repeated +/// references and lying entries can't go past the budget, and nothing is inflated after it. +pub(super) struct Pictures { + /// Entry name → its bytes (`None`: missing, unreadable or over the budget; not retried). + seen: HashMap>>, + /// Bytes still allowed. + pub(super) room: u64, +} + +impl Pictures { + pub(super) fn new(budget: u64) -> Self { + Pictures { seen: HashMap::new(), room: budget } + } + + /// The bytes of entry `name`, or `None` when it can't be read or doesn't fit. + pub(super) fn get(&mut self, z: &mut crate::Package<'_>, name: &str) -> Option> { + if let Some(hit) = self.seen.get(name) { + let d = hit.as_ref().filter(|d| d.len() as u64 <= self.room)?; + self.room = self.room.saturating_sub(d.len() as u64); + return Some(d.clone()); + } + let d = self.inflate(z, name); + self.seen.insert(name.to_string(), d.clone()); + d + } + + /// Inflate entry `name`, charging the bytes inflated; `None` when it doesn't fit or fails. + fn inflate(&mut self, z: &mut crate::Package<'_>, name: &str) -> Option> { + let room = self.room; + if room == 0 || z.declared_size(name)? > room { + return None; + } + // An entry that fits by what it declares but doesn't read back as it declares (one that + // inflates to more, damaged data) uses the budget up: nothing is inflated after it. + // shortcut: `Package::read` inflates up to the per-file limit, not to `room`; a capped + // read would need one in `Package`. + match z.read(name) { + Ok(Some(d)) if d.len() as u64 <= room => { + self.room = room - d.len() as u64; + Some(d) + } + _ => { + self.room = 0; + None + } + } + } +} + +/// What `meta.xml` says of the document's changes: who changed it last (`dc:creator`), when +/// (`dc:date`, kept as written, with `Z` when it gives no zone) and how often it was saved +/// (`meta:editing-cycles`). +pub(super) fn read_changes(xml: &[u8]) -> (String, String, u32) { + let mut cur: Option = None; + let (mut by, mut date, mut cycles) = (String::new(), String::new(), String::new()); + events(xml, |ev| match ev { + Ev::Start(q, _) => cur = Some(q), + Ev::End(_) => cur = None, + Ev::Text(t) => match cur.as_deref() { + Some("dc:creator") if by.len() < 256 => by.push_str(&t), + Some("dc:date") if date.len() < 64 => date.push_str(&t), + Some("meta:editing-cycles") if cycles.len() < 16 => cycles.push_str(&t), + _ => {} + }, + }); + let date = date.trim(); + // Zoned already: `Z`, or an offset (`+hh:mm`, or `-hh:mm` in the time part), as `read_meta`. + let zoned = date.ends_with(['Z', 'z']) || date.contains('+') || date.split_once('T').is_some_and(|(_, time)| time.contains('-')); + let date = if date.is_empty() || zoned { date.to_string() } else { format!("{}Z", date.split('.').next().unwrap_or("")) }; + (by.trim().to_string(), date, cycles.trim().parse::().unwrap_or(0)) +} + +pub(super) fn read_meta(xml: &[u8]) -> Meta { + let mut m = Meta::default(); + let mut cur: Option = None; + let mut keywords: Vec = Vec::new(); + let mut creator = String::new(); + let mut mod_date = String::new(); + events(xml, |ev| match ev { + Ev::Start(q, _) => cur = Some(q), + Ev::End(_) => cur = None, + Ev::Text(t) => match cur.as_deref() { + Some("dc:title") => m.title.push_str(&t), + Some("dc:subject") => m.subject.push_str(&t), + Some("dc:description") => m.description.push_str(&t), + Some("meta:initial-creator") => m.author.push_str(&t), + Some("dc:creator") => creator.push_str(&t), + Some("meta:keyword") => keywords.push(t), + Some("meta:creation-date") => m.created.push_str(&t), + Some("dc:date") => mod_date.push_str(&t), + _ => {} + }, + }); + if m.author.is_empty() { + m.author = creator; + } + if m.created.is_empty() { + m.created = mod_date; + } + m.keywords = keywords.join(", "); + let created = m.created.trim(); + if !created.is_empty() { + let has_tz = + created.ends_with(['Z', 'z']) || created.contains('+') || created.split_once('T').map(|(_, time)| time.contains('-')).unwrap_or(false); + if !has_tz { + let base = created.split('.').next().unwrap_or(""); + m.created = format!("{base}Z"); + } else { + m.created = created.to_string(); + } + } else { + m.created.clear(); + } + m +} + +/// Every paragraph of `blocks`, in table cells, notes and comments too. +fn each_para(blocks: &mut [FBlock], depth: usize, f: &mut dyn FnMut(&mut Para)) { + if depth > model::MAX_DEPTH { + return; + } + for b in blocks { + match b { + FBlock::Para(p) => { + f(p); + for i in &mut p.inlines { + each_inline_para(i, depth + 1, f); + } + } + FBlock::Table(t) => { + for c in t.rows.iter_mut().flatten() { + each_para(&mut c.blocks, depth + 1, f); + } + } + } + } +} + +/// [`each_para`] in the paragraphs `i` holds: a note's, a comment's, a text box's, or those of +/// the text boxes in a group. +fn each_inline_para(i: &mut model::Inline, depth: usize, f: &mut dyn FnMut(&mut Para)) { + if depth > model::MAX_DEPTH { + return; + } + match i { + model::Inline::Note { blocks, .. } | model::Inline::Shape { blocks, .. } => each_para(blocks, depth, f), + model::Inline::Comment { comment, .. } => each_para(&mut comment.blocks, depth, f), + model::Inline::Group { members, .. } => { + for m in members { + each_inline_para(&mut m.inline, depth + 1, f); + } + } + _ => {} + } +} + +/// The custom character styles the text of `blocks` names (by ODF name, in [`model::Fmt::extra`]) +/// with their custom parents, parents first, each with its document style id, added to `found`; +/// the text then names them by id. `para_ids` are the paragraph styles' ids, which a character +/// style's id doesn't take. +fn custom_char_styles(blocks: &mut [FBlock], styles: &Styles, para_ids: &[String], found: &mut Vec<(String, String)>) { + let id_for = |odt: &str| { + let id = super::styles::style_id(odt); + if para_ids.contains(&id) { format!("{id}Char") } else { id } + }; + each_para(blocks, 0, &mut |p| { + for i in &mut p.inlines { + let fmt = match i { + model::Inline::Text(_, f) | model::Inline::Field { fmt: f, .. } | model::Inline::Ruby { fmt: f, .. } => f, + _ => continue, + }; + let Some(odt) = fmt.extra.as_mut().and_then(|x| x.style.take()) else { continue }; + let mut chain: Vec = Vec::new(); + let mut cur = Some(odt.clone()); + while let Some(n) = cur.take() { + if found.iter().any(|(o, _)| *o == n) || chain.contains(&n) || chain.len() >= 16 { + break; + } + cur = styles.named_text(&n).and_then(|(_, parent, ..)| parent); + chain.push(n); + } + for n in chain.into_iter().rev() { + let id = id_for(&n); + found.push((n, id)); + } + if let Some(x) = fmt.extra.as_mut() { + x.style = found.iter().find(|(o, _)| *o == odt).map(|(_, id)| id.clone()); + } + } + }); +} + +/// Add the custom character styles `found` (ODF name, document id; parents first) built from +/// `defs` (name, parent's ODF name, own formatting), alongside. +fn add_char_styles(doc: &mut Document, found: &[(String, String)], defs: &[(String, Option, model::Fmt, [bool; 4])]) { + for ((_, id), (name, parent, fmt, off)) in found.iter().zip(defs) { + let based_on = parent.as_ref().and_then(|p| found.iter().find(|(o, _)| o == p).map(|(_, id)| id.clone())).filter(|b| b != id); + Arc::make_mut(&mut doc.styles).upsert(craft_wordmodel::styles::Style { + id: id.clone(), + name: name.clone(), + kind: craft_wordmodel::styles::StyleKind::Character, + based_on, + chr: turned_off(model::char_props(fmt), *off), + ..Default::default() + }); + } +} + +/// The custom styles the paragraphs of `blocks` name (by ODF name, [`super::styles::ParaLook::named`]) +/// with their parents, parents first; the paragraphs then name them by document style id. +/// `odt` holds the styles already found (in other parts), which aren't added again. +fn custom_styles(blocks: &mut [FBlock], styles: &Styles, odt: Vec) -> (Vec, Vec) { + let mut odt = odt; + let mut defs: Vec = Vec::new(); + each_para(blocks, 0, &mut |p| { + let Some(name) = p.extra.as_mut().and_then(|x| x.style.take()) else { return }; + // The style and its custom parents, parents first, each once (a loop ends the chain). + let mut chain: Vec<(String, super::styles::NamedStyle)> = Vec::new(); + let mut cur = Some(name.clone()); + while let Some(n) = cur.take() { + if odt.contains(&n) || chain.iter().any(|(m, _)| *m == n) || chain.len() >= 16 { + break; + } + let Some(def) = styles.named_style(&n) else { break }; + if let super::styles::Parent::Named(p) = &def.parent { + cur = Some(p.clone()); + } + chain.push((n, def)); + } + for (n, def) in chain.into_iter().rev() { + odt.push(n); + defs.push(def); + } + // A custom style, or an index style WordCraft has under its own id (TOC 1…). + if (odt.contains(&name) || super::writer::generating_style(&name).is_some()) + && let Some(x) = p.extra.as_mut() + { + x.style = Some(super::styles::style_id(&name)); + } + }); + (odt, defs) +} + +/// Add the custom styles `named` (ODF names `odt`, parents first) to the document: each with its +/// name, the style it is based on and its own formatting. A style WordCraft has under the same id +/// takes the file's definition. +fn add_custom_styles(doc: &mut Document, named: &[super::styles::NamedStyle], odt: &[String], tabs_from_indent: bool) { + use super::styles::{Parent, style_id}; + for (def, name) in named.iter().zip(odt) { + let id = style_id(name); + let based_on = match &def.parent { + Parent::Normal => "Normal".to_string(), + Parent::Kind(model::Kind::Heading(n)) => format!("Heading{}", (*n).clamp(1, 6)), + Parent::Kind(model::Kind::Title) => "Title".into(), + Parent::Kind(model::Kind::Quote) => "Quote".into(), + Parent::Kind(model::Kind::Code) if doc.styles.get(model::CODE_STYLE).is_some() => model::CODE_STYLE.into(), + Parent::Kind(_) => "Normal".into(), + Parent::Named(p) => style_id(p), + }; + let look = &def.look; + let sp = &look.spacing; + let para = craft_wordmodel::props::ParaProps { + space_before: sp.before, + space_after: sp.after, + line_spacing: sp.line, + indent_left: sp.left, + indent_right: sp.right, + indent_first: sp.first, + align: look.align, + keep_next: look.keep_next.then_some(true), + borders: look.borders, + shading: look.shading, + // LibreOffice measures tabs from the indent (`TabsRelativeToIndent`); Word from the + // margin. + tabs: look.tabs.clone().map(|mut tabs| { + if tabs_from_indent { + let indent = sp.left.unwrap_or(0.0); + for t in &mut tabs { + t.pos = (t.pos + indent).clamp(-1584.0, 1584.0); + } + } + tabs + }), + contextual_spacing: look.contextual, + bidi: look.rtl.filter(|r| *r), + // The document's nine levels; ODF's tenth is the last of them. + outline_level: def.outline.map(|o| o.clamp(1, 9) - 1), + ..Default::default() + }; + let name = super::writer::generating_style(name).map_or_else(|| def.name.clone(), |(_, n)| n); + let style = craft_wordmodel::styles::Style { + id: id.clone(), + name, + kind: craft_wordmodel::styles::StyleKind::Paragraph, + based_on: (based_on != id).then_some(based_on), + para, + chr: turned_off(model::char_props(&look.fmt), def.off), + ..Default::default() + }; + Arc::make_mut(&mut doc.styles).upsert(style); + } +} + +/// `chr` with what a style turns off (`off`: bold, italic, underline, strike-through) said so: +/// otherwise the style it is based on would show through (LibreOffice's agenda dates, not bold +/// under bold titles). +fn turned_off(mut chr: craft_wordmodel::props::CharProps, off: [bool; 4]) -> craft_wordmodel::props::CharProps { + let [bold, italic, underline, strike] = off; + if bold { + chr.bold = Some(false); + } + if italic { + chr.italic = Some(false); + } + if underline { + chr.underline = Some(craft_wordmodel::props::Underline::None); + } + if strike { + (chr.strike, chr.double_strike) = (Some(false), Some(false)); + } + chr +} + +/// Give the document's style for paragraph kind `kind` the look the file gives its own: its text +/// formatting, spacing, alignment and keep with next replace WordCraft's (a heading the file +/// leaves black isn't blue). Its outline level and other settings stay. +fn apply_look(doc: &mut Document, kind: model::Kind, look: &super::styles::ParaLook) { + let id = match kind { + model::Kind::Heading(n) => format!("Heading{}", n.clamp(1, 6)), + model::Kind::Title => "Title".into(), + model::Kind::Quote => "Quote".into(), + model::Kind::Code => model::CODE_STYLE.into(), + model::Kind::Normal | model::Kind::Rule => return, + }; + let Some(st) = Arc::make_mut(&mut doc.styles).get_mut(&id) else { return }; + st.chr = model::char_props(&look.fmt); + let sp = &look.spacing; + let p = &mut st.para; + (p.space_before, p.space_after, p.line_spacing) = (sp.before, sp.after, sp.line); + (p.indent_left, p.indent_right, p.indent_first) = (sp.left, sp.right, sp.first); + p.align = look.align; + p.keep_next = look.keep_next.then_some(true); + if look.contextual.is_some() { + p.contextual_spacing = look.contextual; + } +} + +/// Number the headings as the outline style `levels` does (ODF 1.3 §16.34): one list whose +/// levels carry the outline's formats, each linked to its heading style, which numbers its +/// paragraphs through it, as Word's numbered headings do. Levels without a number are left so. +fn number_headings(doc: &mut Document, levels: &[super::styles::ListLevel]) { + use craft_wordmodel::numbering::{AbstractNum, ListKind, Num, levels_for}; + use craft_wordmodel::props::NumRef; + let numbered: Vec = (0..9).map(|i| levels.get(i).is_some_and(|l| l.ordered)).collect(); + if !numbered.contains(&true) { + return; + } + let mut out = levels_for(ListKind::Numbered); + for (i, level) in out.iter_mut().enumerate() { + let src = levels.get(i).copied().unwrap_or_default(); + // Headings sit where the outline level puts them, else at the margin; their number is + // followed by a tab. + (level.indent, level.hanging) = src.label.and_then(|l| l.position()).unwrap_or((0.0, 0.0)); + if let Some(a) = src.label.and_then(|l| l.align) { + level.align = a; + } + match src.label.filter(|_| src.ordered) { + Some(l) => { + level.format = l.format; + level.text = l.text(i as u8); + level.start = src.start; + } + None => { + level.format = craft_wordmodel::section::NumFormat::None; + level.text = String::new(); + } + } + if i < 6 { + level.style = Some(format!("Heading{}", i + 1)); + } + } + let numbering = Arc::make_mut(&mut doc.numbering); + let aid = numbering.abstracts.iter().map(|a| a.id + 1).max().unwrap_or(0); + numbering.abstracts.push(AbstractNum { id: aid, name: None, levels: out, ..Default::default() }); + let nid = numbering.nums.iter().map(|n| n.id + 1).max().unwrap_or(1).max(1); + numbering.nums.push(Num { id: nid, abstract_id: aid, ..Default::default() }); + let styles = Arc::make_mut(&mut doc.styles); + for (i, on) in numbered.iter().enumerate() { + if *on && let Some(st) = styles.get_mut(&format!("Heading{}", i + 1)) { + st.para.numbering = Some(NumRef { num: nid, level: i as u8 }); + } + } + // LibreOffice numbers a custom style at an outline level along with that level's headings. + for st in &mut styles.styles { + if let Some(l) = st.para.outline_level.filter(|l| numbered.get(usize::from(*l)) == Some(&true)) { + st.para.numbering = Some(NumRef { num: nid, level: l }); + } + } + // A style at a level the outline doesn't number doesn't take its parent's number from it. + let of_outline = |st: &craft_wordmodel::styles::Style| { + let at = craft_wordmodel::props::ParaProps { style: Some(st.id.clone()), ..Default::default() }; + st.para.numbering.is_none() + && st.para.outline_level.is_some_and(|l| numbered.get(usize::from(l)) != Some(&true)) + && styles.resolve_para(&at).numbering.is_some_and(|n| n.num == nid) + }; + let unnumbered: Vec = styles.styles.iter().filter(|st| of_outline(st)).map(|st| st.id.clone()).collect(); + for id in unnumbered { + if let Some(st) = styles.get_mut(&id) { + // (`num: 0` switches the number off.) + st.para.numbering = Some(NumRef { num: 0, level: 0 }); + } + } + // A heading's own outline level goes before its style's: LibreOffice numbers it by that level. + // A heading in a list is that list's. + let styles = doc.styles.clone(); + each_body_para(&mut doc.body, 0, &mut |p| { + let Some(l) = p.props.outline_level.filter(|_| p.props.numbering.is_none()) else { return }; + let style = styles.resolve_para(&craft_wordmodel::props::ParaProps { style: p.props.style.clone(), ..Default::default() }); + if style.outline_level.is_none() { + return; + } + let own = (numbered.get(usize::from(l)) == Some(&true)).then_some(NumRef { num: nid, level: l }); + if own != style.numbering { + // (`num: 0` switches the style's number off.) + p.props.numbering = Some(own.unwrap_or(NumRef { num: 0, level: 0 })); + } + }); +} + +/// Each paragraph of `blocks`, those in tables too (to 16 levels deep). +fn each_body_para(blocks: &mut [Arc], depth: usize, f: &mut dyn FnMut(&mut craft_wordmodel::Paragraph)) { + for b in blocks { + match Arc::make_mut(b) { + craft_wordmodel::Block::Para(p) => f(p), + craft_wordmodel::Block::Table(t) if depth < 16 => { + for cell in t.rows.iter_mut().flat_map(|r| r.cells.iter_mut()) { + each_body_para(&mut cell.blocks, depth + 1, f); + } + } + craft_wordmodel::Block::Table(_) => {} + } + } +} + +/// What a run's complex size comes from: its paragraph's style, its character style, and the size +/// and complex size it states (as bits). +type SizeSource<'a> = (Option<&'a str>, Option<&'a str>, Option, Option); + +/// A complex-script size the same as the text's (what WordCraft writes for every script) is no size +/// of its own: a later change of size takes it along. The default's and the styles' go together, +/// unless a run's complex size would change: a size stated alone, or another complex size between +/// them, shows what's above it, so they stay as the file says. A run's goes where its stays the same. +fn one_size_for_all_scripts(doc: &mut Document) { + let mut sources = std::collections::BTreeSet::new(); + for blocks in std::iter::once(&doc.body).chain(doc.parts.values().map(|part| &part.blocks)) { + size_sources(blocks, 0, &mut sources); + } + let sizes = |styles: &craft_wordmodel::styles::StyleSheet| -> Vec { + let of = |(para, chr, size, size_cs): &SizeSource| { + let run = craft_wordmodel::props::CharProps { + style: chr.map(str::to_string), + size: size.map(f32::from_bits), + size_cs: size_cs.map(f32::from_bits), + ..Default::default() + }; + styles.resolve_char(*para, &run).size_cs + }; + sources.iter().map(of).collect() + }; + let as_read = Arc::clone(&doc.styles); + let was = sizes(&as_read); + let styles = Arc::make_mut(&mut doc.styles); + let one_size = |c: &mut craft_wordmodel::props::CharProps| { + if c.size.is_some() && c.size_cs == c.size { + c.size_cs = None; + } + }; + one_size(&mut styles.default_chr); + styles.styles.iter_mut().for_each(|st| one_size(&mut st.chr)); + if sizes(&doc.styles) != was { + doc.styles = as_read; + } + let styles = Arc::clone(&doc.styles); + let stories = std::iter::once(&mut doc.body).chain(doc.parts.values_mut().map(|part| &mut part.blocks)); + for blocks in stories { + runs_one_size(blocks, 0, &styles); + } +} + +/// What the runs in `blocks` (and their tables, to 16 levels deep) take their complex sizes from. +fn size_sources<'a>(blocks: &'a [Arc], depth: usize, out: &mut std::collections::BTreeSet>) { + for b in blocks { + match &**b { + craft_wordmodel::Block::Para(p) => { + for r in &p.runs { + out.insert(( + p.props.style.as_deref(), + r.props.style.as_deref(), + r.props.size.map(f32::to_bits), + r.props.size_cs.map(f32::to_bits), + )); + } + } + craft_wordmodel::Block::Table(t) if depth < 16 => { + for cell in t.rows.iter().flat_map(|r| r.cells.iter()) { + size_sources(&cell.blocks, depth + 1, out); + } + } + craft_wordmodel::Block::Table(_) => {} + } + } +} + +/// [`one_size_for_all_scripts`] for the runs in `blocks` (and their tables, to 16 levels deep). +fn runs_one_size(blocks: &mut [Arc], depth: usize, styles: &craft_wordmodel::styles::StyleSheet) { + for b in blocks { + match Arc::make_mut(b) { + craft_wordmodel::Block::Para(p) => { + let style = p.props.style.clone(); + for r in &mut p.runs { + let Some(said) = r.props.size_cs.filter(|s| Some(*s) == r.props.size) else { continue }; + r.props.size_cs = None; + if styles.resolve_char(style.as_deref(), &r.props).size_cs != said { + r.props.size_cs = Some(said); + } + } + } + craft_wordmodel::Block::Table(t) if depth < 16 => { + for cell in t.rows.iter_mut().flat_map(|r| r.cells.iter_mut()) { + runs_one_size(&mut cell.blocks, depth + 1, styles); + } + } + craft_wordmodel::Block::Table(_) => {} + } + } +} + +/// The boolean configuration item `name` of `settings.xml` (ODF 1.3 §3.10), when it has one. +fn setting(xml: &[u8], name: &str) -> Option { + let mut inside = false; + let mut value = None; + events(xml, |ev| match ev { + Ev::Start(q, a) => inside = q == "config:config-item" && get(&a, "config:name") == Some(name) && value.is_none(), + Ev::Text(t) if inside => value = Some(t.trim() == "true"), + _ => inside = false, + }); + value +} + +/// Parse an `.odt` package into the flow model. +pub fn parse(bytes: &[u8]) -> Result { + parse_full(bytes).map(|(flow, _)| flow) +} + +/// [`parse`], with the last section's properties (the others are on the paragraphs that end them). +fn parse_full(bytes: &[u8]) -> Result<(Flow, SectionProps), String> { + parse_tracked(bytes, default_paper()).map(|p| (p.flow, p.last)) +} + +/// The paragraphs kept as read that their links tie together (see `model::Group`), in `blocks`: +/// each group given its id and size when every link it needs is there, its paragraphs written +/// from the model otherwise (a comment's end in a paragraph not kept, say), and a run kept +/// whole (a list) with them. +fn resolve_groups(blocks: &mut [FBlock]) { + // The linked paragraphs, in order, and the paragraphs holding each key. + let mut links: Vec> = Vec::new(); + walk_grouped(blocks, 0, &mut |g| links.push(g.links.clone())); + if links.is_empty() { + return; + } + let mut holders: std::collections::HashMap<&str, Vec> = std::collections::HashMap::new(); + for (i, l) in links.iter().enumerate() { + for (k, _) in l { + let h = holders.entry(k.as_str()).or_default(); + if h.last() != Some(&i) { + h.push(i); + } + } + } + // Groups: paragraphs sharing a key one of them needs. + let mut root: Vec = (0..links.len()).collect(); + fn find(root: &mut [usize], mut i: usize) -> usize { + while let Some(&r) = root.get(i) { + if r == i { + break; + } + i = r; + } + i + } + let needed: std::collections::HashSet<&str> = links.iter().flatten().filter(|(_, n)| *n).map(|(k, _)| k.as_str()).collect(); + for (k, h) in &holders { + if needed.contains(k) + && let Some(&first) = h.first() + { + for &i in h.iter().skip(1) { + let (a, b) = (find(&mut root, first), find(&mut root, i)); + if let Some(r) = root.get_mut(b) { + *r = a; + } + } + } + } + // A group is whole when each key one of its paragraphs needs another holds. + let mut whole: std::collections::HashMap = std::collections::HashMap::new(); + let mut size: std::collections::HashMap = std::collections::HashMap::new(); + for (i, l) in links.iter().enumerate() { + let r = find(&mut root, i); + *size.entry(r).or_default() += 1; + let ok = l.iter().filter(|(_, n)| *n).all(|(k, _)| holders.get(k.as_str()).is_some_and(|h| h.len() >= 2)); + let w = whole.entry(r).or_insert(true); + *w &= ok; + } + let mut ids: std::collections::HashMap = std::collections::HashMap::new(); + let mut verdicts: Vec>> = Vec::with_capacity(links.len()); + for i in 0..links.len() { + let r = find(&mut root, i); + let s = size.get(&r).copied().unwrap_or(1); + let needs = links.get(i).is_some_and(|l| l.iter().any(|(_, n)| *n)); + verdicts.push(match (whole.get(&r).copied().unwrap_or(false), s) { + // Alone, needing nothing: its own (what it has, nobody needed). + (true, 1) if !needs => Some(None), + (true, s) if s > 1 => { + let next = u32::try_from(ids.len()).unwrap_or(u32::MAX); + Some(Some((*ids.entry(r).or_insert(next), s))) + } + _ => None, + }); + } + let mut at = 0usize; + apply_groups(blocks, 0, &verdicts, &mut at); +} + +/// Each paragraph kept as read with links in `blocks` (its [`model::Group`]), in order. +fn walk_grouped(blocks: &[FBlock], depth: usize, f: &mut dyn FnMut(&craft_flow::Group)) { + for b in blocks { + match b { + FBlock::Para(p) => { + if let (Some(_), Some(g)) = (&p.kept, &p.group) { + f(g); + } + } + FBlock::Table(t) if depth < craft_flow::MAX_DEPTH => { + for c in t.rows.iter().flatten() { + walk_grouped(&c.blocks, depth + 1, f); + } + } + FBlock::Table(_) => {} + } + } +} + +/// The groups' `verdicts` (in [`walk_grouped`]'s order, from `at`) applied to `blocks`: a group's +/// id and size, none, or the paragraph written from the model (with a run kept whole holding it). +fn apply_groups(blocks: &mut [FBlock], depth: usize, verdicts: &[Option>], at: &mut usize) { + let mut dropped = Vec::new(); + for (i, b) in blocks.iter_mut().enumerate() { + match b { + FBlock::Para(p) => { + if p.kept.is_some() && p.group.is_some() { + match verdicts.get(*at).copied().flatten() { + Some(None) => p.group = None, + Some(Some(id)) => { + if let Some(g) = &mut p.group { + g.links.clear(); + g.id = Some(id); + } + } + None => { + (p.kept, p.group) = (None, None); + dropped.push(i); + } + } + *at += 1; + } else if p.kept.is_none() { + p.group = None; + } + } + FBlock::Table(t) if depth < craft_flow::MAX_DEPTH => { + for c in t.rows.iter_mut().flatten() { + apply_groups(&mut c.blocks, depth + 1, verdicts, at); + } + } + FBlock::Table(_) => {} + } + } + // A run kept whole (a list) holding a paragraph written from the model is too. + if !dropped.is_empty() { + for (i, b) in blocks.iter_mut().enumerate() { + if let FBlock::Para(p) = b + && let Some(ix) = p.index.as_deref() + && dropped.iter().any(|d| *d >= i && *d < i + ix.1) + { + p.index = None; + } + } + } +} + +/// The markup kept as read in `blocks` (paragraphs', tables', the paragraphs' in tables not +/// kept whole) that links to package files, added to `out`. +fn kept_markup(blocks: &[FBlock], depth: usize, out: &mut Vec<(String, std::sync::Arc)>) { + for b in blocks { + match b { + FBlock::Para(p) => out.extend(p.kept.iter().filter(|x| x.contains("xlink:href")).map(|x| (String::new(), x.clone()))), + FBlock::Table(t) => match &t.kept { + Some(x) => out.extend(x.contains("xlink:href").then(|| (String::new(), x.clone()))), + None if depth < craft_flow::MAX_DEPTH => { + for c in t.rows.iter().flatten() { + kept_markup(&c.blocks, depth + 1, out); + } + } + None => {} + }, + } + } +} + +/// Drop the markup kept as read (of paragraphs and tables, as [`kept_markup`] finds it) that +/// links to one of the package files in `missing`. +fn unkeep_linking(blocks: &mut [FBlock], depth: usize, missing: &std::collections::BTreeSet) { + if missing.is_empty() { + return; + } + for b in blocks { + match b { + FBlock::Para(p) => { + if p.kept.as_deref().is_some_and(|x| super::kept::links_to(x, missing)) { + p.kept = None; + } + } + FBlock::Table(t) => { + if t.kept.as_deref().is_some_and(|x| super::kept::links_to(x, missing)) { + t.kept = None; + } + if t.kept.is_none() && depth < craft_flow::MAX_DEPTH { + for c in t.rows.iter_mut().flatten() { + unkeep_linking(&mut c.blocks, depth + 1, missing); + } + } + } + } + } +} + +/// What [`parse_tracked`] reads besides the flow. +struct Parsed { + flow: Flow, + /// Who changed it last, when, and how often it was saved (`meta.xml`). + changes: (String, String, u32), + /// The style and font definitions kept paragraphs may refer to (see `super::kept`). + kept_defs: Vec<(String, std::sync::Arc)>, + /// The package files those definitions refer to. + kept_files: Vec<(String, Vec)>, + /// Each section's master page as the file named it. + section_masters: Vec>, + /// The space between paragraphs is their spacings' sum (else the larger). + sum_spacing: bool, + /// The last section's properties. + last: SectionProps, + /// Whether change tracking is on (`None`: the document has no tracked changes). + track: Option, + /// The document's default font. + font: Option, + /// The document's default font size, points. + size: Option, + /// The document's default text colour. + color: Option, + /// The custom paragraph styles the paragraphs use, parents included, parents first. + named: Vec, + /// Their ODF names, alongside. + named_odt: Vec, + /// The custom character styles the text uses: ODF name and document id, parents first. + char_styles: Vec<(String, String)>, + /// Their names, parents and own formatting, alongside. + char_defs: Vec<(String, Option, model::Fmt, [bool; 4])>, + /// The file's `settings.xml`, kept for the next save. + settings: Option>, + /// The heading numbering (`text:outline-style`), level by level. + outline: Option>, + /// How the file's heading, title, quotation and preformatted styles look. + looks: Vec<(model::Kind, super::styles::ParaLook)>, + /// The document's default paragraph spacing. + spacing: model::Spacing, + /// Whether the document kerns pairs of letters by default. + kern: bool, + /// Widow and orphan control the document states for every paragraph, if it does. + widows: Option, + /// The document's default Asian and complex-script fonts and languages. + scripts: craft_wordmodel::props::CharProps, + /// The document's default language, if it states one. + default_lang: Option, + /// The default tab interval, points. + default_tab: Option, + /// Tab stops are measured from the indent (LibreOffice's `TabsRelativeToIndent`). + tabs_from_indent: bool, + /// The page settings that are the document's. + page_doc: PageDoc, + parts: page_parts::Parts, + /// Some paragraph hyphenates: the document hyphenates (see [`hyphenation`]). + hyphenate: bool, + /// Custom document properties (`meta:user-defined`) and user fields. + custom: Vec, + /// The bibliography sources the citations carry. + sources: Vec, + /// The package's parts WordCraft doesn't read itself (macros…), kept for the next save. + kept: super::parts::Kept, + /// Hidden form controls, kept for the next save. + hidden_controls: Vec, + /// The page background picture's bytes and fill. + page_image: Option<(Vec, &'static str)>, +} + +/// Hyphenation is a paragraph's in ODF (`fo:hyphenate`) and the document's in WordCraft (as in +/// Word): when any paragraph of `blocks` hyphenates (its `suppress_hyphens` read as +/// `Some(false)`), the document does, and every other paragraph opts out; otherwise nothing is +/// marked. Whether the document hyphenates. +fn hyphenation(blocks: &mut [FBlock]) -> bool { + let mut any = false; + each_para(blocks, 0, &mut |p| any |= p.extra.as_ref().is_some_and(|x| x.suppress_hyphens == Some(false))); + each_para(blocks, 0, &mut |p| { + let hyphenates = p.extra.as_ref().is_some_and(|x| x.suppress_hyphens == Some(false)); + if any && !hyphenates { + p.extra.get_or_insert_with(Default::default).suppress_hyphens = Some(true); + } else if let Some(x) = &mut p.extra { + x.suppress_hyphens = None; + } + }); + any +} + +/// The content of a package that has none: an empty text. +const EMPTY_CONTENT: &str = r#""#; + +/// The page size of a page layout that states none: WordCraft's default page's. +fn default_paper() -> (f32, f32) { + let d = SectionProps::default(); + (d.page_w, d.page_h) +} + +/// [`parse_full`], plus what the document holds outside the flow. A page layout that states no +/// size (§20.215 `fo:page-width` is optional) is `paper` (width, height) in size. +fn parse_tracked(bytes: &[u8], paper: (f32, f32)) -> Result { + crate::zip_guard::check(bytes, crate::Limits::DEFAULT.max_entries).map_err(|e| format!("ODT: {e}"))?; + let mut z = crate::Package::open(bytes, &LIMITS).map_err(|e| format!("ODT: {e}"))?; + if let Some(mt) = read_entry(&mut z, "mimetype") { + let mt = String::from_utf8_lossy(&mt); + if !mt.trim().starts_with("application/vnd.oasis.opendocument.text") { + // Its media type named only when it reads as one (a damaged package's may be a picture). + let mt = mt.trim(); + let shown = mt.len() <= 100 && mt.chars().all(|c| c.is_ascii_graphic()); + return Err(if shown { format!("ODT: not a text document ({mt})") } else { "ODT: not a text document".to_string() }); + } + } + let styles_xml = read_entry(&mut z, "styles.xml"); + // A template of styles only (LibreOffice's style templates) has no content: an empty text. + let content = match read_entry(&mut z, "content.xml") { + Some(c) => c, + None if styles_xml.is_some() => EMPTY_CONTENT.as_bytes().to_vec(), + None => return Err("ODT: content.xml is missing".into()), + }; + let mut styles = Styles::default(); + let mut pages = Pages::default(); + if let Some(s) = &styles_xml { + styles.read(s, false); + pages = read_pages(s, paper); + } + // No page layout at all (no styles.xml, or no master page). + if pages.first.is_none() { + pages.first = Some(unstated_page(paper)); + } + styles.read(&content, true); + let settings = read_entry(&mut z, "settings.xml"); + // Where tab stops are measured from: as the document's settings say (LibreOffice's own), else + // the margin, as Word (the standard ties it only to indexes, §19.512.3). + styles.tabs_from_indent = settings.as_deref().and_then(|s| setting(s, "TabsRelativeToIndent")).unwrap_or(false); + // How the space between two paragraphs adds up: as LibreOffice's `AddParaTableSpacing` says + // (false: the larger of the two, as Word), else the larger (the standard doesn't say). + let sum_spacing = settings.as_deref().and_then(|s| setting(s, "AddParaTableSpacing")).unwrap_or(false); + let meta_xml = read_entry(&mut z, "meta.xml"); + let meta = meta_xml.as_deref().map(read_meta).unwrap_or_default(); + let changes = meta_xml.as_deref().map(read_changes).unwrap_or_default(); + // Custom document properties, then the user fields not among them. + let mut custom = meta_xml.as_deref().map(super::fields::meta_props).unwrap_or_default(); + for f in super::fields::user_fields(&content) { + if custom.len() < super::fields::MAX_PROPS && !custom.iter().any(|c| c.name == f.name) { + custom.push(f); + } + } + let entries: Vec = z.names().take(100_000).map(str::to_string).collect(); + let manifest = read_entry(&mut z, "META-INF/manifest.xml").map(|m| super::objects::manifest(&m)).unwrap_or_default(); + let kept = super::parts::read(&mut z, &entries, &manifest); + let mut pictures = Pictures::new(MAX_ENTRY); + let mut fetch = |path: &str| -> Option> { + // A frame may point at any entry, as often as it likes: `Pictures` keeps that bounded. + pictures.get(&mut z, path) + }; + // The page background picture's bytes. + let page_image = pages.doc.page_image.clone().and_then(|(href, fill)| fetch(&href).map(|b| (b, fill))); + let objects = (entries.as_slice(), &manifest); + // One `text:s` budget for the headers and footers and then the body. + let mut spaces_left = MAX_EXTRA_SPACES; + let mut parts = + styles_xml.as_deref().map(|x| page_parts::read(x, &styles, &mut pages, &mut fetch, objects, &mut spaces_left)).unwrap_or_default(); + first_pages(&mut pages); + let mut body = Body::new(&styles, &pages, &mut fetch); + (body.entries, body.manifest) = (objects.0, Some(objects.1)); + body.raw = &content; + body.spaces_left = spaces_left; + super::xml::events_at(&content, |ev, at| body.feed_at(ev, at)); + let mut blocks = body.take_blocks(); + // The file's own definitions: what kept paragraphs refer to, and the style definitions + // written as read while they stay unchanged. + let mut kept_defs = super::kept::defs(&content, true); + kept_defs.extend(meta_xml.as_deref().map(super::kept::meta_defs).unwrap_or_default()); + kept_defs.extend(styles_xml.as_deref().map(|x| super::kept::defs(x, false)).unwrap_or_default()); + let hidden_controls = body.take_hidden_controls(); + let (mut named_odt, mut named) = custom_styles(&mut blocks, &styles, Vec::new()); + for b in parts.blocks_mut() { + let (o, n) = custom_styles(b, &styles, named_odt); + named_odt = o; + named.extend(n); + } + let para_ids: Vec = named_odt.iter().map(|n| super::styles::style_id(n)).collect(); + let mut char_styles = Vec::new(); + custom_char_styles(&mut blocks, &styles, ¶_ids, &mut char_styles); + for b in parts.blocks_mut() { + custom_char_styles(b, &styles, ¶_ids, &mut char_styles); + } + let char_defs = char_styles.iter().filter_map(|(o, _)| styles.named_text(o)).collect(); + // An open section with nothing in it (after a column section at the end) isn't one: the + // section before it ends the document. + let mut last = body.sec; + if blocks.len() <= body.sec_from + && let Some(FBlock::Para(p)) = blocks.last_mut() + && let Some(s) = p.section.take() + { + last = *s; + } + if blocks.is_empty() { + blocks.push(FBlock::Para(Para::default())); + } + let hyphenate = hyphenation(&mut blocks); + let track = body.track; + let sources = std::mem::take(&mut body.sources); + resolve_groups(&mut blocks); + let section_masters = std::mem::take(&mut body.section_masters); + // And the pictures paragraphs and tables kept as read show. + let mut linking: Vec<(String, std::sync::Arc)> = kept_defs.clone(); + kept_markup(&blocks, 0, &mut linking); + for b in parts.blocks_mut() { + kept_markup(b, 0, &mut linking); + } + let (kept_files, missing) = super::kept::files(&linking, &entries, &mut fetch); + // Markup linking to a file that couldn't be read is written from the model: as read, it + // would point at a file the saved package hasn't got. + unkeep_linking(&mut blocks, 0, &missing); + for b in parts.blocks_mut() { + unkeep_linking(b, 0, &missing); + } + // Their manifest entries, and their folders' (an object's says what it is). + for (path, _) in &kept_files { + let dir = path.rsplit_once('/').map(|(d, _)| format!("{d}/")); + for p in std::iter::once(path.clone()).chain(dir) { + if let Some(mime) = manifest.get(&p).or_else(|| manifest.get(p.trim_end_matches('/'))) { + kept_defs.push((format!("manifest:{p}"), std::sync::Arc::from(mime.as_str()))); + } + } + } + Ok(Parsed { + flow: Flow { blocks, meta }, + last, + track, + font: styles.doc_font(), + size: styles.doc_size(), + color: styles.doc_color(), + spacing: styles.doc_spacing(), + looks: styles.kind_looks(), + named, + named_odt, + char_styles, + char_defs, + settings: settings.filter(|s| s.len() <= super::package::MAX_SETTINGS), + outline: styles.outline.clone(), + kern: styles.default_kern, + widows: styles.stated_widows(), + scripts: styles.default_scripts.clone(), + default_lang: styles.document_lang(), + default_tab: styles.default_tab, + tabs_from_indent: styles.tabs_from_indent, + page_doc: pages.doc.clone(), + parts, + hyphenate, + custom, + sources, + kept, + hidden_controls, + page_image, + sum_spacing, + changes, + kept_defs, + kept_files, + section_masters, + }) +} + +/// Parse an `.odt` package into a document. +pub fn import(bytes: &[u8]) -> Result { + import_on(bytes, &SectionProps::default()) +} + +/// [`import`], with `page` the default page (the user's region's): a page layout that states no +/// size is `page`'s size, as LibreOffice makes it its default paper's. +pub fn import_on(bytes: &[u8], page: &SectionProps) -> Result { + model::begin_import(); + let Parsed { + flow, + last, + track, + font, + size, + color, + spacing, + looks, + named, + named_odt, + char_styles, + char_defs, + settings, + outline, + kern, + widows, + scripts, + default_lang, + default_tab, + tabs_from_indent, + page_doc, + parts, + hyphenate, + custom, + sources, + kept, + hidden_controls, + page_image, + sum_spacing, + changes, + kept_defs, + kept_files, + section_masters, + } = parse_tracked(bytes, (page.page_w, page.page_h))?; + let mut doc = model::to_doc(&flow); + kept.store(&mut doc); + // Elements can't hold a NUL: it parts them. + if !hidden_controls.is_empty() { + doc.passthrough.insert(super::forms::HIDDEN_PART.into(), std::sync::Arc::new(hidden_controls.join("\0").into_bytes())); + } + doc.custom_props = custom; + (doc.core.last_modified_by, doc.core.modified, doc.core.revision) = changes; + doc.sources = sources; + doc.settings.auto_hyphenation |= hyphenate; + if font.is_some() { + Arc::make_mut(&mut doc.styles).default_chr.font = font; + } + if size.is_some() { + Arc::make_mut(&mut doc.styles).default_chr.size = size; + } + if let Some(c) = color { + Arc::make_mut(&mut doc.styles).default_chr.color = Some(craft_wordmodel::props::TextColor::Rgb(c)); + } + Arc::make_mut(&mut doc.styles).default_chr.kern = kern.then_some(1.0); + Arc::make_mut(&mut doc.styles).default_chr.overlay(&scripts); + // The file's own, or none when it states none (each application then uses its own default): + // not the English of WordCraft's new documents. + Arc::make_mut(&mut doc.styles).default_chr.lang = default_lang; + if let Some(t) = default_tab { + doc.settings.default_tab = t; + } + doc.settings.page_color = page_doc.page_color; + // Laid out with the standard's rules, its tab stops measured as the file says. + doc.settings.odf = Some(craft_wordmodel::OdfLayout { tabs_from_indent }); + doc.settings.sum_paragraph_spacing = sum_spacing; + // The page background picture (Design › Page Color › Fill Effects › Picture). + if let Some((bytes, fill)) = page_image + && let Some(ext) = model::sniff_image(&bytes) + { + let media = doc.add_media(bytes, ext); + doc.settings.page_picture = Some(craft_wordmodel::PagePicture { media, fill_type: fill.to_string(), title: String::new() }); + } + if let Some((fmt, props)) = page_doc.notes[0] { + (doc.settings.footnote_format, doc.settings.footnote_pr) = (fmt, props); + } + if let Some((fmt, props)) = page_doc.notes[1] { + (doc.settings.endnote_format, doc.settings.endnote_pr) = (fmt, props); + } + let d = &mut Arc::make_mut(&mut doc.styles).default_para; + (d.space_before, d.space_after, d.line_spacing) = (spacing.before, spacing.after, spacing.line); + (d.indent_left, d.indent_right, d.indent_first) = (spacing.left, spacing.right, spacing.first); + // Unstated: on, XSL's initial value of 2 lines (§20.312 and §20.244 refer to it), and Word's. + d.widow_control = Some(widows.unwrap_or(true)); + for (kind, look) in &looks { + apply_look(&mut doc, *kind, look); + } + if let Some(s) = settings { + doc.passthrough.insert(super::package::SETTINGS_PART.into(), std::sync::Arc::new(s)); + } + add_custom_styles(&mut doc, &named, &named_odt, tabs_from_indent); + add_char_styles(&mut doc, &char_styles, &char_defs); + if let Some(levels) = &outline { + number_headings(&mut doc, levels); + } + doc.last_section = last; + page_parts::add(&mut doc, parts); + one_size_for_all_scripts(&mut doc); + if let Some(t) = track { + doc.settings.track_changes = t; + } + // The markup kept for paragraphs nothing changed while reading, filed for the writer. + if doc.kept.get().is_none() { + doc.kept = craft_wordmodel::kept::Kept::new(Default::default()); + } + if let Some(k) = doc.kept.get_mut() { + k.format = "odt".into(); + for (key, xml) in kept_defs { + k.keep_def(key, xml); + } + for (path, bytes) in kept_files { + k.keep_file(path, bytes); + } + k.section_masters = section_masters; + } + doc.seal_kept(); + let (fp, pages) = (craft_wordmodel::kept::styles_fingerprint(&doc), craft_wordmodel::kept::pages_fingerprint(&doc)); + if let Some(k) = doc.kept.get_mut() { + (k.styles_fp, k.pages_fp) = (Some(fp), Some(pages)); + } + Ok(doc) +} + +/// The page setup of each master page in `styles.xml` (ODF 1.3 §16.9), and of the first one, +/// which pages use until a paragraph names another. +#[derive(Default)] +pub(super) struct Pages { + pub(super) masters: HashMap, + pub(super) first: Option, + pub(super) first_name: Option, + /// Each master page's next one (§19.511 `style:next-style-name`): the page style the pages + /// after its first take. + pub(super) next: HashMap, + /// What the document's first page style and its note and line numbering settings say for + /// the whole document. + pub(super) doc: PageDoc, +} + +/// Page settings that are the document's in WordCraft (and Word): the page colour (the first +/// page style's background), footnote and endnote numbering (`text:notes-configuration`). +#[derive(Clone, Default)] +pub(super) struct PageDoc { + pub(super) page_color: Option, + /// The first page style's background picture (§17.3 `style:background-image`): its package + /// path and WordCraft's fill for its repeat (tiled, stretched, or centred at its size). + pub(super) page_image: Option<(String, &'static str)>, + /// Footnotes', then endnotes', number format and options. + pub(super) notes: [Option<(craft_wordmodel::section::NumFormat, craft_wordmodel::section::NoteProps)>; 2], +} + +/// A `text:notes-configuration` (ODF 1.3 §16.29): which notes it is for (endnotes: true), their +/// number format, first number (ODF counts from 0), restart, and footnotes at the page's foot +/// or the document's end. +fn notes_config(a: &Attrs) -> (bool, craft_wordmodel::section::NumFormat, craft_wordmodel::section::NoteProps) { + use craft_wordmodel::section::{MAX_NOTE_START, NotePos, NoteProps, NoteRestart}; + let endnote = get(a, "text:note-class").is_some_and(|c| c.trim() == "endnote"); + let fmt = super::styles::num_format(get(a, "style:num-format").unwrap_or("1")); + let num_start = + get(a, "text:start-value").and_then(|v| v.trim().parse::().ok()).map(|v| v.saturating_add(1).min(u64::from(MAX_NOTE_START)) as u32); + let num_restart = get(a, "text:start-numbering-at").map(|v| match v.trim() { + "page" => NoteRestart::EachPage, + "chapter" => NoteRestart::EachSect, + _ => NoteRestart::Continuous, + }); + let pos = match (endnote, get(a, "text:footnotes-position").map(str::trim)) { + (false, Some("document")) => Some(NotePos::DocEnd), + (false, Some("page")) => Some(NotePos::PageBottom), + _ => None, + }; + (endnote, fmt, NoteProps { pos, num_fmt: None, num_start, num_restart }) +} + +/// A `text:linenumbering-configuration` (ODF 1.3 §16.31) that numbers lines: every how many, +/// how far from the text, restarting on each page or not. +fn line_numbering(a: &Attrs) -> Option { + use craft_wordmodel::section::{LineNumberRestart, LineNumbering}; + // On unless it says otherwise (§19.845). + if get(a, "text:number-lines").is_some_and(|v| v.trim() == "false") { + return None; + } + let count_by = get(a, "text:increment").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(1, 100); + let distance = get(a, "text:offset").and_then(length).filter(|v| v.is_finite()).map_or(0.0, |v| v.clamp(0.0, 1584.0)); + let restart = + if get(a, "text:restart-on-page").is_some_and(|v| v.trim() == "true") { LineNumberRestart::Page } else { LineNumberRestart::Continuous }; + Some(LineNumbering { count_by, start: 1, distance, restart }) +} + +/// A page layout's header or footer area (`style:header-style`/`style:footer-style`, ODF 1.3 +/// §16.6–16.7): its minimum height and its spacing from the text, in points. +#[derive(Clone, Copy, Default)] +struct Band { + /// Its height (`fo:min-height`), when stated: the standard gives no default (see + /// [`Band::room`]). + height: Option, + spacing: f32, + /// Content taller than the band takes its spacing first (§20.264 `style:dynamic-spacing`, + /// as Word does), instead of keeping it below; `None` when unstated (§20.264 has no default; + /// see [`Band::room`]). + dynamic: Option, +} + +impl Band { + /// Where the text starts below the band's start (the page margin), and the gap kept below + /// content taller than the band. What the standard leaves open: the spacing comes after the + /// height, an unstated height is none (as tall as its text), and unstated dynamic spacing is + /// off (on, with no height stated, it would always drop the spacing the file states). + fn room(&self) -> (f32, f32) { + let room = self.height.unwrap_or(0.0) + self.spacing; + let gap = if self.dynamic.unwrap_or(false) { 0.0 } else { self.spacing }; + (room, gap) + } +} + +/// A page layout read from `styles.xml`: its properties, columns, and header and footer areas. +#[derive(Default)] +struct Layout { + attrs: Attrs, + /// Its background picture: package path and fill (see [`PageDoc::page_image`]). + image: Option<(String, &'static str)>, + columns: Option, + column_specs: Vec, + header: Band, + footer: Band, +} + +/// A page stating nothing, `paper` (width, height) in size: no margins (§20.206–20.209 refer to +/// XSL's margins, whose initial value is 0; whether it comes with the reference the standard +/// doesn't say). +fn unstated_page(paper: (f32, f32)) -> SectionProps { + SectionProps { + page_w: paper.0, + page_h: paper.1, + margin_top: 0.0, + margin_bottom: 0.0, + margin_left: 0.0, + margin_right: 0.0, + ..SectionProps::default() + } +} + +fn read_pages(xml: &[u8], paper: (f32, f32)) -> Pages { + let mut layouts: HashMap = HashMap::new(); + let mut cur: Option = None; + // Inside a layout's footer style (else its header style). + let mut in_footer = false; + // Master pages: name, layout, whether a header and a footer show. + let mut masters: Vec<(String, String, bool, bool)> = Vec::new(); + let mut in_master = false; + let mut nexts: HashMap = HashMap::new(); + let mut doc = PageDoc::default(); + let mut lines = None; + events(xml, |ev| match ev { + Ev::Start(q, a) => match local(&q) { + "notes-configuration" if q.starts_with("text:") => { + let (endnote, fmt, props) = notes_config(&a); + if let Some(slot) = doc.notes.get_mut(usize::from(endnote)) { + *slot = Some((fmt, props)); + } + } + "linenumbering-configuration" if q.starts_with("text:") => lines = line_numbering(&a), + "page-layout" if q.starts_with("style:") => { + cur = get(&a, "style:name").map(String::from); + // A layout stating nothing is still one: the defaults' page. + if let Some(n) = &cur + && layouts.len() < MAX_SECTIONS + { + layouts.entry(n.clone()).or_default(); + } + } + // A page background picture (§17.3, in `style:page-layout-properties`): §20.249 + // `style:repeat` (repeat is the default) as WordCraft's tile, stretch, or centre. + "background-image" if q.starts_with("style:") => { + if let (Some(l), Some(href)) = + (cur.as_ref().and_then(|n| layouts.get_mut(n)), get(&a, "xlink:href").map(str::trim).filter(|h| !h.is_empty() && h.len() <= 1024)) + { + let fill = match get(&a, "style:repeat").map(str::trim) { + Some("no-repeat") => "center", + Some("stretch") => "frame", + _ => "tile", + }; + l.image = Some((href.trim_start_matches("./").to_string(), fill)); + } + } + // A band whose height isn't stated is LibreOffice's default, 0.5 cm (measured; see + // `Band::default`). + "header-style" => in_footer = false, + "footer-style" => in_footer = true, + "header-footer-properties" => { + if let Some(l) = cur.as_ref().and_then(|n| layouts.get_mut(n)) { + let len = |k: &str| get(&a, k).and_then(length).filter(|v| *v >= 0.0).map(|v| v.min(1584.0)); + let band = if in_footer { &mut l.footer } else { &mut l.header }; + band.height = len("fo:min-height").or_else(|| len("svg:height")); + band.spacing = len(if in_footer { "fo:margin-top" } else { "fo:margin-bottom" }).unwrap_or(0.0); + band.dynamic = get(&a, "style:dynamic-spacing").map(|v| v.trim() == "true"); + } + } + "page-layout-properties" => { + if let Some(l) = cur.as_ref().and_then(|n| layouts.get_mut(n)) { + l.attrs = a; + } + } + "columns" if q.starts_with("style:") => { + if let Some(l) = cur.as_ref().and_then(|n| layouts.get_mut(n)) { + l.columns = Some(columns_of(&a)); + } + } + "column" if q.starts_with("style:") => { + if let Some(l) = cur + .as_ref() + .and_then(|n| layouts.get_mut(n)) + .filter(|l| l.columns.is_some() && l.column_specs.len() < super::styles::MAX_COLUMN_SPECS) + && let Some(spec) = super::styles::column_spec(&a) + { + l.column_specs.push(spec); + } + } + "column-sep" => { + if let Some(c) = cur.as_ref().and_then(|n| layouts.get_mut(n)).and_then(|l| l.columns.as_mut()) { + c.separator = column_sep(&a); + } + } + "master-page" if masters.len() < MAX_SECTIONS => { + in_master = false; + if let (Some(n), Some(l)) = (get(&a, "style:name"), get(&a, "style:page-layout-name")) { + masters.push((n.to_string(), l.to_string(), false, false)); + in_master = true; + if let Some(next) = get(&a, "style:next-style-name").map(str::trim).filter(|x| !x.is_empty() && *x != n) { + nexts.insert(n.to_string(), next.to_string()); + } + } + } + "header" | "footer" if in_master && q.starts_with("style:") && get(&a, "style:display") != Some("false") => { + if let Some(m) = masters.last_mut() { + if local(&q) == "header" { + m.2 = true; + } else { + m.3 = true; + } + } + } + _ => {} + }, + Ev::End(q) => match local(&q) { + "page-layout" => cur = None, + "master-page" => in_master = false, + _ => {} + }, + Ev::Text(_) => {} + }); + let mut pages = Pages { doc, next: nexts, ..Default::default() }; + for (name, layout, header, footer) in masters { + let Some(Layout { attrs: a, columns: cols, column_specs, header: hb, footer: fb, image }) = layouts.get(&layout) else { continue }; + // What a layout leaves unsaid: the default paper, and no margins. + let mut sec = unstated_page(paper); + apply_layout(a, &mut sec); + sec.line_numbers.clone_from(&lines); + if pages.first.is_none() { + pages.doc.page_color = get(a, "fo:background-color").and_then(model::parse_color); + pages.doc.page_image.clone_from(image); + } + // ODF starts a header at the page margin and the text after the header's band; Word + // measures the header's distance and the text's margin from the edge. How the band's + // height and spacing add up, and what an unstated height or spacing rule is, the + // standard leaves open (see `Band::room`). + let limit = sec.page_w.max(sec.page_h); + if header { + let (room, gap) = hb.room(); + sec.header = sec.margin_top; + sec.margin_top = (sec.margin_top + room).min(limit); + // Content taller than the band pushes the text down, the spacing below it kept or not. + sec.header_gap = gap; + } + if footer { + let (room, gap) = fb.room(); + sec.footer = sec.margin_bottom; + sec.margin_bottom = (sec.margin_bottom + room).min(limit); + sec.footer_gap = gap; + } + // A one-column layout keeps the default columns (and their gap). + if let Some(c) = cols.as_ref().filter(|c| c.count > 1) { + sec.columns = super::styles::with_widths(c.clone(), column_specs, sec.text_width()); + } + if pages.first.is_none() { + pages.first = Some(sec.clone()); + pages.first_name = Some(name.clone()); + } + pages.masters.entry(name).or_insert(sec); + } + pages +} + +/// A master page followed by another (`style:next-style-name`, LibreOffice's "First Page" +/// before "Default") is a section with a different first page: its own headers and footers on +/// the first page (none there is an empty one), the next master's on the pages after it. Its +/// page size and margins are its own (the first page's). +fn first_pages(pages: &mut Pages) { + let names: Vec<(String, String)> = pages.next.iter().map(|(a, b)| (a.clone(), b.clone())).collect(); + // The masters as read: a master the loop has already changed (Left and Right pages naming each + // other) gives the next one its own headers, whatever order the map yields them in. + let read = pages.masters.clone(); + for (name, next) in names { + let Some(after) = read.get(&next).cloned() else { continue }; + let Some(sec) = pages.masters.get_mut(&name) else { continue }; + let (own_header, own_footer) = (sec.headers.default, sec.footers.default); + sec.headers = after.headers; + sec.footers = after.footers; + sec.headers.first = own_header; + sec.footers.first = own_footer; + sec.title_page = true; + if pages.first_name.as_deref() == Some(name.as_str()) { + pages.first = Some(sec.clone()); + } + } +} + +/// Replace `sec` with the page setup of the first master page in `styles.xml`, if any. +#[cfg(test)] +pub(super) fn apply_page_layout(xml: &[u8], sec: &mut SectionProps) { + if let Some(first) = read_pages(xml, default_paper()).first { + *sec = first; + } +} + +/// A `style:page-layout-properties` element's page size, orientation and margins into `sec`. +fn apply_layout(a: &Attrs, sec: &mut SectionProps) { + let len = |k: &str| get(a, k).and_then(length).filter(|v| *v >= 0.0); + // Like the DOCX reader: a page is at least half an inch; at most 22 inches (Word's limit), so a hostile + // `1e30pt` can't reach layout. + let page = |k: &str| len(k).filter(|v| *v >= 36.0).map(|v| v.min(1584.0)); + if let (Some(w), Some(h)) = (page("fo:page-width"), page("fo:page-height")) { + sec.page_w = w; + sec.page_h = h; + } + sec.landscape = match get(a, "style:print-orientation") { + Some("landscape") => true, + Some("portrait") => false, + _ => sec.page_w > sec.page_h, + }; + for (k, slot) in [ + ("fo:margin-top", &mut sec.margin_top), + ("fo:margin-bottom", &mut sec.margin_bottom), + ("fo:margin-left", &mut sec.margin_left), + ("fo:margin-right", &mut sec.margin_right), + ] { + if let Some(v) = len(k) { + // No margin is larger than the page's longer side (as `w:pgMar` in the DOCX reader). + *slot = v.min(sec.page_w.max(sec.page_h)); + } + } + // Page numbers' format (§19.500 `style:num-format`; an empty one shows the numbers anyway). + if let Some(f) = get(a, "style:num-format").filter(|f| !f.trim().is_empty()) { + sec.page_num_format = super::styles::num_format(f); + } + // Page borders (§20.183 `fo:border`, §20.210 `fo:padding`: the space inside them). + let side = |s: &str| { + let b = get(a, &format!("fo:border-{s}")).or_else(|| get(a, "fo:border")).and_then(super::styles::parse_border)?; + let pad = get(a, &format!("fo:padding-{s}")).or_else(|| get(a, "fo:padding")).and_then(length).filter(|v| v.is_finite()); + (b.style != craft_wordmodel::props::BorderStyle::None) + // Any distance on the page (Word's 31 pt limit is for its own measure, from the edge). + .then(|| craft_wordmodel::props::Border { space: pad.map_or(0.0, |p| p.clamp(0.0, 1584.0)), ..b }) + }; + // ODF's border is at the margin, the text inside its padding; WordCraft measures the border + // from the page edge and the text margin from the edge too. + let limit = sec.page_w.max(sec.page_h); + let inset = |b: Option, margin: &mut f32| { + b.map(|b| { + let edge = *margin; + *margin = (edge + b.width + b.space).min(limit); + craft_wordmodel::props::Border { space: edge, ..b } + }) + }; + let borders = craft_wordmodel::props::Borders { + top: inset(side("top"), &mut sec.margin_top), + bottom: inset(side("bottom"), &mut sec.margin_bottom), + left: inset(side("left"), &mut sec.margin_left), + right: inset(side("right"), &mut sec.margin_right), + ..Default::default() + }; + if borders.any_visible() { + sec.page_borders = Some(borders); + } +} diff --git a/crates/odf/src/text/kept.rs b/crates/odf/src/text/kept.rs new file mode 100644 index 000000000..e9efebdec --- /dev/null +++ b/crates/odf/src/text/kept.rs @@ -0,0 +1,833 @@ +//! Markup kept from the file an ODT document was opened from (see `craft_wordmodel::kept`): the +//! definitions kept paragraphs refer to, read here, and written back with them. +//! +//! A kept paragraph names its automatic styles (`P12`, `T5`), and through them common styles, +//! list styles, data styles and font faces. They are kept as read, by name: `auto:` for +//! `content.xml`'s automatic styles, `common:` for `styles.xml`'s common styles, `font:` for +//! font faces. On save, the ones a written paragraph needs (and what they need in turn) go back +//! in; WordCraft's own automatic styles take another prefix, so names can't meet. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use super::xml::{Ev, events_at, get}; + +/// Most definitions kept from one part (hostile input). +const MAX_DEFS: usize = 100_000; + +/// The definitions in `xml` (a `content.xml` or `styles.xml`), each by the key it is kept +/// under: a child of `office:automatic-styles` as `auto:` in `content.xml` (`content` +/// true), `sauto:` in `styles.xml`; of `office:styles` as `common:`; of +/// `office:font-face-decls` as `font:`; of `office:master-styles` as `master:`; the +/// namespaces the root declares as `ns:` (the URI); and in `content.xml`, the document's +/// `office:scripts` as `top:office:scripts` and the declarations opening the text +/// (`text:variable-decls`…) as `decls:`. +pub(super) fn defs(xml: &[u8], content: bool) -> Vec<(String, Arc)> { + let mut out = Vec::new(); + let mut path: Vec = Vec::new(); + let mut open: Option<(String, usize, usize)> = None; + let mut before_1_3 = false; + let mut prefixes = std::collections::BTreeSet::new(); + events_at(xml, |ev, at| match ev { + Ev::Start(q, a) => { + path.push(q.clone()); + let depth = path.len(); + // The namespaces the root declares, which kept markup may use. + if depth == 1 { + // Older files' unitless gradient angles are tenths of a degree (see `to_1_3`). + before_1_3 = super::strokes::legacy_angles(get(&a, "office:version")); + for (k, v) in &a { + if let Some(p) = k.strip_prefix("xmlns:").filter(|p| !p.is_empty() && p.len() <= 64) + && out.len() < MAX_DEFS + { + prefixes.insert(p.to_string()); + out.push((format!("ns:{p}"), Arc::from(v.as_str()))); + } + } + return; + } + // Those a part's container declares too: what is kept of it leaves the container behind. + if depth == 2 { + for (k, v) in &a { + if let Some(p) = k.strip_prefix("xmlns:").filter(|p| !p.is_empty() && p.len() <= 64) + && out.len() < MAX_DEFS + && prefixes.insert(p.to_string()) + { + out.push((format!("ns:{p}"), Arc::from(v.as_str()))); + } + } + } + // A reference mark (§6.6.2), which `text:reference-ref` names: WordCraft reads it as a + // bookmark, and writes it as both. + if content + && matches!(q.as_str(), "text:reference-mark" | "text:reference-mark-start") + && let Some(n) = get(&a, "text:name").filter(|n| !n.is_empty() && n.len() <= 256) + && out.len() < MAX_DEFS + { + out.push((format!("refmark:{n}"), Arc::from(""))); + } + if open.is_some() || out.len() >= MAX_DEFS { + return; + } + let parent = path.get(depth.wrapping_sub(2)).map(String::as_str).unwrap_or(""); + let key = match (depth, parent) { + (2, _) if content && q == "office:scripts" => Some("top:office:scripts".to_string()), + // Whether the text holds soft page breaks (§19.857), which markup kept may. + (3, "office:body") + if content + && q == "office:text" + && get(&a, "text:use-soft-page-breaks").is_some_and(|v| v.trim() == "true") + && out.len() < MAX_DEFS => + { + out.push(("soft-page-breaks".to_string(), Arc::from("true"))); + None + } + (3, "office:automatic-styles") => child_key(if content { "auto" } else { "sauto" }, &q, &a), + (3, "office:styles") => child_key("common", &q, &a), + (3, "office:font-face-decls") => child_key("font", &q, &a), + (3, "office:master-styles") => child_key("master", &q, &a), + (4, "office:text") if content && q.starts_with("text:") && q.ends_with("-decls") => Some(format!("decls:{q}")), + (4, "office:text") if content && q == "office:forms" => Some("forms".to_string()), + // A tracked change (§5.5.2), by its id: a paragraph written as read names it. + (5, "text:tracked-changes") if content && q == "text:changed-region" => { + get(&a, "text:id").or_else(|| get(&a, "xml:id")).filter(|i| !i.is_empty() && i.len() <= 256).map(|i| format!("change:{i}")) + } + _ => None, + }; + if let Some(key) = key { + open = Some((key, at.0, depth)); + } + } + Ev::End(_) => { + if let Some((_, from, d)) = &open + && *d == path.len() + { + let from = *from; + if let Some((key, ..)) = open.take() + && let Some(s) = xml.get(from..at.1).and_then(|b| std::str::from_utf8(b).ok()) + { + let mut s = if before_1_3 { to_1_3(s) } else { s.to_string() }; + // ODF 1.3 names a changed region by `xml:id` (§5.5.2); an older file's by + // `text:id` alone. + if let Some(id) = key.strip_prefix("change:") + && s.split('>').next().is_some_and(|h| !h.contains("xml:id=")) + { + s.insert_str(" {} + }); + out +} + +/// Markup kept from a file older than ODF 1.3, written as ODF 1.3: a gradient's, transparency +/// gradient's or hatch's angle without a unit is given its unit, as the file meant it — tenths +/// of a degree (`300` → `30deg`). That reading deviates from the standard out of necessity +/// (§18.3.1 says degrees): see `strokes::legacy_angles`. +fn to_1_3(xml: &str) -> String { + if !(xml.contains("').map_or(tail.len(), |e| e + 1); + let (tag, after) = tail.split_at(end); + if tag.starts_with("().ok()).filter(|v| v.is_finite()) + { + t.replace_range(a..a + len, &format!("{}deg", v / 10.0)); + } + } + out.push_str(&t); + } else { + out.push_str(tag); + } + rest = after; + } + out.push_str(rest); + out +} + +/// The key a definition (element `q` with attributes `a`, a child of group `g`) is kept under: +/// `:` (its `style:name`, or a drawing definition's `draw:name`); an unnamed one (a +/// default style, the outline style, a note or line numbering configuration, a layer set) as +/// `~~`. +fn child_key(g: &str, q: &str, a: &super::xml::Attrs) -> Option { + // Drawing definitions (dashes, gradients, arrowheads…) are named by `draw:name`. + match get(a, "style:name").or_else(|| get(a, "draw:name")).filter(|n| !n.is_empty()) { + Some(name) => Some(format!("{g}:{name}{}", space(q, get(a, "style:family").unwrap_or("")))), + None => { + let which = get(a, "style:family").or_else(|| get(a, "text:note-class")).unwrap_or(""); + Some(format!("{g}~{q}~{which}")) + } + } +} + +/// The name space a definition (element `q`, of style family `family`) belongs to, as a key's +/// suffix: names are unique only within one (§16.2: a style's within its family; a list style, +/// a data style, a gradient may share a paragraph style's name, as a graphic style may: +/// LibreOffice's "Marginalia" is both). Paragraph styles, font faces and master pages have none; +/// another family's style is `|`, a list style `|list`, a data style `|data`, another +/// definition `|`. +fn space(q: &str, family: &str) -> String { + match q { + "style:style" if family.trim() == "paragraph" || family.is_empty() => String::new(), + "style:style" => format!("|{}", family.trim()), + "style:font-face" | "style:master-page" => String::new(), + "text:list-style" => "|list".to_string(), + q if q.starts_with("number:") => "|data".to_string(), + q => format!("|{}", q.split_once(':').map_or(q, |(_, l)| l)), + } +} + +/// Whether `xml` (a part WordCraft wrote) already defines what `def` (a kept definition) does: +/// an element of its kind by its name (a list style and a paragraph style may share one). +pub(super) fn defined(xml: &str, def: &str) -> bool { + let head = def.split('>').next().unwrap_or(""); + let el = head.trim_start_matches('<').split([' ', '/', '\t', '\n', '\r']).next().unwrap_or(""); + let name = ["style:name", "draw:name"].into_iter().find_map(|a| Some((a, raw_attr(head, a)?))); + let (Some((attr, name)), false) = (name, el.is_empty()) else { return false }; + let open = format!("<{el} "); + xml.match_indices(&open).any(|(i, _)| { + xml.get(i..).and_then(|t| t.split('>').next()).is_some_and(|tag| raw_attr(tag, attr).is_some_and(|n| attr_text(n) == attr_text(name))) + }) +} + +/// An attribute's value as it reads, however XML spells it (`O'Brien`, `O'Brien`). +pub(super) fn attr_text(raw: &str) -> std::borrow::Cow<'_, str> { + std::borrow::Cow::Owned(craft_xml::unescape(raw)) +} + +/// Attribute `name`'s value as `tag` (a start tag) writes it, spaced and quoted however XML allows. +pub(super) fn raw_attr<'a>(tag: &'a str, name: &str) -> Option<&'a str> { + let past_space = |at: usize| Some(at + tag.get(at..)?.find(|c: char| !c.is_ascii_whitespace())?); + let mut at = tag.find(|c: char| c.is_ascii_whitespace())?; + loop { + let name_at = past_space(at)?; + let eq = name_at + tag.get(name_at..)?.find('=')?; + let value_at = past_space(eq + 1)?; + let rest = tag.get(value_at..)?; + let quote = rest.chars().next().filter(|c| matches!(c, '"' | '\''))?; + let len = rest.get(1..)?.find(quote)?; + if tag.get(name_at..eq)?.trim_end() == name { + return rest.get(1..1 + len); + } + at = value_at + 1 + len + 1; + } +} + +/// The children of `container` (a child of the root) in `xml` (a part WordCraft wrote): each +/// one's key in `group` (as [`defs`] gives it) and its markup's byte range; and the range of +/// the container's content (`None` when there is no such container). +pub(super) fn children(xml: &str, container: &str, group: &str) -> (Vec<(String, (usize, usize))>, Option<(usize, usize)>) { + let mut out = Vec::new(); + let (mut depth, mut inside) = (0usize, false); + let mut open: Option<(String, usize)> = None; + let mut content: Option<(usize, usize)> = None; + events_at(xml.as_bytes(), |ev, at| match ev { + Ev::Start(q, a) => { + depth += 1; + if depth == 2 { + inside = q == container; + if inside { + content = Some((at.1, at.1)); + } + } else if depth == 3 + && inside + && let Some(key) = child_key(group, &q, &a) + { + open = Some((key, at.0)); + } + } + Ev::End(_) => { + if depth == 3 + && let Some((key, from)) = open.take() + { + out.push((key, (from, at.1))); + } + if depth == 2 && inside { + // An empty element (``) has no content to replace: none. + content = content.filter(|c| c.0 < at.1).map(|c| (c.0, at.0.max(c.0))); + inside = false; + } + depth = depth.saturating_sub(1); + } + Ev::Text(_) => {} + }); + (out, content) +} + +/// `xml` (a part WordCraft wrote) with `container`'s children as read: the file's (those of +/// `group` in `defs`, in `order`; `pick` may choose WordCraft's instead, or neither), then +/// WordCraft's own the file had none of by that key. +fn with_as_read( + xml: &str, + container: &str, + group: &str, + defs: &BTreeMap>, + order: &[String], + pick: &dyn Fn(&str, &str, Option<&str>) -> Option, +) -> String { + let (kids, content) = children(xml, container, group); + let Some((from, to)) = content else { return xml.to_string() }; + let ours = |k: &str| kids.iter().find(|(key, _)| key == k).and_then(|(_, (a, b))| xml.get(*a..*b)); + let prefix = (format!("{group}:"), format!("{group}~")); + let mut body = String::new(); + for k in order.iter().filter(|k| k.starts_with(&prefix.0) || k.starts_with(&prefix.1)) { + if let Some(d) = defs.get(k) + && let Some(chosen) = pick(k, d, ours(k)) + { + body.push_str(&chosen); + } + } + for (k, (a, b)) in &kids { + if !defs.contains_key(k) + && let Some(x) = xml.get(*a..*b) + { + body.push_str(x); + } + } + let (Some(head), Some(tail)) = (xml.get(..from), xml.get(to..)) else { return xml.to_string() }; + format!("{head}{body}{tail}") +} + +/// Whether a definition (its markup) is a dash, gradient, marker or fill picture: WordCraft's +/// where both have one by that name, as the drawings it writes from the model name theirs. A +/// graphic style (the `Graphics` and `Frame` styles drawings derive from) is the file's: its +/// drawings written as read derive from it, and those written from the model derived from it +/// when read (what WordCraft models of it is in theirs). +fn drawing_def(xml: &str) -> bool { + xml.split('>').next().unwrap_or("").starts_with(">, order: &[String]) -> String { + with_as_read(styles, "office:styles", "common", defs, order, &|_, d, ours| match ours { + Some(o) if drawing_def(o) => Some(o.to_string()), + // (The default graphic style is the file's: WordCraft's drawings say what they need.) + _ => Some(d.to_string()), + }) +} + +/// `styles` (a `styles.xml` WordCraft wrote) with its page layouts, master pages and their +/// automatic styles as read, WordCraft's own added where the file had none by that name. For a +/// document whose page setup is as read. +pub(super) fn with_pages_as_read(styles: &str, defs: &BTreeMap>, order: &[String]) -> String { + let keep_file = |_: &str, d: &str, _: Option<&str>| Some(d.to_string()); + let s = with_as_read(styles, "office:automatic-styles", "sauto", defs, order, &keep_file); + with_as_read(&s, "office:master-styles", "master", defs, order, &keep_file) +} + +/// Attribute names whose values name a definition kept markup may refer to. +fn refers(attr: &str) -> bool { + attr.ends_with("style-name") || attr.starts_with("style:font-name") || attr == "style:list-style-name" +} + +/// The names `xml` refers to (the values of its style- and font-name attributes). +pub(super) fn names(xml: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + super::xml::events(xml.as_bytes(), |ev| { + if let Ev::Start(_, a) = ev { + for (k, v) in a { + if refers(&k) && !v.is_empty() { + out.insert(v); + } + } + } + }); + out +} + +/// Most package files kept definitions may refer to. +const MAX_FILES: usize = 1000; + +/// The package files `defs` refer to (`xlink:href` to a path inside the package, as a style's +/// fill picture; to a folder, as an embedded object in a header: the files in it), read with +/// `fetch` (`entries`: the package's paths): a definition written as read needs them. +/// Also the paths linked to that are in the package but couldn't be read (over the picture +/// budget, unreadable): markup linking to one can't be written as read. +pub(super) fn files( + defs: &[(String, Arc)], + entries: &[String], + fetch: &mut dyn FnMut(&str) -> Option>, +) -> (Vec<(String, Vec)>, BTreeSet) { + let mut paths = BTreeSet::new(); + for (_, xml) in defs { + hrefs(xml, &mut paths); + } + let mut out = Vec::new(); + let mut missing = BTreeSet::new(); + for p in paths { + if entries.contains(&p) { + match fetch(&p) { + Some(b) => out.push((p, b)), + None => { + missing.insert(p); + } + } + } else { + // A folder (an object): the files in it. + let dir = format!("{}/", p.trim_end_matches('/')); + for e in entries.iter().filter(|e| e.starts_with(&dir) && !e.ends_with('/')).take(MAX_FILES) { + match fetch(e) { + Some(b) => out.push((e.clone(), b)), + None => { + missing.insert(p.clone()); + } + } + } + } + } + (out, missing) +} + +/// Whether markup `xml` links to one of the package files in `missing` (see [`files`]). +pub(super) fn links_to(xml: &str, missing: &BTreeSet) -> bool { + if missing.is_empty() { + return false; + } + let mut paths = BTreeSet::new(); + hrefs(xml, &mut paths); + paths.iter().any(|p| missing.contains(p)) +} + +/// The package files `xml` links to (`xlink:href` to a path inside the package), added to +/// `paths` (at most [`MAX_FILES`]). +pub(super) fn hrefs(xml: &str, paths: &mut BTreeSet) { + if !xml.contains("xlink:href") { + return; + } + super::xml::events(xml.as_bytes(), |ev| { + if let Ev::Start(_, a) = ev + && let Some(h) = get(&a, "xlink:href") + && inside_package(h) + && paths.len() < MAX_FILES + { + paths.insert(h.trim_start_matches("./").to_string()); + } + }); +} + +/// Whether link `h` names a file inside the package (a relative path, not a URL or a fragment). +fn inside_package(h: &str) -> bool { + !h.is_empty() && !h.contains(':') && !h.starts_with(['#', '/']) && !h.contains("..") +} + +/// The document properties WordCraft writes from its model (§4.3); the file's others are kept. +const META_MODELLED: [&str; 11] = [ + "meta:generator", + "dc:title", + "dc:subject", + "dc:description", + "meta:initial-creator", + "dc:creator", + "dc:date", + "meta:editing-cycles", + "meta:keyword", + "meta:creation-date", + "meta:user-defined", +]; + +/// The children of `meta.xml`'s `office:meta` WordCraft doesn't model (the template, the +/// statistics, the editing time, the printing…), as read, in order. +pub(super) fn meta_defs(xml: &[u8]) -> Vec<(String, Arc)> { + let mut out = Vec::new(); + let mut depth = 0usize; + let mut open: Option<(String, usize)> = None; + events_at(xml, |ev, at| match ev { + Ev::Start(q, _) => { + depth += 1; + if depth == 3 && open.is_none() && !META_MODELLED.contains(&q.as_str()) && out.len() < MAX_DEFS { + open = Some((format!("meta~{}~{q}", out.len()), at.0)); + } + } + Ev::End(_) => { + if depth == 3 + && let Some((key, from)) = open.take() + && let Some(s) = xml.get(from..at.1).and_then(|b| std::str::from_utf8(b).ok()) + { + out.push((key, Arc::from(s))); + } + depth = depth.saturating_sub(1); + } + Ev::Text(_) => {} + }); + out +} + +/// `meta` (a `meta.xml` WordCraft wrote) with the file's document properties WordCraft doesn't +/// model (`defs`' `meta~` ones) added, in their order. +pub(super) fn with_kept_meta(meta: String, defs: &BTreeMap>, order: &[String]) -> String { + let kept: String = order.iter().filter(|k| k.starts_with("meta~")).filter_map(|k| defs.get(k)).map(|d| &**d).collect(); + match meta.rfind("") { + Some(at) if !kept.is_empty() => format!("{}{kept}{}", meta.get(..at).unwrap_or(""), meta.get(at..).unwrap_or("")), + _ => meta, + } +} + +/// Whether a child of a `form:form` is a form control (else the form's own properties or +/// event listeners). +fn is_control(q: &str) -> bool { + q.starts_with("form:") && !matches!(q, "form:properties" | "form:form" | "form:connection-resource") +} + +/// The children `depth` deep in `xml` (the root is 1): each one's element, `form:name` and +/// markup. +fn children_at(xml: &str, at_depth: usize) -> Vec<(String, Option, &str)> { + let mut out = Vec::new(); + let mut depth = 0usize; + let mut open: Option<(String, Option, usize)> = None; + events_at(xml.as_bytes(), |ev, at| match ev { + Ev::Start(q, a) => { + depth += 1; + if depth == at_depth && open.is_none() { + open = Some((q, get(&a, "form:name").map(str::to_string), at.0)); + } + } + Ev::End(_) => { + if depth == at_depth + && let Some((q, name, from)) = open.take() + && let Some(x) = xml.get(from..at.1) + { + out.push((q, name, x)); + } + depth = depth.saturating_sub(1); + } + Ev::Text(_) => {} + }); + out +} + +/// Control `ours` (as WordCraft writes it) with what the file's control `theirs` had that +/// WordCraft doesn't model and so can't have changed: its other attributes (not its ids) and its +/// properties and event listeners. +fn with_unmodelled(ours: &str, theirs: &str) -> String { + let attrs_of = |xml: &str| { + let mut out: Vec<(String, String)> = Vec::new(); + let mut first = true; + super::xml::events(xml.as_bytes(), |ev| { + if let Ev::Start(_, a) = ev + && std::mem::take(&mut first) + { + out = a; + } + }); + out + }; + let have: BTreeSet = attrs_of(ours).into_iter().map(|(k, _)| k).collect(); + let extra: String = attrs_of(theirs) + .into_iter() + .filter(|(k, _)| !have.contains(k) && !matches!(k.as_str(), "form:id" | "xml:id" | "form:name") && !k.starts_with("xmlns")) + .map(|(k, v)| format!(" {k}=\"{}\"", super::writer::x(&v))) + .collect(); + let kids: String = children_at(theirs, 2) + .into_iter() + .filter(|(q, ..)| matches!(q.as_str(), "form:properties" | "office:event-listeners")) + .filter(|(q, ..)| !ours.contains(&format!("<{q}"))) + .map(|(.., x)| x) + .collect(); + // Into its start tag, and before its end (an empty one opened for them). + let Some(end) = ours.find('>') else { return ours.to_string() }; + let (head, rest) = ours.split_at(end); + let empty = head.ends_with('/'); + let head = head.trim_end_matches('/'); + let name = head.trim_start_matches('<').split(char::is_whitespace).next().unwrap_or(""); + if empty { + if kids.is_empty() { format!("{head}{extra}/>") } else { format!("{head}{extra}>{kids}") } + } else { + let close = format!(""); + let body = rest.get(1..).unwrap_or(""); + match body.rfind(&close) { + Some(at) => format!("{head}{extra}>{}{kids}{}", body.get(..at).unwrap_or(""), body.get(at..).unwrap_or("")), + None => format!("{head}{extra}>{body}"), + } + } +} + +/// The document's forms (`office:forms`, §13.2) as the file had them (`orig`), with the +/// controls WordCraft writes (`ours`, its own `office:forms`): the file's forms element and each +/// form's own start tag, properties and event listeners as read; the controls the document has +/// in the first form (the file's controls are the document's, read into it and written back); +/// `ours` alone when the file had none, its own form when the file had none. +pub(super) fn merge_forms(orig: Option<&str>, ours: String) -> String { + let Some(orig) = orig else { return ours }; + // `ours`: the controls (children of its one form). + let ours_controls = children_at(&ours, 3); + let orig_controls: Vec<(String, Option, &str)> = children_at(orig, 3).into_iter().filter(|(q, ..)| is_control(q)).collect(); + // Each of ours with what the file's control by that name (else in that place) had and + // WordCraft doesn't model. + let mut used = vec![false; orig_controls.len()]; + let mut controls = String::new(); + for (i, (q, name, xml)) in ours_controls.iter().enumerate() { + let same = |(k, (oq, oname, _)): &(usize, &(String, Option, &str))| !used[*k] && oq == q && oname == name; + let found = orig_controls.iter().enumerate().find(|c| same(c)).map(|(k, _)| k).or_else(|| { + let nth = ours_controls.iter().take(i).filter(|(oq, ..)| oq == q).count(); + orig_controls.iter().enumerate().filter(|(k, (oq, ..))| !used[*k] && oq == q).nth(nth).map(|(k, _)| k) + }); + match found.and_then(|k| orig_controls.get(k).map(|c| (k, c.2))) { + Some((k, theirs)) => { + used[k] = true; + controls.push_str(&with_unmodelled(xml, theirs)); + } + None => controls.push_str(xml), + } + } + // `orig`: its forms element, each form's start tag and non-control children. + let mut head: Option = None; + let mut forms: Vec<(String, String)> = Vec::new(); + let mut child: Option<(usize, bool)> = None; + let mut depth = 0usize; + events_at(orig.as_bytes(), |ev, at| match ev { + Ev::Start(q, _) => { + depth += 1; + let tag = orig.get(at.0..at.1).unwrap_or(""); + let opened = || tag.strip_suffix("/>").map_or_else(|| tag.to_string(), |t| format!("{}>", t.trim_end())); + match depth { + 1 => head = Some(opened()), + 2 if q == "form:form" => forms.push((opened(), String::new())), + 3 if child.is_none() => child = Some((at.0, is_control(&q))), + _ => {} + } + } + Ev::End(_) => { + if depth == 3 + && let Some((from, control)) = child.take() + && !control + && let (Some(f), Some(x)) = (forms.last_mut(), orig.get(from..at.1)) + { + f.1.push_str(x); + } + depth = depth.saturating_sub(1); + } + Ev::Text(_) => {} + }); + let Some(head) = head else { return ours }; + let mut out = head; + if forms.is_empty() && !controls.is_empty() { + out.push_str(&format!("{controls}")); + } + for (i, (start, kids)) in forms.iter().enumerate() { + out.push_str(start); + out.push_str(kids); + if i == 0 { + out.push_str(&controls); + } + out.push_str(""); + } + out.push_str(""); + out +} + +/// A list of declarations as the file had it (`orig`, a `text:variable-decls` or the like), +/// with those of WordCraft's (`ours`, the same list) it doesn't have by name; `ours` alone when +/// the file had none. +pub(super) fn merge_decls(orig: Option<&str>, ours: String) -> String { + let Some(orig) = orig else { return ours }; + let named = |xml: &str| -> Vec<(String, (usize, usize))> { + let mut out = Vec::new(); + let mut depth = 0usize; + events_at(xml.as_bytes(), |ev, at| match ev { + Ev::Start(_, a) => { + depth += 1; + if depth == 2 + && let Some(n) = get(&a, "text:name") + { + out.push((n.to_string(), at)); + } + } + Ev::End(_) => { + if depth == 2 + && let Some(last) = out.last_mut() + { + last.1.1 = at.1; + } + depth = depth.saturating_sub(1); + } + Ev::Text(_) => {} + }); + out + }; + let have: BTreeSet = named(orig).into_iter().map(|(n, _)| n).collect(); + let extra: String = named(&ours).into_iter().filter(|(n, _)| !have.contains(n)).filter_map(|(_, (a, b))| ours.get(a..b)).collect(); + if extra.is_empty() { + return orig.to_string(); + } + match orig.rfind("') && !orig.trim_end().ends_with("/>") => { + format!("{}{extra}{}", orig.get(..at).unwrap_or(""), orig.get(at..).unwrap_or("")) + } + // ``: opened and closed around them. + _ => { + let el = orig.trim_start_matches('<').split([' ', '/', '>']).next().unwrap_or(""); + format!("{}>{extra}", orig.trim_end().trim_end_matches("/>")) + } + } +} + +/// The tracked changes (§5.5.7 `text:change-id`) `xml` marks, by id. +pub(super) fn changes(xml: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + if xml.contains("text:change-id") { + super::xml::events(xml.as_bytes(), |ev| { + if let Ev::Start(_, a) = ev + && let Some(id) = get(&a, "text:change-id").filter(|i| !i.is_empty() && i.len() <= 256) + && out.len() < MAX_DEFS + { + out.insert(id.to_string()); + } + }); + } + out +} + +/// The names of the bookmarks (§6.2.1) `xml` begins (at a point too) and ends. +pub(super) fn bookmarks(xml: &str) -> (BTreeSet, BTreeSet) { + let (mut starts, mut ends) = (BTreeSet::new(), BTreeSet::new()); + if xml.contains(" { + starts.insert(n.to_string()); + ends.insert(n.to_string()); + } + "text:bookmark-start" => { + starts.insert(n.to_string()); + } + "text:bookmark-end" => { + ends.insert(n.to_string()); + } + _ => {} + } + } + }); + } + (starts, ends) +} + +/// The sequences (§7.4.13 `text:sequence`) `xml` counts, which the document declares. +pub(super) fn sequences(xml: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + if xml.contains(">, BTreeMap>, BTreeMap>); + +pub(super) fn needed(defs: &BTreeMap>, start: BTreeSet, masters: bool) -> Option { + let (mut auto, mut common, mut fonts) = (BTreeMap::new(), BTreeMap::new(), BTreeMap::new()); + let mut todo: Vec = start.into_iter().collect(); + let mut seen = BTreeSet::new(); + while let Some(n) = todo.pop() { + if !seen.insert(n.clone()) || seen.len() > MAX_DEFS { + continue; + } + // Every definition by that name (in any name space: see `space`), from the first group + // that has one: automatic styles, then common ones, then font faces. + let mut found = false; + for (group, into) in [("auto", &mut auto), ("common", &mut common), ("font", &mut fonts)] { + let base = format!("{group}:{n}"); + for (k, def) in defs.range(base.clone()..).take_while(|(k, _)| k.starts_with(&base)) { + let Some(rest) = k.strip_prefix(&base).filter(|r| r.is_empty() || r.starts_with('|')) else { continue }; + if !masters && def.contains("style:master-page-name=\"") && !def.contains("style:master-page-name=\"\"") { + return None; + } + into.insert(format!("{n}{rest}"), def.clone()); + todo.extend(names(def)); + found = true; + } + if found { + break; + } + } + } + Some((auto, common, fonts)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CONTENT: &str = r#""#; + + #[test] + fn definitions_are_kept_by_name_and_followed() { + let d: BTreeMap> = defs(CONTENT.as_bytes(), true).into_iter().collect(); + assert_eq!(d.keys().cloned().collect::>(), ["auto:L1|list", "auto:P1", "auto:P2", "font:Liberation Serif"]); + assert!(d["auto:P1"].starts_with("")); + let (auto, common, fonts) = needed(&d, names(r#"x"#), false).unwrap(); + assert_eq!(auto.keys().cloned().collect::>(), ["L1|list", "P1"]); + // "Body" and "Bullets" weren't kept here (they're common styles in styles.xml). + assert!(common.is_empty()); + assert_eq!(fonts.keys().cloned().collect::>(), ["Liberation Serif"]); + // A paragraph starting a page isn't written as kept. + assert!(needed(&d, names(r#""#), false).is_none()); + } + + #[test] + fn angles_from_an_older_file_get_their_unit() { + let xml = r#""#; + let d: BTreeMap> = defs(xml.as_bytes(), false).into_iter().collect(); + assert!(d["common:G|gradient"].contains(r#"draw:angle="30deg""#), "{}", d["common:G|gradient"]); + assert!(d["common:H|hatch"].contains(r#"draw:rotation="45deg""#)); + assert!(d["common:D|gradient"].contains(r#"draw:angle="45deg""#)); + // From a 1.3 file: as it is. + let d: BTreeMap> = defs(xml.replace("1.2", "1.3").as_bytes(), false).into_iter().collect(); + assert!(d["common:G|gradient"].contains(r#"draw:angle="300""#)); + } + + #[test] + fn definitions_sharing_a_name_are_each_kept() { + // A list style may have a paragraph style's name (§16.2: names are unique within a kind + // of definition), as LibreOffice's "Numbering 1" both are; each is kept, and markup + // naming it needs both. + let xml = r#""#; + let d: BTreeMap> = defs(xml.as_bytes(), false).into_iter().collect(); + assert!(d["common:N"].starts_with(""#), false).unwrap(); + assert_eq!(common.keys().cloned().collect::>(), ["N", "N|list"]); + // One is defined where an element of its kind has the name, not the other kind. + let ours = r#""#; + assert!(defined(ours, &d["common:N"]) && !defined(ours, &d["common:N|list"])); + } + + #[test] + fn hostile_markup_keeps_what_it_can() { + assert!(defs(b"> = + [("auto:A".to_string(), Arc::from(r#""#))].into(); + assert_eq!(needed(&d, ["A".to_string()].into(), false).unwrap().0.len(), 1); + } +} diff --git a/crates/odf/src/text/limits.rs b/crates/odf/src/text/limits.rs new file mode 100644 index 000000000..75b6457c2 --- /dev/null +++ b/crates/odf/src/text/limits.rs @@ -0,0 +1,25 @@ +/// Largest zip entry we inflate. +pub(super) const MAX_ENTRY: u64 = 256 << 20; + +/// Deepest element nesting handled in content.xml. +pub(super) const MAX_XML_DEPTH: usize = 512; + +/// Most sections an import makes (page-style changes and column sections); later ones are ignored. +pub(super) const MAX_SECTIONS: usize = 10_000; + +/// Longest shown text a field keeps, in bytes. +pub(super) const MAX_FIELD_RESULT: usize = 64 << 10; + +/// Most comments read; later ones are left out. +pub(super) const MAX_COMMENTS: usize = 100_000; + +/// Most `text:changed-region`s read (hostile input). +pub(super) const MAX_REGIONS: usize = 10_000; + +/// Most insertions open at once (`text:change-start` without its end yet). +pub(super) const MAX_OPEN_INSERTIONS: usize = 64; + +/// Longest change author or date kept, in characters. +pub(super) const MAX_INFO_CHARS: usize = 256; +/// Most spaces the document's `text:s` elements add beyond the one each stands for (hostile input). +pub(super) const MAX_EXTRA_SPACES: usize = 4 << 20; diff --git a/crates/odf/src/text/objects.rs b/crates/odf/src/text/objects.rs new file mode 100644 index 000000000..e0585fed5 --- /dev/null +++ b/crates/odf/src/text/objects.rs @@ -0,0 +1,112 @@ +//! Embedded objects WordCraft shows but doesn't edit (ODF 1.3 §10.4.6 `draw:object` holding a +//! sub-document folder, §10.4.7 `draw:object-ole` an OLE file): their files gathered from the +//! package so the writer can put them back, and a placeholder picture for one whose replacement +//! picture WordCraft can't show (LibreOffice writes its own metafile format there). + +use std::collections::HashMap; +use std::sync::Arc; + +use craft_flow::OdfObject; +use craft_flow::odf_object::MAX_FILES; + +/// The media type of an OLE object (§10.4.7; LibreOffice's manifest names it so). +pub(in crate::text) const OLE_MIME: &str = "application/vnd.sun.star.oleobject"; + +/// The package path an object link names (`./Object 1` → `Object 1`), when it is one inside the +/// package: not empty, not absolute, not leaving the package, not a URL. +fn inside(href: &str) -> Option<&str> { + let p = href.trim().trim_start_matches("./").trim_end_matches('/'); + let bad = p.is_empty() || p.starts_with('/') || p.contains(':') || p.split('/').any(|s| s == ".." || s == "." || s.is_empty()); + (!bad).then_some(p) +} + +/// The media type a sub-document's `content.xml` says it is, by its body's content. +fn guess_mime(content: &[u8]) -> &'static str { + let head = String::from_utf8_lossy(content.get(..content.len().min(64 * 1024)).unwrap_or_default()); + for (el, mime) in [ + (">, + fetch: &mut dyn FnMut(&str) -> Option>, +) -> Option { + let path = inside(href)?; + let mut files = Vec::new(); + if ole { + files.push((String::new(), Arc::new(fetch(path)?))); + } else { + let prefix = format!("{path}/"); + for name in entries.iter().filter(|n| n.starts_with(&prefix) && !n.ends_with('/')).take(MAX_FILES) { + let rel = name.get(prefix.len()..).unwrap_or_default(); + if inside(rel).is_some() + && let Some(bytes) = fetch(name) + { + files.push((rel.to_string(), Arc::new(bytes))); + } + } + } + if files.is_empty() { + return None; + } + let mime = manifest.and_then(|m| m.get(&format!("{path}/")).or_else(|| m.get(path))).cloned().unwrap_or_else(|| { + if ole { + OLE_MIME.to_string() + } else { + let content = files.iter().find(|(n, _)| n == "content.xml").map(|(_, b)| b.as_slice()).unwrap_or_default(); + guess_mime(content).to_string() + } + }); + Some(OdfObject { mime, ole, files, replacement: None }) +} + +/// Each `manifest:file-entry`'s path and media type in a package's `META-INF/manifest.xml`. +pub(in crate::text) fn manifest(xml: &[u8]) -> HashMap { + let mut out = HashMap::new(); + super::events(xml, |ev| { + if let super::Ev::Start(q, a) = ev + && q == "manifest:file-entry" + && out.len() < 100_000 + && let (Some(p), Some(m)) = (super::get(&a, "manifest:full-path"), super::get(&a, "manifest:media-type")) + { + out.insert(p.to_string(), m.to_string()); + } + }); + out +} + +/// A placeholder picture for an object whose own picture WordCraft can't show: a pale panel +/// with a frame, ruled as a grid for a spreadsheet. Drawn here, in WordCraft's own greys. +pub(in crate::text) fn placeholder(mime: &str) -> Option> { + let (w, h) = (120u32, 72u32); + let grid = mime.ends_with("spreadsheet"); + let img = image::RgbImage::from_fn(w, h, |x, y| { + let edge = x == 0 || y == 0 || x == w - 1 || y == h - 1; + let rule = grid && (x % 30 == 0 || y % 12 == 0); + image::Rgb(if edge { + [150, 150, 150] + } else if rule { + [214, 214, 214] + } else { + [246, 246, 246] + }) + }); + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).ok()?; + Some(png) +} diff --git a/crates/odf/src/text/package.rs b/crates/odf/src/text/package.rs new file mode 100644 index 000000000..97520027c --- /dev/null +++ b/crates/odf/src/text/package.rs @@ -0,0 +1,1544 @@ +use super::writer::{NS, Writer, align_attr, props_text, pt, spacing_props, x}; +use super::{Columns, Document, FBlock, Meta, SectionProps, SectionStart, model}; + +/// `style:columns` (ODF 1.3 §17.12) of equal columns with their gap and, when on, the line between. +fn columns_xml(c: &Columns) -> String { + let sep = if c.separator { "" } else { "" }; + let count = c.count.clamp(1, 45); + // Unequal columns (§17.12): each one's share of the width with the half gaps beside it as + // its indents, as LibreOffice writes them. + let mut each = String::new(); + if c.widths.len() == count as usize && count > 1 { + let clean = |v: f32| if v.is_finite() { v.clamp(0.0, 1584.0) } else { 0.0 }; + let mut start = 0.0; + for (w, after) in &c.widths { + let end = clean(*after) / 2.0; + let rel = ((clean(*w) + start + end) * 100.0).round().max(1.0) as u32; + each.push_str(&format!("", pt(start), pt(end))); + start = end; + } + } + // LibreOffice ignores a page's column widths beside a gap: the gaps are in the indents. + let gap = if each.is_empty() { format!(" fo:column-gap=\"{}\"", pt(c.space)) } else { String::new() }; + format!("{sep}{each}") +} + +/// One master page per entry of `pages` (name, setup), each with its own page layout `pm1`, `pm2`… +/// (§16.5, §16.9); the first is the default one, "Standard". +/// The styles WordCraft's paragraph kinds are written in (Heading 1–6, Title, Quotations, +/// Preformatted Text), each with the look of the document's own style: font, size, bold, +/// italic, colour and the rest of its text formatting, spacing, indents, alignment and keep with +/// next, written out in full so other programs show them as WordCraft does. +fn kind_styles(doc: &Document, on_outline: &std::collections::BTreeSet) -> String { + let mut s = String::new(); + let mut kinds: Vec<(String, String, String, Option, &str)> = + (1..=6).map(|n| (format!("Heading_20_{n}"), format!("Heading {n}"), format!("Heading{n}"), Some(n), "text")).collect(); + kinds.push(("Title".into(), "Title".into(), "Title".into(), None, "chapter")); + kinds.push(("Quotations".into(), "Quotations".into(), "Quote".into(), None, "html")); + kinds.push(("Preformatted_20_Text".into(), "Preformatted Text".into(), model::CODE_STYLE.into(), None, "html")); + for (name, display, id, outline, class) in kinds { + let (pp, cp) = match doc.styles.get(&id) { + Some(_) => doc.styles.para_style_props(&id), + // Preformatted text in a document without its style: monospaced, no spacing. + None => ( + craft_wordmodel::props::ParaProps { space_before: Some(0.0), space_after: Some(0.0), ..Default::default() }, + craft_wordmodel::props::CharProps { font: Some(model::MONO_FONT.into()), size: Some(10.0), ..Default::default() }, + ), + }; + let mut para = spacing_props(&model::Spacing { + before: pp.space_before, + after: pp.space_after, + line: pp.line_spacing, + // Zero indents are the defaults; stating them would override the indents of the + // heading numbering in LibreOffice (its number then sits against the text). + left: pp.indent_left.filter(|v| *v != 0.0), + right: pp.indent_right.filter(|v| *v != 0.0), + first: pp.indent_first.filter(|v| *v != 0.0), + ..Default::default() + }); + para.push_str(align_attr(pp.align)); + if pp.keep_next == Some(true) { + para.push_str(" fo:keep-with-next=\"always\""); + } + if let Some(c) = pp.contextual_spacing { + para.push_str(&format!(" style:contextual-spacing=\"{c}\"")); + } + let outline = outline.map(|n| outline_attrs(doc, n, &pp, on_outline.contains(&id))).unwrap_or_default(); + s.push_str(&format!( + "", + props_text(&cp) + )); + } + s +} + +/// The styles of the headings in `blocks` (and their tables, to 16 levels deep) with a number by +/// their own outline level. +fn styles_numbered_by_own_level(blocks: &[std::sync::Arc], depth: usize, out: &mut std::collections::BTreeSet) { + for b in blocks { + match &**b { + craft_wordmodel::Block::Para(p) => { + if p.props.outline_level.is_some() + && p.props.numbering.is_some_and(|n| n.num != 0) + && let Some(s) = &p.props.style + { + out.insert(s.clone()); + } + } + craft_wordmodel::Block::Table(t) if depth < 16 => { + for cell in t.rows.iter().flat_map(|r| r.cells.iter()) { + styles_numbered_by_own_level(&cell.blocks, depth + 1, out); + } + } + craft_wordmodel::Block::Table(_) => {} + } + } +} + +/// A paragraph style's outline level `n` (`style:default-outline-level`), its properties being `pp`. +fn outline_attrs(doc: &Document, n: u8, pp: &craft_wordmodel::props::ParaProps, on_outline: bool) -> String { + // A level the outline doesn't number names no list style: LibreOffice would otherwise give + // it the outline's (unnumbered) level, and its indent. Unless a heading in the style has a + // number by its own level (`on_outline`), which that would take from it. + let numbered = pp + .numbering + .and_then(|r| doc.numbering.level(r.num, r.level)) + .is_some_and(|l| !matches!(l.format, craft_wordmodel::section::NumFormat::None | craft_wordmodel::section::NumFormat::Bullet)); + let detach = if numbered || on_outline { "" } else { " style:list-style-name=\"\"" }; + format!(" style:default-outline-level=\"{n}\"{detach}") +} + +/// The ODF parent of document style `id`: Standard for Normal, the kinds' styles for theirs, a +/// custom style by its id. +fn odf_parent(doc: &Document, id: Option<&str>) -> String { + match id { + None | Some("Normal") => "Standard".into(), + Some("Title") => "Title".into(), + Some("Quote") => "Quotations".into(), + Some(id) if id == model::CODE_STYLE => "Preformatted_20_Text".into(), + Some(id) => match id.strip_prefix("Heading").and_then(|n| n.parse::().ok()).filter(|n| (1..=9).contains(n)) { + Some(n) => format!("Heading_20_{}", n.min(6)), + None if model::is_custom_style(doc, id) => super::writer::odf_name(id), + None => "Standard".into(), + }, + } +} + +/// The custom paragraph styles in `named` and the custom styles they are based on, as common +/// styles (ODF 1.3 §16.2): named by their document id, shown by their name, each with its own +/// paragraph and text formatting over its parent's. +fn custom_styles(doc: &Document, named: &std::collections::BTreeSet, on_outline: &std::collections::BTreeSet) -> String { + let mut todo: Vec = named.iter().cloned().collect(); + let mut done = std::collections::BTreeSet::new(); + let mut s = String::new(); + while let Some(id) = todo.pop() { + if !done.insert(id.clone()) || done.len() > 10_000 { + continue; + } + let Some(st) = doc.styles.get(&id) else { continue }; + if let Some(b) = st.based_on.as_deref().filter(|b| model::is_custom_style(doc, b)) { + todo.push(b.to_string()); + } + let pp = &st.para; + let mut attrs = spacing_props(&model::Spacing { + before: pp.space_before, + after: pp.space_after, + line: pp.line_spacing, + left: pp.indent_left, + right: pp.indent_right, + first: pp.indent_first, + ..Default::default() + }); + attrs.push_str(align_attr(pp.align)); + if pp.keep_next == Some(true) { + attrs.push_str(" fo:keep-with-next=\"always\""); + } + if let Some(c) = pp.contextual_spacing { + attrs.push_str(&format!(" style:contextual-spacing=\"{c}\"")); + } + if pp.bidi == Some(true) { + attrs.push_str(" style:writing-mode=\"rl-tb\""); + } + attrs.push_str(&super::writer::border_attrs(pp.borders)); + if let Some(bg) = pp.shading { + attrs.push_str(&format!(" fo:background-color=\"#{}\"", bg.hex().to_ascii_lowercase())); + } + let tabs = pp + .tabs + .as_deref() + .map(|tabs| { + // From the style's indent where the document measures them so (as they are read). + let indent = if model::tabs_from_indent(doc) { pp.indent_left.unwrap_or(0.0) } else { 0.0 }; + let tabs: Vec<_> = + tabs.iter().map(|t| craft_wordmodel::props::TabStop { pos: (t.pos - indent).clamp(-1584.0, 1584.0), ..*t }).collect(); + super::writer::tab_stops(&tabs) + }) + .unwrap_or_default(); + let parent = odf_parent(doc, st.based_on.as_deref()); + let outline = pp.outline_level.filter(|l| *l < 9).map(|l| outline_attrs(doc, l + 1, pp, on_outline.contains(&id))).unwrap_or_default(); + s.push_str(&format!( + "{tabs}", + super::writer::odf_name(&id), + x(&super::writer::generated_style(&id).map_or_else(|| st.name.clone(), |(_, d)| d)), + props_text(&with_complex_size(doc, &id, &st.chr)) + )); + } + s +} + +/// Paragraph style `id`'s own text formatting `chr`, with the complex-script size it takes when it +/// states only a size: a parent's complex size where one sets it (the model takes it from there, +/// and the writer would otherwise give it this style's size), else its own size. +fn with_complex_size(doc: &Document, id: &str, chr: &craft_wordmodel::props::CharProps) -> craft_wordmodel::props::CharProps { + let mut chr = chr.clone(); + if chr.size.is_some() && chr.size_cs.is_none() { + let cs = doc.styles.resolve_char(Some(id), &craft_wordmodel::props::CharProps::default()).size_cs; + chr.size_cs = (chr.size != Some(cs)).then_some(cs); + } + chr +} + +/// The custom character styles in `named` and those they are based on, as common text styles. +fn char_styles(doc: &Document, named: &std::collections::BTreeSet) -> String { + let mut todo: Vec = named.iter().cloned().collect(); + let mut done = std::collections::BTreeSet::new(); + let mut s = String::new(); + while let Some(id) = todo.pop() { + if !done.insert(id.clone()) || done.len() > 10_000 { + continue; + } + let Some(st) = doc.styles.get(&id) else { continue }; + let parent = match st.based_on.as_deref().filter(|b| model::is_custom_char_style(doc, b)) { + Some(b) => { + todo.push(b.to_string()); + format!(" style:parent-style-name=\"{}\"", super::writer::odf_name(b)) + } + None => String::new(), + }; + s.push_str(&format!( + "", + super::writer::odf_name(&id), + x(&st.name), + props_text(&st.chr) + )); + } + s +} + +/// `text:outline-style` (ODF 1.3 §16.34): the numbering of Heading 1–6, from the list level each +/// heading style numbers its paragraphs with; levels without one show no number. +fn outline_style(doc: &Document) -> String { + let mut s = String::from(""); + for n in 1..=10u8 { + let level = (n <= 6) + .then(|| doc.styles.para_style_props(&format!("Heading{n}")).0.numbering) + .flatten() + .and_then(|r| doc.numbering.level(r.num, r.level).cloned().map(|l| (r.level, l))); + let (indent, hanging) = level.as_ref().map(|(_, l)| (l.indent, l.hanging)).unwrap_or((0.0, 0.0)); + let attrs = match level { + Some((lv, l)) if !matches!(l.format, craft_wordmodel::section::NumFormat::None | craft_wordmodel::section::NumFormat::Bullet) => { + let label = model::Label::of_level(l.format, &l.text); + let mut a = format!(" style:num-format=\"{}\"", super::writer::odf_num_format(l.format)); + for (k, v) in [("style:num-prefix", label.prefix.as_str()), ("style:num-suffix", label.suffix.as_str())] { + if !v.is_empty() { + a.push_str(&format!(" {k}=\"{}\"", x(v))); + } + } + if label.shown > 1 { + a.push_str(&format!(" text:display-levels=\"{}\"", label.shown.min(lv.saturating_add(1)))); + } + if l.start != 1 { + a.push_str(&format!(" text:start-value=\"{}\"", l.start)); + } + a + } + _ => " style:num-format=\"\"".into(), + }; + // The number at the level's indent, followed by a tab (to the next default tab stop + // when the level has no indent), as Word lays out numbered headings. + let fit = |v: f32| if v.is_finite() { v.clamp(0.0, 1584.0) } else { 0.0 }; + // A level with no indent states none, so LibreOffice uses its own (a tab after the number). + let place = if fit(indent) > 0.0 || fit(hanging) > 0.0 { + format!( + " text:list-tab-stop-position=\"{}\" fo:text-indent=\"{}\" fo:margin-left=\"{}\"", + pt(fit(indent)), + pt(-fit(hanging)), + pt(fit(indent)) + ) + } else { + String::new() + }; + s.push_str(&format!( + "" + )); + } + s.push_str(""); + s +} + +/// A master page's headers and footers (ODF 1.3 §16.10), written. +#[derive(Default)] +struct PageParts { + /// The `style:header…`/`style:footer…` elements. + xml: String, + header: bool, + footer: bool, +} + +/// The headers and footers section `sec` shows, written by `w` (its styles go in `styles.xml`): +/// the default one for every page, the even one as left pages' when the document has different +/// odd and even pages, the first one with a different first page. A slot Word leaves empty is an +/// empty element, so the page shows nothing there rather than the default one. +fn page_parts(doc: &Document, sec: &SectionProps, w: &mut Writer) -> PageParts { + let mut out = PageParts::default(); + let even = doc.settings.even_odd_headers; + // A watermark is a shape in every header (one written for it alone when there is none). + let wm = doc.settings.watermark.as_ref().filter(|w| !w.text.trim().is_empty()); + let area = (sec.page_w - sec.margin_left - sec.margin_right, sec.page_h - sec.margin_top - sec.margin_bottom); + for (set, name) in [(&sec.headers, "header"), (&sec.footers, "footer")] { + let wanted = + set.default.is_some() || (even && set.even.is_some()) || (sec.title_page && set.first.is_some()) || (name == "header" && wm.is_some()); + if !wanted { + continue; + } + for (id, el, on) in + [(set.default, name.to_string(), true), (set.even, format!("{name}-left"), even), (set.first, format!("{name}-first"), sec.title_page)] + { + if !on { + continue; + } + let mut body = String::new(); + if let Some(part) = id.and_then(|i| doc.parts.get(&i)) { + let blocks = model::part_flow( + doc, + &part.blocks, + // Pictures, shapes, text boxes, charts and objects too, as in the body. + model::FlowOpts { fields: true, lists: true, chars: true, paras: true, boxes: true, ..Default::default() }, + ); + w.blocks(&blocks, &mut body, 0); + } + if body.is_empty() { + body.push_str(""); + } + if name == "header" + && let Some(wm) = wm + { + body = with_watermark(&body, &w.watermark(wm, (sec.page_w, sec.page_h), area)); + } + out.xml.push_str(&format!("{body}")); + } + if name == "header" { + out.header = true; + } else { + out.footer = true; + } + } + out +} + +/// A header's content with `shape` (anchored to a character) at the start of its first +/// paragraph, or in a paragraph of its own before a table that starts it. +fn with_watermark(body: &str, shape: &str) -> String { + if let Some(rest) = body.strip_prefix("") { + return format!("{shape}{rest}"); + } + if body.starts_with("') { + let (tag, rest) = body.split_at(end + 1); + if let Some(open) = tag.strip_suffix("/>") { + let name = if tag.starts_with("{shape}{rest}"); + } + return format!("{tag}{shape}{rest}"); + } + } + format!("{shape}{body}") +} + +/// `styles.xml` of `doc`: the default style/// `styles.xml` of `doc`: the default style from the document defaults (Normal's spacing, the +/// body font, pair kerning, the tab interval), the paragraph kinds' styles, and one master page +/// per entry of `pages` with `parts`, its headers and footers, whose automatic styles are +/// `part_styles`. +fn styles_xml( + doc: &Document, + pages: &[(String, craft_wordmodel::SectionProps)], + parts: &[PageParts], + part_styles: &str, + named: &std::collections::BTreeSet, +) -> String { + // WordCraft's page layouts take another prefix next to the file's own (see `super::kept`). + let pm = if doc.kept.of("odt").is_some() { "wcpm" } else { "pm" }; + // The document defaults: Normal's font, spacing and indents, pair kerning and tab interval. + let body_font = doc.styles.default_chr.font.clone().unwrap_or_else(|| craft_wordmodel::styles::BODY_FONT.to_string()); + let normal = doc.styles.resolve_para(&Default::default()); + let defaults = &model::Spacing { + before: Some(normal.space_before), + after: Some(normal.space_after), + line: Some(normal.line_spacing), + left: Some(normal.indent_left), + right: Some(normal.indent_right), + first: Some(normal.indent_first), + ..Default::default() + }; + let kern = doc.styles.default_chr.kern.is_some_and(|k| k > 0.0); + // Normal's font size (Standard, written empty, takes the default style's). + let size = doc.styles.resolve_char(None, &Default::default()).size; + let size = if size.is_finite() { size.clamp(1.0, 1638.0) } else { 12.0 }; + let tab = doc.settings.default_tab.clamp(1.0, 1584.0); + let mut s = format!("\n"); + s.push_str(&format!( + "", + spacing_props(defaults), + x(&body_font), + // Widow and orphan control as Normal has it, said either way (the standard leaves it open + // when unsaid, and readers differ: docs/odf-interpretation.md). + widows = if normal.widow_control { " fo:widows=\"2\" fo:orphans=\"2\"" } else { " fo:widows=\"0\" fo:orphans=\"0\"" }, + size = pt(size), + size_cs = if doc.styles.default_chr.size_cs.is_none() { format!(" style:font-size-complex=\"{}\"", pt(size)) } else { String::new() }, + // The document's text colour (a colour, not automatic). + color = match doc.styles.default_chr.color { + Some(craft_wordmodel::props::TextColor::Rgb(c)) => format!(" fo:color=\"#{}\"", c.hex().to_ascii_lowercase()), + _ => String::new(), + }, + tab = pt(tab), + // The document's language, without which each reader uses its own default: Normal's, as + // WordCraft has it (a Word file's Normal may state another than its defaults). + lang = doc.styles.resolve_char(None, &Default::default()).lang.as_deref().map(|l| super::writer::lang_attrs(l, "")).unwrap_or_default(), + scripts = super::writer::script_attrs(&doc.styles.default_chr) + )); + s.push_str(""); + let mut on_outline = std::collections::BTreeSet::new(); + styles_numbered_by_own_level(&doc.body, 0, &mut on_outline); + s.push_str(&kind_styles(doc, &on_outline)); + s.push_str(&outline_style(doc)); + s.push_str(&custom_styles(doc, named, &on_outline)); + // The note text styles, as WordCraft's Footnote Text and Endnote Text: 10 pt, no spacing. + for (name, class) in [("Footnote", "extra"), ("Endnote", "extra")] { + s.push_str(&format!( + "" + )); + } + s.push_str(¬es_configuration(doc)); + if let Some(l) = pages.first().and_then(|(_, sec)| sec.line_numbers.as_ref()) { + s.push_str(&line_numbering_xml(l)); + } + s.push_str(""); + s.push_str(part_styles); + let none = PageParts::default(); + for (i, (_, sec)) in pages.iter().enumerate() { + let pp = parts.get(i).unwrap_or(&none); + // A page of several columns carries them in its layout. + let cols = if sec.columns.count > 1 { columns_xml(&sec.columns) } else { String::new() }; + // Word measures the header from the page edge and the text from the page edge; ODF + // starts the header at the page margin and the text after the header's height. So with + // a header the page margin is the header's distance, and the header is tall enough to + // reach the text (it grows with its content, as Word's pushes the text down). + let (top, header_h) = if pp.header { (sec.header, (sec.margin_top - sec.header).max(0.0)) } else { (sec.margin_top, 0.0) }; + let (bottom, footer_h) = if pp.footer { (sec.footer, (sec.margin_bottom - sec.footer).max(0.0)) } else { (sec.margin_bottom, 0.0) }; + // A page border is at ODF's margin, the text (or header, footer) inside its padding. + let (borders, [top, bottom, left, right]) = page_borders(sec.page_borders, [top, bottom, sec.margin_left, sec.margin_right]); + // The band's height takes in its spacing (see the reader), which content taller than + // the band keeps below it. + let gap = |g: f32, h: f32| if g.is_finite() { g.clamp(0.0, h) } else { 0.0 }; + // Spacing kept below a taller header says so, as the standard leaves the default open. + let keeps = |g: f32| if g.is_finite() && g > 0.0 { " style:dynamic-spacing=\"false\"" } else { "" }; + let header = if pp.header { + format!( + "", + pt(header_h), + pt(gap(sec.header_gap, header_h)), + keeps(sec.header_gap) + ) + } else { + String::new() + }; + let footer = if pp.footer { + format!( + "", + pt(footer_h), + pt(gap(sec.footer_gap, footer_h)), + keeps(sec.footer_gap) + ) + } else { + String::new() + }; + s.push_str(&format!( + "{cols}{header}{footer}", + i + 1, + pt(sec.page_w), + pt(sec.page_h), + pt(top), + pt(bottom), + pt(left), + pt(right), + if sec.page_w > sec.page_h { "landscape" } else { "portrait" }, + page_look(doc, sec) + &borders + )); + } + s.push_str(""); + for (i, (name, _)) in pages.iter().enumerate() { + let xml = parts.get(i).map(|p| p.xml.as_str()).unwrap_or(""); + s.push_str(&format!("{xml}", x(name), i + 1)); + } + s.push_str("\n"); + s +} + +/// `Document::passthrough` key of an ODT's own `settings.xml`, kept for the next save. +pub(super) const SETTINGS_PART: &str = "odt/settings.xml"; + +/// Largest `settings.xml` kept, in bytes. +pub(super) const MAX_SETTINGS: usize = 1 << 20; + +/// LibreOffice's layout settings for a new document (observed in what LibreOffice 24 saves): +/// written in full, because LibreOffice gives any it doesn't find in a document's set its old, +/// pre-OpenOffice value (observed), which changes how the document is laid out. +const LO_DEFAULTS: &[(&str, bool)] = &[ + ("NoNumberingShowFollowBy", false), + ("HyphenateURLs", false), + ("FootnoteInColumnToPageEnd", true), + ("GutterAtTop", false), + ("ContinuousEndnotes", false), + ("FrameAutowidthWithMorePara", false), + ("SubtractFlysAnchoredAtFlys", false), + ("SurroundTextWrapSmall", false), + ("TreatSingleColumnBreakAsPageBreak", false), + ("TabOverSpacing", false), + ("AutoFirstLineIndentDisregardLineSpace", true), + ("TabOverMargin", false), + ("ApplyTextAttrToEmptyLineAtEndOfParagraph", true), + ("DisableOffPagePositioning", false), + ("BackgroundParaOverDrawings", false), + ("TabOverflow", true), + ("AddVerticalFrameOffsets", false), + ("AddFrameOffsets", false), + ("TableRowKeep", false), + ("ApplyParagraphMarkFormatToNumbering", false), + ("JustifyLinesWithShrinking", false), + ("CollapseEmptyCellPara", true), + ("UnbreakableNumberings", false), + ("MsWordCompTrailingBlanks", false), + ("AddParaTableSpacing", true), + ("OutlineLevelYieldsNumbering", false), + ("UseOldNumbering", false), + ("AddParaSpacingToTableCells", true), + ("PropLineSpacingShrinksFirstLine", true), + ("TabAtLeftIndentForParagraphsInList", false), + ("DoNotCaptureDrawObjsOnPage", false), + ("HeaderSpacingBelowLastPara", false), + ("ClippedPictures", false), + ("MathBaselineAlignment", true), + ("AlignTabStopPosition", true), + ("UseFormerLineSpacing", false), + ("AddParaLineSpacingToTableCells", true), + ("IsKernAsianPunctuation", false), + ("DoNotJustifyLinesWithManualBreak", false), + ("UseVariableWidthNBSP", false), + ("AddParaTableSpacingAtStart", true), + ("InvertBorderSpacing", false), + ("AddExternalLeading", true), + ("UseFormerObjectPositioning", false), + ("UnxForceZeroExtLeading", false), + ("UseFormerTextWrapping", false), + ("ConsiderTextWrapOnObjPos", false), + ("StylesNoDefault", false), + ("IgnoreFirstLineIndentInNumbering", false), + ("NoGapAfterNoteNumber", false), + ("EmptyDbFieldHidesPara", true), + ("DoNotResetParaAttrsForNumFont", false), + ("IgnoreTabsAndBlanksForLineCalculation", false), + ("ClipAsCharacterAnchoredWriterFlyFrames", false), + ("DropCapPunctuation", true), + ("UseOldPrinterMetrics", false), + ("TabsRelativeToIndent", true), + ("MsWordCompMinLineHeightByFly", false), + ("SmallCapsPercentage66", false), +]; + +/// LibreOffice's layout compatibility switches (`ooo:configuration-settings`) for a document +/// laid out as Word lays it out, as LibreOffice sets them for Word files. +const WORD_COMPAT: &[(&str, bool)] = &[ + ("NoNumberingShowFollowBy", true), + ("HyphenateURLs", false), + ("FootnoteInColumnToPageEnd", true), + ("GutterAtTop", false), + ("ContinuousEndnotes", false), + ("FrameAutowidthWithMorePara", true), + ("SubtractFlysAnchoredAtFlys", false), + ("SurroundTextWrapSmall", true), + ("TreatSingleColumnBreakAsPageBreak", true), + ("TabOverSpacing", true), + ("AutoFirstLineIndentDisregardLineSpace", true), + ("TabOverMargin", false), + ("ApplyTextAttrToEmptyLineAtEndOfParagraph", true), + ("DisableOffPagePositioning", true), + ("BackgroundParaOverDrawings", true), + ("TabOverflow", true), + ("AddVerticalFrameOffsets", true), + ("AddFrameOffsets", false), + ("TableRowKeep", true), + ("ApplyParagraphMarkFormatToNumbering", true), + ("JustifyLinesWithShrinking", true), + ("CollapseEmptyCellPara", true), + ("UnbreakableNumberings", true), + ("MsWordCompTrailingBlanks", true), + ("OutlineLevelYieldsNumbering", false), + ("UseOldNumbering", false), + ("AddParaSpacingToTableCells", true), + ("PropLineSpacingShrinksFirstLine", true), + ("TabAtLeftIndentForParagraphsInList", true), + ("DoNotCaptureDrawObjsOnPage", false), + ("HeaderSpacingBelowLastPara", true), + ("ClippedPictures", true), + ("MathBaselineAlignment", true), + ("AlignTabStopPosition", true), + ("UseFormerLineSpacing", false), + ("DoNotMirrorRtlDrawObjs", true), + ("AddParaLineSpacingToTableCells", true), + ("IsKernAsianPunctuation", false), + ("DoNotJustifyLinesWithManualBreak", false), + ("UseVariableWidthNBSP", false), + ("AddParaTableSpacingAtStart", true), + ("InvertBorderSpacing", true), + ("AddExternalLeading", true), + ("UseFormerObjectPositioning", false), + ("UnxForceZeroExtLeading", false), + ("UseFormerTextWrapping", false), + ("ConsiderTextWrapOnObjPos", true), + ("StylesNoDefault", false), + ("IgnoreFirstLineIndentInNumbering", false), + ("NoGapAfterNoteNumber", true), + ("EmptyDbFieldHidesPara", false), + ("DoNotResetParaAttrsForNumFont", false), + ("IgnoreTabsAndBlanksForLineCalculation", true), + ("ClipAsCharacterAnchoredWriterFlyFrames", false), + ("DropCapPunctuation", true), + ("UseOldPrinterMetrics", false), + ("MsWordCompMinLineHeightByFly", false), +]; + +/// `settings.xml` (ODF 1.3 §3.10) for `doc`: an ODT's own, kept from opening it, or LibreOffice's +/// switches for Word's layout. Either way tab stops are measured from the margin, as WordCraft +/// writes them (LibreOffice's own default measures them from the paragraph's indent). +/// `settings.xml`: LibreOffice's layout settings for a document laid out as WordCraft lays it +/// out. The set's name, `ooo:configuration-settings`, is a qualified name: without the `ooo` +/// namespace declared LibreOffice ignores the whole set (observed). +fn settings_xml(doc: &Document) -> String { + // What WordCraft's own layout of the document needs LibreOffice to do the same: tab stops + // from the margin (as written), and the space between paragraphs as WordCraft lays it out + // (ODF leaves it open; LibreOffice's `AddParaTableSpacing` false takes the larger of the two). + let ours = [("TabsRelativeToIndent", model::tabs_from_indent(doc)), ("AddParaTableSpacing", doc.settings.sum_paragraph_spacing)]; + let item = |name: &str, on: bool| format!("{on}"); + if let Some(own) = doc.passthrough.get(SETTINGS_PART).filter(|b| b.len() <= MAX_SETTINGS).and_then(|b| std::str::from_utf8(b).ok()) { + let mut out = own.to_string(); + let mut ok = true; + for (name, on) in ours { + match with_setting(&out, name, on, &item(name, on)) { + Some(next) => out = next, + None => ok = false, + } + } + if ok { + return out; + } + } + let mut s = String::from( + "\n", + ); + // LibreOffice's defaults; Word's layout rules for a document WordCraft lays out as Word does + // (one opened from ODF is laid out by ODF's rules, `Settings::odf`, as LibreOffice's + // defaults have it); and what WordCraft's layout of this document needs. + let mut all: Vec<(&str, bool)> = LO_DEFAULTS.to_vec(); + let word = if doc.settings.odf.is_none() { WORD_COMPAT } else { &[] }; + for (name, on) in word.iter().copied().chain(ours) { + match all.iter_mut().find(|(n, _)| *n == name) { + Some(slot) => slot.1 = on, + None => all.push((name, on)), + } + } + for (name, on) in all { + s.push_str(&item(name, on)); + } + // Metrics independent of any printer, as LibreOffice's own documents. + s.push_str("high-resolution"); + s.push_str("\n"); + s +} + +/// A file's own `settings.xml` (`own`) with LibreOffice's setting `name` made `on`: its item's +/// value changed, or `item` added at the start of the configuration settings. `None` when it has +/// neither. +fn with_setting(own: &str, name: &str, on: bool, item: &str) -> Option { + let key = format!("config:name=\"{name}\""); + if let Some(at) = own.find(&key) { + let rest = own.get(at..)?; + let gt = rest.find('>')?; + // A self-closed item (another application's, or a hand-edited file's) is replaced whole: + // its close tag would be the next item's. + if rest.get(..gt).is_some_and(|t| t.ends_with('/')) { + let start = own.get(..at)?.rfind('<')?; + return Some(format!("{}{item}{}", own.get(..start).unwrap_or(""), own.get(at + gt + 1..).unwrap_or(""))); + } + let lt = rest.find("")?; + return (gt < lt).then(|| format!("{}{on}{}", own.get(..at + gt + 1).unwrap_or(""), own.get(at + lt..).unwrap_or(""))); + } + let set = "config:name=\"ooo:configuration-settings\""; + let at = own.find(set).and_then(|at| own.get(at..)?.find('>').map(|gt| at + gt + 1))?; + Some(format!("{}{item}{}", own.get(..at).unwrap_or(""), own.get(at..).unwrap_or(""))) +} + +fn meta_xml(m: &Meta, core: &craft_wordmodel::CoreProps, props: &[craft_wordmodel::CustomProp]) -> String { + let mut s = + format!("\nWordCraft"); + if !m.title.is_empty() { + s.push_str(&format!("{}", x(&m.title))); + } + if !m.subject.is_empty() { + s.push_str(&format!("{}", x(&m.subject))); + } + if !m.description.is_empty() { + s.push_str(&format!("{}", x(&m.description))); + } + // Who made it, and who changed it last (§4.3.2.6 `dc:creator`, §4.3.2.10 `meta:initial-creator`). + let last_by = if core.last_modified_by.trim().is_empty() { m.author.as_str() } else { core.last_modified_by.as_str() }; + if !m.author.is_empty() { + s.push_str(&format!("{}", x(&m.author))); + } + if !last_by.is_empty() { + s.push_str(&format!("{}", x(last_by))); + } + // When it was last changed (§4.3.2.7 `dc:date`), and how often it was saved (§4.3.2.12 + // `meta:editing-cycles`). + let modified = core.modified.trim().trim_end_matches(['Z', 'z']); + if !modified.is_empty() && modified.len() < 64 { + s.push_str(&format!("{}", x(modified))); + } + if core.revision > 0 { + s.push_str(&format!("{}", core.revision)); + } + for k in m.keywords.split([',', ';']).map(str::trim).filter(|k| !k.is_empty()) { + s.push_str(&format!("{}", x(k))); + } + if !m.created.is_empty() { + s.push_str(&format!("{}", x(m.created.trim_end_matches('Z')))); + } + // Custom document properties (§4.3.2.14); ones kept as raw XML from a Word file aren't. + // (User fields go in the content instead: `user_field_decls`.) + for p in props.iter().filter(|p| p.kind != "raw" && !p.odf_user_field) { + let (t, value) = odf_value(p); + s.push_str(&format!("{}", x(&p.name), x(&value))); + } + s.push_str("\n"); + s +} + +/// A custom property's ODF value type and value. +fn odf_value(p: &craft_wordmodel::CustomProp) -> (&'static str, String) { + let t = match p.kind.as_str() { + "r4" | "r8" | "i1" | "i2" | "i4" | "i8" | "int" | "ui1" | "ui2" | "ui4" | "ui8" | "uint" | "decimal" | "cy" => "float", + "bool" => "boolean", + "filetime" | "date" => "date", + _ => "string", + }; + let value = if t == "boolean" { if matches!(p.value.trim(), "true" | "1") { "true" } else { "false" }.to_string() } else { p.value.clone() }; + (t, value) +} + +/// The document's user fields (§7.4.7 `text:user-field-decls`), declared at the start of the text. +fn user_field_decls(props: &[craft_wordmodel::CustomProp]) -> String { + let decls: String = props + .iter() + .filter(|p| p.odf_user_field && p.kind != "raw") + .map(|p| { + let (t, value) = odf_value(p); + let attr = match t { + "float" => "office:value", + "boolean" => "office:boolean-value", + "date" => "office:date-value", + _ => "office:string-value", + }; + let value = if t == "date" { value.trim_end_matches(['Z', 'z']).to_string() } else { value }; + format!("", x(&p.name), x(&value)) + }) + .collect(); + if decls.is_empty() { String::new() } else { format!("{decls}") } +} + +/// The sections of a flow: each one's setup and the top-level blocks it holds. A paragraph that +/// carries a section's properties ends it; the document's last section follows the last such +/// paragraph (and is left out when nothing follows it, unless it is the only one). +fn flow_sections(blocks: &[FBlock], last: &SectionProps) -> Vec<(SectionProps, std::ops::Range)> { + let mut out = Vec::new(); + let mut from = 0; + for (i, b) in blocks.iter().enumerate() { + if let FBlock::Para(p) = b + && let Some(s) = &p.section + { + out.push(((**s).clone(), from..i + 1)); + from = i + 1; + } + } + if from < blocks.len() || out.is_empty() { + out.push((last.clone(), from..blocks.len())); + } + out +} + +/// Write an `.odt` package. +/// The kinds of the shapes with drawing ids in `blocks` (paragraphs, tables, groups), by id. +fn conn_kinds(blocks: &[FBlock]) -> super::HashMap { + fn walk(blocks: &[FBlock], depth: usize, out: &mut super::HashMap) { + if depth > model::MAX_DEPTH { + return; + } + for b in blocks { + match b { + FBlock::Para(p) => p.inlines.iter().for_each(|i| inline(i, 0, out)), + FBlock::Table(t) => t.rows.iter().flatten().for_each(|c| walk(&c.blocks, depth + 1, out)), + } + } + } + fn inline(i: &super::Inline, depth: usize, out: &mut super::HashMap) { + match i { + super::Inline::Shape { kind, look, .. } => { + if let Some(c) = look.conn.as_deref().filter(|c| c.id != 0) { + out.insert(c.id, *kind); + } + } + super::Inline::Group { members, .. } if depth < model::MAX_GROUP_DEPTH => members.iter().for_each(|m| inline(&m.inline, depth + 1, out)), + _ => {} + } + } + let mut out = super::HashMap::new(); + walk(blocks, 0, &mut out); + out +} + +/// `styles` with the file's own style definitions while `doc`'s are as read (see +/// `super::kept`), and the font faces they name added to those kept paragraphs need (`used`). +fn styles_as_read(doc: &Document, kept: Option<&craft_wordmodel::kept::KeptXml>, styles: String, used: &mut super::kept::Needed) -> String { + let Some(k) = kept.filter(|k| k.styles_as_read(doc)) else { return styles }; + // With the font faces they name (any of the file's, by its names). + for (key, d) in &k.defs { + if let Some(name) = key.strip_prefix("font:") { + used.2.entry(name.to_string()).or_insert_with(|| d.clone()); + } + } + super::kept::with_common_as_read(&styles, &k.defs, &k.order) +} + +/// The package files the file's own definitions refer to, and those the markup written as read +/// links to (`used`), written as read with them (added to `files` and `manifest` where no file +/// `taken` has the path). A picture only an edited paragraph showed isn't written. +fn with_kept_files( + kept: Option<&craft_wordmodel::kept::KeptXml>, + used: &std::collections::BTreeSet, + taken: &dyn Fn(&str) -> bool, + files: &mut Vec<(String, std::sync::Arc>)>, + manifest: &mut String, +) { + let Some(k) = kept else { return }; + let mut wanted = used.clone(); + for d in k.defs.values() { + super::kept::hrefs(d, &mut wanted); + } + // A file linked, or one in a folder linked (an object's). + let linked = |p: &str| wanted.contains(p) || wanted.iter().any(|w| p.strip_prefix(w.trim_end_matches('/')).is_some_and(|r| r.starts_with('/'))); + let entry = |manifest: &mut String, path: &str, mime: &str| { + let e = format!("manifest:full-path=\"{}\"", x(path)); + if !manifest.contains(&e) { + manifest.push_str(&format!("\n", x(mime))); + } + }; + for (path, bytes) in &k.files { + if linked(path) && !taken(path) && !files.iter().any(|(p, _)| p == path) { + // As the file's manifest had it (the folder too), else by its extension. + if let Some((dir, _)) = path.rsplit_once('/') + && let Some(mime) = k.defs.get(&format!("manifest:{dir}/")) + { + entry(manifest, &format!("{dir}/"), mime); + } + let ext = path.rsplit('.').next().unwrap_or(""); + let mime = k.defs.get(&format!("manifest:{path}")).map_or_else(|| model::mime_of(ext).to_string(), |m| m.to_string()); + entry(manifest, path, &mime); + files.push((path.clone(), bytes.clone())); + } + } +} + +/// `office:text`'s statement that the text holds the file's soft page breaks (§19.857 +/// `text:use-soft-page-breaks`), when the file said so and markup kept as read in `body` holds +/// them. +fn soft_page_breaks(kept: Option<&craft_wordmodel::kept::KeptXml>, body: &str) -> &'static str { + if kept.is_some_and(|k| k.defs.contains_key("soft-page-breaks")) && body.contains(" std::collections::HashSet { + fn count(blocks: &[FBlock], depth: usize, seen: &mut std::collections::HashMap) { + for b in blocks { + match b { + FBlock::Para(p) => { + if let (Some(_), Some((id, size))) = (&p.kept, p.group.as_ref().and_then(|g| g.id)) { + let e = seen.entry(id).or_insert((0, size)); + e.0 = e.0.saturating_add(1); + } + } + FBlock::Table(t) if depth < model::MAX_DEPTH => { + for c in t.rows.iter().flatten() { + count(&c.blocks, depth + 1, seen); + } + } + FBlock::Table(_) => {} + } + } + } + let mut seen = std::collections::HashMap::new(); + count(blocks, 0, &mut seen); + seen.into_iter().filter(|(_, (n, size))| n == size).map(|(id, _)| id).collect() +} + +/// The prefix of WordCraft's own automatic style names in a document with paragraphs kept as +/// read (theirs keep the names they were read with). +const KEPT_PREFIX: &str = "wc"; + +/// `content` and `styles` with the definitions kept paragraphs need (`used`: automatic, common, +/// font faces), each where it goes and only where its name isn't defined already. +fn with_kept_defs( + mut content: String, + mut styles: String, + used: &super::kept::Needed, + defs: &std::collections::BTreeMap>, +) -> (String, String) { + let (auto, common, fonts) = used; + if auto.is_empty() && common.is_empty() && fonts.is_empty() && defs.is_empty() { + return (content, styles); + } + // The namespaces the file declared (kept markup may use any), where they aren't declared. + for (xml, root) in [(&mut content, "')).map_or(at, |e| at + e); + let head = xml.get(at..head_end).unwrap_or("").to_string(); + let missing: String = defs + .iter() + .filter_map(|(k, uri)| k.strip_prefix("ns:").map(|p| (p, uri))) + .filter(|(p, _)| !head.contains(&format!("xmlns:{p}="))) + .map(|(p, uri)| format!("xmlns:{p}=\"{}\" ", x(uri))) + .collect(); + xml.insert_str(at, &missing); + } + let add = |xml: &mut String, close: &str, defs: &std::collections::BTreeMap>| { + let missing: String = defs.iter().filter(|(_, d)| !super::kept::defined(xml, d)).map(|(_, d)| &**d).collect(); + if missing.is_empty() { + return; + } + let empty = close.replacen("', "/>", 1); + if let Some(at) = xml.find(close) { + xml.insert_str(at, &missing); + } else if let Some(at) = xml.find(&empty) { + xml.replace_range(at..at + empty.len(), &format!("{}{missing}{close}", close.replacen("", auto); + add(&mut styles, "", common); + // A font the file declares is declared as the file does: a reader without the font picks its + // substitute by this, and a different one would reflow every page. + let faces: Vec<&str> = defs.iter().filter(|(k, _)| k.starts_with("font:")).map(|(_, d)| &**d).collect(); + with_file_faces(&mut content, &faces); + with_file_faces(&mut styles, &faces); + for (xml, before) in [(&mut content, ""); + } + add(xml, "", fonts); + } + (content, styles) +} + +/// `xml` (a part WordCraft wrote) with each font it declares that `faces` declare too (the file's +/// own `style:font-face` elements) as `faces` has it: under that name, or else of that family (a +/// face named `F1` of the family `Mystery`, which WordCraft names by its family), then keeping +/// WordCraft's name for it. +fn with_file_faces(xml: &mut String, faces: &[&str]) { + use super::kept::{attr_text, raw_attr}; + let mut named = std::collections::HashMap::new(); + let mut of_family = std::collections::HashMap::new(); + for face in faces { + let Some(head) = face.split('>').next() else { continue }; + if let Some(name) = raw_attr(head, "style:name") { + named.entry(attr_text(name).into_owned()).or_insert(*face); + } + if let Some(family) = raw_attr(head, "svg:font-family") { + let family = attr_text(family); + let first = family.split(',').next().unwrap_or("").trim().trim_matches(['\'', '"']); + if let Some(font) = model::kept_font(first) { + of_family.entry(font).or_insert(*face); + } + } + } + // WordCraft's own declarations are empty elements. + let (kids, _) = super::kept::children(xml, "office:font-face-decls", "font"); + let mut out = String::with_capacity(xml.len()); + let mut done = 0; + for (_, (from, to)) in kids { + let (Some(before), Some(tag)) = (xml.get(done..from), xml.get(from..to)) else { continue }; + out.push_str(before); + let name = raw_attr(tag, "style:name").map(attr_text); + let same = name.as_deref().and_then(|n| named.get(n)).map(|face| face.to_string()); + let family = || name.as_deref().and_then(|n| of_family.get(n).and_then(|face| face_named(face, n))); + let empty = tag.find('>').is_some_and(|end| end + 1 == tag.len()); + out.push_str(same.or_else(family).filter(|_| empty).as_deref().unwrap_or(tag)); + done = to; + } + out.push_str(xml.get(done..).unwrap_or("")); + *xml = out; +} + +/// `face` (a `style:font-face` element) with the name `name`. +fn face_named(face: &str, name: &str) -> Option { + let head = face.split('>').next()?; + let was = super::kept::raw_attr(head, "style:name")?; + // `was` is a piece of `head`, which starts `face`. + let at = (was.as_ptr() as usize).checked_sub(head.as_ptr() as usize)?; + Some(format!("{}{}{}", face.get(..at)?, x(name), face.get(at + was.len()..)?)) +} + +/// An OpenDocument text document's media type, and a template's (ODF 1.3 Part 2 §3.3, Appendix C). +pub const TEXT_MIME: &str = "application/vnd.oasis.opendocument.text"; +pub const TEMPLATE_MIME: &str = "application/vnd.oasis.opendocument.text-template"; + +/// `doc` as an `.odt` package. +pub fn export(doc: &Document) -> Result, String> { + export_as(doc, TEXT_MIME) +} + +/// `doc` as an OpenDocument text template (`.ott`): the same package with the template's +/// media type. +pub fn export_template(doc: &Document) -> Result, String> { + export_as(doc, TEMPLATE_MIME) +} + +fn export_as(doc: &Document, mime: &str) -> Result, String> { + let flow = model::from_doc_with( + doc, + model::FlowOpts { tracked: true, fields: true, notes: true, comments: true, lists: true, chars: true, paras: true, boxes: true }, + ); + let accents = doc.settings.theme_colors.get(4..10).map(<[_]>::to_vec).unwrap_or_default(); + // Paragraphs kept as read name their own automatic styles: WordCraft's take another prefix. + let kept = doc.kept.of("odt"); + let mut w = if kept.is_some() { Writer::with_prefix(KEPT_PREFIX) } else { Writer::default() }; + w.note_numbers = Some([false, true].map(|endnote| { + let o = doc.note_options(&doc.last_section, endnote); + (o.num_fmt, o.num_start) + })); + w.kept_defs = std::sync::Arc::new(kept.map(|k| k.defs.clone()).unwrap_or_default()); + w.reserved = std::sync::Arc::new(kept.map(|k| k.files.keys().cloned().collect()).unwrap_or_default()); + w.groups_whole = whole_groups(&flow.blocks); + // The file's master pages are written as read: what was kept may start them. + w.kept_pages = kept.is_some_and(|k| k.pages_as_read(doc)); + w.accents.clone_from(&accents); + w.conn_kinds = conn_kinds(&flow.blocks); + w.sources = doc.sources.iter().map(|s| (s.tag.clone(), s.clone())).collect(); + w.custom_props = doc.custom_props.iter().filter(|p| p.kind != "raw").map(|p| p.name.clone()).collect(); + w.user_fields = doc.custom_props.iter().filter(|p| p.odf_user_field && p.kind != "raw").map(|p| p.name.clone()).collect(); + w.variables = variables(doc); + w.forms_allowed = true; + w.ranged = model::ranged_anchors(&flow.blocks); + let mut body = String::new(); + let sections = flow_sections(&flow.blocks, &doc.last_section); + // One master page per section that starts a page; the first one is "Standard". + let mut pages: Vec<(String, SectionProps)> = Vec::new(); + // The columns of the page in force: a continuous section with others is a `text:section`. + let mut page_cols = Columns::default(); + for (k, (sec, range)) in sections.iter().enumerate() { + let new_page = k == 0 || sec.start != SectionStart::Continuous; + if new_page { + // The file's own master page while its pages are as read. + let own = kept + .filter(|kx| kx.pages_as_read(doc) && kx.section_masters.len() == sections.len()) + .and_then(|kx| kx.section_masters.get(k).cloned().flatten()); + let name = match &own { + Some(m) => m.clone(), + None if k == 0 => "Standard".to_string(), + None => format!("{}Page{}", w.prefix, pages.len() + 1), + }; + pages.push((name.clone(), sec.clone())); + page_cols = sec.columns.clone(); + // The first section needs its paragraph to name the page only to restart numbering + // (or when it is the file's own other than Standard). + if k > 0 || sec.page_num_start.is_some() || own.as_deref().is_some_and(|m| m != "Standard") { + w.pending_start = Some((name, sec.page_num_start)); + } + } + // A continuous section whose columns differ from the page's is a `text:section` (§5.4) + // with them (one column included). A restart of page numbering isn't written for it: ODF + // applies `style:page-number` only together with a master page. + let wrapped = k > 0 + && sec.start == SectionStart::Continuous + && (sec.columns.count != page_cols.count || (sec.columns.count > 1 && sec.columns != page_cols)); + // A list cut at the boundary goes on numbering in the new section. + let list_at = |b: Option<&FBlock>| matches!(b, Some(FBlock::Para(p)) if p.list.is_some()); + w.continue_list = k > 0 && list_at(flow.blocks.get(range.start)) && list_at(range.start.checked_sub(1).and_then(|i| flow.blocks.get(i))); + if wrapped { + w.close_ins(&mut body); + let n = format!("{}{}", w.prefix, w.section_styles.len() + 1); + // Whether its columns end balanced, said so every reader does the same: as WordCraft + // lays it out (balanced when text follows on the page, or when the section says). + let next_continuous = sections.get(k + 1).is_some_and(|(s, _)| s.start == SectionStart::Continuous); + let balanced = match sec.balance_columns { + Some(b) => b && (next_continuous || k + 1 == sections.len()), + None => next_continuous, + }; + w.section_styles.push(format!( + "{}", + !balanced, + columns_xml(&sec.columns) + )); + body.push_str(&format!("")); + } + w.blocks(flow.blocks.get(range.clone()).unwrap_or(&[]), &mut body, 0); + // A kept section doesn't cross a page or column section's edge. + w.close_sections(&mut body); + w.pending_start = None; + w.continue_list = false; + if wrapped { + w.close_ins(&mut body); + body.push_str("\n"); + } + } + w.close_ins(&mut body); + let tracked = if w.regions.is_empty() && !doc.settings.track_changes { + String::new() + } else { + format!("{}\n", doc.settings.track_changes, w.regions.concat()) + }; + // The sequences fields count (§7.4.11), declared after any tracked changes. + // The variables, declared (§7.4.2). + let var_decls = if w.variables.is_empty() { + String::new() + } else { + let d: String = w.variables.iter().map(|n| format!("", x(n))).collect(); + format!("{d}") + }; + let decls = if w.sequences.is_empty() { + String::new() + } else { + let d: String = + w.sequences.iter().map(|n| format!("", x(n))).collect(); + format!("{d}") + }; + let user_decls = user_field_decls(&doc.custom_props); + // The file's own declarations, with WordCraft's added to them; its scripts. + let kept_decl = |el: &str, ours: String| super::kept::merge_decls(kept.and_then(|k| k.defs.get(&format!("decls:{el}"))).map(|d| &**d), ours); + let var_decls = kept_decl("text:variable-decls", var_decls); + let user_decls = kept_decl("text:user-field-decls", user_decls); + let decls = kept_decl("text:sequence-decls", decls); + let other_decls: String = kept + .map(|k| { + k.defs + .iter() + .filter(|(key, _)| { + key.starts_with("decls:") && !["variable", "user-field", "sequence"].iter().any(|d| key.contains(&format!("text:{d}-decls"))) + }) + .map(|(_, d)| &**d) + .collect() + }) + .unwrap_or_default(); + let scripts = kept.and_then(|k| k.defs.get("top:office:scripts")).map(|d| d.to_string()).unwrap_or_default(); + let soft = soft_page_breaks(kept, &body); + let content = format!( + "\n{scripts}{}\n{}{tracked}{var_decls}{decls}{user_decls}{other_decls}{body}\n", + w.automatic_styles(), + // The file's forms as read, with the document's controls. + super::kept::merge_forms(kept.and_then(|k| k.defs.get("forms")).map(|d| &**d), super::forms::forms_xml(&w.forms, &hidden_controls(doc))) + ); + let mut hw = Writer::with_prefix(if kept.is_some() { "wcM" } else { "M" }); + // Headers' and footers' new pictures and objects skip the kept files' names too. + hw.reserved = std::sync::Arc::clone(&w.reserved); + hw.accents = accents; + hw.variables.clone_from(&w.variables); + let parts: Vec = pages.iter().map(|(_, sec)| page_parts(doc, sec, &mut hw)).collect(); + // A custom style kept paragraphs name is written from the model (as edited), not as read. + for st in &doc.styles.styles { + let name = super::writer::odf_name(&st.id); + if w.kept_used.1.contains_key(&name) || w.kept_used.1.contains_key(&format!("{name}|text")) { + if model::is_custom_style(doc, &st.id) { + w.named.insert(st.id.clone()); + } else if model::is_custom_char_style(doc, &st.id) { + w.named_text.insert(st.id.clone()); + } + } + } + let mut named = std::mem::take(&mut w.named); + named.append(&mut hw.named); + let mut named_text = std::mem::take(&mut w.named_text); + named_text.append(&mut hw.named_text); + let named_text_xml = char_styles(doc, &named_text); + // The dashes, gradients and arrowheads graphic styles name (§16.39–16.41). + let mut draw_defs = std::mem::take(&mut w.draw_defs.defs); + draw_defs.append(&mut hw.draw_defs.defs); + let mut draw_xml: String = draw_defs.into_values().collect(); + // The plain character style index entries' links are in (LibreOffice's Index Link). + if w.indexes + hw.indexes > 0 { + draw_xml.push_str(""); + } + let styles = page_background(doc, &mut w, styles_xml(doc, &pages, &parts, &hw.style_defs(), &named)); + let styles = styles.replacen( + "", + // The Frame style text boxes derive from (it states nothing: their own styles do). + &format!("{draw_xml}{named_text_xml}"), + 1, + ); + w.pictures.append(&mut hw.pictures); + w.objects.append(&mut hw.objects); + w.object_dirs.append(&mut hw.object_dirs); + let meta = meta_xml(&flow.meta, &doc.core, &doc.custom_props); + // With the file's properties WordCraft doesn't model. + let meta = match kept { + Some(k) => super::kept::with_kept_meta(meta, &k.defs, &k.order), + None => meta, + }; + let mut manifest = format!( + "\n\n\n\n\n\n\n", + ); + for (dir, mime) in &w.object_dirs { + manifest.push_str(&format!( + "\n", + x(dir), + x(mime) + )); + } + for (name, _, mime) in &w.pictures { + manifest.push_str(&format!("\n", x(name), x(mime))); + } + for (dir, _, mime) in &w.objects { + manifest.push_str(&format!( + "\n\n", + x(dir) + )); + } + // The parts WordCraft doesn't read, as they were, unless one would take a path written here. + let (kept_files, kept_entries) = super::parts::kept(doc); + let dirs: Vec = + w.objects.iter().map(|(d, _, _)| d).chain(w.object_dirs.iter().map(|(d, _)| d)).map(|d| format!("{}/", d.trim_end_matches('/'))).collect(); + let taken = |p: &str| w.pictures.iter().any(|(n, _, _)| n == p) || dirs.iter().any(|d| p.starts_with(d.as_str())); + let mut kept_files: Vec<_> = kept_files.into_iter().filter(|(p, _)| !taken(p)).collect(); + with_kept_files(kept, &w.kept_files_used, &taken, &mut kept_files, &mut manifest); + for (path, mime) in &kept_entries { + if (path.ends_with('/') && kept_files.iter().any(|(p, _)| p.starts_with(path.as_str()))) || kept_files.iter().any(|(p, _)| p == path) { + manifest.push_str(&format!("\n", x(path), x(mime))); + } + } + manifest.push_str("\n"); + + let mut z = crate::PackageWriter::new(mime); + z.manifest_xml(manifest); + // Each font declared, for readers without it (§16.21). + let (content, styles) = (with_font_faces(&content), with_font_faces(&styles)); + // What the paragraphs written as read need, as read (see `super::kept`). + // Style definitions as read while nobody changed them: the file's own `office:styles`. + let styles = styles_as_read(doc, kept, styles, &mut w.kept_used); + // Page setup as read while nobody changed it: the file's own page layouts and master pages. + let styles = match kept.filter(|k| k.pages_as_read(doc)) { + Some(k) => super::kept::with_pages_as_read(&styles, &k.defs, &k.order), + None => styles, + }; + let (content, styles) = with_kept_defs(content, styles, &w.kept_used, &w.kept_defs); + // The manifest lists every file already: the writer adds the files only. + for (name, data) in [ + ("content.xml", content.as_bytes()), + ("styles.xml", styles.as_bytes()), + ("meta.xml", meta.as_bytes()), + ("settings.xml", settings_xml(doc).as_bytes()), + ] { + z.file(name, "", data); + } + for (dir, xml, _) in &w.objects { + z.file(&format!("{dir}/content.xml"), "", xml.as_bytes()); + } + for (name, data, _) in &w.pictures { + z.stored(name, "", data); + } + for (name, data) in &kept_files { + z.file(name, "", data); + } + z.finish().map_err(|e| format!("ODT: {e}")) +} + +/// A page layout's page number format (§19.500 `style:num-format`), the document's page colour +/// (§20.175 `fo:background-color`) and the section's page borders with the space inside them. +fn page_look(doc: &Document, sec: &SectionProps) -> String { + let mut a = String::new(); + let f = super::writer::odf_num_format(sec.page_num_format); + if f != "1" { + a.push_str(&format!(" style:num-format=\"{f}\"")); + } + if let Some(c) = doc.settings.page_color { + a.push_str(&format!(" fo:background-color=\"#{}\"", c.hex().to_ascii_lowercase())); + } + a +} + +/// Page borders `b` as ODF writes them, and the margins (top, bottom, left, right) of the page +/// they go round: each border at its distance from the edge (WordCraft's), which becomes the +/// margin, with the rest of the way to the text as its padding. +fn page_borders(b: Option, margins: [f32; 4]) -> (String, [f32; 4]) { + let Some(b) = b.filter(|b| b.any_visible()) else { return (String::new(), margins) }; + let mut out = margins; + let mut attrs = String::new(); + for (i, (side, edge)) in [("top", b.top), ("bottom", b.bottom), ("left", b.left), ("right", b.right)].into_iter().enumerate() { + let (Some(e), Some(m)) = (edge.filter(|e| e.is_visible()), out.get_mut(i)) else { continue }; + // A hostile margin (negative or not a number) leaves no room, so clamp cannot panic. + let room = if m.is_finite() { m.max(0.0) } else { 0.0 }; + let at = if e.space.is_finite() { e.space.clamp(0.0, room) } else { 0.0 }; + let pad = (*m - at - e.width).max(0.0); + *m = at; + attrs.push_str(&format!(" fo:border-{side}=\"{}\" fo:padding-{side}=\"{}\"", super::writer::odf_border(Some(e)), pt(pad))); + } + (attrs, out) +} + +/// The footnotes' and endnotes' numbering (§16.29 `text:notes-configuration`): format, first +/// number (ODF counts from 0), restart and, for footnotes, page foot or document end. +fn notes_configuration(doc: &Document) -> String { + use craft_wordmodel::section::{NotePos, NoteRestart}; + let mut s = String::new(); + for (class, fmt, pr) in + [("footnote", doc.settings.footnote_format, &doc.settings.footnote_pr), ("endnote", doc.settings.endnote_format, &doc.settings.endnote_pr)] + { + // Only what the document sets: the rest stays the reader's default. + let start = pr.num_start.map(|n| format!(" text:start-value=\"{}\"", n.max(1) - 1)).unwrap_or_default(); + let restart = pr + .num_restart + .map(|r| { + let at = match r { + NoteRestart::EachPage => "page", + NoteRestart::EachSect => "chapter", + NoteRestart::Continuous => "document", + }; + format!(" text:start-numbering-at=\"{at}\"") + }) + .unwrap_or_default(); + let position = match (class, pr.pos) { + ("footnote", Some(NotePos::DocEnd)) => " text:footnotes-position=\"document\"", + ("footnote", Some(_)) => " text:footnotes-position=\"page\"", + _ => "", + }; + s.push_str(&format!( + "", + super::writer::odf_num_format(fmt) + )); + } + s +} + +/// Line numbering (§16.30 `text:linenumbering-configuration`): every how many lines, how far +/// from the text, restarting on each page or not. +fn line_numbering_xml(l: &craft_wordmodel::section::LineNumbering) -> String { + let restart = l.restart == craft_wordmodel::section::LineNumberRestart::Page; + format!( + "", + l.count_by.clamp(1, 100), + pt(if l.distance.is_finite() { l.distance.clamp(0.0, 1584.0) } else { 0.0 }) + ) +} + +/// The hidden form controls kept in `doc` (see [`super::forms::HIDDEN_PART`]). +fn hidden_controls(doc: &Document) -> Vec { + doc.passthrough + .get(super::forms::HIDDEN_PART) + .and_then(|b| std::str::from_utf8(b).ok()) + .map(|s| s.split('\0').filter(|h| h.starts_with(" std::collections::BTreeSet { + let mut out = std::collections::BTreeSet::new(); + let stories = std::iter::once(craft_wordmodel::StoryRef::Body).chain(doc.parts.keys().map(|k| craft_wordmodel::StoryRef::Part(*k))); + for story in stories { + for path in doc.para_paths(story) { + let Some(p) = doc.para(story, &path) else { continue }; + for o in &p.objects { + if let craft_wordmodel::InlineObject::Field { instr, .. } = o + && let Some((name, _)) = super::fields::set_of(instr) + && out.len() < 10_000 + && !name.contains(['"', '<', '&']) + { + out.insert(name.to_string()); + } + } + } + } + out +} + +/// `styles` with the page background picture (Design › Page Color › Fill Effects › Picture) in +/// each page layout (§17.3 `style:background-image`, first in `style:page-layout-properties`), +/// the picture written into the package by `w`: tiled, stretched, or centred at its own size +/// (§20.249 `style:repeat`). +fn page_background(doc: &Document, w: &mut Writer, styles: String) -> String { + let Some(pic) = doc.settings.page_picture.as_ref() else { return styles }; + let Some(data) = doc.media.get(&pic.media) else { return styles }; + let Some(ext) = model::sniff_image(data) else { return styles }; + let name = format!("Pictures/page-background.{ext}"); + if !w.pictures.iter().any(|(n, _, _)| *n == name) { + w.pictures.push((name.clone(), data.clone(), model::mime_of(ext).to_string())); + } + let repeat = match pic.fill_type.to_ascii_lowercase().as_str() { + "tile" => "repeat", + "center" => "no-repeat", + _ => "stretch", + }; + let el = format!( + "", + x(&name) + ); + // After each layout's properties' opening tag. + let mut out = String::with_capacity(styles.len() + el.len() * 4); + let mut rest = styles.as_str(); + while let Some(at) = rest.find("')).map(|e| at + e) else { break }; + let self_closing = rest.get(end.saturating_sub(1)..end) == Some("/"); + if self_closing { + out.push_str(rest.get(..end.saturating_sub(1)).unwrap_or_default()); + out.push('>'); + out.push_str(&el); + out.push_str(""); + } else { + out.push_str(rest.get(..=end).unwrap_or_default()); + out.push_str(&el); + } + rest = rest.get(end + 1..).unwrap_or_default(); + } + out.push_str(rest); + out +} + +/// `xml` (content.xml or styles.xml) with each font family it uses declared once (§16.21 +/// `style:font-face`: its family, generic family and pitch) and named where it is used +/// (`style:font-name` beside `fo:font-family`, and the Asian and complex ones): a reader without +/// the font picks a like one. +fn with_font_faces(xml: &str) -> String { + const ATTRS: [(&str, &str); 3] = [ + ("fo:font-family=\"", "style:font-name"), + ("style:font-family-asian=\"", "style:font-name-asian"), + ("style:font-family-complex=\"", "style:font-name-complex"), + ]; + let mut families: Vec = Vec::new(); + let mut out = String::with_capacity(xml.len() + 1024); + let mut rest = xml; + loop { + // The next font family attribute. + let next = ATTRS.iter().filter_map(|(a, n)| rest.find(a).map(|at| (at, *a, *n))).min_by_key(|(at, ..)| *at); + let Some((at, attr, name_attr)) = next else { break }; + let value_at = at + attr.len(); + let Some(end) = rest.get(value_at..).and_then(|r| r.find('"')).map(|e| value_at + e) else { break }; + let value = rest.get(value_at..end).unwrap_or(""); + let family = value.replace("'", "'").trim().trim_matches(['\'', '"']).trim().to_string(); + out.push_str(rest.get(..at).unwrap_or("")); + if !family.is_empty() && family.len() <= 200 && !family.contains(',') { + if !families.contains(&family) && families.len() < 1000 { + families.push(family.clone()); + } + out.push_str(&format!("{name_attr}=\"{}\" ", super::writer::x(&family))); + } + out.push_str(rest.get(at..=end).unwrap_or("")); + rest = rest.get(end + 1..).unwrap_or(""); + } + out.push_str(rest); + if families.is_empty() { + return out; + } + let decls: String = families + .iter() + .map(|f| { + let (generic, pitch) = font_kind(f); + format!( + "", + super::writer::x(f) + ) + }) + .collect(); + let decls = format!("{decls}"); + // Before the styles: first in office:document-styles, after the scripts in the content. + for before in ["", "", "", "", ""] { + if let Some(at) = out.find(before) { + out.insert_str(at, &decls); + return out; + } + } + out +} + +/// A font family's generic family and pitch (§20.270, §20.271), from its name: monospaced +/// families are `modern` and fixed, serif ones `roman`, the rest `swiss`. +fn font_kind(family: &str) -> (&'static str, &'static str) { + let f = family.to_ascii_lowercase(); + if model::is_mono(family) { + return ("modern", "fixed"); + } + const SERIF: [&str; 16] = [ + "serif", + "times", + "georgia", + "cambria", + "caladea", + "garamond", + "palatino", + "book", + "roman", + "libertine", + "minion", + "baskerville", + "bodoni", + "didot", + "charter", + "constantia", + ]; + if SERIF.iter().any(|s| f.contains(s)) && !f.contains("sans") { + return ("roman", "variable"); + } + ("swiss", "variable") +} + +#[cfg(test)] +mod setting_tests { + /// A page border round a page whose margin is negative or not a number is written at no + /// distance, instead of panicking. + #[test] + fn a_page_border_round_a_hostile_margin_is_written_at_no_distance() { + let mut edge = craft_wordmodel::props::Border::single(1.0); + edge.space = 12.0; + let b = craft_wordmodel::props::Borders::all(edge); + let (attrs, out) = super::page_borders(Some(b), [-10.0, f32::NAN, 36.0, 5.0]); + assert!(attrs.contains("fo:border-top="), "{attrs}"); + assert_eq!(out[0], 0.0); + assert_eq!(out[1], 0.0); + assert_eq!(out[2], 12.0); + assert_eq!(out[3], 5.0); + } + + /// A self-closed setting is replaced whole, leaving the item after it as it was. + #[test] + fn a_self_closed_setting_is_replaced_whole() { + let item = "true"; + let own = "false"; + let out = super::with_setting(own, "A", true, item).unwrap_or_default(); + assert_eq!(out, format!("{item}false")); + // An item with a value has just its value changed. + let own = "false"; + assert_eq!(super::with_setting(own, "A", true, item).as_deref(), Some(format!("{item}").as_str())); + } +} diff --git a/crates/odf/src/text/page_parts.rs b/crates/odf/src/text/page_parts.rs new file mode 100644 index 000000000..6a67da3af --- /dev/null +++ b/crates/odf/src/text/page_parts.rs @@ -0,0 +1,198 @@ +//! Headers and footers of the master pages in `styles.xml` (ODF 1.3 §16.10), read with the body +//! reader (#863). +use std::collections::HashSet; + +use craft_wordmodel::{Document, PartKind}; + +use super::{Body, Ev, FBlock, Pages, SectionProps, Styles, events, get, model}; + +/// Most XML events kept for all headers and footers together; a slot past it is left out. +const MAX_EVENTS: usize = 1 << 20; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Slot { + Header, + HeaderLeft, + HeaderFirst, + Footer, + FooterLeft, + FooterFirst, +} + +impl Slot { + fn of(q: &str) -> Option { + Some(match q { + "style:header" => Self::Header, + "style:header-left" => Self::HeaderLeft, + "style:header-first" | "loext:header-first" => Self::HeaderFirst, + "style:footer" => Self::Footer, + "style:footer-left" => Self::FooterLeft, + "style:footer-first" | "loext:footer-first" => Self::FooterFirst, + _ => return None, + }) + } + + fn footer(self) -> bool { + matches!(self, Self::Footer | Self::FooterLeft | Self::FooterFirst) + } + + fn set(self, sec: &mut SectionProps, id: u32) { + let set = if self.footer() { &mut sec.footers } else { &mut sec.headers }; + match self { + Self::Header | Self::Footer => set.default = Some(id), + Self::HeaderLeft | Self::FooterLeft => set.even = Some(id), + Self::HeaderFirst | Self::FooterFirst => { + set.first = Some(id); + sec.title_page = true; + } + } + } +} + +/// The headers and footers read, in part-id order from 1, and whether one is for left pages. +#[derive(Default)] +pub(super) struct Parts { + parts: Vec<(PartKind, Vec)>, + even_odd: bool, + /// The watermark a header holds (the first found). + pub(super) watermark: Option, +} + +impl Parts { + /// The blocks of every header and footer read. + pub(super) fn blocks_mut(&mut self) -> impl Iterator> { + self.parts.iter_mut().map(|(_, b)| b) + } +} + +/// Read each visible header and footer of the master pages in `pages`, and name it on its master +/// page's section as part id 1, 2, … in order, so the sections the body starts carry them. +/// [`add`] puts those parts into the document. +/// `objects`: the package's entry names and manifest media types, for embedded objects. +/// `spaces_left` is the document's [`MAX_EXTRA_SPACES`] budget, which the body read after them +/// goes on with. +pub(super) fn read( + xml: &[u8], + styles: &Styles, + pages: &mut Pages, + fetch: &mut dyn FnMut(&str) -> Option>, + objects: (&[String], &std::collections::HashMap), + spaces_left: &mut usize, +) -> Parts { + let mut found: Vec<(String, Slot, Vec)> = Vec::new(); + let mut master: Option = None; + // The slot being read, its nesting depth and whether it is kept. + let mut slot: Option<(usize, bool)> = None; + let mut seen = HashSet::new(); + let mut room = MAX_EVENTS; + events(xml, |ev| { + if let Some((depth, keep)) = &mut slot { + match &ev { + Ev::End(_) if *depth == 0 => { + slot = None; + return; + } + Ev::Start(..) => *depth += 1, + Ev::End(_) => *depth -= 1, + Ev::Text(_) => {} + } + if *keep && room == 0 { + *keep = false; + found.pop(); + } + if *keep && let Some((.., evs)) = found.last_mut() { + room -= 1; + evs.push(ev); + } + return; + } + match ev { + Ev::Start(q, a) if q == "style:master-page" => { + master = get(&a, "style:name").filter(|n| pages.masters.contains_key(*n)).map(str::to_string); + } + Ev::End(q) if q == "style:master-page" => master = None, + Ev::Start(q, a) => { + if let (Some(m), Some(kind)) = (&master, Slot::of(&q)) { + let keep = get(&a, "style:display") != Some("false") && seen.insert((m.clone(), q.clone())); + if keep { + found.push((m.clone(), kind, Vec::new())); + } + slot = Some((0, keep)); + } + } + _ => {} + } + }); + let none = Pages::default(); + let mut out = Parts::default(); + for (name, kind, evs) in found { + let mut reader = Body::new(styles, &none, fetch); + reader.in_text = true; + (reader.entries, reader.manifest) = (objects.0, Some(objects.1)); + reader.spaces_left = *spaces_left; + for ev in evs { + reader.feed(ev); + } + *spaces_left = reader.spaces_left; + out.watermark = out.watermark.take().or(reader.watermark.take()); + out.parts.push((if kind.footer() { PartKind::Footer } else { PartKind::Header }, reader.take_blocks())); + if let Some(sec) = pages.masters.get_mut(&name) { + kind.set(sec, out.parts.len() as u32); + } + out.even_odd |= matches!(kind, Slot::HeaderLeft | Slot::FooterLeft); + } + // Each master page stands alone, but a section without its own header or footer continues + // the previous one's: give those an empty one, and left pages the right pages' one. + for footer in [false, true] { + let pick = |s: &SectionProps| if footer { s.footers } else { s.headers }; + if !pages.masters.values().any(|s| pick(s).default.is_some() || pick(s).even.is_some()) { + continue; + } + let mut empty = None; + for sec in pages.masters.values_mut() { + let set = if footer { &mut sec.footers } else { &mut sec.headers }; + if set.default.is_none() { + set.default = Some(*empty.get_or_insert_with(|| { + out.parts.push((if footer { PartKind::Footer } else { PartKind::Header }, Vec::new())); + out.parts.len() as u32 + })); + } + if out.even_odd && set.even.is_none() { + set.even = set.default; + } + } + } + if let Some(first) = pages.first_name.as_ref().and_then(|n| pages.masters.get(n)) { + pages.first = Some(first.clone()); + } + out +} + +/// Put the parts [`read`] found into `doc`. The sections name them 1, 2…, in the order read; +/// `doc`'s notes, comments and text boxes already hold ids, so each part takes a free one and +/// the sections are pointed at it. +pub(super) fn add(doc: &mut Document, parts: Parts) { + if parts.watermark.is_some() { + doc.settings.watermark.clone_from(&parts.watermark); + } + let mut ids: Vec> = Vec::with_capacity(parts.parts.len()); + for (kind, blocks) in parts.parts { + let blocks = model::part_blocks(doc, &blocks); + ids.push(doc.try_add_part(kind, blocks).ok()); + } + let new_id = |old: Option| old.and_then(|o| o.checked_sub(1)).and_then(|i| ids.get(i as usize).copied().flatten()); + let remap = |sec: &mut SectionProps| { + for set in [&mut sec.headers, &mut sec.footers] { + (set.default, set.first, set.even) = (new_id(set.default), new_id(set.first), new_id(set.even)); + } + }; + remap(&mut doc.last_section); + for b in &mut doc.body { + if let craft_wordmodel::Block::Para(p) = std::sync::Arc::make_mut(b) + && let Some(sec) = p.section.as_deref_mut() + { + remap(sec); + } + } + doc.settings.even_odd_headers |= parts.even_odd; +} diff --git a/crates/odf/src/text/parts.rs b/crates/odf/src/text/parts.rs new file mode 100644 index 000000000..a3f1ff655 --- /dev/null +++ b/crates/odf/src/text/parts.rs @@ -0,0 +1,140 @@ +//! Package parts WordCraft doesn't read (ODF 1.3 Part 2 §3: any file a package lists), kept as +//! they are and written back: macros and dialogs (`Basic/`, `Dialogs/`, `Scripts/`), +//! LibreOffice's interface configuration (`Configurations2/`), metadata (`manifest.rdf`) and +//! files of other applications. +//! +//! Not kept: what WordCraft reads and writes itself (content, styles, meta, settings, pictures, +//! embedded objects), the thumbnail (it would show the document as it was), and digital +//! signatures (`META-INF/*signatures*.xml`): they sign the package as it was, so a saved, changed +//! document can't carry them. + +use std::sync::Arc; + +use craft_wordmodel::Document; + +/// `Document::passthrough` key prefix of a kept part (its path follows), and the key of the list +/// of kept paths with their media types (`path\tmedia type` lines; directories end in `/`). +pub(super) const PREFIX: &str = "odf-part/"; +pub(super) const LIST: &str = "odf-parts"; + +/// Most parts kept, and most bytes in all. +const MAX_PARTS: usize = 1000; +const MAX_BYTES: u64 = 64 << 20; + +/// Whether package path `name` is one WordCraft reads or writes itself, or drops. +fn own(name: &str, manifest: &std::collections::HashMap) -> bool { + const OWN: [&str; 6] = ["mimetype", "content.xml", "styles.xml", "meta.xml", "settings.xml", "META-INF/manifest.xml"]; + if OWN.contains(&name) || name.ends_with('/') { + return true; + } + if ["Pictures/", "Thumbnails/", "ObjectReplacements/"].iter().any(|d| name.starts_with(d)) { + return true; + } + if name.starts_with("META-INF/") && name.contains("signatures") { + return true; + } + // An embedded object's files: its directory is listed as an OpenDocument. + let mut dir = name; + while let Some((up, _)) = dir.rsplit_once('/') { + if manifest + .get(&format!("{up}/")) + .is_some_and(|m| m.starts_with("application/vnd.oasis.opendocument.") || m.starts_with("application/vnd.sun.xml.math")) + { + return true; + } + dir = up; + } + false +} + +/// Whether `name` is a path safe to write back: relative, without `.` or `..` steps or empty +/// ones, and of no more than 512 bytes. +fn safe(name: &str) -> bool { + !name.is_empty() + && name.len() <= 512 + && !name.starts_with('/') + && !name.contains('\\') + && name.split('/').all(|s| !s.is_empty() && s != "." && s != "..") + && !name.chars().any(char::is_control) +} + +/// The parts of a package WordCraft doesn't read itself: each file's path and bytes, and the +/// list of their paths with media types (see [`LIST`]). +#[derive(Default)] +pub(super) struct Kept { + files: Vec<(String, Vec)>, + list: String, +} + +impl Kept { + /// Keeps them in `doc`, for the next save. + pub(super) fn store(self, doc: &mut Document) { + for (name, bytes) in self.files { + doc.passthrough.insert(format!("{PREFIX}{name}"), Arc::new(bytes)); + } + if !self.list.is_empty() { + doc.passthrough.insert(LIST.into(), Arc::new(self.list.into_bytes())); + } + } +} + +/// Reads the parts of package `z` (entries `entries`) WordCraft doesn't read itself, with their +/// media types from `manifest`. +pub(super) fn read(z: &mut crate::Package<'_>, entries: &[String], manifest: &std::collections::HashMap) -> Kept { + let mut kept = Kept::default(); + let mut budget = MAX_BYTES; + let mut dirs = std::collections::BTreeSet::new(); + let clean = |m: &str| m.replace(['\t', '\n', '\r'], " "); + for name in entries { + if kept.files.len() >= MAX_PARTS || own(name, manifest) || !safe(name) { + continue; + } + if z.declared_size(name).is_none_or(|n| n > budget) { + continue; + } + let Some(bytes) = z.read(name).ok().flatten().filter(|b| b.len() as u64 <= budget) else { continue }; + budget = budget.saturating_sub(bytes.len() as u64); + let mime = manifest.get(name.as_str()).map_or("", String::as_str); + kept.list.push_str(&format!("{name}\t{}\n", clean(mime))); + // The directories it is in that the manifest lists (a Basic library, a configuration). + let mut dir = name.as_str(); + while let Some((up, _)) = dir.rsplit_once('/') { + let d = format!("{up}/"); + if let Some(m) = manifest.get(&d) { + dirs.insert((d, clean(m))); + } + dir = up; + } + kept.files.push((name.clone(), bytes)); + } + for (d, m) in dirs { + kept.list.push_str(&format!("{d}\t{m}\n")); + } + kept +} + +/// The parts kept in `doc` to write back: (path, bytes) of each file, and (path, media type) of +/// each manifest entry (directories too). Paths WordCraft writes itself are left out. +pub(super) fn kept(doc: &Document) -> (Vec<(String, Arc>)>, Vec<(String, String)>) { + let mut files = Vec::new(); + let mut entries = Vec::new(); + let Some(list) = doc.passthrough.get(LIST).and_then(|l| std::str::from_utf8(l).ok()) else { return (files, entries) }; + let empty = std::collections::HashMap::new(); + for line in list.lines().take(MAX_PARTS * 2) { + let (path, mime) = line.split_once('\t').unwrap_or((line, "")); + let dir = path.ends_with('/'); + let name = path.trim_end_matches('/'); + if !safe(name) || own(path, &empty) && !dir { + continue; + } + if dir { + entries.push((path.to_owned(), mime.to_owned())); + } else if let Some(b) = doc.passthrough.get(&format!("{PREFIX}{path}")) { + // Every file has its entry (ODF 1.3 Part 2 §4.3; LibreOffice refuses a package with a + // file its manifest doesn't list), its media type empty when the file's was. + files.push((path.to_owned(), b.clone())); + entries.push((path.to_owned(), mime.to_owned())); + } + } + (files, entries) +} diff --git a/crates/odf/src/text/path.rs b/crates/odf/src/text/path.rs new file mode 100644 index 000000000..567002272 --- /dev/null +++ b/crates/odf/src/text/path.rs @@ -0,0 +1,416 @@ +//! Geometry of polygons, polylines, paths and regular polygons (ODF 1.3 §10.3.4–10.3.7, §19.1xx +//! `draw:points`, `svg:d`, `svg:viewBox`): their points, in their view box's coordinates, into +//! a WordCraft freeform (straight segments: curves and arcs are followed by short steps). + +use craft_wordmodel::freeform::{FreePath, Freeform, MAX_PATHS, MAX_POINTS}; + +/// Steps a curve or arc is followed by. +const CURVE_STEPS: usize = 16; +/// Most numbers read from one `svg:d` or `draw:points` (hostile files). +const MAX_NUMBERS: usize = 200_000; +/// Largest coordinate kept. +const MAX_COORD: f64 = 1.0e6; + +/// A view box (`svg:viewBox`, §19.570: min x, min y, width, height). +pub(super) fn view_box(v: Option<&str>) -> Option<[f32; 4]> { + let n: Vec = numbers(v?).take(4).collect(); + let [x, y, w, h] = n.try_into().ok()?; + Some([x, y, w, h]) +} + +/// The numbers in `s`, separated by spaces or commas (an SVG number list), at most +/// [`MAX_NUMBERS`]; what doesn't read as a number ends it. +fn numbers(s: &str) -> impl Iterator + '_ { + s.split(|c: char| c.is_whitespace() || c == ',') + .filter(|t| !t.is_empty()) + .take(MAX_NUMBERS) + .map_while(|t| t.parse::().ok()) + .map(|v| if v.is_finite() { v.clamp(-MAX_COORD as f32, MAX_COORD as f32) } else { 0.0 }) +} + +/// A freeform from `paths` (view box coordinates) in view box `vb` (none, or an empty one: the +/// box around the points): points from its corner. +fn freeform(paths: Vec, vb: Option<[f32; 4]>) -> Option { + let [x0, y0, w, h] = match vb { + Some(vb @ [_, _, w, h]) if w > 0.0 && h > 0.0 => vb, + _ => { + let all = || paths.iter().flat_map(|p| p.pts.iter()); + let (x0, y0) = all().fold((f32::MAX, f32::MAX), |(x, y), p| (x.min(p[0]), y.min(p[1]))); + let (x1, y1) = all().fold((f32::MIN, f32::MIN), |(x, y), p| (x.max(p[0]), y.max(p[1]))); + if x0 > x1 { [0.0, 0.0, 0.0, 0.0] } else { [x0, y0, x1 - x0, y1 - y0] } + } + }; + let paths: Vec = paths + .into_iter() + .take(MAX_PATHS) + .map(|p| FreePath { pts: p.pts.into_iter().take(MAX_POINTS).map(|[x, y]| [x - x0, y - y0]).collect(), closed: p.closed }) + .filter(|p| !p.pts.is_empty()) + .collect(); + (!paths.is_empty()).then(|| Freeform { w: w.max(0.0), h: h.max(0.0), paths, ..Default::default() }.sanitized()) +} + +/// A polygon's or polyline's points (`draw:points`: "x,y x,y …"), closed or not. +pub(super) fn points(v: &str, closed: bool, vb: Option<[f32; 4]>) -> Option { + let n: Vec = numbers(v).collect(); + let pts = n.as_chunks::<2>().0.iter().map(|c| [c[0], c[1]]).take(MAX_POINTS).collect(); + freeform(vec![FreePath { pts, closed }], vb) +} + +/// A regular polygon (§10.3.6) of `corners` corners in a `w` × `h` box, its first corner at the +/// top middle; concave (a star), its inner corners `sharpness` (0–1) of the way in. +pub(super) fn regular(corners: u32, concave: bool, sharpness: f32, w: f32, h: f32) -> Option { + let n = corners.clamp(3, 1000) as usize; + let (rx, ry) = (f64::from(w) / 2.0, f64::from(h) / 2.0); + let inner = 1.0 - f64::from(if sharpness.is_finite() { sharpness.clamp(0.0, 1.0) } else { 0.0 }); + let step = std::f64::consts::TAU / n as f64; + let mut pts = Vec::with_capacity(n * 2); + for i in 0..n { + let a = -std::f64::consts::FRAC_PI_2 + step * i as f64; + pts.push([(rx + rx * a.cos()) as f32, (ry + ry * a.sin()) as f32]); + if concave { + let b = a + step / 2.0; + pts.push([(rx + rx * inner * b.cos()) as f32, (ry + ry * inner * b.sin()) as f32]); + } + } + freeform(vec![FreePath { pts, closed: true }], Some([0.0, 0.0, w, h])) +} + +/// A path's SVG path data (`svg:d`, §19.523: moves, lines, cubic and quadratic curves and +/// elliptical arcs, absolute and relative), each subpath a freeform path. +pub(super) fn path(d: &str, vb: Option<[f32; 4]>) -> Option { + let mut p = PathReader::default(); + let mut tokens = Tokens { s: d, at: 0, read: 0, peeked: None }; + let mut cmd = None; + while let Some(t) = tokens.next() { + let c = match t { + Token::Cmd(c) => c, + // More numbers repeat the command (a move's are lines). + Token::Num(_) => match cmd { + Some('M') => 'L', + Some('m') => 'l', + // A close takes no numbers: a number after one ends the data (it would be read + // again for ever otherwise). + Some('Z' | 'z') | None => break, + Some(c) => c, + }, + }; + if let Token::Num(_) = t { + tokens.peeked = Some(t); + } + cmd = Some(c); + if !p.command(c, &mut tokens) { + break; + } + } + p.end(false); + freeform(p.paths, vb) +} + +#[derive(Clone, Copy)] +enum Token { + Cmd(char), + Num(f64), +} + +/// The commands and numbers of path data. +struct Tokens<'a> { + s: &'a str, + at: usize, + read: usize, + /// A number read as the start of a repeated command, to be read again as its first. + peeked: Option, +} + +impl Tokens<'_> { + fn next(&mut self) -> Option { + if let Some(t) = self.peeked.take() { + return Some(t); + } + self.read += 1; + if self.read > MAX_NUMBERS { + return None; + } + let rest = self.s.get(self.at..)?; + let skip = rest.len() - rest.trim_start_matches(|c: char| c.is_whitespace() || c == ',').len(); + self.at += skip; + let rest = self.s.get(self.at..)?; + let c = rest.chars().next()?; + if c.is_ascii_alphabetic() && c != 'e' && c != 'E' { + self.at += c.len_utf8(); + return Some(Token::Cmd(c)); + } + // A number: sign, digits, a point, an exponent; a second point or sign starts the next. + let b = rest.as_bytes(); + let mut end = 0; + let (mut dot, mut exp) = (false, false); + while let Some(&ch) = b.get(end) { + let ok = match ch { + b'0'..=b'9' => true, + b'+' | b'-' => end == 0 || matches!(b.get(end - 1), Some(b'e' | b'E')), + b'.' if !dot && !exp => { + dot = true; + true + } + b'e' | b'E' if !exp && end > 0 => { + exp = true; + true + } + _ => false, + }; + if !ok { + break; + } + end += 1; + } + if end == 0 { + // Not a number: stop. + self.at = self.s.len(); + return None; + } + let v = rest.get(..end).and_then(|n| n.parse::().ok()).filter(|v| v.is_finite()).unwrap_or(0.0); + self.at += end; + Some(Token::Num(v.clamp(-MAX_COORD, MAX_COORD))) + } + + fn num(&mut self) -> Option { + match self.next() { + Some(Token::Num(v)) => Some(v), + Some(Token::Cmd(_)) => { + // A command where a number belongs: the data is malformed; stop. + self.at = self.s.len(); + None + } + None => None, + } + } +} + +#[derive(Default)] +struct PathReader { + paths: Vec, + cur: Vec<[f32; 2]>, + at: (f64, f64), + start: (f64, f64), + /// The last curve's second control point, for a smooth curve's reflection. + last_ctrl: Option<(f64, f64, bool)>, +} + +impl PathReader { + fn push(&mut self, p: (f64, f64)) { + if self.cur.len() < MAX_POINTS { + let c = |v: f64| v.clamp(-MAX_COORD, MAX_COORD) as f32; + self.cur.push([c(p.0), c(p.1)]); + } + self.at = p; + } + + /// A drawing command without a move before it starts its subpath where the pen is. + fn begin(&mut self) { + if self.cur.is_empty() { + self.push(self.at); + } + } + + fn end(&mut self, closed: bool) { + let pts = std::mem::take(&mut self.cur); + if !pts.is_empty() && self.paths.len() < MAX_PATHS { + self.paths.push(FreePath { pts, closed }); + } + } + + /// Command `c` with its numbers from `t`; false when the data runs out or is malformed. + fn command(&mut self, c: char, t: &mut Tokens<'_>) -> bool { + let rel = c.is_ascii_lowercase(); + let (ox, oy) = if rel { self.at } else { (0.0, 0.0) }; + let pt = |t: &mut Tokens<'_>| Some((t.num()? + ox, t.num()? + oy)); + let mut quad = false; + match c.to_ascii_uppercase() { + 'M' => { + let Some(p) = pt(t) else { return false }; + self.end(false); + self.start = p; + self.push(p); + } + 'L' => { + let Some(p) = pt(t) else { return false }; + self.begin(); + self.push(p); + } + 'H' => { + let Some(x) = t.num() else { return false }; + let x = if rel { x + self.at.0 } else { x }; + self.begin(); + self.push((x, self.at.1)); + } + 'V' => { + let Some(y) = t.num() else { return false }; + let y = if rel { y + self.at.1 } else { y }; + self.begin(); + self.push((self.at.0, y)); + } + 'C' | 'S' => { + let c1 = if c.eq_ignore_ascii_case(&'S') { self.reflected(false) } else { pt(t) }; + let (Some(c1), Some(c2), Some(e)) = (c1, pt(t), pt(t)) else { return false }; + self.cubic(c1, c2, e); + self.last_ctrl = Some((c2.0, c2.1, false)); + return true; + } + 'Q' | 'T' => { + let q = if c.eq_ignore_ascii_case(&'T') { self.reflected(true) } else { pt(t) }; + let (Some(q), Some(e)) = (q, pt(t)) else { return false }; + // A quadratic curve is the cubic with controls two thirds of the way to its own. + let s = self.at; + let c1 = (s.0 + 2.0 / 3.0 * (q.0 - s.0), s.1 + 2.0 / 3.0 * (q.1 - s.1)); + let c2 = (e.0 + 2.0 / 3.0 * (q.0 - e.0), e.1 + 2.0 / 3.0 * (q.1 - e.1)); + self.cubic(c1, c2, e); + self.last_ctrl = Some((q.0, q.1, true)); + quad = true; + } + 'A' => { + let (Some(rx), Some(ry), Some(rot), Some(large), Some(sweep), Some(e)) = (t.num(), t.num(), t.num(), t.num(), t.num(), pt(t)) else { + return false; + }; + self.arc((rx, ry), rot, large != 0.0, sweep != 0.0, e); + } + 'Z' => { + self.end(true); + // A new subpath after it starts where this one did, unless it moves. + self.at = self.start; + } + _ => return false, + } + if !quad { + self.last_ctrl = None; + } + true + } + + /// The first control point of a smooth curve: the last one's reflected (`quad`: of a + /// quadratic curve), or the current point. + fn reflected(&self, quad: bool) -> Option<(f64, f64)> { + Some(match self.last_ctrl { + Some((x, y, q)) if q == quad => (2.0 * self.at.0 - x, 2.0 * self.at.1 - y), + _ => self.at, + }) + } + + fn cubic(&mut self, c1: (f64, f64), c2: (f64, f64), e: (f64, f64)) { + let s = self.at; + self.begin(); + for i in 1..CURVE_STEPS { + let t = i as f64 / CURVE_STEPS as f64; + let u = 1.0 - t; + let (a, b, c, d) = (u * u * u, 3.0 * u * u * t, 3.0 * u * t * t, t * t * t); + self.push((a * s.0 + b * c1.0 + c * c2.0 + d * e.0, a * s.1 + b * c1.1 + c * c2.1 + d * e.1)); + } + self.push(e); + } + + /// An elliptical arc to `e` (SVG's endpoint form, turned into its centre form). + fn arc(&mut self, r: (f64, f64), rot_deg: f64, large: bool, sweep: bool, e: (f64, f64)) { + let s = self.at; + self.begin(); + let (mut rx, mut ry) = (r.0.abs(), r.1.abs()); + if rx < 1e-9 || ry < 1e-9 || (s.0 - e.0).abs() + (s.1 - e.1).abs() < 1e-9 { + self.push(e); + return; + } + let (sin, cos) = (rot_deg % 360.0).to_radians().sin_cos(); + // The start relative to the chord's middle, in the ellipse's own axes. + let (dx, dy) = ((s.0 - e.0) / 2.0, (s.1 - e.1) / 2.0); + let (x1, y1) = (cos * dx + sin * dy, -sin * dx + cos * dy); + // Radii too small to reach are scaled up just enough. + let k = (x1 * x1) / (rx * rx) + (y1 * y1) / (ry * ry); + if k > 1.0 { + (rx, ry) = (rx * k.sqrt(), ry * k.sqrt()); + } + let num = (rx * rx * ry * ry - rx * rx * y1 * y1 - ry * ry * x1 * x1).max(0.0); + let den = rx * rx * y1 * y1 + ry * ry * x1 * x1; + let mut f = if den > 0.0 { (num / den).sqrt() } else { 0.0 }; + if large == sweep { + f = -f; + } + let (cx1, cy1) = (f * rx * y1 / ry, -f * ry * x1 / rx); + let (cx, cy) = (cos * cx1 - sin * cy1 + (s.0 + e.0) / 2.0, sin * cx1 + cos * cy1 + (s.1 + e.1) / 2.0); + let angle = |ux: f64, uy: f64| uy.atan2(ux); + let a0 = angle((x1 - cx1) / rx, (y1 - cy1) / ry); + let a1 = angle((-x1 - cx1) / rx, (-y1 - cy1) / ry); + let mut da = a1 - a0; + if sweep && da < 0.0 { + da += std::f64::consts::TAU; + } else if !sweep && da > 0.0 { + da -= std::f64::consts::TAU; + } + if !da.is_finite() || !cx.is_finite() || !cy.is_finite() { + self.push(e); + return; + } + for i in 1..CURVE_STEPS { + let a = a0 + da * i as f64 / CURVE_STEPS as f64; + let (px, py) = (rx * a.cos(), ry * a.sin()); + self.push((cos * px - sin * py + cx, sin * px + cos * py + cy)); + } + self.push(e); + } +} + +/// `freeform`'s paths as SVG path data in a view box `vw` × `vh` (whole numbers, as ODF's view +/// box is): each path moved to, lined through, and closed when it is. +pub(super) fn path_data(f: &Freeform, vw: u32, vh: u32) -> String { + path_text(f, vw, vh, false) +} + +/// Whether `freeform` has both closed and open paths: drawn filled and unfilled. +pub(super) fn is_mixed(f: &Freeform) -> bool { + f.paths.iter().any(|p| p.closed) && f.paths.iter().any(|p| !p.closed) +} + +/// `freeform`'s paths as a custom shape's path (§19.145 `draw:enhanced-path`) in a view box +/// `vw` × `vh`: each path ended (`N`), open ones unfilled (`F`). +pub(super) fn enhanced_path_data(f: &Freeform, vw: u32, vh: u32) -> String { + path_text(f, vw, vh, true) +} + +fn path_text(f: &Freeform, vw: u32, vh: u32, enhanced: bool) -> String { + let sx = if f.w > 0.0 { vw as f32 / f.w } else { 1.0 }; + let sy = if f.h > 0.0 { vh as f32 / f.h } else { 1.0 }; + let n = |v: f32| { + let v = if v.is_finite() { v.clamp(-MAX_COORD as f32, MAX_COORD as f32) } else { 0.0 }; + format!("{}", v.round() as i64) + }; + let mut d = String::new(); + for p in f.paths.iter().take(MAX_PATHS) { + for (i, [x, y]) in p.pts.iter().take(MAX_POINTS).enumerate() { + d.push_str(if i == 0 { "M" } else { " L" }); + if enhanced { + d.push(' '); + } + d.push_str(&format!("{} {}", n(x * sx), n(y * sy))); + } + if p.closed { + d.push_str(" Z"); + } + if enhanced { + d.push_str(if p.closed { " N" } else { " F N" }); + } + d.push(' '); + } + d.trim_end().to_string() +} + +#[cfg(test)] +mod tests { + /// A number after a close ends the path data; it used to be read again for ever (a hang on a + /// hostile or malformed `svg:d`). + #[test] + fn a_number_after_a_close_ends_the_path() { + let f = super::path("M 0 0 L 10 0 L 10 10 Z 5 5", Some([0.0, 0.0, 10.0, 10.0])); + let paths = f.map(|f| f.paths).unwrap_or_default(); + assert_eq!(paths.len(), 1, "{paths:?}"); + assert!(paths[0].closed); + assert_eq!(paths[0].pts.len(), 3); + // Lower case too, and a close ending the data as before. + assert!(super::path("m 0 0 l 10 0 l 0 10 z 1", None).is_some()); + assert!(super::path("M 0 0 L 10 0 L 10 10 Z", None).is_some()); + } +} diff --git a/crates/odf/src/text/shapes.rs b/crates/odf/src/text/shapes.rs new file mode 100644 index 000000000..7c08151aa --- /dev/null +++ b/crates/odf/src/text/shapes.rs @@ -0,0 +1,171 @@ +//! Drawing shapes (ODF 1.3 §10.3): `draw:rect`, `draw:ellipse`, `draw:line` and +//! `draw:custom-shape`, whose `draw:enhanced-geometry` (§10.6.2) names its predefined geometry +//! in `draw:type`. WordCraft's shape kinds both ways; LibreOffice rebuilds a predefined +//! geometry from its type alone, so only the type is written. + +use super::xml::{Attrs, get, length}; +use craft_wordmodel::para::ShapeKind; + +/// Largest size or offset read, points (22 inches). +const MAX_LEN: f32 = 1584.0; + +/// Whether `q` is a drawing shape element read as a shape. +pub(super) fn is_shape(q: &str) -> bool { + matches!( + q, + "draw:rect" + | "draw:ellipse" + | "draw:circle" + | "draw:line" + | "draw:custom-shape" + | "draw:polygon" + | "draw:polyline" + | "draw:path" + | "draw:regular-polygon" + | "draw:connector" + ) +} + +/// The kind of shape element `name` (local name); a custom shape's geometry says more. +pub(super) fn element_kind(name: &str) -> ShapeKind { + match name { + "ellipse" | "circle" => ShapeKind::Ellipse, + "line" => ShapeKind::Line, + "connector" => ShapeKind::ElbowConnector, + "polygon" | "polyline" | "path" | "regular-polygon" => ShapeKind::Freeform, + _ => ShapeKind::Rectangle, + } +} + +/// The kind of a custom shape of `draw:type` `t`: LibreOffice's own type names, and the +/// DrawingML preset names it writes as `ooxml-…` for shapes from Word documents. A geometry +/// WordCraft has no shape for is a rectangle. +/// A preset geometry type WordCraft has no kind for: drawn as a rectangle (see [`kind_of`]), its +/// geometry kept to be written back. +pub(super) fn unknown_preset(t: &str) -> bool { + let t = t.trim(); + kind_of(t) == ShapeKind::Rectangle + && !matches!(t.strip_prefix("ooxml-").unwrap_or(t), "" | "rectangle" | "rect" | "flowchart-process" | "flowChartProcess" | "non-primitive") +} + +pub(super) fn kind_of(t: &str) -> ShapeKind { + let t = t.trim(); + match t.strip_prefix("ooxml-").unwrap_or(t) { + "round-rectangle" | "roundRect" => ShapeKind::RoundedRectangle, + "ellipse" | "flowchart-connector" | "flowChartConnector" => ShapeKind::Ellipse, + "flowchart-or" | "flowChartOr" => ShapeKind::FlowchartOr, + "isosceles-triangle" | "triangle" => ShapeKind::Triangle, + "right-triangle" | "rtTriangle" => ShapeKind::RightTriangle, + "diamond" | "flowchart-decision" | "flowChartDecision" => ShapeKind::Diamond, + "right-arrow" | "rightArrow" => ShapeKind::Arrow, + "left-arrow" | "leftArrow" => ShapeKind::ArrowLeft, + "up-arrow" | "upArrow" => ShapeKind::ArrowUp, + "down-arrow" | "downArrow" => ShapeKind::ArrowDown, + "star5" => ShapeKind::Star, + "heart" => ShapeKind::Heart, + "line" | "straightConnector1" => ShapeKind::Line, + _ => ShapeKind::Rectangle, + } +} + +/// A connector's kind by its `draw:type` (§19.229): `line` straight, `curve` curved, `standard` +/// (the default) and `lines` with bends. +pub(super) fn connector_kind(t: Option<&str>) -> ShapeKind { + match t.map(str::trim) { + Some("line") => ShapeKind::StraightConnector, + Some("curve") => ShapeKind::CurvedConnector, + _ => ShapeKind::ElbowConnector, + } +} + +/// A connector kind's `draw:type`. +pub(super) fn connector_type(kind: ShapeKind) -> &'static str { + match kind { + ShapeKind::StraightConnector => "line", + ShapeKind::CurvedConnector => "curve", + _ => "standard", + } +} + +/// WordCraft's connection site of a `kind` shape for ODF glue point `g` (§9.2.19: a shape's +/// default glue points are 0 top, 1 right, 2 bottom, 3 left; see +/// [`craft_wordmodel::connector::sites`]). `None` for a glue point of its own (4 on) or a shape +/// without sides. +pub(super) fn site_of_glue(kind: ShapeKind, g: u32) -> Option { + let sides: [u32; 4] = match kind { + ShapeKind::Ellipse => [0, 6, 4, 2], + ShapeKind::Triangle => [0, 5, 3, 1], + k if k.is_open() || k == ShapeKind::RightTriangle => return None, + _ => [0, 3, 2, 1], + }; + sides.get(g as usize).copied() +} + +/// The ODF glue point of site `site` of a `kind` shape (the inverse of [`site_of_glue`]). +pub(super) fn glue_of_site(kind: ShapeKind, site: u32) -> Option { + (0..4).find(|g| site_of_glue(kind, *g) == Some(site)) +} + +/// How shape `kind` is written: its element, and a custom shape's `draw:type`. +pub(super) fn element_of(kind: ShapeKind) -> (&'static str, Option<&'static str>) { + let custom = |t| ("draw:custom-shape", Some(t)); + match kind { + ShapeKind::Ellipse => ("draw:ellipse", None), + ShapeKind::Line => ("draw:line", None), + ShapeKind::StraightConnector | ShapeKind::ElbowConnector | ShapeKind::CurvedConnector => ("draw:connector", None), + ShapeKind::RoundedRectangle => custom("round-rectangle"), + ShapeKind::FlowchartOr => custom("flowchart-or"), + ShapeKind::Triangle => custom("isosceles-triangle"), + ShapeKind::RightTriangle => custom("right-triangle"), + ShapeKind::Diamond => custom("diamond"), + ShapeKind::Arrow => custom("right-arrow"), + ShapeKind::ArrowLeft => custom("left-arrow"), + ShapeKind::ArrowUp => custom("up-arrow"), + ShapeKind::ArrowDown => custom("down-arrow"), + ShapeKind::Star => custom("star5"), + ShapeKind::Heart => custom("heart"), + ShapeKind::Freeform => ("draw:path", None), + ShapeKind::Rectangle | ShapeKind::TextBox => ("draw:rect", None), + } +} + +/// A shape's size and offset (`svg:width`, `svg:height`, `svg:x`, `svg:y`), points, and which +/// way it is drawn (from right to left, from bottom to top). A line's come from its two ends +/// (`svg:x1`…): its box spans them, and it starts at the first end, so a line drawn leftwards +/// or upwards is flipped that way (its arrowheads stay at its ends). +pub(super) fn geometry(kind: ShapeKind, a: &Attrs) -> (f32, f32, Option, Option, (bool, bool)) { + let len = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite()).map(|v| v.clamp(-MAX_LEN, MAX_LEN)); + if kind.is_open() { + let [x1, y1, x2, y2] = ["svg:x1", "svg:y1", "svg:x2", "svg:y2"].map(|k| len(k).unwrap_or(0.0)); + let size = |d: f32| d.abs().min(MAX_LEN); + return (size(x2 - x1), size(y2 - y1), Some(x1.min(x2)), Some(y1.min(y2)), (x2 < x1, y2 < y1)); + } + // A circle may be given by its centre and radius (§19.546 `svg:cx`, §19.554 `svg:r`). + if get(a, "svg:width").is_none() + && let Some(r) = len("svg:r").map(f32::abs) + { + let (cx, cy) = (len("svg:cx").unwrap_or(r), len("svg:cy").unwrap_or(r)); + let d = (r * 2.0).clamp(1.0, MAX_LEN); + return (d, d, Some(cx - r), Some(cy - r), (false, false)); + } + let size = |k: &str| len(k).map_or(72.0, |v| v.clamp(1.0, MAX_LEN)); + (size("svg:width"), size("svg:height"), len("svg:x"), len("svg:y"), (false, false)) +} + +/// The paths of a polygon, polyline, path or regular polygon `name` (local) with attributes +/// `a`, `w` × `h` points: in its view box (§19.570 `svg:viewBox`; none: around its points). +pub(super) fn freeform(name: &str, a: &Attrs, w: f32, h: f32) -> Option { + use super::path; + let vb = path::view_box(get(a, "svg:viewBox")); + match name { + "polygon" | "polyline" => path::points(get(a, "draw:points")?, name == "polygon", vb), + "path" => path::path(get(a, "svg:d")?, vb), + "regular-polygon" => { + let corners = get(a, "draw:corners").and_then(|c| c.trim().parse::().ok()).unwrap_or(3); + let concave = get(a, "draw:concave").is_some_and(|c| c.trim() == "true"); + let sharpness = get(a, "draw:sharpness").and_then(|v| v.trim().strip_suffix('%')?.trim().parse::().ok()).map_or(0.0, |p| p / 100.0); + path::regular(corners, concave, sharpness, w, h) + } + _ => None, + } +} diff --git a/crates/odf/src/text/strokes.rs b/crates/odf/src/text/strokes.rs new file mode 100644 index 000000000..06b8ab0fc --- /dev/null +++ b/crates/odf/src/text/strokes.rs @@ -0,0 +1,348 @@ +//! The named drawing definitions graphic styles use (ODF 1.3 §16.39–16.41): line dashes +//! (`draw:stroke-dash`), gradients (`draw:gradient`) and arrowheads (`draw:marker`), read into +//! WordCraft's dash patterns and gradient fills and written from them. + +use super::writer::pt; +use super::xml::{Attrs, get, length}; +use craft_flow::parse_color; +use craft_wordmodel::fill::{Fill, Gradient, GradientKind, PathShape}; +use craft_wordmodel::line_dash::{DASH_PERCENT, LineDash, MAX_DASH_STOPS}; +use craft_wordmodel::props::Rgb; +use craft_wordmodel::wordart::GradStop; +use std::sync::Arc; + +/// The arrowhead WordCraft writes (`draw:marker`, §16.39): a plain triangle pointing along the +/// line, its tip at the line's end. +pub(super) const ARROW: &str = "WordCraft_20_Arrow"; +pub(super) const ARROW_DEF: &str = + ""; + +/// A dash or space length: a fraction of the line width, or an absolute length in points. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Len { + Width(f32), + Points(f32), +} + +impl Len { + fn read(v: Option<&str>) -> Option { + let v = v?.trim(); + match v.strip_suffix('%') { + Some(p) => p.trim().parse::().ok().filter(|p| p.is_finite()).map(|p| Len::Width((p / 100.0).clamp(0.0, 100.0))), + None => length(v).filter(|l| l.is_finite()).map(|l| Len::Points(l.clamp(0.0, 1584.0))), + } + } + + /// In 100000ths of a line `width` points wide. + fn of(self, width: f32) -> u32 { + let ratio = match self { + Len::Width(r) => r, + Len::Points(p) => { + let w = if width.is_finite() && width > 0.0 { width } else { 0.75 }; + p / w + } + }; + (ratio.clamp(0.0, 1000.0) * DASH_PERCENT as f32).round() as u32 + } +} + +/// A `draw:stroke-dash`: `dots1` dashes of one length then `dots2` of another, each followed by +/// the same space. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Dash { + dots1: u32, + len1: Len, + dots2: u32, + len2: Len, + space: Len, +} + +impl Dash { + pub(super) fn read(a: &Attrs) -> Dash { + let count = |k: &str| get(a, k).and_then(|v| v.trim().parse::().ok()).unwrap_or(0).min(MAX_DASH_STOPS as u32); + // A dash without a length is a dot as long as the line is wide. + let len = |k: &str| Len::read(get(a, k)).unwrap_or(Len::Width(1.0)); + Dash { + dots1: count("draw:dots1"), + len1: len("draw:dots1-length"), + dots2: count("draw:dots2"), + len2: len("draw:dots2-length"), + space: len("draw:distance"), + } + } + + /// The pattern for a line `width` points wide: one of WordCraft's named dashes when it is one. + pub(super) fn pattern(&self, width: f32) -> LineDash { + let space = self.space.of(width); + let pairs: Vec<(u32, u32)> = std::iter::repeat_n((self.len1.of(width), space), self.dots1 as usize) + .chain(std::iter::repeat_n((self.len2.of(width), space), self.dots2 as usize)) + .take(MAX_DASH_STOPS) + .collect(); + if pairs.is_empty() { + return LineDash::Solid; + } + PRESETS.iter().find(|p| ratios(p) == pairs).cloned().unwrap_or(LineDash::Custom(pairs)) + } +} + +/// WordCraft's named dashes. +const PRESETS: [LineDash; 10] = [ + LineDash::Dot, + LineDash::Dash, + LineDash::LongDash, + LineDash::DashDot, + LineDash::LongDashDot, + LineDash::LongDashDotDot, + LineDash::SystemDash, + LineDash::SystemDot, + LineDash::SystemDashDot, + LineDash::SystemDashDotDot, +]; + +/// A dash's (dash, space) pairs in 100000ths of the line width. +fn ratios(d: &LineDash) -> Vec<(u32, u32)> { + if let LineDash::Custom(pairs) = d { + return pairs.iter().take(MAX_DASH_STOPS).copied().collect(); + } + let p: Vec = d.pattern(1.0).iter().map(|v| (v * DASH_PERCENT as f32).round() as u32).collect(); + p.chunks(2).map(|c| (c.first().copied().unwrap_or(0), c.get(1).copied().unwrap_or(0))).collect() +} + +/// The `draw:stroke-dash` named `name` for `dash`, or `None` for a solid line. ODF's dash has +/// at most two kinds of dash and one space: a pattern with more is written as its first two. +pub(super) fn dash_def(name: &str, dash: &LineDash) -> Option { + let pairs = ratios(dash); + let (&(l1, space), rest) = pairs.split_first()?; + let dots1 = 1 + rest.iter().take_while(|p| **p == (l1, space)).count(); + let second = rest.get(dots1 - 1).map(|p| p.0); + let dots2 = second.map_or(0, |l2| rest.iter().skip(dots1 - 1).take_while(|p| p.0 == l2).count()); + let pc = |v: u32| format!("{}%", v as f32 * 100.0 / DASH_PERCENT as f32); + let mut s = format!("", pc(space))); + Some(s) +} + +/// An angle (§18.3.1): degrees, radians or grads by its unit, tenths of a degree without one +/// (as ODF 1.2 wrote a gradient's). +/// Whether a part of ODF `version` (its `office:version`; none: before 1.2) has its gradients' +/// and hatches' unitless angles in tenths of a degree. +/// +/// **A deviation from the standard, out of necessity.** ODF's angle (§18.3.1, since ODF 1.2) +/// is in degrees when it has no unit, so `draw:angle="300"` (§19.112) and `draw:rotation` +/// (§19.209) mean 300°. But LibreOffice and OpenOffice wrote these angles in tenths of a degree +/// in their files before ODF 1.3 (`300` for 30°), and LibreOffice still reads such files so +/// (observed: the same gradient turns 30° in an ODF 1.2 file and 300° in an ODF 1.3 one). Read +/// by the standard, the gradients of every older LibreOffice document would turn the wrong +/// way. So for files older than 1.3 WordCraft reads them as written; from 1.3 on, as the +/// standard says. WordCraft writes them with a unit (`30deg`), which both read the same. +pub(super) fn legacy_angles(version: Option<&str>) -> bool { + version.is_none_or(|v| v.trim() < "1.3") +} + +fn angle(v: &str, legacy: bool) -> Option { + let v = v.trim(); + let (n, scale) = if let Some(n) = v.strip_suffix("deg") { + (n, 1.0) + } else if let Some(n) = v.strip_suffix("grad") { + (n, 0.9) + } else if let Some(n) = v.strip_suffix("rad") { + (n, 180.0 / std::f32::consts::PI) + } else if legacy { + // A deviation from the standard, out of necessity: see `legacy_angles`. + (v, 0.1) + } else { + // §18.3.1: without a unit, degrees. + (v, 1.0) + }; + n.trim().parse::().ok().map(|a| a * scale).filter(|a| a.is_finite()).map(|a| a.rem_euclid(360.0)) +} + +/// A percentage (`50%`), 0–100. +fn percent(v: Option<&str>) -> Option { + v?.trim().strip_suffix('%')?.trim().parse::().ok().filter(|p| p.is_finite()).map(|p| p.clamp(0.0, 100.0)) +} + +/// A `draw:gradient` (§16.40) as a fill. Linear and axial gradients run along their angle (ODF's +/// 0° runs top to bottom, turning anticlockwise; WordCraft's 0° left to right, clockwise); +/// radial, ellipsoid, square and rectangular ones run from their end colour in the middle to +/// their start colour at the edge. The border is where the start colour stays plain; intensity +/// darkens a colour. +pub(super) fn gradient(a: &Attrs, legacy: bool) -> Option { + let color = |k: &str, intensity: &str, default: Rgb| { + let c = get(a, k).and_then(parse_color).unwrap_or(default); + let i = percent(get(a, intensity)).unwrap_or(100.0) / 100.0; + let dim = |v: u8| (f32::from(v) * i).round().clamp(0.0, 255.0) as u8; + Rgb(dim(c.0), dim(c.1), dim(c.2)) + }; + let start = color("draw:start-color", "draw:start-intensity", Rgb::BLACK); + let end = color("draw:end-color", "draw:end-intensity", Rgb::WHITE); + let border = percent(get(a, "draw:border")).unwrap_or(0.0); + let stop = |pos: f32, color: Rgb| GradStop { pos, color, transparency: 0.0 }; + let along = (90.0 - get(a, "draw:angle").and_then(|v| angle(v, legacy)).unwrap_or(0.0)).rem_euclid(360.0); + let centre = || { + let (cx, cy) = (percent(get(a, "draw:cx")).unwrap_or(50.0), percent(get(a, "draw:cy")).unwrap_or(50.0)); + [cx, cy, 100.0 - cx, 100.0 - cy] + }; + let (kind, stops) = match get(a, "draw:style").map(str::trim).unwrap_or("linear") { + "axial" => (GradientKind::Linear { angle: along, scaled: false }, vec![stop(0.0, start), stop(50.0, end), stop(100.0, start)]), + style @ ("radial" | "ellipsoid" | "square" | "rectangular") => { + let shape = if matches!(style, "square" | "rectangular") { PathShape::Rect } else { PathShape::Circle }; + let mut stops = vec![stop(0.0, end)]; + if border > 0.0 { + stops.push(stop(100.0 - border, start)); + } + stops.push(stop(100.0, start)); + (GradientKind::Path { shape, focus: centre() }, stops) + } + _ => { + let mut stops = vec![stop(0.0, start)]; + if border > 0.0 { + stops.push(stop(border, start)); + } + stops.push(stop(100.0, end)); + (GradientKind::Linear { angle: along, scaled: false }, stops) + } + }; + Fill::Gradient(Arc::new(Gradient { stops, kind, tile: None, rot_with_shape: true, flip: Default::default() })).sanitized() +} + +/// The `draw:gradient` named `name` for gradient fill `g` (the reverse of [`gradient`]): a +/// gradient running out from its middle and back is axial; more colours than ODF's two keep +/// the first and last. +pub(super) fn gradient_def(name: &str, g: &Gradient) -> Option { + let hex = |c: Rgb| format!("#{}", c.hex().to_ascii_lowercase()); + let (first, last) = (g.stops.first()?, g.stops.last()?); + let body = match g.kind { + GradientKind::Linear { angle, .. } => { + let odf = (90.0 - if angle.is_finite() { angle } else { 0.0 }).rem_euclid(360.0); + let axial = g.stops.len() == 3 && first.color == last.color && g.stops.get(1).is_some_and(|m| (m.pos - 50.0).abs() < 0.5); + let (style, end) = match g.stops.get(1) { + Some(mid) if axial => ("axial", mid.color), + _ => ("linear", last.color), + }; + format!( + "draw:style=\"{style}\" draw:start-color=\"{}\" draw:end-color=\"{}\" draw:angle=\"{}deg\"", + hex(first.color), + hex(end), + odf.round() + ) + } + GradientKind::Path { shape, focus: [l, t, r, b] } => { + let style = if shape == PathShape::Rect { "rectangular" } else { "radial" }; + let mid = |a: f32, b: f32| if a.is_finite() && b.is_finite() { ((a + 100.0 - b) / 2.0).clamp(0.0, 100.0) } else { 50.0 }; + format!( + "draw:style=\"{style}\" draw:cx=\"{}%\" draw:cy=\"{}%\" draw:start-color=\"{}\" draw:end-color=\"{}\"", + mid(l, r), + mid(t, b), + hex(last.color), + hex(first.color) + ) + } + }; + Some(format!("")) +} + +/// The width an arrowhead is drawn at on a line `width` points wide: three times as wide, and +/// at least 5 points. +pub(super) fn arrow_width(width: f32) -> String { + pt(if width.is_finite() { (width * 3.0).clamp(5.0, 36.0) } else { 5.0 }) +} + +/// Most named dashes and gradients kept (hostile files). +const MAX_DEFS: usize = 10_000; + +/// The named dashes and gradients a document's styles define. +#[derive(Default)] +pub(super) struct Defs { + dashes: std::collections::HashMap, + gradients: std::collections::HashMap, + /// The part read is older than ODF 1.3: unitless angles are tenths of a degree (see + /// [`legacy_angles`]). + pub(super) legacy_angles: bool, +} + +impl Defs { + /// Element `q` with attributes `a`: a dash or gradient definition, or a part's root, whose + /// version says how unitless gradient angles are read (see [`legacy_angles`]). + pub(super) fn element(&mut self, q: &str, a: &Attrs) { + match q { + "office:document-styles" | "office:document-content" | "office:document" => self.legacy_angles = legacy_angles(get(a, "office:version")), + "draw:stroke-dash" => self.read("stroke-dash", a), + "draw:gradient" => self.read("gradient", a), + _ => {} + } + } + + /// Element `name` (local) with attributes `a`, when it is a definition. + pub(super) fn read(&mut self, name: &str, a: &Attrs) { + let Some(id) = get(a, "draw:name").map(String::from) else { return }; + if self.dashes.len() + self.gradients.len() >= MAX_DEFS { + return; + } + match name { + "stroke-dash" => { + self.dashes.insert(id, Dash::read(a)); + } + "gradient" => { + if let Some(g) = gradient(a, self.legacy_angles) { + self.gradients.insert(id, g); + } + } + _ => {} + } + } + + pub(super) fn dash(&self, name: &str) -> Option<&Dash> { + self.dashes.get(name) + } + + pub(super) fn gradient(&self, name: &str) -> Option<&Fill> { + self.gradients.get(name) + } +} + +/// The named definitions a writer's graphic styles use, for `styles.xml`. +#[derive(Default)] +pub(super) struct DrawDefs { + /// Each definition's name by its content (one name for equal definitions). + names: std::collections::BTreeMap, + /// Name → definition. + pub(super) defs: std::collections::BTreeMap, +} + +impl DrawDefs { + /// The name of the definition `make` writes for `kind` (with this writer's `prefix`). + fn named(&mut self, kind: &str, prefix: &str, make: impl Fn(&str) -> Option) -> Option { + let key = make("")?; + if let Some(n) = self.names.get(&key) { + return Some(n.clone()); + } + let name = format!("WordCraft_20_{kind}_20_{prefix}{}", self.names.len() + 1); + let def = make(&name)?; + self.names.insert(key, name.clone()); + self.defs.insert(name.clone(), def); + Some(name) + } + + /// The `draw:stroke-dash` name for `dash`; `None` for a solid line. + pub(super) fn dash(&mut self, dash: &LineDash, prefix: &str) -> Option { + if dash.is_solid() { + return None; + } + self.named("Dash", prefix, |n| dash_def(n, dash)) + } + + /// The `draw:gradient` name for `g`. + pub(super) fn gradient(&mut self, g: &Gradient, prefix: &str) -> Option { + self.named("Gradient", prefix, |n| gradient_def(n, g)) + } + + /// The arrowhead's `draw:marker` name. + pub(super) fn arrow(&mut self) -> &'static str { + self.defs.entry(ARROW.to_string()).or_insert_with(|| ARROW_DEF.to_string()); + ARROW + } +} diff --git a/crates/odf/src/text/styles.rs b/crates/odf/src/text/styles.rs new file mode 100644 index 000000000..ff4a44eeb --- /dev/null +++ b/crates/odf/src/text/styles.rs @@ -0,0 +1,1750 @@ +use super::xml::{Attrs, Ev, events, get, length, local}; +use super::{Align, Arc, Columns, Fmt, HashMap, Kind, Rgb, VAlign, model}; +use craft_flow::Spacing; +use craft_wordmodel::props::{Border, BorderStyle, Borders, LineSpacing, TabAlign, TabLeader, TabStop, Underline}; +use craft_wordmodel::section::NumFormat; + +// --------------------------------------------------------------------------------------------- +// Import + +#[derive(Clone, Debug, Default)] +struct FmtDelta { + bold: Option, + italic: Option, + underline: Option, + strike: Option, + sup: Option, + sub: Option, + color: Option, + background: Option, + size: Option, + /// A size in percent of the inherited one (`fo:font-size="130%"`). + size_pct: Option, + font: Option>, + code: Option, + /// What [`Fmt::extra`] holds, as set here (unset fields leave the inherited ones). + extra: craft_wordmodel::props::CharProps, + /// `style:text-position` raising or lowering text at full size: percent of its height. + raise: Option, +} + +impl FmtDelta { + /// Whether the properties read so far underline the text. + fn fmt_underline_on(&self) -> bool { + self.underline == Some(true) + } + + fn apply(&self, f: &mut Fmt) { + macro_rules! set { + ($($k:ident),*) => { $( if let Some(v) = self.$k.clone() { f.$k = v; } )* }; + } + set!(bold, italic, underline, strike, sup, sub, code); + // What this turns off stays said on the text, or the style of its paragraph or span would + // show through; turned back on, it no longer is. + if self.bold.is_some() || self.italic.is_some() || self.underline.is_some() || self.strike.is_some() { + use craft_wordmodel::props::Underline; + let mut x = f.extra.take().map(|b| *b).unwrap_or_default(); + if self.strike == Some(false) { + x.double_strike = None; + } + for (state, said) in [(self.bold, &mut x.bold), (self.italic, &mut x.italic), (self.strike, &mut x.strike)] { + if let Some(on) = state { + *said = (!on).then_some(false); + } + } + match self.underline { + Some(false) => x.underline = Some(Underline::None), + Some(true) if x.underline == Some(Underline::None) => x.underline = None, + _ => {} + } + f.extra = (x != craft_wordmodel::props::CharProps::default()).then(|| Box::new(x)); + } + if self.color.is_some() { + f.color = self.color; + } + if self.background.is_some() { + f.background = self.background; + } + if self.size.is_some() { + f.size = self.size; + } + if let Some(p) = self.size_pct { + f.size = Some((f.size.unwrap_or(12.0) * p / 100.0).clamp(1.0, 1638.0)); + } + if let Some(fam) = &self.font { + f.font = Some(fam.clone()); + } + if self.extra != craft_wordmodel::props::CharProps::default() || self.raise.is_some() { + let mut x = f.extra.take().map(|b| *b).unwrap_or_default(); + x.overlay(&self.extra); + if let Some(r) = self.raise { + let size = f.size.unwrap_or(12.0); + x.position = Some((r / 100.0 * size).clamp(-1584.0, 1584.0)).filter(|p| *p != 0.0); + } + // An explicit "off" only cancels what a parent style set (capitals are kept: a + // paragraph's or span's style may set them too). + if self.strike == Some(false) { + x.double_strike = None; + } + for b in [&mut x.double_strike, &mut x.hidden, &mut x.outline, &mut x.shadow, &mut x.emboss, &mut x.engrave] { + if *b == Some(false) { + *b = None; + } + } + x.spacing = x.spacing.filter(|s| *s != 0.0); + x.scale = x.scale.filter(|s| *s != 100.0); + f.extra = (x != craft_wordmodel::props::CharProps::default()).then(|| Box::new(x)); + } + } +} + +/// The underline `style:text-underline-style`, `-type`, `-width` and `-mode` describe (ODF 1.3 +/// §20.390–20.393), when underlined. +fn underline_kind(a: &Attrs) -> Underline { + let style = get(a, "style:text-underline-style").map(str::trim).unwrap_or("solid"); + let double = get(a, "style:text-underline-type").is_some_and(|t| t.trim() == "double"); + let bold = get(a, "style:text-underline-width").is_some_and(|w| matches!(w.trim(), "bold" | "thick" | "medium")); + let words = get(a, "style:text-underline-mode").is_some_and(|m| m.trim() == "skip-white-space"); + match style { + "dotted" => Underline::Dotted, + "dash" | "long-dash" => Underline::Dash, + "dot-dash" => Underline::DotDash, + "dot-dot-dash" => Underline::DotDotDash, + "wave" if double => Underline::DoubleWave, + "wave" => Underline::Wave, + _ if double => Underline::Double, + _ if bold => Underline::Thick, + _ if words => Underline::Words, + _ => Underline::Single, + } +} + +/// The rest of a `style:text-properties` element: what only [`Fmt::extra`] holds. +/// The Asian and complex-script fonts, complex-script size, weight and posture and both +/// languages of a `style:text-properties` element (ODF 1.3 §20.2xx `-asian`/`-complex` +/// attributes), into `x`. +fn script_properties(a: &Attrs, x: &mut craft_wordmodel::props::CharProps, faces: &HashMap) { + let family = |fam: &str, name: &str| { + let v = get(a, fam).or_else(|| get(a, name).map(|n| faces.get(n).map_or(n, String::as_str)))?; + let v = v.split(',').next().unwrap_or("").trim().trim_matches(['\'', '"']); + (!v.is_empty() && v.len() <= model::MAX_FONT_NAME).then(|| v.to_string()) + }; + if let Some(f) = family("style:font-family-asian", "style:font-name-asian") { + x.font_ea = Some(f); + } + if let Some(f) = family("style:font-family-complex", "style:font-name-complex") { + x.font_cs = Some(f); + } + if let Some(s) = get(a, "style:font-size-complex").and_then(length).filter(|s| (1.0..=1638.0).contains(s)) { + x.size_cs = Some(s); + } + if let Some(w) = get(a, "style:font-weight-complex") { + let w = w.trim(); + x.bold_cs = Some(w.eq_ignore_ascii_case("bold") || w.parse::().is_ok_and(|n| n >= 600)); + } + if let Some(s) = get(a, "style:font-style-complex") { + x.italic_cs = Some(matches!(s.trim(), "italic" | "oblique")); + } + if let Some(l) = language_tag(a, "style:language-asian", "style:script-asian", "style:country-asian") { + x.lang_ea = Some(l); + } + if let Some(l) = language_tag(a, "style:language-complex", "style:script-complex", "style:country-complex") { + x.lang_bidi = Some(l); + } +} + +fn extra_text_properties(a: &Attrs, d: &mut FmtDelta) { + let underlined = d.fmt_underline_on(); + let x = &mut d.extra; + if underlined { + x.underline = Some(underline_kind(a)); + if let Some(c) = get(a, "style:text-underline-color").filter(|c| c.trim() != "font-color").and_then(model::parse_color) { + x.underline_color = Some(c); + } + } + if let Some(t) = get(a, "style:text-line-through-type") { + x.double_strike = Some(t.trim() == "double"); + } + // A border around the text (LibreOffice's `loext:border`, or its sides): Word's text border, + // one line all round (the first side with a line), its padding the border's distance. + let side = ["loext:border", "loext:border-top", "loext:border-left", "loext:border-bottom", "loext:border-right"] + .into_iter() + .find_map(|k| get(a, k).and_then(parse_border).filter(|b| b.is_visible())); + if let Some(mut b) = side { + let pad = ["loext:padding", "loext:padding-top", "loext:padding-left"].into_iter().find_map(|k| get(a, k).and_then(length)); + b.space = pad.filter(|p| p.is_finite()).map_or(0.0, |p| p.clamp(0.0, 31.0)); + x.border = Some(b); + } + if let Some(t) = get(a, "fo:text-transform") { + x.caps = Some(t.trim() == "uppercase"); + } + if let Some(v) = get(a, "fo:font-variant") { + x.small_caps = Some(v.trim() == "small-caps"); + } + if let Some(v) = get(a, "text:display") { + x.hidden = Some(v.trim() == "none"); + } + // Pair kerning (§20.317): on as Word's at every size, or off. + if let Some(v) = get(a, "style:letter-kerning") { + x.kern = Some(if v.trim() == "true" { 1.0 } else { 0.0 }); + } + if let Some(v) = get(a, "fo:letter-spacing") { + x.spacing = Some(if v.trim() == "normal" { 0.0 } else { length(v).unwrap_or(0.0).clamp(-1584.0, 1584.0) }); + } + if let Some(v) = get(a, "style:text-scale") + .and_then(|v| v.trim().strip_suffix('%')) + .and_then(|v| v.trim().parse::().ok()) + .filter(|v| v.is_finite() && *v > 0.0) + { + x.scale = Some(v.clamp(1.0, 600.0)); + } + // Emphasis marks (§20.370 `style:text-emphasize`): Word's dot, comma (an accent), circle + // (a disc is a dot) and dot below. + if let Some(v) = get(a, "style:text-emphasize") { + use craft_wordmodel::props::EmphasisMark as E; + let mut words = v.split_whitespace(); + let below = v.split_whitespace().nth(1) == Some("below"); + x.emphasis = Some(match words.next() { + Some("dot" | "disc") if below => E::UnderDot, + Some("dot" | "disc") => E::Dot, + Some("accent") => E::Comma, + Some("circle") => E::Circle, + _ => E::None, + }); + } + if let Some(v) = get(a, "style:text-outline") { + x.outline = Some(v.trim() == "true"); + } + if let Some(v) = get(a, "fo:text-shadow") { + x.shadow = Some(v.trim() != "none"); + } + if let Some(v) = get(a, "style:font-relief") { + (x.emboss, x.engrave) = (Some(v.trim() == "embossed"), Some(v.trim() == "engraved")); + } + if let Some(l) = language(a) { + x.lang = Some(l); + } +} + +/// The language `fo:language`, `fo:script` and `fo:country` state: see [`language_tag`]. +fn language(a: &Attrs) -> Option { + language_tag(a, "fo:language", "fo:script", "fo:country") +} + +/// The language the attributes `language`, `script` and `country` state, as a tag (`de-DE`, +/// `sr-Latn-RS`); `None` when they state none, or no language at all (`none`, `zxx`). A script of +/// any other shape than a script's is left out: written back, it would read as a country. +fn language_tag(a: &Attrs, language: &str, script: &str, country: &str) -> Option { + fn tag(v: Option<&str>) -> Option<&str> { + v.map(str::trim).filter(|v| (1..=8).contains(&v.len()) && v.chars().all(|c| c.is_ascii_alphanumeric()) && *v != "none") + } + let language = tag(get(a, language)).filter(|l| *l != "zxx")?; + let script = tag(get(a, script)).filter(|s| super::writer::is_script(s)); + Some([Some(language), script, tag(get(a, country))].into_iter().flatten().collect::>().join("-")) +} + +#[derive(Clone, Debug, Default)] +struct StyleInfo { + parent: Option, + display: Option, + auto: bool, + fmt: FmtDelta, + align: Option, + page_break: bool, + /// `fo:break-before="column"`: its paragraphs start a column. + column_break: bool, + /// `fo:keep-with-next`: `Some(true)` for "always", `Some(false)` for "auto". + keep_next: Option, + outline: Option, + border_bottom: bool, + /// `fo:background-color` of a table-cell style's `style:table-cell-properties`. + cell_background: Option>, + /// A table-cell style's background picture (§17.3 `style:background-image`): its package + /// path, `style:repeat` and `style:position`. + cell_picture: Option<(String, String, String)>, + /// `style:vertical-align` of a table-cell style's `style:table-cell-properties`. + cell_valign: Option, + cell_direction: Option, + /// `fo:border*` of a table-cell style: each side as stated (`none` an explicit no line). + cell_borders: Option, + /// `fo:padding*` of a table-cell style (top, left, bottom, right), each as stated. + cell_padding: [Option; 4], + /// `style:column-width` of a table-column style's `style:table-column-properties`, in points. + column_width: Option, + /// `style:rel-column-width` of the same: its share of the table (`1*`). + column_share: Option, + /// A table style's `style:table-properties`: `table:align`, and its width (`style:width`, + /// points, or `style:rel-width`, percent of the text width). + table_align: Option, + table_width: Option, + table_rel_width: Option, + /// A table style's own space above and below (`fo:margin-top`, `fo:margin-bottom`), points. + table_space: [Option; 2], + /// A table-row style's `style:table-row-properties`: its height (`style:row-height`, exact) + /// or least height (`style:min-row-height`), points. + row_height: Option<(f32, bool)>, + /// `fo:background-color` of a table or table-row style (§20.175): what shows behind cells + /// with none of their own. `Some(None)`: `transparent`, which clears a parent style's. + table_background: Option>, + /// `fo:keep-together` of a table-row style: the row doesn't break across pages. + row_keep: Option, + /// `style:master-page-name` of a paragraph or table style: its paragraphs (tables) start a + /// page of that master page. Empty: explicitly none, which stops the lookup up the chain. + master_page: Option, + /// `style:page-number` of a paragraph or table style (applied only with a master page): `Some(Some(n))` restarts page numbering at n, + /// `Some(None)` ("auto") continues it. + page_number: Option>, + /// `style:columns` of a section style. + columns: Option, + /// The section style's `style:column`s, as [`with_widths`] gives them widths. + column_specs: Vec, + /// `text:dont-balance-text-columns` of `style:section-properties`, as stated. + dont_balance: Option, + /// Margins, first-line indent and line spacing of `style:paragraph-properties`. + spacing: Spacing, + /// `style:family` (paragraph, text, table…). + family: Option, + /// Declared among automatic styles (of either file): never a custom style. + declared_auto: bool, + /// `style:contextual-spacing` of a paragraph style. + contextual: Option, + /// `style:writing-mode` of a paragraph style: right to left or not. + rtl: Option, + /// A drop cap (§17.9 `style:drop-cap`): lines it spans and its distance from the text. + drop_cap: Option<(u8, f32)>, + /// How its lines break across pages: `fo:keep-together` (`always`: kept together), + /// `fo:widows`/`fo:orphans` (widow and orphan control on with any, off with none), and + /// `fo:break-after` (`page`: the next paragraph starts a page). + keep_lines: Option, + widow_control: Option, + break_after: bool, + /// `fo:hyphenate` of its text properties: its words may be hyphenated. + hyphenate: Option, + /// `fo:border*` of a paragraph style, each side's padding as its distance from the text. + para_borders: Option, + /// `fo:background-color` of a paragraph style (`Some(None)`: transparent, stated). + para_shading: Option>, + /// `style:graphic-properties` of a graphic style: a frame's placement. + graphic: Option, + /// `style:tab-stops`: the style's whole set (it replaces the parent's, §17.7). + tabs: Option>, +} + +/// Most tab stops a paragraph style keeps (Word's limit). +const MAX_TABS: usize = 64; + +/// A `style:tab-stop` (ODF 1.3 §17.8), its position from where the style measures; `None` when +/// it has no usable position. +fn tab_stop(a: &Attrs) -> Option { + let pos = get(a, "style:position").and_then(length).filter(|p| p.is_finite())?.clamp(-MAX_LEN, MAX_LEN); + let align = match get(a, "style:type").map(str::trim) { + Some("center") => TabAlign::Center, + Some("right") => TabAlign::Right, + Some("char") => TabAlign::Decimal, + _ => TabAlign::Left, + }; + let style = get(a, "style:leader-style").map(str::trim).unwrap_or("none"); + let leader = if style == "none" { + TabLeader::None + } else { + match get(a, "style:leader-text").and_then(|t| t.chars().next()) { + Some('.') => TabLeader::Dot, + Some('-') => TabLeader::Hyphen, + Some('_') => TabLeader::Underscore, + Some('·') => TabLeader::MiddleDot, + _ if style == "dotted" => TabLeader::Dot, + _ if matches!(style, "dash" | "long-dash") => TabLeader::Hyphen, + _ => TabLeader::Underscore, + } + }; + Some(TabStop { pos, align, leader }) +} + +/// Paragraph spacing, indents and line height of a `style:paragraph-properties` element (ODF 1.3 +/// §20.204–209, §20.225, §20.317–318) over `sp`. Percentages of the parent's margins aren't +/// followed (left as they were); spacing is clamped to 0..=1584 pt, indents to ±1584 pt. +fn paragraph_spacing(a: &Attrs, sp: &mut Spacing) { + let len = |k: &str| get(a, k).and_then(length); + let space = |k: &str| len(k).map(|v| v.clamp(0.0, MAX_LEN)); + let indent = |k: &str| len(k).map(|v| v.clamp(-MAX_LEN, MAX_LEN)); + if let Some(v) = get(a, "fo:margin").and_then(length) { + sp.before = Some(v.clamp(0.0, MAX_LEN)); + sp.after = sp.before; + sp.left = Some(v.clamp(-MAX_LEN, MAX_LEN)); + sp.right = sp.left; + } + sp.before = space("fo:margin-top").or(sp.before); + sp.after = space("fo:margin-bottom").or(sp.after); + sp.left = indent("fo:margin-left").or(sp.left); + sp.right = indent("fo:margin-right").or(sp.right); + sp.first = indent("fo:text-indent").or(sp.first); + if let Some(v) = get(a, "fo:line-height").map(str::trim) { + if v.eq_ignore_ascii_case("normal") { + sp.line = Some(LineSpacing::Multiple(1.0)); + } else if let Some(pc) = v.strip_suffix('%').and_then(|n| n.trim().parse::().ok()).filter(|n| n.is_finite() && *n > 0.0) { + // Word's range: a sixteenth of a line up to 132 lines. + sp.line = Some(LineSpacing::Multiple((pc / 100.0).clamp(0.06, 132.0))); + } else if let Some(h) = length(v).filter(|h| *h > 0.0) { + sp.line = Some(LineSpacing::Exactly(h.clamp(0.7, MAX_LEN))); + } + } + if let Some(h) = len("style:line-height-at-least").filter(|h| *h >= 0.0) { + sp.line = Some(LineSpacing::AtLeast(h.clamp(0.7, MAX_LEN))); + } + // Leading (§20.336: extra space between lines, above each line as LibreOffice draws it) has + // no WordCraft mode: at least a 12 pt line and the leading, which also puts the extra space + // above the text. + // LibreOffice also leaves the leading after the last line: space after the paragraph. + if let Some(l) = len("style:line-spacing").filter(|l| *l >= 0.0) { + let l = l.min(MAX_LEN); + sp.line = Some(LineSpacing::AtLeast((LEADING_LINE + l).clamp(0.7, MAX_LEN))); + sp.after = Some((sp.after.unwrap_or(0.0) + l).clamp(0.0, MAX_LEN)); + } +} + +/// Largest spacing or indent read, in points (22 inches, Word's page limit). +const MAX_LEN: f32 = 1584.0; + +/// Height of the single line `style:line-spacing` leading is added to (12 pt text). +const LEADING_LINE: f32 = 14.0; + +/// `over` laid on `base`: what `over` sets wins. +fn overlay_spacing(base: &mut Spacing, over: &Spacing) { + base.before = over.before.or(base.before); + base.after = over.after.or(base.after); + base.line = over.line.or(base.line); + base.left = over.left.or(base.left); + base.right = over.right.or(base.right); + base.first = over.first.or(base.first); +} + +/// The paragraph kind a style name (lower case, `_20_` as spaces) stands for. +fn kind_named(n: &str) -> Option { + if let Some(d) = n.strip_prefix("heading ").and_then(|r| r.trim().parse::().ok()) { + Some(Kind::Heading(d.clamp(1, 6))) + } else { + match n { + "title" => Some(Kind::Title), + "quotations" | "quote" => Some(Kind::Quote), + "preformatted text" | "source text" | "code" => Some(Kind::Code), + _ => None, + } + } +} + +/// How a paragraph style looks ([`Styles::para`]). +#[derive(Clone, Debug, Default)] +pub(super) struct ParaLook { + pub(super) kind: Kind, + pub(super) align: Option, + pub(super) page_break: bool, + /// It starts a column (`fo:break-before="column"`). + pub(super) column_break: bool, + pub(super) rule: bool, + pub(super) keep_next: bool, + pub(super) fmt: Fmt, + pub(super) spacing: Spacing, + /// Tab stops, from where the document measures them. + pub(super) tabs: Option>, + /// No spacing between paragraphs of one style (`style:contextual-spacing`). + pub(super) contextual: Option, + /// Right to left (`style:writing-mode`). + pub(super) rtl: Option, + /// A drop cap: lines it spans (2–10) and its distance from the text, points. + pub(super) drop_cap: Option<(u8, f32)>, + /// Lines kept together, widow and orphan control, a page break after it, hyphenation. + pub(super) keep_lines: Option, + pub(super) widow_control: Option, + pub(super) break_after: bool, + pub(super) hyphenate: Option, + /// Borders (when a side shows) and background. + pub(super) borders: Option, + pub(super) shading: Option, + /// The custom style (a common style other than Standard and the kinds' styles) the + /// paragraph is in, by its ODF name; the formatting above is then only what the automatic + /// styles over it add. + pub(super) named: Option, + /// The paragraph's left indent over its whole style chain, points: where tab stops are + /// measured from when they are measured from the indent. + pub(super) tab_origin: f32, +} + +/// Where a custom style's formatting comes from besides its own ([`NamedStyle::parent`]). +#[derive(Clone, Debug, PartialEq)] +pub(super) enum Parent { + /// The document defaults (Standard, or no parent). + Normal, + /// The style of a paragraph kind. + Kind(Kind), + /// Another custom style, by ODF name. + Named(String), +} + +/// A custom paragraph style as a document style: its name, parent and own formatting. +#[derive(Clone, Debug)] +pub(super) struct NamedStyle { + pub(super) name: String, + pub(super) parent: Parent, + pub(super) look: ParaLook, + /// What it turns off, which a style it is based on may turn on: bold, italic, underline, + /// strike-through (`fo:font-weight="normal"`…). + pub(super) off: [bool; 4], + /// Its outline level, 1–10 (`style:default-outline-level`). + pub(super) outline: Option, +} + +/// Reads date and time styles (`number:date-style`, `number:time-style`) as Word pictures. +#[derive(Default)] +struct DateStyles { + /// The style being read: its name and parts. + data: Option<(String, Vec)>, + /// It orders its parts as the locale does (§19.341 `number:automatic-order`). + auto_order: bool, + /// In its `number:text`. + in_text: bool, +} + +impl DateStyles { + /// Event `ev` of a style file, into `out` (name → picture); false when it isn't one of a + /// date or time style. + fn event(&mut self, ev: &Ev, out: &mut HashMap) -> bool { + use super::datastyle::{Part, part, picture}; + match ev { + Ev::Start(q, a) => match local(q) { + "date-style" | "time-style" => { + self.data = get(a, "style:name").map(|n| (n.to_string(), Vec::new())); + self.auto_order = local(q) == "date-style" && get(a, "number:automatic-order").is_some_and(|v| v.trim() == "true"); + } + "text" => self.in_text = self.data.is_some(), + name => { + let on = |k: &str, v: &str| get(a, k).is_some_and(|x| x.trim() == v); + if let (Some((_, parts)), Some(p)) = (&mut self.data, part(name, on("number:style", "long"), on("number:textual", "true"))) + && parts.len() < 64 + { + parts.push(p); + } + } + }, + Ev::End(q) => match local(q) { + "text" => self.in_text = false, + "date-style" | "time-style" => { + if let Some((name, parts)) = self.data.take() + && out.len() < MAX_DATA_STYLES + { + let parts = if std::mem::take(&mut self.auto_order) { super::datastyle::auto_ordered(&parts) } else { parts }; + out.insert(name, picture(&parts)); + } + } + _ => {} + }, + Ev::Text(t) => { + if !self.in_text { + return false; + } + if let Some((_, parts)) = &mut self.data { + let n = parts.len(); + if let Some(Part::Text(s)) = parts.last_mut() + && s.len() < 200 + { + s.push_str(t); + } else if n < 64 { + parts.push(Part::Text(t.chars().take(200).collect())); + } + } + } + } + true + } +} + +/// Most date and time styles kept (hostile files). +const MAX_DATA_STYLES: usize = 10_000; + +/// Longest style id made, in characters. +const MAX_STYLE_ID: usize = 64; + +/// The document style id for an ODF style: its name's letters and digits (LibreOffice's +/// `Text_20_body` is `Textbody`), at most [`MAX_STYLE_ID`] of them. +pub(super) fn style_id(odt: &str) -> String { + // LibreOffice's index styles are WordCraft's TOC styles. + if let Some((id, _)) = super::writer::generating_style(odt) { + return id; + } + let id: String = odt.replace("_20_", " ").chars().filter(|c| c.is_alphanumeric()).take(MAX_STYLE_ID).collect(); + if id.is_empty() { "Custom".into() } else { id } +} + +/// An ODF border (`0.5pt solid #000000`, `none`; ODF 1.3 §20.183): width, style and colour in any +/// order. `none` and `hidden` are an explicit no line; `None` when nothing in it reads. +pub(super) fn parse_border(v: &str) -> Option { + let mut b = Border { style: BorderStyle::Single, width: 0.75, color: None, space: 0.0 }; + let mut any = false; + for t in v.split_whitespace() { + let t = t.to_ascii_lowercase(); + match t.as_str() { + "none" | "hidden" => return Some(Border { style: BorderStyle::None, width: 0.0, color: None, space: 0.0 }), + "solid" | "groove" | "ridge" | "inset" | "outset" => b.style = BorderStyle::Single, + "double" => b.style = BorderStyle::Double, + "dotted" => b.style = BorderStyle::Dotted, + "dashed" => b.style = BorderStyle::Dashed, + "dot-dash" | "dot-dot-dash" => b.style = BorderStyle::DotDash, + "wave" => b.style = BorderStyle::Wave, + "thin" => b.width = 0.5, + "medium" => b.width = 1.0, + "thick" => b.width = 2.0, + _ => { + if let Some(c) = model::parse_color(&t) { + b.color = Some(c); + } else if let Some(w) = length(&t).filter(|w| w.is_finite()) { + b.width = w.clamp(0.0, 12.0); + } else { + continue; + } + } + } + any = true; + } + any.then_some(b) +} + +/// The properties element a table column's or row's `style:default-style` (of `family`) holds. +fn table_default_props(family: Option<&str>) -> Option<&'static str> { + match family { + Some("table-column") => Some("table-column-properties"), + Some("table-row") => Some("table-row-properties"), + _ => None, + } +} + +/// A table column's share of its table (`style:rel-column-width`, `1*`). +fn column_share_of(a: &Attrs) -> Option { + get(a, "style:rel-column-width").and_then(|v| v.trim().strip_suffix('*')?.trim().parse::().ok()).filter(|r| r.is_finite() && *r > 0.0) +} + +/// A property element `name` (`q` in full) of the style being read, into `s`: true when it is +/// one (paragraph, table, column, cell, section and graphic properties, tab stops). +/// What `fo:background-color` says of a table or row: a colour, or none (`transparent`). +fn table_background_of(a: &Attrs) -> Option> { + let bg = get(a, "fo:background-color")?; + if bg.trim() == "transparent" { Some(None) } else { model::parse_color(bg).map(Some) } +} + +fn style_child(name: &str, q: &str, a: &Attrs, s: &mut StyleInfo) -> bool { + match name { + "paragraph-properties" => paragraph_properties(a, s), + "tab-stops" => s.tabs = Some(Vec::new()), + // One line drops nothing. + "drop-cap" => { + let lines = get(a, "style:lines").and_then(|l| l.trim().parse::().ok()).unwrap_or(1); + let distance = get(a, "style:distance").and_then(length).filter(|d| d.is_finite()).map_or(0.0, |d| d.clamp(0.0, 1584.0)); + s.drop_cap = Some(((lines.min(10) as u8), distance)).filter(|(l, _)| *l > 1); + } + "tab-stop" => { + if let Some(t) = s.tabs.as_mut().filter(|t| t.len() < MAX_TABS) + && let Some(stop) = tab_stop(a) + { + t.push(stop); + } + } + "table-properties" => { + // A table starting a page (§20.191 `fo:break-before`). + if get(a, "fo:break-before").is_some_and(|b| b.trim().eq_ignore_ascii_case("page")) { + s.page_break = true; + } + if let Some(v) = get(a, "style:page-number") { + s.page_number = Some(v.trim().parse::().ok().map(|n| n.min(32_767))); + } + // §20.300 `table:align`: `margins` fills the text width, as no alignment does. + s.table_background = table_background_of(a).or(s.table_background); + s.table_align = match get(a, "table:align").map(str::trim) { + Some("left") => Some(Align::Left), + Some("center") => Some(Align::Center), + Some("right") => Some(Align::Right), + _ => s.table_align, + }; + if let Some(w) = get(a, "style:width").and_then(length).filter(|w| w.is_finite() && *w > 0.0) { + s.table_width = Some(w.min(MAX_LEN)); + } + if let Some(p) = + get(a, "style:rel-width").and_then(|v| v.trim().strip_suffix('%')?.trim().parse::().ok()).filter(|p| p.is_finite() && *p > 0.0) + { + s.table_rel_width = Some(p.min(100.0)); + } + for (i, k) in ["fo:margin-top", "fo:margin-bottom"].iter().enumerate() { + if let (Some(v), Some(slot)) = (get(a, k).and_then(length).filter(|v| v.is_finite()), s.table_space.get_mut(i)) { + *slot = Some(v.clamp(0.0, MAX_LEN)); + } + } + } + "table-row-properties" => { + s.table_background = table_background_of(a).or(s.table_background); + if let Some(h) = get(a, "style:row-height").and_then(length).filter(|h| h.is_finite() && *h > 0.0) { + s.row_height = Some((h.min(MAX_LEN), true)); + } else if let Some(h) = get(a, "style:min-row-height").and_then(length).filter(|h| h.is_finite() && *h > 0.0) { + s.row_height = Some((h.min(MAX_LEN), false)); + } + if let Some(k) = get(a, "fo:keep-together") { + s.row_keep = Some(k.trim() == "always"); + } + } + "table-column-properties" => { + if let Some(w) = get(a, "style:column-width").and_then(length) { + s.column_width = Some(w); + } + if let Some(r) = column_share_of(a) { + s.column_share = Some(r); + } + } + "table-cell-properties" => table_cell_properties(a, s), + "section-properties" => { + s.dont_balance = get(a, "text:dont-balance-text-columns").map(|v| v.trim() == "true"); + } + "columns" if q.starts_with("style:") => { + s.columns = Some(columns_of(a)); + } + "column" if q.starts_with("style:") && s.columns.is_some() => { + if let Some(spec) = column_spec(a).filter(|_| s.column_specs.len() < MAX_COLUMN_SPECS) { + s.column_specs.push(spec); + } + } + "column-sep" => { + if let Some(c) = &mut s.columns { + c.separator = column_sep(a); + } + } + "graphic-properties" => s.graphic = Some(super::frames::Graphic::read(a)), + "background-image" if q.starts_with("style:") && s.family.as_deref() == Some("table-cell") => { + if let Some(href) = get(a, "xlink:href").filter(|h| !h.trim().is_empty() && h.len() < 1024) { + let attr = |k: &str| get(a, k).unwrap_or_default().trim().to_string(); + s.cell_picture = Some((href.trim().to_string(), attr("style:repeat"), attr("style:position"))); + } + } + _ => return false, + } + true +} + +/// The text direction a `style:writing-mode` names: top to bottom turned down, bottom to top +/// turned up, the rest across. +pub(super) fn writing_direction(m: &str) -> craft_wordmodel::props::TextDirection { + use craft_wordmodel::props::TextDirection; + match m.trim() { + "tb-rl" | "tb-lr" | "tb" => TextDirection::Down, + "bt-lr" => TextDirection::Up, + _ => TextDirection::Horizontal, + } +} + +/// A table-cell style's `style:table-cell-properties` (ODF 1.3 §17.18): background, borders, +/// padding and vertical alignment. +fn table_cell_properties(a: &Attrs, s: &mut StyleInfo) { + if let Some(c) = table_background_of(a) { + s.cell_background = Some(c); + } + cell_borders_and_padding(a, s); + // Which way its text runs (§20.404 `style:writing-mode`; LibreOffice's upward `bt-lr` is + // its own `loext:writing-mode`). + if let Some(m) = get(a, "loext:writing-mode").or_else(|| get(a, "style:writing-mode")) { + s.cell_direction = Some(writing_direction(m)); + } + if let Some(va) = get(a, "style:vertical-align") { + s.cell_valign = match va.trim() { + "top" => Some(VAlign::Top), + "middle" | "center" => Some(VAlign::Center), + "bottom" => Some(VAlign::Bottom), + _ => None, + }; + } +} + +/// A table-cell style's `fo:border*` and `fo:padding*` (a side's own value wins over the +/// shorthand) into `s`. +fn cell_borders_and_padding(a: &Attrs, s: &mut StyleInfo) { + let all = get(a, "fo:border").and_then(parse_border); + let side = |k: &str| get(a, k).and_then(parse_border).or(all); + let (top, left, bottom, right) = (side("fo:border-top"), side("fo:border-left"), side("fo:border-bottom"), side("fo:border-right")); + if top.is_some() || left.is_some() || bottom.is_some() || right.is_some() { + s.cell_borders = Some(Borders { top, left, bottom, right, ..Default::default() }); + } + let pad = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite()).map(|v| v.clamp(0.0, MAX_LEN)); + let every = pad("fo:padding"); + let sides = [pad("fo:padding-top"), pad("fo:padding-left"), pad("fo:padding-bottom"), pad("fo:padding-right")]; + if every.is_some() || sides.iter().any(Option::is_some) { + s.cell_padding = sides.map(|v| v.or(every)); + } +} + +/// Columns of a `style:columns` element (ODF 1.3 §17.12): count and gap, clamped as the RTF and +/// DOCX readers do (each column's own width comes from its `style:column`, see [`with_widths`]). +/// Without `fo:column-gap`, whose default the standard doesn't give (§19.233), the gap is +/// [`Columns::default`]'s half inch, Word's. +pub(super) fn columns_of(a: &Attrs) -> Columns { + let mut c = Columns::default(); + if let Some(n) = get(a, "fo:column-count").and_then(|v| v.trim().parse::().ok()) { + c.count = n.clamp(1, 45) as u32; + } + if let Some(g) = get(a, "fo:column-gap").and_then(length).filter(|v| *v >= 0.0) { + c.space = g.min(1584.0); + } + c +} + +/// A `style:column` (ODF 1.3 §17.12): its relative width (`style:rel-width`, a share of the +/// columns' total) and its space before and after (`fo:start-indent`, `fo:end-indent`), points. +pub(super) type ColumnSpec = (f32, f32, f32); + +/// Most `style:column`s read for one set of columns (as many as Word's 45 columns). +pub(super) const MAX_COLUMN_SPECS: usize = 45; + +pub(super) fn column_spec(a: &Attrs) -> Option { + let rel = get(a, "style:rel-width")?.trim().trim_end_matches('*').trim().parse::().ok().filter(|v| v.is_finite() && *v > 0.0)?; + let len = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite()).map_or(0.0, |v| v.clamp(0.0, 1584.0)); + Some((rel.min(1.0e9), len("fo:start-indent"), len("fo:end-indent"))) +} + +/// `c` with the widths its `style:column`s give, in a text `tw` points wide: each column's +/// share of the width less its indents, and the space after it its end indent with the next +/// column's start indent. Equal columns keep equal widths and take the gap between them. +pub(super) fn with_widths(mut c: Columns, specs: &[ColumnSpec], tw: f32) -> Columns { + let total: f32 = specs.iter().map(|s| s.0).sum(); + if c.count < 2 || specs.len() != c.count as usize || !(total.is_finite() && total > 0.0) || !(tw.is_finite() && tw > 0.0) { + return c; + } + let boxes: Vec<(f32, f32)> = specs + .iter() + .enumerate() + .map(|(i, (rel, start, end))| { + let next_start = specs.get(i + 1).map_or(0.0, |s| s.1); + let after = if i + 1 < specs.len() { end + next_start } else { 0.0 }; + ((rel / total * tw - start - end).max(18.0), after) + }) + .collect(); + let (w0, gap) = boxes.first().copied().unwrap_or_default(); + let equal = boxes.iter().all(|(w, _)| (w - w0).abs() < 0.5) && boxes.iter().rev().skip(1).all(|(_, a)| (a - gap).abs() < 0.5); + if equal { + c.space = gap; + c.widths.clear(); + } else { + c.widths = boxes; + } + c +} + +/// A `style:column-sep` (ODF 1.3 §17.13) draws a line between columns unless its style is "none". +pub(super) fn column_sep(a: &Attrs) -> bool { + get(a, "style:style") != Some("none") +} + +/// The font family `a` names: `fo:font-family`, else the family of the font face `style:font-name` +/// names (ODF 1.3 §20.194, §16.23), else that name itself. +fn family_of<'a>(a: &'a Attrs, faces: &'a HashMap) -> Option<&'a str> { + get(a, "fo:font-family").or_else(|| get(a, "style:font-name").map(|n| faces.get(n).map_or(n, String::as_str))) +} + +fn default_font(a: &Attrs, faces: &HashMap) -> Option { + let family = family_of(a, faces)?; + let family = family.split(',').next().unwrap_or("").trim().trim_matches(['\'', '"']); + (!family.is_empty() && !model::is_mono(family)).then(|| family.to_string()) +} + +fn text_properties(a: &Attrs, style: &mut StyleInfo, faces: &HashMap) { + let fmt = &mut style.fmt; + let is_bold = |weight: &str| { + let weight = weight.trim(); + weight.eq_ignore_ascii_case("bold") || weight.parse::().is_ok_and(|n| n >= 600) + }; + // Latin attributes take precedence; Asian and complex-script attributes are fallbacks, to turn + // it on: one of those saying normal says nothing of Latin text. + if let Some(weight) = get(a, "fo:font-weight") { + fmt.bold = Some(is_bold(weight)); + } else if get(a, "style:font-weight-asian").or_else(|| get(a, "style:font-weight-complex")).is_some_and(is_bold) { + fmt.bold = Some(true); + } + let is_italic = |value: &str| value.trim().eq_ignore_ascii_case("italic") || value.trim().eq_ignore_ascii_case("oblique"); + if let Some(value) = get(a, "fo:font-style") { + fmt.italic = Some(is_italic(value)); + } else if get(a, "style:font-style-asian").or_else(|| get(a, "style:font-style-complex")).is_some_and(is_italic) { + fmt.italic = Some(true); + } + let is_not_none = |value: &str| !value.trim().eq_ignore_ascii_case("none"); + if let Some(value) = get(a, "style:text-underline-style").or_else(|| get(a, "style:text-underline-type")) { + fmt.underline = Some(is_not_none(value)); + } + if let Some(value) = get(a, "style:text-line-through-style").or_else(|| get(a, "style:text-line-through-type")) { + fmt.strike = Some(is_not_none(value)); + } + if let Some(value) = get(a, "style:text-position") { + // `super`/`sub` or a percentage raising the text, then its size (§20.374): raised text + // at full size is a position, smaller text superscript or subscript. + let mut parts = value.split_whitespace(); + let first = parts.next().unwrap_or(""); + let pct = first.trim_end_matches('%').parse::().ok().filter(|p| p.is_finite()).unwrap_or(0.0); + let full = parts.next().and_then(|s| s.trim_end_matches('%').parse::().ok()).is_some_and(|s| s >= 100.0); + let keyword = first.eq_ignore_ascii_case("super") || first.eq_ignore_ascii_case("sub"); + if full && !keyword { + fmt.sup = Some(false); + fmt.sub = Some(false); + fmt.raise = Some(pct.clamp(-10_000.0, 10_000.0)); + } else { + fmt.sup = Some(first.eq_ignore_ascii_case("super") || pct > 0.0); + fmt.sub = Some(first.eq_ignore_ascii_case("sub") || pct < 0.0); + fmt.raise = Some(0.0); + } + } + if let Some(color) = get(a, "fo:color").and_then(model::parse_color) { + fmt.color = Some(color); + } + if let Some(color) = get(a, "fo:background-color").and_then(model::parse_color) { + fmt.background = Some(color); + } + if let Some(pct) = get(a, "fo:font-size") + .and_then(|v| v.trim().strip_suffix('%')) + .and_then(|v| v.trim().parse::().ok()) + .filter(|p| p.is_finite() && *p > 0.0) + { + fmt.size_pct = Some(pct.min(10_000.0)); + } else if let Some(size) = get(a, "fo:font-size").and_then(length).filter(|size| (1.0..=1638.0).contains(size)) { + fmt.size = Some(size); + } + if let Some(family) = family_of(a, faces) { + let family = family.split(',').next().unwrap_or("").trim().trim_matches(['\'', '"']).to_string(); + // A monospaced font keeps its name (it is not "code" here, which would be Courier New). + if !family.is_empty() { + fmt.font = model::kept_font(&family); + fmt.code = Some(false); + } + } + extra_text_properties(a, &mut style.fmt); + script_properties(a, &mut style.fmt.extra, faces); +} + +/// A paragraph style's `style:paragraph-properties` (ODF 1.3 §17.6) into `s`. +fn paragraph_properties(a: &Attrs, s: &mut StyleInfo) { + s.align = match get(a, "fo:text-align").map(|v| v.trim().to_ascii_lowercase()).as_deref() { + Some("center") => Some(Align::Center), + Some("end") | Some("right") => Some(Align::Right), + Some("justify") => Some(Align::Justify), + Some("start") | Some("left") => Some(Align::Left), + _ => s.align, + }; + if get(a, "fo:break-before").is_some_and(|b| b.trim().eq_ignore_ascii_case("column")) { + s.column_break = true; + } + if get(a, "fo:break-before").is_some_and(|b| b.trim().eq_ignore_ascii_case("page")) { + s.page_break = true; + } + if get(a, "fo:break-after").is_some_and(|b| b.trim().eq_ignore_ascii_case("page")) { + s.break_after = true; + } + match get(a, "fo:keep-together").map(str::trim) { + Some("always") => s.keep_lines = Some(true), + Some("auto") => s.keep_lines = Some(false), + _ => {} + } + if let Some(w) = widows(a) { + s.widow_control = Some(w); + } + match get(a, "fo:keep-with-next") { + Some("always") => s.keep_next = Some(true), + Some("auto") => s.keep_next = Some(false), + _ => {} + } + if let Some(v) = get(a, "style:page-number") { + s.page_number = Some(v.trim().parse::().ok().map(|n| n.min(32_767))); + } + if get(a, "fo:border-bottom").is_some_and(|b| !b.trim().eq_ignore_ascii_case("none")) { + s.border_bottom = true; + } + paragraph_spacing(a, &mut s.spacing); + if let Some(b) = border_sides(a) { + s.para_borders = Some(b); + } + if let Some(c) = get(a, "style:contextual-spacing") { + s.contextual = Some(c.trim() == "true"); + } + // `page` (the page's direction) is left to the section. + match get(a, "style:writing-mode").map(str::trim) { + Some("rl-tb" | "rl" | "tb-rl") => s.rtl = Some(true), + Some("lr-tb" | "lr" | "tb-lr") => s.rtl = Some(false), + _ => {} + } + if let Some(bg) = get(a, "fo:background-color") { + s.para_shading = Some(if bg.trim() == "transparent" { None } else { model::parse_color(bg) }); + } +} + +/// The most padding a side without a line keeps, points (22 inches). +const MAX_PADDING: f32 = 1584.0; + +/// `fo:border`, `fo:border-top`… of `a` (a side's own value wins over the shorthand), each side's +/// `fo:padding*` as its distance from the text (Word's 31 pt at most beside a line); a side +/// with padding and no line is an invisible border holding it. `None` when none is stated. +fn border_sides(a: &Attrs) -> Option { + let all = get(a, "fo:border").and_then(parse_border); + let pad_all = get(a, "fo:padding").and_then(length); + let side = |b: &str, p: &str| { + let pad = get(a, p).and_then(length).or(pad_all).filter(|v| v.is_finite() && *v > 0.0); + // Padding without a line still takes room (LibreOffice): a side with no line holds it. + let Some(mut line) = get(a, b).and_then(parse_border).or(all) else { + return pad.map(|sp| Border { style: BorderStyle::None, width: 0.0, color: None, space: sp.min(MAX_PADDING) }); + }; + if let Some(sp) = pad { + line.space = if line.is_visible() { sp.min(31.0) } else { sp.min(MAX_PADDING) }; + } + Some(line) + }; + let b = Borders { + top: side("fo:border-top", "fo:padding-top"), + left: side("fo:border-left", "fo:padding-left"), + bottom: side("fo:border-bottom", "fo:padding-bottom"), + right: side("fo:border-right", "fo:padding-right"), + ..Default::default() + }; + (b.top.is_some() || b.left.is_some() || b.bottom.is_some() || b.right.is_some()).then_some(b) +} + +/// A list style's level (ODF 1.3 §16.31–16.33): numbered or not, its label and start value. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub(super) struct ListLevel { + pub(super) ordered: bool, + pub(super) label: Option, + pub(super) start: u32, +} + +/// ODF number format (`style:num-format`, ODF 1.3 §19.500) → WordCraft's; an empty format shows +/// no number. +pub(super) fn num_format(f: &str) -> NumFormat { + match f.trim() { + "" => NumFormat::None, + "i" => NumFormat::LowerRoman, + "I" => NumFormat::UpperRoman, + "a" => NumFormat::LowerLetter, + "A" => NumFormat::UpperLetter, + _ => NumFormat::Decimal, + } +} + +/// A level's `style:list-level-properties` (its label's alignment, `fo:text-align`) or +/// `style:list-level-label-alignment` (its indent, `fo:margin-left`, and the label's hanging, +/// `fo:text-indent`), ODF 1.3 §17.19–17.20, into `slot`'s label. +fn level_position(slot: &mut ListLevel, local: &str, a: &Attrs) { + let Some(label) = slot.label.as_mut() else { return }; + if local == "list-level-properties" { + label.align = match get(a, "fo:text-align").map(str::trim) { + Some("end" | "right") => Some(Align::Right), + Some("center") => Some(Align::Center), + Some("start" | "left") => Some(Align::Left), + _ => label.align, + }; + return; + } + let len = |k: &str| get(a, k).and_then(length).filter(|v| v.is_finite()); + if let Some(m) = len("fo:margin-left") { + label.indent = model::Label::twips(m.clamp(-MAX_LEN, MAX_LEN)); + label.hanging = model::Label::twips((-len("fo:text-indent").unwrap_or(0.0)).clamp(-MAX_LEN, MAX_LEN)); + } +} + +/// A `text:list-level-style-number`/`-bullet`/`-image` element's level. +fn list_level(local: &str, a: &Attrs) -> ListLevel { + let shown = get(a, "text:display-levels").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(1, 9); + let start = get(a, "text:start-value").and_then(|v| v.trim().parse::().ok()).unwrap_or(1).clamp(0, 32_767); + let affix = |k: &str| model::Affix::new(get(a, k).unwrap_or("")); + match local { + "list-level-style-number" => { + let format = num_format(get(a, "style:num-format").unwrap_or("")); + let ordered = format != NumFormat::None; + let label = model::Label { + format, + prefix: affix("style:num-prefix"), + suffix: affix("style:num-suffix"), + shown, + indent: None, + hanging: None, + align: None, + }; + ListLevel { ordered, label: ordered.then_some(label), start } + } + _ => { + // A picture bullet shows as the default bullet. + let bullet = get(a, "text:bullet-char").and_then(|b| b.chars().find(|c| !c.is_control() && !c.is_whitespace())); + ListLevel { ordered: false, label: Some(model::Label::bulleted(bullet)), start: 1 } + } + } +} + +/// Widow and orphan control as `fo:widows`/`fo:orphans` state it (on with either above 0, off +/// with both 0); `None` when neither is stated. +fn widows(a: &Attrs) -> Option { + let lines = |k: &str| get(a, k).and_then(|v| v.trim().parse::().ok()); + let (w, o) = (lines("fo:widows"), lines("fo:orphans")); + (w.is_some() || o.is_some()).then(|| w.unwrap_or(0) > 0 || o.unwrap_or(0) > 0) +} + +/// Most font faces kept from `office:font-face-decls` (hostile input). +const MAX_FACES: usize = 10_000; + +#[derive(Default)] +pub(super) struct Styles { + map: HashMap, + /// Font faces (`style:font-face`): name → family. + faces: HashMap, + /// The levels of `text:outline-style` (heading numbering, ODF 1.3 §16.34), when there is one. + pub(super) outline: Option>, + pub(super) lists: HashMap>, + /// Font of `style:default-style` for paragraphs (the document default, ODF 1.3 §16.4). + default_font: Option, + /// The default paragraph style's font size (`fo:font-size` in `style:default-style`), points. + default_size: Option, + /// Spacing of `style:default-style` for paragraphs. + default_spacing: Spacing, + /// Tab stops are measured from the paragraph's indent (LibreOffice's `TabsRelativeToIndent` + /// setting in `settings.xml`, on unless it says otherwise). + pub(super) tabs_from_indent: bool, + /// The default tab interval (`style:tab-stop-distance` of the default style), points. + pub(super) default_tab: Option, + /// The Asian and complex-script fonts and languages of `style:default-style` for paragraphs. + pub(super) default_scripts: craft_wordmodel::props::CharProps, + /// The language the default paragraph style states. + pub(super) default_lang: Option, + /// Pair kerning of `style:default-style` for paragraphs (`style:letter-kerning`, §20.317; + /// off unless it says so). + pub(super) default_kern: bool, + /// Widow and orphan control `style:default-style` for paragraphs states, if it does. + default_widows: Option, + /// The text colour of `style:default-style` for paragraphs (`fo:color`), if it gives one. + default_color: Option, + /// `style:default-style` for graphics: what every drawing shape's style falls back on. + default_graphic: super::frames::Graphic, + /// `style:default-style` for table columns: the share of a column whose style gives none. + default_column_share: Option, + /// `style:default-style` for table rows: `fo:keep-together`, for a row whose style doesn't say. + default_row_keep: Option, + /// The named dashes and gradients graphic styles use. + pub(super) defs: super::strokes::Defs, + /// Date and time styles (`number:date-style`, `number:time-style`) as Word pictures. + data_styles: HashMap, +} + +impl Styles { + /// A table column's or row's `style:default-style` properties (`props` names which): what a + /// column or row whose own style doesn't say falls back on. + fn read_table_default(&mut self, props: &str, a: &Attrs) { + if props == "table-row-properties" { + self.default_row_keep = get(a, "fo:keep-together").map(|k| k.trim() == "always"); + } else { + self.default_column_share = column_share_of(a); + } + } + + pub(super) fn read(&mut self, xml: &[u8], auto_section_is_auto: bool) { + let mut in_auto = false; + let mut cur: Option<(String, StyleInfo)> = None; + let mut in_default = false; + let mut in_graphic_default = false; + // In a table column's or row's `style:default-style`: the properties element it reads. + let mut in_table_default: Option<&str> = None; + let mut dates = DateStyles::default(); + let mut list: Option<(String, Vec)> = None; + // The list or outline level being read. + let mut level_at: Option = None; + let mut outline_at: Option = None; + events(xml, |ev| match ev { + Ev::Start(q, _) | Ev::End(q) if q.starts_with("number:") && dates.event(&ev, &mut self.data_styles) => {} + Ev::Text(_) if dates.event(&ev, &mut self.data_styles) => {} + Ev::Start(q, a) => match local(&q) { + "automatic-styles" => in_auto = true, + "style" if q.starts_with("style:") => { + if let Some(name) = get(&a, "style:name") { + let info = StyleInfo { + parent: get(&a, "style:parent-style-name").map(String::from), + family: get(&a, "style:family").map(String::from), + display: get(&a, "style:display-name").map(String::from), + auto: in_auto && auto_section_is_auto, + declared_auto: in_auto, + outline: get(&a, "style:default-outline-level").and_then(|v| v.trim().parse::().ok()), + master_page: get(&a, "style:master-page-name").map(String::from), + ..Default::default() + }; + cur = Some((name.to_string(), info)); + } + } + "default-style" if q.starts_with("style:") => { + in_default = get(&a, "style:family") == Some("paragraph"); + in_graphic_default = get(&a, "style:family") == Some("graphic"); + in_table_default = table_default_props(get(&a, "style:family")); + } + "graphic-properties" if in_graphic_default && cur.is_none() => self.default_graphic = super::frames::Graphic::read(&a), + props if in_table_default == Some(props) && cur.is_none() => self.read_table_default(props, &a), + "text-properties" if in_default && cur.is_none() => { + // A later default style without a (usable) font keeps the earlier one (#4501). + self.default_font = default_font(&a, &self.faces).or(self.default_font.take()); + if let Some(sz) = get(&a, "fo:font-size").and_then(length).filter(|v| v.is_finite() && *v >= 1.0) { + self.default_size = Some(sz.min(1638.0)); + } + self.default_kern = get(&a, "style:letter-kerning").is_some_and(|k| k.trim() == "true"); + self.default_color = get(&a, "fo:color").and_then(model::parse_color); + script_properties(&a, &mut self.default_scripts, &self.faces); + self.default_lang = language(&a); + } + "paragraph-properties" if in_default && cur.is_none() => { + paragraph_spacing(&a, &mut self.default_spacing); + self.default_widows = widows(&a); + if let Some(d) = get(&a, "style:tab-stop-distance").and_then(length).filter(|d| d.is_finite() && *d > 0.0) { + self.default_tab = Some(d.clamp(1.0, MAX_LEN)); + } + } + "text-properties" => { + if let Some((_, s)) = &mut cur { + text_properties(&a, s, &self.faces); + if let Some(h) = get(&a, "fo:hyphenate") { + s.hyphenate = Some(h.trim() == "true"); + } + } + } + name if cur.as_mut().is_some_and(|(_, s)| style_child(name, &q, &a, s)) => {} + "font-face" if q.starts_with("style:") && self.faces.len() < MAX_FACES => { + if let (Some(n), Some(f)) = (get(&a, "style:name"), get(&a, "svg:font-family")) { + let family = f.split(',').next().unwrap_or("").trim().trim_matches(['\'', '"']); + if !family.is_empty() && family.len() <= model::MAX_FONT_NAME { + self.faces.insert(n.to_string(), family.to_string()); + } + } + } + // Dashes and gradients; and the part's root, whose version they depend on. + "stroke-dash" | "gradient" | "document-styles" | "document-content" | "document" => self.defs.element(&q, &a), + "list-style" => list = get(&a, "style:name").map(|n| (n.to_string(), vec![ListLevel::default(); 10])), + "outline-style" => self.outline = Some(vec![ListLevel::default(); 10]), + "outline-level-style" => { + let lv = get(&a, "text:level").and_then(|l| l.trim().parse::().ok()).unwrap_or(1).clamp(1, 10); + if let Some(slot) = self.outline.as_mut().and_then(|o| o.get_mut(lv - 1)) { + *slot = list_level("list-level-style-number", &a); + } + outline_at = Some(lv - 1); + } + "list-level-style-number" | "list-level-style-bullet" | "list-level-style-image" => { + if let Some((_, v)) = &mut list { + let lv = get(&a, "text:level").and_then(|l| l.trim().parse::().ok()).unwrap_or(1).clamp(1, 10); + if let Some(slot) = v.get_mut(lv - 1) { + *slot = list_level(local(&q), &a); + } + level_at = Some(lv - 1); + } + } + "list-level-properties" | "list-level-label-alignment" => { + let slot = match (&mut list, level_at) { + (Some((_, v)), Some(i)) => v.get_mut(i), + _ => self.outline.as_mut().zip(outline_at).and_then(|(o, i)| o.get_mut(i)), + }; + if let Some(slot) = slot { + level_position(slot, local(&q), &a); + } + } + _ => {} + }, + Ev::End(q) => match local(&q) { + "automatic-styles" => in_auto = false, + "default-style" if q.starts_with("style:") => (in_default, in_graphic_default, in_table_default) = (false, false, None), + "style" if q.starts_with("style:") => { + if let Some((n, s)) = cur.take() + && self.map.len() < 100_000 + { + self.map.insert(n, s); + } + } + "list-style" => { + if let Some((n, v)) = list.take() { + self.lists.insert(n, v); + } + } + "list-level-style-number" | "list-level-style-bullet" | "list-level-style-image" => level_at = None, + "outline-level-style" => outline_at = None, + _ => {} + }, + Ev::Text(_) => {} + }); + } + + /// The document's default font size: the Standard paragraph style's (the document's + /// defaults, as for [`Styles::doc_font`]), else the default style's. + /// The document's default text colour: the Standard paragraph style's, else the default + /// style's. + pub(super) fn doc_color(&self) -> Option { + self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.color).or(self.default_color) + } + + pub(super) fn doc_size(&self) -> Option { + self.map + .get("Standard") + .filter(|s| !s.auto) + .and_then(|s| s.fmt.size) + .or(self.default_size) + .filter(|v| v.is_finite() && *v >= 1.0) + .map(|v| v.min(1638.0)) + } + + /// The document's language: its Standard style's, else its default paragraph style's. (Written + /// from the model, Standard is empty and the default style says it for both.) + pub(super) fn document_lang(&self) -> Option { + self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.extra.lang.clone()).or_else(|| self.default_lang.clone()) + } + + /// The document's default font: the Standard paragraph style's, else the default style's. + pub(super) fn doc_font(&self) -> Option { + self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.as_deref().map(str::to_string)).or_else(|| self.default_font.clone()) + } + + /// The document's default paragraph spacing: the default style's, with the Standard + /// paragraph style's over it, on ODF's initial values (no margins, single lines). + pub(super) fn doc_spacing(&self) -> Spacing { + let mut sp = Spacing { + before: Some(0.0), + after: Some(0.0), + line: Some(LineSpacing::Multiple(1.0)), + left: Some(0.0), + right: Some(0.0), + first: Some(0.0), + ..Default::default() + }; + overlay_spacing(&mut sp, &self.default_spacing); + if let Some(s) = self.map.get("Standard").filter(|s| !s.auto) { + overlay_spacing(&mut sp, &s.spacing); + } + sp + } + + fn chain<'a>(&'a self, name: &'a str) -> Vec<(&'a str, &'a StyleInfo)> { + let mut out = Vec::new(); + let mut cur = Some(name); + while let Some(n) = cur { + if out.len() >= 16 { + break; + } + let Some(s) = self.map.get(n) else { break }; + if out.iter().any(|(m, _): &(&str, &StyleInfo)| *m == n) { + break; + } + out.push((n, s)); + cur = s.parent.as_deref(); + } + out + } + + /// Text formatting of a text style (whole chain). + /// Whether `odt` names a custom character style: a common text style other than those + /// LibreOffice gives links, note marks and list labels. + pub(super) fn is_named_text(&self, odt: &str) -> bool { + let Some(s) = self.map.get(odt).filter(|s| !s.declared_auto && s.family.as_deref() == Some("text")) else { return false }; + let special = |n: &str| { + let n = n.to_ascii_lowercase().replace("_20_", " "); + n.contains("internet link") || n.contains("footnote") || n.contains("endnote") || n.ends_with("symbols") || n.contains("anchor") + }; + !(special(odt) || s.display.as_deref().is_some_and(special)) + } + + /// A custom character style ([`Self::is_named_text`]): its name, the custom character style + /// it is based on (by ODF name) and its own formatting. + /// A custom character style: its name, parent, own formatting, and what it turns off (bold, + /// italic, underline, strike-through: see [`NamedStyle::off`]). + pub(super) fn named_text(&self, odt: &str) -> Option<(String, Option, Fmt, [bool; 4])> { + if !self.is_named_text(odt) { + return None; + } + let s = self.map.get(odt)?; + let mut fmt = Fmt::default(); + s.fmt.apply(&mut fmt); + let parent = s.parent.clone().filter(|p| self.is_named_text(p)); + let name: String = s.display.clone().unwrap_or_else(|| odt.replace("_20_", " ")).chars().take(model::MAX_FONT_NAME).collect(); + let off = [s.fmt.bold, s.fmt.italic, s.fmt.underline, s.fmt.strike].map(|v| v == Some(false)); + Some((name, parent, fmt, off)) + } + + /// The formatting of a span in text style `name` over `base`: the automatic styles' over the + /// nearest custom character style, which the span then names ([`Fmt::extra`]'s style, by ODF + /// name until import makes it a document style id); without one, the whole chain's. + /// The size of text in paragraph style `name` by its named styles, not its automatic ones, + /// which are the paragraph's own: what they say on what the document says. + pub(super) fn named_size(&self, name: &str) -> Option { + // The default style's: Standard's counts only for a style based on it, in the chain. + let size = self.default_size.filter(|v| v.is_finite() && *v >= 1.0).map(|v| v.min(1638.0)); + let mut f = Fmt { size, ..Fmt::default() }; + for (_, s) in self.chain(name).iter().rev().filter(|(_, s)| !s.auto) { + s.fmt.apply(&mut f); + } + f.size + } + + /// Whether paragraph style `name`'s named styles state a size. Those that don't leave the size + /// to WordCraft's style for its kind, so text of any size isn't known to be the style's. + pub(super) fn states_size(&self, name: &str) -> bool { + self.chain(name).iter().any(|(_, s)| !s.auto && (s.fmt.size.is_some() || s.fmt.size_pct.is_some())) + } + + pub(super) fn span_fmt(&self, name: &str, base: &Fmt) -> Fmt { + let chain = self.chain(name); + let Some(at) = chain.iter().position(|(n, _)| self.is_named_text(n)) else { return self.text_fmt(name, base) }; + let mut f = base.clone(); + // What the span's character style says is its own: its paragraph turning that off doesn't + // reach past it (ODF: the span's style wins). + if let Some(x) = f.extra.as_deref_mut() { + use craft_wordmodel::props::Underline; + for (_, s) in chain.iter().skip(at) { + let (d, e) = (&s.fmt, &s.fmt.extra); + if d.bold.is_some() && x.bold == Some(false) { + x.bold = None; + } + if d.italic.is_some() && x.italic == Some(false) { + x.italic = None; + } + if d.underline.is_some() && x.underline == Some(Underline::None) { + x.underline = None; + } + if (d.strike.is_some() || e.double_strike.is_some()) && x.strike == Some(false) { + (x.strike, x.double_strike) = (None, None); + } + if e.caps.is_some() && x.caps == Some(false) { + x.caps = None; + } + if e.small_caps.is_some() && x.small_caps == Some(false) { + x.small_caps = None; + } + } + } + for (_, s) in chain.iter().take(at).rev() { + s.fmt.apply(&mut f); + } + if let Some((n, _)) = chain.get(at) { + f.extra.get_or_insert_with(Default::default).style = Some(n.to_string()); + } + f + } + + pub(super) fn text_fmt(&self, name: &str, base: &Fmt) -> Fmt { + let mut f = base.clone(); + for (_, s) in self.chain(name).iter().rev() { + s.fmt.apply(&mut f); + } + f + } + + /// The page a paragraph style starts (`style:master-page-name` and `style:page-number`, + /// ODF 1.3 §16.2 and §17.6): the nearest master page and page-number setting in its chain. + pub(super) fn page_start<'a>(&'a self, name: &'a str) -> (Option<&'a str>, Option>) { + let chain = self.chain(name); + let master = chain.iter().find_map(|(_, s)| s.master_page.as_deref()).filter(|m| !m.is_empty()); + (master, chain.iter().find_map(|(_, s)| s.page_number)) + } + + /// Columns of a section style (whole chain). + /// A section style's columns in a text `tw` points wide (whole chain). + pub(super) fn columns(&self, name: &str, tw: f32) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.columns.clone().map(|c| with_widths(c, &s.column_specs, tw))) + } + + /// Whether a section style's columns are balanced at its end, when it says (the attribute's + /// name, `text:dont-balance-text-columns`, read as it reads; §20.428's value descriptions + /// say the opposite). + pub(super) fn section_balance(&self, name: &str) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.dont_balance).map(|d| !d) + } + + /// Width in points of a table-column style (whole chain). + pub(super) fn column_width(&self, name: &str) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.column_width) + } + + /// A table column's share of its table (`style:rel-column-width`): its style's, whole chain, + /// else the default style's. + pub(super) fn column_share(&self, name: Option<&str>) -> Option { + name.and_then(|n| self.chain(n).iter().find_map(|(_, s)| s.column_share)).or(self.default_column_share) + } + + /// A table style's alignment, width (points) and relative width (percent), whole chain. + /// Whether a table style starts a page (`fo:break-before="page"`, whole chain). + pub(super) fn table_breaks(&self, name: &str) -> bool { + self.chain(name).iter().any(|(_, s)| s.page_break) + } + + pub(super) fn table_look(&self, name: &str) -> (Option, Option, Option) { + let chain = self.chain(name); + ( + chain.iter().find_map(|(_, s)| s.table_align), + chain.iter().find_map(|(_, s)| s.table_width), + chain.iter().find_map(|(_, s)| s.table_rel_width), + ) + } + + /// A table style's own space above and below (points; each side from the nearest style in + /// the chain giving it). + pub(super) fn table_space(&self, name: &str) -> [Option; 2] { + let chain = self.chain(name); + [0, 1].map(|i| chain.iter().find_map(|(_, s)| s.table_space.get(i).copied().flatten())) + } + + /// A table-row style's height (points) and whether it is exact, whole chain. + pub(super) fn row_height(&self, name: &str) -> Option<(f32, bool)> { + self.chain(name).iter().find_map(|(_, s)| s.row_height) + } + + /// Whether a table row doesn't break across pages: its style says (whole chain), else the + /// default style. + pub(super) fn row_keep(&self, name: Option<&str>) -> bool { + name.and_then(|n| self.chain(n).iter().find_map(|(_, s)| s.row_keep)).or(self.default_row_keep).unwrap_or(false) + } + + /// Background colour of a table-cell style (whole chain). + /// A table-cell style's background picture: its package path, repeat and position. + pub(super) fn cell_picture(&self, name: &str) -> Option<(String, String, String)> { + self.chain(name).iter().find_map(|(_, s)| s.cell_picture.clone()) + } + + pub(super) fn table_background(&self, name: &str) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.table_background).flatten() + } + + /// Background of a table-cell style (whole chain). + pub(super) fn cell_background(&self, name: &str) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.cell_background).flatten() + } + + /// Borders of a table-cell style (whole chain). + pub(super) fn cell_borders(&self, name: &str) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.cell_borders) + } + + /// Padding of a table-cell style (top, left, bottom, right; whole chain), when it gives all + /// four sides; missing sides of a partial one are 0, as LibreOffice reads them. + pub(super) fn cell_padding(&self, name: &str) -> Option<[f32; 4]> { + let p = self.chain(name).iter().map(|(_, s)| s.cell_padding).find(|p| p.iter().any(Option::is_some))?; + Some(p.map(|v| v.unwrap_or(0.0))) + } + + /// A graphic style's frame placement (whole chain). + pub(super) fn graphic(&self, name: &str) -> super::frames::Graphic { + self.chain(name).iter().filter_map(|(_, s)| s.graphic.as_ref()).fold(super::frames::Graphic::default(), |g, p| g.or(p)) + } + + /// The graphic properties of drawing shape style `name` (none stated: the default style's), + /// over its chain and then the default graphic style. + pub(super) fn shape_graphic(&self, name: Option<&str>) -> super::frames::Graphic { + name.map(|n| self.graphic(n)).unwrap_or_default().or(&self.default_graphic) + } + + /// The Word date-time picture of date or time style `name`. + pub(super) fn date_picture(&self, name: &str) -> Option<&str> { + self.data_styles.get(name).map(String::as_str).filter(|p| !p.is_empty()) + } + + /// Vertical alignment of a table-cell style (whole chain). + pub(super) fn cell_valign(&self, name: &str) -> Option { + self.chain(name).iter().find_map(|(_, s)| s.cell_valign) + } + + /// Which way a table-cell style's text runs (whole chain). + pub(super) fn cell_direction(&self, name: &str) -> craft_wordmodel::props::TextDirection { + self.chain(name).iter().find_map(|(_, s)| s.cell_direction).unwrap_or_default() + } + + /// Paragraph kind, alignment, page break, rule, keep-with-next and base formatting. The text formatting of + /// every style in the chain applies (automatic and common, ODF 1.3 §16.2 inheritance), except + /// the style that gives the paragraph its kind and that style's ancestors: their formatting + /// is the kind's own look (a heading's size and colour), not direct formatting. `Standard`, + /// the default paragraph style, holds document defaults and is left out too. + /// The look the file gives each style that maps to a paragraph kind (Heading 1–6, Title, + /// Quotations, Preformatted Text): its text formatting, spacing, alignment and keep with + /// next, over the whole chain but Standard (the document defaults). A style named for the + /// kind wins over one only displayed so. + pub(super) fn kind_looks(&self) -> Vec<(Kind, ParaLook)> { + let mut found: Vec<(Kind, bool, &str)> = Vec::new(); + for (n, s) in self.map.iter().filter(|(_, s)| !s.auto) { + let own = kind_named(&n.to_ascii_lowercase().replace("_20_", " ")); + let shown = s.display.as_deref().and_then(|d| kind_named(&d.to_ascii_lowercase())); + let Some((kind, canonical)) = own.map(|k| (k, true)).or(shown.map(|k| (k, false))) else { continue }; + match found.iter_mut().find(|(k, ..)| *k == kind) { + Some(slot) if !slot.1 && canonical => *slot = (kind, canonical, n), + Some(slot) if slot.1 == canonical && n.as_str() < slot.2 => *slot = (kind, canonical, n), + Some(_) => {} + None => found.push((kind, canonical, n)), + } + } + found + .into_iter() + .map(|(kind, _, name)| { + let chain: Vec<_> = self.chain(name).into_iter().filter(|(n, _)| *n != "Standard").collect(); + let mut fmt = Fmt::default(); + let mut spacing = Spacing::default(); + for (_, s) in chain.iter().rev() { + s.fmt.apply(&mut fmt); + overlay_spacing(&mut spacing, &s.spacing); + } + let align = chain.iter().find_map(|(_, s)| s.align); + let keep_next = chain.iter().find_map(|(_, s)| s.keep_next).unwrap_or(false); + let contextual = chain.iter().find_map(|(_, s)| s.contextual); + (kind, ParaLook { kind, align, keep_next, fmt, spacing, contextual, ..Default::default() }) + }) + .collect() + } + + /// The custom style `odt` names (a common paragraph style other than Standard and the + /// kinds' styles), with its parent and own formatting; `None` for any other style. + /// Whether `odt` names a custom style ([`Self::named_style`]), without building it. + pub(super) fn is_named(&self, odt: &str) -> bool { + let Some(s) = self.map.get(odt).filter(|s| !s.auto && !s.declared_auto && odt != "Standard") else { return false }; + let lower = |n: &str| n.to_ascii_lowercase().replace("_20_", " "); + // The kinds' styles, and the note text styles (WordCraft's Footnote Text and Endnote + // Text stand for them), are no custom styles. + let special = |n: &str| kind_named(n).is_some() || matches!(n, "footnote" | "endnote"); + !(special(&lower(odt)) || s.display.as_deref().is_some_and(|d| special(&lower(d)))) + } + + /// Widow and orphan control the document states for every paragraph: in its default style, + /// else in Standard; `None` when neither says. + pub(super) fn stated_widows(&self) -> Option { + self.default_widows.or_else(|| self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.widow_control)) + } + + pub(super) fn named_style(&self, odt: &str) -> Option { + if !self.is_named(odt) { + return None; + } + let s = self.map.get(odt)?; + let lower = |n: &str| n.to_ascii_lowercase().replace("_20_", " "); + let parent = match s.parent.as_deref() { + None | Some("Standard") => Parent::Normal, + Some(p) => match self.map.get(p) { + None => Parent::Normal, + Some(ps) => match kind_named(&lower(p)).or_else(|| ps.display.as_deref().and_then(|d| kind_named(&lower(d)))) { + Some(k) => Parent::Kind(k), + None if ps.auto => Parent::Normal, + None => Parent::Named(p.to_string()), + }, + }, + }; + let mut fmt = Fmt::default(); + s.fmt.apply(&mut fmt); + // A percent is of what's under it: the parent's size, or with none the default style's. + if s.fmt.size_pct.is_some() { + fmt.size = self.named_size(odt); + } + // A style with no parent inherits the default style, not Standard (the document's + // defaults here): its size, when Standard's differs. + if s.parent.is_none() && fmt.size.is_none() && self.doc_size() != self.default_size { + fmt.size = self.default_size; + } + // And its language, likewise. + if s.parent.is_none() + && let Some(lang) = self.default_lang.as_ref().filter(|l| self.document_lang().as_ref() != Some(*l)) + && fmt.extra.as_ref().is_none_or(|x| x.lang.is_none()) + { + fmt.extra.get_or_insert_with(Default::default).lang = Some(lang.clone()); + } + let look = ParaLook { + align: s.align, + keep_next: s.keep_next == Some(true), + fmt, + spacing: s.spacing, + tabs: s.tabs.clone(), + borders: s.para_borders.filter(|b| b.takes_room()), + shading: s.para_shading.flatten(), + contextual: s.contextual, + rtl: s.rtl, + ..Default::default() + }; + let name = s.display.clone().unwrap_or_else(|| odt.replace("_20_", " ")); + let off = [s.fmt.bold, s.fmt.italic, s.fmt.underline, s.fmt.strike].map(|v| v == Some(false)); + let outline = s.outline.filter(|o| (1..=10).contains(o)); + Some(NamedStyle { name: name.chars().take(craft_flow::MAX_FONT_NAME).collect(), parent, look, off, outline }) + } + + pub(super) fn para(&self, name: &str) -> ParaLook { + let chain = self.chain(name); + let mut kind = Kind::Normal; + let mut kind_at = chain.len(); + // The nearest outline level; a style with one is no kind's style: its look is its own. + let mut outline = None; + for (i, (n, s)) in chain.iter().enumerate() { + kind_at = i; + let dn = s.display.as_deref().unwrap_or(n).to_ascii_lowercase().replace("_20_", " "); + let nn = n.to_ascii_lowercase().replace("_20_", " "); + for cand in [dn.as_str(), nn.as_str()] { + if let Some(d) = cand.strip_prefix("heading ").and_then(|r| r.trim().parse::().ok()) { + kind = Kind::Heading(d.clamp(1, 6)); + } else if cand == "title" { + kind = Kind::Title; + } else if cand == "quotations" || cand == "quote" { + kind = Kind::Quote; + } else if cand == "preformatted text" || cand == "source text" || cand == "code" { + kind = Kind::Code; + } + } + if kind != Kind::Normal { + break; + } + outline = outline.or(s.outline.filter(|o| (1..=10).contains(o))); + kind_at = chain.len(); + } + if let Some(o) = outline { + kind = Kind::Heading(o.min(6)); + } + // The nearest custom style: its formatting stays in it; the automatic styles over it give + // the paragraph's own. + let named_at = chain.iter().take(kind_at).position(|(n, _)| self.is_named(n)); + let named = named_at.and_then(|i| chain.get(i)).map(|(n, _)| n.to_string()).or_else(|| { + // An index style the file names but doesn't define (WordCraft has its own). + let undefined = chain.last().map_or(Some(name), |(_, s)| s.parent.as_deref()); + undefined.filter(|n| super::writer::generating_style(n).is_some()).map(String::from) + }); + let kind_at = named_at.unwrap_or(kind_at); + let align = if named.is_some() { chain.iter().take(kind_at).find_map(|(_, s)| s.align) } else { chain.iter().find_map(|(_, s)| s.align) }; + let pb = chain.iter().take_while(|(_, s)| s.auto).any(|(_, s)| s.page_break); + let column_break = !pb && chain.iter().take_while(|(_, s)| s.auto).any(|(_, s)| s.column_break); + let rule = chain.iter().any(|(_, s)| s.border_bottom); + let mut f = Fmt::default(); + let mut spacing = Spacing::default(); + for (_, s) in chain.iter().take(kind_at).rev().filter(|(n, s)| s.auto || *n != "Standard") { + s.fmt.apply(&mut f); + overlay_spacing(&mut spacing, &s.spacing); + } + // Like the text formatting, the kind style's own `fo:keep-with-next` (a heading's) is the kind's look, not direct formatting. + let keep_next = chain.iter().take(kind_at).find_map(|(_, s)| s.keep_next).unwrap_or(false); + // A style's tab stops replace its parent's; the kind's style and Standard give none. + let tabs = chain.iter().take(kind_at).filter(|(n, s)| s.auto || *n != "Standard").find_map(|(_, s)| s.tabs.clone()); + // Borders and background: the nearest in the chain, the kind's style and Standard aside. + let own = || chain.iter().take(kind_at).filter(|(n, s)| s.auto || *n != "Standard"); + let borders = own().find_map(|(_, s)| s.para_borders).filter(|b| b.takes_room()); + let shading = own().find_map(|(_, s)| s.para_shading).flatten(); + let contextual = own().find_map(|(_, s)| s.contextual); + let rtl = own().find_map(|(_, s)| s.rtl); + let drop_cap = own().find_map(|(_, s)| s.drop_cap); + let keep_lines = own().find_map(|(_, s)| s.keep_lines); + let widow_control = own().find_map(|(_, s)| s.widow_control); + // A page break after the paragraph: its own style's (automatic or common). + let break_after = own().take(1).any(|(_, s)| s.break_after); + let hyphenate = chain.iter().find_map(|(_, s)| s.hyphenate); + // Where LibreOffice measures the tab stops from: the indent the paragraph ends up with, + // whichever style in the chain gives it. + let tab_origin = chain.iter().find_map(|(_, s)| s.spacing.left).filter(|v| v.is_finite()).unwrap_or(0.0); + ParaLook { + kind, + align, + page_break: pb, + column_break, + rule, + keep_next, + fmt: f, + spacing, + tabs, + borders, + shading, + named, + contextual, + rtl, + tab_origin, + drop_cap, + keep_lines, + widow_control, + break_after, + hyphenate, + } + } +} diff --git a/crates/odf/src/text/tests.rs b/crates/odf/src/text/tests.rs new file mode 100644 index 000000000..8d7af18dc --- /dev/null +++ b/crates/odf/src/text/tests.rs @@ -0,0 +1,1507 @@ +use super::*; +use std::io::{Cursor, Write}; + +#[test] +fn escaper_drops_noncharacters() { + assert_eq!(x("a\u{FFFF}b\u{FFFE}c\u{1}d\te"), "abcd\te"); +} + +#[test] +fn lengths() { + assert_eq!(length("1in"), Some(72.0)); + assert_eq!(length("12pt"), Some(12.0)); + assert!((length("2.54cm").unwrap() - 72.0).abs() < 0.01); + assert_eq!(length("1IN"), Some(72.0)); + assert_eq!(length("12PT"), Some(12.0)); + assert!((length("2.54CM").unwrap() - 72.0).abs() < 0.01); + assert!((length("25.4MM").unwrap() - 72.0).abs() < 0.01); + assert_eq!(length("1pc"), Some(12.0)); + assert_eq!(length("10PX"), Some(7.5)); + assert_eq!(length("x"), None); +} + +#[test] +fn default_paragraph_style_font_becomes_document_font() { + // LibreOffice keeps the font in the Standard style or in `style:default-style`; the import + // used to skip both and the text fell back to the built-in body font. + let content = format!( + r#"Hi"# + ); + let standard = format!( + r#""# + ); + let doc = import(&package(&content, Some(&standard))).unwrap(); + assert_eq!(doc.styles.default_chr.font.as_deref(), Some("Times New Roman")); + let only_default = format!( + r#""# + ); + let doc = import(&package(&content, Some(&only_default))).unwrap(); + assert_eq!(doc.styles.default_chr.font.as_deref(), Some("DejaVu Serif")); +} + +#[test] +fn the_default_text_colour_round_trips() { + // Google Docs' résumé template greys all its text in the default style; it came back black. + let content = format!( + r#"Hi"# + ); + let styles = format!( + r##""## + ); + let grey = Some(craft_wordmodel::props::TextColor::Rgb(craft_wordmodel::props::Rgb(0x66, 0x66, 0x66))); + let doc = import(&package(&content, Some(&styles))).unwrap(); + assert_eq!(doc.styles.default_chr.color, grey); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(back.styles.default_chr.color, grey); + // None stated: none written (automatic). + let plain = import(&package(&content, None)).unwrap(); + let styles = entry(&export(&plain).unwrap(), "styles.xml"); + let default = + styles.split("").nth(1).and_then(|s| s.split("").next()).unwrap(); + assert!(!default.contains("fo:color"), "{default}"); +} + +#[test] +fn rejects_garbage() { + assert!(parse(b"not a zip").is_err()); +} + +/// A package holding `entries`, deflated. +pub(super) fn zip_of(entries: &[(&str, &[u8])]) -> Vec { + let mut z = zip::ZipWriter::new(Cursor::new(Vec::new())); + let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, data) in entries { + z.start_file(*name, o).unwrap(); + z.write_all(data).unwrap(); + } + z.finish().unwrap().into_inner() +} + +/// `bytes` (a package) with the size its central directory declares for `name` set to `size`. +fn declare_size(mut bytes: Vec, name: &str, size: u32) -> Vec { + let at = (0..bytes.len()) + .find(|&i| bytes[i..].starts_with(b"PK\x01\x02") && bytes.get(i + 46..).is_some_and(|r| r.starts_with(name.as_bytes()))) + .unwrap(); + bytes[at + 24..at + 28].copy_from_slice(&size.to_le_bytes()); + bytes +} + +fn archive(bytes: &[u8]) -> crate::Package<'_> { + crate::Package::open(bytes, &super::import::LIMITS).unwrap() +} + +#[test] +fn pictures_are_inflated_once_and_every_use_is_charged() { + // Every frame on a picture inflated it again, even once the budget was used up. + let a = zip_of(&[("p.png", &[1u8; 300])]); + let b = zip_of(&[("p.png", &[2u8; 300]), ("q.png", &[3u8; 10])]); + let (mut za, mut zb) = (archive(&a), archive(&b)); + let mut pics = Pictures::new(1000); + assert_eq!(pics.get(&mut za, "p.png"), Some(vec![1u8; 300])); + // Kept from the first read: the entry isn't read again (another package's would differ). + assert_eq!(pics.get(&mut zb, "p.png"), Some(vec![1u8; 300])); + assert_eq!(pics.get(&mut zb, "p.png"), Some(vec![1u8; 300])); + assert_eq!(pics.room, 100); + // A fourth copy doesn't fit; an entry that couldn't be read isn't looked up again. + assert_eq!(pics.get(&mut za, "p.png"), None); + assert_eq!(pics.get(&mut za, "q.png"), None); + assert_eq!(pics.get(&mut zb, "q.png"), None); + assert_eq!(pics.room, 100); +} + +#[test] +fn pictures_over_the_budget_are_not_inflated() { + // An entry declaring more than is left is refused before inflating, and costs nothing. + let pkg = zip_of(&[("big.png", &[0u8; 1200]), ("ok.png", &[5u8; 300])]); + let mut z = archive(&pkg); + let mut pics = Pictures::new(1000); + assert_eq!(pics.get(&mut z, "big.png"), None); + assert_eq!(pics.room, 1000); + assert_eq!(pics.get(&mut z, "ok.png").map(|d| d.len()), Some(300)); + // One inflating to more than it declares stops a byte past the budget and uses it up: + // nothing is inflated after it. + let liar = declare_size(zip_of(&[("liar.png", &[0u8; 5000]), ("ok.png", &[5u8; 10])]), "liar.png", 10); + let mut z = archive(&liar); + let mut pics = Pictures::new(1000); + assert_eq!(pics.get(&mut z, "liar.png"), None); + assert_eq!(pics.room, 0); + assert_eq!(pics.get(&mut z, "ok.png"), None); +} + +#[test] +fn frames_sharing_a_picture_all_show_it() { + let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7u8; 300]].concat(); + let frames = r#""#.repeat(3); + let content = format!( + r#"{frames}"# + ); + let pkg = zip_of(&[("content.xml", content.as_bytes()), ("Pictures/a.png", &png)]); + let flow = parse(&pkg).unwrap(); + let imgs: Vec<&model::Img> = + paras(&flow).into_iter().flat_map(|p| &p.inlines).filter_map(|i| if let Inline::Image(img) = i { Some(img) } else { None }).collect(); + assert_eq!(imgs.len(), 3); + assert!(imgs.iter().all(|i| i.data.as_slice() == png.as_slice())); + assert_eq!(import(&pkg).unwrap().media.len(), 1); +} + +#[test] +fn embedded_formula_object_reads_its_mathml() { + // #3099: LibreOffice saves an equation as a `draw:object` (a formula sub-document whose + // own `content.xml` is MathML) plus a `draw:image` replacement preview in LibreOffice's + // own metafile format, which WordCraft can't decode as a picture. The object's MathML + // must be read instead of silently dropping the frame (blank equation). + let math = r#"E=mc2E=mc^2"#; + let frame = r#""#; + let content = format!( + r#"{frame}"# + ); + let pkg = zip_of(&[ + ("content.xml", content.as_bytes()), + ("Object 1/content.xml", math.as_bytes()), + // Not a format `sniff_image` recognizes: proves the equation wins over this fallback. + ("ObjectReplacements/Object 1", &[0u8; 50]), + ]); + let flow = parse(&pkg).unwrap(); + let eqs: Vec<(String, bool)> = paras(&flow) + .into_iter() + .flat_map(|p| &p.inlines) + .filter_map(|i| if let Inline::Equation { linear, display } = i { Some((linear.clone(), *display)) } else { None }) + .collect(); + assert_eq!(eqs, vec![("E=mc^2".to_string(), true)]); +} + +#[test] +fn a_trailing_slash_on_the_object_href_still_resolves() { + // Some ODF producers write the object folder with a trailing slash + // (`xlink:href="./Object 1/"`); that must not double the slash before `content.xml`. + let math = r#"x"#; + let frame = r#""#; + let content = format!( + r#"{frame}"# + ); + let pkg = zip_of(&[("content.xml", content.as_bytes()), ("Object 1/content.xml", math.as_bytes())]); + let flow = parse(&pkg).unwrap(); + let eqs: Vec<(String, bool)> = paras(&flow) + .into_iter() + .flat_map(|p| &p.inlines) + .filter_map(|i| if let Inline::Equation { linear, display } = i { Some((linear.clone(), *display)) } else { None }) + .collect(); + assert_eq!(eqs, vec![("x".to_string(), false)]); +} + +#[test] +fn object_without_mathml_falls_back_to_its_replacement_image() { + // A non-formula embedded object (or one whose content.xml has no `math:math`) has no + // equation to read: the frame still shows its `draw:image` replacement, as before. + let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7u8; 300]].concat(); + let frame = r#""#; + let content = format!( + r#"{frame}"# + ); + let pkg = zip_of(&[("content.xml", content.as_bytes()), ("Object 1/content.xml", b""), ("Pictures/a.png", &png)]); + let flow = parse(&pkg).unwrap(); + let imgs: Vec<&model::Img> = + paras(&flow).into_iter().flat_map(|p| &p.inlines).filter_map(|i| if let Inline::Image(img) = i { Some(img) } else { None }).collect(); + assert_eq!(imgs.len(), 1); + assert!(imgs[0].data.as_slice() == png.as_slice()); +} + +/// A picture of `w` by `h` pixels, as a PNG. +fn png_of(w: u32, h: u32) -> Vec { + let mut png = Cursor::new(Vec::new()); + image::RgbImage::new(w, h).write_to(&mut png, image::ImageFormat::Png).unwrap(); + png.into_inner() +} + +#[test] +fn a_picture_stated_on_one_side_keeps_its_own_proportions() { + // A hundred times as tall as wide: only the cap on a page's size may shrink it, in proportion. + let frame = r#""#; + let content = format!( + r#"{frame}"# + ); + let pkg = zip_of(&[("content.xml", content.as_bytes()), ("Pictures/a.png", &png_of(10, 1000))]); + let flow = parse(&pkg).unwrap(); + let img = paras(&flow).into_iter().flat_map(|p| &p.inlines).find_map(|i| if let Inline::Image(img) = i { Some(img) } else { None }).unwrap(); + assert!((img.w - 15.84).abs() < 0.01 && (img.h - 1584.0).abs() < 0.01, "{} {}", img.w, img.h); +} + +#[test] +fn an_objects_picture_with_no_stated_size_is_fitted_to_the_text() { + let styles = format!( + r#""# + ); + let frame = + r#""#; + let content = format!( + r#"{frame}"# + ); + let pkg = zip_of(&[ + ("content.xml", content.as_bytes()), + ("styles.xml", styles.as_bytes()), + ("Object 1/content.xml", b""), + ("Pictures/a.png", &png_of(1000, 500)), + ]); + let flow = parse(&pkg).unwrap(); + let img = paras(&flow).into_iter().flat_map(|p| &p.inlines).find_map(|i| if let Inline::Image(img) = i { Some(img) } else { None }).unwrap(); + assert!(img.object.is_some(), "still the object"); + assert!((img.w - 306.14).abs() < 0.5 && (img.h - 153.07).abs() < 0.5, "{} {}", img.w, img.h); +} + +#[test] +fn a_frame_stating_a_size_of_nothing_is_fitted_to_the_text() { + let styles = format!( + r#""# + ); + let frame = r#""#; + let content = format!( + r#"{frame}"# + ); + let pkg = zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/a.png", &png_of(1000, 500))]); + let flow = parse(&pkg).unwrap(); + let img = paras(&flow).into_iter().flat_map(|p| &p.inlines).find_map(|i| if let Inline::Image(img) = i { Some(img) } else { None }).unwrap(); + assert!((img.w - 306.14).abs() < 0.5 && (img.h - 153.07).abs() < 0.5, "{} {}", img.w, img.h); +} + +#[test] +fn a_picture_in_line_raised_or_turned_with_no_size_is_fitted_to_the_text() { + let styles = format!( + r#""# + ); + let auto = r#""#; + for attrs in [r#" draw:style-name="fr1""#, r#" draw:transform="rotate (0.5) translate (0cm 0cm)""#] { + let frame = format!(r#""#); + let content = format!( + r#"{auto}{frame}"# + ); + let pkg = zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/a.png", &png_of(1000, 500))]); + let flow = parse(&pkg).unwrap(); + let img = paras(&flow).into_iter().flat_map(|p| &p.inlines).find_map(|i| if let Inline::Image(img) = i { Some(img) } else { None }).unwrap(); + assert!(img.w <= 306.5, "{attrs}: {} {}", img.w, img.h); + } +} + +pub(super) const NS_T: &str = r#"xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0""#; + +/// A package with `content.xml` and, when given, `styles.xml`. +pub(super) fn package(content: &str, styles: Option<&str>) -> Vec { + let mut z = zip::ZipWriter::new(Cursor::new(Vec::new())); + let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + for (name, data) in + [("mimetype", "application/vnd.oasis.opendocument.text"), ("content.xml", content)].into_iter().chain(styles.map(|s| ("styles.xml", s))) + { + z.start_file(name, o).unwrap(); + z.write_all(data.as_bytes()).unwrap(); + } + z.finish().unwrap().into_inner() +} + +pub(super) fn paras(f: &Flow) -> Vec<&Para> { + f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect() +} + +#[test] +fn common_paragraph_styles_give_text_formatting() { + // #436: a common paragraph style's text properties reach its paragraphs, directly, + // through a derived common style and through an automatic style's parent. + let props = r##""##; + let styles = format!( + r#"{props}"# + ); + let content = format!( + r#"PlainCommonDerivedBridge"# + ); + // The text shows them (they stay in the custom styles the paragraphs are in). + let d = import(&package(&content, Some(&styles))).unwrap(); + let shown: Vec<_> = + d.body.iter().filter_map(|b| b.as_para()).map(|p| d.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props)).collect(); + assert_eq!(shown.len(), 4); + assert!(!shown[0].bold && shown[0].size != 24.0); + for c in &shown[1..] { + assert!(c.bold && c.color == craft_wordmodel::props::TextColor::Rgb(Rgb(255, 0, 0)) && c.size == 24.0, "{c:?}"); + } +} + +#[test] +fn text_after_a_drawing_shape_stays_in_its_paragraph() { + // #1654: a shape's own text:p closed the paragraph it is anchored in, and the text after + // the shape was dropped. A shape's text is its own (`odt/tests_more/shapes.rs`); the + // paragraph keeps its text. + let draw = + r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0""#; + let content = format!( + r#"In the shapeBefore middle In the groupafterNext"# + ); + let styles = format!( + r#""# + ); + let f = parse(&package(&content, Some(&styles))).unwrap(); + let ps = paras(&f); + assert_eq!(ps.iter().map(|p| p.text()).collect::>(), ["Before middle after", "Next"]); + // The whole paragraph keeps its style's bold. + let d = import(&package(&content, Some(&styles))).unwrap(); + let p = d.body[0].as_para().unwrap(); + assert!(p.runs.iter().all(|r| d.styles.resolve_char(p.props.style.as_deref(), &r.props).bold), "{:?}", p.runs); +} + +#[test] +fn page_size_orientation_and_margins_round_trip() { + // #810: the page setup written on export is read back on import. + let mut doc = Document::new(); + doc.last_section.page_w = 841.0; + doc.last_section.page_h = 595.0; + doc.last_section.landscape = true; + doc.last_section.margin_left = 36.0; + doc.last_section.margin_top = 54.0; + let back = import(&export(&doc).unwrap()).unwrap(); + let s = &back.last_section; + assert!(s.landscape); + assert_eq!((s.page_w, s.page_h), (841.0, 595.0)); + assert_eq!((s.margin_left, s.margin_top), (36.0, 54.0)); +} + +#[test] +fn page_layout_follows_the_first_master_page() { + let xml = br#" + + + + + + + + "#; + let mut sec = craft_wordmodel::SectionProps::default(); + apply_page_layout(xml, &mut sec); + assert!(sec.landscape); + // A margin left unsaid is XSL's initial value, 0. + assert_eq!((sec.page_w, sec.page_h, sec.margin_left, sec.margin_top), (720.0, 360.0, 72.0, 0.0)); +} + +#[test] +fn hostile_page_layout_values_are_clamped() { + let styles = |w: &str, h: &str, m: &str| { + format!( + r#" + + + + "# + ) + }; + // Huge values: the page is capped at 22 inches and margins at the page's longer side. + let mut sec = craft_wordmodel::SectionProps::default(); + apply_page_layout(styles("1e30pt", "1e30in", "1e30pt").as_bytes(), &mut sec); + assert_eq!((sec.page_w, sec.page_h), (1584.0, 1584.0)); + assert_eq!((sec.margin_top, sec.margin_left), (1584.0, 1584.0)); + // Negative, zero, tiny and non-numeric values leave the defaults alone: the default page, and + // XSL's margins (0). + let def = craft_wordmodel::SectionProps { margin_top: 0.0, margin_right: 0.0, ..Default::default() }; + for (w, h, m) in [("-10in", "-5in", "-1in"), ("0pt", "0pt", "NaNpt"), ("1pt", "2pt", "infpt"), ("NaN", "inf", "x")] { + let mut sec = craft_wordmodel::SectionProps::default(); + apply_page_layout(styles(w, h, m).as_bytes(), &mut sec); + assert_eq!((sec.page_w, sec.page_h), (def.page_w, def.page_h), "{w} {h}"); + assert_eq!((sec.margin_top, sec.margin_right), (def.margin_top, def.margin_right), "{m}"); + assert!(sec.page_w.is_finite() && sec.margin_left.is_finite()); + } +} + +#[test] +fn table_column_styles_are_the_grid_widths() { + // A column's `style:column-width` (and `table:number-columns-repeated`) sets its width. + let content = format!( + r##"ABC"## + ); + let f = parse(&package(&content, None)).unwrap(); + let Some(FBlock::Table(t)) = f.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table") }; + assert_eq!(t.widths, vec![72.0, 144.0, 144.0]); +} + +#[test] +fn table_column_widths_survive_a_round_trip() { + // A width changed in the document is written and read back; the file's own widths are + // what the first import shows. + let content = format!( + r##"AB"## + ); + let grid = + |doc: &Document| match doc.body.iter().find_map(|b| if let craft_wordmodel::Block::Table(t) = &**b { Some(t.grid.clone()) } else { None }) { + Some(g) => g, + None => panic!("no table"), + }; + let mut doc = import(&package(&content, None)).unwrap(); + assert_eq!(grid(&doc), vec![72.0, 216.0]); + let widths = vec![150.0, 100.0]; + for b in doc.body.iter_mut() { + if let craft_wordmodel::Block::Table(t) = std::sync::Arc::make_mut(b) { + t.grid = widths.clone(); + } + } + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(grid(&back), widths); +} + +#[test] +fn table_without_column_widths_keeps_the_equal_split() { + let content = format!( + r##"AB"## + ); + let f = parse(&package(&content, None)).unwrap(); + let Some(FBlock::Table(t)) = f.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table") }; + assert!(t.widths.is_empty()); +} + +#[test] +fn table_cell_style_background_is_shading() { + // #437: `fo:background-color` on `style:table-cell-properties` is the cell's shading. + let content = format!( + r##"RedPlain"## + ); + let f = parse(&package(&content, None)).unwrap(); + let Some(FBlock::Table(t)) = f.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table") }; + let row: Vec<_> = t.rows[0].iter().map(|c| (c.text(), c.shading)).collect(); + assert_eq!(row, vec![("Red".to_string(), Some(Rgb(255, 0, 0))), ("Plain".to_string(), None)]); +} + +#[test] +fn table_cell_style_vertical_alignment() { + let content = format!( + r##"MidBot"## + ); + let f = parse(&package(&content, None)).unwrap(); + let Some(FBlock::Table(t)) = f.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table") }; + assert_eq!(t.rows[0][0].valign, Some(VAlign::Center)); + assert_eq!(t.rows[0][1].valign, Some(VAlign::Bottom)); + let doc = craft_flow::to_doc(&f); + let bytes = export(&doc).unwrap(); + let f2 = parse(&bytes).unwrap(); + let Some(FBlock::Table(t2)) = f2.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("no table in export") }; + assert_eq!(t2.rows[0][0].valign, Some(VAlign::Center)); + assert_eq!(t2.rows[0][1].valign, Some(VAlign::Bottom)); +} + +/// #1379: three master pages (A4 upright; A4 wide with page numbering restarting at 1; A4 +/// upright again with numbering continuing) and a two-column `text:section` in between. +fn sections_package() -> Vec { + let styles = format!( + r#" + + + + + + + "# + ); + let content = format!( + r#" + + + + + + Opening wordsMore opening words + Wide part + AB + Left column textRight column text + Closing wordsLast line + "# + ); + package(&content, Some(&styles)) +} + +fn part_text(d: &Document, id: Option) -> String { + d.plain_text(craft_wordmodel::StoryRef::Part(id.expect("part"))) +} + +#[test] +fn headers_and_footers_follow_their_master_page() { + // #863: headers and footers live on the master pages in styles.xml. + let styles = format!( + r#" + + + + + + ACME Letterhead + Left page + First page + Page 1 + Hidden + + Wide header + "# + ); + let content = format!( + r#" + + + Upright bodyWide body + "# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let secs = d.sections(); + assert_eq!(secs.len(), 2); + let (up, wide) = (&secs[0].1, &secs[1].1); + assert_eq!(part_text(&d, up.headers.default), "ACME Letterhead"); + assert_eq!(part_text(&d, up.headers.even), "Left page"); + assert_eq!(part_text(&d, up.headers.first), "First page"); + assert_eq!(part_text(&d, up.footers.default), "Page 1"); + assert!(up.title_page && d.settings.even_odd_headers); + // The hidden left footer: left pages show the right pages' footer. + assert_eq!((up.footers.even, up.footers.first), (up.footers.default, None)); + assert_eq!(part_text(&d, wide.headers.default), "Wide header"); + assert_eq!((wide.headers.even, wide.title_page), (wide.headers.default, false)); + // Its master page has no footer: an empty one, not the previous section's. + assert_eq!(part_text(&d, wide.footers.default), ""); + assert_eq!(d.parts.len(), 6); + let id = up.headers.default.unwrap(); + let Some(craft_wordmodel::Block::Para(p)) = d.parts[&id].blocks.first().map(|b| &**b) else { panic!("no header paragraph") }; + assert!(p.runs.iter().all(|r| d.styles.resolve_char(p.props.style.as_deref(), &r.props).bold), "{:?}", p.runs); +} + +#[test] +fn documents_without_headers_get_none() { + let content = format!( + r#"Body"# + ); + let styles = format!( + r#""# + ); + for d in [import(&package(&content, Some(&styles))).unwrap(), import(&package(&content, None)).unwrap()] { + assert!(d.parts.is_empty()); + assert_eq!(d.last_section.headers, Default::default()); + assert!(!d.last_section.title_page && !d.settings.even_odd_headers); + } +} + +#[test] +fn deeply_nested_header_content_does_not_panic() { + let n = 4 * MAX_XML_DEPTH; + let (open, close) = ("".repeat(n), "".repeat(n)); + let styles = format!( + r#"{open}Deep{close}After"# + ); + let content = format!( + r#"Body"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + assert!(d.last_section.headers.default.is_some()); + assert_eq!(part_text(&d, d.last_section.footers.default), "After"); +} + +#[test] +fn headers_and_footers_share_the_text_s_budget_with_the_body() { + // Each header and footer was read with a budget of its own (#3700). + let run = r#""#.repeat(3_000); + let styles = format!( + r#"h{run}f{run}"# + ); + let content = format!( + r#"b{run}"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let texts = + [part_text(&d, d.last_section.headers.default), part_text(&d, d.last_section.footers.default), d.plain_text(craft_wordmodel::StoryRef::Body)]; + // One space per element in all three, and one budget of extra spaces between them. + assert_eq!(texts.iter().map(|t| t.matches(' ').count()).sum::(), 3 * 3_000 + MAX_EXTRA_SPACES); + assert!(texts[0].starts_with("h ") && texts[1].starts_with("f ") && texts[2].starts_with("b ")); +} + +#[test] +fn master_pages_and_column_sections_become_sections() { + use craft_wordmodel::section::SectionStart; + let doc = import(§ions_package()).unwrap(); + let secs = doc.sections(); + let summary: Vec<_> = + secs.iter().map(|(_, s)| (s.start, s.landscape, s.page_w.round(), s.page_h.round(), s.columns.count, s.page_num_start)).collect(); + assert_eq!( + summary, + [ + (SectionStart::NextPage, false, 595.0, 842.0, 1, None), + (SectionStart::NextPage, true, 842.0, 595.0, 1, Some(1)), + (SectionStart::Continuous, true, 842.0, 595.0, 2, None), + (SectionStart::NextPage, false, 595.0, 842.0, 1, None), + ] + ); + // Margins come from each section's page layout; the columns from the section style. + assert!((secs[0].1.margin_left - 70.87).abs() < 0.01 && (secs[1].1.margin_left - 42.52).abs() < 0.01); + let cols = &secs[2].1.columns; + assert!((cols.space - 28.35).abs() < 0.01 && cols.separator, "{cols:?}"); + // Each section ends on its own last paragraph; the wide one ends after its table. + let text = |i: usize| match doc.body.get(i).map(|b| &**b) { + Some(craft_wordmodel::Block::Para(p)) => p.plain_text().to_string(), + _ => String::from(""), + }; + assert_eq!(text(secs[0].0), "More opening words"); + assert!(matches!(doc.body.get(secs[1].0 - 1).map(|b| &**b), Some(craft_wordmodel::Block::Table(_)))); + assert_eq!(text(secs[2].0), "Right column text"); + assert_eq!(text(secs[3].0), "Last line"); +} + +/// (start, landscape, width, height, columns, restart, margins) of each section, and the +/// text of the paragraph ending it. +fn section_summary(doc: &Document) -> Vec<(craft_wordmodel::section::SectionStart, bool, f32, f32, u32, bool, Option, [f32; 4], String)> { + doc.sections() + .iter() + .map(|(end, s)| { + let text = match doc.body.get(*end).map(|b| &**b) { + Some(craft_wordmodel::Block::Para(p)) => p.plain_text().to_string(), + _ => String::new(), + }; + ( + s.start, + s.landscape, + s.page_w.round(), + s.page_h.round(), + s.columns.count, + s.columns.separator, + s.page_num_start, + [s.margin_top, s.margin_bottom, s.margin_left, s.margin_right].map(f32::round), + text, + ) + }) + .collect() +} + +#[test] +fn export_writes_sections_that_import_back() { + // Portrait; landscape on a new page restarting at 1; continuous two columns with a + // separator; portrait again. Export, import: the same sections. + let doc = import(§ions_package()).unwrap(); + let before = section_summary(&doc); + assert_eq!(before.len(), 4, "{before:?}"); + let back = import(&export(&doc).unwrap()).unwrap(); + let after = section_summary(&back); + assert_eq!(after, before); + let (a, b) = (&doc.sections(), &back.sections()); + for (x, y) in a.iter().zip(b.iter()) { + assert!((x.1.columns.space - y.1.columns.space).abs() < 0.1, "{:?} vs {:?}", x.1.columns, y.1.columns); + assert!((x.1.margin_top - y.1.margin_top).abs() < 0.1 && (x.1.margin_right - y.1.margin_right).abs() < 0.1); + } +} + +#[test] +fn export_starts_a_section_at_a_table_and_keeps_its_restart() { + // A section that starts with a table names its master page on the table's style. + let styles = format!( + r#""# + ); + let content = format!( + r#"IntroCellAfter"# + ); + let doc = import(&package(&content, Some(&styles))).unwrap(); + let before = section_summary(&doc); + assert_eq!(before.len(), 2, "{before:?}"); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(section_summary(&back), before); + assert_eq!(back.sections()[1].1.page_num_start, Some(5)); +} + +/// Master pages `Standard` (portrait) and `Cols` (landscape, two columns with a line) for the +/// round-trip tests below. +fn cols_styles() -> String { + format!( + r#""# + ) +} + +/// Import `body` (with automatic `styles`) over [`cols_styles`], export, import: both summaries. +fn round_trip(styles: &str, body: &str) -> (Vec, Vec) { + let content = format!( + r#"{styles}{body}"# + ); + let doc = import(&package(&content, Some(&cols_styles()))).unwrap(); + let back = import(&export(&doc).unwrap()).unwrap(); + let show = |d: &Document| section_summary(d).iter().map(|s| format!("{s:?}")).collect::>(); + (show(&doc), show(&back)) +} + +#[test] +fn export_keeps_a_columned_page_and_the_one_column_part_after_it() { + let styles = r#""#; + let body = r#"IntroTwo columnsOne column"#; + let (before, after) = round_trip(styles, body); + assert_eq!(before.len(), 3, "{before:?}"); + assert!(before[1].contains(", 2, true,") && before[2].contains("Continuous") && before[2].contains(", 1, false,"), "{before:?}"); + assert_eq!(after, before); +} + +#[test] +fn plain_one_column_text_sections_do_not_make_sections() { + let styles = r#""#; + let body = r#"ABC"#; + let (before, after) = round_trip(styles, body); + assert_eq!(before.len(), 1, "{before:?}"); + assert_eq!(after, before); +} + +#[test] +fn export_keeps_a_section_that_starts_on_a_list_item() { + let styles = r#""#; + let body = r#"IntroFirstSecondAfter"#; + let (before, after) = round_trip(styles, body); + assert_eq!(before.len(), 2, "{before:?}"); + assert_eq!(after, before); +} + +#[test] +fn a_list_cut_at_a_section_boundary_goes_on_numbering() { + let styles = r#""#; + let body = r#"OneTwo"#; + let content = format!( + r#"{styles}{body}"# + ); + let doc = import(&package(&content, Some(&cols_styles()))).unwrap(); + let bytes = crate::text::tests::export_model(&doc).unwrap(); + let xml = String::from_utf8(read_entry(&mut archive(&bytes), "content.xml").unwrap()).unwrap(); + assert_eq!(xml.matches("text:continue-list=\"").count(), 1, "{xml}"); + // The continuation points at the first list's xml:id. + assert!(xml.contains("xml:id=\"lst1\"") && xml.contains("text:continue-list=\"lst1\""), "{xml}"); + assert_eq!(section_summary(&import(&bytes).unwrap()), section_summary(&doc)); +} + +#[test] +fn export_keeps_a_page_number_restart_on_the_first_section() { + let styles = r#""#; + let body = r#"FirstSecond"#; + let (before, after) = round_trip(styles, body); + assert_eq!(before.len(), 1, "{before:?}"); + assert!(before[0].contains("Some(7)"), "{before:?}"); + assert_eq!(after, before); +} + +#[test] +fn export_of_a_one_section_document_stays_simple() { + let doc = import(§ions_package()).unwrap(); + let one = import(&crate::text::tests::export_model(&doc).unwrap()).unwrap(); + let mut single = one.clone(); + for b in single.body.iter_mut() { + if let craft_wordmodel::Block::Para(p) = std::sync::Arc::make_mut(b) { + p.section = None; + } + } + let bytes = crate::text::tests::export_model(&single).unwrap(); + let mut z = archive(&bytes); + let styles = String::from_utf8(read_entry(&mut z, "styles.xml").unwrap()).unwrap(); + let content = String::from_utf8(read_entry(&mut z, "content.xml").unwrap()).unwrap(); + assert_eq!(styles.matches(""# + ); + let content = format!( + r#"OneTwo"# + ); + let doc = import(&package(&content, Some(&styles))).unwrap(); + assert_eq!(doc.sections().len(), 1); + assert_eq!(doc.last_section.page_w.round(), 595.0); +} + +#[test] +fn hostile_sections_are_capped_and_clamped() { + // 20,000 master-page paragraphs give at most 10,000 sections; absurd column counts and + // gaps are clamped; a column section at the very end leaves no empty section after it. + let mut body = String::new(); + for _ in 0..20_000 { + body.push_str(r#"x"#); + } + body.push_str(r#"cols"#); + let content = format!( + r#"{body}"# + ); + let styles = format!( + r#""# + ); + let doc = import(&package(&content, Some(&styles))).unwrap(); + assert_eq!(doc.sections().len(), 10_000); + let content = format!( + r#"beforecols"# + ); + let doc = import(&package(&content, None)).unwrap(); + let secs = doc.sections(); + assert_eq!(secs.len(), 2); + assert_eq!(secs[1].1.columns.count, 45); + assert!(secs[1].1.columns.space <= 1584.0); +} + +#[test] +fn master_page_on_a_table_style_starts_a_section() { + // Review of #1379: a table style can name a master page too (ODF 1.3 §19.509), with its + // page-number restart on `style:table-properties`. An empty name on a child style means + // "no page style here", and a page-number restart without a master page starts nothing. + use craft_wordmodel::section::SectionStart; + let styles = format!( + r#" + + + + + "# + ); + let content = format!( + r#" + + + + + IntroNot a new page stylePage break only + Cell + After + "# + ); + let doc = import(&package(&content, Some(&styles))).unwrap(); + let secs = doc.sections(); + let summary: Vec<_> = secs.iter().map(|(i, s)| (*i, s.start, s.landscape, s.page_num_start)).collect(); + assert_eq!(summary, [(2, SectionStart::NextPage, false, None), (4, SectionStart::NextPage, true, Some(1))]); + assert!(matches!(doc.body.get(3).map(|b| &**b), Some(craft_wordmodel::Block::Table(_)))); + // A one-column page layout keeps the default column gap. + assert_eq!(secs[0].1.columns, craft_wordmodel::section::Columns::default()); +} + +#[test] +fn keep_with_next_survives_import_and_export() { + let content = format!( + r#" + + + + KeptLoosePlain + "# + ); + let doc = import(&package(&content, None)).unwrap(); + let flags = |d: &Document| d.body.iter().filter_map(|b| b.as_para()).map(|p| p.props.keep_next).collect::>(); + assert_eq!(flags(&doc), [Some(true), None, None]); + let again = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(flags(&again), [Some(true), None, None]); +} + +// --- Tracked changes (ODF 1.3 Part 3 §5.5) ------------------------------------------------- + +const NS_TC: &str = r#"xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:dc="http://purl.org/dc/elements/1.1/""#; + +/// An `.odt` whose body holds `regions` in `text:tracked-changes` and then `body`. +fn tracked(regions: &str, body: &str) -> Vec { + let content = format!( + r#"{regions}{body}"# + ); + package(&content, None) +} + +fn region(id: &str, kind: &str, author: &str, date: &str, content: &str) -> String { + format!( + r#"{author}{date}a remark on the change{content}"# + ) +} + +/// Each body paragraph as (text, revision kind) runs, plus its mark's revision kind. +type RevRuns = Vec<(Vec<(String, Option)>, Option)>; + +fn rev_runs(doc: &Document) -> RevRuns { + let kind = |i: Option| i.and_then(|i| doc.revisions.get(i as usize)).map(|r| r.kind); + doc.body + .iter() + .filter_map(|b| if let craft_wordmodel::Block::Para(p) = &**b { Some(p) } else { None }) + .map(|p| { + let runs = p.run_ranges().map(|(r, cp)| (p.text[r].to_string(), kind(cp.del).or(kind(cp.ins)))).collect(); + (runs, kind(p.mark.del).or(kind(p.mark.ins))) + }) + .collect() +} + +fn insertion_and_deletion() -> Vec { + let regions = region("ct1", "insertion", "Alice", "2024-03-01T10:00:00", "") + + ®ion("ct2", "deletion", "Bob", "2024-03-02T11:30:00", "old"); + tracked( + ®ions, + r#"Keep new words."#, + ) +} + +#[test] +fn tracked_insertion_and_deletion_import_with_authors_and_dates() { + use craft_wordmodel::RevisionKind::{Delete, Insert}; + let doc = import(&insertion_and_deletion()).unwrap(); + let s = |v: &str| v.to_string(); + assert_eq!(rev_runs(&doc), vec![(vec![(s("Keep "), None), (s("new "), Some(Insert)), (s("old "), Some(Delete)), (s("words."), None)], None)]); + let revs: Vec<_> = doc.revisions.iter().map(|r| (r.kind, r.author.as_str(), r.date.as_str())).collect(); + assert_eq!(revs, vec![(Insert, "Alice", "2024-03-01T10:00:00"), (Delete, "Bob", "2024-03-02T11:30:00")]); + assert!(doc.settings.track_changes, "text:track-changes=\"true\" turns tracking on"); +} + +#[test] +fn tracked_deletion_across_paragraphs_keeps_the_deleted_paragraph_mark() { + use craft_wordmodel::RevisionKind::Delete; + // "First tail¶head rest." with "tail¶head " deleted: the body holds the merged paragraph. + let regions = region("d1", "deletion", "Bob", "2024-03-02T11:30:00", "tailhead"); + let doc = import(&tracked(®ions, r#"First rest."#)).unwrap(); + let s = |v: &str| v.to_string(); + assert_eq!( + rev_runs(&doc), + vec![(vec![(s("First "), None), (s("tail"), Some(Delete))], Some(Delete)), (vec![(s("head "), Some(Delete)), (s("rest."), None)], None),] + ); +} + +#[test] +fn tracked_changes_round_trip_through_export() { + let regions = region("ct1", "insertion", "Alice", "2024-03-01T10:00:00", "") + + ®ion("d1", "deletion", "Bob", "2024-03-02T11:30:00", "tailhead"); + let body = r#"First rest addedwhole newEnd."#; + let doc = import(&tracked(®ions, body)).unwrap(); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(rev_runs(&back), rev_runs(&doc)); + let revs = |d: &Document| d.revisions.iter().map(|r| (r.kind, r.author.clone(), r.date.clone())).collect::>(); + assert_eq!(revs(&back), revs(&doc)); + assert_eq!(back.settings.track_changes, doc.settings.track_changes); + // The inserted paragraph mark survives: "added¶" is part of the insertion. + assert_eq!(rev_runs(&doc)[2].1, Some(craft_wordmodel::RevisionKind::Insert)); +} + +#[test] +fn an_inserted_ruby_keeps_its_tracked_insertion() { + // A ruby inside a tracked insertion is written as its base text, so that it stays inside + // the change range rather than becoming plain untracked text. + use craft_wordmodel::RevisionKind::Insert; + let props = craft_wordmodel::props::CharProps::default(); + let mut p = craft_wordmodel::Paragraph::with_text("before ", props.clone()); + let n = p.len(); + let mut ins = props.clone(); + ins.ins = Some(0); + p.insert_object(n, craft_flow::ruby_object("漢字".into(), "かんじ".into(), 20.0), &ins).unwrap(); + let mut d = Document::new(); + d.settings.track_changes = true; + d.revisions.push(craft_wordmodel::Revision::new(Insert, "Alice", "2024-03-01T10:00:00")); + d.body = vec![craft_wordmodel::para_block(p)]; + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(rev_runs(&back), vec![(vec![("before ".to_string(), None), ("漢字".to_string(), Some(Insert))], None)]); +} + +#[test] +fn hostile_tracked_changes_are_bounded() { + // Unknown and duplicate ids, a change-end without a start, nested regions, a deletion + // inside a deletion, a change mark inside the deleted text and tracked changes inside a paragraph. + let regions = region("d1", "deletion", "B", "x", r#"gone"#) + + r#"inner"# + + ®ion("dd", "deletion", "B", "x", "deeperok") + + ®ion("d1", "insertion", "C", "y", ""); + let body = r#"abc"#; + let doc = import(&tracked(®ions, body)).unwrap(); + let text: String = rev_runs(&doc).iter().flat_map(|(r, _)| r.iter().map(|(t, _)| t.clone())).collect(); + assert_eq!(text.matches("gone").count(), 1, "a deletion is placed once: {text}"); + assert!(!text.contains("deeper"), "{text}"); + assert!(text.starts_with('a') && text.ends_with('c'), "{text}"); + + // A huge number of regions is capped. + let many: String = (0..20_000).map(|i| format!(r#""#)).collect(); + let starts: String = (0..20_000).map(|i| format!(r#"x"#)).collect(); + let doc = import(&tracked(&many, &format!("{starts}"))).unwrap(); + assert!(doc.revisions.len() <= 10_000, "{}", doc.revisions.len()); +} + +/// Every paragraph (body and table cells, in order) as (text, inserted, deleted) runs plus +/// its mark's (inserted, deleted). +type Marks = Vec<(Vec<(String, bool, bool)>, (bool, bool))>; + +fn marks(doc: &Document) -> Marks { + fn walk(bl: &craft_wordmodel::Blocks, out: &mut Marks) { + for b in bl { + match &**b { + craft_wordmodel::Block::Para(p) => out.push(( + p.run_ranges().map(|(r, cp)| (p.text[r].to_string(), cp.ins.is_some(), cp.del.is_some())).collect(), + (p.mark.ins.is_some(), p.mark.del.is_some()), + )), + craft_wordmodel::Block::Table(t) => { + for r in &t.rows { + for c in &r.cells { + walk(&c.blocks, out); + } + } + } + } + } + } + let mut out = Vec::new(); + walk(&doc.body, &mut out); + out +} + +#[test] +fn inserted_then_deleted_text_keeps_both_marks() { + // "new" was inserted by Alice, then Bob deleted "old new": the deletion's content holds + // the insertion as a change range. + let regions = region("i1", "insertion", "Alice", "2024-03-01T10:00:00", "") + + ®ion( + "d1", + "deletion", + "Bob", + "2024-03-02T11:30:00", + r#"old new"#, + ); + let doc = import(&tracked(®ions, r#"Keep end."#)).unwrap(); + let s = |v: &str| v.to_string(); + let want = vec![(vec![(s("Keep "), false, false), (s("old "), false, true), (s("new"), true, true), (s("end."), false, false)], (false, false))]; + assert_eq!(marks(&doc), want); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(marks(&back), want, "both marks survive a save"); + assert_eq!(back.revisions.len(), doc.revisions.len(), "one deletion stays one region and one revision: {:?}", back.revisions); +} + +#[test] +fn a_deleted_mark_inside_an_insertion_is_not_inserted() { + // "A" inserted by Alice, then a paragraph break Bob deleted, then "C" inserted by Alice: + // the insertion range runs across the deletion point, but the deleted mark wasn't inserted. + let regions = region("i1", "insertion", "Alice", "2024-03-01T10:00:00", "") + + ®ion("d1", "deletion", "Bob", "2024-03-02T11:30:00", ""); + let body = + r#"AC"#; + let doc = import(&tracked(®ions, body)).unwrap(); + let s = |v: &str| v.to_string(); + let want = vec![(vec![(s("A"), true, false)], (false, true)), (vec![(s("C"), true, false)], (false, false))]; + assert_eq!(marks(&doc), want); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(marks(&back), want, "after a save"); +} + +#[test] +fn insertions_end_before_a_table_and_inside_a_cell() { + let regions = region("i1", "insertion", "Alice", "2024-03-01T10:00:00", "") + ®ion("i2", "insertion", "Bob", "2024-03-02T10:00:00", ""); + let table = r#"Cell added"#; + let body = format!( + r#"StartNew paragraph{table}End."# + ); + let content = format!( + r#"{regions}{body}"# + ); + let doc = import(&package(&content, None)).unwrap(); + let s = |v: &str| v.to_string(); + let want = vec![ + (vec![(s("Start"), false, false)], (false, false)), + (vec![(s("New paragraph"), true, false)], (true, false)), + (vec![(s("Cell "), false, false), (s("added"), true, false)], (true, false)), + (vec![(s("End."), false, false)], (false, false)), + ]; + assert_eq!(marks(&doc), want); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(marks(&back), want); + let authors = |d: &Document| d.revisions.iter().map(|r| r.author.clone()).collect::>(); + assert_eq!(authors(&back), authors(&doc)); +} + +#[test] +fn exported_changes_always_have_a_date_and_keep_paragraph_separators() { + // A change without a date gets one on save (`dc:date` is required); a U+2029 in deleted + // text is text, not a paragraph break. + let regions = r#"Boba
b"#; + let doc = import(&tracked(regions, r#"xy"#)).unwrap(); + assert_eq!(doc.revisions.first().map(|r| r.date.as_str()), Some("")); + let back = import(&export(&doc).unwrap()).unwrap(); + assert!(back.revisions.first().is_some_and(|r| r.date.len() >= 19 && r.date.starts_with("20")), "{:?}", back.revisions); + assert_eq!(marks(&back), marks(&doc)); + assert_eq!(marks(&back).len(), 1); +} + +#[test] +fn spaces_after_a_tab_survive_a_round_trip() { + for src in ["a\t b", "a\t b", "a\t éa", "é\t 😀a", "a\t\t b"] { + let doc = Document::from_text(src); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(back.plain_text(craft_wordmodel::StoryRef::Body).trim_end(), src, "{src:?}"); + } +} + +#[test] +fn whitespace_in_attributes_is_trimmed() { + let content = format!( + r#" + Heading + AB + + + + + Cell + + + + "# + ); + let flow = parse(&package(&content, None)).unwrap(); + let h = match &flow.blocks[0] { + FBlock::Para(p) => p.kind, + _ => panic!("expected heading para"), + }; + assert_eq!(h, Kind::Heading(2)); + + let p_text = match &flow.blocks[1] { + FBlock::Para(p) => p.text(), + _ => panic!("expected paragraph"), + }; + assert_eq!(p_text, "A B"); + + let colspan = match &flow.blocks[2] { + FBlock::Table(t) => t.rows[0][0].colspan, + _ => panic!("expected table"), + }; + assert_eq!(colspan, 2); +} + +#[test] +fn explicit_spaces_at_paragraph_ends_are_kept() { + // `text:s` spaces are content; only collapsible XML whitespace is trimmed at the ends. + let content = format!( + r#"leadtrail plain x "# + ); + let f = parse(&package(&content, None)).unwrap(); + let texts: Vec = paras(&f).iter().map(|p| p.text()).collect(); + assert_eq!(texts, vec![" lead", "trail ", "plain", " x "]); + let d = Document::from_text(" indented code "); + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(back.plain_text(craft_wordmodel::StoryRef::Body).trim_end_matches('\n'), " indented code "); + for t in ["trail ", "trail "] { + let d = Document::from_text(t); + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(back.plain_text(craft_wordmodel::StoryRef::Body).trim_end_matches('\n'), t); + } +} + +#[test] +fn text_s_spaces_have_a_budget_for_the_whole_document() { + // 5,000 elements asking for 1,000 spaces each would add 5 million (#3700). + let tags = 5_000; + let run = r#""#.repeat(tags); + let content = format!( + r#"a{run}bcd"# + ); + craft_flow::begin_import(); + let f = parse(&package(&content, None)).unwrap(); + let texts: Vec = paras(&f).iter().map(|p| p.text()).collect(); + // Once the budget is spent, each element still stands for one space, and the cut is logged. + assert_eq!(texts[0].len(), 2 + tags + MAX_EXTRA_SPACES); + assert!(texts[0].starts_with("a ") && texts[0].ends_with(" b")); + assert_eq!(texts[1], "c d"); + assert!(craft_flow::Dropped::Spaces.reported()); +} + +#[test] +fn text_s_outside_a_paragraph_spends_no_budget() { + // Spaces outside a paragraph are dropped, so they mustn't use up the budget (#3700). + let stray = r#""#.repeat(5_000); + let content = format!( + r#"{stray}ab"# + ); + craft_flow::begin_import(); + let f = parse(&package(&content, None)).unwrap(); + let texts: Vec = paras(&f).iter().map(|p| p.text()).collect(); + assert_eq!(texts, vec![format!("a{}b", " ".repeat(1000))]); + assert!(!craft_flow::Dropped::Spaces.reported(), "nothing was cut"); +} + +#[test] +fn test_asian_complex_and_extended_text_properties() { + let content = format!( + r#" + + + + + + + Asian + Complex + Underlined + Struck + Centered + "# + ); + let f = parse(&package(&content, None)).unwrap(); + let ps = paras(&f); + assert_eq!(ps.len(), 5); + + let fmt = |p: &Para| p.inlines.iter().find_map(|i| if let Inline::Text(_, f) = i { Some(f.clone()) } else { None }).unwrap(); + + assert!(fmt(ps[0]).bold); + assert!(fmt(ps[1]).italic); + assert!(fmt(ps[2]).underline); + assert!(fmt(ps[3]).strike); + assert!(fmt(ps[4]).bold); + assert_eq!(ps[4].align, Some(Align::Center)); + assert!(ps[4].page_break); +} + +#[test] +fn latin_attributes_win_over_asian_and_line_types() { + let content = format!( + r#" + + + + Latin + Plain + "# + ); + let f = parse(&package(&content, None)).unwrap(); + let ps = paras(&f); + let fmt = |p: &Para| p.inlines.iter().find_map(|i| if let Inline::Text(_, f) = i { Some(f.clone()) } else { None }).unwrap(); + assert!(!fmt(ps[0]).bold, "Asian bold does not make Latin text bold"); + assert!(!fmt(ps[0]).italic, "complex-script italic does not make Latin text italic"); + assert!(!fmt(ps[1]).underline, "a line type does not override a line style of none"); + assert!(!fmt(ps[1]).strike); +} + +#[test] +fn meta_creation_date_timezones_and_fallbacks() { + let xml_neg = br#" + 2024-03-05T13:45:00-05:00 + Modifier + Original Author + "#; + let m = read_meta(xml_neg); + assert_eq!(m.created, "2024-03-05T13:45:00-05:00"); + assert_eq!(m.author, "Original Author"); + + let xml_neg_frac = br#" + 2024-03-05T13:45:00.500-05:00 + "#; + assert_eq!(read_meta(xml_neg_frac).created, "2024-03-05T13:45:00.500-05:00"); + + let xml_local_frac = br#" + 2024-03-05T13:45:00.123 + "#; + assert_eq!(read_meta(xml_local_frac).created, "2024-03-05T13:45:00Z"); + + let xml_z_lower = br#" + 2024-03-05T13:45:00z + "#; + assert_eq!(read_meta(xml_z_lower).created, "2024-03-05T13:45:00z"); + + let xml_dc_date_fallback = br#" + 2024-03-05T13:45:00Z + Solo Creator + "#; + let m2 = read_meta(xml_dc_date_fallback); + assert_eq!(m2.created, "2024-03-05T13:45:00Z"); + assert_eq!(m2.author, "Solo Creator"); +} + +#[test] +fn a_later_default_style_without_a_font_keeps_the_default_font() { + // #4501 follow-up: styles read twice (styles.xml, then content.xml) must not wipe the font. + let mut styles = styles::Styles::default(); + let with_font = format!( + r#""# + ); + let without = format!( + r#""# + ); + styles.read(with_font.as_bytes(), false); + styles.read(without.as_bytes(), false); + assert_eq!(styles.doc_font().as_deref(), Some("DejaVu Serif")); +} + +#[test] +fn a_later_monospaced_default_style_keeps_the_proportional_default_font() { + // #4501 follow-up: a monospaced default font is rejected and must not wipe the earlier one. + let mut styles = styles::Styles::default(); + let with_font = format!( + r#""# + ); + let mono = format!( + r#""# + ); + styles.read(with_font.as_bytes(), false); + styles.read(mono.as_bytes(), false); + assert_eq!(styles.doc_font().as_deref(), Some("DejaVu Serif")); +} + +/// The `content.xml` of a package. +pub(super) fn content_of(bytes: &[u8]) -> String { + let mut z = archive(bytes); + String::from_utf8(read_entry(&mut z, "content.xml").unwrap()).unwrap() +} + +/// Entry `name` of a package, as text. +/// `d` saved from its model alone, as after an edit: nothing written as it was read (for tests +/// of how WordCraft writes what it models). +pub(crate) fn export_model(d: &craft_wordmodel::Document) -> Result, String> { + let mut d = d.clone(); + d.kept.clear(); + super::export(&d) +} + +pub(super) fn entry(bytes: &[u8], name: &str) -> String { + let mut z = archive(bytes); + String::from_utf8(read_entry(&mut z, name).unwrap()).unwrap() +} + +/// The `styles.xml` of a package. +pub(super) fn styles_of(bytes: &[u8]) -> String { + let mut z = archive(bytes); + String::from_utf8(read_entry(&mut z, "styles.xml").unwrap()).unwrap() +} + +#[test] +fn the_default_styles_font_size_is_the_documents() { + // LibreOffice's letter templates: 10 pt in `style:default-style`; the Standard style's own + // size (the document's defaults) wins over it. + let content = format!( + r#"Hi"# + ); + let default_only = format!( + r#""# + ); + let doc = import(&package(&content, Some(&default_only))).unwrap(); + assert_eq!(doc.styles.default_chr.size, Some(10.0)); + let p = doc.body[0].as_para().unwrap(); + assert_eq!(doc.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props).size, 10.0); + let standard = default_only.replace( + "", + r#""#, + ); + let d = import(&package(&content, Some(&standard))).unwrap(); + assert_eq!(d.styles.default_chr.size, Some(11.0)); + let p = d.body[0].as_para().unwrap(); + assert_eq!(d.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props).size, 11.0); + // Written back as the default style's. + let s = entry(&crate::text::tests::export_model(&doc).unwrap(), "styles.xml"); + assert!(s.contains(r#"fo:font-size="10pt""#), "{s}"); + // Standard's 11 pt too: LibreOffice's letters grew a page at 12 pt. + let s = entry(&crate::text::tests::export_model(&d).unwrap(), "styles.xml"); + assert!( + s.contains(r#"HiPlain"# + ); + let styles = format!( + r#""# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let sizes: Vec = + d.body.iter().filter_map(|b| b.as_para()).map(|p| d.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props).size).collect(); + assert_eq!(sizes, [12.0, 11.0]); +} + +#[test] +fn an_empty_paragraph_keeps_its_marks_size() { + // LibreOffice's fax template: an empty 6 pt line above "Important!". Its size is the mark's, + // the paragraph's height, and is written back on the paragraph's style. + let content = format!( + r#"Text"# + ); + let doc = import(&package(&content, None)).unwrap(); + let marks = |d: &Document| d.body.iter().filter_map(|b| b.as_para()).map(|p| p.mark.size).collect::>(); + // Only the empty paragraph's: the other's text has the size itself. + assert_eq!(marks(&doc), [Some(6.0), None]); + let s = entry(&export(&doc).unwrap(), "content.xml"); + assert!(s.contains(r#"fo:font-size="6pt""#), "{s}"); + let back = import(&export(&doc).unwrap()).unwrap(); + assert_eq!(marks(&back), [Some(6.0), None]); + let p = back.body[1].as_para().unwrap(); + assert_eq!(p.runs[0].props.size, Some(6.0)); +} + +#[test] +fn unequal_columns_keep_their_widths_and_columns_without_a_gap_have_none() { + // LibreOffice's one-page résumé: a narrow first column and a wide second one (shares of the + // width, `style:rel-width`), written back as shares (and without a gap, which would make + // LibreOffice ignore them on a page). + let styles = format!( + r#""# + ); + let content = format!( + r#"x"# + ); + let doc = import(&package(&content, Some(&styles))).unwrap(); + let widths = + |d: &Document| d.last_section.columns.widths.iter().map(|(w, a)| ((w * 10.0).round() / 10.0, (a * 10.0).round() / 10.0)).collect::>(); + // 6.5 in of text: 2.1667 in less 0.1 in, then 4.3333 in less 0.1 in; 0.2 in between. + assert_eq!(widths(&doc), [(148.8, 14.4), (304.8, 0.0)]); + let out = entry(&export(&doc).unwrap(), "styles.xml"); + assert!(out.contains(r#""#, ""); + let c = import(&package(&content, Some(&bare))).unwrap().last_section.columns; + assert_eq!((c.count, c.space, c.widths.len()), (2, 36.0, 0)); +} + +#[test] +fn a_paragraph_starting_a_column_starts_with_a_column_break() { + // LibreOffice's résumé: "Education" starts the second column (`fo:break-before="column"`), + // Word's column break before it, written back on the paragraph. + let content = format!( + r#"SideEducation"# + ); + let doc = import(&package(&content, None)).unwrap(); + let text = |d: &Document| d.body.iter().filter_map(|b| b.as_para()).map(|p| p.text.clone()).collect::>(); + assert_eq!(text(&doc), ["Side", "\u{e}Education"]); + let out = content_of(&export(&doc).unwrap()); + assert!(out.contains(r#"fo:break-before="column""#) && !out.contains('\u{e}'), "{out}"); + assert_eq!(text(&import(&export(&doc).unwrap()).unwrap()), ["Side", "\u{e}Education"]); +} + +#[test] +fn meta_modified_date_keeps_a_negative_offset() { + let meta = |date: &str| format!("{date}"); + // A `-hh:mm` offset is a zone: no `Z` after it (an invalid xsd:dateTime written back on save). + assert_eq!(super::import::read_changes(meta("2024-03-05T13:45:00-05:00").as_bytes()).1, "2024-03-05T13:45:00-05:00"); + assert_eq!(super::import::read_changes(meta("2024-03-05T13:45:00+01:00").as_bytes()).1, "2024-03-05T13:45:00+01:00"); + assert_eq!(super::import::read_changes(meta("2024-03-05T13:45:00Z").as_bytes()).1, "2024-03-05T13:45:00Z"); + // Zoneless (the date's own `-`s are not an offset): UTC, fraction dropped. + assert_eq!(super::import::read_changes(meta("2024-03-05T13:45:00.25").as_bytes()).1, "2024-03-05T13:45:00Z"); + assert_eq!(super::import::read_changes(meta("").as_bytes()).1, ""); +} + +/// WordCraft's dispatch by extension, for the tests that went through it. +pub(crate) fn import_ext(ext: &str, bytes: &[u8]) -> Option> { + let r = match ext { + "odt" | "ott" => super::import(bytes), + "fodt" => super::import_flat(bytes), + _ => return None, + }; + Some(r.map(|mut d| { + d.ensure_nonempty(); + d + })) +} + +/// [`import_ext`] onto `page`. +pub(crate) fn import_on_ext(ext: &str, bytes: &[u8], page: &craft_wordmodel::SectionProps) -> Option> { + match ext { + "odt" | "ott" => Some(super::import_on(bytes, page)), + "fodt" => Some(super::import_flat_on(bytes, page)), + _ => None, + } +} + +/// WordCraft's export by extension. +pub(crate) fn export_ext(ext: &str, doc: &Document) -> Option, String>> { + Some(match ext { + "odt" => super::export(doc), + "ott" => super::export_template(doc), + "fodt" => super::export_flat(doc), + _ => return None, + }) +} diff --git a/crates/odf/src/text/tests_more.rs b/crates/odf/src/text/tests_more.rs new file mode 100644 index 000000000..7488f85ef --- /dev/null +++ b/crates/odf/src/text/tests_more.rs @@ -0,0 +1,49 @@ +//! Tests for the ODT features added after the first reader and writer: one module per feature. + +mod bookmarks; +mod cell_formulas; +mod char_styles; +mod chars; +mod charts; +mod comments; +mod connectors; +mod custom_geometry; +mod drop_caps; +mod encryption; +mod equations; +mod fields; +mod flat; +mod floats; +mod flow_breaks; +mod fonts; +mod format_changes; +mod forms; +mod groups; +mod headers; +mod indexes; +mod kept; +mod lists; +mod lossless; +mod named_styles; +mod notes; +mod numbering_fields; +mod objects; +mod outline; +mod page_setup; +mod para_borders; +mod parts; +mod placeholders; +mod polygons; +mod rotation; +mod rtl; +mod ruby; +mod settings; +mod shapes; +mod spacing; +mod strokes; +mod style_look; +mod tables; +mod tabs; +mod text_boxes; +mod text_direction; +mod unstated; diff --git a/crates/odf/src/text/tests_more/bookmarks.rs b/crates/odf/src/text/tests_more/bookmarks.rs new file mode 100644 index 000000000..9191f251e --- /dev/null +++ b/crates/odf/src/text/tests_more/bookmarks.rs @@ -0,0 +1,82 @@ +//! Bookmarks (ODF 1.3 §6.2.1 `text:bookmark`, `text:bookmark-start`/`-end`): a point or the text +//! between its start and end, in one paragraph or across several, kept as a WordCraft bookmark +//! over that text (what a reference to it shows) and written back the same way. + +use crate::text::tests::{NS_T, content_of, package}; +use crate::text::{export, import}; +use craft_wordmodel::Document; +use craft_wordmodel::para::{InlineObject, OBJ}; + +/// Each bookmark: (name, the text it covers, paragraphs joined by '|'). +fn bookmarks(d: &Document) -> Vec<(String, String)> { + let mut open: Vec<(String, String)> = Vec::new(); + let mut done = Vec::new(); + for (i, b) in d.body.iter().enumerate() { + let Some(p) = b.as_para() else { continue }; + if i > 0 { + for o in &mut open { + o.1.push('|'); + } + } + let mut k = 0; + for ch in p.text.chars() { + if ch == OBJ { + match p.objects.get(k) { + Some(InlineObject::BookmarkStart { name }) => open.push((name.clone(), String::new())), + Some(InlineObject::BookmarkEnd { name }) => { + if let Some(at) = open.iter().position(|(n, _)| n == name) { + done.push(open.remove(at)); + } + } + _ => {} + } + k += 1; + } else { + for o in &mut open { + o.1.push(ch); + } + } + } + } + done.sort(); + done +} + +fn open() -> Document { + let content = format!( + r#"Before important words after.From hereto there."# + ); + import(&package(&content, None)).unwrap() +} + +#[test] +fn a_bookmark_covers_its_text() { + assert_eq!( + bookmarks(&open()), + [("span".to_string(), "here|to there".to_string()), ("spot".into(), String::new()), ("words".into(), "important words".into())] + ); +} + +#[test] +fn bookmarks_round_trip() { + let d = open(); + let bytes = export(&d).unwrap(); + let c = content_of(&bytes); + assert!(c.contains(r#""#) && c.contains(r#""#), "{c}"); + assert_eq!(bookmarks(&import(&bytes).unwrap()), bookmarks(&d)); +} + +#[test] +fn hostile_bookmarks_do_not_panic() { + for body in [ + r#"x"#, + r#"no end"#, + r#"twice"#, + r#"nameless"#, + ] { + let content = + format!(r#"{body}"#); + let d = import(&package(&content, None)).unwrap(); + let _ = export(&d).unwrap(); + } +} diff --git a/crates/odf/src/text/tests_more/cell_formulas.rs b/crates/odf/src/text/tests_more/cell_formulas.rs new file mode 100644 index 000000000..b12aa04e6 --- /dev/null +++ b/crates/odf/src/text/tests_more/cell_formulas.rs @@ -0,0 +1,117 @@ +//! Table cell formulas (ODF 1.3 §19.646 `table:formula`, with the cell's value +//! `office:value`): LibreOffice's (`ooow:sum `) open as Word formula fields +//! (`=SUM(A1:A3)`) showing the cell's value, so Update Field recalculates them, and formula +//! fields in cells are written back as cell formulas. + +use crate::text::cell_formula::{odf_of, word_of}; +use crate::text::import; +use crate::text::tests::{NS_T, content_of, package}; +use craft_wordmodel::InlineObject; + +#[test] +fn libreoffice_formulas_say_what_word_formulas_do() { + for (odf, word) in [ + ("ooow:sum ", "=SUM(A1:A3)"), + ("ooow:*+", "=B1*B2+B3"), + ("ooow:mean ", "=AVERAGE(A1:B3)"), + ("ooow:sum ||3", "=SUM(A1,B2,3)"), + ("ooow: add mul 2", "=A1+A2*2"), + ("ooow:max (-1)|", "=MAX(A1-1,B1)"), + ("=-(/4)", "=-(C2/4)"), + // Brackets round the whole say nothing; others stay. + ("ooow:((sum ))", "=SUM(A1:A2)"), + ("ooow:()+()", "=(A1)+(A2)"), + ] { + assert_eq!(word_of(odf).as_deref(), Some(word), "{odf}"); + } + // What Word can't say: another table's cells, logic written as words, junk. + for odf in ["ooow:+1", "ooow: and ", "ooow:sum", "ooow:|", "ooow:$x", "", "ooow:)))"] { + assert_eq!(word_of(odf), None, "{odf}"); + } +} + +#[test] +fn word_formulas_are_written_as_libreoffice_says_them() { + // In the last row (row 3 of 4, from 0) of a 4 × 3 table, column B. + let at = ((3, 1), (4, 3)); + for (word, odf) in [ + ("=SUM(ABOVE)", "ooow:(sum )"), + ("=SUM(LEFT)", "ooow:(sum )"), + ("=sum(right) \\# \"0.00\"", "ooow:(sum )"), + ("=AVERAGE(A1:B3,4)", "ooow:(mean |4)"), + ("=B1*B2+B3", "ooow: * + "), + ("=(A1-2)/4", "ooow:( - 2 ) / 4"), + ] { + assert_eq!(odf_of(word, at.0, at.1).as_deref(), Some(odf), "{word}"); + } + // Nothing below the last row; logic Word has that LibreOffice writes otherwise; not a formula. + for word in ["=SUM(BELOW)", "=IF(A1>2,1,0)", "=SUM(A1", "PAGE", "=", "=SUM(ABOVE,(A1))", "=Sheet!A1"] { + assert_eq!(odf_of(word, at.0, at.1), None, "{word}"); + } + // Above the first row and left of the first column there is nothing. + assert_eq!(odf_of("=SUM(ABOVE)", (0, 0), (2, 2)), None); + assert_eq!(odf_of("=SUM(LEFT)", (0, 0), (2, 2)), None); +} + +const TABLE: &str = r#" +12.5 +34 +42.5 +"#; + +fn open() -> craft_wordmodel::Document { + let content = + format!(r#"{TABLE}"#); + import(&package(&content, None)).unwrap() +} + +/// The fields in the table's cells: (row, column, code, result). +fn fields(d: &craft_wordmodel::Document) -> Vec<(usize, usize, String, String)> { + let mut out = Vec::new(); + let Some(t) = d.body.iter().find_map(|b| b.as_table()) else { return out }; + for (r, row) in t.rows.iter().enumerate() { + for (c, cell) in row.cells.iter().enumerate() { + for p in cell.blocks.iter().filter_map(|b| b.as_para()) { + for o in &p.objects { + if let InlineObject::Field { instr, result, .. } = o { + out.push((r, c, instr.clone(), result.clone())); + } + } + } + } + } + out +} + +#[test] +fn a_cell_formula_opens_as_a_formula_field_showing_its_value() { + let d = open(); + assert_eq!(fields(&d), [(2, 0, "=SUM(A1:A2)".to_string(), "4".to_string())]); + // One Word can't say keeps its value as text. + let t = d.body.iter().find_map(|b| b.as_table()).unwrap(); + let last = t.rows[2].cells[1].blocks[0].as_para().unwrap(); + assert_eq!(last.text, "2.5"); +} + +#[test] +fn formula_fields_in_cells_are_written_as_cell_formulas() { + let d = open(); + let c = content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#"table:formula="ooow:(sum <A1:A2>)" office:value-type="float" office:value="4""#), "{c}"); + // Read back, the same field. + let again = import(&crate::text::tests::export_model(&d).unwrap()).unwrap(); + assert_eq!(fields(&again), fields(&d)); +} + +#[test] +fn cells_holding_numbers_are_written_as_numbers() { + let c = content_of(&crate::text::tests::export_model(&open()).unwrap()); + assert!(c.contains(r#"office:value-type="float" office:value="2.5">x1,5-1.2.3 -7 "# + ); + let c = content_of(&crate::text::tests::export_model(&import(&package(&content, None)).unwrap()).unwrap()); + assert_eq!(c.matches(r#"office:value-type="float""#).count(), 1, "{c}"); + assert!(c.contains(r#"office:value="-7""#), "{c}"); +} diff --git a/crates/odf/src/text/tests_more/char_styles.rs b/crates/odf/src/text/tests_more/char_styles.rs new file mode 100644 index 000000000..4ffcb7bc1 --- /dev/null +++ b/crates/odf/src/text/tests_more/char_styles.rs @@ -0,0 +1,91 @@ +//! Custom character styles (ODF 1.3 §16.2 common `style:style` of family text): kept as document +//! character styles, the text in them keeping the style; written back as common text styles. + +use crate::text::tests::{NS_T, package}; +use crate::text::{export, import}; +use craft_wordmodel::Document; + +fn open() -> Document { + let styles = format!( + r##""## + ); + let content = format!( + r#"a em b loud c big"# + ); + import(&package(&content, Some(&styles))).unwrap() +} + +/// Each run of the first paragraph: (its text, its character style's name, resolved bold, italic, size). +fn runs(d: &Document) -> Vec<(String, Option, bool, bool, f32)> { + let p = d.body[0].as_para().unwrap(); + p.run_ranges() + .map(|(r, props)| { + let c = d.styles.resolve_char(p.props.style.as_deref(), props); + let name = props.style.as_ref().and_then(|id| d.styles.get(id)).map(|s| s.name.clone()); + (p.text.get(r).unwrap_or("").to_string(), name, c.bold, c.italic, c.size) + }) + .filter(|r| r.0.trim().len() > 1) + .collect() +} + +#[test] +fn text_keeps_its_character_style() { + let d = open(); + let r = runs(&d); + assert_eq!( + r.iter().map(|x| (x.0.as_str(), x.1.as_deref())).collect::>(), + [("em", Some("Emphasis")), ("loud", Some("Loud words")), ("big", Some("Loud words"))] + ); + // As shown: its style's formatting, and the parent's. + assert!(r[0].3 && !r[0].2); + assert!(r[1].2 && r[1].3); + assert!(r[2].2 && r[2].3 && r[2].4 == 18.0, "{:?}", r[2]); + // The style's own formatting stays in it. + let p = d.body[0].as_para().unwrap(); + let loud = p.run_ranges().find(|(r, _)| p.text.get(r.clone()) == Some("loud")).map(|(_, c)| c.clone()).unwrap(); + assert!(loud.bold.is_none() && loud.italic.is_none() && loud.color.is_none(), "{loud:?}"); +} + +#[test] +fn character_styles_round_trip() { + let d = open(); + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(runs(&back), runs(&d)); +} + +#[test] +fn what_a_character_style_turns_off_stays_off() { + // "Plain" is based on the bold, italic "Loud" and turns both off. + let styles = format!( + r#""# + ); + let content = format!( + r#"ab"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let p = d.body[0].as_para().unwrap(); + let shown: Vec<(bool, bool)> = + p.runs.iter().map(|r| d.styles.resolve_char(p.props.style.as_deref(), &r.props)).map(|c| (c.bold, c.italic)).collect(); + assert_eq!(shown, [(true, true), (false, false)]); + let s = crate::text::tests::entry(&export(&d).unwrap(), "styles.xml"); + let plain = s.split(r#"style:name="Plain""#).nth(1).and_then(|r| r.split("").next()).unwrap_or("missing"); + assert!(plain.contains(r#"fo:font-weight="normal""#) && plain.contains(r#"fo:font-style="normal""#), "{plain}"); +} + +#[test] +fn a_character_style_based_on_a_missing_one_keeps_its_own_look() { + // "Orphan" names a parent the file doesn't define; "Tilted" comes after it. + let styles = format!( + r#""# + ); + let content = format!( + r#"aa bb"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let r = runs(&d); + assert_eq!( + r.iter().map(|x| (x.0.as_str(), x.1.as_deref(), x.2, x.3)).collect::>(), + [("aa", Some("Orphan"), true, false), ("bb", Some("Tilted"), false, true)] + ); + assert!(d.styles.get("Nowhere").is_none(), "no style for the missing parent"); +} diff --git a/crates/odf/src/text/tests_more/chars.rs b/crates/odf/src/text/tests_more/chars.rs new file mode 100644 index 000000000..338e8b2bf --- /dev/null +++ b/crates/odf/src/text/tests_more/chars.rs @@ -0,0 +1,838 @@ +//! Character formatting beyond bold, italic and colour (ODF 1.3 §20.2xx `style:text-properties`): +//! underline styles and colour, double strike, caps, small caps, hidden text, letter spacing, +//! scale, raised and lowered text, kerning, outline, shadow, relief and language. + +use crate::text::tests::{NS_T, package}; +use crate::text::{export, import}; +use craft_wordmodel::props::{CharProps, Underline}; +use craft_wordmodel::{Document, Paragraph, Rgb, para_block}; + +fn spans(props: &[&str]) -> Document { + let styles: String = props + .iter() + .enumerate() + .map(|(i, p)| format!(r#""#)) + .collect(); + let body: String = (0..props.len()).map(|i| format!(r#"s{i}|"#)).collect(); + let content = format!( + r#"{styles}{body}"# + ); + import(&package(&content, None)).unwrap() +} + +/// The formatting of each `sN` span. +fn runs(d: &Document) -> Vec { + let p = d.body[0].as_para().unwrap(); + p.run_ranges() + .filter(|(r, _)| p.text.get(r.clone()).is_some_and(|t| t.trim_start_matches('|').starts_with('s'))) + .map(|(_, props)| props.clone()) + .collect() +} + +#[test] +fn underline_styles_and_strikes_are_read() { + let d = spans(&[ + r#"style:text-underline-style="solid" style:text-underline-type="double""#, + r#"style:text-underline-style="dotted""#, + r#"style:text-underline-style="wave""#, + r#"style:text-underline-style="solid" style:text-underline-width="bold""#, + r##"style:text-underline-style="dash" style:text-underline-color="#ff0000""##, + r#"style:text-underline-style="solid" style:text-underline-mode="skip-white-space""#, + r#"style:text-line-through-style="solid" style:text-line-through-type="double""#, + ]); + let r = runs(&d); + let u: Vec> = r.iter().map(|p| p.underline).collect(); + assert_eq!( + u[..6], + [ + Some(Underline::Double), + Some(Underline::Dotted), + Some(Underline::Wave), + Some(Underline::Thick), + Some(Underline::Dash), + Some(Underline::Words) + ] + ); + assert_eq!(r[4].underline_color, Some(Rgb(255, 0, 0))); + assert_eq!((r[6].double_strike, r[6].strike), (Some(true), None)); +} + +#[test] +fn caps_spacing_and_effects_are_read() { + let d = spans(&[ + r#"fo:text-transform="uppercase""#, + r#"fo:font-variant="small-caps""#, + r#"text:display="none""#, + r#"fo:letter-spacing="2pt""#, + r#"style:text-scale="80%""#, + r#"style:text-outline="true""#, + r#"fo:text-shadow="1pt 1pt""#, + r#"style:font-relief="embossed""#, + r#"style:font-relief="engraved""#, + r#"fo:language="nb" fo:country="NO""#, + r#"style:text-position="33% 100%" fo:font-size="12pt""#, + ]); + let r = runs(&d); + assert_eq!(r[0].caps, Some(true)); + assert_eq!(r[1].small_caps, Some(true)); + assert_eq!(r[2].hidden, Some(true)); + assert_eq!(r[3].spacing, Some(2.0)); + assert_eq!(r[4].scale, Some(80.0)); + assert_eq!(r[5].outline, Some(true)); + assert_eq!(r[6].shadow, Some(true)); + assert_eq!(r[7].emboss, Some(true)); + assert_eq!(r[8].engrave, Some(true)); + assert_eq!(r[9].lang.as_deref(), Some("nb-NO")); + // Raised by a third of the text's height, at full size: a position, not superscript. + assert!(r[10].position.is_some_and(|p| (p - 3.96).abs() < 0.01) && r[10].vert_align.is_none(), "{:?}", r[10]); +} + +#[test] +fn character_formatting_round_trips() { + let want = [ + CharProps { underline: Some(Underline::Double), ..Default::default() }, + CharProps { underline: Some(Underline::Wave), underline_color: Some(Rgb(0, 0, 255)), ..Default::default() }, + CharProps { underline: Some(Underline::Words), ..Default::default() }, + CharProps { underline: Some(Underline::DotDotDash), ..Default::default() }, + CharProps { double_strike: Some(true), ..Default::default() }, + CharProps { caps: Some(true), ..Default::default() }, + CharProps { small_caps: Some(true), ..Default::default() }, + CharProps { hidden: Some(true), ..Default::default() }, + CharProps { spacing: Some(-0.5), ..Default::default() }, + CharProps { scale: Some(150.0), ..Default::default() }, + CharProps { position: Some(3.0), size: Some(12.0), ..Default::default() }, + CharProps { outline: Some(true), shadow: Some(true), ..Default::default() }, + CharProps { emboss: Some(true), ..Default::default() }, + CharProps { lang: Some("de-DE".into()), ..Default::default() }, + ]; + let mut p = Paragraph::new(); + for (i, props) in want.iter().enumerate() { + let s = format!("s{i}|"); + p.text.push_str(&s); + p.runs.push(craft_wordmodel::para::Run { len: s.len() - 1, props: props.clone() }); + p.runs.push(craft_wordmodel::para::Run { len: 1, props: Default::default() }); + } + let mut d = Document::new(); + d.body = vec![para_block(p)]; + let back = import(&export(&d).unwrap()).unwrap(); + let got = runs(&back); + for (i, (g, w)) in got.iter().zip(&want).enumerate() { + assert_eq!(g, w, "run {i}"); + } + assert_eq!(got.len(), want.len()); +} + +#[test] +fn hostile_text_properties_are_clamped() { + let d = spans(&[ + r#"fo:letter-spacing="1e30pt" style:text-scale="1e9%""#, + r#"fo:letter-spacing="-1e30pt" style:text-scale="-5%" style:text-position="1e9% 100%""#, + r#"fo:language="" fo:country="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx""#, + ]); + let r = runs(&d); + assert!(r[0].spacing.is_some_and(|s| s <= 1584.0) && r[0].scale.is_some_and(|s| s <= 600.0), "{:?}", r[0]); + assert!(r[1].spacing.is_some_and(|s| s >= -1584.0) && r[1].scale.is_none_or(|s| s >= 1.0), "{:?}", r[1]); + assert!(r[1].position.is_none_or(|p| p.abs() <= 1584.0)); + assert!(r[2].lang.as_ref().is_none_or(|l| l.len() <= 64)); + let _ = export(&d).unwrap(); +} + +#[test] +fn a_monospaced_font_keeps_its_name() { + // A fixed-pitch font was read as "code" and written as Courier New. + let d = spans(&[r#"fo:font-family="'Liberation Mono'" style:font-pitch="fixed""#, r#"style:font-name="DejaVu Sans Mono""#]); + let fonts: Vec> = runs(&d).into_iter().map(|p| p.font).collect(); + assert_eq!(fonts, [Some("Liberation Mono".to_string()), Some("DejaVu Sans Mono".to_string())]); + let back = import(&export(&d).unwrap()).unwrap(); + let fonts: Vec> = runs(&back).into_iter().map(|p| p.font).collect(); + assert_eq!(fonts, [Some("Liberation Mono".to_string()), Some("DejaVu Sans Mono".to_string())]); +} + +#[test] +fn the_default_letter_kerning_round_trips() { + // LibreOffice kerns pairs by default (`style:letter-kerning` on the default style); without + // it lines broke elsewhere after a save. + let styles = format!( + r#""# + ); + let content = format!( + r#"x"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + assert!(d.styles.default_chr.kern.is_some_and(|k| k > 0.0)); + let none = format!( + r#""# + ); + let off = import(&package(&content, Some(&none))).unwrap(); + assert!(off.styles.default_chr.kern.is_none_or(|k| k == 0.0)); + let back = import(&export(&d).unwrap()).unwrap(); + assert!(back.styles.default_chr.kern.is_some_and(|k| k > 0.0)); + let back = import(&export(&off).unwrap()).unwrap(); + assert!(back.styles.default_chr.kern.is_none_or(|k| k == 0.0)); +} + +#[test] +fn kerning_a_span_turns_on_or_off_round_trips() { + // ONLYOFFICE turns kerning on span by span, the default style saying nothing; without it + // LibreOffice set the text wider and every page broke elsewhere after a save. + let d = spans(&[r#"style:letter-kerning="true""#, r#"style:letter-kerning="false""#, r#"fo:font-weight="bold""#]); + let kerns = |d: &Document| runs(d).iter().map(|r| r.kern).collect::>(); + assert_eq!(kerns(&d), [Some(1.0), Some(0.0), None]); + // Written where it differs from the document's default (here off). + let on = |d: &Document| runs(d).iter().map(|r| r.kern.or(d.styles.default_chr.kern).is_some_and(|k| k > 0.0)).collect::>(); + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(on(&back), [true, false, false]); +} + +#[test] +fn asian_and_complex_script_fonts_and_languages() { + let d = spans(&[ + r#"style:font-name-complex="Noto Sans Hebrew" style:font-size-complex="14pt" style:font-weight-complex="bold" style:language-complex="he" style:country-complex="IL""#, + r#"style:font-name-asian="Noto Sans CJK JP" style:language-asian="ja" style:country-asian="JP""#, + ]); + let r = runs(&d); + assert_eq!((r[0].font_cs.as_deref(), r[0].size_cs, r[0].bold_cs), (Some("Noto Sans Hebrew"), Some(14.0), Some(true))); + assert_eq!(r[0].lang_bidi.as_deref(), Some("he-IL")); + // Latin bold isn't taken from the complex-script weight once the run has its own. + assert_eq!((r[1].font_ea.as_deref(), r[1].lang_ea.as_deref()), (Some("Noto Sans CJK JP"), Some("ja-JP"))); + let back = import(&export(&d).unwrap()).unwrap(); + let b = runs(&back); + assert_eq!( + (b[0].font_cs.as_deref(), b[0].size_cs, b[0].bold_cs, b[0].lang_bidi.as_deref()), + (Some("Noto Sans Hebrew"), Some(14.0), Some(true), Some("he-IL")) + ); + assert_eq!((b[1].font_ea.as_deref(), b[1].lang_ea.as_deref()), (Some("Noto Sans CJK JP"), Some("ja-JP"))); +} + +#[test] +fn the_default_styles_script_fonts_are_the_document_defaults() { + let styles = format!( + r#""# + ); + let content = format!( + r#"x"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let c = &d.styles.default_chr; + assert_eq!((c.font_ea.as_deref(), c.font_cs.as_deref(), c.lang_bidi.as_deref()), (Some("WenQuanYi Zen Hei"), Some("FreeSans"), Some("hi-IN"))); + let back = import(&export(&d).unwrap()).unwrap(); + let c = &back.styles.default_chr; + assert_eq!((c.font_ea.as_deref(), c.font_cs.as_deref(), c.lang_bidi.as_deref()), (Some("WenQuanYi Zen Hei"), Some("FreeSans"), Some("hi-IN"))); +} + +#[test] +fn a_border_around_text_is_read_and_written() { + // LibreOffice's text border (`loext:border`, or its sides, and `loext:padding`): Word's border + // around text, one line all round. + let content = format!( + r#"A boxed word."# + ); + let d = crate::text::import(&crate::text::tests::package(&content, None)).unwrap(); + let p = d.body.iter().find_map(|b| b.as_para()).unwrap(); + let b = p.runs.iter().find_map(|r| r.props.border).expect("a text border"); + assert_eq!((b.color, (b.width * 100.0).round(), (b.space * 100.0).round()), (Some(craft_wordmodel::props::Rgb(255, 0, 0)), 74.0, 144.0)); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#"loext:border="0.74pt solid #ff0000" loext:padding="1.44pt""#), "{c}"); +} + +#[test] +fn emphasis_marks_are_read_and_written() { + use craft_wordmodel::props::EmphasisMark as E; + let style = |n: &str, v: &str| { + format!(r#""#) + }; + let styles = [ + style("T1", "dot above"), + style("T2", "accent above"), + style("T3", "circle above"), + style("T4", "dot below"), + style("T5", "disc above"), + style("T6", "none"), + ] + .concat(); + let spans: String = (1..=6).map(|i| format!(r#"{i}"#)).collect(); + let content = format!( + r#"{styles}{spans}"#, + crate::text::tests::NS_T + ); + let d = crate::text::import(&crate::text::tests::package(&content, None)).unwrap(); + let marks: Vec> = d.body[0].as_para().unwrap().runs.iter().map(|r| r.props.emphasis).collect(); + assert_eq!(marks, [E::Dot, E::Comma, E::Circle, E::UnderDot, E::Dot, E::None].map(Some)); + let saved = crate::text::tests::export_model(&d).unwrap(); + let c = crate::text::tests::content_of(&saved); + for v in ["dot above", "accent above", "circle above", "dot below"] { + assert!(c.contains(&format!(r#"style:text-emphasize="{v}""#)), "{v} in {c}"); + } + // Read back the same (an explicit "none" with nothing to turn off is unset: the same look). + let back = crate::text::import(&saved).unwrap(); + let back: Vec> = back.body[0].as_para().unwrap().runs.iter().map(|r| r.props.emphasis).collect(); + assert_eq!(back, [Some(E::Dot), Some(E::Comma), Some(E::Circle), Some(E::UnderDot), Some(E::Dot), None]); +} + +#[test] +fn objects_in_hidden_text_are_kept() { + // Hidden text is written, marked hidden: so are the bookmarks and fields it holds. + let hidden = CharProps { hidden: Some(true), ..Default::default() }; + let mut p = Paragraph::with_text("Shown. Hidden words.", CharProps::default()); + let n = p.len(); + p.format(7, n, &|c| c.hidden = Some(true)).unwrap(); + p.insert_object(14, craft_wordmodel::para::InlineObject::BookmarkStart { name: "Secret".into() }, &hidden).unwrap(); + let mut d = Document::new(); + d.body = vec![para_block(p)]; + let back = import(&export(&d).unwrap()).unwrap(); + let p = back.body[0].as_para().unwrap(); + assert!(p.text.contains("Hidden"), "{:?}", p.text); + assert!( + p.objects.iter().any(|o| matches!(o, craft_wordmodel::para::InlineObject::BookmarkStart { name } if name == "Secret")), + "{:?}", + p.objects + ); +} + +/// A paragraph in style `style` holding `span`, with `auto` among its automatic styles. +fn off_doc(styles: &str, style: &str, auto: &str, span: &str) -> Document { + let content = format!( + r#"{auto}{span}"# + ); + let styles = format!("{styles}"); + import(&crate::text::tests::package(&content, Some(&styles))).unwrap() +} + +#[test] +fn text_turning_off_what_its_style_turns_on_stays_off() { + // A word not bold in a bold heading: it showed bold, on screen and after a save. + let on = r#"fo:font-weight="bold" fo:font-style="italic" style:text-underline-style="solid" style:text-line-through-style="solid" fo:font-variant="small-caps" fo:text-transform="uppercase""#; + let off = r#"fo:font-weight="normal" fo:font-style="normal" style:text-underline-style="none" style:text-line-through-style="none" fo:font-variant="normal" fo:text-transform="none""#; + let styles = format!(r#""#); + let auto = format!(r#""#); + let check = |d: &Document, what: &str| { + let p = d.body[0].as_para().unwrap(); + let shown = |run: usize| d.styles.resolve_char(p.props.style.as_deref(), &p.runs[run].props); + assert_eq!(p.text, "onoff", "{what}"); + let (a, b) = (shown(0), shown(1)); + assert_eq!((a.bold, a.italic, a.strike, a.small_caps, a.caps), (true, true, true, true, true), "{what}: the style's"); + assert!(a.underline != Underline::None, "{what}"); + assert_eq!((b.bold, b.italic, b.strike, b.small_caps, b.caps), (false, false, false, false, false), "{what}: turned off"); + assert_eq!(b.underline, Underline::None, "{what}"); + }; + let d = off_doc(&styles, "Laut", &auto, r#"onoff"#); + check(&d, "read"); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap(), "written from the model"); + check(&import(&export(&d).unwrap()).unwrap(), "written as read"); + // Turned back on inside it: on. + let auto = format!( + r#"{auto}"# + ); + let d = off_doc(&styles, "Laut", &auto, r#"x"#); + let p = d.body[0].as_para().unwrap(); + let x = d.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props); + assert!(x.bold); + assert!(x.underline != Underline::None, "{x:?}"); + assert!(!x.italic, "still off"); + // A character style's: off in the automatic style based on it. + let styles = r#""#; + let auto = r#""#; + let d = off_doc(styles, "Standard", auto, r#"x"#); + let p = d.body[0].as_para().unwrap(); + let x = d.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props); + assert_eq!((x.bold, x.underline), (false, Underline::None), "{x:?}"); + // Text nothing turned on says nothing. + let d = off_doc("", "Standard", "", "plain"); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(!c.contains(r#"fo:font-weight="normal""#) && !c.contains(r#"style:text-underline-style="none""#), "{c}"); +} + +/// A weight or slant only for Asian or complex-script text says nothing of Latin text when it is +/// normal: the heading's bold stays, on screen and saved. Latin text's own "normal" still counts. +#[test] +fn an_asian_or_complex_normal_doesnt_turn_latin_text_off() { + let styles = format!( + r#""# + ); + let open = |attr: &str| { + let content = format!( + r#"Bold word"# + ); + import(&package(&content, Some(&styles))).unwrap() + }; + let run = |d: &Document| d.body[0].as_para().unwrap().runs.last().unwrap().props.clone(); + for attr in [ + r#"style:font-weight-complex="normal""#, + r#"style:font-weight-asian="normal""#, + r#"style:font-style-complex="normal""#, + r#"style:font-style-asian="normal""#, + ] { + let d = open(attr); + assert_eq!((run(&d).bold, run(&d).italic), (None, None), "{attr}"); + let saved = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(!saved.contains(r#"fo:font-weight="normal""#) && !saved.contains(r#"fo:font-style="normal""#), "{attr}: {saved}"); + } + assert_eq!(run(&open(r#"fo:font-weight="normal""#)).bold, Some(false)); + assert_eq!(run(&open(r#"fo:font-style="normal""#)).italic, Some(false)); +} + +/// Strike-through turned off over a double one is off, not the double one again. +#[test] +fn strike_turned_off_over_a_double_strike_stays_off() { + let double = r#""#; + let none = r#""#; + let check = |d: &Document, what: &str| { + let p = d.body[0].as_para().unwrap(); + let c = d.styles.resolve_char(p.props.style.as_deref(), &p.runs.last().unwrap().props); + assert_eq!((c.strike, c.double_strike), (false, false), "{what}"); + }; + // A style over its parent. + let styles = format!( + r#"{double}{none}"# + ); + let d = off_doc(&styles, "Kid", "", "One"); + check(&d, "a style, read"); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap(), "a style, written from the model"); + // A span over its paragraph's style. + let styles = format!(r#"{double}"#); + let auto = format!(r#"{none}"#); + let d = off_doc(&styles, "Parent", &auto, r#"One"#); + check(&d, "a span, read"); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap(), "a span, written from the model"); +} + +/// A span's character style is its own: a paragraph turning bold or italic off doesn't reach past it. +#[test] +fn a_spans_character_style_wins_over_its_paragraph_turning_off() { + let styles = r#""#; + let auto = r#""#; + let d = off_doc(styles, "P1", auto, r#"One"#); + let check = |d: &Document, what: &str| { + let p = d.body[0].as_para().unwrap(); + let c = d.styles.resolve_char(p.props.style.as_deref(), &p.runs.last().unwrap().props); + assert_eq!((c.bold, c.italic), (true, true), "{what}: {:?}", p.runs.last().unwrap().props); + }; + check(&d, "read"); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap(), "written from the model"); +} + +/// `default` as a document's default paragraph style's text properties, with `standard` as its +/// Standard style's. +fn with_default_text(default: &str, standard: &str) -> Vec { + let styles = format!( + r#""# + ); + let content = format!( + r#"Hallo"# + ); + crate::text::tests::package(&content, Some(&styles)) +} + +/// `styles`' default paragraph style. +fn default_style(styles: &str) -> &str { + let at = styles.find(r#""#).expect("a default paragraph style"); + styles[at..].split("").next().unwrap_or_default() +} + +/// A document's language is the one its file states, and a model-written save says it. +#[test] +fn the_default_language_is_the_files_and_written_back_from_the_model() { + let d = import(&with_default_text(r#"fo:language="de" fo:country="DE""#, "")).unwrap(); + assert_eq!(d.styles.default_chr.lang.as_deref(), Some("de-DE")); + assert_eq!(d.styles.resolve_char(None, &Default::default()).lang.as_deref(), Some("de-DE")); + // Written from the model (a style or list definition changed), it is still the file's: without + // it LibreOffice shows the document in its own default language. + let saved = crate::text::tests::export_model(&d).unwrap(); + let styles = crate::text::tests::entry(&saved, "styles.xml"); + assert!(default_style(&styles).contains(r#"fo:language="de" fo:country="DE""#), "{styles}"); + assert!(!styles.contains(r#"fo:language="en""#), "no style says English: {styles}"); + assert_eq!(import(&saved).unwrap().styles.default_chr.lang.as_deref(), Some("de-DE")); + // Saved as read, the styles WordCraft adds beside the file's don't say English either. + let styles = crate::text::tests::entry(&export(&d).unwrap(), "styles.xml"); + assert!(!styles.contains(r#"fo:language="en""#), "{styles}"); +} + +/// The document's language as WordCraft has it is Normal's (a Word file's Normal may state +/// another than its defaults): that is what the default style says when written from the model. +#[test] +fn the_documents_language_written_is_normals() { + use std::sync::Arc; + let mut d = Document::new(); + Arc::make_mut(&mut d.styles).default_chr.lang = Some("en-US".into()); + Arc::make_mut(&mut d.styles).get_mut("Normal").unwrap().chr.lang = Some("fr-FR".into()); + let saved = crate::text::tests::export_model(&d).unwrap(); + let s = crate::text::tests::styles_of(&saved); + let def = s.split(r#"").next()).unwrap(); + assert!(def.contains(r#"fo:language="fr""#) && def.contains(r#"fo:country="FR""#), "{def}"); + let back = import(&saved).unwrap(); + assert_eq!(back.styles.resolve_char(None, &Default::default()).lang.as_deref(), Some("fr-FR")); +} + +/// A file stating no language, or `none` or `zxx`, has none: not the English of new documents. +#[test] +fn a_file_stating_no_language_has_none_and_writes_none() { + let d = import(&with_default_text("", "")).unwrap(); + assert_eq!(d.styles.default_chr.lang, None, "not the English of WordCraft's new documents"); + let saved = crate::text::tests::export_model(&d).unwrap(); + assert!(!crate::text::tests::entry(&saved, "styles.xml").contains("fo:language")); + assert_eq!(import(&saved).unwrap().styles.default_chr.lang, None); + // `none` and `zxx` are no language. + for none in ["none", "zxx"] { + let d = import(&with_default_text(&format!(r#"fo:language="{none}" fo:country="none""#), "")).unwrap(); + assert_eq!(d.styles.default_chr.lang, None, "{none}"); + } +} + +/// A language only Standard states is the document's, and a model-written save says it in the +/// default style. +#[test] +fn a_language_only_the_standard_style_states_survives_a_rewrite() { + let d = import(&with_default_text("", r#"fo:language="nb" fo:country="NO""#)).unwrap(); + assert_eq!(d.styles.default_chr.lang.as_deref(), Some("nb-NO")); + let saved = crate::text::tests::export_model(&d).unwrap(); + let styles = crate::text::tests::entry(&saved, "styles.xml"); + assert!(default_style(&styles).contains(r#"fo:language="nb" fo:country="NO""#), "{styles}"); + let back = import(&saved).unwrap(); + assert_eq!(back.styles.default_chr.lang.as_deref(), Some("nb-NO")); + let p = back.body[0].as_para().unwrap(); + let lang = + p.runs.first().and_then(|r| r.props.lang.clone()).or_else(|| back.styles.resolve_char(p.props.style.as_deref(), &Default::default()).lang); + assert_eq!(lang.as_deref(), Some("nb-NO"), "{}", crate::text::tests::entry(&saved, "styles.xml")); +} + +/// A language with a script keeps it, read and written: the document's, a style's for each kind +/// of text, and a run's. +#[test] +fn a_language_with_a_script_keeps_it() { + let d = import(&with_default_text(r#"fo:language="sr" fo:script="Latn" fo:country="RS""#, "")).unwrap(); + assert_eq!(d.styles.default_chr.lang.as_deref(), Some("sr-Latn-RS")); + let saved = crate::text::tests::export_model(&d).unwrap(); + let styles = crate::text::tests::entry(&saved, "styles.xml"); + assert!(default_style(&styles).contains(r#"fo:language="sr" fo:script="Latn" fo:country="RS""#), "{styles}"); + assert!(!styles.contains(r#"fo:country="Latn""#), "{styles}"); + assert_eq!(import(&saved).unwrap().styles.default_chr.lang.as_deref(), Some("sr-Latn-RS")); + + let (latin, asian, complex) = ( + r#"fo:language="sr" fo:script="Cyrl" fo:country="BA""#, + r#"style:language-asian="zh" style:script-asian="Hant" style:country-asian="TW""#, + r#"style:language-complex="az" style:script-complex="Arab" style:country-complex="IR""#, + ); + let run = r#"fo:language="bs" fo:script="Latn" fo:country="BA""#; + let styles = format!( + r#""# + ); + let content = format!( + r#"Own span"# + ); + let check = |d: &craft_wordmodel::Document| { + let chr = &d.styles.get("Eigen").unwrap().chr; + assert_eq!( + (chr.lang.as_deref(), chr.lang_ea.as_deref(), chr.lang_bidi.as_deref()), + (Some("sr-Cyrl-BA"), Some("zh-Hant-TW"), Some("az-Arab-IR")) + ); + assert_eq!(d.body[0].as_para().unwrap().runs.iter().find_map(|r| r.props.lang.as_deref()), Some("bs-Latn-BA")); + }; + let d = import(&crate::text::tests::package(&content, Some(&styles))).unwrap(); + check(&d); + let saved = crate::text::tests::export_model(&d).unwrap(); + let written = crate::text::tests::entry(&saved, "styles.xml"); + assert!(written.contains(latin) && written.contains(asian) && written.contains(complex), "{written}"); + assert!(crate::text::tests::entry(&saved, "content.xml").contains(run)); + check(&import(&saved).unwrap()); + + // No country, a region of digits, and what isn't a script. + let attrs = crate::text::writer::lang_attrs; + assert_eq!(attrs("zh-Hant", ""), r#" fo:language="zh" fo:script="Hant""#); + assert_eq!(attrs("es-419", ""), r#" fo:language="es" fo:country="419""#); + assert_eq!(attrs("de-DE", "-asian"), r#" style:language-asian="de" style:country-asian="DE""#); + let d = import(&with_default_text(r#"fo:language="sr" fo:script="Latin1" fo:country="RS""#, "")).unwrap(); + assert_eq!(d.styles.default_chr.lang.as_deref(), Some("sr-RS")); +} + +/// A style with no parent inherits the default style's language, not Standard's (the document's +/// here), and one based on Standard takes Standard's. +#[test] +fn a_style_with_no_parent_has_the_default_styles_language() { + let styles = format!( + r#""# + ); + let content = format!( + r#"abc"# + ); + let check = |d: &craft_wordmodel::Document| { + assert_eq!(d.styles.default_chr.lang.as_deref(), Some("de-DE")); + let lang = |id: &str| d.styles.resolve_char(Some(id), &Default::default()).lang; + assert_eq!(lang("Eigen").as_deref(), Some("en-US"), "no parent: the default style's"); + assert_eq!(lang("Gesagt").as_deref(), Some("fr-FR"), "its own"); + assert_eq!(lang("Kind").as_deref(), Some("de-DE"), "based on Standard"); + }; + let d = import(&crate::text::tests::package(&content, Some(&styles))).unwrap(); + check(&d); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()); +} + +/// WordCraft's own documents say their language, so a reader doesn't use its own default. +#[test] +fn a_new_documents_language_is_stated() { + let styles = crate::text::tests::entry(&export(&craft_wordmodel::Document::new()).unwrap(), "styles.xml"); + assert!(default_style(&styles).contains(r#"fo:language="en" fo:country="US""#), "{styles}"); +} + +#[test] +fn a_size_is_every_scripts_size() { + // Chinese text set to 24 pt stayed at the default size in LibreOffice: only the Latin size was + // written. + let mut d = Document::new(); + let mut p = Paragraph::with_text("漢字", CharProps { size: Some(24.0), ..Default::default() }); + p.props = Default::default(); + d.body = vec![para_block(p)]; + let saved = crate::text::tests::export_model(&d).unwrap(); + let c = crate::text::tests::content_of(&saved); + assert!(c.contains(r#"fo:font-size="24pt" style:font-size-asian="24pt" style:font-size-complex="24pt""#), "{c}"); + let styles = crate::text::tests::entry(&saved, "styles.xml"); + assert!(styles.contains(r#"fo:font-size="12pt" style:font-size-asian="12pt" style:font-size-complex="12pt""#), "{styles}"); + // Read back every script has it. + let back = import(&saved).unwrap(); + let run = &back.body[0].as_para().unwrap().runs[0].props; + assert_eq!(back.styles.resolve_char(None, run).size_cs, 24.0); + // A complex-script size of its own stays its own, written once. + let mut d = Document::new(); + d.body = vec![para_block(Paragraph::with_text("x", CharProps { size: Some(24.0), size_cs: Some(30.0), ..Default::default() }))]; + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert_eq!(c.matches("style:font-size-complex").count(), 1, "{c}"); + assert!(c.contains(r#"style:font-size-complex="30pt""#), "{c}"); +} + +/// Text and its complex-script size stated alike, over a default style with a smaller one (as +/// LibreOffice's files have): that size stays, on screen and saved, for a run and for a heading. +#[test] +fn a_complex_size_like_the_texts_stays_over_the_default() { + let styles = format!( + r#""# + ); + let content = format!( + r#"a bigh"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let check = |d: &Document| { + let p = d.body[0].as_para().unwrap(); + let c = d.styles.resolve_char(p.props.style.as_deref(), &p.runs.last().unwrap().props); + assert_eq!((c.size, c.size_cs), (24.0, 24.0)); + let h = d.body[1].as_para().unwrap(); + let c = d.styles.resolve_char(h.props.style.as_deref(), &h.runs[0].props); + assert_eq!((c.size, c.size_cs), (18.0, 18.0)); + }; + check(&d); + let saved = crate::text::tests::export_model(&d).unwrap(); + let written = crate::text::tests::entry(&saved, "styles.xml"); + let heading = written.split(r#"style:name="Heading_20_1""#).nth(1).and_then(|t| t.split("").next()).unwrap(); + assert!(heading.contains(r#"style:font-size-complex="18pt""#) && !heading.contains(r#"style:font-size-complex="14pt""#), "{heading}"); + check(&import(&saved).unwrap()); + + // A default stating one size for all goes with that size: Normal changed takes it along. + let same = styles.replace(r#"style:font-size-complex="14pt""#, r#"style:font-size-complex="12pt""#); + let d = import(&package(&content, Some(&same))).unwrap(); + assert_eq!(d.styles.default_chr.size_cs, None); +} + +/// A complex size equal to the text's goes only where no run's complex size changes without it: +/// not under a character style over another size, nor under a size stated alone. +#[test] +fn a_complex_size_something_shows_stays() { + let open = |s: &str, auto: &str, body: &str| { + let styles = format!( + r#"{s}"# + ); + let content = format!( + r#"{auto}{body}"# + ); + import(&package(&content, Some(&styles))).unwrap() + }; + let sizes = |d: &Document| { + let p = d.body[0].as_para().unwrap(); + let c = d.styles.resolve_char(p.props.style.as_deref(), &p.runs.last().unwrap().props); + (c.size, c.size_cs) + }; + // A character style's, over a paragraph style with another complex size. + let d = open( + r#""#, + "", + r#"One"#, + ); + assert_eq!(sizes(&d), (24.0, 24.0)); + let parent = r#""#; + // A parent's, under a child stating only a size. + let kid = format!( + r#"{parent}"# + ); + assert_eq!(sizes(&open(&kid, "", r#"One"#)), (20.0, 16.0)); + // A style's, under a run stating only a size. + let d = open( + parent, + r#""#, + r#"One"#, + ); + assert_eq!(sizes(&d), (30.0, 16.0)); + // Nothing shows it: one size for all, so a change of size takes it along. + let d = open(parent, "", r#"One"#); + assert_eq!(sizes(&d), (16.0, 16.0)); + assert_eq!(d.styles.get("Parent").unwrap().chr.size_cs, None); +} + +/// A size stated alone, by a run or a style, under a different complex size above it: saved from the +/// model, and saved again, the complex size is still the one above, not the stated size. +#[test] +fn a_size_stated_alone_keeps_the_complex_size_above_it_when_saved() { + let sizes = |d: &Document, i: usize| { + let p = d.body[i].as_para().unwrap(); + let c = d.styles.resolve_char(p.props.style.as_deref(), &p.runs.last().unwrap().props); + (c.size, c.size_cs) + }; + let twice = |d: Document| { + let once = import(&crate::text::tests::export_model(&d).unwrap()).unwrap(); + let again = import(&crate::text::tests::export_model(&once).unwrap()).unwrap(); + (once, again) + }; + // From a file: a run stating only a size, and a style stating only a size, under a parent + // style with its own complex size. + let styles = format!( + r#""# + ); + let content = format!( + r#"OneTwo"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + assert_eq!((sizes(&d, 0), sizes(&d, 1)), ((30.0, 16.0), (20.0, 16.0))); + let (once, again) = twice(d); + assert_eq!((sizes(&once, 0), sizes(&once, 1)), ((30.0, 16.0), (20.0, 16.0)), "saved once"); + assert_eq!((sizes(&again, 0), sizes(&again, 1)), ((30.0, 16.0), (20.0, 16.0)), "saved twice"); + // Built in the model (a DOCX run with `w:sz` and no `w:szCs`): the default's complex size. + let mut d = Document::new(); + let styles = std::sync::Arc::make_mut(&mut d.styles); + styles.default_chr.size = Some(12.0); + styles.default_chr.size_cs = Some(16.0); + let mut p = Paragraph::with_text("x", CharProps { size: Some(30.0), ..Default::default() }); + p.props = Default::default(); + d.body = vec![para_block(p)]; + assert_eq!(sizes(&d, 0), (30.0, 16.0)); + let (once, again) = twice(d); + assert_eq!((sizes(&once, 0), sizes(&again, 0)), ((30.0, 16.0), (30.0, 16.0))); +} + +#[test] +fn a_headings_text_of_another_size_keeps_it() { + // Only the size its style gives goes with the style: a smaller word in a heading came back + // from a save at the heading's size. + let styles = format!( + r#""# + ); + let content = format!( + r#"bigsmall"# + ); + let check = |d: &Document, what: &str| { + let p = d.body[0].as_para().unwrap(); + let size = |run: usize| d.styles.resolve_char(p.props.style.as_deref(), &p.runs[run].props).size; + assert_eq!((size(0), size(1)), (16.0, 10.0), "{what}"); + }; + let d = import(&package(&content, Some(&styles))).unwrap(); + check(&d, "read"); + assert_eq!(d.body[0].as_para().unwrap().runs[0].props.size, None, "the style's stays the style's"); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap(), "written from the model"); +} + +/// In a heading style with no parent, where Standard's size and the default's differ, text stating +/// a size keeps that size. +#[test] +fn a_parentless_headings_text_keeps_its_size() { + let styles = format!( + r#""# + ); + for size in ["10pt", "14pt"] { + let content = format!( + r#"OneTwo"# + ); + let want: f32 = size.trim_end_matches("pt").parse().unwrap(); + let check = |d: &Document, what: &str| { + let p = d.body[0].as_para().unwrap(); + assert_eq!(p.runs[1].props.size, None, "{size} {what}: a span stating no size has none"); + assert_eq!( + d.styles.resolve_char(p.props.style.as_deref(), &p.runs[0].props).size, + want, + "{size} {what}: {:?} {:?}", + p.props.style, + p.runs[0].props.size + ); + }; + let d = import(&package(&content, Some(&styles))).unwrap(); + check(&d, "read"); + check(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap(), "written from the model"); + } +} + +/// A percent in a heading style with no parent is of the default style's size, not Standard's. +#[test] +fn a_percent_in_a_parentless_heading_is_of_the_default_size() { + let styles = format!( + r#""# + ); + let content = format!( + r#"One"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + assert_eq!(d.body[0].as_para().unwrap().runs[0].props.size, Some(5.0)); +} + +/// A heading style with no parent and a percent size is that percent of the default's size, so text +/// stating that size is at the style's size. +#[test] +fn a_parentless_heading_style_in_percent_is_of_the_default_size() { + let styles = format!( + r#""# + ); + let content = format!( + r#"OneTwo"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let p = d.body[0].as_para().unwrap(); + for r in &p.runs { + assert_eq!(d.styles.resolve_char(p.props.style.as_deref(), &r.props).size, 5.0, "{:?} {:?}", p.props.style, r.props.size); + } +} + +/// Text at its heading's size goes with the heading style, stated or not, and a size in percent +/// is of the heading's size; text of another size keeps it, and so does a paragraph's own. +#[test] +fn a_headings_text_sizes_are_measured_against_the_heading() { + let styles = format!( + r#""# + ); + let span = |name: &str, size: &str| { + format!(r#""#) + }; + let content = format!( + r#"{}{}{}plain same pct otherown"#, + span("Same", "16pt"), + span("Pct", "50%"), + span("Other", "10pt"), + ); + let mut d = import(&package(&content, Some(&styles))).unwrap(); + let sizes = |d: &Document| { + let p = d.body[0].as_para().unwrap(); + let of = |word: &str| { + let at = p.text.find(word).unwrap(); + let mut from = 0; + let run = p.runs.iter().find(|r| { + from += r.len; + at < from + }); + d.styles.resolve_char(p.props.style.as_deref(), &run.unwrap().props).size + }; + [of("plain"), of("same"), of("pct"), of("other")] + }; + let want = |t: f32| [t, t, 8.0, 10.0]; + let plain = |d: &Document| d.body[0].as_para().unwrap().runs.iter().filter(|r| r.props.size.is_some()).count(); + assert_eq!(sizes(&d), want(16.0)); + assert_eq!(plain(&d), 2, "only the two of another size state one"); + // The paragraph's own size stays with its text. + let own = d.body[1].as_para().unwrap(); + assert_eq!(d.styles.resolve_char(own.props.style.as_deref(), &own.runs[0].props).size, 30.0); + // A change to the style reaches what goes with it. + let id = d.body[0].as_para().unwrap().props.style.clone().unwrap(); + std::sync::Arc::make_mut(&mut d.styles).get_mut(&id).unwrap().chr.size = Some(24.0); + assert_eq!(sizes(&d), want(24.0)); +} diff --git a/crates/odf/src/text/tests_more/charts.rs b/crates/odf/src/text/tests_more/charts.rs new file mode 100644 index 000000000..9ac0b6b96 --- /dev/null +++ b/crates/odf/src/text/tests_more/charts.rs @@ -0,0 +1,355 @@ +//! Charts (ODF 1.3 §10.4.3 embedded objects holding an `office:chart` sub-document, Part 3 +//! §11): read into WordCraft's editable chart (type, categories, series, title, legend, data +//! labels) from the chart's own data table, so they're drawn and edited like charts WordCraft +//! makes. Charts WordCraft can't edit keep their replacement picture. + +use std::sync::Arc; + +use crate::text::parse; +use crate::text::tests::{NS_T, paras, zip_of}; +use craft_flow::Inline; +use craft_wordmodel::chart::{ChartSeries, ChartSpec, ChartType, LegendPos}; + +const NS_C: &str = r#"xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:xlink="http://www.w3.org/1999/xlink""#; + +/// LibreOffice's data table for categories Q1–Q4 and series North and South (South has a gap +/// in Q2, written `NaN`), with its labels in the first row and column. +const TABLE: &str = r#"NorthSouthQ11012Q214NaNQ3811Q41613"#; + +/// The two series over [`TABLE`], each `class`, with style `ser` (and its data points). +fn series(class: &str) -> String { + format!( + r#""# + ) +} + +/// A chart sub-document's `content.xml` as LibreOffice writes it: chart class `class`, plot +/// area chart properties `plot`, series style chart properties `ser`, and `inner` (title, +/// legend) before the plot area, holding `series`. +fn chart_xml(class: &str, plot: &str, ser: &str, inner: &str, series: &str) -> String { + format!( + r#"{inner}{series}{TABLE}"# + ) +} + +/// What the document holding chart object `chart` (with a replacement picture) reads as: the +/// paragraph's charts, and whether a picture was read instead. +fn read(chart: &str) -> (Vec, bool) { + let frame = r#""#; + let content = format!( + r#"Before {frame}"# + ); + let img = image::RgbaImage::from_fn(4, 2, |_, _| image::Rgba([0, 80, 160, 255])); + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap(); + let flow = + parse(&zip_of(&[("content.xml", content.as_bytes()), ("Object 1/content.xml", chart.as_bytes()), ("ObjectReplacements/Object 1", &png)])) + .unwrap(); + let inlines: Vec = paras(&flow).into_iter().flat_map(|p| p.inlines.clone()).collect(); + let charts = inlines.iter().filter_map(|i| if let Inline::Chart(c) = i { Some(c.clone()) } else { None }).collect(); + (charts, inlines.iter().any(|i| matches!(i, Inline::Image(_)))) +} + +fn spec(chart: &str) -> Option { + let (charts, _) = read(chart); + charts.first().map(|c| (*c.spec).clone()) +} + +fn north_south() -> Vec { + vec![ + ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)] }, + ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] }, + ] +} + +#[test] +fn a_column_chart_reads_its_data_title_and_legend() { + let inner = r#"Sales by quarter"#; + let xml = chart_xml("chart:bar", r#"chart:vertical="false""#, r#"chart:data-label-number="none""#, inner, &series("chart:bar")); + let (charts, picture) = read(&xml); + assert!(!picture, "the chart replaces its preview picture"); + let c = charts.first().expect("a chart"); + assert_eq!( + *c.spec, + ChartSpec { + kind: ChartType::Column, + categories: vec!["Q1".into(), "Q2".into(), "Q3".into(), "Q4".into()], + series: north_south(), + title: Some("Sales by quarter".into()), + legend: Some(LegendPos::Right), + data_labels: false, + } + ); + // The frame's size, in points. + assert_eq!(((c.w * 10.0).round(), (c.h * 10.0).round()), (4252.0, 2126.0)); +} + +#[test] +fn bar_stacking_legend_and_labels_come_from_the_styles() { + let cases = [ + (r#"chart:vertical="true""#, ChartType::Bar), + (r#"chart:vertical="true" chart:stacked="true""#, ChartType::BarStacked), + (r#"chart:stacked="true""#, ChartType::ColumnStacked), + ]; + for (plot, kind) in cases { + let xml = chart_xml( + "chart:bar", + plot, + r#"chart:data-label-number="value""#, + r#""#, + &series("chart:bar"), + ); + let s = spec(&xml).expect(plot); + assert_eq!((s.kind, s.legend, s.data_labels, s.title.clone()), (kind, Some(LegendPos::Bottom), true, None), "{plot}"); + assert_eq!(s.series, north_south()); + } + // No legend element: no legend. Legend corners and sides. + for (pos, want) in [ + ("", None), + ("start", Some(LegendPos::Left)), + ("top", Some(LegendPos::Top)), + ("top-end", Some(LegendPos::Top)), + ("bottom-start", Some(LegendPos::Bottom)), + ] { + let legend = if pos.is_empty() { String::new() } else { format!(r#""#) }; + let s = spec(&chart_xml("chart:bar", "", "", &legend, &series("chart:bar"))).unwrap(); + assert_eq!(s.legend, want, "{pos}"); + } +} + +#[test] +fn line_area_pie_doughnut_and_scatter_classes() { + for (class, ser_class, kind) in [ + ("chart:line", "chart:line", ChartType::Line), + ("chart:area", "chart:area", ChartType::Area), + ("chart:circle", "chart:circle", ChartType::Pie), + // LibreOffice writes a doughnut's series as `chart:circle` in a `chart:ring` chart. + ("chart:ring", "chart:circle", ChartType::Doughnut), + ] { + let s = spec(&chart_xml(class, "", "", "", &series(ser_class))).expect(class); + assert_eq!(s.kind, kind, "{class}"); + assert_eq!(s.series, north_south()); + } + // A scatter chart's x values are its first series' domain: they're its categories. + let scatter = r#""#; + let s = spec(&chart_xml("chart:scatter", "", "", "", scatter)).unwrap(); + assert_eq!(s.kind, ChartType::Scatter); + assert_eq!(s.categories, vec!["10", "14", "8", "16"]); + assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] }]); +} + +#[test] +fn charts_wordcraft_cannot_edit_keep_their_picture() { + // Percent-stacked, a radar chart, a chart mixing columns and a line, 3D: the picture. + let mixed = format!( + "{}{}", + series("chart:bar").split("").next().unwrap_or_default().to_string() + "", + r#""# + ); + for xml in [ + chart_xml("chart:bar", r#"chart:stacked="true" chart:percentage="true""#, "", "", &series("chart:bar")), + chart_xml("chart:radar", "", "", "", &series("chart:radar")), + chart_xml("chart:bar", "", "", "", &mixed), + chart_xml("chart:bar", r#"chart:three-dimensional="true""#, "", "", &series("chart:bar")), + // Not a chart at all, or broken. + "".to_string(), + "".to_string(), + ] { + let (charts, picture) = read(&xml); + assert!(charts.is_empty() && picture, "{xml}"); + } +} + +#[test] +fn hostile_chart_tables_are_bounded() { + // Huge repeat counts, references far outside the table and into reversed or broken ranges: + // never a panic or a huge allocation, and the series stay within the chart's limits. + let table = r#""#; + let series = r#""#; + let xml = format!( + r#"{series}{table}"# + ); + let s = spec(&xml).expect("a chart"); + assert!(s.series.len() <= craft_wordmodel::chart::MAX_SERIES); + assert!(s.series.iter().all(|x| x.values.len() <= craft_wordmodel::chart::MAX_POINTS)); + assert!(s.categories.len() <= craft_wordmodel::chart::MAX_POINTS); +} + +#[test] +fn an_imported_chart_is_an_editable_chart_object() { + let xml = chart_xml("chart:line", "", "", "", &series("chart:line")); + let frame = r#""#; + let content = format!( + r#"{frame}"# + ); + let doc = crate::text::import(&zip_of(&[("content.xml", content.as_bytes()), ("Object 1/content.xml", xml.as_bytes())])).unwrap(); + let graphics: Vec> = doc + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects_with_offsets()) + .filter_map( + |(_, o)| if let craft_wordmodel::para::InlineObject::Graphic { graphic, w, h, .. } = o { Some((graphic.clone(), *w, *h)) } else { None }, + ) + .map(|(g, w, h)| { + assert_eq!((w, h), (288.0, 144.0)); + g + }) + .collect(); + let g = graphics.first().expect("a chart object"); + assert_eq!(g.kind, craft_wordmodel::graphic::GraphicKind::Chart); + assert_eq!(g.chart.as_ref().map(|c| c.kind), Some(ChartType::Line)); +} + +/// A document holding `spec` as a chart `w` × `h` points, after some text. +fn doc_with(spec: ChartSpec, float: bool) -> craft_wordmodel::Document { + let flow = craft_flow::Flow { + blocks: vec![craft_flow::FBlock::Para(craft_flow::Para { + inlines: vec![ + Inline::Text("See ".into(), Default::default()), + Inline::Chart(craft_flow::Chart { + spec: Arc::new(spec), + w: 300.0, + h: 180.0, + alt: "Sales chart".into(), + float: float.then(|| { + Box::new(craft_wordmodel::para::Float { wrap: craft_wordmodel::para::Wrap::Square, x: 20.0, y: 10.0, ..Default::default() }) + }), + source: None, + }), + ], + ..Default::default() + })], + ..Default::default() + }; + craft_flow::to_doc(&flow) +} + +/// The charts the saved document reads back as. +fn reread(bytes: &[u8]) -> Vec { + paras(&parse(bytes).unwrap()) + .into_iter() + .flat_map(|p| p.inlines.clone()) + .filter_map(|i| if let Inline::Chart(c) = i { Some(c) } else { None }) + .collect() +} + +#[test] +fn every_chart_type_is_saved_as_a_chart_object_and_reads_back() { + for kind in ChartType::ALL { + let mut spec = ChartSpec::sample(kind); + spec.title = Some("Q & A ".into()); + spec.data_labels = kind == ChartType::Bar; + spec.legend = Some(LegendPos::Left); + if let Some(s) = spec.series.first_mut() + && let Some(v) = s.values.get_mut(1) + { + *v = None; + } + let bytes = crate::text::export(&doc_with(spec.clone(), false)).unwrap(); + let back = reread(&bytes); + assert_eq!(back.len(), 1, "{kind:?}"); + assert_eq!(*back[0].spec, spec.clone().sanitized(), "{kind:?}"); + assert_eq!((back[0].w, back[0].h, back[0].alt.as_str()), (300.0, 180.0, "Sales chart"), "{kind:?}"); + // The package lists the chart object. + let z = crate::Package::open(&bytes, &crate::Limits::DEFAULT).unwrap(); + assert!(z.names().any(|n| n == "Object 1/content.xml"), "{kind:?}"); + let manifest = crate::text::tests::entry(&bytes, "META-INF/manifest.xml"); + assert!( + manifest + .contains(r#"manifest:full-path="Object 1/" manifest:version="1.3" manifest:media-type="application/vnd.oasis.opendocument.chart""#) + ); + } +} + +#[test] +fn a_floating_chart_keeps_its_place_and_wrap() { + let bytes = crate::text::export(&doc_with(ChartSpec::sample(ChartType::Column), true)).unwrap(); + let back = reread(&bytes); + let f = back.first().and_then(|c| c.float.as_deref()).expect("a floating chart"); + assert_eq!((f.wrap, f.x.round(), f.y.round()), (craft_wordmodel::para::Wrap::Square, 20.0, 10.0)); +} + +#[test] +fn a_flat_document_holds_its_chart_inline() { + let flat = crate::text::export_flat(&doc_with(ChartSpec::sample(ChartType::Pie), false)).unwrap(); + let text = String::from_utf8(flat.clone()).unwrap(); + assert!(text.contains(" = crate::text::import_flat(&flat) + .unwrap() + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.clone()) + .filter_map(|o| match o { + craft_wordmodel::para::InlineObject::Graphic { graphic, w, h, alt, .. } => { + graphic.chart.clone().map(|spec| craft_flow::Chart { spec, w, h, alt, float: None, source: None }) + } + _ => None, + }) + .collect(); + assert_eq!(back.first().map(|c| c.spec.kind), Some(ChartType::Pie)); +} + +#[test] +fn an_unchanged_chart_is_written_back_as_it_was_and_an_edited_one_anew() { + // A chart keeps the look WordCraft doesn't model (its fonts, lines and colours) only if its + // object is written back whole; once edited, it's written from WordCraft's chart. + let original = chart_xml( + "chart:bar", + r#"chart:vertical="false""#, + "", + r#"Sales"#, + &series("chart:bar"), + ) + .replace( + "", + r#""#, + ); + // With its replacement picture (what a reader that can't draw charts shows). + let frame = r#""#; + let content = format!( + r#"Before {frame}"# + ); + let picture: &[u8] = b"VCLMTF the chart as drawn"; + let doc = crate::text::import(&zip_of(&[ + ("content.xml", content.as_bytes()), + ("Object 1/content.xml", original.as_bytes()), + ("ObjectReplacements/Object 1", picture), + ])) + .unwrap(); + let replacement_of = |bytes: &[u8]| { + let mut z = crate::Package::open(bytes, &crate::Limits::DEFAULT).unwrap(); + let name = z.names().map(str::to_string).collect::>().into_iter().find(|n| n.starts_with("ObjectReplacements/"))?; + z.read(&name).unwrap() + }; + let saved = crate::text::export(&doc).unwrap(); + assert_eq!(replacement_of(&saved).as_deref(), Some(picture)); + assert!(crate::text::tests::entry(&saved, "content.xml").contains(r#">().into_iter().find(|n| n.ends_with("/content.xml") && n.starts_with("Object")).unwrap(); + String::from_utf8(z.read(&name).unwrap().unwrap()).unwrap() + }; + assert_eq!(chart_of(&crate::text::export(&doc).unwrap()), original); + // Edited: a new title. + let mut edited = doc.clone(); + for b in edited.body.iter_mut() { + if let craft_wordmodel::Block::Para(p) = Arc::make_mut(b) { + for o in &mut p.objects { + if let craft_wordmodel::para::InlineObject::Graphic { graphic, .. } = o { + let g = Arc::make_mut(graphic); + let mut spec = (**g.chart.as_ref().unwrap()).clone(); + spec.title = Some("Revenue".into()); + g.chart = Some(Arc::new(spec)); + } + } + } + } + let saved = crate::text::export(&edited).unwrap(); + let out = chart_of(&saved); + assert!(out.contains("Revenue") && !out.contains(r#"fo:font-size="18pt""#), "{out}"); + // The old picture would show the old chart. + assert_eq!(replacement_of(&saved), None); +} diff --git a/crates/odf/src/text/tests_more/comments.rs b/crates/odf/src/text/tests_more/comments.rs new file mode 100644 index 000000000..a4f47be2b --- /dev/null +++ b/crates/odf/src/text/tests_more/comments.rs @@ -0,0 +1,187 @@ +//! Comments (ODF 1.3 §14.1 `office:annotation`, §14.2 `office:annotation-end`): read into the +//! document's comments with their ranges, replies and resolved state, and written back. + +use crate::text::tests::{NS_T, package}; +use crate::text::{export, import}; +use craft_wordmodel::para::{InlineObject, OBJ}; +use craft_wordmodel::{Document, StoryRef}; + +const NS_C: &str = r#"xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0""#; + +fn body(xml: &str) -> Vec { + package( + &format!(r#"{xml}"#), + None, + ) +} + +/// Each comment: (author, initials, date, resolved, parent's text, its text, the text it covers). +type Summary = (String, String, String, bool, Option, String, String); + +fn text_of(d: &Document, part: u32) -> String { + d.parts + .get(&part) + .map(|p| p.blocks.iter().filter_map(|b| b.as_para()).map(|p| p.text.replace(OBJ, "")).collect::>().join("|")) + .unwrap_or_default() +} + +/// The body text between each comment's start and end. +fn covered(d: &Document, id: u32) -> String { + let mut out = String::new(); + let mut inside = false; + for b in d.body.iter() { + let Some(p) = b.as_para() else { continue }; + let offs = p.object_offsets(); + let mut last = 0; + for (k, o) in p.objects.iter().enumerate() { + let at = offs.get(k).copied().unwrap_or(0); + if inside { + out.push_str(&p.text[last..at].replace(OBJ, "")); + } + match o { + InlineObject::CommentStart { id: i } if *i == id => inside = true, + InlineObject::CommentEnd { id: i } if *i == id => inside = false, + _ => {} + } + last = at + OBJ.len_utf8(); + } + if inside { + out.push_str(&p.text[last.min(p.text.len())..].replace(OBJ, "")); + out.push('|'); + } + } + out +} + +fn comments(d: &Document) -> Vec { + d.comments + .iter() + .map(|(id, c)| { + let parent = c.parent.and_then(|p| d.comments.get(&p)).map(|p| text_of(d, p.part)); + (c.author.clone(), c.initials.clone(), c.date.clone(), c.resolved, parent, text_of(d, c.part), covered(d, *id)) + }) + .collect() +} + +const COMMENTS: &str = concat!( + r#"Before Ada Lovelace2026-10-31T10:00:00ALFirst comment.Second line.commented words and"#, + r#"Grace0000-00-00T00:00:00A point. after."#, +); + +#[test] +fn comments_open_with_their_ranges() { + let d = import(&body(COMMENTS)).unwrap(); + let got = comments(&d); + assert_eq!(got.len(), 2, "{got:?}"); + assert_eq!( + got[0], + ( + "Ada Lovelace".into(), + "AL".into(), + "2026-10-31T10:00:00Z".into(), + false, + None, + "First comment.|Second line.".into(), + "commented words".into() + ) + ); + // A comment without a range covers nothing; LibreOffice's empty date is no date. + assert_eq!(got[1], ("Grace".into(), String::new(), String::new(), true, None, "A point.".into(), String::new())); + // The comment's text isn't body text. + assert_eq!(d.plain_text(StoryRef::Body).trim(), "Before commented words and after."); +} + +#[test] +fn a_comment_across_paragraphs_keeps_its_range() { + let d = import(&body(r#"a Aconetwo b"#)).unwrap(); + let (id, _) = d.comments.iter().next().unwrap(); + assert_eq!(covered(&d, *id), "one|two"); +} + +#[test] +fn replies_follow_their_parent() { + let xml = concat!( + r#"AdaQuestion?text"#, + r#"AlanAnswer."#, + ); + let d = import(&body(xml)).unwrap(); + let got = comments(&d); + assert_eq!(got.len(), 2); + assert_eq!(got[1].4.as_deref(), Some("Question?")); + assert_eq!(got[1].5, "Answer."); + // A reply has no place in the text of its own. + let reply_id = *d.comments.iter().find(|(_, c)| c.parent.is_some()).unwrap().0; + let anchored = d + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .any(|o| matches!(o, InlineObject::CommentStart { id } if *id == reply_id)); + assert!(!anchored); +} + +#[test] +fn comments_round_trip() { + let xml = concat!( + r#"Before Ada Lovelace2026-10-31T10:00:00ALFirst comment.commentedwords"#, + r#"AlanReply. and"#, + r#"GraceA point. after."#, + ); + let d = import(&body(xml)).unwrap(); + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(comments(&back), comments(&d)); + assert_eq!(back.plain_text(StoryRef::Body), d.plain_text(StoryRef::Body)); +} + +#[test] +fn unmatched_and_hostile_comments_do_not_panic() { + for xml in [ + // An end without a start, a start whose end never comes, a nested annotation. + r#"ab"#.to_string(), + r#"acb"#.to_string(), + r#"xy"# + .to_string(), + r#""#.to_string(), + format!(r#"{}long"#, "n".repeat(100_000)), + r#"orphan"#.to_string(), + ] { + let d = import(&body(&xml)).unwrap(); + let again = import(&export(&d).unwrap()).unwrap(); + // Every comment kept is whole: its story exists and its range closes. + for (id, c) in &again.comments { + assert!(again.parts.contains_key(&c.part), "{xml}"); + let starts = again + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter(|o| matches!(o, InlineObject::CommentStart { id: i } if i == id)) + .count(); + let ends = again + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter(|o| matches!(o, InlineObject::CommentEnd { id: i } if i == id)) + .count(); + assert_eq!(starts, ends, "{xml}"); + } + } +} + +#[test] +fn a_reply_to_a_reply_is_kept() { + let xml = concat!( + r#"ARootx"#, + r#"BReply"#, + r#"CReply to reply"#, + ); + let d = import(&body(xml)).unwrap(); + assert_eq!(d.comments.len(), 3); + // LibreOffice's threads are flat: every reply answers the comment that starts them. + let back = import(&export(&d).unwrap()).unwrap(); + let got = comments(&back); + assert_eq!(got.len(), 3, "{got:?}"); + assert_eq!(got[2].4.as_deref(), Some("Root")); + assert_eq!(got[2].5, "Reply to reply"); +} diff --git a/crates/odf/src/text/tests_more/connectors.rs b/crates/odf/src/text/tests_more/connectors.rs new file mode 100644 index 000000000..0aedd8c62 --- /dev/null +++ b/crates/odf/src/text/tests_more/connectors.rs @@ -0,0 +1,102 @@ +//! Connectors (ODF 1.3 §10.3.10 `draw:connector`): lines glued to other shapes' glue points +//! (§9.2.19), read as WordCraft connectors glued to the shapes' connection sites, placed between +//! them, and written back glued. + +use crate::text::import; +use crate::text::tests::{NS_T, content_of, package}; +use craft_wordmodel::para::{InlineObject, ShapeKind}; + +const NS_D: &str = + r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0""#; + +/// A rectangle (`id1`) and an ellipse (`id2`) anchored to one paragraph, and a connector of +/// `kind` from the rectangle's bottom (glue point 2) to the ellipse's top (0), its own +/// coordinates in LibreOffice's drawing layer (far from the shapes). +fn doc(kind: &str) -> craft_wordmodel::Document { + let content = format!( + r#"Text."# + ); + import(&package(&content, None)).unwrap() +} + +/// The paragraph's shapes: (kind, frame x, y, w, h, flips, drawing id, glued ends). +#[allow(clippy::type_complexity)] +fn shapes(d: &craft_wordmodel::Document) -> Vec<(ShapeKind, [f32; 4], (bool, bool), u32, Option<(u32, u32)>, Option<(u32, u32)>)> { + d.body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter_map(|o| match o { + InlineObject::Shape { kind, w, h, float, extra, .. } => Some(( + *kind, + [float.x, float.y, *w, *h].map(|v| (v * 10.0).round() / 10.0), + (float.flip_h, float.flip_v), + extra.id, + extra.start.map(|e| (e.id, e.site)), + extra.end.map(|e| (e.id, e.site)), + )), + _ => None, + }) + .collect() +} + +#[test] +fn a_connector_is_glued_between_its_shapes() { + let s = shapes(&doc("")); + assert_eq!(s.len(), 3); + assert_eq!((s[0].0, s[0].3), (ShapeKind::Rectangle, 1)); + assert_eq!((s[1].0, s[1].3), (ShapeKind::Ellipse, 2)); + // From the rectangle's bottom (its site 2) to the ellipse's top (site 0), and placed so: + // (36, 36) to (252, 72). + assert_eq!(s[2], (ShapeKind::ElbowConnector, [36.0, 36.0, 216.0, 36.0], (false, false), 0, Some((1, 2)), Some((2, 0)))); + // Its type: straight and curved. + assert_eq!(shapes(&doc(r#" draw:type="line""#))[2].0, ShapeKind::StraightConnector); + assert_eq!(shapes(&doc(r#" draw:type="curve""#))[2].0, ShapeKind::CurvedConnector); +} + +#[test] +fn connectors_are_written_glued_and_read_back() { + let d = doc(""); + let c = content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#""# + ); + let s = shapes(&import(&package(&content, None)).unwrap()); + assert_eq!((s[1].4, s[1].5), (None, None)); + assert_eq!(s[2].4, None); +} + +#[test] +fn a_connector_in_a_group_is_glued_by_its_shapes_sites() { + // In a group, a connector glued to the group's ellipse: its glue point is the ellipse's site + // (right: glue point 1 is the ellipse's site 6), not taken as a site number itself. + let content = format!( + r#""# + ); + let d = import(&package(&content, None)).unwrap(); + let ends: Vec<(Option<(u32, u32)>, Option<(u32, u32)>)> = d + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter_map(|o| if let InlineObject::Group { children, .. } = o { Some(children) } else { None }) + .flatten() + .filter_map(|c| match &c.obj { + InlineObject::Shape { kind, extra, .. } if kind.is_connector() => { + Some((extra.start.map(|e| (e.id, e.site)), extra.end.map(|e| (e.id, e.site)))) + } + _ => None, + }) + .collect(); + assert_eq!(ends, [(Some((1, 6)), Some((2, 1)))]); +} diff --git a/crates/odf/src/text/tests_more/custom_geometry.rs b/crates/odf/src/text/tests_more/custom_geometry.rs new file mode 100644 index 000000000..176b3d478 --- /dev/null +++ b/crates/odf/src/text/tests_more/custom_geometry.rs @@ -0,0 +1,240 @@ +//! Custom shapes whose type WordCraft has no kind for (ODF 1.3 §10.6.2 +//! `draw:enhanced-geometry`): drawn from their own path (§19.145 `draw:enhanced-path`), its +//! values worked out from the shape's equations (§10.6.3 `draw:equation`) and adjustment values +//! (`draw:modifiers`), as freeforms. + +use crate::text::tests::{NS_T, package}; +use crate::text::{export, import}; +use craft_wordmodel::Document; +use craft_wordmodel::para::{InlineObject, ShapeKind}; + +const NS_D: &str = + r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0""#; + +/// A 1in square custom shape with geometry attributes `geo` and equations `eqs`. +fn shape(geo: &str, eqs: &str) -> String { + format!( + r#"{eqs}"# + ) +} + +fn open(shapes: &str) -> Document { + let content = format!( + r#"{shapes}Text."# + ); + import(&package(&content, None)).unwrap() +} + +/// Each shape's kind and its paths in frame points (rounded), closed or not. +#[allow(clippy::type_complexity)] +fn drawn(d: &Document) -> Vec<(ShapeKind, Vec<(Vec<(f32, f32)>, bool)>)> { + let r = |v: f32| v.round(); + d.body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter_map(|o| match o { + InlineObject::Shape { kind, w, h, freeform, .. } => { + let paths = freeform + .as_ref() + .map(|f| f.placed(0.0, 0.0, *w, *h).into_iter().map(|(pts, c)| (pts.iter().map(|[x, y]| (r(*x), r(*y))).collect(), c)).collect()) + .unwrap_or_default(); + Some((*kind, paths)) + } + _ => None, + }) + .collect() +} + +#[test] +fn a_hexagon_is_drawn_from_its_equations_and_adjustment_value() { + let d = open(&shape( + r#"svg:viewBox="0 0 21600 21600" draw:type="hexagon" draw:modifiers="5400" draw:enhanced-path="M ?f0 0 L ?f1 0 21600 10800 ?f1 21600 ?f0 21600 0 10800 Z N""#, + r#""#, + )); + let pts = vec![(18.0, 0.0), (54.0, 0.0), (72.0, 36.0), (54.0, 72.0), (18.0, 72.0), (0.0, 36.0)]; + let got = drawn(&d); + assert_eq!(got.len(), 1); + assert_eq!(got[0].0, ShapeKind::Freeform); + let (p, closed) = &got[0].1[0]; + assert!(*closed); + assert_eq!(p.get(..6), Some(pts.as_slice()), "{p:?}"); +} + +#[test] +fn formulas_use_functions_operators_and_other_equations() { + // f2 = if(f1, 10800, 0) with f1 = max(abs(-3), sqrt(16)) * (2 + 1) / 4 = 3: 10800. + let d = open(&shape( + r#"draw:type="mso-spt99" draw:enhanced-path="M 0 0 L ?f2 ?f3 N""#, + r#""#, + )); + assert_eq!(drawn(&d)[0].1, [(vec![(0.0, 0.0), (36.0, 36.0)], false)]); +} + +#[test] +fn quarter_ellipses_run_along_one_axis_then_the_other() { + // A terminator: straight top and bottom, rounded ends. + let d = open(&shape( + r#"draw:type="flowchart-terminator" draw:enhanced-path="M 3470 21600 X 0 10800 3470 0 L 18130 0 X 21600 10800 18130 21600 Z N""#, + "", + )); + let (_, paths) = &drawn(&d)[0]; + let pts = &paths[0].0; + // Bulging out to the frame's left and right edges, never beyond. + let (min_x, max_x) = pts.iter().fold((f32::MAX, f32::MIN), |(a, b), p| (a.min(p.0), b.max(p.0))); + assert_eq!((min_x, max_x), (0.0, 72.0)); + assert!(pts.contains(&(0.0, 36.0)) && pts.contains(&(72.0, 36.0)), "{pts:?}"); + // The left end leaves the bottom point going left (along x) first: its next point is lower + // than half way. + let next = pts.get(1).copied().unwrap_or_default(); + assert!(next.0 < 12.0 && next.1 > 54.0, "{next:?}"); +} + +#[test] +fn ellipses_and_arcs_are_followed_in_steps() { + // A circle (U: centre, radii, angles), and the upper half of the frame's ellipse, drawn + // anticlockwise from its right to its left (A: box, from, to). + let d = + open(&shape(r#"draw:type="smiley" draw:enhanced-path="U 10800 10800 10800 10800 0 360 Z N B 0 0 21600 21600 21600 10800 0 10800 N""#, "")); + let (_, paths) = &drawn(&d)[0]; + assert_eq!(paths.len(), 2, "{paths:?}"); + let circle = &paths[0].0; + assert!(circle.len() > 32 && circle.iter().all(|(x, y)| ((x - 36.0).hypot(y - 36.0) - 36.0).abs() <= 1.0), "{circle:?}"); + let arc = &paths[1].0; + assert_eq!((arc.first(), arc.last()), (Some(&(72.0, 36.0)), Some(&(0.0, 36.0)))); + // Anticlockwise on the page (y down): over the top. + assert!(arc.iter().all(|(_, y)| *y <= 36.0), "{arc:?}"); +} + +#[test] +fn a_custom_shape_keeps_its_drawn_geometry_when_saved() { + let d = open(&shape(r#"draw:type="octagon" draw:enhanced-path="M 0 0 L 21600 0 10800 21600 Z N""#, "")); + let again = import(&export(&d).unwrap()).unwrap(); + assert_eq!(drawn(&again), drawn(&d)); + assert_eq!(drawn(&again)[0].0, ShapeKind::Freeform); +} + +#[test] +fn known_and_plain_types_keep_their_kinds() { + let d = open(&format!( + "{}{}", + shape(r#"draw:type="ellipse" draw:enhanced-path="M 0 0 L 1 1 N""#, ""), + shape(r#"draw:type="rectangle" draw:enhanced-path="M 0 0 L 21600 0 21600 21600 0 21600 Z N""#, "") + )); + let kinds: Vec = drawn(&d).iter().map(|s| s.0).collect(); + assert_eq!(kinds, [ShapeKind::Ellipse, ShapeKind::Rectangle]); +} + +#[test] +fn hostile_geometry_never_panics_and_draws_what_it_can() { + let deep = format!("{}1{}", "(".repeat(5000), ")".repeat(5000)); + let unary = "-".repeat(5000); + let mut eqs = String::new(); + // A chain of equations each referring to the next, and one referring to itself. + for i in 0..2000 { + eqs.push_str(&format!(r#""#, i + 1)); + } + eqs.push_str(r#""#); + for (path, eq) in [ + ("M ?f0 0 L 10 10 N", eqs.as_str()), + ("M ?me ?nothing L $99 $-1 N", eqs.as_str()), + ("M 0 0 L ?d 1 N", &*format!(r#""#)), + ("M 0 0 L ?u 1 N", &*format!(r#""#)), + ("M 1e308 1e308 L -1e308 1e308 C NaN inf 0 0 0 0 N", ""), + ("U 0 0 1e300 1e300 0 1e300 Z B 0 0 0 0 0 0 0 0 W 1 1 1 1 1 1 1 1 X Y X 5 5 Y", ""), + ("M 0 0 L 1", ""), + ("Q", ""), + (&*"L 1 1 ".repeat(30_000), ""), + (&*format!("M 0 0 {}", "U 10 10 10 10 0 360 ".repeat(5000)), ""), + ] { + let d = open(&shape(&format!(r#"draw:type="hostile" svg:viewBox="0 0 21600 21600" draw:enhanced-path="{path}""#), eq)); + let _ = drawn(&d); + let _ = export(&d).unwrap(); + } + // An empty or broken view box falls back to a rectangle. + let d = open(&shape(r#"draw:type="hostile" svg:viewBox="0 0 0 0" draw:enhanced-path="M 0 0 L 1 1 N""#, "")); + assert_eq!(drawn(&d)[0].0, ShapeKind::Rectangle); +} + +#[test] +fn a_freeform_with_open_and_closed_paths_is_saved_as_a_custom_shape() { + // A face and a mouth: the face filled, the mouth not. A path with an open part would be + // drawn unfilled whole, so it is saved as a custom shape whose path marks the mouth unfilled. + let mut d = open(&shape(r#"draw:type="smiley" draw:enhanced-path="M 0 0 L 21600 0 21600 21600 Z N M 5000 15000 L 16000 15000 F N""#, "")); + // As WordCraft's own freeform (the geometry read is written back whole otherwise). + for b in d.body.iter_mut() { + if let craft_wordmodel::Block::Para(p) = std::sync::Arc::make_mut(b) { + for o in &mut p.objects { + if let InlineObject::Shape { extra, .. } = o { + extra.odf_geometry = None; + } + } + } + } + let c = crate::text::tests::content_of(&export(&d).unwrap()); + assert!(c.contains(r#"draw:type="non-primitive""#) && c.contains(" F N"), "{c}"); + assert!(!c.contains(">(), [true, false]); +} + +#[test] +fn equations_referring_twice_to_ones_that_fail_are_worked_out_once() { + // Each equation refers twice to the next, and the last to itself: worked out naively, 2^200 + // steps. Each is worked out once, and all fail, so the path has nothing to draw. + let mut eqs = String::new(); + for i in 0..200 { + eqs.push_str(&format!(r#""#, i + 1)); + } + eqs.push_str(r#""#); + let start = std::time::Instant::now(); + let d = open(&shape(r#"draw:type="hostile" draw:enhanced-path="M 0 0 L ?f0 10 L 20 20 N""#, &eqs)); + assert!(start.elapsed().as_secs() < 5); + // The line with the unknown end is left out; the rest is drawn. + assert_eq!(drawn(&d)[0].1, [(vec![(0.0, 0.0), (0.0, 0.0)], false)]); +} + +#[test] +fn points_far_outside_the_view_box_stay_finite() { + let d = open(&shape(r#"draw:type="hostile" draw:enhanced-path="M 0 0 L 1e300 -1e300 N""#, "")); + let Some(InlineObject::Shape { freeform: Some(f), .. }) = d.body[0].as_para().unwrap().objects.first() else { panic!("a freeform") }; + assert!(f.paths.iter().flat_map(|p| p.pts.iter()).all(|[x, y]| x.is_finite() && y.is_finite())); +} + +#[test] +fn a_custom_shapes_geometry_is_written_back_as_it_was_read() { + // LibreOffice's shape presets: saved as their own geometry (type, path, equations, handles), + // so LibreOffice draws its own shape, not WordCraft's path of it; a preset WordCraft draws as + // a rectangle (a cloud) keeps its geometry too. A shape made another kind loses it. + let hexagon = shape( + r#"svg:viewBox="0 0 21600 21600" draw:type="hexagon" draw:modifiers="5400" draw:mirror-horizontal="true" draw:enhanced-path="M ?f0 0 L ?f1 0 21600 10800 ?f1 21600 ?f0 21600 0 10800 Z N""#, + r#""#, + ); + let cloud = r#""#; + let d = open(&format!("{hexagon}{cloud}")); + assert_eq!(drawn(&d).iter().map(|s| s.0).collect::>(), [ShapeKind::Freeform, ShapeKind::Rectangle]); + let out = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(out.contains(r#"draw:type="hexagon" draw:modifiers="5400""#) && out.contains(r#"draw:formula="right-$0""#), "{out}"); + assert!(out.contains(r#""#), "{out}"); + // Its mirroring is the shape's own, written once. + assert_eq!(out.matches("draw:mirror-horizontal=\"true\"").count(), 1, "{out}"); + assert!(out.contains(r#""#), "{out}"); + // Read back, it is the same shape again. + let back = import(&crate::text::tests::export_model(&d).unwrap()).unwrap(); + assert_eq!(drawn(&back), drawn(&d)); + // Made an ellipse: written as one. + let mut changed = d.clone(); + for b in changed.body.iter_mut() { + if let craft_wordmodel::Block::Para(p) = std::sync::Arc::make_mut(b) { + for o in &mut p.objects { + if let InlineObject::Shape { kind, extra, .. } = o { + *kind = ShapeKind::Ellipse; + extra.odf_geometry = None; + } + } + } + } + let out = crate::text::tests::content_of(&crate::text::tests::export_model(&changed).unwrap()); + assert!(!out.contains("hexagon") && !out.contains("cloud"), "{out}"); +} diff --git a/crates/odf/src/text/tests_more/drop_caps.rs b/crates/odf/src/text/tests_more/drop_caps.rs new file mode 100644 index 000000000..f7a6720b7 --- /dev/null +++ b/crates/odf/src/text/tests_more/drop_caps.rs @@ -0,0 +1,44 @@ +//! Drop caps (ODF 1.3 §17.9 `style:drop-cap`): a paragraph's first letter over several lines, +//! its distance from the text, kept as WordCraft's drop cap and written back. + +use crate::text::tests::{NS_T, content_of, package}; +use crate::text::{export, import}; +use craft_wordmodel::Document; + +fn open(drop: &str) -> Document { + let content = format!( + r#"{drop}Once upon a time there was a long paragraph.Plain."# + ); + import(&package(&content, None)).unwrap() +} + +/// Each paragraph's drop cap: (lines, distance). +fn caps(d: &Document) -> Vec<(Option, Option)> { + d.body.iter().filter_map(|b| b.as_para()).map(|p| (p.props.drop_cap, p.props.drop_cap_distance.map(|v| (v * 100.0).round() / 100.0))).collect() +} + +#[test] +fn a_drop_cap_opens() { + assert_eq!(caps(&open(r#""#)), [(Some(3), Some(7.2)), (None, None)]); + // One line is no drop cap (LibreOffice's "1"). + assert_eq!(caps(&open(r#""#)), [(None, None), (None, None)]); +} + +#[test] +fn drop_caps_round_trip() { + let d = open(r#""#); + let bytes = crate::text::tests::export_model(&d).unwrap(); + assert!(content_of(&bytes).contains(r#""#), "{}", content_of(&bytes)); + assert_eq!(caps(&import(&bytes).unwrap()), caps(&d)); +} + +#[test] +fn hostile_drop_caps_do_not_panic() { + for drop in + [r#""#, r#""#] + { + let d = open(drop); + assert!(caps(&d).iter().all(|(l, dist)| l.is_none_or(|l| l <= 10) && dist.is_none_or(|v| v.is_finite() && v <= 1584.0)), "{drop}"); + let _ = export(&d).unwrap(); + } +} diff --git a/crates/odf/src/text/tests_more/encryption.rs b/crates/odf/src/text/tests_more/encryption.rs new file mode 100644 index 000000000..2d998e6c9 --- /dev/null +++ b/crates/odf/src/text/tests_more/encryption.rs @@ -0,0 +1,78 @@ +//! Password-protected ODT (ODF 1.3 Part 2 §3.4): saved encrypted as LibreOffice encrypts +//! (AES-256-CBC, SHA-256 start key, PBKDF2), opened with the right password, refused with a +//! wrong one, and hostile encryption data never panics or hangs. + +use crate::text::tests::{NS_T, package, zip_of}; +use crate::text::{CryptError, decrypt, encrypt, export, import, is_encrypted}; +use craft_wordmodel::Document; + +fn secret_doc() -> Vec { + let content = format!( + r#"Top secret words."# + ); + export(&import(&package(&content, None)).unwrap()).unwrap() +} + +fn text(d: &Document) -> String { + d.body.iter().filter_map(|b| b.as_para()).map(|p| p.text.clone()).collect::>().join("\n") +} + +#[test] +fn an_encrypted_package_opens_with_its_password_only() { + let plain = secret_doc(); + assert!(!is_encrypted(&plain)); + let enc = encrypt(&plain, "pässword").unwrap(); + assert!(is_encrypted(&enc)); + // Nothing readable is left: the content is encrypted, the manifest says how. + let z = crate::Package::open(&enc, &crate::Limits::DEFAULT).unwrap(); + assert_eq!(z.names().next(), Some("mimetype")); + assert!(!enc.windows(10).any(|w| w == b"Top secret")); + let manifest = crate::text::tests::entry(&enc, "META-INF/manifest.xml"); + for want in [ + r#""# + ) + }; + let pkg = |m: String, content: &[u8]| { + zip_of(&[("mimetype", b"application/vnd.oasis.opendocument.text"), ("META-INF/manifest.xml", m.as_bytes()), ("content.xml", content)]) + }; + // Blowfish: said, not guessed at. + let blowfish = pkg(manifest("Blowfish CFB", r#"manifest:iteration-count="1024""#), &[0u8; 64]); + assert!(is_encrypted(&blowfish)); + assert!(matches!(decrypt(&blowfish, "x"), Err(CryptError::Unsupported(w)) if w.contains("Blowfish"))); + // An iteration count that would hang, a vector of the wrong size, ragged data: refused. + for (extra, data) in [ + (r#"manifest:iteration-count="4000000000""#, vec![0u8; 32]), + (r#"manifest:iteration-count="1""#, vec![0u8; 33]), + (r#"manifest:iteration-count="1""#, vec![]), + ] { + assert!(decrypt(&pkg(manifest("http://www.w3.org/2001/04/xmlenc#aes256-cbc", extra), &data), "x").is_err(), "{extra}"); + } + // Junk that isn't a package. + assert!(!is_encrypted(b"not a zip")); + assert!(decrypt(b"not a zip", "x").is_err()); +} + +#[test] +fn a_package_declaring_too_many_entries_is_refused_before_it_is_decrypted() { + // #3660: refused before `zip` reserves and parses them, on the password path too. + let bomb = crate::zip_guard::zip64_entry_count_sample(crate::Limits::DEFAULT.max_entries + 1); + assert!(!is_encrypted(&bomb)); + assert!(matches!(decrypt(&bomb, "x"), Err(CryptError::Unsupported(e)) if e.contains("declares"))); +} diff --git a/crates/odf/src/text/tests_more/equations.rs b/crates/odf/src/text/tests_more/equations.rs new file mode 100644 index 000000000..1909da459 --- /dev/null +++ b/crates/odf/src/text/tests_more/equations.rs @@ -0,0 +1,61 @@ +//! Equations written as ODF formula objects (ODF 1.3 §10.4.3 `draw:object`, Part 2 packages: a +//! formula sub-document holding MathML), read back by the reader that opens LibreOffice's. + +use crate::text::{export, import}; +use craft_wordmodel::para::{InlineObject, OBJ}; +use craft_wordmodel::{Document, Paragraph, para_block}; + +fn eq_para(before: &str, linear: &str, display: bool) -> std::sync::Arc { + let mut p = Paragraph::new(); + p.text = format!("{before}{OBJ}"); + p.objects.push(InlineObject::Equation { linear: linear.into(), display, math: Default::default() }); + p.runs = vec![craft_wordmodel::para::Run { len: p.text.len(), props: Default::default() }]; + para_block(p) +} + +fn equations(d: &Document) -> Vec<(String, bool)> { + d.body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter_map(|o| if let InlineObject::Equation { linear, display, .. } = o { Some((linear.clone(), *display)) } else { None }) + .collect() +} + +#[test] +fn equations_round_trip_as_formula_objects() { + let mut d = Document::new(); + d.body = vec![eq_para("Inline ", "x^2+y^2", false), eq_para("", "a/b", true)]; + let bytes = export(&d).unwrap(); + // A formula sub-document per equation, declared in the manifest. + let mut z = crate::Package::open(bytes.as_slice(), &crate::Limits::DEFAULT).unwrap(); + let names: Vec = z.names().map(String::from).collect(); + assert!(names.iter().any(|n| n == "Object 1/content.xml") && names.iter().any(|n| n == "Object 2/content.xml"), "{names:?}"); + let manifest = String::from_utf8(z.read("META-INF/manifest.xml").unwrap().unwrap()).unwrap(); + assert!( + manifest + .contains(r#"manifest:full-path="Object 1/" manifest:version="1.3" manifest:media-type="application/vnd.oasis.opendocument.formula""#), + "{manifest}" + ); + // A formula document's content.xml has a `math:math` root (ODF 1.3 Part 3 §2.2.9 B), not an + // `office:document-content` (the schema's start pattern doesn't cover it; the spec does). + let object = String::from_utf8(z.read("Object 1/content.xml").unwrap().unwrap()).unwrap(); + let root = object.trim_start_matches(|c| c != '>').get(1..).unwrap_or("").trim_start(); + assert!(root.starts_with(""# + ); + let math = br#"x"#; + let bytes = + zip_of(&[("mimetype", b"application/vnd.oasis.opendocument.text"), ("content.xml", content.as_bytes()), ("Object 1/content.xml", math)]); + assert_eq!(equations(&import(&bytes).unwrap()), [("x".to_string(), true)]); +} diff --git a/crates/odf/src/text/tests_more/fields.rs b/crates/odf/src/text/tests_more/fields.rs new file mode 100644 index 000000000..bd826253d --- /dev/null +++ b/crates/odf/src/text/tests_more/fields.rs @@ -0,0 +1,368 @@ +//! Text fields (ODF 1.3 §7): page number and count, dates and times, document information and +//! bookmark references open as WordCraft fields and are written back as ODF fields. + +use crate::text::tests::{NS_T, package}; +use crate::text::{export, import}; +use craft_wordmodel::para::InlineObject; +use craft_wordmodel::{Document, Paragraph, para_block}; + +/// Every field in the body: (code, cached result). +pub(super) fn fields(d: &Document) -> Vec<(String, String)> { + let mut out = Vec::new(); + for b in d.body.iter() { + if let Some(p) = b.as_para() { + for o in &p.objects { + if let InlineObject::Field { instr, result, .. } = o { + out.push((instr.clone(), result.clone())); + } + } + } + } + out +} + +fn body(xml: &str) -> Vec { + package( + &format!(r#"{xml}"#), + None, + ) +} + +#[test] +fn fields_open_as_fields_with_their_shown_text() { + let d = import(&body(concat!( + r#"Page 3 of 9."#, + r#"31/10/2026 12:00"#, + r#"My title|Subj|Ada|f.odt"#, + r#"See 2 and Intro"#, + ))) + .unwrap(); + let got = fields(&d); + let want = [ + ("PAGE", "3"), + ("NUMPAGES", "9"), + ("DATE", "31/10/2026"), + ("TIME", "12:00"), + ("TITLE", "My title"), + ("SUBJECT", "Subj"), + ("AUTHOR", "Ada"), + ("FILENAME", "f.odt"), + ("PAGEREF intro \\h", "2"), + ("REF intro \\h", "Intro"), + ]; + assert_eq!(got.iter().map(|(a, b)| (a.as_str(), b.as_str())).collect::>(), want); + // The text around the fields stays where it was. + let p0 = d.body[0].as_para().unwrap(); + assert_eq!(p0.text.replace(craft_wordmodel::para::OBJ, "#"), "Page # of #."); +} + +#[test] +fn a_fixed_date_is_a_locked_field_and_unknown_fields_are_plain_text() { + // A fixed date (§19.807 `text:fixed`) keeps what it shows: a locked field, written back fixed. + let d = import(&body(r#"1 May, 4, 5"#)).unwrap(); + assert_eq!(fields(&d), [("DATE".to_string(), "1 May".to_string())]); + let locked = d.body[0].as_para().unwrap().objects.iter().any(|o| matches!(o, InlineObject::Field { locked: true, .. })); + assert!(locked); + // Written back: with no picture to read its value under, as its text (a reader would show a + // fixed date without a value as today's). + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(!c.contains(" :1 May 2026 14:30"# + ), + None, + )) + .unwrap(); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#"text:date-value="2026-05-01""#) && c.contains(">1 May 2026"), "{c}"); + assert!(c.contains(r#"text:time-value="1899-12-30T14:30:00""#) && c.contains(">14:30"), "{c}"); + assert_eq!(c.matches(r#"text:fixed="true""#).count(), 2, "{c}"); + // A fixed date with neither a value nor text (LibreOffice's templates) shows the day: a + // date field. + let d = import(&body(r#"."#)).unwrap(); + assert_eq!(fields(&d).len(), 1); +} + +fn field_para(parts: &[(&str, Option<&str>)]) -> std::sync::Arc { + // Text pieces, and fields given as (code, Some(result)). + let mut p = Paragraph::new(); + for (s, field) in parts { + match field { + Some(result) => { + p.text.push(craft_wordmodel::para::OBJ); + p.objects.push(InlineObject::Field { + instr: s.to_string(), + result: result.to_string(), + locked: false, + dirty: false, + code: false, + form: String::new(), + }); + } + None => p.text.push_str(s), + } + } + p.runs = vec![craft_wordmodel::para::Run { len: p.text.len(), props: Default::default() }]; + para_block(p) +} + +#[test] +fn fields_round_trip() { + let mut d = Document::new(); + d.body = vec![field_para(&[ + ("Page ", None), + ("PAGE", Some("1")), + (" of ", None), + ("NUMPAGES", Some("2")), + (" ", None), + ("DATE \\@ \"d MMMM yyyy\"", Some("31 October 2026")), + (" ", None), + ("TITLE", Some("T")), + (" ", None), + ("PAGEREF intro \\h", Some("4")), + ])]; + let xml = export(&d).unwrap(); + let back = import(&xml).unwrap(); + let got: Vec = fields(&back).into_iter().map(|(c, _)| c).collect(); + // The date keeps its picture (as a date style). + assert_eq!(got, ["PAGE", "NUMPAGES", "DATE \\@ \"d MMMM yyyy\"", "TITLE", "PAGEREF intro \\h"]); + assert_eq!(back.body[0].as_para().unwrap().text.matches(craft_wordmodel::para::OBJ).count(), 5); +} + +#[test] +fn a_field_odf_has_no_element_for_keeps_its_result_as_text() { + let mut d = Document::new(); + d.body = vec![field_para(&[("x ", None), ("SECTIONPAGES", Some("7")), (" y", None)])]; + let back = import(&export(&d).unwrap()).unwrap(); + assert!(fields(&back).is_empty()); + assert_eq!(back.body[0].as_para().unwrap().text, "x 7 y"); +} + +#[test] +fn hostile_fields_do_not_panic() { + // Unclosed, nested and empty fields, and a long reference name. + let long = "n".repeat(100_000); + for xml in [ + "".to_string(), + "1".to_string(), + "".to_string(), + format!(r#"1"#), + ] { + let d = import(&body(&xml)).unwrap(); + let _ = export(&d).unwrap(); + } +} + +#[test] +fn an_element_inside_a_field_leaves_the_formatting_after_it_alone() { + // A span inside a field was skipped at its start but still ended, popping the enclosing + // span's formatting: the text after the field lost its bold. + let content = format!( + r#"x1y"# + ); + let d = import(&package(&content, None)).unwrap(); + let p = d.body[0].as_para().unwrap(); + assert_eq!(fields(&d), [("PAGE".to_string(), "1".to_string())]); + assert!(p.runs.iter().all(|r| r.props.bold == Some(true)), "{:?}", p.runs); +} + +#[test] +fn a_paragraph_ending_inside_a_field_still_ends() { + let d = import(&body("a1b")).unwrap(); + let texts: Vec = d.body.iter().filter_map(|b| b.as_para()).map(|p| p.text.replace(craft_wordmodel::para::OBJ, "#")).collect(); + assert_eq!(texts, ["a#", "b"]); +} + +#[test] +fn custom_properties_and_user_fields_are_docproperty_fields() { + // A custom property (`meta:user-defined`) shown by `text:user-defined`, and a user field + // (`text:user-field-decl`) shown by `text:user-field-get`: Word's custom document properties + // and `DOCPROPERTY` fields. + let meta = r#"Kiln1200.5"#; + let content = format!( + r#"For Kiln by Ada."# + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("meta.xml", meta.as_bytes())]); + let d = crate::text::import(&pkg).unwrap(); + let props: Vec<(String, String, String)> = d.custom_props.iter().map(|p| (p.name.clone(), p.kind.clone(), p.value.clone())).collect(); + assert_eq!( + props, + [ + ("Project".into(), "lpwstr".into(), "Kiln".into()), + ("Budget".into(), "r8".into(), "1200.5".into()), + ("Owner".into(), "lpwstr".into(), "Ada".into()) + ] + ); + let instrs: Vec = d + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter_map(|o| if let craft_wordmodel::para::InlineObject::Field { instr, .. } = o { Some(instr.clone()) } else { None }) + .collect(); + assert_eq!(instrs, [r#"DOCPROPERTY "Project""#, r#"DOCPROPERTY "Owner""#]); + // Saved: the properties in `meta.xml` with `text:user-defined` fields, the user field as one. + let saved = crate::text::export(&d).unwrap(); + let m = crate::text::tests::entry(&saved, "meta.xml"); + assert!(m.contains(r#"1200.5"#), "{m}"); + assert!(!m.contains(r#"meta:name="Owner""#), "{m}"); + let c = crate::text::tests::content_of(&saved); + assert!(c.contains(r#"Kiln"#), "{c}"); + assert!(c.contains(r#"Ada"#), "{c}"); + assert_eq!(crate::text::import(&saved).unwrap().custom_props.len(), 3); +} + +#[test] +fn custom_date_properties_are_full_utc_dates() { + for (v, kind, want) in [ + ("2026-11-01", "filetime", "2026-11-01T00:00:00Z"), + ("2026-11-01T10:30:00", "filetime", "2026-11-01T10:30:00Z"), + ("2026-11-01T10:30:00Z", "filetime", "2026-11-01T10:30:00Z"), + ("soon", "lpwstr", "soon"), + ] { + let p = crate::text::fields::custom_prop("When", Some("date"), v).unwrap(); + assert_eq!((p.kind.as_str(), p.value.as_str()), (kind, want), "{v}"); + } +} + +#[test] +fn bibliography_marks_are_citations_of_sources() { + // `text:bibliography-mark`: a citation carrying its source, which becomes the document's. + let content = format!( + r#"As shown [Smi20], and again [Smi20]."# + ); + let d = import(&package(&content, None)).unwrap(); + assert_eq!(d.sources.len(), 1); + let s = &d.sources[0]; + assert_eq!( + (s.tag.as_str(), s.kind.as_str(), s.author.as_str(), s.title.as_str(), s.year.as_str(), s.publisher.as_str()), + ("Smi20", "book", "Smith, Jane", "Kilns", "2020", "Clay Press") + ); + let fields: Vec<(String, String)> = d + .body + .iter() + .filter_map(|b| b.as_para()) + .flat_map(|p| p.objects.iter()) + .filter_map(|o| if let InlineObject::Field { instr, result, .. } = o { Some((instr.clone(), result.clone())) } else { None }) + .collect(); + assert_eq!(fields, [("CITATION Smi20".to_string(), "[Smi20]".to_string()), ("CITATION Smi20".to_string(), "[Smi20]".to_string())]); + let c = crate::text::tests::content_of(&export(&d).unwrap()); + assert!( + c.contains(r#"[Smi20]"#), + "{c}" + ); +} + +const VARIABLES: &str = r#"Set: Ada "AL". Hidden: . Get: Grace. Number: 42."#; + +#[test] +fn variables_open_as_set_and_ref_fields() { + let d = import(&body(VARIABLES)).unwrap(); + let want: Vec<(String, String)> = [ + (r#"SET who "Ada \"AL\"""#, ""), + ("REF who", r#"Ada "AL""#), + (r#"SET who "Grace""#, ""), + ("REF who", "Grace"), + (r#"SET n "42""#, ""), + ("REF n", "42"), + ] + .into_iter() + .map(|(a, b)| (a.to_string(), b.to_string())) + .collect(); + assert_eq!(fields(&d), want); + // The text around them, as LibreOffice shows it. + assert_eq!(d.body[0].as_para().unwrap().text.replace(craft_wordmodel::para::OBJ, ""), "Set: . Hidden: . Get: . Number: ."); +} + +#[test] +fn variables_are_written_back_declared() { + let d = import(&body(VARIABLES)).unwrap(); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + for want in [ + r#""#, + r#""#, + r#"Grace"#, + ] { + assert!(c.contains(want), "{want} in {c}"); + } + assert_eq!(fields(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()), fields(&d)); +} + +#[test] +fn hostile_variables_never_panic() { + let long = "v".repeat(100_000); + for xml in [ + r#""#.to_string(), + r#""#.to_string(), + format!(r#""#), + r#""#.to_string(), + ] { + let d = import(&body(&xml)).unwrap(); + let _ = export(&d).unwrap(); + } +} + +#[test] +fn a_date_in_the_locales_order_shows_so() { + // LibreOffice's letters: year, month by name and day, `number:automatic-order`: shown in + // the locale's order, "November 2, 2026" in US English. + let styles = format!( + r#""# + ); + let content = format!( + r#"2026November2 2112026"# + ); + let d = import(&package(&content, Some(&styles))).unwrap(); + let codes: Vec = fields(&d).into_iter().map(|(c, _)| c).collect(); + assert_eq!(codes, [r#"DATE \@ "MMMM d, yyyy""#, r#"DATE \@ "M/d/yyyy""#]); +} + +#[test] +fn user_fields_stay_user_fields() { + // LibreOffice's report templates: user fields (§7.4.7 `text:user-field-decl`, §7.4.9 + // `text:user-field-get`) are written back as such, not as document properties. + let content = format!( + r#"Group: Group one, 3"# + ); + let d = import(&package(&content, None)).unwrap(); + assert!(d.custom_props.iter().all(|p| p.odf_user_field), "{:?}", d.custom_props); + let saved = crate::text::tests::export_model(&d).unwrap(); + let c = crate::text::tests::content_of(&saved); + assert!(c.contains(r#""#), "{c}"); + assert!(c.contains(r#"office:value-type="float" office:value="3""#), "{c}"); + assert!(c.contains(r#"Group one"#), "{c}"); + let meta = crate::text::tests::entry(&saved, "meta.xml"); + assert!(!meta.contains("GroupField1"), "not a document property too: {meta}"); + let back = import(&saved).unwrap(); + assert_eq!(back.custom_props, d.custom_props); +} + +#[test] +fn sender_fields_stay_fields() { + // LibreOffice's letter and fax templates: the sender's details (§7.3.6), shown as last filled + // in (WordCraft's `SENDER` field code), written back as the same fields. + let d = import(&body(r#"Ada Lovelace, ada@example.org"#)).unwrap(); + assert_eq!( + fields(&d), + [("SENDER firstname".into(), "Ada".into()), ("SENDER lastname".into(), "Lovelace".into()), ("SENDER email".into(), "ada@example.org".into())] + ); + let c = crate::text::tests::content_of(&export(&d).unwrap()); + assert!( + c.contains("Ada") && c.contains("ada@example.org"), + "{c}" + ); +} + +#[test] +fn a_field_that_follows_the_document_says_so() { + // §19.807: not fixed unless said; LibreOffice reads an author field without `text:fixed` as + // fixed (an empty one stayed empty instead of showing the author), so WordCraft says it, as + // LibreOffice does. Sender fields it writes without, and so does WordCraft. + let d = import(&body(r#" Ann"#)).unwrap(); + let c = crate::text::tests::content_of(&export(&d).unwrap()); + assert!(c.contains(r#"Ann"), "{c}"); +} diff --git a/crates/odf/src/text/tests_more/flat.rs b/crates/odf/src/text/tests_more/flat.rs new file mode 100644 index 000000000..7ddc29ed7 --- /dev/null +++ b/crates/odf/src/text/tests_more/flat.rs @@ -0,0 +1,169 @@ +//! Flat OpenDocument text (`.fodt`, ODF 1.3 Part 3 §2.1.2: the whole document in one +//! `office:document` element, pictures inline as `office:binary-data`, formulas as +//! `math:math` inside their object): opened and saved like a package. + +use crate::text::{export_flat, import_flat}; +use craft_wordmodel::Document; +use craft_wordmodel::para::InlineObject; + +/// A 1×1 PNG, base64. +const PNG_B64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + +fn flat() -> String { + format!( + r#" + + Flat title + + + + + + The footer + + Some bold words. + +{PNG_B64} + + xx + +"# + ) +} + +/// What a flat document holds: (its title, page width, body text, bold text, the paragraph +/// style, its footer's text, its pictures' sizes in bytes, its equations). +#[allow(clippy::type_complexity)] +fn summary(d: &Document) -> (String, f32, Vec, Vec, Option, String, Vec, Vec) { + let paras: Vec<_> = d.body.iter().filter_map(|b| b.as_para()).collect(); + let text = paras.iter().map(|p| p.text.replace(craft_wordmodel::para::OBJ, "#")).collect(); + let bold = paras + .iter() + .flat_map(|p| p.run_ranges().filter(|(_, c)| c.bold == Some(true)).map(|(r, _)| p.text.get(r).unwrap_or("").to_string()).collect::>()) + .collect(); + let style = paras.first().and_then(|p| p.props.style.clone()); + let footer = d + .last_section + .footers + .default + .and_then(|id| d.parts.get(&id)) + .map(|p| p.blocks.iter().filter_map(|b| b.as_para()).map(|p| p.text.clone()).collect()) + .unwrap_or_default(); + let mut pictures = Vec::new(); + let mut equations = Vec::new(); + for p in ¶s { + for o in &p.objects { + match o { + InlineObject::Image { media, .. } => pictures.push(d.media.get(media).map_or(0, |m| m.len())), + InlineObject::Equation { linear, .. } => equations.push(linear.clone()), + _ => {} + } + } + } + (d.core.title.clone(), d.last_section.page_w, text, bold, style, footer, pictures, equations) +} + +#[test] +fn a_flat_document_opens() { + let d = import_flat(flat().as_bytes()).unwrap(); + let s = summary(&d); + assert_eq!(s.0, "Flat title"); + assert_eq!(s.1, 360.0); + assert_eq!(s.2, ["Some bold words.", "#", "#"]); + assert_eq!(s.3, ["bold"]); + assert_eq!(s.4.as_deref(), Some("Quiet")); + assert_eq!(s.5, "The footer"); + // The picture's own bytes, decoded. + assert_eq!(s.6, [70]); + assert_eq!(s.7, ["x"]); +} + +#[test] +fn a_flat_document_round_trips() { + let d = import_flat(flat().as_bytes()).unwrap(); + let bytes = export_flat(&d).unwrap(); + let xml = String::from_utf8(bytes.clone()).unwrap(); + assert!(xml.contains("") && !xml.contains("Pictures/"), "pictures inline: {xml}"); + assert_eq!(xml.matches("").count(), 1, "one set of automatic styles"); + assert_eq!(summary(&import_flat(&bytes).unwrap()), summary(&d)); + // And by its extension. + assert_eq!(summary(&crate::text::tests::import_ext("fodt", &bytes).unwrap().unwrap()), summary(&d)); +} + +#[test] +fn hostile_flat_documents_do_not_panic() { + let f = flat(); + for bad in [ + String::new(), + "".to_string(), + f.replace("", ""), + f.replace(PNG_B64, "!!!not base64!!!"), + f.replace("", ""), + f.replace("", ""), + format!("{}", "".repeat(10_000)), + "\u{feff}".to_string(), + ] { + if let Ok(d) = import_flat(bad.as_bytes()) { + let _ = export_flat(&d).unwrap(); + } + } +} + +#[test] +fn an_inline_sub_document_does_not_end_the_body() { + // A chart object inline holds an `office:document` with an `office:body` of its own: the + // outer body goes on past it (the text after the chart stays), and deep nesting is bounded. + let chart = r#""#; + let xml = format!( + r#"Before{chart}After the chart."# + ); + let d = import_flat(xml.as_bytes()).unwrap(); + let text: Vec = d.body.iter().filter_map(|b| b.as_para()).map(|p| p.text.replace(craft_wordmodel::para::OBJ, "")).collect(); + assert_eq!(text, ["Before", "After the chart."]); + let deep = format!("{}", "".repeat(5000)); + let _ = import_flat(deep.as_bytes()); +} + +#[test] +fn a_template_is_saved_and_opened_as_one() { + // An OpenDocument text template (`.ott`): the package with the template's media type, in + // `mimetype` and as the manifest's root entry (ODF 1.3 Part 2 §3.3). + let d = crate::text::tests::import_ext("ott", &crate::text::tests::package(&format!("Template text.", crate::text::tests::NS_T), None)) + .unwrap() + .unwrap(); + let bytes = crate::text::tests::export_ext("ott", &d).unwrap().unwrap(); + assert_eq!(crate::text::tests::entry(&bytes, "mimetype"), crate::text::TEMPLATE_MIME); + assert!( + crate::text::tests::entry(&bytes, "META-INF/manifest.xml") + .contains(r#"manifest:full-path="/" manifest:version="1.3" manifest:media-type="application/vnd.oasis.opendocument.text-template""#) + ); + let back = crate::text::tests::import_ext("ott", &bytes).unwrap().unwrap(); + assert_eq!(back.body[0].as_para().map(|p| p.text.as_str()), Some("Template text.")); + // A document is still a document. + assert_eq!(crate::text::tests::entry(&crate::text::tests::export_ext("odt", &d).unwrap().unwrap(), "mimetype"), crate::text::TEXT_MIME); +} + +#[test] +fn a_template_of_styles_only_opens_empty_with_its_styles() { + // LibreOffice's style templates hold `styles.xml` and no `content.xml`. + use std::io::Write; + let styles = format!( + r#""#, + crate::text::tests::NS_T + ); + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + for (name, data) in [("mimetype", crate::text::TEMPLATE_MIME), ("styles.xml", styles.as_str())] { + z.start_file(name, o).unwrap(); + z.write_all(data.as_bytes()).unwrap(); + } + let d = crate::text::tests::import_ext("ott", &z.finish().unwrap().into_inner()).unwrap().unwrap(); + assert_eq!(d.body.len(), 1); + assert_eq!(d.styles.default_chr.font.as_deref(), Some("DejaVu Serif")); + // Without styles either, it isn't a document. + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + z.start_file("mimetype", o).unwrap(); + z.write_all(crate::text::TEXT_MIME.as_bytes()).unwrap(); + assert!(crate::text::tests::import_ext("odt", &z.finish().unwrap().into_inner()).unwrap().is_err()); +} diff --git a/crates/odf/src/text/tests_more/floats.rs b/crates/odf/src/text/tests_more/floats.rs new file mode 100644 index 000000000..f8ac5fe99 --- /dev/null +++ b/crates/odf/src/text/tests_more/floats.rs @@ -0,0 +1,458 @@ +//! Floating pictures (ODF 1.3 §9.2 frames anchored to a paragraph, character or page, §20.3xx +//! `style:wrap`, `style:horizontal-pos/-rel`, `style:vertical-pos/-rel`): placed and wrapped as +//! the file says, and written back the same way. + +use crate::text::tests::{NS_T, zip_of}; +use crate::text::{export, import}; +use craft_wordmodel::Document; +use craft_wordmodel::para::{Anchor, Float, FloatAlign, InlineObject, Wrap, WrapText}; + +const NS_D: &str = r#"xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:xlink="http://www.w3.org/1999/xlink""#; + +fn open(graphic: &str, frame: &str) -> Document { + let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7u8; 64]].concat(); + let content = format!( + r#"Text beside."# + ); + crate::text::import(&zip_of(&[("content.xml", content.as_bytes()), ("Pictures/a.png", &png)])).unwrap() +} + +fn float(d: &Document) -> Float { + d.body[0] + .as_para() + .unwrap() + .objects + .iter() + .find_map(|o| if let InlineObject::Image { float, .. } = o { Some(float.clone()) } else { None }) + .expect("a picture") +} + +#[test] +fn a_paragraph_anchored_frame_floats_with_text_around_it() { + let f = float(&open( + r#"style:wrap="parallel" style:horizontal-pos="right" style:horizontal-rel="paragraph" style:vertical-pos="top" style:vertical-rel="paragraph" fo:margin-left="0.1in" fo:margin-right="0.1in" fo:margin-top="0.05in" fo:margin-bottom="0.05in""#, + r#"text:anchor-type="paragraph""#, + )); + assert_eq!((f.wrap, f.wrap_text), (Wrap::Square, WrapText::BothSides)); + assert_eq!((f.h_rel, f.h_align), (Anchor::Paragraph, Some(FloatAlign::End))); + assert_eq!((f.v_rel, f.v_align), (Anchor::Paragraph, Some(FloatAlign::Start))); + assert!((f.dist - 7.2).abs() < 0.01 && (f.dist_top - 3.6).abs() < 0.01, "{f:?}"); +} + +#[test] +fn wrap_modes_and_offsets() { + let f = float(&open( + r#"style:wrap="none" style:horizontal-pos="from-left" style:vertical-pos="from-top""#, + r#"text:anchor-type="paragraph" svg:x="1in" svg:y="0.5in""#, + )); + assert_eq!(f.wrap, Wrap::TopAndBottom); + assert_eq!((f.h_align, f.x, f.v_align, f.y), (None, 72.0, None, 36.0)); + let f = float(&open(r#"style:wrap="run-through" style:run-through="background""#, r#"text:anchor-type="page""#)); + assert_eq!((f.wrap, f.h_rel, f.v_rel), (Wrap::BehindText, Anchor::Page, Anchor::Page)); + assert_eq!(float(&open(r#"style:wrap="run-through""#, r#"text:anchor-type="char""#)).wrap, Wrap::InFrontOfText); + assert_eq!(float(&open(r#"style:wrap="left""#, r#"text:anchor-type="paragraph""#)).wrap_text, WrapText::Left); + assert_eq!(float(&open(r#"style:wrap="dynamic""#, r#"text:anchor-type="paragraph""#)).wrap_text, WrapText::Largest); + // In line with the text. + assert_eq!(float(&open("", r#"text:anchor-type="as-char""#)).wrap, Wrap::Inline); +} + +#[test] +fn an_inline_frame_placed_against_the_text_is_raised_from_its_size() { + // LibreOffice's style templates: a logo centred on the text (`middle` relative to `text`) + // stands with its middle at the text's (measured: half the text's ascent less its descent + // above the baseline), in the paragraph's 12 pt text. + let pic = { + let img = image::RgbaImage::from_fn(4, 4, |_, _| image::Rgba([10, 20, 30, 255])); + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap(); + png + }; + let raise = |vpos: &str, rel: &str| { + let content = format!( + r#"Title"#, + NS = NS_T + ); + let doc = import(&zip_of(&[("content.xml", content.as_bytes()), ("Pictures/p.png", &pic)])).unwrap(); + let p = doc.body.iter().find_map(|b| b.as_para()).unwrap(); + let (off, _) = p.objects_with_offsets().next().unwrap(); + p.props_at(off + craft_wordmodel::para::OBJ.len_utf8()).position.map_or(0.0, |v| (v * 10.0).round() / 10.0) + }; + // (0.891 - 0.216) / 2 × 12 − 36. + assert_eq!(raise("middle", "text"), -32.0); + assert_eq!(raise("middle", "char"), -32.0); + assert_eq!(raise("top", "text"), -61.3); + assert_eq!(raise("bottom", "text"), -2.6); + // Against the line: on the baseline. + assert_eq!(raise("middle", "line"), 0.0); +} + +#[test] +fn an_inline_picture_is_saved_standing_on_the_baseline() { + // LibreOffice puts an in-line frame without placement at the line's top (measured): a + // picture standing on the baseline, as in Word, says so. + let d = float_doc_inline(); + let out = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(out.contains(r#"style:vertical-pos="top" style:vertical-rel="baseline""#), "{out}"); +} + +/// A document with one picture in line with its text. +fn float_doc_inline() -> Document { + open("", r#"text:anchor-type="as-char""#) +} + +#[test] +fn a_picture_alone_in_its_paragraph_keeps_its_raise_and_the_paragraphs_size() { + // LibreOffice's elegant agenda: a rule raised 0.2 in above the baseline in a 9 pt paragraph. + // Its parent style's `vertical-rel="paragraph"` is no in-line frame's (the standard doesn't + // define it): it stands on the baseline. Placed against the baseline, it is raised. + let pic = { + let img = image::RgbaImage::from_fn(4, 4, |_, _| image::Rgba([10, 20, 30, 255])); + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap(); + png + }; + let content = format!( + r#""#, + NS = crate::text::tests::NS_T + ); + let styles = format!( + r#""#, + NS = crate::text::tests::NS_T + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", styles.as_bytes()), ("Pictures/p.png", &pic)]); + let look = |doc: &Document| { + let p = doc.body.iter().find_map(|b| b.as_para()).unwrap(); + let (off, _) = p.objects_with_offsets().next().unwrap(); + let props = p.props_at(off + craft_wordmodel::para::OBJ.len_utf8()); + (props.position.map(|v| (v * 10.0).round() / 10.0), props.size, p.mark.size) + }; + let doc = crate::text::import(&pkg).unwrap(); + assert_eq!(look(&doc), (None, Some(9.0), Some(9.0))); + let baseline = styles.replace(r#"style:vertical-rel="paragraph""#, r#"style:vertical-rel="baseline""#); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("styles.xml", baseline.as_bytes()), ("Pictures/p.png", &pic)]); + let doc = crate::text::import(&pkg).unwrap(); + assert_eq!(look(&doc), (Some(10.8), Some(9.0), Some(9.0))); + // Saved, it says so in terms every reader takes the same way. + let again = crate::text::import(&crate::text::tests::export_model(&doc).unwrap()).unwrap(); + assert_eq!(look(&again), (Some(10.8), Some(9.0), Some(9.0))); +} + +#[test] +fn a_page_anchored_frames_paragraph_relations_are_its_text_area() { + // LibreOffice's letter templates: a page-anchored frame "from the paragraph" or "from the + // paragraph's content" (combinations the standard doesn't define) is placed from the page's + // text area. + let f = float(&open( + r#"style:wrap="run-through" style:horizontal-pos="from-left" style:horizontal-rel="paragraph" style:vertical-pos="from-top" style:vertical-rel="paragraph""#, + r#"text:anchor-type="page" svg:x="2cm" svg:y="2cm""#, + )); + assert_eq!((f.h_rel, f.v_rel), (Anchor::Margin, Anchor::Margin)); + let f = float(&open( + r#"style:wrap="run-through" style:horizontal-pos="from-left" style:horizontal-rel="paragraph-content" style:vertical-rel="char""#, + r#"text:anchor-type="page" svg:x="2cm""#, + )); + assert_eq!((f.h_rel, f.v_rel), (Anchor::Margin, Anchor::Margin)); + // Anchored to a paragraph, they stay the paragraph's. + let f = float(&open(r#"style:horizontal-rel="paragraph-content" style:vertical-rel="char""#, r#"text:anchor-type="char""#)); + assert_eq!((f.h_rel, f.v_rel), (Anchor::Column, Anchor::Line)); +} + +#[test] +fn a_frames_style_gives_the_anchor_and_place_it_leaves_unsaid() { + // LibreOffice's business card: the Labels style holds the card in line, Graphics puts the + // credit at the top of the page; neither frame says so itself. + let f = float(&open(r#"text:anchor-type="page" svg:x="1in" svg:y="0.5in" style:wrap="run-through""#, "")); + assert_eq!((f.h_rel, f.v_rel, f.x, f.y), (Anchor::Page, Anchor::Page, 72.0, 36.0)); + assert_eq!(float(&open(r#"text:anchor-type="as-char""#, "")).wrap, Wrap::Inline); + // The frame's own anchor and place win. + let f = float(&open(r#"text:anchor-type="page" svg:x="1in""#, r#"text:anchor-type="paragraph" svg:x="2in""#)); + assert_eq!((f.h_rel, f.x), (Anchor::Paragraph, 144.0)); +} + +#[test] +fn floating_pictures_round_trip() { + for (graphic, frame) in [ + ( + r#"style:wrap="parallel" style:horizontal-pos="right" style:horizontal-rel="paragraph" style:vertical-pos="top" style:vertical-rel="paragraph" fo:margin-left="0.1in" fo:margin-right="0.1in""#, + r#"text:anchor-type="paragraph""#, + ), + ( + r#"style:wrap="none" style:horizontal-pos="from-left" style:horizontal-rel="page" style:vertical-pos="from-top" style:vertical-rel="page""#, + r#"text:anchor-type="page" svg:x="1in" svg:y="2in""#, + ), + ( + r#"style:wrap="run-through" style:run-through="background" style:horizontal-pos="center" style:horizontal-rel="page-content""#, + r#"text:anchor-type="paragraph""#, + ), + ("", r#"text:anchor-type="as-char""#), + ] { + let d = open(graphic, frame); + let back = import(&export(&d).unwrap()).unwrap(); + let (a, b) = (float(&d), float(&back)); + assert_eq!( + (a.wrap, a.wrap_text, a.h_rel, a.v_rel, a.h_align, a.v_align), + (b.wrap, b.wrap_text, b.h_rel, b.v_rel, b.h_align, b.v_align), + "{graphic}" + ); + assert!((a.x - b.x).abs() < 0.01 && (a.y - b.y).abs() < 0.01 && (a.dist - b.dist).abs() < 0.01, "{graphic}: {a:?} {b:?}"); + } +} + +#[test] +fn hostile_frame_positions_are_clamped() { + let f = float(&open(r#"style:wrap="parallel" fo:margin-left="1e30in""#, r#"text:anchor-type="paragraph" svg:x="-1e30in" svg:y="NaNin""#)); + assert!(f.x.is_finite() && f.x.abs() <= 1584.0 && f.y.is_finite() && f.dist.is_finite() && f.dist <= 1584.0, "{f:?}"); +} + +#[test] +fn without_a_wrap_text_runs_on_both_sides() { + // ODF 1.3 §20.400 gives no default; LibreOffice wraps text on both sides of a frame, shape or + // group whose styles state none. + let f = float(&open(r#"style:horizontal-pos="from-left""#, r#"text:anchor-type="paragraph" svg:x="1in""#)); + assert_eq!((f.wrap, f.wrap_text), (Wrap::Square, WrapText::BothSides)); + let f = float(&open(r#"style:wrap="biggest""#, r#"text:anchor-type="paragraph""#)); + assert_eq!((f.wrap, f.wrap_text), (Wrap::Square, WrapText::Largest)); +} + +#[test] +fn an_inline_frame_keeps_its_place_on_the_baseline() { + // A frame in line with the text, hanging from the baseline (`from-top` 0, as LibreOffice + // places a shape added by its API), centred on it or hanging from it (`bottom`): lowered by + // its run's position, as Word lowers a picture, drawn so and written back. + let pic = { + let img = image::RgbaImage::from_fn(4, 4, |_, _| image::Rgba([10, 20, 30, 255])); + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap(); + png + }; + for (vpos, y, want) in [ + ("from-top", r#" svg:y="0in""#, -72.0), + ("from-top", r#" svg:y="-0.5in""#, -36.0), + ("middle", "", -36.0), + ("bottom", "", -72.0), + ("top", "", 0.0), + ] { + let content = format!( + r#"Line text"#, + NS = crate::text::tests::NS_T + ); + let pkg = crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("Pictures/p.png", &pic)]); + let doc = crate::text::import(&pkg).unwrap(); + let p = doc.body.iter().find_map(|b| b.as_para()).unwrap(); + let (off, _) = p.objects_with_offsets().next().unwrap(); + let pos = p.props_at(off + craft_wordmodel::para::OBJ.len_utf8()).position.unwrap_or(0.0); + assert_eq!((pos * 10.0).round() / 10.0, want, "{vpos}{y}"); + // Saved: a lowered one says how far its top is from the baseline. + let saved = crate::text::tests::content_of(&crate::text::tests::export_model(&doc).unwrap()); + if want != 0.0 { + assert!(saved.contains(r#"style:vertical-pos="from-top" style:vertical-rel="baseline""#), "{saved}"); + let top = -(72.0 + want) + 0.0; + assert!(saved.contains(&format!(r#"svg:y="{}pt""#, top)), "{vpos}{y}: {saved}"); + } + let again = crate::text::import(&crate::text::tests::export_model(&doc).unwrap()).unwrap(); + let p2 = again.body.iter().find_map(|b| b.as_para()).unwrap(); + let (off2, _) = p2.objects_with_offsets().next().unwrap(); + assert_eq!( + p2.props_at(off2 + craft_wordmodel::para::OBJ.len_utf8()).position.map(|v| (v * 10.0).round() / 10.0), + (want != 0.0).then_some(want) + ); + } +} + +#[test] +fn a_cropped_picture_keeps_its_crop() { + // LibreOffice's crop (`fo:clip`: top, right, bottom, left, in lengths of the picture's own + // size — 200 × 100 px is 150 × 75 pt): a quarter off its left and its bottom. + let png = { + let img = image::RgbaImage::from_fn(200, 100, |x, _| image::Rgba([if x < 100 { 200 } else { 20 }, 80, 160, 255])); + let mut b = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut b), image::ImageFormat::Png).unwrap(); + b + }; + let content = format!( + r#""#, + NS = crate::text::tests::NS_T + ); + let doc = crate::text::import(&crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("Pictures/q.png", &png)])).unwrap(); + let crop = doc + .body + .iter() + .find_map(|b| b.as_para()) + .and_then(|p| p.objects.iter().find_map(|o| if let craft_wordmodel::para::InlineObject::Image { crop, .. } = o { Some(*crop) } else { None })) + .unwrap(); + assert_eq!(crop.map(|c| (c * 100.0).round() / 100.0), [0.25, 0.0, 0.0, 0.25]); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&doc).unwrap()); + // 0.2602 in is 18.73 pt, 0.5209 in 37.5 pt. + assert!(c.contains(r#"fo:clip="rect(0pt, 0pt, 18.73pt, 37.5pt)""#), "{c}"); + // An absurd crop (nothing left) is none. + assert_eq!(crate::text::body::drawing::crop_fractions([0.0, 100.0, 0.0, 100.0], &png), [0.0; 4]); +} + +#[test] +fn contour_wrapping_and_its_outline_are_kept() { + // `style:wrap-contour` with `outside` (Word's Tight) or `full` (Through), and the outline + // (`draw:contour-polygon`) in the frame's view box, as WordCraft's wrap polygon. + let png = { + let img = image::RgbaImage::from_fn(10, 10, |_, _| image::Rgba([0, 0, 0, 255])); + let mut b = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut b), image::ImageFormat::Png).unwrap(); + b + }; + for (mode, want) in [("outside", craft_wordmodel::para::Wrap::Tight), ("full", craft_wordmodel::para::Wrap::Through)] { + let content = format!( + r#"Text around it."#, + NS = crate::text::tests::NS_T + ); + let doc = crate::text::import(&crate::text::tests::zip_of(&[("content.xml", content.as_bytes()), ("Pictures/p.png", &png)])).unwrap(); + let float = |d: &Document| { + d.body.iter().find_map(|b| b.as_para()).and_then(|p| p.objects.iter().find_map(|o| o.frame().map(|(_, _, f)| f.clone()))).unwrap() + }; + let f = float(&doc); + assert_eq!(f.wrap, want, "{mode}"); + assert_eq!(f.wrap_polygon.as_deref().map(|p| p.points.clone()), Some(vec![[10800, 0], [21600, 21600], [0, 21600]]), "{mode}"); + let saved = crate::text::tests::export_model(&doc).unwrap(); + let c = crate::text::tests::content_of(&saved); + assert!(c.contains(&format!(r#"style:wrap-contour="true" style:wrap-contour-mode="{mode}""#)), "{c}"); + assert!(c.contains(r#"draw:points="10800,0 21600,21600 0,21600""#), "{c}"); + let back = float(&crate::text::import(&saved).unwrap()); + assert_eq!((back.wrap, back.wrap_polygon), (f.wrap, f.wrap_polygon)); + } +} + +#[test] +fn a_contour_outside_its_view_box_is_worked_out_instead() { + // Points far past the view box: not the frame's coordinates, so no polygon (the outline is + // derived from the object), though the wrap stays a contour one. + let a: crate::text::xml::Attrs = vec![("svg:viewBox".into(), "0 0 100 100".into()), ("draw:points".into(), "0,0 100,150 0,100".into())]; + assert!(crate::text::frames::contour_polygon(&a).is_none()); + let a: crate::text::xml::Attrs = vec![("svg:viewBox".into(), "0 0 100 100".into()), ("draw:points".into(), "0,0 100,100 0,100".into())]; + assert!(crate::text::frames::contour_polygon(&a).is_some()); +} + +#[test] +fn a_drawing_outside_any_paragraph_belongs_to_the_next_one() { + // LibreOffice puts page-anchored frames before the first paragraph. They join it rather + // than making a paragraph of their own, which would start the document on the default page + // style instead of the one the first paragraph asks for. + let ns = crate::text::tests::NS_T; + let styles = format!( + r#"Running head"# + ); + let content = format!( + r#"AddressDear reader,Body."# + ); + let d = import(&crate::text::tests::package(&content, Some(&styles))).unwrap(); + let paras: Vec<&craft_wordmodel::Paragraph> = d.body.iter().filter_map(|b| b.as_para()).collect(); + assert_eq!(paras.len(), 2, "no paragraph of the frame's own"); + assert_eq!(paras[0].text.replace(craft_wordmodel::para::OBJ, ""), "Dear reader,"); + assert_eq!(paras[0].objects.len(), 1); + // One section, starting on the first page style: a different first page without the head. + assert!(d.last_section.title_page && d.last_section.headers.default.is_some()); + // Drawings with no paragraph after them still get one. + let tail = content.replace(r#"Dear reader,Body."#, ""); + let d = import(&crate::text::tests::package(&tail, Some(&styles))).unwrap(); + assert_eq!(d.body.iter().filter_map(|b| b.as_para()).map(|p| p.objects.len()).sum::(), 1); +} + +#[test] +fn a_frame_keeps_its_place_in_the_stacking_order() { + // §19.231 `draw:z-index`: 0 at the bottom; WordCraft's z counts from 1 (0 is unknown). + let d = open(r#"style:wrap="run-through""#, r#"text:anchor-type="paragraph" draw:z-index="3""#); + assert_eq!(float(&d).z, 4); + let saved = crate::text::tests::export_model(&d).unwrap(); + assert!(crate::text::tests::entry(&saved, "content.xml").contains(r#"draw:z-index="3""#)); + assert_eq!(float(&import(&saved).unwrap()).z, 4); + // Unknown (inserted in WordCraft): at the bottom, in document order. + let d = open(r#"style:wrap="run-through""#, r#"text:anchor-type="paragraph""#); + assert_eq!(float(&d).z, 0); + assert!(crate::text::tests::entry(&crate::text::tests::export_model(&d).unwrap(), "content.xml").contains(r#"draw:z-index="0""#)); +} + +#[test] +fn a_frame_anchored_to_a_page_stays_on_that_page() { + // LibreOffice's page-anchored frames name their page. Written without it, a frame on page 3 + // came back on page 1 in LibreOffice. + let frame = |attrs: &str| { + let content = format!( + r#"On itOne"# + ); + import(&crate::text::tests::package(&content, None)).unwrap() + }; + let d = frame(r#"text:anchor-type="page" text:anchor-page-number="3""#); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#"text:anchor-type="page" text:anchor-page-number="3""#), "{c}"); + let back = + crate::text::tests::content_of(&crate::text::tests::export_model(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()).unwrap()); + assert!(back.contains(r#"text:anchor-page-number="3""#), "again: {back}"); + // A frame anchored to its paragraph names no page, and junk is none. + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&frame(r#"text:anchor-type="paragraph""#)).unwrap()); + assert!(!c.contains("anchor-page-number"), "{c}"); + for junk in ["-1", "abc", "", "99999999999999"] { + let d = frame(&format!(r#"text:anchor-type="page" text:anchor-page-number="{junk}""#)); + let c = crate::text::tests::content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(!c.contains("anchor-page-number"), "{junk}: {c}"); + } +} + +/// A page number only says something on a page anchor. On another LibreOffice ignores it, so it +/// doesn't make a drawing anchored to a page when saved. +#[test] +fn a_page_number_on_another_anchor_is_no_page_anchor() { + let saved = |styles: &str, body: &str| { + let content = format!( + r#"{styles}{body}"# + ); + crate::text::tests::content_of(&crate::text::tests::export_model(&import(&crate::text::tests::package(&content, None)).unwrap()).unwrap()) + }; + let frame = |attrs: &str| { + format!( + r#"OneOn it"# + ) + }; + for anchor in ["paragraph", "char"] { + let c = saved("", &frame(&format!(r#"text:anchor-type="{anchor}" text:anchor-page-number="3""#))); + assert!(!c.contains(r#"text:anchor-type="page""#) && !c.contains("anchor-page-number"), "{anchor}: {c}"); + } + let c = saved( + "", + r#"One"#, + ); + assert!(!c.contains(r#"text:anchor-type="page""#) && !c.contains("anchor-page-number"), "a line: {c}"); + // An anchor its style gives counts as one the frame states. + let by_style = r#""#; + let c = saved(by_style, &frame(r#"draw:style-name="fr" text:anchor-page-number="3""#)); + assert!(c.contains(r#"text:anchor-type="page" text:anchor-page-number="3""#), "{c}"); + let by_style = r#""#; + let c = saved(by_style, &frame(r#"draw:style-name="fr" text:anchor-page-number="3""#)); + assert!(!c.contains("anchor-page-number"), "{c}"); +} + +#[test] +fn a_frame_offering_several_formats_shows_the_first_it_can_read() { + // LibreOffice's letter templates: its own metafile, then a PNG. Taking the first left nothing + // to show, and the picture was gone once the paragraph was written from the model. + let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7u8; 64]].concat(); + let other = [b"\x89PNG\r\n\x1a\n".as_slice(), &[9u8; 64]].concat(); + let svm = b"VCLMTF a metafile WordCraft can't draw".to_vec(); + let frame = |images: &[&str]| { + let images: String = images.iter().map(|h| format!(r#""#)).collect(); + format!( + r#"{images}"# + ) + }; + let pictures = |d: &Document| d.body[0].as_para().unwrap().objects.iter().filter(|o| matches!(o, InlineObject::Image { .. })).count(); + let both = frame(&["Pictures/a.svm", "Pictures/a.png"]); + let d = import(&zip_of(&[("content.xml", both.as_bytes()), ("Pictures/a.svm", &svm), ("Pictures/a.png", &png)])).unwrap(); + assert_eq!(pictures(&d), 1, "the PNG"); + let back = import(&crate::text::tests::export_model(&d).unwrap()).unwrap(); + assert_eq!(pictures(&back), 1, "written from the model"); + // The first it can read, not the last. + let order = frame(&["Pictures/a.png", "Pictures/b.png"]); + let d = import(&zip_of(&[("content.xml", order.as_bytes()), ("Pictures/a.png", &png), ("Pictures/b.png", &other)])).unwrap(); + assert_eq!(pictures(&d), 1); + assert_eq!(d.media.values().next().and_then(|m| m.get(8).copied()), Some(7), "the first"); + // Nothing it can read: no picture, and no failure. + let only = frame(&["Pictures/a.svm"]); + let d = import(&zip_of(&[("content.xml", only.as_bytes()), ("Pictures/a.svm", &svm)])).unwrap(); + assert_eq!(pictures(&d), 0); +} diff --git a/crates/odf/src/text/tests_more/flow_breaks.rs b/crates/odf/src/text/tests_more/flow_breaks.rs new file mode 100644 index 000000000..04282c795 --- /dev/null +++ b/crates/odf/src/text/tests_more/flow_breaks.rs @@ -0,0 +1,470 @@ +//! How text flows across pages and tables sit on the page: a page break after a paragraph +//! (`fo:break-after`), lines kept together (`fo:keep-together`), widow and orphan control +//! (`fo:widows`, `fo:orphans`), hyphenation (`fo:hyphenate`), a table's alignment and width +//! (`table:align`, `style:rel-width`) and its rows' heights (`style:row-height`, +//! `style:min-row-height`), read into WordCraft's document and written back. + +use crate::text::tests::{NS_T, content_of, package}; +use crate::text::{export, import}; +use craft_wordmodel::Align; +use craft_wordmodel::props::HeightRule; + +fn doc(styles: &str, body: &str) -> craft_wordmodel::Document { + let content = format!( + r#"{styles}{body}"# + ); + import(&package(&content, None)).unwrap() +} + +fn para_style(name: &str, para: &str, text: &str) -> String { + format!( + r#""# + ) +} + +fn paras(d: &craft_wordmodel::Document) -> Vec<&craft_wordmodel::props::ParaProps> { + d.body.iter().filter_map(|b| b.as_para()).map(|p| &p.props).collect() +} + +#[test] +fn a_page_break_after_starts_the_next_paragraph_on_a_page() { + let d = doc( + ¶_style("P1", r#"fo:break-after="page""#, ""), + r#"One.Two.Three."#, + ); + let breaks: Vec = paras(&d).iter().map(|p| p.page_break_before == Some(true)).collect(); + assert_eq!(breaks, [false, true, false]); + // Written back as a page break before the second paragraph. + let c = content_of(&export(&d).unwrap()); + assert_eq!(c.matches(r#"fo:break-before="page""#).count(), 1, "{c}"); +} + +#[test] +fn kept_lines_and_widow_control_are_read_and_written() { + let styles = format!( + "{}{}", + para_style("P1", r#"fo:keep-together="always" fo:widows="3" fo:orphans="3""#, ""), + para_style("P2", r#"fo:widows="0" fo:orphans="0""#, "") + ); + let d = doc(&styles, r#"Kept.Loose."#); + let p = paras(&d); + assert_eq!((p[0].keep_lines, p[0].widow_control), (Some(true), None)); + assert_eq!((p[1].keep_lines, p[1].widow_control), (None, Some(false))); + let c = content_of(&export(&d).unwrap()); + assert!(c.contains(r#"fo:keep-together="always""#) && c.contains(r#"fo:widows="0" fo:orphans="0""#), "{c}"); +} + +#[test] +fn a_hyphenated_paragraph_makes_the_document_hyphenate() { + // Hyphenation is a paragraph's in ODF, the document's in WordCraft: the document hyphenates + // and the other paragraphs opt out. + let d = doc(¶_style("P1", "", r#"fo:hyphenate="true""#), r#"Hyphenated.Not."#); + assert!(d.settings.auto_hyphenation); + let p = paras(&d); + assert_eq!((p[0].suppress_hyphens, p[1].suppress_hyphens), (None, Some(true))); + let c = content_of(&export(&d).unwrap()); + assert_eq!(c.matches(r#"fo:hyphenate="true""#).count(), 1, "{c}"); + // No hyphenated paragraph: nothing changes. + let plain = doc("", "Plain."); + assert!(!plain.settings.auto_hyphenation && paras(&plain)[0].suppress_hyphens.is_none()); +} + +#[test] +fn a_tables_alignment_width_and_row_heights_are_kept() { + let styles = r#""#; + let body = r#"abcdef"#; + let d = doc(styles, body); + let t = d.body.iter().find_map(|b| b.as_table()).expect("a table"); + assert_eq!((t.props.align, t.props.width_pct), (Some(Align::Center), Some(50.0))); + let heights: Vec<(Option, HeightRule)> = t.rows.iter().map(|r| (r.props.height, r.props.height_rule)).collect(); + assert_eq!(heights, [(Some(36.0), HeightRule::Exact), (Some(18.0), HeightRule::AtLeast), (None, HeightRule::Auto)]); + let c = content_of(&crate::text::tests::export_model(&d).unwrap()); + for want in [r#"style:rel-width="50%" table:align="center""#, r#"style:row-height="36pt""#, r#"style:min-row-height="18pt""#] { + assert!(c.contains(want), "{want} in {c}"); + } + // A table filling the margins stays so: the text width. + let plain = doc("", r#"x"#); + let pt = plain.body.iter().find_map(|b| b.as_table()).unwrap(); + assert_eq!((pt.props.align, pt.props.width_pct), (None, Some(100.0))); + assert!(content_of(&crate::text::tests::export_model(&plain).unwrap()).contains(r#"table:align="margins""#)); +} + +#[test] +fn columns_are_written_with_their_share_of_the_table() { + // A table filling the margins of a page narrower than it was made for keeps its columns' + // proportions only when they are given as shares too (LibreOffice scales by them). + let styles = r#""#; + let d = doc( + styles, + r#"ab"#, + ); + let c = content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#"style:column-width="72pt" style:rel-column-width="7200*""#) && c.contains(r#"style:rel-column-width="21600*""#), "{c}"); + // An aligned table keeps its widths: LibreOffice lays one out oddly with shares. + let left = r#""#; + let d = doc( + &format!("{styles}{left}"), + r#"ab"#, + ); + let c = content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(c.contains(r#"table:align="left""#) && !c.contains("rel-column-width"), "{c}"); +} + +#[test] +fn a_tables_columns_keep_their_widths() { + // LibreOffice never fits a table's columns to their content (Word's fixed layout); a table + // filling the margins is fitted to them, its columns in their proportions. + let styles = r#""#; + let d = doc( + styles, + r#"aa much longer text in the second column"#, + ); + let t = d.body.iter().find_map(|b| b.as_table()).expect("a table"); + assert!(t.props.fixed); + assert_eq!(t.grid, [288.0, 288.0]); + assert_eq!(t.props.width_pct, Some(100.0)); +} + +#[test] +fn a_tables_own_space_above_and_below_is_kept() { + // LibreOffice's report templates: 0.3 cm below each table (`fo:margin-bottom` on the table). + let styles = r#""#; + let d = doc( + styles, + r#"CellAfter."#, + ); + let space = |d: &craft_wordmodel::Document| { + let t = d.body.iter().find_map(|b| b.as_table()).expect("a table"); + (t.props.space_before.map(|v| (v * 100.0).round() / 100.0), t.props.space_after.map(|v| (v * 100.0).round() / 100.0)) + }; + assert_eq!(space(&d), (Some(7.2), Some(8.5))); + let out = content_of(&crate::text::tests::export_model(&d).unwrap()); + assert!(out.contains(r#"fo:margin-top="7.2pt" fo:margin-bottom="8.5"#), "{out}"); + assert_eq!(space(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()), (Some(7.2), Some(8.5))); +} + +#[test] +fn a_table_starting_a_page_starts_it() { + // LibreOffice's agenda: the minutes table starts a new page (`fo:break-before` on the table). + let styles = r#""#; + let d = doc( + styles, + r#"Before.First cellAfter."#, + ); + let t = d.body.iter().find_map(|b| b.as_table()).expect("a table"); + let first = t.rows[0].cells[0].blocks[0].as_para().unwrap(); + assert_eq!(first.props.page_break_before, Some(true)); + // Nothing else starts a page. + assert!(paras(&d).iter().all(|p| p.page_break_before != Some(true))); + let c = content_of(&crate::text::tests::export_model(&d).unwrap()); + assert_eq!(c.matches(r#"fo:break-before="page""#).count(), 2, "{c}"); +} + +#[test] +fn columns_stating_only_their_shares_keep_their_proportions() { + // LibreOffice's CV template: a table filling the margins whose columns give only + // `style:rel-column-width`. They were read as equal. + let column = |name: &str, share: &str| { + format!( + r#""# + ) + }; + let table = r#"ab"#; + let grid = |d: &craft_wordmodel::Document| d.body.iter().find_map(|b| b.as_table()).map(|t| t.grid.clone()).unwrap_or_default(); + let d = doc(&format!("{}{}", column("A", "1000*"), column("B", "3000*")), table); + let g = grid(&d); + assert_eq!(g.len(), 2, "{g:?}"); + assert!((g[1] / g[0] - 3.0).abs() < 0.01, "{g:?}"); + // Rewritten from the model, the proportions stay. + let back = import(&crate::text::tests::export_model(&d).unwrap()).unwrap(); + let g = grid(&back); + assert!((g[1] / g[0] - 3.0).abs() < 0.01, "{g:?}"); + // A column without a share leaves the equal split, and junk shares are none. + for junk in ["0*", "-1*", "NaN*", "1e39*", "abc"] { + let d = doc(&format!("{}{}", column("A", "1000*"), column("B", junk)), table); + let g = grid(&d); + assert_eq!(g.len(), 2, "{junk}: {g:?}"); + assert!((g[0] - g[1]).abs() < 0.01, "{junk}: {g:?}"); + } + // A table filling the margins with a width stated beside each share keeps the widths: files + // give them in proportion, and the table is fitted to the margins. + let stated = r#""#; + let g = grid(&doc(stated, table)); + assert!((g[0] / g[1] - 2.0).abs() < 0.01, "{g:?}"); +} + +/// A share-only table wider than the text keeps its stated width, as in LibreOffice; a column with +/// only a width takes it first, and the shares divide the rest; a column with no share of its own +/// takes the default style's. +#[test] +fn columns_by_shares_divide_the_right_width() { + let column = |name: &str, props: &str| { + format!(r#""#) + }; + let table_style = |props: &str| format!(r#""#); + let table = |names: &[&str]| { + let columns: String = names + .iter() + .map(|n| if n.is_empty() { "".to_string() } else { format!(r#""#) }) + .collect(); + let cells: String = names.iter().map(|_| "x").collect(); + format!(r#"{columns}{cells}"#) + }; + let grid = |d: &craft_wordmodel::Document| d.body.iter().find_map(|b| b.as_table()).map(|t| t.grid.clone()).unwrap_or_default(); + let near = |got: &[f32], want: &[f32]| { + assert!(got.len() == want.len() && got.iter().zip(want).all(|(a, b)| (a - b).abs() < 0.5), "{got:?}, not {want:?}") + }; + // 576 pt, aligned, in a narrower text area. + let d = doc( + &format!( + "{}{}{}", + table_style(r#"style:width="8in" table:align="left""#), + column("A", r#"style:rel-column-width="1000*""#), + column("B", r#"style:rel-column-width="3000*""#) + ), + &table(&["A", "B"]), + ); + near(&grid(&d), &[144.0, 432.0]); + near(&grid(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()), &[144.0, 432.0]); + // 72 pt stated, the other 300 by shares. + let d = doc( + &format!( + "{}{}{}{}", + table_style(r#"style:width="372pt" table:align="left""#), + column("A", r#"style:column-width="72pt""#), + column("B", r#"style:rel-column-width="1000*""#), + column("C", r#"style:rel-column-width="3000*""#) + ), + &table(&["A", "B", "C"]), + ); + near(&grid(&d), &[72.0, 75.0, 225.0]); + // A share on every column goes before a stated width, as in LibreOffice. + let d = doc( + &format!( + "{}{}{}", + table_style(r#"style:width="400pt" table:align="left""#), + column("A", r#"style:column-width="72pt" style:rel-column-width="1000*""#), + column("B", r#"style:rel-column-width="3000*""#) + ), + &table(&["A", "B"]), + ); + near(&grid(&d), &[100.0, 300.0]); + // An aligned table whose every column states both: the shares (measured in LibreOffice 26.8). + let d = doc( + &format!( + "{}{}{}", + table_style(r#"style:width="400pt" table:align="left""#), + column("A", r#"style:column-width="300pt" style:rel-column-width="1*""#), + column("B", r#"style:column-width="100pt" style:rel-column-width="1*""#) + ), + &table(&["A", "B"]), + ); + near(&grid(&d), &[200.0, 200.0]); + // A share too small for a column is the narrowest one, not the grid lost to the equal split. + let d = doc( + &format!( + "{}{}{}", + table_style(r#"style:width="400pt" table:align="left""#), + column("A", r#"style:rel-column-width="1*""#), + column("B", r#"style:rel-column-width="65535*""#) + ), + &table(&["A", "B"]), + ); + near(&grid(&d), &[6.0, 400.0]); + // The default style's share, for a column with none and one whose style says none. + let styles = r#""#; + let content = format!( + r#"{}{}{}{}"#, + table_style(r#"style:width="400pt" table:align="left""#), + column("B", r#"style:rel-column-width="3000*""#), + column("E", r#"style:use-optimal-column-width="false""#), + table(&["", "B"]) + ); + near(&grid(&import(&package(&content, Some(styles))).unwrap()), &[100.0, 300.0]); +} + +/// A table inside a cell doesn't know its width from the page: with only shares and no width of +/// its own to divide, or with some columns stating widths, its columns split it equally, as before. +#[test] +fn a_table_in_a_cell_shares_out_no_page_width() { + let styles = r#""#; + for (style, columns) in [ + (r#" table:style-name="L""#, r#""#), + ("", r#""#), + ] { + let inner = format!( + r#"{columns}ab"# + ); + let body = format!( + r#"{inner}c"# + ); + let d = doc(styles, &body); + let outer = d.body.iter().find_map(|b| b.as_table()).unwrap(); + let inner = outer.rows[0].cells[0].blocks.iter().find_map(|b| b.as_table()).unwrap(); + // The equal split main gives, which layout fits to the cell (not two page-wide columns). + let equal = inner.grid.windows(2).all(|w| (w[0] - w[1]).abs() < 0.01); + assert!(equal && inner.grid.first().is_some_and(|w| *w < 300.0), "{style}: {:?}", inner.grid); + } +} + +#[test] +fn a_tables_and_rows_background_show_behind_their_cells() { + // Table Properties › Background: it was dropped, on screen and in LibreOffice after a save. + let styles = r##""##; + let table = r#"tablecellrowcell"#; + let colours = |d: &craft_wordmodel::Document| { + d.body + .iter() + .find_map(|b| b.as_table()) + .map(|t| t.rows.iter().flat_map(|r| r.cells.iter().map(|c| c.props.shading.map(|s| s.hex().to_ascii_lowercase()))).collect::>()) + .unwrap_or_default() + }; + let want = |t: &str, r: &str, c: &str| vec![Some(t.to_string()), Some(c.to_string()), Some(r.to_string()), Some(c.to_string())]; + let d = doc(styles, table); + assert_eq!(colours(&d), want("ddffdd", "ffdddd", "ddddff")); + assert_eq!(colours(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()), want("ddffdd", "ffdddd", "ddddff")); +} + +/// As LibreOffice shows them: a cell's own style, else its column's default cell style (a row's +/// isn't applied), else the row's background, else the table's. A row or table clearing its +/// background (`transparent`) doesn't take its parent's; the table's shows behind that row. +#[test] +fn a_cells_background_comes_from_its_column_then_row_then_table() { + let cell = |n: &str, c: &str| { + format!( + r##""## + ) + }; + let styles = format!( + r##"{}{}"##, + cell("InRow", "ff0000"), + cell("InColumn", "00ff00"), + ); + let c = "x"; + let table = format!( + r#"{c}{c}{c}{c}{c}{c}"# + ); + let colours = |d: &craft_wordmodel::Document| { + d.body + .iter() + .find_map(|b| b.as_table()) + .map(|t| t.rows.iter().flat_map(|r| r.cells.iter().map(|c| c.props.shading.map(|s| s.hex().to_ascii_lowercase()))).collect::>()) + .unwrap_or_default() + }; + let s = |v: &str| Some(v.to_string()); + let want = vec![s("00ff00"), s("0000ff"), s("00ff00"), s("ffff00"), s("00ff00"), s("ffff00")]; + let d = doc(&styles, &table); + assert_eq!(colours(&d), want); + assert_eq!(colours(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()), want); +} + +/// A column default after a spanning cell belongs to the column the cell is in. +#[test] +fn a_column_default_counts_spanned_columns() { + let styles = + r##""##; + let table = r#"ab"#; + let d = doc(styles, table); + let t = d.body.iter().find_map(|b| b.as_table()).unwrap(); + let last = t.rows[0].cells.iter().rev().find(|c| !c.blocks.is_empty()).unwrap(); + assert_eq!(last.props.shading.map(|s| s.hex().to_ascii_lowercase()).as_deref(), Some("00ff00")); +} + +/// A column's default cell style that names nothing there is no background, however long its name. +#[test] +fn a_column_default_naming_no_style_is_no_background() { + let long = "x".repeat(100_000); + let table = format!( + r#"a"# + ); + let d = doc("", &table); + let t = d.body.iter().find_map(|b| b.as_table()).unwrap(); + assert_eq!(t.rows[0].cells[0].props.shading, None); +} + +/// A cell style saying `transparent` over a coloured parent has no background of its own: what's +/// behind it shows, in LibreOffice too, from a column's default cell style or the cell's own. +#[test] +fn a_transparent_cell_style_stops_its_parents_colour() { + let styles = r##""##; + for (column, cell) in [(r#" table:default-cell-style-name="KC""#, ""), ("", r#" table:style-name="KC""#)] { + let table = format!( + r#"a"# + ); + let d = doc(styles, &table); + let t = d.body.iter().find_map(|b| b.as_table()).unwrap(); + assert_eq!(t.rows[0].cells[0].props.shading, craft_flow::parse_color("#0000ff"), "{column}{cell}"); + } +} + +#[test] +fn a_row_that_keeps_together_says_so() { + // Table › Text Flow: a row not to break across pages. It came back from a save free to break. + let styles = r#""#; + let row = |s: &str, t: &str| format!(r#"{t}"#); + let table = format!( + "{}{}{}", + row(r#" table:style-name="R""#, "kept"), + row("", "free"), + row(r#" table:style-name="H""#, "kept and tall") + ); + let shape = |d: &craft_wordmodel::Document| { + d.body + .iter() + .find_map(|b| b.as_table()) + .map(|t| t.rows.iter().map(|r| (r.props.cant_split, r.props.height)).collect::>()) + .unwrap_or_default() + }; + let d = doc(styles, &table); + assert_eq!(shape(&d), [(true, None), (false, None), (true, Some(72.0))]); + let saved = crate::text::tests::export_model(&d).unwrap(); + assert!(content_of(&saved).contains(r#"fo:keep-together="always""#)); + assert_eq!(shape(&import(&saved).unwrap()), [(true, None), (false, None), (true, Some(72.0))]); +} + +/// A row whose style says nothing of it, or a row with no style, follows the default style; one +/// that says otherwise doesn't. +#[test] +fn a_rows_keep_together_defaults_to_the_default_style() { + let styles = r#""#; + let row = |s: &str| format!(r#"x"#); + let content = format!( + r#"{}{}{}"#, + row(""), + row(r#" table:style-name="H""#), + row(r#" table:style-name="F""#) + ); + let keeps = |d: &craft_wordmodel::Document| { + d.body.iter().find_map(|b| b.as_table()).map(|t| t.rows.iter().map(|r| r.props.cant_split).collect::>()).unwrap_or_default() + }; + let d = import(&package(&content, Some(styles))).unwrap(); + assert_eq!(keeps(&d), [true, true, false]); + assert_eq!(keeps(&import(&crate::text::tests::export_model(&d).unwrap()).unwrap()), [true, true, false]); +} + +/// A row free to break under a default row style that keeps rows together says so when written from +/// the model, or a save takes that away: the file's default style stays. +#[test] +fn a_row_free_to_break_stays_so_over_the_default_style() { + let styles = r#""#; + let row = |s: &str| format!(r#"x"#); + let content = format!( + r#"{}{}"#, + row(r#" table:style-name="F""#), + row("") + ); + let mut d = import(&package(&content, Some(styles))).unwrap(); + let keeps = |d: &craft_wordmodel::Document| { + d.body.iter().find_map(|b| b.as_table()).map(|t| t.rows.iter().map(|r| r.props.cant_split).collect::>()).unwrap_or_default() + }; + assert_eq!(keeps(&d), [false, true]); + if let Some(craft_wordmodel::Block::Table(t)) = d.body.get_mut(0).map(std::sync::Arc::make_mut) + && let Some(craft_wordmodel::Block::Para(p)) = t.rows[0].cells[0].blocks.get_mut(0).map(std::sync::Arc::make_mut) + { + p.text.push('!'); + if let Some(r) = p.runs.last_mut() { + r.len += 1; + } + } + assert_eq!(keeps(&import(&export(&d).unwrap()).unwrap()), [false, true], "after an edit and a save"); +} diff --git a/crates/odf/src/text/tests_more/fonts.rs b/crates/odf/src/text/tests_more/fonts.rs new file mode 100644 index 000000000..845c33bf5 --- /dev/null +++ b/crates/odf/src/text/tests_more/fonts.rs @@ -0,0 +1,75 @@ +//! Font names (ODF 1.3 §16.23 `style:font-face`): `style:font-name` names a font face declared in +//! `office:font-face-decls`, whose `svg:font-family` is the font. + +use crate::text::import; +use crate::text::tests::{NS_T, package}; + +const NS_SVG: &str = r#"xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0""#; + +fn fonts(decls: &str, props: &[&str], styles: Option<&str>) -> Vec> { + let auto: String = props + .iter() + .enumerate() + .map(|(i, p)| format!(r#""#)) + .collect(); + let body: String = (0..props.len()).map(|i| format!(r#"s{i}|"#)).collect(); + let content = format!( + r#"{decls}{auto}{body}"# + ); + let d = import(&package(&content, styles)).unwrap(); + let p = d.body[0].as_para().unwrap(); + p.run_ranges() + .filter(|(r, _)| p.text.get(r.clone()).is_some_and(|t| t.trim_start_matches('|').starts_with('s'))) + .map(|(_, c)| c.font.clone()) + .collect() +} + +#[test] +fn a_font_name_is_the_family_its_face_declares() { + let decls = r#""#; + let got = fonts(decls, &[r#"style:font-name="FreeSans1""#, r#"style:font-name="Serif""#, r#"style:font-name="Undeclared Font""#], None); + assert_eq!(got, [Some("FreeSans".into()), Some("Liberation Serif".into()), Some("Undeclared Font".into())]); +} + +#[test] +fn faces_declared_in_styles_xml_count_too() { + let styles = format!( + r#""# + ); + assert_eq!(fonts("", &[r#"style:font-name="F2""#], Some(&styles)), [Some("DejaVu Sans".into())]); +} + +#[test] +fn hostile_font_faces_do_not_panic() { + let many: String = (0..20_000).map(|i| format!(r#""#)).collect(); + let long = "n".repeat(10_000); + let decls = format!(r#"{many}"#); + let got = fonts(&decls, &[r#"style:font-name="L""#, r#"style:font-name="F5""#], None); + assert!(got[0].as_ref().is_none_or(|f| f.len() <= 256), "{:?}", got[0].as_ref().map(String::len)); +} + +#[test] +fn each_font_is_declared_with_its_kind() { + // §16.21 `style:font-face`: a reader without the font picks a like one by its generic family + // and pitch; each use names its declaration (`style:font-name`). + let mut d = craft_wordmodel::Document::new(); + let mono = craft_wordmodel::props::CharProps { font: Some("Courier New".into()), ..Default::default() }; + let serif = craft_wordmodel::props::CharProps { font: Some("Georgia".into()), ..Default::default() }; + let mut p = craft_wordmodel::Paragraph::with_text("code", mono); + let end = p.text.len(); + p.insert_text(end, " prose", &serif).unwrap(); + d.body = vec![craft_wordmodel::para_block(p)]; + let saved = crate::text::export(&d).unwrap(); + let content = crate::text::tests::content_of(&saved); + let styles = crate::text::tests::entry(&saved, "styles.xml"); + assert!(content.contains(r#""#), "{content}"); + assert!(content.contains(r#"style:font-family-generic="roman""#)); + assert!(content.contains(r#"style:font-name="Georgia""#)); + // Declared before the styles that use them, in each part. + assert!(content.find("") < content.find("")); + assert!(styles.contains("") && styles.find("") < styles.find("")); + // Read back the same. + let back = crate::text::import(&saved).unwrap(); + let fonts: Vec<_> = back.body[0].as_para().unwrap().runs.iter().map(|r| r.props.font.clone()).collect(); + assert_eq!(fonts, [Some("Courier New".to_string()), Some("Georgia".to_string())]); +} diff --git a/crates/odf/src/text/tests_more/format_changes.rs b/crates/odf/src/text/tests_more/format_changes.rs new file mode 100644 index 000000000..720770750 --- /dev/null +++ b/crates/odf/src/text/tests_more/format_changes.rs @@ -0,0 +1,74 @@ +//! Tracked formatting changes (ODF 1.3 §5.5.7 `text:format-change`, marked by +//! `text:change-start`/`-end`): the text keeps a formatting revision with its author and date. +//! ODF records no old formatting, so the old is the text's own (accepting keeps it, rejecting +//! changes nothing); WordCraft's formatting revisions are written as format changes. + +use crate::text::tests::{NS_T, content_of, package}; +use crate::text::{export, import}; +use craft_wordmodel::Document; +use craft_wordmodel::RevisionKind; + +const NS_DC: &str = r#"xmlns:dc="http://purl.org/dc/elements/1.1/""#; + +/// Each run with a formatting revision: (its text, the revision's author, whether it is bold). +fn changed(d: &Document) -> Vec<(String, String, bool)> { + let mut out = Vec::new(); + for b in d.body.iter() { + let Some(p) = b.as_para() else { continue }; + for (r, c) in p.run_ranges() { + if let Some(ch) = &c.fmt_change { + let rev = d.revisions.get(ch.rev as usize).expect("its revision"); + assert_eq!(rev.kind, RevisionKind::Format); + out.push((p.text.get(r).unwrap_or("").to_string(), rev.author.clone(), c.bold == Some(true))); + } + } + } + out +} + +/// LibreOffice's own (a document it saved). +fn open() -> Document { + let content = format!( + r#"Ada2026-11-01T11:29:13Some words that turned bold."# + ); + import(&package(&content, None)).unwrap() +} + +#[test] +fn a_format_change_opens_as_a_formatting_revision() { + assert_eq!(changed(&open()), [("words".to_string(), "Ada".to_string(), true)]); +} + +#[test] +fn format_changes_round_trip() { + let d = open(); + let bytes = export(&d).unwrap(); + let c = content_of(&bytes); + assert!(c.contains("") && c.contains(""#), "{c}"); + assert_eq!(changed(&import(&bytes).unwrap()), changed(&d)); +} + +#[test] +fn a_region_named_by_xml_id_alone_opens() { + let content = format!( + r#"Bo2026-01-01T00:00:00shown"# + ); + assert_eq!(changed(&import(&package(&content, None)).unwrap()), [("shown".to_string(), "Bo".to_string(), false)]); +} + +#[test] +fn hostile_format_changes_do_not_panic() { + for body in [ + r#"x"#, + r#"never endsnext"#, + r#"ends first"#, + ] { + let content = format!( + r#"A2026-01-01T00:00:00{body}"# + ); + let d = import(&package(&content, None)).unwrap(); + let _ = export(&d).unwrap(); + } +} diff --git a/crates/odf/src/text/tests_more/forms.rs b/crates/odf/src/text/tests_more/forms.rs new file mode 100644 index 000000000..808e0096d --- /dev/null +++ b/crates/odf/src/text/tests_more/forms.rs @@ -0,0 +1,248 @@ +//! Form controls (ODF 1.3 §13): `office:forms` controls placed by `draw:control`, read as +//! WordCraft content controls and written back as form controls LibreOffice shows. + +use crate::text::tests::{NS_T, entry, package, paras}; +use crate::text::{export, import, parse}; +use craft_flow::Inline; +use craft_wordmodel::control::{ControlKind, ListItem}; + +const NS_F: &str = r#"xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0""#; + +/// LibreOffice's form controls: a text field, a check box, a list box (Green chosen), a combo +/// box, a multi-line field, a radio button, a date field, an empty field and a button. +const FORMS: &str = r#""#; + +fn doc_xml() -> String { + let ctl = |id: &str| format!(r#""#); + let p: String = (1..=9).map(|i| format!("Field {i}: {}", ctl(&format!("control{i}")))).collect(); + format!( + r#"{FORMS}{p}After {}"#, + ctl("nothing") + ) +} + +/// Each paragraph's controls: (kind, tag, title, text, showing placeholder). +fn controls(flow: &craft_flow::Flow) -> Vec<(ControlKind, String, String, String, bool)> { + paras(flow) + .into_iter() + .flat_map(|p| &p.inlines) + .filter_map(|i| match i { + Inline::Control { control, text, .. } => { + Some((control.kind.clone(), control.tag.clone(), control.title.clone(), text.clone(), control.showing_placeholder)) + } + _ => None, + }) + .collect() +} + +#[test] +fn form_controls_open_as_content_controls() { + let flow = parse(&package(&doc_xml(), None)).unwrap(); + let got = controls(&flow); + assert_eq!(got.len(), 9, "a missing control is left out: {got:?}"); + // A button, which WordCraft has no kind for, shows its label (see `buttons_are_kept`). + assert_eq!((got[8].1.as_str(), got[8].3.as_str()), ("Submit", "Send")); + assert_eq!(got[0], (ControlKind::Text { multi_line: false }, "FullName".into(), String::new(), "Ada Lovelace".into(), false)); + assert!(matches!(got[1].0, ControlKind::CheckBox { checked: true, checked_char: '\u{2612}', unchecked_char: '\u{2610}', .. })); + assert_eq!((got[1].1.as_str(), got[1].2.as_str(), got[1].3.as_str()), ("Subscribe", "Yes please", "\u{2612}")); + let items = vec![ + ListItem { display: "Red".into(), value: String::new() }, + ListItem { display: "Green".into(), value: String::new() }, + ListItem { display: "Blue".into(), value: "b".into() }, + ]; + assert_eq!(got[2], (ControlKind::DropDown { items, last_value: "Green".into() }, "Colour".into(), String::new(), "Green".into(), false)); + assert!(matches!(&got[3].0, ControlKind::ComboBox { items, .. } if items.len() == 2) && got[3].3 == "Bergen"); + assert_eq!((got[4].0.clone(), got[4].3.as_str()), (ControlKind::Text { multi_line: true }, "Line one")); + assert!(matches!(got[5].0, ControlKind::CheckBox { checked: true, checked_char: '\u{25C9}', .. }) && got[5].3 == "\u{25C9}"); + assert!(matches!(&got[6].0, ControlKind::Date { full_date, .. } if full_date == "2026-11-01T00:00:00Z") && got[6].3 == "2026-11-01"); + // An empty field shows its placeholder. + assert!(got[7].4 && got[7].3 == "Type here."); + // The forms' own elements aren't text. + assert!(paras(&flow).iter().all(|p| !matches!(p.inlines.first(), Some(Inline::Text(t, _)) if t.contains('y') && t.len() == 1))); +} + +#[test] +fn content_controls_are_saved_as_form_controls() { + let doc = import(&package(&doc_xml(), None)).unwrap(); + let saved = export(&doc).unwrap(); + let content = entry(&saved, "content.xml"); + for want in [ + r#""#, + r#""#, + r#"\n"#), "{content}"); + // And they open again the same. + assert_eq!(controls(&parse(&saved).unwrap()), controls(&parse(&package(&doc_xml(), None)).unwrap())); +} + +#[test] +fn hostile_forms_are_bounded() { + let many: String = (0..20_000).map(|i| format!(r#""#)).collect(); + let items: String = (0..5000).map(|i| format!(r#""#)).collect(); + let content = format!( + r#"{many}{items}"# + ); + let flow = parse(&package(&content, None)).unwrap(); + let got = controls(&flow); + // Ids past the limit aren't read; a list keeps at most its limit of entries. + assert!(got.iter().all(|(k, ..)| k.items().is_none_or(|i| i.len() <= 1000))); + assert!(got.len() <= 2); +} + +#[test] +fn controls_keep_their_size() { + let saved = export(&import(&package(&doc_xml(), None)).unwrap()).unwrap(); + let content = entry(&saved, "content.xml"); + // Each control 1in × 0.2in, as it was, not sized to its text. + assert_eq!(content.matches(r#"svg:width="72pt" svg:height="14.4pt""#).count(), 9, "{content}"); +} + +#[test] +fn buttons_are_kept() { + let button = r#""#; + let content = format!( + r#"{button}Press "# + ); + let doc = import(&package(&content, None)).unwrap(); + // It shows its label, which can't be edited. + let p = doc.body[0].as_para().unwrap(); + assert!(p.text.contains("Send & close"), "{:?}", p.text); + let saved = export(&doc).unwrap(); + let c = entry(&saved, "content.xml"); + assert!( + c.contains(r#"".into(); + let kept = crate::text::forms::form_xml(&c, "Go", "control1", 1).unwrap().0; + assert!(kept.starts_with(r#""#), "{kept}"); + c.kind = ControlKind::CheckBox { + checked: true, + checked_char: '\u{2612}', + checked_font: String::new(), + unchecked_char: '\u{2610}', + unchecked_font: String::new(), + }; + let made = crate::text::forms::form_xml(&c, "\u{2612}", "control1", 1).unwrap().0; + assert!(made.starts_with("".repeat(5000), "".repeat(5000)); + let wide = + r#""# + .repeat(5000); + let content = format!( + r#"{deep}{wide}"# + ); + let doc = import(&package(&content, None)).unwrap(); + let c = entry(&export(&doc).unwrap(), "content.xml"); + // Too long to keep whole: left out; a control inside a kept one is part of it. + assert_eq!(c.matches(""# + ); + let c = entry(&export(&import(&package(&content, None)).unwrap()).unwrap(), "content.xml"); + assert_eq!(c.matches(""#; + let styles = format!( + r#"{decls}"# + ); + let content = format!( + r#"{decls}x"# + ); + let saved = export(&import(&package(&content, Some(&styles))).unwrap()).unwrap(); + for part in [styles_of(&saved), content_of(&saved)] { + assert!(part.contains(r#"x:odd="1""#), "{part}"); + assert!(part.contains(r#"xmlns:x="urn:example:x""#), "{part}"); + } +} diff --git a/crates/odf/src/text/tests_more/lists.rs b/crates/odf/src/text/tests_more/lists.rs new file mode 100644 index 000000000..079413aa8 --- /dev/null +++ b/crates/odf/src/text/tests_more/lists.rs @@ -0,0 +1,168 @@ +//! Lists (ODF 1.3 §5.3, §16.30–16.33): each `text:list` is a list of its own unless it continues +//! one; its list style gives each level's number format, prefix and suffix, the levels shown, +//! the bullet character and the start value. + +use crate::text::tests::{NS_T, package}; +use crate::text::{export, import}; +use craft_wordmodel::Document; +use craft_wordmodel::section::NumFormat; + +const NS_X: &str = r#"xmlns:xml="http://www.w3.org/XML/1998/namespace""#; + +fn doc(styles: &str, body: &str) -> Document { + let content = format!( + r#"{styles}{body}"# + ); + import(&package(&content, None)).unwrap() +} + +fn number(name: &str, format: &str, prefix: &str, suffix: &str, extra: &str) -> String { + format!( + r#""# + ) +} + +fn bullet(name: &str, ch: &str) -> String { + format!(r#""#) +} + +fn list(style: &str, items: &[&str], attrs: &str) -> String { + let items: String = items.iter().map(|t| format!("{t}")).collect(); + format!(r#"{items}"#) +} + +/// Each list paragraph: (its text, its list instance, level 0's format, label text and start). +fn items(d: &Document) -> Vec<(String, u32, NumFormat, String, u32)> { + d.body + .iter() + .filter_map(|b| b.as_para()) + .filter_map(|p| { + let n = p.props.numbering?; + let lv = d.numbering.level(n.num, 0)?; + Some((p.text.clone(), n.num, lv.format, lv.text.clone(), lv.start)) + }) + .collect() +} + +fn styles() -> String { + [number("N1", "1", "", ".", ""), number("N2", "i", "", ".", ""), number("N3", "A", "(", ")", ""), bullet("B1", "–")].concat() +} + +fn lists_body() -> String { + [list("N1", &["One", "Two"], ""), list("N2", &["i one", "i two"], ""), list("N3", &["A one"], ""), list("B1", &["dash"], "")].concat() +} + +#[test] +fn separate_lists_stay_separate_with_their_formats() { + let d = doc(&styles(), &lists_body()); + let got = items(&d); + let nums: Vec = got.iter().map(|i| i.1).collect(); + assert_eq!(nums[0], nums[1]); + assert!(nums[1] != nums[2] && nums[3] != nums[4] && nums[4] != nums[5], "each text:list is its own list: {nums:?}"); + let shown: Vec<(NumFormat, &str)> = got.iter().map(|i| (i.2, i.3.as_str())).collect(); + assert_eq!( + shown, + [ + (NumFormat::Decimal, "%1."), + (NumFormat::Decimal, "%1."), + (NumFormat::LowerRoman, "%1."), + (NumFormat::LowerRoman, "%1."), + (NumFormat::UpperLetter, "(%1)"), + (NumFormat::Bullet, "–"), + ] + ); +} + +#[test] +fn a_list_can_continue_another() { + let body = [ + list("N1", &["One", "Two"], r#"xml:id="first""#), + "Between".to_string(), + list("N1", &["Three"], r#"text:continue-numbering="true""#), + list("N2", &["i"], ""), + list("N1", &["Four"], r#"text:continue-list="first""#), + ] + .concat(); + let d = doc(&styles(), &body); + let got = items(&d); + assert_eq!(got.len(), 5); + assert_eq!(got[0].1, got[2].1, "continue-numbering goes on with the list before"); + assert_eq!(got[0].1, got[4].1, "continue-list goes on with the list it names"); + assert_ne!(got[0].1, got[3].1); +} + +#[test] +fn start_values_and_levels_shown() { + let styles = [number("S", "1", "", ".", r#"text:start-value="5" text:display-levels="1""#), { + r#""#.to_string() + }] + .concat(); + let body = [list("S", &["five"], ""), r#"11.1"#.to_string()].concat(); + let d = doc(&styles, &body); + let got = items(&d); + assert_eq!(got[0].4, 5); + let n = d.body.iter().filter_map(|b| b.as_para()).find(|p| p.text == "1.1").and_then(|p| p.props.numbering).unwrap(); + assert_eq!(n.level, 1); + assert_eq!(d.numbering.level(n.num, 1).unwrap().text, "%1.%2."); +} + +#[test] +fn lists_round_trip() { + let d = + doc(&styles(), &[lists_body(), "Between".to_string(), list("N1", &["Three"], r#"text:continue-numbering="true""#)].concat()); + let back = import(&export(&d).unwrap()).unwrap(); + let strip = |v: Vec<(String, u32, NumFormat, String, u32)>| { + // Instance ids may be renumbered; which items share one may not. + let firsts: Vec = v.iter().map(|i| i.1).collect(); + let same: Vec = firsts.iter().map(|n| firsts.iter().position(|m| m == n).unwrap_or(0)).collect(); + v.into_iter().zip(same).map(|((t, _, f, l, s), k)| (t, k, f, l, s)).collect::>() + }; + assert_eq!(strip(items(&back)), strip(items(&d))); +} + +#[test] +fn hostile_list_styles_are_bounded() { + let styles = [ + number("H1", "?", "((((((((((((((((((((", "))))))))))))))))))", r#"text:start-value="99999999999" text:display-levels="99""#), + r#""#.to_string(), + ] + .concat(); + let d = doc(&styles, &[list("H1", &["x"], ""), list("H2", &["y"], r#"text:continue-list="nope""#)].concat()); + let got = items(&d); + assert_eq!(got.len(), 2); + assert!(got[0].3.chars().count() <= 64, "{}", got[0].3); + let _ = export(&d).unwrap(); +} + +#[test] +fn a_word_before_the_number_is_kept() { + let d = doc(&number("C", "1", "Chapter ", ":", ""), &list("C", &["x"], "")); + assert_eq!(items(&d)[0].3, "Chapter %1:"); + let back = import(&export(&d).unwrap()).unwrap(); + assert_eq!(items(&back)[0].3, "Chapter %1:"); +} + +#[test] +fn list_level_positions_are_read_and_written() { + // Each level's indent, hanging and number alignment (ODF 1.3 §17.19–17.20, + // `style:list-level-properties` in label-alignment mode). + let styles = r#""#; + let d = doc(styles, &[list("P", &["one"], ""), list("Q", &["dot"], "")].concat()); + let level = |i: usize| { + let p = d.body.iter().filter_map(|b| b.as_para()).nth(i).unwrap(); + let n = p.props.numbering.unwrap(); + d.numbering.level(n.num, n.level).unwrap().clone() + }; + let (a, b) = (level(0), level(1)); + assert_eq!((a.indent, a.hanging, a.align), (36.0, 18.0, craft_wordmodel::Align::Right)); + assert_eq!((b.indent, b.hanging), (14.4, 7.2)); + let back = import(&export(&d).unwrap()).unwrap(); + let again = |i: usize| { + let p = back.body.iter().filter_map(|b| b.as_para()).nth(i).unwrap(); + let n = p.props.numbering.unwrap(); + back.numbering.level(n.num, n.level).unwrap().clone() + }; + let (c, e) = (again(0), again(1)); + assert_eq!((c.indent, c.hanging, c.align), (a.indent, a.hanging, a.align)); + assert!((e.indent - b.indent).abs() < 0.01 && (e.hanging - b.hanging).abs() < 0.01); +} diff --git a/crates/odf/src/text/tests_more/lossless.rs b/crates/odf/src/text/tests_more/lossless.rs new file mode 100644 index 000000000..7d05e3491 --- /dev/null +++ b/crates/odf/src/text/tests_more/lossless.rs @@ -0,0 +1,896 @@ +//! The no-edit loss gate: a document opened and saved without changes loses nothing. Each +//! sample below is saved without edits and compared, part by part, with what was read: every +//! element, every attribute, every formatting property's value and every word must still be +//! there. What may go is listed in [`ALLOWED`], each with its reason. +//! +//! The samples are written here, by us, for the purpose (see `kept`'s module docs): never a file +//! or text taken from someone else. + +use std::collections::BTreeMap; + +use crate::text::xml::{Ev, events}; +use crate::text::{export, import}; + +/// What a save may lose, and why. Matched against the start of a loss (see [`inventory`]). +const ALLOWED: &[(&str, &str)] = &[]; + +const NS: &str = r#"xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0" xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0" xmlns:wcx="urn:example:wordcraft-test" xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0" office:version="1.3""#; + +/// A sample package: `content.xml`, `styles.xml` and `meta.xml` bodies (inside their roots). +struct Sample { + name: &'static str, + content: &'static str, + styles: &'static str, + meta: &'static str, +} + +fn package(s: &Sample) -> Vec { + let parts = [ + ("mimetype", "application/vnd.oasis.opendocument.text".to_string()), + ("content.xml", format!("{}", s.content)), + ("styles.xml", format!("{}", s.styles)), + ("meta.xml", format!("{}", s.meta)), + ]; + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + for (name, data) in parts { + z.start_file(name, o).unwrap(); + std::io::Write::write_all(&mut z, data.as_bytes()).unwrap(); + } + // The picture a sample may show, and a formula object with its preview. + for (name, data) in [ + ("META-INF/manifest.xml", MANIFEST.as_bytes()), + ("Pictures/dot.png", DOT), + ("Object 9/content.xml", FORMULA.as_bytes()), + ("ObjectReplacements/Object 9", DOT), + ] { + z.start_file(name, o).unwrap(); + std::io::Write::write_all(&mut z, data).unwrap(); + } + z.finish().unwrap().into_inner() +} + +/// The samples' manifest (ODF 1.3 Part 2 §4): the formula object is one. +const MANIFEST: &str = r#""#; + +/// A formula object's content (MathML, ODF 1.3 §10.4.3): x². +const FORMULA: &str = r#"x2x^2"#; + +/// A one-pixel picture of our own. +const DOT: &[u8] = &[ + 0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d, b'I', b'H', b'D', b'R', 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 0x1f, 0x15, 0xc4, + 0x89, 0, 0, 0, 0x0d, b'I', b'D', b'A', b'T', 0x78, 0xda, 0x63, 0xfc, 0xcf, 0xc0, 0x50, 0x0f, 0, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, + 0, 0, 0, 0, b'I', b'E', b'N', b'D', 0xae, 0x42, 0x60, 0x82, +]; + +/// A length's value in points to a tenth, else the value as written. +fn norm(v: &str) -> String { + let v = v.trim(); + let units = [("cm", 72.0 / 2.54), ("mm", 72.0 / 25.4), ("in", 72.0), ("pt", 1.0), ("pc", 12.0), ("px", 0.75)]; + for (u, f) in units { + if let Some(n) = v.strip_suffix(u).and_then(|n| n.parse::().ok()) { + return format!("{:.1}pt", n * f); + } + } + v.to_lowercase() +} + +/// Attributes whose values are names or places (their presence counts, not their value). +fn nameish(a: &str) -> bool { + ["name", "id", "ref", "href", "class", "index", "parent", "master", "list-style", "data-style"] + .iter() + .any(|n| a.ends_with(n) || a.contains("style-name")) +} + +/// What a package holds, as counts: `el `, `attr @`, `val @=` (the +/// values of formatting properties), and `word `. +fn inventory(bytes: &[u8]) -> BTreeMap { + let mut z = crate::Package::open(bytes, &crate::Limits::DEFAULT).unwrap(); + let mut inv = BTreeMap::new(); + let mut add = |k: String| *inv.entry(k).or_insert(0usize) += 1; + for part in ["content.xml", "styles.xml", "meta.xml"] { + let Some(xml) = z.read(part).unwrap() else { continue }; + let mut text = String::new(); + events(&xml, |ev| match ev { + Ev::Start(q, a) => { + add(format!("el {q}")); + let props = q.ends_with("-properties"); + for (k, v) in a { + if k.starts_with("xmlns") { + continue; + } + if props && !nameish(&k) { + add(format!("val {q}@{k}={}", norm(&v))); + } else { + add(format!("attr {q}@{k}")); + } + } + if matches!(q.as_str(), "text:p" | "text:h" | "text:tab" | "text:s" | "text:line-break") { + text.push(' '); + } + } + Ev::End(_) => {} + Ev::Text(t) => text.push_str(&t), + }); + if part == "content.xml" { + for w in text.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()) { + add(format!("word {w}")); + } + } + } + inv +} + +/// What `saved` lacks that `orig` has (none of it allowed): fewer of a word or an element, or +/// an attribute or a value it has none of. +fn losses(orig: &[u8], saved: &[u8]) -> Vec { + let (a, b) = (inventory(orig), inventory(saved)); + a.into_iter() + // Fewer words or elements (a section of three), or an attribute or value gone. + .filter(|(k, n)| b.get(k).copied().unwrap_or(0) < *n && (k.starts_with("word ") || k.starts_with("el ") || !b.contains_key(k))) + .map(|(k, _)| k) + .filter(|k| !ALLOWED.iter().any(|(p, _)| k.starts_with(p))) + .collect() +} + +fn check(s: &Sample) { + let orig = package(s); + let d = import(&orig).unwrap(); + let saved = export(&d).unwrap(); + let lost = losses(&orig, &saved); + assert!(lost.is_empty(), "{}: lost on a save without edits:\n {}", s.name, lost.join("\n ")); + // Saved again: nothing more goes. + let again = export(&import(&saved).unwrap()).unwrap(); + let lost = losses(&saved, &again); + assert!(lost.is_empty(), "{}: lost on a second save:\n {}", s.name, lost.join("\n ")); +} + +/// Paragraphs and text WordCraft partly models: unknown elements, attributes and namespaces. +const TEXT: Sample = Sample { + name: "text", + content: r##"Orchard notesPears ripen late in the year, and apples early.Second paragraph with a 1Footnote words. note."##, + styles: r##""##, + meta: "", +}; + +/// Style definitions WordCraft doesn't use: an unused list style, note and line numbering +/// settings, an outline style, unknown style attributes. +const STYLES: Sample = Sample { + name: "styles", + content: r##"Only text."##, + styles: r##""##, + meta: "", +}; + +#[test] +fn a_save_without_edits_loses_nothing() { + for s in SAMPLES { + check(&s); + } +} + +#[test] +fn the_gate_sees_a_loss() { + let with = Sample { + name: "with", + content: r##"Kept words"##, + styles: "", + meta: "", + }; + let without = + Sample { name: "without", content: r##"Kept"##, styles: "", meta: "" }; + let lost = losses(&package(&with), &package(&without)); + assert_eq!(lost, ["attr text:p@wcx:a", "el wcx:b", "word words"]); +} + +/// Document properties WordCraft doesn't model. +const META: Sample = Sample { + name: "meta", + content: r##"Body."##, + styles: "", + meta: r##"Sample WriterLossless sampleen-GBPT1H2M3S2026-01-02T03:04:05Sample Printertrue"##, +}; + +/// Declarations at the start of the text and the document's scripts. +const DECLS: Sample = Sample { + name: "decls", + content: r##"Decls."##, + styles: "", + meta: "", +}; + +/// Page layouts and master pages WordCraft partly models: the footnote separator, header +/// properties, a layout grid, a header with its own paragraph. +const PAGES: Sample = Sample { + name: "pages", + content: r##"Page."##, + styles: r##"Running words"##, + meta: "", +}; + +/// A table WordCraft partly models: unknown table, row and cell attributes. +const TABLE: Sample = Sample { + name: "table", + content: r##"OneTwoAfter."##, + styles: "", + meta: "", +}; + +/// Named sections, one protected. +const SECTIONS: Sample = Sample { + name: "sections", + content: r##"Before.Inside the terms.Inside the lock.A cell alone.Item one.Notes under the agenda.Closing the outer one.Ada Lovelace12 Example StreetBelow the address.A boxed sender.After."##, + styles: "", + meta: "", +}; + +/// A form with its own properties and event listeners, and a field placed in the text. +const FORMS: Sample = Sample { + name: "forms", + content: r##"Name: "##, + styles: "", + meta: "", +}; + +/// A table of contents and an alphabetical index with their settings and templates. +const INDEXES: Sample = Sample { + name: "indexes", + content: r##"ContentsContentsOrchards1OrchardsPears and apples.WordsWordsapples1"##, + styles: "", + meta: "", +}; + +/// Pictures and shapes in the text: a cropped picture with its look (adjustments WordCraft +/// doesn't model), a wrapped one with a contour, a shape, a group, and a picture in a table. +const DRAWINGS: Sample = Sample { + name: "drawings", + content: r##"A picture A dotOne pixel of our own.beside the text.A shape and a group after. in a cell"##, + styles: r##""##, + meta: "", +}; + +/// Comments in a paragraph: one over a range, with a reply, resolved, its text styled; one at a +/// point; with what WordCraft doesn't model (an attribute, a styled paragraph in the comment). +const COMMENTS: Sample = Sample { + name: "comments", + content: r##"Before Ada Lovelace2026-10-31T10:00:00ALA firm remark.Alan Turing2026-10-31T11:00:00Agreed.the words remarked on and a pointGrace Hopper2026-10-31T12:00:00Here. after."##, + styles: "", + meta: "", +}; + +/// Tracked changes: an insertion, a deletion (of text inserted before it, in part), a format +/// change, a change's own remark (`office:change-info` paragraphs), and what WordCraft doesn't +/// model on them. +const CHANGES: Sample = Sample { + name: "changes", + content: r##"Ada Lovelace2026-10-30T09:00:00Added for clarity.Ada Lovelace2026-10-30T09:05:00Alan Turing2026-10-30T10:00:00gone and newGrace Hopper2026-10-30T11:00:00Some inserted words.Some removed words.A restyled word."##, + styles: "", + meta: "", +}; + +/// A text box with styled text in a styled frame, and a shape holding text, each with what +/// WordCraft doesn't model. +const TEXT_BOXES: Sample = Sample { + name: "text boxes", + content: r##"Our addressA box Inside the box.A second line.beside the text.A shape Stopwith words."##, + styles: r##""##, + meta: "", +}; + +/// Bookmarks: at a point in a table's cell (with LibreOffice's relative column widths), and over +/// text from one paragraph into the next. +const BOOKMARKS: Sample = Sample { + name: "bookmarks", + content: r##"ItemWhoFrom hereto there."##, + styles: "", + meta: "", +}; + +/// A footnote with styled text, and a formula object with its preview picture. +const NOTES_AND_FORMULAS: Sample = Sample { + name: "notes and formulas", + content: r##"A claim1A source for it. and a formula in it."##, + styles: "", + meta: "", +}; + +/// Lists as LibreOffice and OpenOffice write them: a label followed by a space, a picture +/// bullet, an unnumbered header, a nested level, and a list going on with the first by its id. +const LISTS: Sample = Sample { + name: "lists", + content: r##"A heading not countedFirstSecondInnerBetween the lists.Third, going onPictured"##, + styles: "", + meta: "", +}; + +/// Index marks over text, a plugin in a frame, and a form's data source: what a check of the +/// schema's every element found no file holding. +const MARKS: Sample = Sample { + name: "marks", + content: r##"A marked heading, a term and a pear.A plugin here."##, + styles: "", + meta: "", +}; + +/// What ties paragraphs together: a comment over text from one paragraph into the next, an +/// insertion likewise, and a text box continuing in another's in a later paragraph. +const GROUPS: Sample = Sample { + name: "groups", + content: r##"Ada Lovelace2026-11-01T09:00:00The remark Alan Turing2026-11-01T10:00:00Both paragraphs.starts hereand ends here; an insertiongoing on to here.A box Words flowing on.here.Its next there."##, + styles: "", + meta: "", +}; + +/// Every sample. +const SAMPLES: [Sample; 18] = [ + TEXT, + STYLES, + META, + DECLS, + TABLE, + PAGES, + SECTIONS, + FORMS, + INDEXES, + DRAWINGS, + COMMENTS, + CHANGES, + TEXT_BOXES, + BOOKMARKS, + NOTES_AND_FORMULAS, + LISTS, + MARKS, + GROUPS, +]; + +/// Whether `xml` is well-formed: every element closed, in order, once. +fn well_formed(xml: &[u8]) -> bool { + let limits = craft_xml::Limits { + max_input: usize::MAX, + max_depth: usize::MAX, + max_nodes: usize::MAX, + max_elements: usize::MAX, + max_attrs: usize::MAX, + max_attrs_total: usize::MAX, + }; + craft_xml::Options::default().strict().with_limits(limits).parse(xml).is_ok() +} + +fn parts_well_formed(bytes: &[u8]) -> Result<(), String> { + let mut z = crate::Package::open(bytes, &crate::Limits::DEFAULT).map_err(|e| e.to_string())?; + for part in ["content.xml", "styles.xml", "meta.xml", "settings.xml", "META-INF/manifest.xml"] { + let xml = z.read(part).map_err(|e| e.to_string())?.ok_or(format!("{part}: missing"))?; + if !well_formed(&xml) { + return Err(format!("{part} isn't well-formed:\n{}", String::from_utf8_lossy(&xml))); + } + } + Ok(()) +} + +#[test] +fn edits_leave_kept_markup_well_formed() { + // Whatever is edited (a paragraph that opens a section deleted, text typed into each + // paragraph, a style changed), what is written stays well-formed and reads back. + use craft_wordmodel::Pos; + for s in SAMPLES { + let d = import(&package(&s)).unwrap(); + let mut edits: Vec = Vec::new(); + for i in 0..d.body.len() { + let mut e = d.clone(); + if e.body.get(i).is_some_and(|b| b.as_para().is_some()) { + e.insert_text(&Pos::body(i, 0), "Edited ", &Default::default()).unwrap(); + } + edits.push(e); + if d.body.len() > 1 { + let mut e = d.clone(); + e.body.remove(i); + edits.push(e); + } + } + let mut styled = d.clone(); + if let Some(st) = std::sync::Arc::make_mut(&mut styled.styles).styles.iter_mut().find(|s| s.id == "Normal") { + st.chr.italic = Some(true); + } + edits.push(styled); + for e in edits { + let saved = export(&e).unwrap(); + if let Err(m) = parts_well_formed(&saved) { + panic!("{}: {m}", s.name); + } + let again = export(&import(&saved).unwrap()).unwrap(); + assert!(parts_well_formed(&again).is_ok(), "{}", s.name); + } + } +} + +#[test] +fn an_edited_index_is_written_from_the_model() { + // Unedited, the index keeps its templates; once an entry changes, the edit is what's saved. + use craft_wordmodel::Pos; + let d = import(&package(&INDEXES)).unwrap(); + let c = content_of(&export(&d).unwrap()); + assert!(c.contains("text:combine-entries-with-pp=\"false\""), "{c}"); + let i = d.body.iter().position(|b| b.as_para().is_some_and(|p| p.plain_text().starts_with("Orchards\t"))).unwrap(); + let mut e = d.clone(); + e.insert_text(&Pos::body(i, 0), "Edited ", &Default::default()).unwrap(); + let c = content_of(&export(&e).unwrap()); + assert!(c.contains("Edited Orchards"), "{c}"); + // The other index, unedited, is still as read. + assert!(c.contains("text:combine-entries-with-pp=\"false\""), "{c}"); +} + +fn content_of(bytes: &[u8]) -> String { + let mut z = crate::Package::open(bytes, &crate::Limits::DEFAULT).unwrap(); + String::from_utf8(z.read("content.xml").unwrap().unwrap()).unwrap() +} + +/// The declarations at the start of the text are written in the schema's order: variables, +/// sequences, then user fields. +#[test] +fn the_text_declarations_are_written_in_the_schemas_order() { + let c = content_of(&export(&import(&package(&DECLS)).unwrap()).unwrap()); + let at = |el: &str| c.find(el).unwrap_or_else(|| panic!("no {el} in {c}")); + assert!(at("").next()).unwrap_or(""); + assert_eq!(in_box.matches("").count(), "{c}"); + } + } +} + +#[test] +fn a_picture_kept_as_read_is_written_with_its_paragraph_only() { + // The picture a paragraph written as read shows is in the package; once that paragraph + // and the table showing it are gone, it isn't (a deleted picture stays deleted). + let d = import(&package(&DRAWINGS)).unwrap(); + let files = |bytes: &[u8]| { + let mut z = crate::Package::open(bytes, &crate::Limits::DEFAULT).unwrap(); + let names: Vec = z.names().map(str::to_string).collect::>().into_iter().collect(); + let dot = z.read("Pictures/dot.png").unwrap(); + (names, dot) + }; + let saved = export(&d).unwrap(); + assert_eq!(files(&saved).1.as_deref(), Some(DOT)); + assert!(crate::text::tests::entry(&saved, "META-INF/manifest.xml").contains(r#"manifest:full-path="Pictures/dot.png""#)); + let mut e = d.clone(); + e.body.retain(|b| b.as_para().is_none_or(|p| !p.text.starts_with("A picture")) && b.as_table().is_none()); + let saved = export(&e).unwrap(); + let (names, dot) = files(&saved); + assert!(dot.is_none(), "{names:?}"); + assert!(!content_of(&saved).contains("dot.png")); +} + +/// A picture that can't be read (its entry is corrupt) isn't pointed at by markup kept as read: +/// what showed it is written from the model, and the saved file links to nothing it hasn't got. +#[test] +fn markup_kept_as_read_never_links_to_a_picture_that_could_not_be_read() { + let mut bytes = package(&DRAWINGS); + let at = bytes.windows(DOT.len()).position(|w| w == DOT).unwrap(); + // The first copy is `Pictures/dot.png`'s: one byte changed fails its checksum. + bytes[at + 20] ^= 0xff; + let d = import(&bytes).unwrap(); + let saved = export(&d).unwrap(); + let c = content_of(&saved); + let z = crate::Package::open(&saved, &crate::Limits::DEFAULT).unwrap(); + assert!(!c.contains("Pictures/dot.png") || z.contains("Pictures/dot.png"), "{c}"); + // The text round it stays. + assert!(c.contains("A picture"), "{c}"); +} + +#[test] +fn a_paragraph_with_comments_is_written_as_read_while_they_are() { + let d = import(&package(&COMMENTS)).unwrap(); + let as_read = |d: &craft_wordmodel::Document| content_of(&export(d).unwrap()).contains(r#"wcx:mark="kept""#); + assert!(as_read(&d)); + // A comment's text edited, resolved, deleted with its reply, or answered: written from the + // model, which has what changed. + let first = |d: &craft_wordmodel::Document| d.comments.iter().find(|(_, c)| c.author == "Ada Lovelace").map(|(id, c)| (*id, c.clone())).unwrap(); + let mut e = d.clone(); + let (_, c) = first(&e); + if let Some(craft_wordmodel::Block::Para(p)) = e.parts.get_mut(&c.part).and_then(|p| p.blocks.first_mut()).map(std::sync::Arc::make_mut) { + p.props.style = Some("Edited".into()); + p.touch(); + } + assert!(!as_read(&e)); + let mut e = d.clone(); + let (id, _) = first(&e); + if let Some(c) = e.comments.get_mut(&id) { + c.resolved = true; + } + assert!(!as_read(&e)); + assert!(content_of(&export(&e).unwrap()).contains(r#"loext:resolved="true""#)); + let mut e = d.clone(); + let (id, c) = first(&e); + let blocks = e.parts.get(&c.part).unwrap().blocks.clone(); + let part = e.try_add_part(craft_wordmodel::PartKind::Comment, blocks).unwrap(); + e.comments.insert(999, craft_wordmodel::Comment { author: "New".into(), parent: Some(id), part, ..Default::default() }); + let c = content_of(&export(&e).unwrap()); + assert!(!c.contains(r#"wcx:mark="kept""#) && c.contains("New"), "{c}"); + // Every save stays well-formed and reads back with its comments. + assert_eq!(import(&export(&e).unwrap()).unwrap().comments.len(), 4); +} + +#[test] +fn a_comment_answered_in_another_paragraph_is_written_from_the_model() { + // Its reply elsewhere in the file: the paragraph holding it isn't kept (WordCraft writes a + // reply with what it answers), and the reply still names it. + let content = COMMENTS.content.replace( + r#"Alan Turing2026-10-31T11:00:00Agreed."#, + "", + ).replace( + "", + r#"Later Alan TuringAgreed.on."#, + ); + let leaked: &'static str = Box::leak(content.into_boxed_str()); + let d = import(&package(&Sample { name: "elsewhere", content: leaked, styles: "", meta: "" })).unwrap(); + let saved = export(&d).unwrap(); + let c = content_of(&saved); + assert!(!c.contains(r#"wcx:mark="kept""#), "{c}"); + let again = import(&saved).unwrap(); + assert_eq!(again.comments.len(), 3); + assert!(again.comments.values().any(|c| c.author == "Alan Turing" && c.parent.is_some())); +} + +#[test] +fn a_change_accepted_is_written_from_the_model_the_others_as_read() { + // The insertion accepted: its paragraph is written from the model and its region isn't + // written; the other paragraphs keep theirs as read, and ids don't meet. + let d = import(&package(&CHANGES)).unwrap(); + let mut e = d.clone(); + let i = e.body.iter().position(|b| b.as_para().is_some_and(|p| p.text.starts_with("Some inserted"))).unwrap(); + if let Some(craft_wordmodel::Block::Para(p)) = e.body.get_mut(i).map(std::sync::Arc::make_mut) { + for r in &mut p.runs { + r.props.ins = None; + } + p.normalize(); + p.touch(); + } + let saved = export(&e).unwrap(); + assert!(parts_well_formed(&saved).is_ok()); + let c = content_of(&saved); + assert!(!c.contains(r#"wcx:region="kept""#), "{c}"); + assert!(c.contains(r#"text:id="ct12""#) && c.contains(r#"text:id="ct11""#) && c.contains(r#"text:id="ct13""#), "{c}"); + let back = import(&saved).unwrap(); + assert_eq!(back.revisions.len(), d.revisions.len() - 1, "{:?}", back.revisions); + // A change made in WordCraft next to the file's: its own id. + let mut e = d.clone(); + e.settings.track_changes = true; + if let Some(craft_wordmodel::Block::Para(p)) = e.body.get_mut(i).map(std::sync::Arc::make_mut) { + p.props.style = Some("Edited".into()); + p.touch(); + } + let c = content_of(&export(&e).unwrap()); + assert!(c.contains(r#"text:id="wcct1""#) && c.contains(r#"text:id="ct13""#), "{c}"); +} + +#[test] +fn a_text_box_edited_is_written_from_the_model() { + // Its text edited: the paragraph showing the box is written from the model, with the edit. + let d = import(&package(&TEXT_BOXES)).unwrap(); + assert!(content_of(&export(&d).unwrap()).contains(r#"wcx:box="kept""#)); + let mut e = d.clone(); + let id = e + .parts + .iter() + .find(|(_, p)| { + p.kind == craft_wordmodel::PartKind::TextBox && p.blocks.iter().any(|b| b.as_para().is_some_and(|p| p.text.starts_with("Inside"))) + }) + .map(|(id, _)| *id) + .unwrap(); + let mut blocks = e.parts.get(&id).unwrap().blocks.clone(); + if let Some(craft_wordmodel::Block::Para(p)) = blocks.get_mut(1).map(std::sync::Arc::make_mut) { + p.props.style = Some("Edited".into()); + p.touch(); + } + if let Some(part) = e.parts.get_mut(&id) { + part.blocks = blocks; + } + let saved = export(&e).unwrap(); + let c = content_of(&saved); + assert!(!c.contains(r#"wcx:box="kept""#) && c.contains("Inside the box."), "{c}"); + assert!(parts_well_formed(&saved).is_ok()); +} + +#[test] +fn linked_text_boxes_keep_their_link() { + // A box continuing in another (§19.112 `draw:chain-next-name`): both written from the model, + // which names them, and the link survives. + let content = r##"First Text that flows on.box.Second box."##; + let d = import(&package(&Sample { name: "linked", content, styles: "", meta: "" })).unwrap(); + let c = content_of(&export(&d).unwrap()); + let next = c.split("draw:chain-next-name=\"").nth(1).and_then(|r| r.split('"').next()).unwrap_or_default().to_string(); + assert!(!next.is_empty() && c.contains(&format!("draw:name=\"{next}\"")), "{c}"); +} + +#[test] +fn a_bookmark_kept_at_one_end_is_written_once() { + // Either paragraph a bookmark spans edited: one start and one end, each once. + let d = import(&package(&BOOKMARKS)).unwrap(); + for start in ["From", "to there"] { + let mut e = d.clone(); + let i = e.body.iter().position(|b| b.as_para().is_some_and(|p| p.text.starts_with(start))).unwrap(); + if let Some(craft_wordmodel::Block::Para(p)) = e.body.get_mut(i).map(std::sync::Arc::make_mut) { + p.props.style = Some("Edited".into()); + p.touch(); + } + let c = content_of(&export(&e).unwrap()); + assert_eq!(c.matches(r#""#).count(), 1, "{c}"); + assert_eq!(c.matches(r#""#).count(), 1, "{c}"); + assert!(c.find("bookmark-start").unwrap_or(0) < c.find("bookmark-end").unwrap_or(0), "{c}"); + } +} + +#[test] +fn a_note_or_formula_edited_is_written_from_the_model() { + let d = import(&package(&NOTES_AND_FORMULAS)).unwrap(); + let saved = export(&d).unwrap(); + assert!(content_of(&saved).contains(r#"wcx:note="kept""#)); + let names = |b: &[u8]| crate::Package::open(b, &crate::Limits::DEFAULT).unwrap().names().map(String::from).collect::>(); + assert!(names(&saved).iter().any(|n| n == "Object 9/content.xml"), "{:?}", names(&saved)); + let manifest = crate::text::tests::entry(&saved, "META-INF/manifest.xml"); + assert!( + manifest.contains(r#"manifest:full-path="Object 9/" manifest:media-type="application/vnd.oasis.opendocument.formula""#) + && manifest.contains(r#"manifest:full-path="ObjectReplacements/Object 9""#), + "{manifest}" + ); + // The note's text edited. + let mut e = d.clone(); + let id = e.parts.iter().find(|(_, p)| p.kind == craft_wordmodel::PartKind::Footnote).map(|(id, _)| *id).unwrap(); + let mut blocks = e.parts.get(&id).unwrap().blocks.clone(); + if let Some(craft_wordmodel::Block::Para(p)) = blocks.first_mut().map(std::sync::Arc::make_mut) { + p.props.style = Some("Edited".into()); + p.touch(); + } + if let Some(part) = e.parts.get_mut(&id) { + part.blocks = blocks; + } + let c = content_of(&export(&e).unwrap()); + assert!(!c.contains(r#"wcx:note="kept""#) && c.contains("for it."), "{c}"); + // The formula edited: written anew, and the old one's files aren't. + let mut e = d.clone(); + for b in e.body.iter_mut() { + if let craft_wordmodel::Block::Para(p) = std::sync::Arc::make_mut(b) { + for o in &mut p.objects { + if let craft_wordmodel::para::InlineObject::Equation { linear, .. } = o { + *linear = "y^3".into(); + } + } + p.touch(); + } + } + let saved = export(&e).unwrap(); + assert!(!names(&saved).iter().any(|n| n.starts_with("Object 9")), "{:?}", names(&saved)); + assert!(parts_well_formed(&saved).is_ok()); +} + +#[test] +fn a_list_kept_as_read_is_gone_on_with_and_an_edited_one_written_anew() { + let d = import(&package(&LISTS)).unwrap(); + let c = content_of(&export(&d).unwrap()); + // As read; the list going on with it (written from the model) names it. + assert!(c.contains(r#""#) && c.contains(r#"text:label-followed-by="space""#), "{c}"); + let going_on = c.split("Between the lists.").nth(1).unwrap_or(""); + assert!(going_on.contains(r#"text:continue-list="list1""#), "{c}"); + // An item edited: the list written from the model, its numbering going on still. + let mut e = d.clone(); + let i = e.body.iter().position(|b| b.as_para().is_some_and(|p| p.text.starts_with("Second"))).unwrap(); + if let Some(craft_wordmodel::Block::Para(p)) = e.body.get_mut(i).map(std::sync::Arc::make_mut) { + p.props.style = Some("Edited".into()); + p.touch(); + } + let saved = export(&e).unwrap(); + let c = content_of(&saved); + assert!(!c.contains(r#"xml:id="list1""#) && c.contains("Second"), "{c}"); + assert!(parts_well_formed(&saved).is_ok()); + let going_on = c.split("Between the lists.").nth(1).unwrap_or(""); + let first = c.split("".to_vec())]); + // Patch the uncompressed size in the central directory and local header to ~4 GB. + let mut patched = 0; + for sig in [[0x50u8, 0x4B, 0x01, 0x02], [0x50, 0x4B, 0x03, 0x04]] { + if let Some(pos) = z.windows(4).position(|w| w == sig) { + let off = if sig[2] == 1 { pos + 24 } else { pos + 22 }; + z[off..off + 4].copy_from_slice(&0xFFFF_FFF0u32.to_le_bytes()); + patched += 1; + } + } + assert_eq!(patched, 2); + assert_eq!(declared_oversized(&z, &LIMITS), Some(("a.xml".to_string(), 0xFFFF_FFF0))); + match Package::open(&z, &LIMITS) { + Err(_) => {} + Ok(p) => assert!(p.get("a.xml").is_none_or(|b| b.len() < 1024)), + } + let mut lazy = LazyPackage::open(&z, &LIMITS).unwrap(); + assert!(lazy.read("a.xml").is_err()); +} + +#[test] +fn parts_are_shared_not_copied() { + let z = zip_of(&[("word/media/pic.png", b"\x89PNG picture bytes".to_vec())]); + let pkg = Package::open(&z, &LIMITS).unwrap(); + let held = Arc::clone(pkg.get_shared("/WORD/media/PIC.png").unwrap()); + assert!(Arc::ptr_eq(&held, pkg.get_shared("word/media/pic.png").unwrap())); + assert_eq!(pkg.get("word/media/pic.png"), Some(&b"\x89PNG picture bytes"[..])); +} + +#[test] +fn skipped_parts_count_against_the_inflate_budget() { + // Kept bytes cap at 100: b, c and d declare more than is left and are left out before + // anything is inflated, so they cost no work and e, which fits, is still read. + let mut zw = zip::ZipWriter::new(Cursor::new(Vec::new())); + let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, len) in [("a", 80), ("b", 80), ("c", 80), ("d", 80), ("e", 5), ("big", 2000)] { + zw.start_file(name, o).unwrap(); + zw.write_all(&vec![b'x'; len]).unwrap(); + } + let z = zw.finish().unwrap().into_inner(); + let skip = |max_work| Limits { max_entries: 100, max_part: 1000, max_total: 100, max_work, oversized: Oversized::Skip }; + let p = Package::open(&z, &skip(130)).unwrap(); + assert!(p.get("a").is_some()); + assert!(p.get("b").is_none() && p.get("d").is_none()); + assert!(p.get("e").is_some(), "parts refused by their declared size cost nothing"); + assert_eq!(p.size(), 85); + // With room to spare the same small part is kept, and a part declaring more than the part + // limit is left out without being inflated. + let p = Package::open(&z, &skip(10_000)).unwrap(); + assert!(p.get("e").is_some()); + assert!(p.get("big").is_none()); + assert_eq!(p.size(), 85); + // The same package under `Refuse` is an error at the first part that does not fit. + let refuse = Limits { oversized: Oversized::Refuse, ..skip(10_000) }; + assert_eq!(Package::open(&z, &refuse).err(), Some(Error::PackageTooLarge { limit: 100 })); +} + +#[test] +fn backslash_zip_entry_names_are_found() { + let z = zip_of(&[("PPT\\Slides\\Slide1.xml", b"".to_vec()), ("/docProps/core.xml", b"".to_vec())]); + let p = Package::open(&z, &LIMITS).unwrap(); + assert!(p.get("ppt/slides/slide1.xml").is_some()); + assert_eq!(p.canonical("ppt/slides/slide1.xml"), Some("PPT/Slides/Slide1.xml")); + assert_eq!(p.canonical("docprops/core.xml"), Some("docProps/core.xml")); + let mut lazy = LazyPackage::open(&z, &LIMITS).unwrap(); + assert_eq!(lazy.names(), ["docprops/core.xml", "ppt/slides/slide1.xml"]); + assert_eq!(lazy.canonical("ppt/slides/slide1.xml").as_deref(), Some("PPT/Slides/Slide1.xml")); + assert_eq!(lazy.canonical("/DOCPROPS/core.xml").as_deref(), Some("docProps/core.xml")); + assert_eq!(lazy.canonical("nothing"), None); + assert_eq!(lazy.read("PPT/slides/slide1.xml").unwrap().as_deref(), Some(&b""[..])); +} + +#[test] +fn xml_parts_are_read_with_the_callers_options() { + let z = zip_of(&[("a.xml", b"".to_vec()), ("bad.xml", b"".to_vec())]); + let p = Package::open(&z, &LIMITS).unwrap(); + let local = craft_xml::Options::local(); + assert_eq!(p.xml("a.xml", &local).unwrap().map(|e| e.name), Some("a".to_string())); + assert_eq!(p.xml("a.xml", &craft_xml::Options::default()).unwrap().map(|e| e.name), Some("w:a".to_string())); + assert_eq!(p.xml("missing.xml", &local), Ok(None)); + assert!(p.xml("bad.xml", &local).unwrap().is_some(), "lenient"); + assert!(matches!(p.xml("bad.xml", &local.strict()), Err(Error::Xml { part, .. }) if part == "bad.xml")); + let tiny = local.with_limits(craft_xml::Limits { max_nodes: 1, ..craft_xml::Limits::DEFAULT }); + assert!(p.xml("a.xml", &tiny).is_err_and(|e| e.is_limit())); + let mut lazy = LazyPackage::open(&z, &LIMITS).unwrap(); + assert_eq!(lazy.read_xml("a.xml", &local).unwrap().map(|e| e.name), Some("a".to_string())); + assert!(lazy.read_xml("bad.xml", &local.strict()).is_err()); +} + +#[test] +fn from_parts_behaves_like_an_opened_package() { + let mut parts = BTreeMap::new(); + parts.insert("/word/document.xml".to_string(), b"".to_vec()); + parts.insert( + "word/_rels/document.xml.rels".to_string(), + br#""#.to_vec(), + ); + parts.insert("word/styles.xml".to_string(), b"".to_vec()); + let p = Package::from_parts(parts); + assert_eq!(p.get("word/document.xml"), Some(&b""[..])); + assert_eq!(p.rels("word/document.xml").first("styles").map(|r| r.target.as_str()), Some("word/styles.xml"), "the stored spelling"); + assert!(p.rels("word/styles.xml").list.is_empty()); + assert_eq!(p.content_types().of("word/document.xml"), None); +} + +/// An over-budget read must spend the package's allowance once and for all: loops over many +/// broken parts otherwise re-decompress the remaining allowance again and again. +#[test] +fn a_failed_lazy_read_ends_the_package_budget() { + let z = zip_of(&[("a.xml", b"0123456789".to_vec()), ("b.xml", b"hello".to_vec())]); + let mut pkg = LazyPackage::open(&z, &limits(100, 12)).unwrap(); + assert!(pkg.has("A.xml") && !pkg.has("c.xml")); + assert_eq!(pkg.read("c.xml"), Ok(None)); + assert!(pkg.read("a.xml").unwrap().is_some()); + assert_eq!(pkg.spent(), 10); + assert_eq!(pkg.read("b.xml"), Err(Error::PackageTooLarge { limit: 12 })); + assert_eq!(pkg.spent(), 12, "the failed read is charged"); + assert_eq!(pkg.read("b.xml"), Err(Error::PackageTooLarge { limit: 12 })); + // The per-read maximum is its own error and costs nothing: the declared size is enough. + let mut pkg = LazyPackage::open(&z, &limits(100, 100)).unwrap(); + assert_eq!(pkg.read_limited("a.xml", 4), Err(Error::PartTooLarge { part: "a.xml".into(), limit: 4 })); + assert_eq!(pkg.spent(), 0); + assert_eq!(pkg.read_limited("b.xml", 5).unwrap().as_deref(), Some(&b"hello"[..])); +} + +/// A part the reader can do without, damaged inside an intact archive, costs its own +/// allowance: the parts after it are still read. +#[test] +fn a_damaged_optional_part_costs_only_its_allowance() { + let mut w = ZipOut::new(); + w.add("a.xml", b"damaged", Compression::Store).unwrap(); + w.add("b.xml", b"hello", Compression::Deflate).unwrap(); + w.add( + "xl/_rels/workbook.xml.rels", + b"", + Compression::Deflate, + ) + .unwrap(); + let mut z = w.finish().unwrap(); + let at = z.windows(7).position(|x| x == b"damaged").unwrap(); + z[at] ^= 0x20; + let max_total = 1 << 20; + let mut pkg = LazyPackage::open(&z, &limits(1 << 16, max_total)).unwrap(); + assert!(matches!(pkg.read_optional_limited("a.xml", 1024), Err(Error::Zip(_)))); + assert_eq!(pkg.spent(), 1025, "the failed read is charged its allowance"); + assert_eq!(pkg.read("b.xml").unwrap().as_deref(), Some(&b"hello"[..])); + let rels = pkg.rels("xl/workbook.xml").unwrap(); + assert_eq!(rels.first("worksheet").map(|r| (r.target.as_str(), r.kind())), Some(("B.XML", "worksheet"))); + assert!(pkg.rels("xl/styles.xml").unwrap().list.is_empty()); + // An ordinary read of it still ends the package's budget. + assert!(pkg.read("a.xml").is_err()); + assert_eq!(pkg.spent(), max_total); +} + +#[test] +fn relationships_being_written_reuse_identical_targets() { + let n = 5_000; + let mut out = RelsOut::default(); + let ids: Vec = (0..n).map(|i| out.add("hyperlink", &format!("https://example.com/{i}"), true)).collect(); + assert_eq!(out.add("hyperlink", "https://example.com/7", true), ids[7], "an identical target reuses its id"); + assert_eq!(out.add("hyperlink", "https://example.com/7", false), format!("rId{}", n + 1), "internal is distinct"); + let fresh = out.add_new("image", "../media/a.png", false); + assert_ne!(out.add_new("image", "../media/a.png", false), fresh); + assert_eq!(out.add("image", "../media/a.png", false), fresh, "add reuses the first add_new made"); + assert_eq!(out.add("http://schemas.microsoft.com/office/2007/relationships/media", "a&b.mp4", false), format!("rId{}", n + 4)); + let back = craft_opc::Rels::parse(&out.to_xml(), "ppt/slides/slide1.xml"); + assert_eq!(back.list.len(), n + 4); + assert_eq!(back.by_id(&fresh).map(|r| (r.ty.clone(), r.target.as_str())), Some((format!("{RT_BASE}image"), "ppt/media/a.png"))); + assert_eq!(back.first("media").map(|r| r.target_raw.as_str()), Some("a&b.mp4"), "escaped on the way out"); + assert!(back.by_id("rId8").is_some_and(|r| r.external)); +} + +#[test] +fn a_written_package_reads_back() { + let mut out = PackageOut::default(); + out.xml_part("ppt/presentation.xml", "application/presentation+xml", b"

".to_vec()); + out.binary("ppt/media/image1.png", "image/png", b"png".to_vec()); + out.binary("ppt/media/image2.png", "image/png", b"png2".to_vec()); + out.binary("ppt/media/odd.png", "image/x-odd", b"odd".to_vec()); + out.raw("docProps/thumbnail.xml", b"".to_vec()); + out.xml_part("PPT/Presentation.xml", "second", b"".to_vec()); + let mut rels = RelsOut::default(); + let id = rels.add("image", "media/image1.png", false); + out.rels("ppt/presentation.xml", &RelsOut::default()); + out.rels("ppt/presentation.xml", &rels); + rels.add("image", "media/image2.png", false); + out.rels("ppt/presentation.xml", &rels); + let mut root = RelsOut::default(); + root.add("officeDocument", "ppt/presentation.xml", false); + out.rels("", &root); + let bytes = out.finish().unwrap(); + + let p = Package::open(&bytes, &LIMITS).unwrap(); + assert_eq!(p.names().next(), Some("[Content_Types].xml"), "names sort it first; it is also the first entry"); + assert!(bytes.windows(19).position(|w| w == b"[Content_Types].xml").is_some_and(|at| at < 64)); + assert_eq!(p.get("ppt/presentation.xml"), Some(&b"

"[..]), "the first part under a name is the one written"); + let ct = p.content_types(); + assert_eq!(ct.of("ppt/presentation.xml"), Some("application/presentation+xml")); + assert_eq!(ct.of("ppt/media/image2.png"), Some("image/png")); + assert_eq!(ct.of("ppt/media/odd.png"), Some("image/x-odd"), "an Override where the extension is taken"); + assert_eq!(ct.of("ppt/_rels/presentation.xml.rels"), Some(craft_opc::CT_RELATIONSHIPS)); + assert_eq!(ct.of("docProps/thumbnail.xml"), Some("application/xml")); + let main = p.rels("").first("officeDocument").map(|r| r.target.clone()).unwrap(); + let r = p.rels(&main); + assert_eq!(r.target(&id), Some("ppt/media/image1.png")); + assert_eq!(r.list.len(), 2, "the relationships part was replaced, not written twice"); + // Writing the same package again gives the same bytes. + assert_eq!(zip_entries(&[("a.xml".into(), b"".to_vec())]), zip_entries(&[("a.xml".into(), b"".to_vec())])); +} + +#[test] +fn packed_media_are_stored() { + assert_eq!(compression_for("word/media/image1.PNG"), Compression::Store); + assert_eq!(compression_for("ppt/media/movie.mp4"), Compression::Store); + assert_eq!(compression_for("word/document.xml"), Compression::Deflate); + assert_eq!(compression_for("word/media/image1.emf"), Compression::Deflate); + assert_eq!(compression_for("noextension"), Compression::Deflate); + let data = vec![b'a'; 100_000]; + let stored = zip_entries(&[("a.png".into(), data.clone())]).unwrap(); + let deflated = zip_entries(&[("a.xml".into(), data.clone())]).unwrap(); + assert!(stored.len() > 100_000 && deflated.len() < 10_000); + for z in [stored, deflated] { + let p = Package::open(&z, &LIMITS).unwrap(); + assert_eq!(p.size(), 100_000); + } +} + +/// Two entries of exactly one name: `zip` keeps one. This crate is lenient about part names +/// (spellings that differ only in case are one part), so the package still opens. +#[test] +fn an_entry_name_repeated_exactly_still_opens() { + let mut z = zip_of(&[("word/a.xml", b"".to_vec()), ("word/b.xml", b"".to_vec())]); + let mut i = 0; + while let Some(at) = z[i..].windows(10).position(|w| w == b"word/b.xml") { + z[i + at + 5] = b'a'; + i += at + 10; + } + assert_eq!(Package::open(&z, &LIMITS).unwrap().names().collect::>(), ["word/a.xml"]); + assert!(LazyPackage::open(&z, &LIMITS).unwrap().read("word/a.xml").unwrap().is_some()); +} diff --git a/crates/openpgp/Cargo.toml b/crates/openpgp/Cargo.toml new file mode 100644 index 000000000..5630c3829 --- /dev/null +++ b/crates/openpgp/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "craft-openpgp" +description = "OpenPGP for mail: PGP/MIME and cleartext Digital signatures checked, attached Public keys and Autocrypt headers read, a secret key file inspected; no I/O of its own." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +# rPGP: the OpenPGP packets and the cryptography (MIT OR Apache-2.0). Pinned: every size cap in +# this crate was written against this version's parser. +pgp = { version = "=0.21.0", default-features = false } +thiserror = { workspace = true } +# rPGP's generation of RustCrypto digests (MIT OR Apache-2.0). +sha2 = { version = "0.10", default-features = false } +sha3 = { version = "0.10", default-features = false } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +# Random numbers in the browser, which rPGP's dependencies ask for even when it only checks. +pgp = { version = "=0.21.0", default-features = false, features = ["wasm"] } + +[dev-dependencies] +proptest = { workspace = true } +# Keys and signatures made in the tests, from a seed: the random numbers rPGP 0.21 takes. +rand = { version = "0.8", default-features = false, features = ["std", "std_rng"] } +sha1 = { version = "0.10", default-features = false } +ed25519-dalek = { version = "2", default-features = false } +rsa = { version = "0.9", default-features = false } diff --git a/crates/openpgp/README.md b/crates/openpgp/README.md new file mode 100644 index 000000000..815432105 --- /dev/null +++ b/crates/openpgp/README.md @@ -0,0 +1,31 @@ +# craft-openpgp + +OpenPGP as mail carries it: Digital signatures checked and Public keys read. No I/O, no clock and +no keys of its own. The caller hands in the message's bytes, the Public keys it keeps, the time, +and whether the owner has verified a key. + +- **`verify`** checks a PGP/MIME message (`multipart/signed; + protocol="application/pgp-signature"`, RFC 3156 §5) over the signed part's canonical bytes, or a + cleartext-signed text (RFC 9580 §7), and gives an `Outcome` (verified, valid but not verified, + bad, or can't be checked) with what the signature covers (`Coverage`). MD5, SHA-1 and + RIPEMD-160 signatures are refused as no longer safe. +- **`keys_in_message`** reads the keys a message carries: attached `application/pgp-keys` parts + and the `Autocrypt:` header (Autocrypt Level 1). Nothing is fetched or sent. +- **`PublicKey`, `KeyRing`, `Fingerprint`, `KeyId`**: transferable public keys (armoured or + binary) with their self-signatures, expiry and revocations checked. +- **`parse_autocrypt`**: one `Autocrypt:` header value. +- **`secret::read`, `secret::holds_secret`**: what a transferable secret key file is (its + fingerprint, addresses, public half, whether every secret part has a passphrase, whether it can + sign), without keeping or using the secret. + +The packets and the cryptography are [rPGP](https://github.com/rpgp/rpgp)'s (`pgp`, MIT OR +Apache-2.0). The armour, the Cleartext Signature Framework and the little MIME needed to find the +signed bytes exactly as they came are read here, from RFC 2045/2046, RFC 3156 and RFC 9580, so +that every size is capped by this crate. + +Everything from outside is hostile: messages past `MAX_MESSAGE` are not read, signatures, keys, +packets, user IDs, subkeys, MIME parts, nesting and the number of signature checks per call are +capped, work is linear in the input, and a panic inside rPGP is caught and reported as an +unreadable signature. + +Moved from MailCraft's `mailcraft-openpgp`. diff --git a/crates/openpgp/proptest-regressions/prop_tests.txt b/crates/openpgp/proptest-regressions/prop_tests.txt new file mode 100644 index 000000000..76b451790 --- /dev/null +++ b/crates/openpgp/proptest-regressions/prop_tests.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 9769ed1f0f9ff8e8afbdb639c2d3a0bad49d0a89c70a977a791d7b53ca178cea # shrinks to bytes = [45, 45, 45, 45, 45, 66, 69, 71, 73, 78, 32, 80, 71, 80, 32, 83, 73, 71, 78, 65, 84, 85, 82, 69, 45, 45, 45, 45, 45, 13, 10, 45, 45, 45, 45, 45, 66, 69, 71, 73, 78, 32, 80, 71, 80, 32, 80, 85, 66, 76, 73, 67, 32, 75, 69, 89, 32, 66, 76, 79, 67, 75, 45, 45, 45, 45, 45, 13, 10, 45, 45, 45, 45, 45, 69, 78, 68, 32, 80, 71, 80, 32, 83, 73, 71, 78, 65, 84, 85, 82, 69, 45, 45, 45, 45, 45, 13, 10], limit = 27, max = 0 diff --git a/crates/openpgp/src/armour.rs b/crates/openpgp/src/armour.rs new file mode 100644 index 000000000..64ba26d57 --- /dev/null +++ b/crates/openpgp/src/armour.rs @@ -0,0 +1,236 @@ +//! OpenPGP armour (RFC 9580 §6.2) and the Cleartext Signature Framework (RFC 9580 §7), read +//! here rather than in rPGP so that every size is ours to cap: the lines of a block are found, +//! its base64 is decoded up to a limit, and a cleartext-signed text is given back as the bytes +//! that were signed. Linear in the input; nothing here fails on any bytes. + +use crate::mime; + +/// The kinds of armour block this crate reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Signature, + PublicKey, + PrivateKey, +} + +impl Kind { + fn label(self) -> &'static [u8] { + match self { + Kind::Signature => b"SIGNATURE", + Kind::PublicKey => b"PUBLIC KEY BLOCK", + Kind::PrivateKey => b"PRIVATE KEY BLOCK", + } + } +} + +/// One line of `text`: its bytes without the line break, and without the white space it ends +/// in as well. +struct Line<'a> { + raw: &'a [u8], + text: &'a [u8], + /// Where it starts in the text, and where the next line starts. + start: usize, + next: usize, +} + +fn lines(text: &[u8]) -> impl Iterator> { + let mut at = 0usize; + std::iter::from_fn(move || { + if at >= text.len() { + return None; + } + let rest = text.get(at..).unwrap_or_default(); + let len = rest.iter().position(|b| *b == b'\n'); + let raw = rest.get(..len.unwrap_or(rest.len())).unwrap_or_default(); + let start = at; + at = len.map_or(text.len(), |n| at.saturating_add(n).saturating_add(1)); + let raw = raw.strip_suffix(b"\r").unwrap_or(raw); + Some(Line { raw, text: trim_end(raw), start, next: at }) + }) +} + +/// `line` without the spaces, tabs and CR it ends in. +pub(crate) fn trim_end(line: &[u8]) -> &[u8] { + let end = line.iter().rposition(|b| !matches!(b, b' ' | b'\t' | b'\r')).map_or(0, |i| i.saturating_add(1)); + line.get(..end).unwrap_or_default() +} + +fn marker(edge: &[u8], label: &[u8]) -> Vec { + let mut m = Vec::with_capacity(label.len().saturating_add(20)); + m.extend_from_slice(b"-----"); + m.extend_from_slice(edge); + m.extend_from_slice(b" PGP "); + m.extend_from_slice(label); + m.extend_from_slice(b"-----"); + m +} + +/// The decoded contents of every armour block of `kind` in `text`, at most `max_blocks`, each +/// at most `limit` bytes (a larger one is skipped). A byte-order mark before a line is allowed. +pub(crate) fn blocks(text: &[u8], kind: Kind, max_blocks: usize, limit: usize) -> Vec> { + let begin = marker(b"BEGIN", kind.label()); + let end = marker(b"END", kind.label()); + let mut out = Vec::new(); + if max_blocks == 0 { + return out; + } + let mut inside: Option = None; + for line in lines(text) { + let line_text = line.text.strip_prefix(b"\xef\xbb\xbf").unwrap_or(line.text); + match inside.as_mut() { + None if line_text == begin.as_slice() => inside = Some(Body::new(limit)), + None => {} + Some(body) if line_text == end.as_slice() => { + if let Some(bytes) = body.finish() { + out.push(bytes); + if out.len() >= max_blocks { + return out; + } + } + inside = None; + } + Some(body) if line_text == begin.as_slice() => *body = Body::new(limit), + Some(body) => body.line(line_text), + } + } + out +} + +/// The base64 of one block as its lines come: armour headers until the first empty line, then +/// data until the checksum line (`=` and four characters), which RFC 9580 lets a reader ignore. +struct Body { + in_headers: bool, + data: Vec, + limit: usize, + too_large: bool, + done: bool, +} + +impl Body { + fn new(limit: usize) -> Body { + Body { in_headers: true, data: Vec::new(), limit, too_large: false, done: false } + } + + fn line(&mut self, line: &[u8]) { + if self.done || self.too_large { + return; + } + if self.in_headers { + if line.is_empty() { + self.in_headers = false; + return; + } + // Armour without headers or the empty line after them: data from the first line. + if line.contains(&b':') { + return; + } + self.in_headers = false; + } + if line.first() == Some(&b'=') { + self.done = true; + return; + } + // Four characters of base64 are three bytes: stop collecting well before the limit. + if self.data.len().saturating_add(line.len()) / 4 * 3 > self.limit { + self.too_large = true; + return; + } + self.data.extend_from_slice(line); + } + + fn finish(&mut self) -> Option> { + if self.too_large { + return None; + } + let bytes = mime::base64(&self.data); + (!bytes.is_empty() && bytes.len() <= self.limit).then_some(bytes) + } +} + +/// The hash names a `Hash:` armour header may give (RFC 9580 §7.1, the names of §9.5). +const HASH_NAMES: [&[u8]; 9] = [b"MD5", b"SHA1", b"RIPEMD160", b"SHA224", b"SHA256", b"SHA384", b"SHA512", b"SHA3-256", b"SHA3-512"]; + +/// Whether `line` is a `Hash:` header whose value is nothing but hash names, separated by +/// commas: any other words there would be shown and aren't signed. +fn hash_header(line: &[u8]) -> bool { + let Some(value) = line.strip_prefix(b"Hash:") else { return false }; + value.split(|b| *b == b',').all(|name| { + let name = name.trim_ascii(); + HASH_NAMES.iter().any(|known| name.eq_ignore_ascii_case(known)) + }) +} + +/// A cleartext-signed text (RFC 9580 §7): the bytes that were signed and the decoded +/// signature block after them. +#[derive(Debug)] +pub(crate) struct Cleartext { + /// The text as it was signed: dash-escaping undone, white space at each line's end removed, + /// lines joined by CR LF and no line break after the last one. + pub(crate) signed: Vec, + /// The signature packets, decoded from their armour. + pub(crate) signature: Vec, + /// Where the block is in the text: from its first line to the end of its last. + pub(crate) span: (usize, usize), + /// Nothing but white space around the block, and so no other block: the block is the text. + pub(crate) alone: bool, + /// Its armour headers are `Hash:` lines only, and no line of the signed text is unescaped + /// armour. One that isn't would show words that aren't signed, so it can't be checked. + pub(crate) well_formed: bool, +} + +/// The first cleartext-signed text in `text`, if there is a whole one: `-----BEGIN PGP SIGNED +/// MESSAGE-----`, its `Hash:` headers and an empty line, the text, and a signature block, which +/// may be at most `limit` bytes. +pub(crate) fn cleartext(text: &[u8], limit: usize) -> Option { + let begin = marker(b"BEGIN", b"SIGNED MESSAGE"); + let sig_begin = marker(b"BEGIN", Kind::Signature.label()); + let sig_end = marker(b"END", Kind::Signature.label()); + #[derive(PartialEq)] + enum At { + Before, + Headers, + Text, + Signature, + } + let mut at = At::Before; + let mut signed: Vec = Vec::new(); + let mut first_line = true; + let mut body = Body::new(limit); + let (mut start, mut alone, mut well_formed) = (0usize, true, true); + for line in lines(text) { + let line_text = line.text.strip_prefix(b"\xef\xbb\xbf").unwrap_or(line.text); + match at { + At::Before if line_text == begin.as_slice() => { + at = At::Headers; + start = line.start; + } + At::Before => alone &= line_text.iter().all(u8::is_ascii_whitespace), + // RFC 9580 §7.1: `Hash:` lines (or none, in newer writers), then an empty line. + // Anything else there would be shown and isn't signed. + At::Headers if line_text.is_empty() => at = At::Text, + At::Headers => well_formed &= hash_header(line_text), + At::Text if line_text == sig_begin.as_slice() => at = At::Signature, + At::Text => { + if !first_line { + signed.extend_from_slice(b"\r\n"); + } + first_line = false; + // Dash-escaping undone (§7.1), then the white space the line ends in removed + // (§7.2). Every line that starts with a dash is escaped: one that isn't is + // armour where text belongs. + let raw = line.raw.strip_prefix(b"\xef\xbb\xbf").unwrap_or(line.raw); + if raw.starts_with(b"-") && !raw.starts_with(b"- ") { + well_formed = false; + } + signed.extend_from_slice(trim_end(raw.strip_prefix(b"- ").unwrap_or(raw))); + } + At::Signature if line_text == sig_end.as_slice() => { + let signature = body.finish()?; + alone &= text.get(line.next..).unwrap_or_default().iter().all(u8::is_ascii_whitespace); + return Some(Cleartext { signed, signature, span: (start, line.next), alone, well_formed }); + } + At::Signature => body.line(line_text), + } + } + None +} diff --git a/crates/openpgp/src/armour_tests.rs b/crates/openpgp/src/armour_tests.rs new file mode 100644 index 000000000..d5df00292 --- /dev/null +++ b/crates/openpgp/src/armour_tests.rs @@ -0,0 +1,38 @@ +use crate::armour::{Kind, blocks, cleartext}; + +#[test] +fn a_block_is_decoded_without_its_headers_or_checksum() { + let text = + b"before\r\n-----BEGIN PGP SIGNATURE-----\r\nComment: made by hand\r\n\r\nSGVs\r\nbG8=\r\n=AAAA\r\n-----END PGP SIGNATURE-----\r\nafter"; + assert_eq!(blocks(text, Kind::Signature, 4, 100), vec![b"Hello".to_vec()]); + // Armour with no headers and no empty line after the first line. + let text = b"-----BEGIN PGP SIGNATURE-----\nSGVsbG8=\n-----END PGP SIGNATURE-----\n"; + assert_eq!(blocks(text, Kind::Signature, 4, 100), vec![b"Hello".to_vec()]); +} + +#[test] +fn blocks_are_capped_in_size_and_number() { + let block = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nSGVsbG8=\n-----END PGP PUBLIC KEY BLOCK-----\n"; + let many = block.repeat(10); + assert_eq!(blocks(many.as_bytes(), Kind::PublicKey, 3, 100).len(), 3); + assert!(blocks(block.as_bytes(), Kind::PublicKey, 3, 4).is_empty()); + // Another kind of block, an unclosed one, and an empty one are not blocks. + assert!(blocks(block.as_bytes(), Kind::Signature, 3, 100).is_empty()); + assert!(blocks(b"-----BEGIN PGP SIGNATURE-----\n\nSGVsbG8=\n", Kind::Signature, 3, 100).is_empty()); + assert!(blocks(b"-----BEGIN PGP SIGNATURE-----\n\n-----END PGP SIGNATURE-----\n", Kind::Signature, 3, 100).is_empty()); +} + +#[test] +fn cleartext_undoes_dash_escaping_and_trailing_white_space() { + let text = b"Intro, not signed\r\n-----BEGIN PGP SIGNED MESSAGE-----\r\nHash: SHA256\r\n\r\nline one \r\n- -dashes\r\n\r\nlast\t\r\n-----BEGIN PGP SIGNATURE-----\r\n\r\nSGVsbG8=\r\n-----END PGP SIGNATURE-----\r\ntrailer"; + let clear = cleartext(text, 100).unwrap(); + assert_eq!(clear.signed, b"line one\r\n-dashes\r\n\r\nlast"); + assert_eq!(clear.signature, b"Hello"); +} + +#[test] +fn an_incomplete_cleartext_is_none() { + assert!(cleartext(b"-----BEGIN PGP SIGNED MESSAGE-----\n\ntext\n", 100).is_none()); + assert!(cleartext(b"-----BEGIN PGP SIGNED MESSAGE-----\n\ntext\n-----BEGIN PGP SIGNATURE-----\n\nSGVsbG8=\n", 100).is_none()); + assert!(cleartext(b"plain text", 100).is_none()); +} diff --git a/crates/openpgp/src/autocrypt.rs b/crates/openpgp/src/autocrypt.rs new file mode 100644 index 000000000..99f4c99c8 --- /dev/null +++ b/crates/openpgp/src/autocrypt.rs @@ -0,0 +1,73 @@ +//! The `Autocrypt:` header (Autocrypt Level 1, §2.1): a sender's Public key carried in every +//! message, for the address it came from. Read, never acted on here: the caller keeps the key +//! for that address, and nothing is sent. + +use crate::keys::{Budget, MAX_CHECKS_PER_CALL, MAX_KEY_BYTES, PublicKey}; +use crate::{Error, mime}; + +/// How long a header value may be: a key for Autocrypt is meant to be small (Level 1 §2.1 +/// recommends one signing and one encryption key, a few KB), and the cap leaves room. +pub(crate) const MAX_HEADER: usize = 64 * 1024; + +/// An `Autocrypt:` header that reads: whose it is, what they prefer, and their key. +#[derive(Clone, Debug)] +pub struct Autocrypt { + /// The address it is for (`addr`), lower case. + pub address: String, + /// `prefer-encrypt=mutual`: the sender wants encrypted replies when both sides can. + pub prefer_encrypt_mutual: bool, + /// The key (`keydata`), its self-signatures checked. + pub key: PublicKey, +} + +/// Reads one `Autocrypt:` header value. [`Error::Unreadable`] when it lacks `addr` or `keydata`, +/// has an attribute it doesn't know that doesn't start with `_` (which Level 1 says makes the +/// whole header one to ignore), or carries no key that checks; [`Error::TooLarge`] when it is +/// longer than a key for Autocrypt needs. +pub fn parse_autocrypt(value: &str) -> Result { + attributes(value)?.read(&mut Budget::new(MAX_CHECKS_PER_CALL)) +} + +/// A header's attributes, read before its key is: the key is the costly part. +pub(crate) struct Attributes<'a> { + pub(crate) address: String, + mutual: bool, + keydata: &'a str, +} + +/// The attributes of an `Autocrypt:` header value, as [`parse_autocrypt`] reads them. +pub(crate) fn attributes(value: &str) -> Result, Error> { + if value.len() > MAX_HEADER { + return Err(Error::TooLarge); + } + let (mut address, mut keydata, mut mutual) = (None, None, false); + for attribute in value.split(';') { + let attribute = attribute.trim(); + if attribute.is_empty() { + continue; + } + let (name, data) = attribute.split_once('=').ok_or(Error::Unreadable)?; + match name.trim() { + "addr" if address.is_none() => address = Some(mime::address(data.trim())), + "keydata" if keydata.is_none() => keydata = Some(data), + "prefer-encrypt" => mutual = data.trim() == "mutual", + name if name.starts_with('_') => {} + _ => return Err(Error::Unreadable), + } + } + let address = address.filter(|a| !a.is_empty()).ok_or(Error::Unreadable)?; + Ok(Attributes { address, mutual, keydata: keydata.ok_or(Error::Unreadable)? }) +} + +impl Attributes<'_> { + /// The header with its key read. + pub(crate) fn read(self, total: &mut Budget) -> Result { + let bytes = mime::base64(self.keydata.as_bytes()); + if bytes.len() > MAX_KEY_BYTES { + return Err(Error::TooLarge); + } + // Level 1 §2.1: the keydata is one Transferable Public Key. + let key = PublicKey::binary(&bytes, 1, total).into_iter().next().ok_or(Error::Unreadable)?; + Ok(Autocrypt { address: self.address, prefer_encrypt_mutual: self.mutual, key }) + } +} diff --git a/crates/openpgp/src/autocrypt_tests.rs b/crates/openpgp/src/autocrypt_tests.rs new file mode 100644 index 000000000..813a0a019 --- /dev/null +++ b/crates/openpgp/src/autocrypt_tests.rs @@ -0,0 +1,23 @@ +use crate::autocrypt::MAX_HEADER; +use crate::{Error, parse_autocrypt}; + +#[test] +fn a_header_without_addr_or_keydata_is_unreadable() { + assert_eq!(parse_autocrypt("keydata=AAAA").unwrap_err(), Error::Unreadable); + assert_eq!(parse_autocrypt("addr=a@example.org").unwrap_err(), Error::Unreadable); + assert_eq!(parse_autocrypt("").unwrap_err(), Error::Unreadable); +} + +#[test] +fn an_unknown_critical_attribute_makes_the_header_unreadable() { + assert_eq!(parse_autocrypt("addr=a@example.org; colour=blue; keydata=AAAA").unwrap_err(), Error::Unreadable); + assert_eq!(parse_autocrypt("addr=a@example.org; novalue; keydata=AAAA").unwrap_err(), Error::Unreadable); + // One starting with `_` is skipped; the keydata here is no key, so it still doesn't read. + assert_eq!(parse_autocrypt("addr=a@example.org; _colour=blue; keydata=AAAA").unwrap_err(), Error::Unreadable); +} + +#[test] +fn a_header_too_long_is_refused_before_it_is_read() { + let long = format!("addr=a@example.org; keydata={}", "A".repeat(MAX_HEADER)); + assert_eq!(parse_autocrypt(&long).unwrap_err(), Error::TooLarge); +} diff --git a/crates/openpgp/src/check.rs b/crates/openpgp/src/check.rs new file mode 100644 index 000000000..3b54eed21 --- /dev/null +++ b/crates/openpgp/src/check.rs @@ -0,0 +1,462 @@ +//! Checking one Digital signature over bytes, and what came of it. +//! +//! Only what a signature's hashed area says is believed: its issuer there may be blamed for a +//! signature that doesn't match, and its unhashed area (which anybody can change) is a hint to +//! find a key that then has to fit, never a reason for a result. What was signed is hashed once +//! for each hash, kind and salt among the signatures, whatever the number of keys tried. + +use std::cell::Cell; + +use pgp::composed::{Deserializable, DetachedSignature}; +use pgp::crypto::hash::HashAlgorithm; +use pgp::crypto::public_key::PublicKeyAlgorithm; +use pgp::line_writer::LineBreak; +use pgp::normalize_lines::NormalizedReader; +use pgp::packet::{Signature, SignatureConfig, SignatureType, SignatureVersion, SignatureVersionSpecific, Subpacket, SubpacketData}; +use pgp::types::{KeyVersion, SignatureBytes, VerifyingKey}; +use sha2::digest::{Digest, DynDigest}; + +use crate::keys::{self, Fingerprint, KeyId, KeyRing, PublicKey, Signing}; + +/// How many signatures over one text are looked at. +pub(crate) const MAX_SIGNATURES: usize = 8; +/// How many bytes a block of signatures may hold, decoded. +pub(crate) const MAX_SIGNATURE_BYTES: usize = 64 * 1024; +/// How many keys one signature is tried with. +pub(crate) const MAX_CANDIDATES: usize = 8; +/// How many signature checks (public-key operations) one message may cost. +pub(crate) const MAX_VERIFICATIONS: usize = 24; + +/// Who made a Digital signature and when, for the details. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Signer { + /// The primary key's fingerprint: what the owner compares and marks Verified. + pub fingerprint: Fingerprint, + /// The fingerprint of the key that signed, when it is a subkey. + pub subkey: Option, + /// The key id of the key that signed. + pub key_id: KeyId, + /// The key's addresses (from its own User IDs), lower case. + pub addresses: Vec, + /// When the signature says it was made, in seconds since 1970. + pub signed: u64, + /// When the primary key was made, in seconds since 1970. + pub key_created: u64, + /// The signing key's algorithm ("RSA", "Ed25519", …). + pub algorithm: &'static str, + /// The signature's hash ("SHA-256", …). + pub hash: &'static str, +} + +/// Who a signature that couldn't be checked names as its signer in its hashed area, if it does. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Issuer { + pub fingerprint: Option, + pub key_id: Option, +} + +/// Why a valid Digital signature isn't Verified: one or more of these. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct NotVerified { + /// The owner hasn't Verified this key for the From address (marked it after comparing the + /// fingerprint, or their GnuPG keyring trusts it fully). + pub key_unverified: bool, + /// The From address isn't one of the key's own (when the message was signed). + pub address_differs: bool, + /// The signature covers only part of the message ([`Coverage`]). + pub partly_signed: bool, +} + +/// Why a Digital signature can't be checked. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CantCheck { + /// No Public key here for the key it names, nor any for the From address. + NoPublicKey, + /// None of the Public keys here for the From address made it, and the key it names (if + /// any) isn't here. + NoMatchingKey, + /// Made with an algorithm that is no longer safe: a hash (MD5, SHA-1, RIPEMD-160) or a key + /// (DSA, RSA under 2048 bits). Its name. + WeakAlgorithm(&'static str), + /// An algorithm or signature version this program doesn't know (or one before 4). + UnknownAlgorithm, + /// The signing key (or its primary key) was revoked by its owner, for this signature. + KeyRevoked, + /// The signing key (or its primary key) had expired when the signature was made. + KeyExpired, + /// The key that matches isn't one that may sign (or its subkey didn't sign back that it + /// belongs to the key). + NotASigningKey, + /// The signature itself has expired. + SignatureExpired, + /// The message is larger than any looked at: it is signed, but isn't checked. + TooLarge, + /// The signature, or what it is over, can't be read, or is malformed. + Unreadable, +} + +/// What came of checking a message's OpenPGP Digital signature: the four results. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// Valid, the key Verified for the From address, which is the key's, over the whole message. + Verified(Signer), + /// Valid, but not Verified: why. + Valid { signer: Signer, why: NotVerified }, + /// The message was changed after it was signed: the key its hashed area names is here and + /// doesn't match it. + Changed(Signer), + /// It can't be checked, and why; who its hashed area names as its signer. + CantCheck { why: CantCheck, issuer: Issuer }, +} + +impl Outcome { + /// Which outcome to show of several signatures over one text: the best. + fn rank(&self) -> u8 { + match self { + Outcome::Verified(_) => 3, + Outcome::Valid { .. } => 2, + Outcome::Changed(_) => 1, + Outcome::CantCheck { .. } => 0, + } + } + + pub(crate) fn cant(why: CantCheck) -> Outcome { + Outcome::CantCheck { why, issuer: Issuer::default() } + } +} + +/// What of a message a Digital signature covers. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Coverage { + /// Whether it is the whole message: a PGP/MIME message at the top, or a plain text body that + /// is one cleartext-signed block with nothing but white space around it. Only then can a + /// message be Verified. + pub whole: bool, + /// The MIME part signed, as the place of each part on the way down from the message (empty: + /// the message itself; `[0]`: its first part). + pub path: Vec, + /// For a cleartext-signed text: where the block is in that part's text (decoded from its + /// transfer encoding), from its first line to the end of its last. + pub span: Option<(usize, usize)>, +} + +/// A message's Digital signature checked: what came of it, and what it covers. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Checked { + pub outcome: Outcome, + pub coverage: Coverage, +} + +/// What the caller hands in to check against, and what checking has cost so far. +pub(crate) struct Context<'a> { + /// The From address, lower case; empty when the From field doesn't hold exactly one. + pub(crate) from: String, + pub(crate) keys: &'a KeyRing, + pub(crate) now: u64, + pub(crate) verified: &'a dyn Fn(&Fingerprint, &str) -> bool, + /// Public-key operations made. + pub(crate) verifications: Cell, + /// Times what was signed was hashed. + pub(crate) data_passes: Cell, +} + +/// A hash over what was signed, before a signature's own fields: cloned for each signature. +#[derive(Clone)] +enum Base { + Sha224(sha2::Sha224), + Sha256(sha2::Sha256), + Sha384(sha2::Sha384), + Sha512(sha2::Sha512), + Sha3_256(sha3::Sha3_256), + Sha3_512(sha3::Sha3_512), +} + +impl Base { + fn new(hash: HashAlgorithm) -> Option { + Some(match hash { + HashAlgorithm::Sha224 => Base::Sha224(Default::default()), + HashAlgorithm::Sha256 => Base::Sha256(Default::default()), + HashAlgorithm::Sha384 => Base::Sha384(Default::default()), + HashAlgorithm::Sha512 => Base::Sha512(Default::default()), + HashAlgorithm::Sha3_256 => Base::Sha3_256(Default::default()), + HashAlgorithm::Sha3_512 => Base::Sha3_512(Default::default()), + _ => return None, + }) + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Base::Sha224(h) => Digest::update(h, bytes), + Base::Sha256(h) => Digest::update(h, bytes), + Base::Sha384(h) => Digest::update(h, bytes), + Base::Sha512(h) => Digest::update(h, bytes), + Base::Sha3_256(h) => Digest::update(h, bytes), + Base::Sha3_512(h) => Digest::update(h, bytes), + } + } + + fn boxed(&self) -> Box { + match self.clone() { + Base::Sha224(h) => Box::new(h), + Base::Sha256(h) => Box::new(h), + Base::Sha384(h) => Box::new(h), + Base::Sha512(h) => Box::new(h), + Base::Sha3_256(h) => Box::new(h), + Base::Sha3_512(h) => Box::new(h), + } + } +} + +/// The hashes over what was signed, one for each hash, kind (binary or text) and salt. +type Bases = Vec<((HashAlgorithm, bool, Vec), Base)>; + +/// Checks the signature packets `packets` (binary) over `data`; the best outcome of them. +pub(crate) fn check(packets: &[u8], data: &[u8], cx: &Context<'_>) -> Outcome { + if packets.len() > MAX_SIGNATURE_BYTES { + return Outcome::cant(CantCheck::Unreadable); + } + let parsed = crate::guard(|| DetachedSignature::from_bytes_many(packets).map(|it| it.take(MAX_SIGNATURES).collect::>())); + let Some(Ok(signatures)) = parsed else { return Outcome::cant(CantCheck::Unreadable) }; + let mut bases: Bases = Vec::new(); + let mut best: Option = None; + for signature in signatures.into_iter().filter_map(Result::ok) { + let outcome = crate::guard(|| one(&signature.signature, data, cx, &mut bases)).unwrap_or_else(|| Outcome::cant(CantCheck::Unreadable)); + if best.as_ref().is_none_or(|b| outcome.rank() > b.rank()) { + best = Some(outcome); + } + } + best.unwrap_or_else(|| Outcome::cant(CantCheck::Unreadable)) +} + +/// The issuers a list of subpackets names. +fn issuers(subpackets: &[Subpacket]) -> (Vec, Vec) { + let mut fps = Vec::new(); + let mut ids = Vec::new(); + for p in subpackets.iter().take(64) { + match &p.data { + SubpacketData::IssuerFingerprint(fp) => fps.push(Fingerprint::of(fp)), + SubpacketData::IssuerKeyId(id) => ids.push(KeyId::of(id)), + _ => {} + } + } + (fps, ids) +} + +/// The digest a signature signs: what was signed (hashed once per hash, kind and salt), then +/// its own fields and trailer (RFC 9580 §5.2.4). `None` when it can't be made. +fn digest(config: &SignatureConfig, data: &[u8], cx: &Context<'_>, bases: &mut Bases) -> Option> { + let text = config.typ == SignatureType::Text; + let salt = match &config.version_specific { + SignatureVersionSpecific::V6 { salt } => { + if config.hash_alg.salt_len() != Some(salt.len()) { + return None; + } + salt.clone() + } + _ => Vec::new(), + }; + let key = (config.hash_alg, text, salt); + let base = match bases.iter().find(|(k, _)| *k == key) { + Some((_, base)) => base.clone(), + None => { + let mut base = Base::new(config.hash_alg)?; + base.update(&key.2); + if text { + // RFC 9580 §5.2.1.2: a text signature is over the text with CR LF line ends. + let mut reader = NormalizedReader::new(data, LineBreak::Crlf); + let mut chunk = [0u8; 8192]; + loop { + let n = std::io::Read::read(&mut reader, &mut chunk).ok()?; + if n == 0 { + break; + } + base.update(chunk.get(..n)?); + } + } else { + base.update(data); + } + cx.data_passes.set(cx.data_passes.get().saturating_add(1)); + bases.push((key, base.clone())); + base + } + }; + let mut hasher = base.boxed(); + let len = config.hash_signature_data(&mut hasher).ok()?; + hasher.update(&config.trailer(len).ok()?); + Some(hasher.finalize().to_vec()) +} + +fn one(sig: &Signature, data: &[u8], cx: &Context<'_>, bases: &mut Bases) -> Outcome { + let Some(config) = sig.config() else { return Outcome::cant(CantCheck::UnknownAlgorithm) }; + let (fps, ids) = issuers(&config.hashed_subpackets); + let (hint_fps, hint_ids) = issuers(&config.unhashed_subpackets); + let issuer = Issuer { fingerprint: fps.first().cloned(), key_id: ids.first().copied() }; + let cant = |why: CantCheck| Outcome::CantCheck { why, issuer: issuer.clone() }; + // Version 3 signatures (and earlier) hash too little of themselves to be trusted. + if !matches!(config.version(), SignatureVersion::V4 | SignatureVersion::V6) { + return cant(CantCheck::UnknownAlgorithm); + } + if !matches!(config.typ, SignatureType::Binary | SignatureType::Text) { + return cant(CantCheck::Unreadable); + } + if keys::is_weak(config.hash_alg) { + return cant(CantCheck::WeakAlgorithm(keys::hash_name(config.hash_alg))); + } + if config.pub_alg == PublicKeyAlgorithm::DSA { + return cant(CantCheck::WeakAlgorithm("DSA")); + } + if !known_signature_algorithm(config.pub_alg) { + return cant(CantCheck::UnknownAlgorithm); + } + // RFC 9580 §5.2.3.11: a signature says when it was made, in its hashed area. + let Some(signed) = config.created().map(|t| u64::from(t.as_secs())) else { return cant(CantCheck::Unreadable) }; + // A signature whose value isn't of the algorithm's shape is malformed, never "changed". + let Some(sig_bytes) = sig.signature().filter(|b| well_shaped(config.pub_alg, b)) else { return cant(CantCheck::Unreadable) }; + let Some(digest) = digest(config, data, cx, bases) else { return cant(CantCheck::UnknownAlgorithm) }; + // The first two bytes of the digest are in the signature: a quick test that it is over this. + let fits = sig.signed_hash_value().is_some_and(|v| digest.get(..2) == Some(&v[..])); + + // The keys to try, at most MAX_CANDIDATES: those the hashed area names; then those the + // unhashed area names (a hint: only a key that fits counts), before any other key of the + // From address, so that keys claiming the address can't push the signer's out; then the + // From address's keys. + let mut candidates: Vec<(&PublicKey, &Signing, bool)> = Vec::new(); + for (k, s) in cx.keys.signers(&fps, &ids) { + push(&mut candidates, k, s, true); + } + for (k, s) in cx.keys.signers(&hint_fps, &hint_ids) { + push(&mut candidates, k, s, false); + } + let from_keys: Vec<&PublicKey> = cx.keys.for_address(&cx.from).collect(); + for k in &from_keys { + for s in &k.signing { + push(&mut candidates, k, s, false); + } + } + + let mut blamed: Option = None; + let mut weak: Option<&'static str> = None; + let mut starved = false; + for (key, signing, named) in candidates { + // A version 6 signature is made by a version 6 key and only by one (RFC 9580 §5.2.3). + if (signing.version == KeyVersion::V6) != (config.version() == SignatureVersion::V6) || signing.algorithm != config.pub_alg { + continue; + } + let signer = Signer { + fingerprint: key.fingerprint().clone(), + subkey: signing.subkey.map(|_| signing.fingerprint.clone()), + key_id: signing.key_id, + addresses: key.addresses().to_vec(), + signed, + key_created: key.created(), + algorithm: keys::algorithm_name(signing.algorithm), + hash: keys::hash_name(config.hash_alg), + }; + if let Some(why) = signing.weak { + if named { + weak = Some(why); + } + continue; + } + if !fits { + if named { + blamed.get_or_insert(signer); + } + continue; + } + if cx.verifications.get() >= MAX_VERIFICATIONS { + // Keys left untried: what came of it isn't known. + starved = true; + break; + } + cx.verifications.set(cx.verifications.get().saturating_add(1)); + if verify_with(config.hash_alg, &digest, sig_bytes, key, signing) { + return judged(sig, key, signing, signer, cx).unwrap_or_else(&cant); + } + if named { + blamed.get_or_insert(signer); + } + } + if let Some(why) = weak { + return cant(CantCheck::WeakAlgorithm(why)); + } + match blamed { + Some(signer) => Outcome::Changed(signer), + None if starved => cant(CantCheck::Unreadable), + None if !from_keys.is_empty() => cant(CantCheck::NoMatchingKey), + None => cant(CantCheck::NoPublicKey), + } +} + +/// Adds a key to try, once, while there is room. +fn push<'a>(list: &mut Vec<(&'a PublicKey, &'a Signing, bool)>, key: &'a PublicKey, signing: &'a Signing, named: bool) { + if list.len() < MAX_CANDIDATES && !list.iter().any(|(_, s, _)| s.fingerprint == signing.fingerprint) { + list.push((key, signing, named)); + } +} + +/// A signature that checks: whether its key (and the primary key over it) could make it then, +/// and whether it is Verified. +fn judged(sig: &Signature, key: &PublicKey, signing: &Signing, signer: Signer, cx: &Context<'_>) -> Result { + let when = signer.signed; + let primary = key.primary(); + if signing.revoked.is_some_and(|r| r.covers(Some(when))) || primary.and_then(|p| p.revoked).is_some_and(|r| r.covers(Some(when))) { + return Err(CantCheck::KeyRevoked); + } + if !signing.may_sign { + return Err(CantCheck::NotASigningKey); + } + // A signature that says it was made before its key was is not one the key made. + if when < u64::from(signing.created) { + return Err(CantCheck::Unreadable); + } + if signing.expired_at(when) || primary.is_some_and(|p| p.expired_at(when)) { + return Err(CantCheck::KeyExpired); + } + if let Some(after) = sig.signature_expiration_time().map(|d| u64::from(d.as_secs())) + && after > 0 + && cx.now >= when.saturating_add(after) + { + return Err(CantCheck::SignatureExpired); + } + let address_on_key = !cx.from.is_empty() && key.has_address_at(&cx.from, when); + let why = NotVerified { + key_unverified: cx.from.is_empty() || !(cx.verified)(&signer.fingerprint, &cx.from), + address_differs: !address_on_key, + partly_signed: false, + }; + Ok(if why.key_unverified || why.address_differs { Outcome::Valid { signer, why } } else { Outcome::Verified(signer) }) +} + +fn verify_with(hash: HashAlgorithm, digest: &[u8], sig: &SignatureBytes, key: &PublicKey, signing: &Signing) -> bool { + let inner = key.inner(); + match signing.subkey { + None => inner.primary_key.verify(hash, digest, sig).is_ok(), + Some(i) => inner.public_subkeys.get(i).is_some_and(|sub| sub.key.verify(hash, digest, sig).is_ok()), + } +} + +/// Whether a signature value has the shape its algorithm gives it (RFC 9580 §5.2.3): one MPI +/// for RSA, two for ECDSA and legacy EdDSA, 64 or 114 bytes for Ed25519 and Ed448. +fn well_shaped(algorithm: PublicKeyAlgorithm, bytes: &SignatureBytes) -> bool { + match (algorithm, bytes) { + (PublicKeyAlgorithm::RSA | PublicKeyAlgorithm::RSASign, SignatureBytes::Mpis(m)) => m.len() == 1, + (PublicKeyAlgorithm::ECDSA | PublicKeyAlgorithm::EdDSALegacy, SignatureBytes::Mpis(m)) => m.len() == 2, + (PublicKeyAlgorithm::Ed25519, SignatureBytes::Native(b)) => b.len() == 64, + (PublicKeyAlgorithm::Ed448, SignatureBytes::Native(b)) => b.len() == 114, + _ => false, + } +} + +/// Algorithms a Digital signature can be made with that are checked (DSA is refused before). +fn known_signature_algorithm(algorithm: PublicKeyAlgorithm) -> bool { + matches!( + algorithm, + PublicKeyAlgorithm::RSA + | PublicKeyAlgorithm::RSASign + | PublicKeyAlgorithm::ECDSA + | PublicKeyAlgorithm::EdDSALegacy + | PublicKeyAlgorithm::Ed25519 + | PublicKeyAlgorithm::Ed448 + ) +} diff --git a/crates/openpgp/src/coverage_tests.rs b/crates/openpgp/src/coverage_tests.rs new file mode 100644 index 000000000..ffad12ce2 --- /dev/null +++ b/crates/openpgp/src/coverage_tests.rs @@ -0,0 +1,136 @@ +//! What a signature covers: only a whole message can be Verified, and words around a signed +//! part never borrow its signature. + +use pgp::crypto::hash::HashAlgorithm; + +use crate::fixtures::*; +use crate::{CantCheck, Checked, Coverage, Fingerprint, KeyRing, MAX_MESSAGE, NotVerified, Outcome, verify}; + +const ALICE: &str = "Alice "; + +fn yes(_: &Fingerprint, _: &str) -> bool { + true +} + +fn alice() -> (pgp::composed::SignedSecretKey, KeyRing) { + let alice = secret(ALICE, Make::default()); + let ring = [public(&alice)].into_iter().collect(); + (alice, ring) +} + +/// A PGP/MIME message by `alice` as a part: its Content-Type field and its body. +fn signed_part(alice: &pgp::composed::SignedSecretKey) -> (String, String) { + let inner = String::from_utf8(signed_message(alice, HashAlgorithm::Sha256)).unwrap(); + let (head, body) = inner.split_once("\r\n\r\n").unwrap(); + let content_type = head.split("\r\n").skip_while(|l| !l.starts_with("Content-Type")).collect::>().join("\r\n"); + (content_type, body.to_string()) +} + +fn partly(checked: &Checked) -> bool { + matches!(&checked.outcome, Outcome::Valid { why: NotVerified { partly_signed: true, .. }, .. }) +} + +#[test] +fn a_pgp_mime_message_at_the_top_covers_the_whole_message() { + let (alice, keys) = alice(); + let checked = verify(&signed_message(&alice, HashAlgorithm::Sha256), ALICE, &keys, LATER, yes).unwrap(); + assert_eq!(checked.coverage, Coverage { whole: true, path: vec![], span: None }); + assert!(matches!(checked.outcome, Outcome::Verified(_))); +} + +#[test] +fn a_signed_alternative_beside_an_unsigned_one_is_not_a_signed_message() { + // The HTML alternative is what most readers show, and it isn't signed. + let (alice, keys) = alice(); + let (content_type, body) = signed_part(&alice); + let message = format!( + "From: {ALICE}\r\nContent-Type: multipart/alternative; boundary=\"alt\"\r\n\r\n--alt\r\n{content_type}\r\n\r\n{body}\r\n--alt\r\nContent-Type: text/html\r\n\r\n

Wire 10000 EUR to Mallory

\r\n--alt--\r\n" + ); + assert_eq!(verify(message.as_bytes(), ALICE, &keys, LATER, yes), None); +} + +#[test] +fn a_signed_part_beside_an_unsigned_one_is_only_partly_signed() { + let (alice, keys) = alice(); + let (content_type, body) = signed_part(&alice); + let message = format!( + "From: {ALICE}\r\nContent-Type: multipart/mixed; boundary=\"mix\"\r\n\r\n--mix\r\n{content_type}\r\n\r\n{body}\r\n--mix\r\nContent-Type: text/plain\r\n\r\nWire 10000 EUR to Mallory\r\n--mix--\r\n" + ); + let checked = verify(message.as_bytes(), ALICE, &keys, LATER, yes).unwrap(); + assert!(!checked.coverage.whole); + assert_eq!(checked.coverage.path, vec![0]); + assert!(partly(&checked), "{checked:?}"); +} + +#[test] +fn an_inline_signed_body_alone_covers_the_whole_message_and_says_where() { + let (alice, keys) = alice(); + let block = cleartext(&alice, "Pay 10 EUR to Bob.\n"); + let body = format!("\r\n{block}\r\n\r\n"); + let checked = verify(&plain(ALICE, "7bit", &body), ALICE, &keys, LATER, yes).unwrap(); + assert!(matches!(checked.outcome, Outcome::Verified(_)), "{checked:?}"); + assert!(checked.coverage.whole); + let (start, end) = checked.coverage.span.unwrap(); + assert_eq!(&body[start..end].trim_end(), &block.trim_end()); +} + +#[test] +fn words_before_or_after_an_inline_signed_block_are_not_covered() { + let (alice, keys) = alice(); + let block = cleartext(&alice, "Pay 10 EUR to Bob.\n"); + for body in [format!("Wire 10000 EUR to Mallory\r\n{block}"), format!("{block}\r\nPS: wire 10000 EUR to Mallory\r\n")] { + let checked = verify(&plain(ALICE, "7bit", &body), ALICE, &keys, LATER, yes).unwrap(); + assert!(!checked.coverage.whole); + assert!(partly(&checked), "{checked:?}"); + } +} + +#[test] +fn two_inline_signed_blocks_are_each_only_part() { + let (alice, keys) = alice(); + let body = format!("{}\r\n{}", cleartext(&alice, "First.\n"), cleartext(&alice, "Second.\n")); + let checked = verify(&plain(ALICE, "7bit", &body), ALICE, &keys, LATER, yes).unwrap(); + assert!(partly(&checked), "{checked:?}"); +} + +#[test] +fn an_armour_header_other_than_hash_or_an_unescaped_dash_line_is_unreadable() { + // Both would show words in the block that aren't signed. + let (alice, keys) = alice(); + let block = cleartext(&alice, "Pay 10 EUR to Bob.\n"); + let header = block.replacen("-----BEGIN PGP SIGNED MESSAGE-----\n", "-----BEGIN PGP SIGNED MESSAGE-----\nNote: and 10000 EUR to Mallory\n", 1); + assert_ne!(header, block); + let dash = block.replacen("Pay 10 EUR to Bob.", "-----BEGIN PGP SIGNATURE-----", 1); + // A `Hash:` header with more than hash names after it. + let with_hash = block.replacen( + "-----BEGIN PGP SIGNED MESSAGE-----\n", + "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256, and pay Mallory 10000 EUR today\n", + 1, + ); + assert_ne!(with_hash, block); + // Hash names alone, as many writers give them, are fine. + let names = block.replacen("-----BEGIN PGP SIGNED MESSAGE-----\n", "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256, sha512 ,SHA3-256\n", 1); + let checked = verify(&plain(ALICE, "7bit", &names), ALICE, &keys, LATER, yes).unwrap(); + assert!(matches!(checked.outcome, Outcome::Verified(_)), "{checked:?}"); + for body in [header, dash, with_hash] { + let checked = verify(&plain(ALICE, "7bit", &body), ALICE, &keys, LATER, yes).unwrap(); + assert!(matches!(checked.outcome, Outcome::CantCheck { why: CantCheck::Unreadable, .. }), "{checked:?}"); + } +} + +#[test] +fn a_signed_message_too_large_to_check_says_so() { + let mut big = signed_message(&secret(ALICE, Make::default()), HashAlgorithm::Sha256); + big.resize(MAX_MESSAGE + 1, b'a'); + let checked = verify(&big, ALICE, &KeyRing::new(), LATER, yes).unwrap(); + assert!(matches!(checked.outcome, Outcome::CantCheck { why: CantCheck::TooLarge, .. })); + // Inline-signed, the block found anywhere in it. + let mut big = plain(ALICE, "7bit", &"a".repeat(1 << 20)); + big.extend_from_slice(b"\r\n-----BEGIN PGP SIGNED MESSAGE-----\r\n"); + big.resize(MAX_MESSAGE + 1, b'a'); + assert!(verify(&big, ALICE, &KeyRing::new(), LATER, yes).is_some()); + // Not signed at all: nothing to say. + let mut big = plain(ALICE, "7bit", "Just text.\r\n"); + big.resize(MAX_MESSAGE + 1, b'a'); + assert_eq!(verify(&big, ALICE, &KeyRing::new(), LATER, yes), None); +} diff --git a/crates/openpgp/src/fixtures_tests.rs b/crates/openpgp/src/fixtures_tests.rs new file mode 100644 index 000000000..04ebcdb94 --- /dev/null +++ b/crates/openpgp/src/fixtures_tests.rs @@ -0,0 +1,222 @@ +//! Keys, signatures and messages made in the tests: synthetic `example.org` identities only. + +use pgp::composed::{ + ArmorOptions, CleartextSignedMessage, DetachedSignature, EncryptionCaps, KeyType, SecretKeyParamsBuilder, SignedPublicKey, SignedSecretKey, + SubkeyParamsBuilder, +}; +use pgp::crypto::hash::HashAlgorithm; +use pgp::packet::{SignatureConfig, SignatureType, Subpacket, SubpacketData}; +use pgp::ser::Serialize; +use pgp::types::{KeyDetails, KeyVersion, Password, SigningKey, Timestamp}; +use rand::SeedableRng; +use rand::rngs::StdRng; + +use crate::PublicKey; + +/// RFC 9580 Appendix A.3: the sample version 6 certificate (no User ID). +pub(crate) const RFC9580_A3_CERT: &str = "-----BEGIN PGP PUBLIC KEY BLOCK----- + +xioGY4d/4xsAAAAg+U2nu0jWCmHlZ3BqZYfQMxmZu52JGggkLq2EVD34laPCsQYf +GwoAAABCBYJjh3/jAwsJBwUVCg4IDAIWAAKbAwIeCSIhBssYbE8GCaaX5NUt+mxy +KwwfHifBilZwj2Ul7Ce62azJBScJAgcCAAAAAK0oIBA+LX0ifsDm185Ecds2v8lw +gyU2kCcUmKfvBXbAf6rhRYWzuQOwEn7E/aLwIwRaLsdry0+VcallHhSu4RN6HWaE +QsiPlR4zxP/TP7mhfVEe7XWPxtnMUMtf15OyA51YBM4qBmOHf+MZAAAAIIaTJINn ++eUBXbki+PSAld2nhJh/LVmFsS+60WyvXkQ1wpsGGBsKAAAALAWCY4d/4wKbDCIh +BssYbE8GCaaX5NUt+mxyKwwfHifBilZwj2Ul7Ce62azJAAAAAAQBIKbpGG2dWTX8 +j+VjFM21J0hqWlEg+bdiojWnKfA5AQpWUWtnNwDEM0g12vYxoWM8Y81W+bHBw805 +I8kWVkXU6vFOi+HWvv/ira7ofJu16NnoUkhclkUrk0mXubZvyl4GBg== +-----END PGP PUBLIC KEY BLOCK----- +"; + +/// RFC 9580 Appendix A.6: the sample cleartext-signed message, signed by [`RFC9580_A3_CERT`]. +pub(crate) const RFC9580_A6_CLEARTEXT: &str = "-----BEGIN PGP SIGNED MESSAGE----- + +What we need from the grocery store: + +- - tofu +- - vegetables +- - noodles + +-----BEGIN PGP SIGNATURE----- + +wpgGARsKAAAAKQWCY5ijYyIhBssYbE8GCaaX5NUt+mxyKwwfHifBilZwj2Ul7Ce6 +2azJAAAAAGk2IHZJX1AhiJD39eLuPBgiUU9wUA9VHYblySHkBONKU/usJ9BvuAqo +/FvLFuGWMbKAdA+epq7V4HOtAPlBWmU8QOd6aud+aSunHQaaEJ+iTFjP2OMW0KBr +NK2ay45cX1IVAQ== +-----END PGP SIGNATURE----- +"; + +/// The primary fingerprint of [`RFC9580_A3_CERT`]. +pub(crate) const RFC9580_A3_FINGERPRINT: &str = "CB186C4F0609A697E4D52DFA6C722B0C1F1E27C18A56708F6525EC27BAD9ACC9"; + +/// A key for the property tests to look signatures up in. +pub(crate) static ALICE_FOR_PROPS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| vec![public(&secret("A ", Make::default()))]); + +/// A time after every signature made in the tests (2100-01-01). +pub(crate) const LATER: u64 = 4_102_444_800; + +/// How a test key is made. +#[derive(Clone, Copy)] +pub(crate) struct Make { + pub(crate) seed: u64, + /// A subkey signs, not the primary key. + pub(crate) subkey: bool, + pub(crate) v6: bool, + /// RSA of this many bits rather than EdDSA: rPGP makes MD5 and SHA-1 signatures only with RSA. + pub(crate) rsa: Option, + /// A second User ID, `Alias `. + pub(crate) alias: bool, + /// An RSA encryption subkey of 2048 bits besides. + pub(crate) rsa_encryption: bool, +} + +impl Default for Make { + fn default() -> Make { + Make { seed: 1, subkey: false, v6: false, rsa: None, alias: false, rsa_encryption: false } + } +} + +/// A secret key for `user_id` (an `example.org` identity). +pub(crate) fn secret(user_id: &str, make: Make) -> SignedSecretKey { + let rng = StdRng::seed_from_u64(make.seed); + let version = if make.v6 { KeyVersion::V6 } else { KeyVersion::V4 }; + let key_type = match (make.rsa, make.v6) { + (Some(bits), _) => KeyType::Rsa(bits as u32), + (None, true) => KeyType::Ed25519, + (None, false) => KeyType::Ed25519Legacy, + }; + let mut params = SecretKeyParamsBuilder::default(); + params.version(version).key_type(key_type.clone()).can_certify(true).can_sign(!make.subkey).primary_user_id(user_id.into()); + if make.alias { + params.user_ids(vec!["Alias ".into()]); + } + let mut subkeys = Vec::new(); + if make.subkey { + let mut sub = SubkeyParamsBuilder::default(); + sub.version(version).key_type(key_type).can_sign(true).can_encrypt(EncryptionCaps::None); + subkeys.push(sub.build().unwrap()); + } + if make.rsa_encryption { + let mut sub = SubkeyParamsBuilder::default(); + sub.version(version).key_type(KeyType::Rsa(2048)).can_sign(false).can_encrypt(EncryptionCaps::All); + subkeys.push(sub.build().unwrap()); + } + if !subkeys.is_empty() { + params.subkeys(subkeys); + } + params.build().unwrap().generate(rng).unwrap() +} + +/// The public half of `secret`, read back as the crate reads keys. +pub(crate) fn public(secret: &SignedSecretKey) -> PublicKey { + let bytes = SignedPublicKey::from(secret.clone()).to_bytes().unwrap(); + PublicKey::parse(&bytes).unwrap().remove(0) +} + +/// The armoured public half of `secret`. +pub(crate) fn armoured_public(secret: &SignedSecretKey) -> String { + SignedPublicKey::from(secret.clone()).to_armored_string(ArmorOptions::default()).unwrap() +} + +/// The key that signs: rPGP takes either kind through this box alone. +#[allow(clippy::redundant_allocation)] +pub(crate) fn signer(secret: &SignedSecretKey) -> Box<&dyn SigningKey> { + match secret.secret_subkeys.first() { + Some(sub) => Box::new(&sub.key), + None => Box::new(&secret.primary_key), + } +} + +/// An armoured detached signature over `data` by `secret` (its signing subkey when it has one). +pub(crate) fn detached(secret: &SignedSecretKey, data: &[u8], hash: HashAlgorithm, text: bool) -> String { + let rng = StdRng::seed_from_u64(7); + let key = signer(secret); + let sig = if text { + DetachedSignature::sign_text_data(rng, &key, &Password::empty(), hash, data).unwrap() + } else { + DetachedSignature::sign_binary_data(rng, &key, &Password::empty(), hash, data).unwrap() + }; + sig.to_armored_string(ArmorOptions::default()).unwrap() +} + +/// When `secret`'s primary key was made, in seconds since 1970. +pub(crate) fn made(secret: &SignedSecretKey) -> u32 { + secret.primary_key.created_at().as_secs() +} + +/// A signature of `typ` by `key` made at `at`, its issuer's fingerprint and `extra` in its +/// hashed area: to be signed. +pub(crate) fn config(key: &impl SigningKey, typ: SignatureType, at: u32, extra: Vec) -> SignatureConfig { + let mut config = SignatureConfig::from_key(StdRng::seed_from_u64(11), key, typ).unwrap(); + config.hashed_subpackets = [SubpacketData::SignatureCreationTime(Timestamp::from_secs(at)), SubpacketData::IssuerFingerprint(key.fingerprint())] + .into_iter() + .chain(extra) + .map(|d| Subpacket::regular(d).unwrap()) + .collect(); + config +} + +/// A PGP/MIME message from alice@example.org over [`SIGNED_PART`], signed by `secret`'s +/// signing key at `at`. +pub(crate) fn signed_at(secret: &SignedSecretKey, at: u32) -> Vec { + let key = signer(secret); + let sig = config(&key, SignatureType::Binary, at, vec![]).sign(&key, &Password::empty(), SIGNED_PART.as_bytes()).unwrap(); + pgp_mime("alice@example.org", SIGNED_PART, &DetachedSignature::new(sig).to_armored_string(ArmorOptions::default()).unwrap()) +} + +/// The keys in `key`, read as the crate reads them, as a ring. +pub(crate) fn ring_of(key: &SignedPublicKey) -> crate::KeyRing { + PublicKey::parse(&key.to_bytes().unwrap()).unwrap().into_iter().collect() +} + +/// A cleartext-signed `text` by `secret`. +pub(crate) fn cleartext(secret: &SignedSecretKey, text: &str) -> String { + let rng = StdRng::seed_from_u64(9); + let key = signer(secret); + CleartextSignedMessage::sign(rng, text, &key, &Password::empty()).unwrap().to_armored_string(ArmorOptions::default()).unwrap() +} + +/// The part a PGP/MIME message signs, CR LF throughout. +pub(crate) const SIGNED_PART: &str = "Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 7bit\r\n\r\nHello Bob,\r\n\r\nthe minutes are attached.\r\n-- \r\nAlice\r\n"; + +/// A PGP/MIME message from `from` whose first part is `part` and whose signature is `armour`. +pub(crate) fn pgp_mime(from: &str, part: &str, armour: &str) -> Vec { + format!( + "From: Alice <{from}>\r\nTo: bob@example.org\r\nSubject: Minutes\r\nMIME-Version: 1.0\r\nContent-Type: multipart/signed; micalg=pgp-sha256;\r\n protocol=\"application/pgp-signature\"; boundary=\"=-b1\"\r\n\r\nThis is an OpenPGP/MIME signed message.\r\n--=-b1\r\n{part}\r\n--=-b1\r\nContent-Type: application/pgp-signature; name=\"signature.asc\"\r\nContent-Disposition: attachment; filename=\"signature.asc\"\r\n\r\n{armour}\r\n--=-b1--\r\n" + ) + .into_bytes() +} + +/// A PGP/MIME message from alice@example.org signed by `secret` over [`SIGNED_PART`]. +pub(crate) fn signed_message(secret: &SignedSecretKey, hash: HashAlgorithm) -> Vec { + let armour = detached(secret, SIGNED_PART.as_bytes(), hash, false); + pgp_mime("alice@example.org", SIGNED_PART, &armour) +} + +/// A one-part message from `from` with `body` as its text, `encoding` its transfer encoding. +pub(crate) fn plain(from: &str, encoding: &str, body: &str) -> Vec { + format!("From: {from}\r\nSubject: Note\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: {encoding}\r\n\r\n{body}") + .into_bytes() +} + +/// `bytes` in base64, a line break after every 76 characters. +pub(crate) fn base64(bytes: &[u8]) -> String { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::new(); + for (n, chunk) in bytes.chunks(3).enumerate() { + let b = [chunk[0], *chunk.get(1).unwrap_or(&0), *chunk.get(2).unwrap_or(&0)]; + let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + for i in 0..4 { + if i <= chunk.len() { + out.push(ALPHABET[((v >> (18 - 6 * i)) & 63) as usize] as char); + } else { + out.push('='); + } + } + if n % 19 == 18 { + out.push_str("\r\n"); + } + } + out +} diff --git a/crates/openpgp/src/keys.rs b/crates/openpgp/src/keys.rs new file mode 100644 index 000000000..c3b083162 --- /dev/null +++ b/crates/openpgp/src/keys.rs @@ -0,0 +1,751 @@ +//! Public keys: read from their bytes (armoured or not), their self-signatures checked once +//! when they are read, and kept in a [`KeyRing`] that finds them by fingerprint, key id or +//! address. A key's addresses are those of the User IDs it signed itself (RFC 9580 §5.2.1, +//! §10.1); nothing here is a web of trust: whether a key is Verified is the caller's to say. + +use pgp::composed::{Deserializable, SignedKeyDetails, SignedPublicKey, SignedPublicSubKey}; +use pgp::crypto::hash::HashAlgorithm; +use pgp::crypto::public_key::PublicKeyAlgorithm; +use pgp::packet::{KeyFlags, RevocationCode, Signature, SignatureType, SignatureVersion, SubpacketData}; +use pgp::ser::Serialize; +use pgp::types::{KeyDetails, KeyVersion, PublicParams, SignedUser, Tag}; + +use crate::armour::{self, Kind}; +use crate::{Error, mime}; + +/// How many bytes a block of Public keys may hold, decoded. +pub(crate) const MAX_KEY_BYTES: usize = 1 << 20; +/// How many keys one block (a file, an attachment) is read for. +pub(crate) const MAX_KEYS_PER_BLOCK: usize = 16; +/// How many User IDs and subkeys one key may have: more is not a key anybody sends. +const MAX_USERS: usize = 64; +const MAX_SUBKEYS: usize = 32; +/// How long a User ID may be: a name and an address are far shorter. (User attributes, such as +/// pictures, are never checked or kept, so cost nothing.) +const MAX_USER_ID: usize = 4 * 1024; +/// How many signature checks reading one key may cost (third-party certifications are never +/// checked, so a real key needs a handful). Room for two copies of the largest key kept, merged: +/// two self-signatures for each User ID, and two with a back-signature each for each subkey. +const MAX_CHECKS: usize = 512; +/// How many packets one block is read for before it is given up on. +const MAX_PACKETS: usize = 4_096; + +/// A key's fingerprint (RFC 9580 §5.5.4): 20 bytes for a version 4 key, 32 for version 6. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Fingerprint(Vec); + +impl Fingerprint { + /// The fingerprint written as hexadecimal digits (spaces and a `0x` in front allowed). + pub fn from_hex(text: &str) -> Option { + let text = text.trim(); + let text = text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")).unwrap_or(text); + let digits: Vec = text.chars().filter(|c| !c.is_whitespace()).map(|c| c.to_digit(16)).collect::>()?; + if !matches!(digits.len(), 40 | 64) { + return None; + } + Some(Fingerprint(digits.chunks(2).map(|p| p.iter().fold(0u8, |acc, d| (acc << 4) | (*d as u8))).collect())) + } + + /// The fingerprint's bytes. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Upper-case hexadecimal, no spaces: what [`Fingerprint::from_hex`] reads back. + pub fn to_hex(&self) -> String { + self.0.iter().map(|b| format!("{b:02X}")).collect() + } + + /// Upper-case hexadecimal in groups of four, for an owner to compare by eye. + pub fn grouped(&self) -> String { + let hex = self.to_hex(); + let groups: Vec<&str> = (0..hex.len()).step_by(4).filter_map(|i| hex.get(i..i.saturating_add(4).min(hex.len()))).collect(); + groups.join(" ") + } + + pub(crate) fn of(fp: &pgp::types::Fingerprint) -> Fingerprint { + Fingerprint(fp.as_bytes().to_vec()) + } +} + +impl std::fmt::Display for Fingerprint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.to_hex()) + } +} + +/// A key id (RFC 9580 §5.5.4): eight bytes that a signature names its signer by. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct KeyId(pub [u8; 8]); + +impl KeyId { + /// Upper-case hexadecimal, sixteen digits. + pub fn to_hex(&self) -> String { + self.0.iter().map(|b| format!("{b:02X}")).collect() + } + + pub(crate) fn of(id: &pgp::types::KeyId) -> KeyId { + let mut bytes = [0u8; 8]; + for (to, from) in bytes.iter_mut().zip(id.as_ref()) { + *to = *from; + } + KeyId(bytes) + } +} + +impl std::fmt::Display for KeyId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.to_hex()) + } +} + +/// A public-key algorithm's name, as an owner may see it in the details. +pub(crate) fn algorithm_name(algorithm: PublicKeyAlgorithm) -> &'static str { + match algorithm { + PublicKeyAlgorithm::RSA | PublicKeyAlgorithm::RSASign | PublicKeyAlgorithm::RSAEncrypt => "RSA", + PublicKeyAlgorithm::DSA => "DSA", + PublicKeyAlgorithm::ECDSA => "ECDSA", + PublicKeyAlgorithm::EdDSALegacy => "EdDSA", + PublicKeyAlgorithm::Ed25519 => "Ed25519", + PublicKeyAlgorithm::Ed448 => "Ed448", + PublicKeyAlgorithm::ECDH => "ECDH", + PublicKeyAlgorithm::X25519 => "X25519", + PublicKeyAlgorithm::X448 => "X448", + PublicKeyAlgorithm::Elgamal | PublicKeyAlgorithm::ElgamalEncrypt => "Elgamal", + _ => "unknown", + } +} + +/// A hash algorithm's name, as an owner may see it in the details. +pub(crate) fn hash_name(hash: HashAlgorithm) -> &'static str { + match hash { + HashAlgorithm::Md5 => "MD5", + HashAlgorithm::Sha1 => "SHA-1", + HashAlgorithm::Ripemd160 => "RIPEMD-160", + HashAlgorithm::Sha224 => "SHA-224", + HashAlgorithm::Sha256 => "SHA-256", + HashAlgorithm::Sha384 => "SHA-384", + HashAlgorithm::Sha512 => "SHA-512", + HashAlgorithm::Sha3_256 => "SHA3-256", + HashAlgorithm::Sha3_512 => "SHA3-512", + _ => "unknown", + } +} + +/// Hashes no longer safe for a signature: MD5, SHA-1 and RIPEMD-160 (RFC 9580 §9.5). +pub(crate) fn is_weak(hash: HashAlgorithm) -> bool { + matches!(hash, HashAlgorithm::Md5 | HashAlgorithm::Sha1 | HashAlgorithm::Ripemd160) +} + +/// Why a key may not be trusted to make a Digital signature at all, whatever its signature +/// says: DSA, and RSA under 2048 bits (RFC 9580 §12.4, §12.5). +pub(crate) fn weak_key(key: &impl KeyDetails) -> Option<&'static str> { + match key.public_params() { + PublicParams::DSA(_) => Some("DSA"), + PublicParams::RSA(params) => { + // An MPI starts with its length in bits (RFC 9580 §3.2): n comes first. + let bytes = params.to_bytes().unwrap_or_default(); + let bits = u16::from_be_bytes([bytes.first().copied().unwrap_or(0), bytes.get(1).copied().unwrap_or(0)]); + (bits < 2048).then_some("RSA under 2048 bits") + } + _ => None, + } +} + +/// A revocation of a key or subkey (RFC 9580 §5.2.3.31): a hard one (compromised, or no reason +/// a reader knows) undoes every signature the key made; a soft one (superseded, retired) only +/// those made at or after it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Revoked { + pub(crate) hard: bool, + pub(crate) at: u64, +} + +impl Revoked { + fn of(sig: &Signature) -> Revoked { + let soft = matches!(sig.revocation_reason_code(), Some(RevocationCode::KeySuperseded | RevocationCode::KeyRetired)); + Revoked { hard: !soft, at: u64::from(created(sig)) } + } + + /// Whether it undoes a signature made at `when` (`None`: unknown, so yes). + pub(crate) fn covers(&self, when: Option) -> bool { + self.hard || when.is_none_or(|t| t >= self.at) + } +} + +/// One key of a [`PublicKey`] that may make Digital signatures: the primary key or a subkey. +#[derive(Clone, Debug)] +pub(crate) struct Signing { + /// `None` for the primary key, else the subkey's place in the key. + pub(crate) subkey: Option, + pub(crate) fingerprint: Fingerprint, + pub(crate) key_id: KeyId, + pub(crate) version: KeyVersion, + pub(crate) algorithm: PublicKeyAlgorithm, + pub(crate) created: u32, + /// When it stops being valid, in seconds since 1970 (its own expiry, or that of the + /// self-signature that binds it); `None` when never. + pub(crate) expires: Option, + /// Its own revocation; a subkey is also undone by the primary key's. + pub(crate) revoked: Option, + /// Whether its self-signature lets it sign (subkeys say so; a primary key that says nothing + /// may, as RFC 9580 §5.2.3.29 leaves it). + pub(crate) may_sign: bool, + /// Why it isn't strong enough to be trusted, if it isn't. + pub(crate) weak: Option<&'static str>, +} + +impl Signing { + /// Whether it was valid at `when`: not before it was made, not after it expired. + pub(crate) fn expired_at(&self, when: u64) -> bool { + self.expires.is_some_and(|end| when >= end) + } +} + +/// A Public key (an OpenPGP certificate, RFC 9580 §10.1) whose self-signatures check. +#[derive(Clone, Debug)] +pub struct PublicKey { + key: SignedPublicKey, + fingerprint: Fingerprint, + addresses: Vec, + /// For each address, when its User ID's self-signature stops being valid. + address_until: Vec>, + /// The primary key first, then each subkey that may sign. + pub(crate) signing: Vec, +} + +/// How many signature checks one call that reads keys may cost in all ([`PublicKey::parse`], +/// [`crate::keys_in_message`]). +pub(crate) const MAX_CHECKS_PER_CALL: usize = 1_024; +/// How many revocations of one key are checked: never cut short; a key with more is refused. +/// Room for two of each part of two merged copies of the largest key kept. +const MAX_REVOCATION_CHECKS: usize = 256; + +/// How many signature checks some reading may cost. +pub(crate) struct Budget { + left: usize, + /// A check was wanted and refused. + pub(crate) starved: bool, +} + +impl Budget { + pub(crate) fn new(left: usize) -> Budget { + Budget { left, starved: false } + } + + /// Whether one more check may be made, counting it. + fn spend(&mut self) -> bool { + if self.left == 0 { + self.starved = true; + return false; + } + self.left -= 1; + true + } +} + +impl PublicKey { + /// Every Public key in `bytes`: armoured (`-----BEGIN PGP PUBLIC KEY BLOCK-----`, as many + /// blocks as there are) or binary, at most 16 in all. A key whose self-signatures don't + /// check, of a version before 4, or larger than a key anybody sends, is left out. + /// [`Error::Unreadable`] when no key at all could be read. + pub fn parse(bytes: &[u8]) -> Result, Error> { + Self::parse_with(bytes, &mut Budget::new(MAX_CHECKS_PER_CALL)) + } + + pub(crate) fn parse_with(bytes: &[u8], total: &mut Budget) -> Result, Error> { + let mut out = Vec::new(); + let blocks = armour::blocks(bytes, Kind::PublicKey, MAX_KEYS_PER_BLOCK, MAX_KEY_BYTES); + if blocks.is_empty() { + if bytes.len() > MAX_KEY_BYTES { + return Err(Error::TooLarge); + } + out.extend(Self::binary(bytes, MAX_KEYS_PER_BLOCK, total)); + } else { + for block in blocks { + let room = MAX_KEYS_PER_BLOCK.saturating_sub(out.len()); + if room == 0 { + break; + } + out.extend(Self::binary(&block, room, total)); + } + } + if out.is_empty() { Err(Error::Unreadable) } else { Ok(out) } + } + + /// At most `room` keys in binary packets. + pub(crate) fn binary(bytes: &[u8], room: usize, total: &mut Budget) -> Vec { + if room == 0 || count_packets(bytes) > MAX_PACKETS { + return Vec::new(); + } + let Some(Ok(keys)) = crate::guard(|| SignedPublicKey::from_bytes_many(bytes).map(|it| it.take(room).collect::>())) else { + return Vec::new(); + }; + keys.into_iter().filter_map(Result::ok).filter_map(|key| crate::guard(|| PublicKey::checked(key, total)).flatten()).collect() + } + + /// The key, once its self-signatures check: its addresses from the User IDs it certified, + /// its revocations, and the keys of it that may sign. `None` when it is no key to trust: + /// nothing certifies it, it is too large, its version is before 4, or a budget ran out + /// before all of it was checked. What is kept of it is what checked and is used: for the + /// key, each User ID and each subkey, its newest self-signature and its strongest + /// revocation; nothing that anybody but the key's owner could have added. + pub(crate) fn checked(key: SignedPublicKey, total: &mut Budget) -> Option { + if !matches!(key.primary_key.version(), KeyVersion::V4 | KeyVersion::V6) + || key.details.users.len() > MAX_USERS + || key.public_subkeys.len() > MAX_SUBKEYS + { + return None; + } + let primary = &key.primary_key; + let mut local = Budget::new(MAX_CHECKS); + let mut revocations = Budget::new(MAX_REVOCATION_CHECKS); + let spend = |budget: &mut Budget, total: &mut Budget| budget.spend() && total.spend(); + + // Revocations first, on a budget of their own, so that none is ever left unchecked. + let mut key_revocation: Option<&Signature> = None; + for sig in key.details.revocation_signatures.iter().chain(&key.details.direct_signatures) { + if sig.typ() == Some(SignatureType::KeyRevocation) + && usable(sig) + && is_self(sig, primary) + && spend(&mut revocations, total) + && sig.verify_key(primary).is_ok() + { + key_revocation = stronger(key_revocation, sig); + } + } + + // The newest valid self-signature on the key or one of its User IDs: its key flags and + // expiry are the primary key's (RFC 9580 §5.2.3.10, §10.1.1). + let mut newest: Option<&Signature> = None; + let mut direct: Option<&Signature> = None; + for sig in &key.details.direct_signatures { + if sig.typ() == Some(SignatureType::Key) + && usable(sig) + && is_self(sig, primary) + && spend(&mut local, total) + && sig.verify_key(primary).is_ok() + { + consider(sig, &mut direct); + } + } + if let Some(sig) = direct { + consider(sig, &mut newest); + } + let (mut addresses, mut address_until, mut users) = (Vec::new(), Vec::new(), Vec::new()); + for user in &key.details.users { + // Every check of a User ID hashes all of it: a longer one than anybody's is left + // out with its signatures, unhashed. + if user.id.id().len() > MAX_USER_ID { + continue; + } + let mut certified: Option<&Signature> = None; + let mut user_revocation: Option<&Signature> = None; + for sig in &user.signatures { + let Some(typ) = sig.typ() else { continue }; + if !usable(sig) || !is_self(sig, primary) { + continue; + } + if typ == SignatureType::CertRevocation { + if spend(&mut revocations, total) && sig.verify_certification(primary, Tag::UserId, &user.id).is_ok() { + consider(sig, &mut user_revocation); + } + continue; + } + let certification = + matches!(typ, SignatureType::CertGeneric | SignatureType::CertPersona | SignatureType::CertCasual | SignatureType::CertPositive); + if certification && spend(&mut local, total) && sig.verify_certification(primary, Tag::UserId, &user.id).is_ok() { + consider(sig, &mut certified); + } + } + if certified.is_some() || user_revocation.is_some() { + users.push(SignedUser::new(user.id.clone(), certified.into_iter().chain(user_revocation).cloned().collect())); + } + let Some(cert) = certified else { continue }; + consider(cert, &mut newest); + // RFC 9580 §5.2.3.31: the newest self-signature on a User ID is what holds, so a + // certification after its revocation takes it back (one at the same second doesn't). + let revoked = user_revocation.is_some_and(|r| created(r) >= created(cert)); + let address = mime::address(&String::from_utf8_lossy(user.id.id())); + if !revoked && !address.is_empty() && !addresses.contains(&address) { + addresses.push(address); + address_until.push(signature_until(cert)); + } + } + let newest = newest?; + let primary_created = primary.created_at().as_secs(); + let mut signing = vec![Signing { + subkey: None, + fingerprint: Fingerprint::of(&primary.fingerprint()), + key_id: KeyId::of(&primary.legacy_key_id()), + version: primary.version(), + algorithm: primary.algorithm(), + created: primary_created, + expires: earlier(expiry(primary_created, newest), signature_until(newest)), + revoked: key_revocation.map(Revoked::of), + may_sign: flags(newest).is_none_or(|f| f.sign()), + weak: weak_key(primary), + }]; + let mut subkeys = Vec::new(); + for sub in &key.public_subkeys { + if !matches!(sub.key.version(), KeyVersion::V4 | KeyVersion::V6) { + continue; + } + let mut binding: Option<&Signature> = None; + let mut sub_revocation: Option<&Signature> = None; + for sig in &sub.signatures { + if !usable(sig) || !is_self(sig, primary) { + continue; + } + match sig.typ() { + Some(SignatureType::SubkeyRevocation) => { + if spend(&mut revocations, total) && sig.verify_subkey_binding(primary, &sub.key).is_ok() { + sub_revocation = stronger(sub_revocation, sig); + } + } + Some(SignatureType::SubkeyBinding) if spend(&mut local, total) && sig.verify_subkey_binding(primary, &sub.key).is_ok() => { + consider(sig, &mut binding); + } + _ => {} + } + } + let Some(binding) = binding else { continue }; + let place = subkeys.len(); + subkeys.push(SignedPublicSubKey::new(sub.key.clone(), std::iter::once(binding).chain(sub_revocation).cloned().collect())); + if !flags(binding).is_some_and(|f| f.sign()) { + continue; + } + // A subkey that signs must sign back that it belongs to this key (§5.2.1): else + // anybody's key could claim somebody else's subkey. + let backed = binding + .embedded_signature() + .is_some_and(|back| spend(&mut local, total) && back.verify_primary_key_binding(&sub.key, primary).is_ok()); + let sub_created = sub.key.created_at().as_secs(); + signing.push(Signing { + subkey: Some(place), + fingerprint: Fingerprint::of(&sub.key.fingerprint()), + key_id: KeyId::of(&sub.key.legacy_key_id()), + version: sub.key.version(), + algorithm: sub.key.algorithm(), + created: sub_created, + expires: earlier(expiry(sub_created, binding), signature_until(binding)), + revoked: sub_revocation.map(Revoked::of), + may_sign: backed, + weak: weak_key(&sub.key), + }); + } + // A revocation that couldn't be checked, or a key not checked whole, is no key at all: + // never a key with a part of it left out. + if revocations.starved || local.starved || total.starved { + return None; + } + let details = SignedKeyDetails::new(key_revocation.into_iter().cloned().collect(), direct.into_iter().cloned().collect(), users, Vec::new()); + let kept = SignedPublicKey::new(key.primary_key.clone(), details, subkeys); + Some(PublicKey { fingerprint: Fingerprint::of(&key.fingerprint()), key: kept, addresses, address_until, signing }) + } + + /// The primary key's fingerprint: what the owner compares, and marks Verified. + pub fn fingerprint(&self) -> &Fingerprint { + &self.fingerprint + } + + /// The primary key's key id. + pub fn key_id(&self) -> KeyId { + KeyId::of(&self.key.primary_key.legacy_key_id()) + } + + /// The addresses of the User IDs the key certified itself (lower case), in its order. + pub fn addresses(&self) -> &[String] { + &self.addresses + } + + /// Whether `address` (an address, or a From field's value) is one of the key's own. + pub fn has_address(&self, address: &str) -> bool { + let address = mime::address(address); + !address.is_empty() && self.addresses.contains(&address) + } + + /// Whether `address` was one of the key's own at `when`: its self-signature hadn't expired. + pub(crate) fn has_address_at(&self, address: &str, when: u64) -> bool { + let address = mime::address(address); + !address.is_empty() && self.addresses.iter().zip(&self.address_until).any(|(a, until)| *a == address && until.is_none_or(|end| when < end)) + } + + /// When the primary key was made, in seconds since 1970. + pub fn created(&self) -> u64 { + u64::from(self.key.primary_key.created_at().as_secs()) + } + + /// The primary key's algorithm, by name ("RSA", "Ed25519", …). + pub fn algorithm(&self) -> &'static str { + algorithm_name(self.key.primary_key.algorithm()) + } + + /// Whether the key's owner revoked it. + pub fn revoked(&self) -> bool { + self.primary().is_some_and(|s| s.revoked.is_some()) + } + + /// When the primary key stops being valid, in seconds since 1970; `None` when never. + pub fn expires(&self) -> Option { + self.primary().and_then(|s| s.expires) + } + + pub(crate) fn primary(&self) -> Option<&Signing> { + self.signing.first() + } + + /// The key as binary OpenPGP packets, to be kept and read back with [`PublicKey::parse`]. + pub fn to_bytes(&self) -> Vec { + self.key.to_bytes().unwrap_or_default() + } + + pub(crate) fn inner(&self) -> &SignedPublicKey { + &self.key + } +} + +/// The earlier of two optional times. +fn earlier(a: Option, b: Option) -> Option { + match (a, b) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } +} + +/// When a signature stops being valid by its own expiry (RFC 9580 §5.2.3.18). +fn signature_until(sig: &Signature) -> Option { + let after = sig.signature_expiration_time()?.as_secs(); + (after > 0).then(|| u64::from(created(sig)).saturating_add(u64::from(after))) +} + +/// Makes `sig` the `newest` when it is at least as new. +fn consider<'a>(sig: &'a Signature, newest: &mut Option<&'a Signature>) { + if newest.is_none_or(|n| created(sig) >= created(n)) { + *newest = Some(sig); + } +} + +/// A signature's creation time, 0 when it has none. +fn created(sig: &Signature) -> u32 { + sig.created().map_or(0, |t| t.as_secs()) +} + +/// A self-signature with a hash that may still be trusted for one: everything but MD5 (SHA-1 +/// self-signatures are still on many keys, and rPGP checks SHA-1 for collision attacks), and +/// of version 4 or 6. +fn usable(sig: &Signature) -> bool { + sig.hash_alg().is_some_and(|h| h != HashAlgorithm::Md5) && matches!(sig.version(), SignatureVersion::V4 | SignatureVersion::V6) +} + +/// Whether `sig` names `key` as its issuer, or names nobody (a hint only: the check decides). +fn is_self(sig: &Signature, key: &impl KeyDetails) -> bool { + let (ids, fps) = (sig.issuer_key_id(), sig.issuer_fingerprint()); + (ids.is_empty() && fps.is_empty()) || ids.iter().any(|id| **id == key.legacy_key_id()) || fps.iter().any(|fp| **fp == key.fingerprint()) +} + +/// The key flags a self-signature states, if it states any. +fn flags(sig: &Signature) -> Option { + sig.config()?.hashed_subpackets.iter().find_map(|p| match &p.data { + SubpacketData::KeyFlags(flags) => Some(flags.clone()), + _ => None, + }) +} + +/// When a key made at `created` stops being valid by self-signature `sig`. +fn expiry(created: u32, sig: &Signature) -> Option { + let after = sig.key_expiration_time()?.as_secs(); + (after > 0).then(|| u64::from(created).saturating_add(u64::from(after))) +} + +/// How many packet headers `bytes` holds, counted without reading them (RFC 9580 §4.2): a +/// cap on work for a block of many tiny packets. +fn count_packets(bytes: &[u8]) -> usize { + packet_tags(bytes, MAX_PACKETS.saturating_add(1)).len() +} + +/// The tags of the first `max` packets of `bytes` (RFC 9580 §4.2), read from their headers +/// alone: as far as a header reads, and a packet of partial length as the last. +pub(crate) fn packet_tags(bytes: &[u8], max: usize) -> Vec { + packets(bytes, max).0 +} + +/// [`packet_tags`], with where the packets read end: the length of `bytes` when they are all +/// there is, short of it when something follows that isn't a packet (or a packet of partial or +/// indeterminate length, or more than `max`), past it when the last says more than is there. +pub(crate) fn packets(bytes: &[u8], max: usize) -> (Vec, usize) { + let mut at = 0usize; + let mut tags = Vec::new(); + while let Some(&tag) = bytes.get(at) { + if tag & 0x80 == 0 || tags.len() >= max { + break; + } + tags.push(if tag & 0x40 != 0 { tag & 0x3f } else { (tag >> 2) & 0x0f }); + let len = |o: usize| bytes.get(at.saturating_add(o)).copied().map_or(0usize, usize::from); + let (header, body) = if tag & 0x40 != 0 { + match len(1) { + 0..=191 => (2, len(1)), + 192..=223 => (3, ((len(1) - 192) << 8) + len(2) + 192), + 255 => (6, (len(2) << 24) | (len(3) << 16) | (len(4) << 8) | len(5)), + // A partial length: the rest is one stream, counted as one. + _ => break, + } + } else { + match tag & 0x03 { + 0 => (2, len(1)), + 1 => (3, (len(1) << 8) | len(2)), + 2 => (5, (len(1) << 24) | (len(2) << 16) | (len(3) << 8) | len(4)), + _ => break, + } + }; + at = at.saturating_add(header).saturating_add(body); + } + (tags, at) +} + +/// Of two revocations of one thing, the one that counts for more: a hard one (it undoes +/// everything), else the earlier. +fn stronger<'a>(a: Option<&'a Signature>, b: &'a Signature) -> Option<&'a Signature> { + Some(match a { + Some(a) if Revoked::of(a).hard => a, + _ if Revoked::of(b).hard => b, + Some(a) if created(a) <= created(b) => a, + _ => b, + }) +} + +/// `a`, then what of `b` isn't in it: each signature once, `a`'s first. Both are what a check +/// kept of a key (signatures its owner made, at most two of a kind for each part), so nothing +/// is cut, and no date anybody wrote decides what stays. +fn union(a: &[Signature], b: &[Signature]) -> Vec { + let mut all = a.to_vec(); + for sig in b { + if !all.contains(sig) { + all.push(sig.clone()); + } + } + all +} + +/// Two checked copies of one certificate as one (RFC 9580 §10.1): every User ID, subkey and +/// self-signature of either, the old copy's first, so that a new copy can only add to it. +fn merge(old: &SignedPublicKey, new: &SignedPublicKey) -> SignedPublicKey { + let mut merged = old.clone(); + merged.details.revocation_signatures = union(&old.details.revocation_signatures, &new.details.revocation_signatures); + merged.details.direct_signatures = union(&old.details.direct_signatures, &new.details.direct_signatures); + for user in &new.details.users { + let users = &mut merged.details.users; + match users.iter().position(|u| u.id == user.id) { + Some(i) => { + if let Some(mine) = users.get_mut(i) { + mine.signatures = union(&mine.signatures, &user.signatures); + } + } + None if users.len() < MAX_USERS => users.push(user.clone()), + None => {} + } + } + for sub in &new.public_subkeys { + let fingerprint = sub.key.fingerprint(); + let subs = &mut merged.public_subkeys; + match subs.iter().position(|s| s.key.fingerprint() == fingerprint) { + Some(i) => { + if let Some(mine) = subs.get_mut(i) { + mine.signatures = union(&mine.signatures, &sub.signatures); + } + } + None if subs.len() < MAX_SUBKEYS => subs.push(sub.clone()), + None => {} + } + } + merged +} + +/// `key` with nothing but its revocations: what a merge must keep when all of it can't be. +fn revocations_only(key: &SignedPublicKey) -> SignedPublicKey { + let mut only = key.clone(); + only.details.direct_signatures.clear(); + only.details.users.clear(); + only.details.user_attributes.clear(); + for sub in &mut only.public_subkeys { + sub.signatures.retain(|s| s.typ() == Some(SignatureType::SubkeyRevocation)); + } + only +} + +/// The Public keys the caller holds: by address, fingerprint and key id. Linear look-ups; the +/// caller decides how many keys it keeps. +#[derive(Clone, Debug, Default)] +pub struct KeyRing { + keys: Vec, +} + +impl KeyRing { + /// An empty ring. + pub fn new() -> KeyRing { + KeyRing::default() + } + + /// Adds `key`. A key already here with the same fingerprint is merged with it: every User + /// ID, subkey and self-signature of both is kept and checked again, so that a copy that + /// lacks a revocation never undoes it and a copy that brings one always counts. Whether + /// the ring changed. + pub fn insert(&mut self, key: PublicKey) -> bool { + let Some(old) = self.keys.iter_mut().find(|k| k.fingerprint == key.fingerprint) else { + self.keys.push(key); + return true; + }; + let recheck = |merged: SignedPublicKey| crate::guard(|| PublicKey::checked(merged, &mut Budget::new(MAX_CHECKS_PER_CALL))).flatten(); + let merged = recheck(merge(&old.key, &key.key)).or_else(|| recheck(merge(&old.key, &revocations_only(&key.key)))); + match merged { + Some(merged) if merged.to_bytes() != old.to_bytes() => { + *old = merged; + true + } + _ => false, + } + } + + /// Every key in the ring. + pub fn keys(&self) -> &[PublicKey] { + &self.keys + } + + /// The keys that have `address` (an address, or a From field's value) as one of their own. + pub fn for_address<'a>(&'a self, address: &str) -> impl Iterator + 'a { + let address = mime::address(address); + self.keys.iter().filter(move |k| !address.is_empty() && k.addresses.contains(&address)) + } + + /// The key with primary fingerprint `fingerprint`. + pub fn get(&self, fingerprint: &Fingerprint) -> Option<&PublicKey> { + self.keys.iter().find(|k| &k.fingerprint == fingerprint) + } + + /// How many keys the ring holds. + pub fn len(&self) -> usize { + self.keys.len() + } + + /// Whether the ring holds no key. + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + /// The signing keys (primary or subkey) named by fingerprint or key id. + pub(crate) fn signers<'a>(&'a self, fps: &'a [Fingerprint], ids: &'a [KeyId]) -> impl Iterator + 'a { + self.keys + .iter() + .flat_map(move |k| k.signing.iter().filter(move |s| fps.contains(&s.fingerprint) || ids.contains(&s.key_id)).map(move |s| (k, s))) + } +} + +impl FromIterator for KeyRing { + fn from_iter>(iter: I) -> KeyRing { + let mut ring = KeyRing::new(); + for key in iter { + ring.insert(key); + } + ring + } +} diff --git a/crates/openpgp/src/keys_tests.rs b/crates/openpgp/src/keys_tests.rs new file mode 100644 index 000000000..3cc53b0bc --- /dev/null +++ b/crates/openpgp/src/keys_tests.rs @@ -0,0 +1,332 @@ +//! Keys over time: revocations that are never lost, expiry of the primary key and of +//! self-signatures, and the cost of reading keys. + +use pgp::composed::{SignedPublicKey, SignedSecretKey}; +use pgp::packet::{RevocationCode, SignatureType, SubpacketData}; +use pgp::ser::Serialize; +use pgp::types::{Duration, Password, Tag}; + +use crate::fixtures::*; +use crate::keys::Budget; +use crate::{CantCheck, Fingerprint, KeyRing, NotVerified, Outcome, PublicKey, verify}; + +const ALICE: &str = "Alice "; + +fn yes(_: &Fingerprint, _: &str) -> bool { + true +} + +fn outcome(message: &[u8], from: &str, keys: &KeyRing) -> Outcome { + verify(message, from, keys, LATER, yes).unwrap().outcome +} + +fn revoked(outcome: &Outcome) -> bool { + matches!(outcome, Outcome::CantCheck { why: CantCheck::KeyRevoked, .. }) +} + +/// `secret`'s public key revoked at `at`, for `reason` (none: no reason given). +fn key_revoked(secret: &SignedSecretKey, at: u32, reason: Option) -> SignedPublicKey { + let mut key = SignedPublicKey::from(secret.clone()); + let extra = reason.map(|r| SubpacketData::RevocationReason(r, Vec::new().into())).into_iter().collect(); + let sig = config(&secret.primary_key, SignatureType::KeyRevocation, at, extra) + .sign_key(&secret.primary_key, &Password::empty(), &key.primary_key) + .unwrap(); + key.details.revocation_signatures.push(sig); + key +} + +/// `secret`'s public key with its first subkey revoked at `at`. +fn subkey_revoked(secret: &SignedSecretKey, at: u32) -> SignedPublicKey { + let mut key = SignedPublicKey::from(secret.clone()); + let sig = config(&secret.primary_key, SignatureType::SubkeyRevocation, at, vec![]) + .sign_subkey_binding(&secret.primary_key, &key.primary_key, &Password::empty(), &key.public_subkeys[0].key) + .unwrap(); + key.public_subkeys[0].signatures.push(sig); + key +} + +/// `secret`'s public key with a new self-signature on its User ID number `user`, made at `at`. +fn recertified(secret: &SignedSecretKey, key: &mut SignedPublicKey, user: usize, at: u32, extra: Vec) { + let id = key.details.users[user].id.clone(); + let sig = config(&secret.primary_key, SignatureType::CertPositive, at, extra) + .sign_certification(&secret.primary_key, &key.primary_key, &Password::empty(), Tag::UserId, &id) + .unwrap(); + key.details.users[user].signatures.push(sig); +} + +#[test] +fn a_copy_without_the_revocation_never_undoes_it() { + let alice = secret(ALICE, Make::default()); + let mut keys = ring_of(&key_revoked(&alice, made(&alice), None)); + assert!(!keys.insert(public(&alice)), "nothing new in the copy"); + assert!(keys.keys()[0].revoked()); + assert!(revoked(&outcome(&signed_message(&alice, pgp::crypto::hash::HashAlgorithm::Sha256), ALICE, &keys))); +} + +#[test] +fn a_copy_that_brings_a_revocation_always_counts() { + let alice = secret(ALICE, Make::default()); + let mut keys = ring_of(&SignedPublicKey::from(alice.clone())); + let copy = PublicKey::parse(&key_revoked(&alice, made(&alice), None).to_bytes().unwrap()).unwrap().remove(0); + assert!(keys.insert(copy)); + assert!(keys.keys()[0].revoked()); +} + +#[test] +fn a_copy_with_a_subkey_revocation_stripped_never_undoes_it() { + let alice = secret(ALICE, Make { subkey: true, ..Make::default() }); + let mut keys = ring_of(&subkey_revoked(&alice, made(&alice))); + let message = signed_at(&alice, made(&alice) + 100); + assert!(revoked(&outcome(&message, ALICE, &keys))); + keys.insert(public(&alice)); + assert!(revoked(&outcome(&message, ALICE, &keys))); +} + +#[test] +fn the_primary_keys_expiry_and_revocation_hold_for_its_subkeys() { + let alice = secret(ALICE, Make { subkey: true, ..Make::default() }); + let t = made(&alice); + // The newest self-signature says the key expires 50 seconds after it was made. + let mut key = SignedPublicKey::from(alice.clone()); + recertified(&alice, &mut key, 0, t + 10, vec![SubpacketData::KeyExpirationTime(Duration::from_secs(50))]); + let keys = ring_of(&key); + assert!(matches!(outcome(&signed_at(&alice, t + 20), ALICE, &keys), Outcome::Verified(_))); + assert!(matches!(outcome(&signed_at(&alice, t + 100), ALICE, &keys), Outcome::CantCheck { why: CantCheck::KeyExpired, .. })); + // A revoked primary key undoes its subkey's signatures. + assert!(revoked(&outcome(&signed_at(&alice, t + 20), ALICE, &ring_of(&key_revoked(&alice, t + 50, None))))); +} + +#[test] +fn a_soft_revocation_undoes_only_what_was_signed_after_it() { + let alice = secret(ALICE, Make::default()); + let t = made(&alice); + for reason in [RevocationCode::KeySuperseded, RevocationCode::KeyRetired] { + let keys = ring_of(&key_revoked(&alice, t + 50, Some(reason))); + assert!(matches!(outcome(&signed_at(&alice, t + 20), ALICE, &keys), Outcome::Verified(_))); + assert!(revoked(&outcome(&signed_at(&alice, t + 100), ALICE, &keys))); + } + // Compromised, or no reason: everything it ever signed. + for reason in [Some(RevocationCode::KeyCompromised), None] { + let keys = ring_of(&key_revoked(&alice, t + 50, reason)); + assert!(revoked(&outcome(&signed_at(&alice, t + 20), ALICE, &keys))); + } +} + +#[test] +fn an_expired_user_id_self_signature_takes_its_address_away() { + let alice = secret(ALICE, Make { alias: true, ..Make::default() }); + let t = made(&alice); + let mut key = SignedPublicKey::from(alice.clone()); + let alias = key.details.users.iter().position(|u| u.id.id().starts_with(b"Alias")).unwrap(); + // The alias's newest self-signature expires 50 seconds after it was made; then a newer one + // on the primary User ID that doesn't, so that the key itself stays valid. + recertified(&alice, &mut key, alias, t + 10, vec![SubpacketData::SignatureExpirationTime(Duration::from_secs(50))]); + recertified(&alice, &mut key, 1 - alias, t + 20, vec![]); + let keys = ring_of(&key); + let from_alias = |at| outcome(&signed_at(&alice, at), "alias@example.org", &keys); + assert!(matches!(from_alias(t + 30), Outcome::Verified(_))); + let Outcome::Valid { why, .. } = from_alias(t + 100) else { panic!() }; + assert_eq!(why, NotVerified { address_differs: true, ..NotVerified::default() }); +} + +#[test] +fn an_expired_subkey_binding_stops_the_subkey() { + let alice = secret(ALICE, Make { subkey: true, ..Make::default() }); + let t = made(&alice); + let mut key = SignedPublicKey::from(alice.clone()); + let back = key.public_subkeys[0].signatures[0].embedded_signature().unwrap().clone(); + let mut flags = pgp::packet::KeyFlags::default(); + flags.set_sign(true); + let extra = vec![ + SubpacketData::KeyFlags(flags), + SubpacketData::SignatureExpirationTime(Duration::from_secs(50)), + SubpacketData::EmbeddedSignature(Box::new(back)), + ]; + let binding = config(&alice.primary_key, SignatureType::SubkeyBinding, t + 10, extra) + .sign_subkey_binding(&alice.primary_key, &key.primary_key, &Password::empty(), &key.public_subkeys[0].key) + .unwrap(); + key.public_subkeys[0].signatures.push(binding); + let keys = ring_of(&key); + assert!(matches!(outcome(&signed_at(&alice, t + 30), ALICE, &keys), Outcome::Verified(_))); + assert!(matches!(outcome(&signed_at(&alice, t + 100), ALICE, &keys), Outcome::CantCheck { why: CantCheck::KeyExpired, .. })); +} + +#[test] +fn rsa_under_2048_bits_is_too_weak_to_trust() { + use pgp::composed::SignedKeyDetails; + use pgp::crypto::public_key::PublicKeyAlgorithm; + use pgp::packet::{PubKeyInner, PublicKey as Packet, SecretKey, UserId}; + use pgp::types::{KeyVersion, PlainSecretParams, PublicParams, RsaPublicParams, SecretParams, SignedUser, Timestamp}; + use rand::SeedableRng; + // rPGP won't make a key this small, so it is put together from one the rsa crate makes. + let private = rsa::RsaPrivateKey::new(&mut rand::rngs::StdRng::seed_from_u64(3), 1024).unwrap(); + let t = 1_704_067_200; + let public_params = PublicParams::RSA(RsaPublicParams { key: private.to_public_key() }); + let inner = PubKeyInner::new(KeyVersion::V4, PublicKeyAlgorithm::RSA, Timestamp::from_secs(t), None, public_params).unwrap(); + let packet = Packet::from_inner(inner).unwrap(); + let secret = SecretKey::new(packet.clone(), SecretParams::Plain(PlainSecretParams::RSA(private.into()))).unwrap(); + let id = UserId::from_str(Default::default(), ALICE).unwrap(); + let mut flags = pgp::packet::KeyFlags::default(); + flags.set_sign(true); + flags.set_certify(true); + let cert = config(&secret, SignatureType::CertPositive, t, vec![SubpacketData::KeyFlags(flags)]) + .sign_certification(&secret, &packet, &Password::empty(), Tag::UserId, &id) + .unwrap(); + let key = SignedPublicKey::new(packet, SignedKeyDetails::new(vec![], vec![], vec![SignedUser::new(id, vec![cert])], vec![]), vec![]); + let keys = ring_of(&key); + let sig = config(&secret, SignatureType::Binary, t + 1, vec![]).sign(&secret, &Password::empty(), SIGNED_PART.as_bytes()).unwrap(); + let armour = pgp::composed::DetachedSignature::new(sig).to_armored_string(Default::default()).unwrap(); + let out = outcome(&pgp_mime("alice@example.org", SIGNED_PART, &armour), ALICE, &keys); + assert!(matches!(out, Outcome::CantCheck { why: CantCheck::WeakAlgorithm("RSA under 2048 bits"), .. }), "{out:?}"); +} + +#[test] +fn reading_keys_stops_when_its_budget_or_room_runs_out() { + let block = armoured_public(&secret(ALICE, Make::default())); + let many = block.repeat(20); + // At most 16 keys from one call, however many blocks. + assert_eq!(PublicKey::parse(many.as_bytes()).unwrap().len(), 16); + // A budget of checks shared by everything one call reads: the key that would overspend it is + // left out, whole, and so is every key after it. + let mut budget = Budget::new(2); + let read = PublicKey::parse_with(block.repeat(4).as_bytes(), &mut budget).unwrap(); + assert_eq!(read.len(), 2); + assert!(budget.starved); +} + +/// A signature of `typ` that names Alice's key as its issuer but is made by Mallory's: junk +/// anybody can add to a copy of Alice's key. +fn forged( + alice: &SignedSecretKey, + mallory: &SignedSecretKey, + typ: SignatureType, + at: u32, + extra: Vec, +) -> pgp::packet::SignatureConfig { + use pgp::types::KeyDetails; + let mut config = config(&mallory.primary_key, typ, at, extra); + for p in &mut config.hashed_subpackets { + if matches!(p.data, SubpacketData::IssuerFingerprint(_)) { + *p = pgp::packet::Subpacket::regular(SubpacketData::IssuerFingerprint(alice.primary_key.fingerprint())).unwrap(); + } + } + config +} + +/// Inserts `bytes` as a copy of a key, when they read as one. +fn offer(keys: &mut KeyRing, bytes: &[u8]) { + for key in PublicKey::parse(bytes).unwrap_or_default() { + keys.insert(key); + } +} + +#[test] +fn a_copy_with_junk_subkey_bindings_from_the_future_never_pushes_out_the_real_one() { + let alice = secret(ALICE, Make { subkey: true, ..Make::default() }); + let mallory = secret("Mallory ", Make { seed: 6, ..Make::default() }); + let mut copy = SignedPublicKey::from(alice.clone()); + let mut flags = pgp::packet::KeyFlags::default(); + flags.set_sign(true); + for i in 0..64 { + // Dated 2096. + let junk = forged(&alice, &mallory, SignatureType::SubkeyBinding, 4_000_000_000 + i, vec![SubpacketData::KeyFlags(flags.clone())]) + .sign_subkey_binding(&mallory.primary_key, ©.primary_key, &Password::empty(), ©.public_subkeys[0].key) + .unwrap(); + copy.public_subkeys[0].signatures.push(junk); + } + let junk = copy.to_bytes().unwrap(); + let message = signed_at(&alice, made(&alice) + 10); + // Either way round, the real key stays whole, and nothing of the junk is kept. + for first_real in [true, false] { + let mut keys = KeyRing::new(); + if first_real { + keys.insert(public(&alice)); + offer(&mut keys, &junk); + } else { + offer(&mut keys, &junk); + keys.insert(public(&alice)); + } + assert!(matches!(outcome(&message, ALICE, &keys), Outcome::Verified(_)), "real first: {first_real}"); + assert_eq!(keys.keys()[0].to_bytes(), public(&alice).to_bytes()); + } +} + +#[test] +fn a_copy_that_costs_more_checks_than_a_key_may_is_refused_not_cut_short() { + let alice = secret(ALICE, Make { subkey: true, ..Make::default() }); + let mallory = secret("Mallory ", Make { seed: 6, ..Make::default() }); + let mut copy = SignedPublicKey::from(alice.clone()); + // Junk User IDs with junk certifications before the subkey: checking them all would leave + // no checks for the subkey's binding. + for n in 0..5 { + let id = pgp::packet::UserId::from_str(Default::default(), format!("Alice {n} ")).unwrap(); + let sigs = (0..110) + .map(|i| { + forged(&alice, &mallory, SignatureType::CertPositive, made(&alice) + i, vec![]) + .sign_certification(&mallory.primary_key, ©.primary_key, &Password::empty(), Tag::UserId, &id) + .unwrap() + }) + .collect(); + copy.details.users.push(pgp::types::SignedUser::new(id, sigs)); + } + let junk = copy.to_bytes().unwrap(); + assert!(PublicKey::parse(&junk).is_err(), "a key not checked whole is no key"); + let mut keys = KeyRing::new(); + keys.insert(public(&alice)); + offer(&mut keys, &junk); + assert!(matches!(outcome(&signed_at(&alice, made(&alice) + 10), ALICE, &keys), Outcome::Verified(_))); +} + +#[test] +fn a_key_whose_revocations_or_the_calls_budget_run_out_is_refused() { + let alice = secret(ALICE, Make { subkey: true, ..Make::default() }); + let mallory = secret("Mallory ", Make { seed: 6, ..Make::default() }); + // More revocations than are checked: one left unchecked could be the real one. + let mut copy = SignedPublicKey::from(alice.clone()); + for i in 0..257 { + let junk = forged(&alice, &mallory, SignatureType::KeyRevocation, made(&alice) + i, vec![]) + .sign_key(&mallory.primary_key, &Password::empty(), ©.primary_key) + .unwrap(); + copy.details.revocation_signatures.push(junk); + } + assert!(PublicKey::parse(©.to_bytes().unwrap()).is_err()); + // The call's budget runs out after the User ID, before the subkey. + let bytes = SignedPublicKey::from(alice).to_bytes().unwrap(); + assert!(PublicKey::parse_with(&bytes, &mut Budget::new(1)).is_err()); + assert!(PublicKey::parse_with(&bytes, &mut Budget::new(3)).is_ok()); +} + +#[test] +fn a_user_id_certified_again_after_its_revocation_is_the_keys_again() { + let alice = secret(ALICE, Make::default()); + let t = made(&alice); + let mut key = SignedPublicKey::from(alice.clone()); + let id = key.details.users[0].id.clone(); + let revocation = config(&alice.primary_key, SignatureType::CertRevocation, t + 10, vec![]) + .sign_certification(&alice.primary_key, &key.primary_key, &Password::empty(), Tag::UserId, &id) + .unwrap(); + key.details.users[0].signatures.push(revocation); + let revoked = ring_of(&key); + assert!(revoked.keys()[0].addresses().is_empty(), "revoked"); + // Certified again in the same second as the revocation: which came last isn't known, so it + // stays revoked. + let mut tie = key.clone(); + recertified(&alice, &mut tie, 0, t + 10, vec![]); + assert!(ring_of(&tie).keys()[0].addresses().is_empty(), "same second"); + recertified(&alice, &mut key, 0, t + 20, vec![]); + assert_eq!(ring_of(&key).keys()[0].addresses(), ["alice@example.org".to_string()]); +} + +#[test] +fn a_user_id_longer_than_any_real_one_is_left_out_unhashed() { + let alice = secret(ALICE, Make::default()); + let mut key = SignedPublicKey::from(alice.clone()); + let long = format!("{} ", "x".repeat(5_000)); + let id = pgp::packet::UserId::from_str(Default::default(), &long).unwrap(); + let cert = config(&alice.primary_key, SignatureType::CertPositive, made(&alice), vec![]) + .sign_certification(&alice.primary_key, &key.primary_key, &Password::empty(), Tag::UserId, &id) + .unwrap(); + key.details.users.push(pgp::types::SignedUser::new(id, vec![cert])); + let keys = ring_of(&key); + assert_eq!(keys.keys()[0].addresses(), ["alice@example.org".to_string()]); +} diff --git a/crates/openpgp/src/lib.rs b/crates/openpgp/src/lib.rs new file mode 100644 index 000000000..01ec0af43 --- /dev/null +++ b/crates/openpgp/src/lib.rs @@ -0,0 +1,257 @@ +//! OpenPGP for mail: checking a message's Digital signature and reading the Public keys it +//! carries. No I/O, no clock and no keys of its own: the caller hands in the message's +//! original bytes (what `Message::original` keeps), the Public keys it keeps +//! ([`KeyRing`]), the time, and whether the owner has Verified a key; so it builds for the +//! browser too, which checks but never signs. +//! +//! - [`verify`] checks a PGP/MIME message (`multipart/signed; +//! protocol="application/pgp-signature"`, RFC 3156 §5) over the signed part's canonical +//! bytes, or a cleartext-signed ("inline") text (RFC 9580 §7), and says which of the four +//! results it is ([`Outcome`]). OpenPGP has no certificate authorities and this crate no web of +//! trust: a key is Verified only when the owner marked it after comparing its fingerprint +//! or their GnuPG keyring trusts it fully, both of which the caller knows and this doesn't. +//! - [`keys_in_message`] reads the keys a message carries: attached `application/pgp-keys` +//! parts and the `Autocrypt:` header (Autocrypt Level 1), for the caller to keep by +//! address. Nothing is fetched or sent. +//! +//! The cryptography is rPGP's (`pgp`, MIT OR Apache-2.0); the MIME around it, the armour and the +//! Cleartext Signature Framework are read here, from RFC 2045/2046, RFC 3156 and RFC 9580, so +//! that every size is capped by this crate. Everything from outside is hostile: nothing here +//! panics on any bytes (a panic inside rPGP is caught and is an unreadable signature), work is +//! linear in the input, and sizes, counts and nesting are capped. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod armour; +mod autocrypt; +mod check; +mod keys; +mod mime; +pub mod secret; + +#[cfg(test)] +mod armour_tests; +#[cfg(test)] +mod autocrypt_tests; +#[cfg(test)] +mod coverage_tests; +#[cfg(test)] +#[path = "fixtures_tests.rs"] +mod fixtures; +#[cfg(test)] +mod keys_tests; +#[cfg(test)] +mod mime_tests; +#[cfg(test)] +mod prop_tests; +#[cfg(test)] +mod secret_tests; +#[cfg(test)] +mod tests; +#[cfg(test)] +mod vector_tests; + +pub use autocrypt::{Autocrypt, parse_autocrypt}; +pub use check::{CantCheck, Checked, Coverage, Issuer, NotVerified, Outcome, Signer}; +pub use keys::{Fingerprint, KeyId, KeyRing, PublicKey}; + +/// The largest message looked at, as large as the largest any server is fetched for. +pub const MAX_MESSAGE: usize = 150 << 20; +/// How deep into wrapping `multipart/mixed` parts the signed part is looked for, and how deep +/// into multiparts an attached key. +const MAX_DEPTH: usize = 8; +/// How many parts of a message are looked at for attached keys. +const MAX_PARTS: usize = 256; + +/// Why key material couldn't be read. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// Not OpenPGP key material, or none of it checks. + #[error("no Public key that can be read")] + Unreadable, + /// Larger than any real key or header. + #[error("larger than a Public key can be")] + TooLarge, + /// More than one key where only one is taken (a secret key to import). + #[error("more than one key")] + SeveralKeys, +} + +/// The address of a From field's value, lower case: as [`verify`] takes it. Empty when the +/// field names no one address (none, or several). +pub fn from_address(value: &str) -> String { + mime::address(value).trim().to_lowercase() +} + +/// Runs `f`, giving `None` instead of letting a panic out of it: rPGP is a large library +/// reading hostile bytes, and a panic in it must not take the program down. Where panics abort (the +/// browser build) this is only a function call. +pub(crate) fn guard(f: impl FnOnce() -> T) -> Option { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).ok() +} + +/// Checks the OpenPGP Digital signature of the message `original` (its bytes as they came), +/// sent from `from` (the From field's value or its address), against `keys`, at `now` (seconds +/// since 1970). `verified(fingerprint, address)` says whether the owner has Verified that key +/// for that address: marked it after comparing its fingerprint, or their GnuPG keyring +/// trusts it fully with that User ID. +/// +/// `None` when the message has no OpenPGP Digital signature (an S/MIME one isn't this crate's, +/// and one inside `multipart/alternative` is none: the other alternative would be shown +/// unsigned). Never fails: what can't be read is [`CantCheck::Unreadable`]. A message is +/// [`Outcome::Verified`] only when the signature covers all of it ([`Coverage::whole`]); one +/// that covers part of it is at best [`Outcome::Valid`] with [`NotVerified::partly_signed`]. +pub fn verify(original: &[u8], from: &str, keys: &KeyRing, now: u64, verified: impl Fn(&Fingerprint, &str) -> bool) -> Option { + if original.len() > MAX_MESSAGE { + return looks_signed(original).then(|| Checked { outcome: Outcome::cant(CantCheck::TooLarge), coverage: Coverage::default() }); + } + let cx = check::Context { + from: mime::address(from), + keys, + now, + verified: &verified, + verifications: std::cell::Cell::new(0), + data_passes: std::cell::Cell::new(0), + }; + let (headers, body) = mime::split(original); + let mut checked = find(&headers, body, &mut Vec::new(), &cx)?; + if !checked.coverage.whole { + checked.outcome = match checked.outcome { + Outcome::Verified(signer) => Outcome::Valid { signer, why: NotVerified { partly_signed: true, ..NotVerified::default() } }, + Outcome::Valid { signer, why } => Outcome::Valid { signer, why: NotVerified { partly_signed: true, ..why } }, + other => other, + }; + } + Some(checked) +} + +/// Whether a message too large to check is a signed one, told cheaply: a PGP/MIME type at the +/// top, or a cleartext-signed block anywhere in it. +fn looks_signed(original: &[u8]) -> bool { + let (headers, _) = mime::split(original.get(..1 << 20).unwrap_or(original)); + let content_type = headers.content_type(); + let pgp_mime = content_type.value == "multipart/signed" + && content_type.get("protocol").is_some_and(|p| p.trim().eq_ignore_ascii_case("application/pgp-signature")); + pgp_mime || original.windows(34).any(|w| w == b"-----BEGIN PGP SIGNED MESSAGE-----") +} + +/// The signed part of a message at `path`, checked. +fn find(headers: &mime::Headers, body: &[u8], path: &mut Vec, cx: &check::Context<'_>) -> Option { + let content_type = headers.content_type(); + match content_type.value.as_str() { + "multipart/signed" => { + let protocol = content_type.get("protocol").unwrap_or_default().trim().to_ascii_lowercase(); + if protocol != "application/pgp-signature" { + return None; + } + // Only a PGP/MIME message at the top is all of the message. + let coverage = Coverage { whole: path.is_empty(), path: path.clone(), span: None }; + let Some(boundary) = content_type.get("boundary").filter(|b| !b.is_empty()) else { + return Some(Checked { outcome: Outcome::cant(CantCheck::Unreadable), coverage }); + }; + let parts = mime::parts(body, boundary, 3); + // RFC 1847 §2.1: exactly two parts, what was signed and the signature. + let [signed, signature] = parts.as_slice() else { return Some(Checked { outcome: Outcome::cant(CantCheck::Unreadable), coverage }) }; + let (sig_headers, sig_body) = mime::split(signature); + let sig_bytes = mime::decode(&sig_headers.encoding(), sig_body); + let packets = signature_packets(&sig_bytes); + // RFC 3156 §5: the signed part with its header, in canonical form (CR LF). + Some(Checked { outcome: check::check(&packets, &mime::crlf(signed), cx), coverage }) + } + "text/plain" => { + let text = mime::decode(&headers.encoding(), body); + let clear = armour::cleartext(&text, check::MAX_SIGNATURE_BYTES)?; + let coverage = Coverage { whole: path.is_empty() && clear.alone, path: path.clone(), span: Some(clear.span) }; + // Words shown in the block that aren't signed: nothing in it can be vouched for. + let outcome = if clear.well_formed { check::check(&clear.signature, &clear.signed, cx) } else { Outcome::cant(CantCheck::Unreadable) }; + Some(Checked { outcome, coverage }) + } + // A mailing list's wrapper: its first part, which is then only part of the message. + // Never into `multipart/alternative`, whose other alternatives would be shown unsigned. + "multipart/mixed" if path.len() < MAX_DEPTH => { + let boundary = content_type.get("boundary").filter(|b| !b.is_empty())?; + let first = *mime::parts(body, boundary, 1).first()?; + let (part_headers, part_body) = mime::split(first); + path.push(0); + find(&part_headers, part_body, path, cx) + } + _ => None, + } +} + +/// The signature packets of a PGP/MIME signature part: armoured, as RFC 3156 §5 says, or +/// binary from a writer that didn't armour them. Empty when it holds neither, which checks as +/// an unreadable signature: the message is still a signed one. +fn signature_packets(bytes: &[u8]) -> Vec { + let blocks = armour::blocks(bytes, armour::Kind::Signature, 1, check::MAX_SIGNATURE_BYTES); + match blocks.into_iter().next() { + Some(block) => block, + None if bytes.first().is_some_and(|b| b & 0x80 != 0) && bytes.len() <= check::MAX_SIGNATURE_BYTES => bytes.to_vec(), + None => Vec::new(), + } +} + +/// The Public keys a message carries, for the caller to keep by address. +#[derive(Clone, Debug, Default)] +pub struct Found { + /// Keys in attached `application/pgp-keys` parts, their self-signatures checked. + pub attached: Vec, + /// The sender's `Autocrypt:` header, when there is exactly one that reads and it is for + /// the From address (Autocrypt Level 1 §2.3). + pub autocrypt: Option, +} + +/// How many keys are taken from one message's attachments. +const MAX_ATTACHED: usize = 16; +/// How many `Autocrypt:` headers of one message are looked at. +const MAX_AUTOCRYPT: usize = 8; + +/// Reads the Public keys in the message `original`: attached key parts and the `Autocrypt:` +/// header. Nothing is fetched or sent; what can't be read is left out. All of it together costs +/// at most a fixed number of signature checks. +pub fn keys_in_message(original: &[u8]) -> Found { + let mut found = Found::default(); + if original.len() > MAX_MESSAGE { + return found; + } + let mut total = keys::Budget::new(keys::MAX_CHECKS_PER_CALL); + let (headers, body) = mime::split(original); + // One From mailbox, or none counts. + let from = headers.get("from").map(mime::address).unwrap_or_default(); + // Level 1 §2.3: not in a report (a bounce quotes the message it is about), and more than + // one header for the address is as good as none. The attributes are read first, and the + // key, which costs, only of the one header that counts. + if headers.content_type().value != "multipart/report" && !from.is_empty() { + let mut headers_for_from = + headers.all("autocrypt").take(MAX_AUTOCRYPT).filter_map(|v| autocrypt::attributes(v).ok()).filter(|a| a.address == from); + if let (Some(one), None) = (headers_for_from.next(), headers_for_from.next()) { + found.autocrypt = one.read(&mut total).ok(); + } + } + let mut parts = MAX_PARTS; + attached(&headers, body, 0, &mut parts, &mut total, &mut found.attached); + found +} + +fn attached(headers: &mime::Headers, body: &[u8], depth: usize, parts: &mut usize, total: &mut keys::Budget, out: &mut Vec) { + if *parts == 0 || out.len() >= MAX_ATTACHED { + return; + } + *parts = parts.saturating_sub(1); + let content_type = headers.content_type(); + let media = content_type.value.as_str(); + if media == "application/pgp-keys" { + let bytes = mime::decode(&headers.encoding(), body); + if let Ok(keys) = PublicKey::parse_with(&bytes, total) { + let room = MAX_ATTACHED.saturating_sub(out.len()); + out.extend(keys.into_iter().take(room)); + } + return; + } + if media.starts_with("multipart/") && depth < MAX_DEPTH { + let Some(boundary) = content_type.get("boundary").filter(|b| !b.is_empty()) else { return }; + for part in mime::parts(body, boundary, *parts) { + let (part_headers, part_body) = mime::split(part); + attached(&part_headers, part_body, depth.saturating_add(1), parts, total, out); + } + } +} diff --git a/crates/openpgp/src/mime.rs b/crates/openpgp/src/mime.rs new file mode 100644 index 000000000..743ffffe4 --- /dev/null +++ b/crates/openpgp/src/mime.rs @@ -0,0 +1,400 @@ +//! Just enough MIME to find what OpenPGP signed, written from RFC 2045, RFC 2046 and RFC 3156: +//! a part's header fields and parameters, the parts of a multipart with their bytes exactly as +//! they came, and the two transfer encodings that carry text. Everything works on borrowed bytes +//! of the original message and is linear in them. + +/// How many header fields of one part are read; the rest are ignored. +const MAX_FIELDS: usize = 1_000; + +/// A part's header fields, names in lower case, folded lines joined. +#[derive(Debug, Default)] +pub(crate) struct Headers { + fields: Vec<(String, String)>, +} + +impl Headers { + /// The first field called `name` (lower case). + pub(crate) fn get(&self, name: &str) -> Option<&str> { + self.fields.iter().find(|(n, _)| n == name).map(|(_, v)| v.as_str()) + } + + /// Every field called `name` (lower case), in order. + pub(crate) fn all<'a>(&'a self, name: &'a str) -> impl Iterator + 'a { + self.fields.iter().filter(move |(n, _)| n == name).map(|(_, v)| v.as_str()) + } + + /// The part's media type and its parameters (`text/plain` when it says none, RFC 2045 §5.2). + pub(crate) fn content_type(&self) -> Params { + let params = self.get("content-type").map(Params::parse).unwrap_or_default(); + if params.value.is_empty() { Params { value: "text/plain".into(), ..params } } else { params } + } + + /// The part's transfer encoding, in lower case. + pub(crate) fn encoding(&self) -> String { + self.get("content-transfer-encoding").map(|e| e.trim().to_ascii_lowercase()).unwrap_or_default() + } +} + +/// Splits a part into its header fields and its body: the body starts after the first empty +/// line (or is empty when there is none). +pub(crate) fn split(part: &[u8]) -> (Headers, &[u8]) { + let (head, body) = match blank_line(part) { + Some((head_end, body_start)) => (part.get(..head_end).unwrap_or_default(), part.get(body_start..).unwrap_or_default()), + None => (part, &[][..]), + }; + (parse_fields(head), body) +} + +/// Where the header of `part` ends and where its body starts. +fn blank_line(part: &[u8]) -> Option<(usize, usize)> { + // A part that starts with an empty line has no header fields at all. + if part.starts_with(b"\r\n") { + return Some((0, 2)); + } + if part.starts_with(b"\n") { + return Some((0, 1)); + } + let mut i = 0usize; + while let Some(off) = part.get(i..).and_then(|rest| rest.iter().position(|b| *b == b'\n')) { + let nl = i.saturating_add(off); + let next = nl.saturating_add(1); + match part.get(next..) { + Some(rest) if rest.starts_with(b"\r\n") => return Some((next, next.saturating_add(2))), + Some(rest) if rest.starts_with(b"\n") => return Some((next, next.saturating_add(1))), + Some(_) => i = next, + None => break, + } + } + None +} + +fn parse_fields(head: &[u8]) -> Headers { + let mut fields: Vec<(String, String)> = Vec::new(); + for raw in head.split(|b| *b == b'\n') { + let line = raw.strip_suffix(b"\r").unwrap_or(raw); + if line.is_empty() { + continue; + } + if matches!(line.first(), Some(b' ' | b'\t')) { + if let Some((_, value)) = fields.last_mut() { + value.push(' '); + value.push_str(String::from_utf8_lossy(line).trim()); + } + continue; + } + if fields.len() >= MAX_FIELDS { + continue; + } + let Some(colon) = line.iter().position(|b| *b == b':') else { continue }; + let name = String::from_utf8_lossy(line.get(..colon).unwrap_or_default()).trim().to_ascii_lowercase(); + let value = String::from_utf8_lossy(line.get(colon.saturating_add(1)..).unwrap_or_default()).trim().to_string(); + if !name.is_empty() { + fields.push((name, value)); + } + } + Headers { fields } +} + +/// A header value with `;` parameters: `multipart/signed; protocol="application/pgp-signature"`. +#[derive(Debug, Default, Clone)] +pub(crate) struct Params { + /// The value before the first `;`, in lower case. + pub(crate) value: String, + params: Vec<(String, String)>, +} + +impl Params { + pub(crate) fn parse(text: &str) -> Params { + let mut pieces = Pieces { text, at: 0 }; + let value = pieces.next().unwrap_or_default().trim().to_ascii_lowercase(); + let mut params = Vec::new(); + for piece in pieces { + let Some((name, raw)) = piece.split_once('=') else { continue }; + let name = name.trim().to_ascii_lowercase(); + if !name.is_empty() && params.len() < MAX_FIELDS { + params.push((name, unquote(raw.trim()))); + } + } + Params { value, params } + } + + /// The parameter called `name` (lower case). + pub(crate) fn get(&self, name: &str) -> Option<&str> { + self.params.iter().find(|(n, _)| n == name).map(|(_, v)| v.as_str()) + } +} + +/// The `;`-separated pieces of a header value, a `;` inside quotes not counting. +struct Pieces<'a> { + text: &'a str, + at: usize, +} + +impl<'a> Iterator for Pieces<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + let rest = self.text.get(self.at..)?; + if rest.is_empty() && self.at > 0 { + return None; + } + let (mut quoted, mut escaped) = (false, false); + for (i, c) in rest.char_indices() { + match c { + _ if escaped => escaped = false, + '\\' if quoted => escaped = true, + '"' => quoted = !quoted, + ';' if !quoted => { + self.at = self.at.saturating_add(i).saturating_add(1); + return rest.get(..i); + } + _ => {} + } + } + self.at = self.text.len().saturating_add(1); + Some(rest) + } +} + +fn unquote(raw: &str) -> String { + let Some(inner) = raw.strip_prefix('"') else { return raw.to_string() }; + let mut out = String::new(); + let mut escaped = false; + for c in inner.chars() { + match c { + _ if escaped => { + out.push(c); + escaped = false; + } + '\\' => escaped = true, + '"' => break, + _ => out.push(c), + } + } + out +} + +/// The parts of the multipart `body` with `boundary` (RFC 2046 §5.1.1), each exactly as it came: +/// from after its delimiter line to before the line break that comes ahead of the next one, +/// which belongs to the delimiter. At most `max` parts; a part after the last delimiter that +/// isn't closed still counts, as mail programs read it. +pub(crate) fn parts<'a>(body: &'a [u8], boundary: &str, max: usize) -> Vec<&'a [u8]> { + let mut out = Vec::new(); + if boundary.is_empty() { + return out; + } + let mut dash = Vec::with_capacity(boundary.len().saturating_add(2)); + dash.extend_from_slice(b"--"); + dash.extend_from_slice(boundary.as_bytes()); + // The start of the current part, once its opening delimiter has been seen. + let mut open: Option = None; + let mut line_start = 0usize; + while line_start <= body.len() { + let rest = body.get(line_start..).unwrap_or_default(); + let line_len = rest.iter().position(|b| *b == b'\n'); + let next = line_len.map_or(body.len().saturating_add(1), |n| line_start.saturating_add(n).saturating_add(1)); + if let Some(after) = rest.strip_prefix(dash.as_slice()) { + let line = after.get(..line_len.map_or(after.len(), |n| n.saturating_sub(dash.len()))).unwrap_or_default(); + let close = line.starts_with(b"--"); + let tail = if close { line.get(2..).unwrap_or_default() } else { line }; + if tail.iter().all(|b| matches!(b, b' ' | b'\t' | b'\r')) { + if let Some(start) = open.take() { + // The line break before the delimiter is the delimiter's. + let mut end = line_start.saturating_sub(1); + if end > start && body.get(end.saturating_sub(1)) == Some(&b'\r') { + end = end.saturating_sub(1); + } + let end = end.max(start); + out.push(body.get(start..end).unwrap_or_default()); + if out.len() >= max { + return out; + } + } + if close { + return out; + } + open = Some(next.min(body.len())); + } + } + line_start = next; + } + if let Some(start) = open { + out.push(body.get(start..).unwrap_or_default()); + } + out.truncate(max); + out +} + +/// `bytes` with every line break as CR LF (RFC 3156 §5: what was signed is the part's +/// canonical form). A lone LF gets its CR; nothing else changes. +pub(crate) fn crlf(bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(bytes.len().saturating_add(bytes.len() / 32)); + let mut previous = 0u8; + for &b in bytes { + if b == b'\n' && previous != b'\r' { + out.push(b'\r'); + } + out.push(b); + previous = b; + } + out +} + +/// A leaf's body without its transfer encoding: base64 and quoted-printable are undone, every +/// other encoding is the bytes themselves. +pub(crate) fn decode(encoding: &str, body: &[u8]) -> Vec { + match encoding { + "base64" => base64(body), + "quoted-printable" => quoted_printable(body), + _ => body.to_vec(), + } +} + +/// Base64 (RFC 2045 §6.8, RFC 4648), tolerant: characters outside the alphabet are skipped and +/// decoding stops at the first `=`. +pub(crate) fn base64(text: &[u8]) -> Vec { + let mut out = Vec::with_capacity(text.len() / 4 * 3); + let (mut acc, mut bits) = (0u32, 0u32); + for &c in text { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + b'/' => 63, + b'=' => break, + _ => continue, + }; + acc = (acc << 6) | u32::from(v); + bits = bits.saturating_add(6); + if bits >= 8 { + bits -= 8; + out.push(((acc >> bits) & 0xff) as u8); + } + acc &= (1 << bits) - 1; + } + out +} + +/// Quoted-printable (RFC 2045 §6.7): `=XX` is a byte, `=` at a line's end joins it to the next. +fn quoted_printable(text: &[u8]) -> Vec { + let mut out = Vec::with_capacity(text.len()); + let mut i = 0usize; + while let Some(&b) = text.get(i) { + if b != b'=' { + out.push(b); + i = i.saturating_add(1); + continue; + } + let hex = |o: usize| text.get(i.saturating_add(o)).and_then(|c| (*c as char).to_digit(16)); + if let (Some(h), Some(l)) = (hex(1), hex(2)) { + out.push(((h << 4) | l) as u8); + i = i.saturating_add(3); + continue; + } + // A soft line break, with any white space the line ends in. + let mut j = i.saturating_add(1); + while matches!(text.get(j), Some(b' ' | b'\t')) { + j = j.saturating_add(1); + } + match (text.get(j), text.get(j.saturating_add(1))) { + (Some(b'\r'), Some(b'\n')) => i = j.saturating_add(2), + (Some(b'\n'), _) => i = j.saturating_add(1), + (None, _) => i = j, + _ => { + out.push(b'='); + i = i.saturating_add(1); + } + } + } + out +} + +/// The address of the one mailbox in a From field (or an address given as it is): what is in +/// its `<…>` outside quoted strings and comments, else the value without its comments, in +/// lower case. Empty when there is no one address in it. +pub(crate) fn address(value: &str) -> String { + // RFC 5322 §3.4: a mailbox is `name ` or a bare addr-spec. Quoted strings and comments + // are read past, so that nothing in them is taken for the address; a field with more than + // one mailbox (§3.6.2: a comma between them, or more than one `<…>`) has no one address. + let (mut quoted, mut escaped, mut comment) = (false, false, 0usize); + let (mut angle, mut angles, mut closed) = (false, 0usize, false); + let (mut inside, mut outside) = (String::new(), String::new()); + for c in value.chars() { + let text = comment == 0; + match c { + _ if escaped => escaped = false, + '\\' if quoted || comment > 0 => { + escaped = true; + continue; + } + '"' if comment == 0 => quoted = !quoted, + _ if quoted => {} + '(' => comment = comment.saturating_add(1), + ')' => comment = comment.saturating_sub(1), + _ if comment > 0 => {} + '<' if angle => return String::new(), + '<' => { + angle = true; + angles = angles.saturating_add(1); + continue; + } + '>' if angle => { + angle = false; + closed = true; + continue; + } + ',' if !angle => return String::new(), + _ => {} + } + // What is in a comment, and the parentheses around it, is neither. + if text && comment == 0 && c != ')' { + if angle { inside.push(c) } else { outside.push(c) } + } + } + let spec = match angles { + 0 => outside, + 1 if closed && !angle => inside, + _ => return String::new(), + }; + let spec = spec.trim(); + // A control or format character (a zero-width space, a direction mark) would make an + // address that looks like another one: none is taken. + let clean = spec.matches('@').count() == 1 + && !spec.chars().any(|c| c.is_whitespace() || c.is_control() || is_format(c) || matches!(c, '<' | '>' | '"' | '(' | ')')); + if clean { spec.to_lowercase() } else { String::new() } +} + +/// Whether `c` is one of Unicode's format characters (general category Cf, Unicode 17.0): the +/// soft hyphen, the Arabic number signs and end-of-ayah marks, the Syriac abbreviation mark, the +/// Arabic pound and piastre marks above, the Mongolian vowel separator, zero-width marks and +/// joiners, bidirectional marks, embeddings, overrides and isolates, the word joiner and +/// invisible operators, the deprecated shaping controls, the byte-order mark, interlinear +/// annotations, the Kaithi number signs, the Egyptian hieroglyph, shorthand and musical format +/// controls, and tags. +pub(crate) fn is_format(c: char) -> bool { + matches!( + c, + '\u{00ad}' + | '\u{0600}'..='\u{0605}' + | '\u{061c}' + | '\u{06dd}' + | '\u{070f}' + | '\u{0890}'..='\u{0891}' + | '\u{08e2}' + | '\u{180e}' + | '\u{200b}'..='\u{200f}' + | '\u{202a}'..='\u{202e}' + | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{206f}' + | '\u{feff}' + | '\u{fff9}'..='\u{fffb}' + | '\u{110bd}' + | '\u{110cd}' + | '\u{13430}'..='\u{1343f}' + | '\u{1bca0}'..='\u{1bca3}' + | '\u{1d173}'..='\u{1d17a}' + | '\u{e0001}' + | '\u{e0020}'..='\u{e007f}' + ) +} diff --git a/crates/openpgp/src/mime_tests.rs b/crates/openpgp/src/mime_tests.rs new file mode 100644 index 000000000..d6fd32b69 --- /dev/null +++ b/crates/openpgp/src/mime_tests.rs @@ -0,0 +1,101 @@ +use crate::mime::{Params, address, base64, crlf, decode, parts, split}; + +#[test] +fn headers_unfold_and_parameters_unquote() { + let (headers, body) = + split(b"Content-Type: multipart/signed;\r\n\tprotocol=\"application/pgp-signature\";\r\n micalg=pgp-sha256; boundary=\"a;b\"\r\n\r\nbody"); + assert_eq!(body, b"body"); + let ct = headers.content_type(); + assert_eq!(ct.value, "multipart/signed"); + assert_eq!(ct.get("protocol"), Some("application/pgp-signature")); + assert_eq!(ct.get("boundary"), Some("a;b")); + assert_eq!(ct.get("micalg"), Some("pgp-sha256")); +} + +#[test] +fn a_part_without_fields_or_type_is_plain_text() { + let (headers, body) = split(b"\r\nonly a body"); + assert_eq!(body, b"only a body"); + assert_eq!(headers.content_type().value, "text/plain"); + let (_, body) = split(b"Subject: no body at all"); + assert!(body.is_empty()); +} + +#[test] +fn parts_are_cut_exactly_with_the_line_break_before_a_delimiter_belonging_to_it() { + let body = b"preamble\r\n--b\r\nA: 1\r\n\r\nfirst\r\n\r\n--b \r\nsecond\r\n--b--\r\nepilogue"; + assert_eq!(parts(body, "b", 10), vec![&b"A: 1\r\n\r\nfirst\r\n"[..], &b"second"[..]]); + // Bare line feeds, and a delimiter that only starts like the boundary isn't one. + let body = b"--b\nx\n--bc\ny\n--b--\n"; + assert_eq!(parts(body, "b", 10), vec![&b"x\n--bc\ny"[..]]); + // A part that is never closed still counts; an empty one too. + assert_eq!(parts(b"--b\r\n--b\r\nrest", "b", 10), vec![&b""[..], &b"rest"[..]]); + assert_eq!(parts(b"--b\r\na\r\n--b\r\nb\r\n--b--", "b", 1), vec![&b"a"[..]]); + assert!(parts(b"no delimiters", "b", 10).is_empty()); + assert!(parts(b"--\r\n", "", 10).is_empty()); +} + +#[test] +fn canonical_form_adds_carriage_returns_only_where_missing() { + assert_eq!(crlf(b"a\nb\r\nc\n"), b"a\r\nb\r\nc\r\n"); + assert_eq!(crlf(b""), b""); +} + +#[test] +fn transfer_encodings_are_undone() { + assert_eq!(base64(b"SGVs\r\nbG8=\r\n"), b"Hello"); + assert_eq!(decode("base64", b"!!QUJD"), b"ABC"); + assert_eq!(decode("quoted-printable", b"a=3Db=\r\nc= \r\nd=ZZ="), b"a=bcd=ZZ"); + assert_eq!(decode("7bit", b"same"), b"same"); +} + +#[test] +fn addresses_are_taken_from_angle_brackets_and_lower_cased() { + assert_eq!(address("Alice "), "alice@example.org"); + assert_eq!(address(" bob@example.org "), "bob@example.org"); + assert_eq!(address("\"a \" "), "c@example.org"); + assert_eq!(address("no address"), ""); + assert_eq!(address("two words@example.org"), ""); + // An address as the display name, as many writers send it. + assert_eq!(address("bob@example.org "), "bob@example.org"); + assert_eq!(address("\"Bob, the builder\" "), "bob@example.org"); + // Two mailboxes: no one address. + assert_eq!(address("alice@example.org, Mallory "), ""); + assert_eq!(address("Alice "), ""); + assert_eq!(address("alice@example.org mallory@example.org"), ""); + // What is in a comment or a quoted string is never the address, wherever it is. + assert_eq!(address("Mallory ()"), "mallory@evil.example"); + assert_eq!(address("Mallory \"\""), "mallory@evil.example"); + assert_eq!(address("mallory@evil.example (alice@example.org)"), "mallory@evil.example"); + assert_eq!(address("Alice