mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa705aef90
|
||
|
|
39260a8817
|
||
|
|
6b9640aa48
|
||
|
|
ad59f518ff
|
||
|
|
e21643063e
|
||
|
|
246fc989f0
|
||
|
|
2e2d1b3506
|
||
|
|
1747d321d8
|
||
|
|
505990e49e
|
||
|
|
95b475ff08
|
||
|
|
2297fdc91c
|
||
|
|
4580b79d4f
|
||
|
|
edb32c23b4
|
||
|
|
4a467641aa
|
||
|
|
bd43a12dee
|
||
|
|
c496e41c55
|
||
|
|
68f026c990
|
||
|
|
62d80c8905
|
||
|
|
78c9c841c6
|
||
|
|
e49d500d23
|
||
|
|
6feabd723b
|
||
|
|
ea72516d5c
|
||
|
|
755330e585
|
||
|
|
c40b074b3c
|
||
|
|
9910da2914
|
||
|
|
ebc0122264
|
||
|
|
44cba8abe0
|
||
|
|
99009aae9a
|
||
|
|
72130b9023
|
||
|
|
734cfc2eb3
|
||
|
|
a8156697fa
|
||
|
|
c093dfe5ac
|
||
|
|
ee6f3e4a27
|
||
|
|
16ed53fead
|
||
|
|
1d9e3629c9
|
||
|
|
b3d87b4f7d
|
||
|
|
98c48b99c0
|
||
|
|
f40639c799
|
||
|
|
51cc079a85
|
||
|
|
aa3083792a
|
||
|
|
6849006ad7
|
||
|
|
042a04ab6e
|
||
|
|
9f28af4e6b
|
||
|
|
d61dba5096
|
||
|
|
f6df3e28cb
|
||
|
|
deb1ef2428
|
||
|
|
b5e5340edc
|
||
|
|
425d1505cf
|
||
|
|
6e40f55440
|
||
|
|
7998114dc3
|
||
|
|
9a96f78177
|
||
|
|
b50f794d52
|
||
|
|
d32fa969d6
|
||
|
|
020e2d01a3
|
||
|
|
3d6f8548c3
|
||
|
|
063e910473
|
||
|
|
b0ced34b4c
|
||
|
|
4060a88031
|
||
|
|
fb43441c56
|
||
|
|
bf628dc999
|
||
|
|
de22aa9b4d
|
||
|
|
65a9554173
|
||
|
|
e755785147
|
||
|
|
5d5b6d56e7
|
||
|
|
d0fdf1c5da
|
||
|
|
1db4ecfafa
|
||
|
|
47039bbe2d
|
||
|
|
bc954e6e0b
|
@@ -0,0 +1,64 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
# Test/lint gate (offline, no secrets) on every push/PR, plus a live-network
|
||||||
|
# job that exercises the real source sites and the token-gated pixiv tests.
|
||||||
|
#
|
||||||
|
# Layering:
|
||||||
|
# test — fmt + clippy + the full offline unit suite. Runs on every push
|
||||||
|
# and PR, including forks (it needs no secrets).
|
||||||
|
# live — the #[ignore]d live-network tests plus the pixiv tests that are
|
||||||
|
# gated on PIXIV_REFRESH_TOKEN. Runs on schedule / manual dispatch
|
||||||
|
# / tag pushes only, because pull requests from forks cannot read
|
||||||
|
# repository secrets. continue-on-error keeps a flaky external site
|
||||||
|
# from blocking, while the run still records the outcome.
|
||||||
|
#
|
||||||
|
# Test gating convention (keep in sync with AGENTS.md "Testing & QA"):
|
||||||
|
# - pure unit tests: plain #[test] / #[tokio::test], always run.
|
||||||
|
# - live-network tests: #[ignore = "live network: ..."], only run here.
|
||||||
|
# - token-gated tests (pixiv): #[tokio::test] with an early return when
|
||||||
|
# PIXIV_REFRESH_TOKEN is absent or empty (empty = unset CI secret).
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [master]
|
||||||
|
pull_request:
|
||||||
|
schedule:
|
||||||
|
# Weekly probe of the live endpoints, so external API changes surface.
|
||||||
|
- cron: '0 3 * * 1'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
components: clippy, rustfmt
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- name: Check formatting
|
||||||
|
run: cargo fmt --check
|
||||||
|
- name: Lint (deny warnings)
|
||||||
|
run: cargo clippy --workspace --all-targets -- -D warnings
|
||||||
|
- name: Run offline tests
|
||||||
|
run: cargo test --workspace
|
||||||
|
|
||||||
|
live:
|
||||||
|
needs: test
|
||||||
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v')
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
PIXIV_REFRESH_TOKEN: ${{ secrets.PIXIV_REFRESH_TOKEN }}
|
||||||
|
TWITTER_AUTH_TOKEN: ${{ secrets.TWITTER_AUTH_TOKEN }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
# Full suite: with the secret present, the pixiv token-gated tests run;
|
||||||
|
# without it they skip themselves. Live tests stay #[ignore]d here.
|
||||||
|
- name: Run token-gated tests
|
||||||
|
run: cargo test --workspace
|
||||||
|
# The live-network tests, by the "live" name filter (all #[ignore]d).
|
||||||
|
- name: Run live-network tests
|
||||||
|
run: cargo test --workspace -- --ignored live
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
Telegram bot (teloxide) that turns post links from X/Twitter, Pixiv, and Bluesky into media messages (images, video, GIF) with the post's title, author, and tags. It supports batch media splitting, retry with persistence, inline queries, forward-channel rebinding with caption templates, and Pixiv ugoira→MP4 transcoding. README and user-facing strings are in Chinese. The project is a Rust port of a Python predecessor (see `queue.rs` comments referencing `utils/task_queue.py`).
|
Telegram bot (teloxide) that turns post links from X/Twitter, Pixiv, and Bluesky into media messages (images, video, GIF) with the post's title, author, and tags. It supports batch media splitting, retry with persistence, inline queries, forward-channel rebinding with caption templates, and Pixiv ugoira→MP4 transcoding. README and user-facing strings are in Chinese. The project is a Rust port of a Python predecessor (see `queue.rs` comments referencing `utils/task_queue.py`).
|
||||||
|
|
||||||
Two-crate Cargo workspace (both v1.0.3, edition 2024, resolver 3):
|
Two-crate Cargo workspace (both v1.2.0, edition 2024, resolver 3):
|
||||||
|
|
||||||
- **`crates/x-media`** — library that fetches and normalizes media from the three sites. Pure, no Telegram knowledge.
|
- **`crates/x-media`** — library that fetches and normalizes media from the three sites. Pure, no Telegram knowledge.
|
||||||
- **`crates/xmedia-bot`** — the bot binary: teloxide dispatcher, SQLite-backed chat state, persistent task queue.
|
- **`crates/xmedia-bot`** — the bot binary: teloxide dispatcher, SQLite-backed chat state, persistent task queue.
|
||||||
@@ -18,7 +18,7 @@ Telegram update → Dispatcher (polling or axum webhook) → dptree branches
|
|||||||
└─ callback_query → "forward" (copy to channel) / "template|<name>" (apply caption template)
|
└─ callback_query → "forward" (copy to channel) / "template|<name>" (apply caption template)
|
||||||
```
|
```
|
||||||
|
|
||||||
Message flow: `message_handler` extracts URLs (from `url`/`text_link` entities, text + caption, deduped) → `x_media::site::fetch(url)` → `Fetched` → builds a `Task` → `send::send_media_sequence` (media groups ≤ 9, caption on first item) or `send::send_animation`. On Telegram URL-fetch failure or size error (`send_batch_via_upload`): download via `x_media::site::download_media` to a temp file (≤ 10 MiB), sniff magic bytes (`sniff_ext`), upload via multipart; oversized items fall back to `fallback_url`. On failure: `enqueue_retry` persists resume-state `Task` into the SQLite queue → single worker leases (120 s lock TTL) → retry with exponential backoff (≤ 30 s, `MAX_RETRIES = 2`) → dead-letter → `notify_failure`. Success → `post_send_actions`: edit-before-forward prompt with inline buttons, or `copy_messages` to the bound forward channel.
|
Message flow: `message_handler` extracts URLs (from `url`/`text_link` entities, text + caption, deduped) → `x_media::site::fetch(url)` → `Fetched` → builds a `Task` → `send::send_media_sequence` (media groups ≤ 9, caption on first item) or `send::send_animation`. On Telegram URL-fetch failure or size error (`send_batch_via_upload`): download via `x_media::site::download_media` to a temp file (≤ 10 MiB), sniff magic bytes (`sniff_ext`), upload via multipart; oversized items fall back to `fallback_url`. On failure: `enqueue_retry` persists resume-state `Task` into the SQLite queue → workers lease (120 s lock TTL) → retry with exponential backoff (≤ 30 s, `MAX_RETRIES = 2`) → dead-letter → `notify_failure`. Success → `post_send_actions`: edit-before-forward prompt with inline buttons, or `copy_messages` to the bound forward channel.
|
||||||
|
|
||||||
The `x-media` library: `site::fetch(url)` dispatches (in order) twitter → bsky → pixiv via per-site regex `PATTERN` and returns `Ok(None)` for unmatched URLs. `Fetched { source_url, caption, title, media: Vec<Media>, sensitive, … }`; `caption_with(format)` substitutes `{url} {author} {author_url} {title} {tags}`.
|
The `x-media` library: `site::fetch(url)` dispatches (in order) twitter → bsky → pixiv via per-site regex `PATTERN` and returns `Ok(None)` for unmatched URLs. `Fetched { source_url, caption, title, media: Vec<Media>, sensitive, … }`; `caption_with(format)` substitutes `{url} {author} {author_url} {title} {tags}`.
|
||||||
|
|
||||||
@@ -28,9 +28,9 @@ The `x-media` library: `site::fetch(url)` dispatches (in order) twitter → bsky
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `crates/x-media/src/` | Fetch library. `site/mod.rs` = dispatcher + `Fetched`/`FetchError`/`download_media`/`media_size`; `media.rs` = `Media` enum; `examples/fetch.rs` = end-to-end usage sample |
|
| `crates/x-media/src/` | Fetch library. `site/mod.rs` = dispatcher + `Fetched`/`FetchError`/`download_media`/`media_size`; `media.rs` = `Media` enum; `examples/fetch.rs` = end-to-end usage sample |
|
||||||
| `crates/x-media/src/site/<twitter\|pixiv\|bsky>/` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, site struct, `From<SiteStruct> for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`) |
|
| `crates/x-media/src/site/<twitter\|pixiv\|bsky>/` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, site struct, `From<SiteStruct> for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`) |
|
||||||
| `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, queue worker start, pixiv validation, 300 s edit-expiry sweep, dptree handler tree, webhook vs polling dispatch |
|
| `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, command registration (`register_commands`), shared `send::BOT` force-init, queue worker start, pixiv validation, 300 s edit-expiry sweep, dptree handler tree, webhook vs polling dispatch |
|
||||||
| `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` |
|
| `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` |
|
||||||
| `crates/xmedia-bot/src/handlers.rs` | `Command` enum (teloxide `BotCommands`), message/inline/callback handlers, URL extraction, global statics; per-URL work spawned with a `Semaphore(8)` cap (teloxide's per-chat workers are sequential — batch-forwards need concurrency) |
|
| `crates/xmedia-bot/src/handlers.rs` | `Command` enum (teloxide `BotCommands`), message/inline/callback handlers, URL extraction, global statics; per-URL work flows through a bounded job channel (256) drained by `URL_WORKERS = 8` workers (`start_url_workers`) — backpressure instead of unbounded spawns (teloxide's per-chat workers are sequential — batch-forwards need concurrency) |
|
||||||
| `crates/xmedia-bot/src/state.rs` | `ChatStore`: parking_lot `Mutex<HashMap>` cache + SQLite write-through (`chat_state` table) |
|
| `crates/xmedia-bot/src/state.rs` | `ChatStore`: parking_lot `Mutex<HashMap>` cache + SQLite write-through (`chat_state` table) |
|
||||||
| `crates/xmedia-bot/src/link_cache.rs` | `LinkCache`: SQLite-backed cache (`link_cache` table) of successfully sent posts — raw caption fields + Telegram `file_id`s; repeat links re-send locally (no fetch/upload), TTL + prune, invalidated on permanent send failure |
|
| `crates/xmedia-bot/src/link_cache.rs` | `LinkCache`: SQLite-backed cache (`link_cache` table) of successfully sent posts — raw caption fields + Telegram `file_id`s; repeat links re-send locally (no fetch/upload), TTL + prune, invalidated on permanent send failure |
|
||||||
| `crates/xmedia-bot/src/queue.rs` | `PersistentTaskQueue`: SQLite-backed queue (`tasks` table), `QUEUE_WORKERS = 4` concurrent workers (lease via `BEGIN IMMEDIATE` + `locked_until` TTL), retry→dead-letter, `Notify::notify_waiters` wakeup, `busy_timeout` on all connections |
|
| `crates/xmedia-bot/src/queue.rs` | `PersistentTaskQueue`: SQLite-backed queue (`tasks` table), `QUEUE_WORKERS = 4` concurrent workers (lease via `BEGIN IMMEDIATE` + `locked_until` TTL), retry→dead-letter, `Notify::notify_waiters` wakeup, `busy_timeout` on all connections |
|
||||||
@@ -48,12 +48,12 @@ cargo clippy --workspace --all-targets # lint (Clippy is the configured IDE lint
|
|||||||
cargo fmt --check # formatting
|
cargo fmt --check # formatting
|
||||||
```
|
```
|
||||||
|
|
||||||
Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb`. Runtime requires **ffmpeg** (built into the image).
|
Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb`. Runtime requires **ffmpeg** (built into the image). The builder fetches crates.io + ffmpeg; on restricted networks pass proxy build args, e.g. `--build-arg HTTP_PROXY=http://host.docker.internal:10808 --build-arg HTTPS_PROXY=…` (Docker Desktop builds can't reach the host loopback — use `host.docker.internal`).
|
||||||
|
|
||||||
## Code Conventions & Common Patterns
|
## Code Conventions & Common Patterns
|
||||||
|
|
||||||
- **No anyhow/thiserror.** Errors are hand-rolled enums with manual `Display`/`source()`/`From` impls: `QueueError` (`Retryable { delay_seconds, payload }` / `Permanent`), `SendError` (Retryable/Permanent), `FetchError` (`Http`/`Json`/`Pixiv`/`NotFound`/`Blocked`), `PixivError`, `Classification`. New errors should follow this pattern.
|
- **No anyhow/thiserror.** Errors are hand-rolled enums with manual `Display`/`source()`/`From` impls: `QueueError` (`Retryable { delay_seconds, payload }` / `Permanent`), `SendError` (Retryable/Permanent), `FetchError` (`Http`/`Json`/`Pixiv`/`NotFound`/`Blocked`), `PixivError`, `Classification`. New errors should follow this pattern.
|
||||||
- **Global state via `std::sync::LazyLock` statics**, not DI: `CONFIG`, `CHAT_STORE`, `TASK_QUEUE` in `handlers.rs`; shared reqwest `CLIENT` in `x-media/src/site/mod.rs`. `Bot` is passed/cloned into handlers; queue workers rebuild `Bot::from_env()`.
|
- **Global state via `std::sync::LazyLock` statics**, not DI: `CONFIG`, `CHAT_STORE`, `TASK_QUEUE` in `handlers.rs`; shared reqwest `CLIENT` in `x-media/src/site/mod.rs`. `Bot` is passed/cloned into handlers; queue workers share the process-wide `send::BOT` (`LazyLock<Bot>`, force-initialized in `main` so a missing token fails at startup).
|
||||||
- **Async**: tokio multi-thread runtime (`#[tokio::main]` default). All rusqlite I/O inside `tokio::task::spawn_blocking`. Long loops use `tokio::select!` with `tokio::sync::{watch, Notify}` stop/wake channels. No streams.
|
- **Async**: tokio multi-thread runtime (`#[tokio::main]` default). All rusqlite I/O inside `tokio::task::spawn_blocking`. Long loops use `tokio::select!` with `tokio::sync::{watch, Notify}` stop/wake channels. No streams.
|
||||||
- **Blocking sync primitives**: `parking_lot::Mutex` for hot caches, `tokio::sync::Mutex` for async-shared state (pixiv token cache), `AtomicBool` for feature gates.
|
- **Blocking sync primitives**: `parking_lot::Mutex` for hot caches, `tokio::sync::Mutex` for async-shared state (pixiv token cache), `AtomicBool` for feature gates.
|
||||||
- **Site adapter convention** (no trait, no enum dispatch — follow the existing convention): each site module exports `PATTERN: LazyLock<Regex>`, `enabled() -> bool`, `fetch_from_url(url) -> Result<Fetched, FetchError>`; `site/mod.rs` re-exports the site struct and `fetch_once` adds one guarded if-branch. Adding a site = new `site/<name>/{mod.rs,interface.rs,model.rs}` + one branch in `fetch_once`.
|
- **Site adapter convention** (no trait, no enum dispatch — follow the existing convention): each site module exports `PATTERN: LazyLock<Regex>`, `enabled() -> bool`, `fetch_from_url(url) -> Result<Fetched, FetchError>`; `site/mod.rs` re-exports the site struct and `fetch_once` adds one guarded if-branch. Adding a site = new `site/<name>/{mod.rs,interface.rs,model.rs}` + one branch in `fetch_once`.
|
||||||
@@ -68,10 +68,11 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `crates/xmedia-bot/src/main.rs` | Startup sequence, webhook vs polling, graceful shutdown (SIGINT via teloxide ctrlc / SIGTERM via `stop_token` for docker, → sweep stop → admin msg → queue stop) |
|
| `crates/xmedia-bot/src/main.rs` | Startup sequence, webhook vs polling, graceful shutdown (SIGINT via teloxide ctrlc / SIGTERM via `stop_token` for docker, → sweep stop → admin msg → queue stop) |
|
||||||
| `crates/xmedia-bot/src/handlers.rs` | `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `data/task_queue.db` **relative to CWD**); command dispatch; URL extraction; retry enqueue |
|
| `crates/xmedia-bot/src/handlers.rs` | `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `data/task_queue.db` **relative to CWD**); command dispatch; URL extraction; retry enqueue |
|
||||||
| `crates/xmedia-bot/src/send.rs` | Constants `MAX_MEDIA_GROUP = 9`, `MAX_UPLOAD_BYTES = 10 MiB`; fallback chain; `classify_request_error` |
|
| `crates/xmedia-bot/src/send.rs` | Constants `MAX_MEDIA_GROUP = 9`; fallback chain; `classify_request_error`; download-and-reupload fallback triggered only by Telegram API errors (`is_media_fetch_failure` / `is_size_error`) |
|
||||||
|
| `crates/xmedia-bot/src/photo.rs` | Pure-Rust photo processing (no ffmpeg): `png` (image-png) decode/encode + `zune-jpeg` decode + `fast_image_resize` Lanczos3 downscale + `jpeg-encoder`. Photos over Telegram's limits (width + height > 10000 px → `PHOTO_INVALID_DIMENSIONS`; bytes > 10 MiB) are decoded, downscaled keeping the format, PNG bit depth > 24 (RGBA 32-bit / 16-bit per channel) reduced to 24-bit RGB with alpha flattened white (≤24-bit untouched, never upconverted), and transcoded to JPEG only if still over the cap; memory budget guarded, otherwise the item's smaller fallback URL |
|
||||||
| `crates/x-media/src/site/mod.rs` | Dispatcher, `Fetched`/`FetchError`, shared `CLIENT`, `download_media` (adds `Referer: https://www.pixiv.net/` for `pximg.net` hotlink protection) |
|
| `crates/x-media/src/site/mod.rs` | Dispatcher, `Fetched`/`FetchError`, shared `CLIENT`, `download_media` (adds `Referer: https://www.pixiv.net/` for `pximg.net` hotlink protection) |
|
||||||
| `crates/x-media/src/site/pixiv/api.rs` | OAuth token exchange (hardcoded app client id/secret), access-token cache, ugoira zip→MP4 via ffmpeg in `spawn_blocking` |
|
| `crates/x-media/src/site/pixiv/api.rs` | OAuth token exchange (hardcoded app client id/secret), access-token cache, ugoira zip→MP4 via ffmpeg in `spawn_blocking` |
|
||||||
| `Dockerfile` | Multi-stage: cached dep layer via stub sources + `touch *.rs` mtime hack, static ffmpeg from ffmpeg.martin-riedl.de (`FFMPEG_URL` arg, `unzip -t` integrity check), `debian:bookworm-slim` runtime, entrypoint |
|
| `Dockerfile` | Multi-stage: cached dep layer via stub sources + `touch *.rs` mtime bump (cargo's freshness is mtime-based and `cargo clean -p` removes 0 files — the touch is what forces the real sources to rebuild while deps stay cached), static ffmpeg from ffmpeg.martin-riedl.de (`FFMPEG_URL` arg, optional `FFMPEG_SHA256` checksum, `unzip -t` integrity check), `debian:bookworm-slim` runtime, entrypoint. Runtime ships **no libssl/libcrypto/CA bundle** — rustls webpki-roots handles all TLS, and the static ffmpeg only processes local files (downloads go through reqwest) |
|
||||||
| `docker-entrypoint.sh` | Privilege drop: `useradd` with `LOCAL_USER_ID` (default 9001) + `setpriv` (no gosu on bookworm-slim) |
|
| `docker-entrypoint.sh` | Privilege drop: `useradd` with `LOCAL_USER_ID` (default 9001) + `setpriv` (no gosu on bookworm-slim) |
|
||||||
| `docker-compose.yml.example` | Deployment env reference (real `docker-compose.yml` is gitignored). Ships nginx-proxy + acme-companion: webhook mode needs TLS termination in front (teloxide's axum listener is HTTP-only; `WEBHOOK_CERT` only feeds `set_webhook`), bot exposes `VIRTUAL_HOST`/`VIRTUAL_PORT` on the shared `proxy` network, no host port; container names `nginx-proxy`/`acme-companion`/`tgxmb`, start order via `depends_on` (proxy → acme → bot) |
|
| `docker-compose.yml.example` | Deployment env reference (real `docker-compose.yml` is gitignored). Ships nginx-proxy + acme-companion: webhook mode needs TLS termination in front (teloxide's axum listener is HTTP-only; `WEBHOOK_CERT` only feeds `set_webhook`), bot exposes `VIRTUAL_HOST`/`VIRTUAL_PORT` on the shared `proxy` network, no host port; container names `nginx-proxy`/`acme-companion`/`tgxmb`, start order via `depends_on` (proxy → acme → bot) |
|
||||||
| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master; **no test step**; buildx gha cache (`cache-from`/`cache-to`, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs |
|
| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master; **no test step**; buildx gha cache (`cache-from`/`cache-to`, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs |
|
||||||
@@ -81,7 +82,8 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
|||||||
|
|
||||||
- **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features.
|
- **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features.
|
||||||
- Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently.
|
- Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently.
|
||||||
- **Two reqwest versions coexist in the lock** (0.12.28 via teloxide, 0.13.3 in x-media) — don't unify casually.
|
- **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls"]` (webpki-roots baked in, so the image ships no CA bundle). One reqwest 0.12.28 in the lock.
|
||||||
|
- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build).
|
||||||
- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only).
|
- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only).
|
||||||
- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `data/task_queue.db` is CWD-relative — run from the workspace root, or `/app` in Docker. Mount `./data` and `./cert` volumes.
|
- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `data/task_queue.db` is CWD-relative — run from the workspace root, or `/app` in Docker. Mount `./data` and `./cert` volumes.
|
||||||
- `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`.
|
- `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`.
|
||||||
@@ -89,10 +91,10 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
|||||||
|
|
||||||
## Testing & QA
|
## Testing & QA
|
||||||
|
|
||||||
- **~51 tests, all inline `#[cfg(test)] mod tests`** — no `tests/` integration directories. Framework: built-in Rust test + `#[tokio::test]` (dev-deps only in `x-media`: tokio macros/rt-multi-thread, dotenv).
|
- **~80 tests, all inline `#[cfg(test)] mod tests`** — no `tests/` integration directories. Framework: built-in Rust test + `#[tokio::test]` (dev-deps only in `x-media`: tokio macros/rt-multi-thread, dotenv).
|
||||||
- No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches.
|
- No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches.
|
||||||
- Live-network tests exist in `site/twitter/interface.rs` (3), `site/bsky/interface.rs` (2), `site/pixiv/interface.rs`/`api.rs` (env-gated on `PIXIV_REFRESH_TOKEN`/dotenv, skip by early return). Run the full suite with `cargo test --workspace`.
|
- Live-network tests exist in `site/twitter/interface.rs` (3), `site/bsky/interface.rs` (2), `site/pixiv/interface.rs`/`api.rs`. Test gating convention (enforced by `.github/workflows/ci.yml`): pure unit tests always run; live-network tests carry `#[ignore = "live network: ..."]` (run via `cargo test --workspace -- --ignored live`); token-gated pixiv tests early-return when `PIXIV_REFRESH_TOKEN` is absent **or empty** (an unset GitHub secret arrives as `""` — `is_err()` alone would run them tokenless and fail). Run the full offline suite with `cargo test --workspace`.
|
||||||
- Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`.
|
- Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`.
|
||||||
- **CI runs no tests** — `.github/workflows/docker.yml` only builds/pushes the image; verification is a local responsibility.
|
- **CI** — `.github/workflows/ci.yml` runs `cargo fmt --check` + `cargo clippy --workspace --all-targets -- -D warnings` + `cargo test --workspace` (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `PIXIV_REFRESH_TOKEN`/`TWITTER_AUTH_TOKEN` from secrets, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds/pushes the image only.
|
||||||
- Untested and hard to test without a mock seam: `handlers.rs` (depends directly on teloxide `Bot`); `main.rs`, `config.rs`, `state.rs`; `media.rs`, `lib.rs`, all `model.rs`.
|
- Untested and hard to test without a mock seam: `handlers.rs` (depends directly on teloxide `Bot`); `main.rs`, `config.rs`, `state.rs`; `media.rs`, `lib.rs`, all `model.rs`.
|
||||||
- No coverage tracking, no lint gate in CI.
|
- No coverage tracking.
|
||||||
|
|||||||
Generated
+142
-494
@@ -16,7 +16,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"cipher",
|
"cipher",
|
||||||
"cpufeatures",
|
"cpufeatures 0.2.17",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -99,28 +99,6 @@ version = "1.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "aws-lc-rs"
|
|
||||||
version = "1.17.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
|
|
||||||
dependencies = [
|
|
||||||
"aws-lc-sys",
|
|
||||||
"zeroize",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "aws-lc-sys"
|
|
||||||
version = "0.41.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
|
|
||||||
dependencies = [
|
|
||||||
"cc",
|
|
||||||
"cmake",
|
|
||||||
"dunce",
|
|
||||||
"fs_extra",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "axum"
|
name = "axum"
|
||||||
version = "0.8.9"
|
version = "0.8.9"
|
||||||
@@ -266,9 +244,20 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg_aliases"
|
name = "cfg_aliases"
|
||||||
version = "0.2.1"
|
version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chacha20"
|
||||||
|
version = "0.10.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures 0.3.0",
|
||||||
|
"rand_core 0.10.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrono"
|
name = "chrono"
|
||||||
@@ -292,15 +281,6 @@ dependencies = [
|
|||||||
"inout",
|
"inout",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "cmake"
|
|
||||||
version = "0.1.58"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
|
|
||||||
dependencies = [
|
|
||||||
"cc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "colored"
|
name = "colored"
|
||||||
version = "3.1.1"
|
version = "3.1.1"
|
||||||
@@ -310,42 +290,12 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "combine"
|
|
||||||
version = "4.6.7"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
|
|
||||||
dependencies = [
|
|
||||||
"bytes",
|
|
||||||
"memchr",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "constant_time_eq"
|
name = "constant_time_eq"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
|
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "core-foundation"
|
|
||||||
version = "0.9.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "core-foundation"
|
|
||||||
version = "0.10.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation-sys"
|
name = "core-foundation-sys"
|
||||||
version = "0.8.7"
|
version = "0.8.7"
|
||||||
@@ -361,6 +311,15 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cpufeatures"
|
||||||
|
version = "0.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crc"
|
name = "crc"
|
||||||
version = "3.4.0"
|
version = "3.4.0"
|
||||||
@@ -505,6 +464,15 @@ dependencies = [
|
|||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "document-features"
|
||||||
|
version = "0.2.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
|
||||||
|
dependencies = [
|
||||||
|
"litrs",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dotenv"
|
name = "dotenv"
|
||||||
version = "0.15.0"
|
version = "0.15.0"
|
||||||
@@ -521,12 +489,6 @@ dependencies = [
|
|||||||
"futures",
|
"futures",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "dunce"
|
|
||||||
version = "1.0.5"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dyn-clone"
|
name = "dyn-clone"
|
||||||
version = "1.0.20"
|
version = "1.0.20"
|
||||||
@@ -539,15 +501,6 @@ version = "1.15.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "encoding_rs"
|
|
||||||
version = "0.8.35"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "env_logger"
|
name = "env_logger"
|
||||||
version = "0.10.2"
|
version = "0.10.2"
|
||||||
@@ -599,12 +552,33 @@ version = "0.1.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fast_image_resize"
|
||||||
|
version = "6.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e9c50201dc184ba6553da1695aac20a042efffbe2d84542cee31917c86c3ab1e"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"document-features",
|
||||||
|
"num-traits",
|
||||||
|
"thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fastrand"
|
name = "fastrand"
|
||||||
version = "2.4.1"
|
version = "2.4.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fdeflate"
|
||||||
|
version = "0.3.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
|
||||||
|
dependencies = [
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "find-msvc-tools"
|
name = "find-msvc-tools"
|
||||||
version = "0.1.9"
|
version = "0.1.9"
|
||||||
@@ -621,33 +595,12 @@ dependencies = [
|
|||||||
"miniz_oxide",
|
"miniz_oxide",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fnv"
|
|
||||||
version = "1.0.7"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "foldhash"
|
name = "foldhash"
|
||||||
version = "0.1.5"
|
version = "0.1.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "foreign-types"
|
|
||||||
version = "0.3.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
|
||||||
dependencies = [
|
|
||||||
"foreign-types-shared",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "foreign-types-shared"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "form_urlencoded"
|
name = "form_urlencoded"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
@@ -657,12 +610,6 @@ dependencies = [
|
|||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fs_extra"
|
|
||||||
version = "1.3.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures"
|
name = "futures"
|
||||||
version = "0.3.32"
|
version = "0.3.32"
|
||||||
@@ -795,29 +742,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
|
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
|
"js-sys",
|
||||||
"libc",
|
"libc",
|
||||||
"r-efi 6.0.0",
|
"r-efi 6.0.0",
|
||||||
|
"rand_core 0.10.1",
|
||||||
"wasip2",
|
"wasip2",
|
||||||
"wasip3",
|
"wasip3",
|
||||||
]
|
"wasm-bindgen",
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "h2"
|
|
||||||
version = "0.4.14"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
|
|
||||||
dependencies = [
|
|
||||||
"atomic-waker",
|
|
||||||
"bytes",
|
|
||||||
"fnv",
|
|
||||||
"futures-core",
|
|
||||||
"futures-sink",
|
|
||||||
"http",
|
|
||||||
"indexmap 2.14.0",
|
|
||||||
"slab",
|
|
||||||
"tokio",
|
|
||||||
"tokio-util",
|
|
||||||
"tracing",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -956,7 +887,6 @@ dependencies = [
|
|||||||
"bytes",
|
"bytes",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"h2",
|
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
"httparse",
|
"httparse",
|
||||||
@@ -981,22 +911,7 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
]
|
"webpki-roots",
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "hyper-tls"
|
|
||||||
version = "0.6.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
|
||||||
dependencies = [
|
|
||||||
"bytes",
|
|
||||||
"http-body-util",
|
|
||||||
"hyper",
|
|
||||||
"hyper-util",
|
|
||||||
"native-tls",
|
|
||||||
"tokio",
|
|
||||||
"tokio-native-tls",
|
|
||||||
"tower-service",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1017,11 +932,9 @@ dependencies = [
|
|||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"socket2",
|
"socket2",
|
||||||
"system-configuration",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
"tracing",
|
"tracing",
|
||||||
"windows-registry",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1246,55 +1159,6 @@ version = "1.0.18"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "jni"
|
|
||||||
version = "0.22.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
"combine",
|
|
||||||
"jni-macros",
|
|
||||||
"jni-sys",
|
|
||||||
"log",
|
|
||||||
"simd_cesu8",
|
|
||||||
"thiserror",
|
|
||||||
"walkdir",
|
|
||||||
"windows-link",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "jni-macros"
|
|
||||||
version = "0.22.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"rustc_version",
|
|
||||||
"simd_cesu8",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "jni-sys"
|
|
||||||
version = "0.4.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
|
|
||||||
dependencies = [
|
|
||||||
"jni-sys-macros",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "jni-sys-macros"
|
|
||||||
version = "0.4.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
|
|
||||||
dependencies = [
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "jobserver"
|
name = "jobserver"
|
||||||
version = "0.1.34"
|
version = "0.1.34"
|
||||||
@@ -1305,6 +1169,12 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jpeg-encoder"
|
||||||
|
version = "0.7.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a0370574b86f7eca156b9f298392b5e69a23f8c86f3f865add60bbc2e79467a6"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.98"
|
version = "0.3.98"
|
||||||
@@ -1352,6 +1222,12 @@ version = "0.8.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "litrs"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lock_api"
|
name = "lock_api"
|
||||||
version = "0.4.14"
|
version = "0.4.14"
|
||||||
@@ -1443,23 +1319,6 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "native-tls"
|
|
||||||
version = "0.2.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"log",
|
|
||||||
"openssl",
|
|
||||||
"openssl-probe",
|
|
||||||
"openssl-sys",
|
|
||||||
"schannel",
|
|
||||||
"security-framework",
|
|
||||||
"security-framework-sys",
|
|
||||||
"tempfile",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-conv"
|
name = "num-conv"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
@@ -1490,49 +1349,6 @@ version = "1.21.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl"
|
|
||||||
version = "0.10.79"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"cfg-if",
|
|
||||||
"foreign-types",
|
|
||||||
"libc",
|
|
||||||
"openssl-macros",
|
|
||||||
"openssl-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-macros"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-probe"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-sys"
|
|
||||||
version = "0.9.115"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"
|
|
||||||
dependencies = [
|
|
||||||
"cc",
|
|
||||||
"libc",
|
|
||||||
"pkg-config",
|
|
||||||
"vcpkg",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.5"
|
version = "0.12.5"
|
||||||
@@ -1604,6 +1420,19 @@ version = "0.3.33"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "png"
|
||||||
|
version = "0.18.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"crc32fast",
|
||||||
|
"fdeflate",
|
||||||
|
"flate2",
|
||||||
|
"miniz_oxide",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "potential_utf"
|
name = "potential_utf"
|
||||||
version = "0.1.5"
|
version = "0.1.5"
|
||||||
@@ -1690,9 +1519,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quinn"
|
name = "quinn"
|
||||||
version = "0.11.9"
|
version = "0.11.11"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
|
checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"cfg_aliases",
|
"cfg_aliases",
|
||||||
@@ -1710,15 +1539,15 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quinn-proto"
|
name = "quinn-proto"
|
||||||
version = "0.11.14"
|
version = "0.11.16"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
|
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
|
||||||
"bytes",
|
"bytes",
|
||||||
"getrandom 0.3.4",
|
"getrandom 0.4.2",
|
||||||
"lru-slab",
|
"lru-slab",
|
||||||
"rand 0.9.4",
|
"rand 0.10.2",
|
||||||
|
"rand_pcg",
|
||||||
"ring",
|
"ring",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
"rustls",
|
"rustls",
|
||||||
@@ -1732,9 +1561,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quinn-udp"
|
name = "quinn-udp"
|
||||||
version = "0.5.14"
|
version = "0.5.15"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
|
checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg_aliases",
|
"cfg_aliases",
|
||||||
"libc",
|
"libc",
|
||||||
@@ -1772,18 +1601,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"rand_chacha 0.3.1",
|
"rand_chacha",
|
||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand"
|
name = "rand"
|
||||||
version = "0.9.4"
|
version = "0.10.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rand_chacha 0.9.0",
|
"chacha20",
|
||||||
"rand_core 0.9.5",
|
"getrandom 0.4.2",
|
||||||
|
"rand_core 0.10.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1796,16 +1626,6 @@ dependencies = [
|
|||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rand_chacha"
|
|
||||||
version = "0.9.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
|
||||||
dependencies = [
|
|
||||||
"ppv-lite86",
|
|
||||||
"rand_core 0.9.5",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand_core"
|
name = "rand_core"
|
||||||
version = "0.6.4"
|
version = "0.6.4"
|
||||||
@@ -1817,11 +1637,17 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand_core"
|
name = "rand_core"
|
||||||
version = "0.9.5"
|
version = "0.10.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_pcg"
|
||||||
|
version = "0.10.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"getrandom 0.3.4",
|
"rand_core 0.10.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1905,21 +1731,22 @@ dependencies = [
|
|||||||
"http-body",
|
"http-body",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-tls",
|
"hyper-rustls",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"js-sys",
|
"js-sys",
|
||||||
"log",
|
"log",
|
||||||
"mime_guess",
|
"mime_guess",
|
||||||
"native-tls",
|
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
"quinn",
|
||||||
|
"rustls",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
"sync_wrapper",
|
"sync_wrapper",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-native-tls",
|
"tokio-rustls",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tower",
|
"tower",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
@@ -1929,47 +1756,7 @@ dependencies = [
|
|||||||
"wasm-bindgen-futures",
|
"wasm-bindgen-futures",
|
||||||
"wasm-streams",
|
"wasm-streams",
|
||||||
"web-sys",
|
"web-sys",
|
||||||
]
|
"webpki-roots",
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "reqwest"
|
|
||||||
version = "0.13.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
|
|
||||||
dependencies = [
|
|
||||||
"base64",
|
|
||||||
"bytes",
|
|
||||||
"encoding_rs",
|
|
||||||
"futures-core",
|
|
||||||
"h2",
|
|
||||||
"http",
|
|
||||||
"http-body",
|
|
||||||
"http-body-util",
|
|
||||||
"hyper",
|
|
||||||
"hyper-rustls",
|
|
||||||
"hyper-util",
|
|
||||||
"js-sys",
|
|
||||||
"log",
|
|
||||||
"mime",
|
|
||||||
"percent-encoding",
|
|
||||||
"pin-project-lite",
|
|
||||||
"quinn",
|
|
||||||
"rustls",
|
|
||||||
"rustls-pki-types",
|
|
||||||
"rustls-platform-verifier",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"serde_urlencoded",
|
|
||||||
"sync_wrapper",
|
|
||||||
"tokio",
|
|
||||||
"tokio-rustls",
|
|
||||||
"tower",
|
|
||||||
"tower-http",
|
|
||||||
"tower-service",
|
|
||||||
"url",
|
|
||||||
"wasm-bindgen",
|
|
||||||
"wasm-bindgen-futures",
|
|
||||||
"web-sys",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2011,18 +1798,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-hash"
|
name = "rustc-hash"
|
||||||
version = "2.1.2"
|
version = "2.1.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rustc_version"
|
|
||||||
version = "0.4.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
|
|
||||||
dependencies = [
|
|
||||||
"semver",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustix"
|
name = "rustix"
|
||||||
@@ -2043,26 +1821,14 @@ version = "0.23.40"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
|
||||||
"once_cell",
|
"once_cell",
|
||||||
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"rustls-webpki",
|
"rustls-webpki",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rustls-native-certs"
|
|
||||||
version = "0.8.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
|
|
||||||
dependencies = [
|
|
||||||
"openssl-probe",
|
|
||||||
"rustls-pki-types",
|
|
||||||
"schannel",
|
|
||||||
"security-framework",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-pki-types"
|
name = "rustls-pki-types"
|
||||||
version = "1.14.1"
|
version = "1.14.1"
|
||||||
@@ -2073,40 +1839,12 @@ dependencies = [
|
|||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rustls-platform-verifier"
|
|
||||||
version = "0.7.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation 0.10.1",
|
|
||||||
"core-foundation-sys",
|
|
||||||
"jni",
|
|
||||||
"log",
|
|
||||||
"once_cell",
|
|
||||||
"rustls",
|
|
||||||
"rustls-native-certs",
|
|
||||||
"rustls-platform-verifier-android",
|
|
||||||
"rustls-webpki",
|
|
||||||
"security-framework",
|
|
||||||
"security-framework-sys",
|
|
||||||
"webpki-root-certs",
|
|
||||||
"windows-sys 0.61.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rustls-platform-verifier-android"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-webpki"
|
name = "rustls-webpki"
|
||||||
version = "0.103.13"
|
version = "0.103.13"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"untrusted",
|
"untrusted",
|
||||||
@@ -2124,24 +1862,6 @@ version = "1.0.23"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "same-file"
|
|
||||||
version = "1.0.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
|
||||||
dependencies = [
|
|
||||||
"winapi-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "schannel"
|
|
||||||
version = "0.1.29"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
|
|
||||||
dependencies = [
|
|
||||||
"windows-sys 0.61.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "schemars"
|
name = "schemars"
|
||||||
version = "0.9.0"
|
version = "0.9.0"
|
||||||
@@ -2172,29 +1892,6 @@ version = "1.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "security-framework"
|
|
||||||
version = "3.7.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"core-foundation 0.10.1",
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
"security-framework-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "security-framework-sys"
|
|
||||||
version = "2.17.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "semver"
|
name = "semver"
|
||||||
version = "1.0.28"
|
version = "1.0.28"
|
||||||
@@ -2306,7 +2003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
|
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"cpufeatures",
|
"cpufeatures 0.2.17",
|
||||||
"digest",
|
"digest",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2332,22 +2029,6 @@ version = "0.3.10"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "simd_cesu8"
|
|
||||||
version = "1.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
|
|
||||||
dependencies = [
|
|
||||||
"rustc_version",
|
|
||||||
"simdutf8",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "simdutf8"
|
|
||||||
version = "0.1.5"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "slab"
|
name = "slab"
|
||||||
version = "0.4.12"
|
version = "0.4.12"
|
||||||
@@ -2432,27 +2113,6 @@ dependencies = [
|
|||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "system-configuration"
|
|
||||||
version = "0.7.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"core-foundation 0.9.4",
|
|
||||||
"system-configuration-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "system-configuration-sys"
|
|
||||||
version = "0.6.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "take_mut"
|
name = "take_mut"
|
||||||
version = "0.2.2"
|
version = "0.2.2"
|
||||||
@@ -2512,7 +2172,7 @@ dependencies = [
|
|||||||
"once_cell",
|
"once_cell",
|
||||||
"pin-project",
|
"pin-project",
|
||||||
"rc-box",
|
"rc-box",
|
||||||
"reqwest 0.12.28",
|
"reqwest",
|
||||||
"rgb",
|
"rgb",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -2664,16 +2324,6 @@ dependencies = [
|
|||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tokio-native-tls"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
|
||||||
dependencies = [
|
|
||||||
"native-tls",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-rustls"
|
name = "tokio-rustls"
|
||||||
version = "0.26.4"
|
version = "0.26.4"
|
||||||
@@ -2859,16 +2509,6 @@ version = "0.9.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "walkdir"
|
|
||||||
version = "2.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
|
||||||
dependencies = [
|
|
||||||
"same-file",
|
|
||||||
"winapi-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "want"
|
name = "want"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
@@ -3025,10 +2665,10 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "webpki-root-certs"
|
name = "webpki-roots"
|
||||||
version = "1.0.7"
|
version = "1.0.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
|
checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
]
|
]
|
||||||
@@ -3083,17 +2723,6 @@ version = "0.2.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "windows-registry"
|
|
||||||
version = "0.6.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
|
|
||||||
dependencies = [
|
|
||||||
"windows-link",
|
|
||||||
"windows-result",
|
|
||||||
"windows-strings",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-result"
|
name = "windows-result"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
@@ -3296,7 +2925,7 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "x-media"
|
name = "x-media"
|
||||||
version = "1.0.6"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
@@ -3304,7 +2933,7 @@ dependencies = [
|
|||||||
"log",
|
"log",
|
||||||
"rand 0.8.6",
|
"rand 0.8.6",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest 0.13.3",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
@@ -3315,15 +2944,18 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "xmedia-bot"
|
name = "xmedia-bot"
|
||||||
version = "1.0.6"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
|
"fast_image_resize",
|
||||||
"html-escape",
|
"html-escape",
|
||||||
|
"jpeg-encoder",
|
||||||
"log",
|
"log",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
|
"png",
|
||||||
"pretty_env_logger",
|
"pretty_env_logger",
|
||||||
"rand 0.8.6",
|
"rand 0.8.6",
|
||||||
"regex",
|
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -3332,6 +2964,7 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
"url",
|
"url",
|
||||||
"x-media",
|
"x-media",
|
||||||
|
"zune-jpeg",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -3535,3 +3168,18 @@ dependencies = [
|
|||||||
"cc",
|
"cc",
|
||||||
"pkg-config",
|
"pkg-config",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-core"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-jpeg"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||||
|
dependencies = [
|
||||||
|
"zune-core",
|
||||||
|
]
|
||||||
|
|||||||
@@ -1,3 +1,12 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
members = ["crates/x-media", "crates/xmedia-bot"]
|
members = ["crates/x-media", "crates/xmedia-bot"]
|
||||||
resolver = "3"
|
resolver = "3"
|
||||||
|
|
||||||
|
# Smaller/faster production binary: strip debug symbols, link-time
|
||||||
|
# optimization across crates, and one codegen unit per crate (bigger LTO
|
||||||
|
# wins). panic=abort is intentionally NOT set: queue workers and db
|
||||||
|
# closures rely on JoinHandle catching panics, which abort would defeat.
|
||||||
|
[profile.release]
|
||||||
|
strip = true
|
||||||
|
lto = "thin"
|
||||||
|
codegen-units = 1
|
||||||
|
|||||||
+17
-9
@@ -11,6 +11,11 @@ ARG APP_NAME=telegram-twitter-media-bot
|
|||||||
# runners. `/redirect/latest/` floats to the newest release build; each build
|
# runners. `/redirect/latest/` floats to the newest release build; each build
|
||||||
# also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images.
|
# also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images.
|
||||||
ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip
|
ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip
|
||||||
|
# Optional sha256 of ffmpeg.zip (pinned releases only): set to verify the
|
||||||
|
# download. The mirror publishes .sha256 sidecars next to pinned builds, e.g.
|
||||||
|
# https://ffmpeg.martin-riedl.de/download/linux/amd64/<id>_9.0/ffmpeg.zip.sha256
|
||||||
|
# (the /redirect/latest/ URL itself has no sidecar — pin the effective URL).
|
||||||
|
ARG FFMPEG_SHA256=
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
@@ -30,17 +35,20 @@ RUN mkdir -p crates/x-media/src crates/xmedia-bot/src \
|
|||||||
# root. `unzip -t` verifies the archive before extraction so a bad
|
# root. `unzip -t` verifies the archive before extraction so a bad
|
||||||
# download fails loudly here instead of a cryptic later error.
|
# download fails loudly here instead of a cryptic later error.
|
||||||
RUN wget -q -O /tmp/ffmpeg.zip "$FFMPEG_URL" \
|
RUN wget -q -O /tmp/ffmpeg.zip "$FFMPEG_URL" \
|
||||||
|
&& if [ -n "$FFMPEG_SHA256" ]; then echo "$FFMPEG_SHA256 /tmp/ffmpeg.zip" | sha256sum -c -; fi \
|
||||||
&& unzip -tq /tmp/ffmpeg.zip \
|
&& unzip -tq /tmp/ffmpeg.zip \
|
||||||
&& unzip -q /tmp/ffmpeg.zip -d /usr/local/bin \
|
&& unzip -q /tmp/ffmpeg.zip -d /usr/local/bin \
|
||||||
&& chmod +x /usr/local/bin/ffmpeg \
|
&& chmod +x /usr/local/bin/ffmpeg \
|
||||||
&& rm /tmp/ffmpeg.zip \
|
&& rm /tmp/ffmpeg.zip \
|
||||||
&& /usr/local/bin/ffmpeg -version >/dev/null
|
&& /usr/local/bin/ffmpeg -version >/dev/null
|
||||||
|
|
||||||
# 3. Real sources last: only our crates recompile on source changes. The
|
# 3. Real sources last: only our crates recompile on source changes. Cargo's
|
||||||
# COPY preserves host mtimes, which predate the stub artifacts from step 1;
|
# freshness check is mtime-based; the COPY'd host files usually predate the
|
||||||
# cargo's mtime-based freshness check would otherwise treat the stub build
|
# step-1 stub build, so cargo would consider the stub up to date and never
|
||||||
# as up-to-date and never compile the real sources. `touch` forces cargo to
|
# compile the real sources. `touch` makes every .rs newer than the stub
|
||||||
# see the real files as newer.
|
# artifacts, forcing a rebuild of just the two crates while the compiled
|
||||||
|
# dependency layer stays cached. (`cargo clean -p` does NOT work here — it
|
||||||
|
# removes 0 files and the stub binary silently ships.)
|
||||||
COPY crates/ ./crates/
|
COPY crates/ ./crates/
|
||||||
RUN find crates -type f -name '*.rs' -exec touch {} + \
|
RUN find crates -type f -name '*.rs' -exec touch {} + \
|
||||||
&& cargo build --release -p xmedia-bot
|
&& cargo build --release -p xmedia-bot
|
||||||
@@ -56,10 +64,10 @@ LABEL org.opencontainers.image.title="${APP_NAME}"
|
|||||||
|
|
||||||
# Everything is copied in — no apt in the runtime stage. Privilege dropping is
|
# Everything is copied in — no apt in the runtime stage. Privilege dropping is
|
||||||
# done by docker-entrypoint.sh with setpriv (util-linux, already in
|
# done by docker-entrypoint.sh with setpriv (util-linux, already in
|
||||||
# bookworm-slim), so no gosu needed.
|
# bookworm-slim), so no gosu needed. TLS is rustls (webpki-roots baked in,
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
# see Cargo.toml feature `rustls`/`rustls-tls`), so no system CA bundle or
|
||||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libssl.so.3* /usr/lib/x86_64-linux-gnu/
|
# libssl are needed; the static ffmpeg only processes local files (all
|
||||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.so.3* /usr/lib/x86_64-linux-gnu/
|
# downloads go through reqwest).
|
||||||
COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
|
COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|||||||
+123
@@ -0,0 +1,123 @@
|
|||||||
|
# TelegramXMediaBot
|
||||||
|
|
||||||
|
A Telegram bot that turns post links from X / Twitter, Pixiv, and Bluesky into media messages (images, video, GIF) with the post's title, author, and tags.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- Sending a link in a private chat fetches and sends the images, videos and GIFs automatically; oversized media is split into batches
|
||||||
|
- Text-only posts report "no media"; unsupported links are silently ignored
|
||||||
|
- Inline queries (`@bot <link>`)
|
||||||
|
- Bind a forward channel for automatic forwarding; edit the caption before forwarding and apply custom templates
|
||||||
|
- Failed sends are retried automatically with persistence; the user is notified after retries are exhausted
|
||||||
|
- Pixiv ugoira animations are transcoded to MP4; Bluesky videos are remuxed (HLS stream → MP4)
|
||||||
|
- Photos exceeding Telegram's size/dimension limits are compressed automatically (original format kept, JPEG fallback only when needed)
|
||||||
|
- Link-result cache: after a successful send the Telegram file ids and caption fields are cached locally, so a repeated link is re-sent from local state — no source-site request, no media file stored (expiry controlled by `LINK_CACHE_TTL_SECONDS`, default 7 days)
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Required: BotFather token; optional: PIXIV_REFRESH_TOKEN (Pixiv is disabled without it)
|
||||||
|
export TELOXIDE_TOKEN=<token>
|
||||||
|
export PIXIV_REFRESH_TOKEN=<token>
|
||||||
|
|
||||||
|
cargo run -p xmedia-bot
|
||||||
|
```
|
||||||
|
|
||||||
|
Docker deployment (see `docker-compose.yml.example`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t tgxmb .
|
||||||
|
docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb
|
||||||
|
```
|
||||||
|
|
||||||
|
Environment variables: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `BOT_ADMIN`, `EDIT_MESSAGE_TTL_SECONDS`, `LINK_CACHE_TTL_SECONDS`, `RUST_LOG`, `WEBHOOK*`, `TWITTER_AUTH_TOKEN` (optional).
|
||||||
|
|
||||||
|
NSFW tweets: the public syndication endpoint does not return sensitive content. Setting `TWITTER_AUTH_TOKEN` (the `auth_token` cookie value of a logged-in x.com session) lets the bot fetch NSFW media in the logged-in state only when it hits a withheld tweet; without it, the bot reports no media.
|
||||||
|
|
||||||
|
### Webhook deployment (needs a reverse proxy)
|
||||||
|
|
||||||
|
`docker-compose.yml.example` ships an [nginx-proxy](https://github.com/nginx-proxy/nginx-proxy) + [acme-companion](https://github.com/nginx-proxy/acme-companion) reverse-proxy orchestration. Pick one deployment shape:
|
||||||
|
|
||||||
|
**With a domain**
|
||||||
|
1. Point a DNS A record at the server
|
||||||
|
2. In compose set `VIRTUAL_HOST` and `WEBHOOK_URL` to the domain, and uncomment `ACME_HOST` (set it to the domain)
|
||||||
|
3. acme-companion issues and renews certificates automatically — nothing manual
|
||||||
|
|
||||||
|
**IP only**
|
||||||
|
Let's Encrypt can issue certificates for public IPs (available since 2026, validity ~7 days, requires the `shortlived` profile). Use [acme.sh](https://github.com/acmesh-official/acme.sh) to issue and renew automatically, no manual certificates:
|
||||||
|
|
||||||
|
1. Add an acme-ip service to compose (issue + daily auto-renewal check):
|
||||||
|
```yaml
|
||||||
|
acme-ip:
|
||||||
|
image: neilpang/acme.sh
|
||||||
|
container_name: acme-ip
|
||||||
|
command: daemon
|
||||||
|
restart: always
|
||||||
|
volumes:
|
||||||
|
- certs:/acme.sh
|
||||||
|
- html:/usr/share/nginx/html
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
networks: [proxy]
|
||||||
|
```
|
||||||
|
2. First issuance (replace `<SERVER_IP>` with the server's public IP; IPv6 works too, repeat `-d` for more):
|
||||||
|
```bash
|
||||||
|
docker compose exec acme-ip acme.sh --issue --server letsencrypt \
|
||||||
|
-d <SERVER_IP> --cert-profile shortlived --days 3 \
|
||||||
|
--webroot /usr/share/nginx/html \
|
||||||
|
--install-cert --cert-file /acme.sh/<SERVER_IP>.crt \
|
||||||
|
--key-file /acme.sh/<SERVER_IP>.key \
|
||||||
|
--reloadcmd "curl --unix-socket /var/run/docker.sock -X POST http://localhost/containers/nginx-proxy/kill?signal=HUP"
|
||||||
|
```
|
||||||
|
3. In compose set `VIRTUAL_HOST: '<SERVER_IP>'` and `WEBHOOK_URL: 'https://<SERVER_IP>/'`; no `WEBHOOK_CERT` needed. Renewal is handled by the acme.sh daemon (`--days 3` = renew every 3 days, buffer against the 7-day validity), and a successful renewal HUP-notifies nginx-proxy to load the new certificate.
|
||||||
|
|
||||||
|
Limitations: certificate validity ~7 days; only http-01/tls-alpn-01 validation (port 80 must be publicly reachable); no DNS-01, private IPs or IP ranges; at most 5 certificates per 168 hours for the same IP set. It is recommended to trial-issue with `--server letsencrypt_test` first, then switch to the production server.
|
||||||
|
|
||||||
|
Telegram only accepts ports 443/80/88/8443.
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>Environment variables</summary>
|
||||||
|
|
||||||
|
| Variable | Description |
|
||||||
|
|---|---|
|
||||||
|
| `TELOXIDE_TOKEN` | Bot token (required) |
|
||||||
|
| `PIXIV_REFRESH_TOKEN` | Pixiv refresh token; Pixiv is disabled without it |
|
||||||
|
| `BOT_ADMIN` | Admin chat IDs, comma-separated; receives start/stop notifications |
|
||||||
|
| `EDIT_MESSAGE_TTL_SECONDS` | Edit-before-forward record expiry in seconds, default 86400 |
|
||||||
|
| `LINK_CACHE_TTL_SECONDS` | Link-result cache expiry in seconds, default 604800 (7 days) |
|
||||||
|
| `RUST_LOG` | Log level |
|
||||||
|
| `TELOXIDE_PROXY` | HTTP proxy (e.g. `http://127.0.0.1:10808`); applies to both the Telegram Bot API and site fetches — required on restricted networks (e.g. behind the GFW) |
|
||||||
|
| `LOCAL_USER_ID` | UID the container runs as, default 9001 |
|
||||||
|
| `VIRTUAL_HOST` | Public domain or IP; nginx-proxy routes by this |
|
||||||
|
| `VIRTUAL_PORT` | Port the bot listens on inside the container; nginx-proxy's forwarding target |
|
||||||
|
| `ACME_HOST` | Domain deployment: when set to the domain, acme-companion issues/renews certificates automatically |
|
||||||
|
| `DEFAULT_HOST` | nginx-proxy routes requests with unknown Host headers to this vhost (needed for IP access) |
|
||||||
|
| `DEFAULT_EMAIL` | acme-companion certificate notification email |
|
||||||
|
| `WEBHOOK` | `true` enables webhook mode (polling by default) |
|
||||||
|
| `WEBHOOK_LISTEN` / `WEBHOOK_PORT` | Listen address/port inside the bot container |
|
||||||
|
| `WEBHOOK_URL` | Public HTTPS URL (`https://domain/` or `https://IP/`) |
|
||||||
|
| `WEBHOOK_CERT` | Optional; self-signed certificate path, only used for Telegram-side validation (TLS is terminated by the reverse proxy) |
|
||||||
|
| `WEBHOOK_SECRET_TOKEN` | Update validation token (`X-Telegram-Bot-Api-Secret-Token`) |
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
| Command | Description |
|
||||||
|
|---|---|
|
||||||
|
| `/start` | Welcome message |
|
||||||
|
| `/help` | List all commands and usage (this command table) |
|
||||||
|
| `/set_forward_channel <channel>` | Set the forward channel: `@channel` or channel ID; media messages are forwarded to it automatically afterwards |
|
||||||
|
| `/remove_forward_channel` | Remove the forward channel |
|
||||||
|
| `/edit_before_forward` | Toggle "edit before forward": when enabled, the bot posts a prompt after forwarding; replying to it edits the first forwarded message's caption (or taps a template button to apply one) |
|
||||||
|
| `/set_template <name>` | Reply to a message containing `[]` to save it as a named template; `[]` is replaced by the original post link when forwarding (used with "edit before forward") |
|
||||||
|
| `/set_format <site> <format>` | Customize the caption format for one site. Sites: `twitter` / `bsky` / `pixiv`. Placeholders: `{url}` `{author}` `{author_url}` `{title}` `{tags}` |
|
||||||
|
| `/clear_cache [link]` | Clear the link cache (admin only); with a link only that entry, otherwise everything |
|
||||||
|
| `/bot_dict` | Show the current chat state (debugging) |
|
||||||
|
|
||||||
|
Link processing works only in private chats; commands work in any chat.
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- State is persisted in `data/task_queue.db`; compose deployments use the bind mount `./data` (keep it a directory for easy backups)
|
||||||
|
- The runtime needs ffmpeg (built into the Docker image)
|
||||||
|
- Tests: `cargo test --workspace`
|
||||||
@@ -9,7 +9,8 @@ Telegram 机器人,将 X / Twitter、Pixiv、Bluesky 的帖子链接转换为
|
|||||||
- 支持内联查询(`@机器人 <链接>`)
|
- 支持内联查询(`@机器人 <链接>`)
|
||||||
- 可绑定转发频道自动转发;支持转发前编辑 caption 与自定义模板
|
- 可绑定转发频道自动转发;支持转发前编辑 caption 与自定义模板
|
||||||
- 发送失败自动重试并持久化,重试耗尽后通知用户
|
- 发送失败自动重试并持久化,重试耗尽后通知用户
|
||||||
- Pixiv ugoira 动图自动转码为 MP4
|
- Pixiv ugoira 动图自动转码为 MP4;Bluesky 视频自动转码(HLS 流 → MP4)
|
||||||
|
- 超过 Telegram 尺寸/大小限制的图片自动压缩(保持原格式,必要时转 JPEG)
|
||||||
- 链接结果本地缓存:成功发送后缓存 Telegram file id 与 caption 等,再次收到相同链接直接本地重发,不再请求源站、不保存媒体文件(`LINK_CACHE_TTL_SECONDS` 控制过期,默认 7 天)
|
- 链接结果本地缓存:成功发送后缓存 Telegram file id 与 caption 等,再次收到相同链接直接本地重发,不再请求源站、不保存媒体文件(`LINK_CACHE_TTL_SECONDS` 控制过期,默认 7 天)
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
@@ -39,16 +40,37 @@ NSFW 推文:公开的 syndication 接口不返回敏感内容。设置 `TWITTE
|
|||||||
|
|
||||||
**有域名**
|
**有域名**
|
||||||
1. DNS A 记录指向服务器
|
1. DNS A 记录指向服务器
|
||||||
2. compose 里设 `VIRTUAL_HOST`、`LETSENCRYPT_HOST` 为域名,`WEBHOOK_URL` 设为 `https://域名/`
|
2. compose 里设 `VIRTUAL_HOST`、`WEBHOOK_URL` 为域名,并取消注释 `ACME_HOST`(设为域名)
|
||||||
3. 证书自动签发与续期,无需手动处理
|
3. acme-companion 自动签发与续期证书,无需手动处理
|
||||||
|
|
||||||
**只有 IP**
|
**只有 IP**
|
||||||
1. 生成自签证书(PEM 格式,见第 3 步):
|
Let's Encrypt 支持为公网 IP 签发证书(2026 年起可用,有效期约 7 天,须 `shortlived` profile)。用 [acme.sh](https://github.com/acmesh-official/acme.sh) 自动签发与续期,无需手动证书:
|
||||||
`openssl req -x509 -newkey rsa:2048 -nodes -days 365 -keyout nginx-certs/default.key -out nginx-certs/default.crt`
|
|
||||||
2. compose 里 nginx-proxy 设 `DEFAULT_HOST`,bot 设 `WEBHOOK_CERT: './cert/cert.pem'`(须与代理所服务的为同一张证书)
|
1. compose 里增加 acme-ip 服务(签发 + 每日检查自动续期):
|
||||||
3. 证书必须是 PEM 编码(ASCII BASE64,以 `-----BEGIN CERTIFICATE-----` 开头)—— Telegram 只接受该格式;若现有证书是 DER 二进制,转换:
|
```yaml
|
||||||
`openssl x509 -in cert.der -inform DER -out cert.pem -outform PEM`
|
acme-ip:
|
||||||
(私钥同理:`openssl rsa -in key.der -inform DER -out key.pem -outform PEM`)
|
image: neilpang/acme.sh
|
||||||
|
container_name: acme-ip
|
||||||
|
command: daemon
|
||||||
|
restart: always
|
||||||
|
volumes:
|
||||||
|
- certs:/acme.sh
|
||||||
|
- html:/usr/share/nginx/html
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
networks: [proxy]
|
||||||
|
```
|
||||||
|
2. 首次签发(把 `<SERVER_IP>` 换成服务器公网 IP,IPv6 同样支持,多个 `-d` 可并列):
|
||||||
|
```bash
|
||||||
|
docker compose exec acme-ip acme.sh --issue --server letsencrypt \
|
||||||
|
-d <SERVER_IP> --cert-profile shortlived --days 3 \
|
||||||
|
--webroot /usr/share/nginx/html \
|
||||||
|
--install-cert --cert-file /acme.sh/<SERVER_IP>.crt \
|
||||||
|
--key-file /acme.sh/<SERVER_IP>.key \
|
||||||
|
--reloadcmd "curl --unix-socket /var/run/docker.sock -X POST http://localhost/containers/nginx-proxy/kill?signal=HUP"
|
||||||
|
```
|
||||||
|
3. compose 里设 `VIRTUAL_HOST: '<SERVER_IP>'`、`WEBHOOK_URL: 'https://<SERVER_IP>/'`,无需 `WEBHOOK_CERT`。续期由 acme.sh daemon 自动完成(`--days 3` = 每 3 天续一次,证书 7 天有效有缓冲),续期成功后自动 HUP 通知 nginx-proxy 加载新证书。
|
||||||
|
|
||||||
|
限制:证书约 7 天有效;验证仅支持 http-01/tls-alpn-01(80 端口必须公网可达);不支持 DNS-01、私有 IP 与 IP 段;同一 IP 集合每 168 小时限签发 5 张。建议先用 `--server letsencrypt_test` 试签,成功后再切正式服务器。
|
||||||
|
|
||||||
Telegram 只接受 443/80/88/8443 端口。
|
Telegram 只接受 443/80/88/8443 端口。
|
||||||
|
|
||||||
@@ -63,16 +85,17 @@ Telegram 只接受 443/80/88/8443 端口。
|
|||||||
| `EDIT_MESSAGE_TTL_SECONDS` | 转发前编辑记录过期秒数,默认 86400 |
|
| `EDIT_MESSAGE_TTL_SECONDS` | 转发前编辑记录过期秒数,默认 86400 |
|
||||||
| `LINK_CACHE_TTL_SECONDS` | 链接结果缓存过期秒数,默认 604800(7 天) |
|
| `LINK_CACHE_TTL_SECONDS` | 链接结果缓存过期秒数,默认 604800(7 天) |
|
||||||
| `RUST_LOG` | 日志级别 |
|
| `RUST_LOG` | 日志级别 |
|
||||||
|
| `TELOXIDE_PROXY` | HTTP 代理(如 `http://127.0.0.1:10808`);同时作用于 Telegram Bot API 与站点抓取请求,网络受限环境(如 GFW)必需 |
|
||||||
| `LOCAL_USER_ID` | 容器内运行用户 UID,默认 9001 |
|
| `LOCAL_USER_ID` | 容器内运行用户 UID,默认 9001 |
|
||||||
| `VIRTUAL_HOST` | 对外域名或 IP,nginx-proxy 按此路由 |
|
| `VIRTUAL_HOST` | 对外域名或 IP,nginx-proxy 按此路由 |
|
||||||
| `VIRTUAL_PORT` | bot 容器内监听端口,nginx-proxy 的转发目标 |
|
| `VIRTUAL_PORT` | bot 容器内监听端口,nginx-proxy 的转发目标 |
|
||||||
| `LETSENCRYPT_HOST` | 设为域名时由 acme-companion 自动签发/续期证书 |
|
| `ACME_HOST` | 域名部署:设为域名时由 acme-companion 自动签发/续期证书 |
|
||||||
| `DEFAULT_HOST` | nginx-proxy 将未知 Host 的请求路由到该 vhost(IP 访问时需要) |
|
| `DEFAULT_HOST` | nginx-proxy 将未知 Host 的请求路由到该 vhost(IP 访问时需要) |
|
||||||
| `DEFAULT_EMAIL` | acme-companion 证书通知邮箱 |
|
| `DEFAULT_EMAIL` | acme-companion 证书通知邮箱 |
|
||||||
| `WEBHOOK` | `true` 启用 webhook 模式(默认轮询) |
|
| `WEBHOOK` | `true` 启用 webhook 模式(默认轮询) |
|
||||||
| `WEBHOOK_LISTEN` / `WEBHOOK_PORT` | bot 容器内监听地址/端口 |
|
| `WEBHOOK_LISTEN` / `WEBHOOK_PORT` | bot 容器内监听地址/端口 |
|
||||||
| `WEBHOOK_URL` | 对外公网 HTTPS 地址(`https://域名/`) |
|
| `WEBHOOK_URL` | 对外公网 HTTPS 地址(`https://域名/` 或 `https://IP/`) |
|
||||||
| `WEBHOOK_CERT` | 自签证书路径(仅 IP 路径需要,须为 PEM 且与代理所服务的一致) |
|
| `WEBHOOK_CERT` | 可选;自签名证书路径,仅用于 Telegram 侧验证(TLS 由反向代理终止) |
|
||||||
| `WEBHOOK_SECRET_TOKEN` | 更新校验令牌(`X-Telegram-Bot-Api-Secret-Token`) |
|
| `WEBHOOK_SECRET_TOKEN` | 更新校验令牌(`X-Telegram-Bot-Api-Secret-Token`) |
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
@@ -81,17 +104,20 @@ Telegram 只接受 443/80/88/8443 端口。
|
|||||||
|
|
||||||
| 命令 | 说明 |
|
| 命令 | 说明 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `/set_forward_channel <频道>` | 设置转发频道 |
|
| `/start` | 欢迎语 |
|
||||||
|
| `/help` | 查看全部命令及用法(即本文档的命令表) |
|
||||||
|
| `/set_forward_channel <频道>` | 设置转发频道,参数为 `@频道名` 或频道 ID;设置后发送的媒体消息会自动转发到该频道 |
|
||||||
| `/remove_forward_channel` | 取消转发频道 |
|
| `/remove_forward_channel` | 取消转发频道 |
|
||||||
| `/edit_before_forward` | 开关转发前编辑 |
|
| `/edit_before_forward` | 开关「转发前编辑」:开启后,转发成功后 bot 会发一条提示消息,回复它可修改第一条转发消息的 caption(或点击模板按钮套用模板) |
|
||||||
| `/set_template <名称>` | 将回复的消息(含 `[]`)保存为模板 |
|
| `/set_template <名称>` | 回复一条含 `[]` 的消息,将其保存为命名模板;转发时 `[]` 会被替换为原帖链接(配合「转发前编辑」使用) |
|
||||||
| `/set_format <站点> <格式>` | 自定义 caption 格式(占位符 `{url}` `{title}` `{tags}` 等) |
|
| `/set_format <站点> <格式>` | 自定义某站点的 caption 格式。站点:`twitter` / `bsky` / `pixiv`。占位符:`{url}` `{author}` `{author_url}` `{title}` `{tags}` |
|
||||||
| `/bot_dict` | 查看聊天状态 |
|
| `/clear_cache [链接]` | 清空链接缓存(仅管理员);带链接只清该条,否则清空全部 |
|
||||||
|
| `/bot_dict` | 查看当前聊天状态(调试用) |
|
||||||
|
|
||||||
链接处理仅限私聊;命令在任意聊天可用。
|
链接处理仅限私聊;命令在任意聊天可用。
|
||||||
|
|
||||||
## 备注
|
## 备注
|
||||||
|
|
||||||
- 数据持久化于 `data/task_queue.db`,容器部署需挂载该目录
|
- 数据持久化于 `data/task_queue.db`,compose 部署使用 bind mount `./data`(保持目录形式便于备份)
|
||||||
- 运行环境需安装 ffmpeg(Docker 镜像已内置)
|
- 运行环境需安装 ffmpeg(Docker 镜像已内置)
|
||||||
- 测试:`cargo test --workspace`
|
- 测试:`cargo test --workspace`
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "x-media"
|
name = "x-media"
|
||||||
version = "1.0.6"
|
version = "1.2.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
reqwest = { version = "0.13", features = ["json", "query", "form"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
regex = "1.12"
|
regex = "1.12"
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
use super::model;
|
use super::model;
|
||||||
use crate::media::Media;
|
use crate::media::Media;
|
||||||
use crate::site::{FetchError, Fetched};
|
use crate::site::{FetchError, Fetched};
|
||||||
use html_escape::encode_text;
|
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
Regex::new(r"bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap()
|
Regex::new(r"^(?:https?://)?bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap()
|
||||||
});
|
});
|
||||||
|
|
||||||
pub fn enabled() -> bool {
|
pub fn enabled() -> bool {
|
||||||
@@ -15,9 +15,169 @@ pub fn enabled() -> bool {
|
|||||||
|
|
||||||
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
||||||
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
|
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
|
||||||
let handle = caps.get(1).map(|m| m.as_str()).ok_or(FetchError::NotFound)?;
|
let handle = caps
|
||||||
let rkey = caps.get(2).map(|m| m.as_str()).ok_or(FetchError::NotFound)?;
|
.get(1)
|
||||||
Ok(fetch(handle, rkey).await?.into())
|
.map(|m| m.as_str())
|
||||||
|
.ok_or(FetchError::NotFound)?;
|
||||||
|
let rkey = caps
|
||||||
|
.get(2)
|
||||||
|
.map(|m| m.as_str())
|
||||||
|
.ok_or(FetchError::NotFound)?;
|
||||||
|
let post = fetch(handle, rkey).await?;
|
||||||
|
let mut fetched: Fetched = post.into();
|
||||||
|
// bsky video embeds expose only an HLS playlist URL, which Telegram
|
||||||
|
// cannot fetch; remux it to a single MP4 (mirrors the pixiv ugoira
|
||||||
|
// encode path — the temp file stays alive via `_keep_alive`). On any
|
||||||
|
// failure the video item is dropped and the post degrades to its text.
|
||||||
|
let mut media = Vec::with_capacity(fetched.media.len());
|
||||||
|
for item in fetched.media {
|
||||||
|
let is_hls = matches!(&item, Media::Video { url, .. }
|
||||||
|
if url.contains("playlist") || url.ends_with(".m3u8"));
|
||||||
|
if !is_hls {
|
||||||
|
media.push(item);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url = item.url().to_string();
|
||||||
|
match resolve_bsky_video(&url).await {
|
||||||
|
Ok(Some((mp4_path, keep_alive))) => {
|
||||||
|
let thumbnail_url = match &item {
|
||||||
|
Media::Video { thumbnail_url, .. } => thumbnail_url.clone(),
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
media.push(Media::Video {
|
||||||
|
title: None,
|
||||||
|
url: mp4_path.to_string_lossy().into_owned(),
|
||||||
|
thumbnail_url,
|
||||||
|
});
|
||||||
|
fetched._keep_alive = Some(keep_alive);
|
||||||
|
}
|
||||||
|
Ok(None) => log::warn!("bsky video remux unavailable for {url}"),
|
||||||
|
Err(e) => log::warn!("bsky video remux failed for {url}: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fetched.media = media;
|
||||||
|
Ok(fetched)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Downloads an HLS playlist (master or media) and remuxes its segments to a
|
||||||
|
/// single MP4 via ffmpeg. Returns the MP4 path plus the temp dir that must
|
||||||
|
/// stay alive until the file is uploaded. `Ok(None)` when ffmpeg is missing.
|
||||||
|
///
|
||||||
|
/// Verified live (2026-08): bsky master playlists carry `#EXT-X-STREAM-INF`
|
||||||
|
/// variant lines (e.g. `720p/video.m3u8?session_id=…`), and the media
|
||||||
|
/// playlists are VOD MPEG-TS segments (`videoN.ts?…`) without EXT-X-MAP, so
|
||||||
|
/// a plain `-f concat -c copy` remux is valid.
|
||||||
|
async fn resolve_bsky_video(
|
||||||
|
playlist_url: &str,
|
||||||
|
) -> Result<Option<(std::path::PathBuf, tempfile::TempDir)>, String> {
|
||||||
|
if !crate::site::ffmpeg_available() {
|
||||||
|
crate::site::log_once_ffmpeg_missing();
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let master = crate::site::download_media_limited(playlist_url, 1_048_576)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("bsky video master playlist: {e}"))?;
|
||||||
|
let master = String::from_utf8_lossy(&master);
|
||||||
|
|
||||||
|
// Master playlist: pick the variant with the highest declared bandwidth.
|
||||||
|
let playlist_url = if master.contains("#EXT-X-STREAM-INF") {
|
||||||
|
let mut best: Option<(u64, String)> = None;
|
||||||
|
let mut lines = master.lines();
|
||||||
|
while let Some(line) = lines.next() {
|
||||||
|
if !line.starts_with("#EXT-X-STREAM-INF") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bandwidth = line
|
||||||
|
.split_once("BANDWIDTH=")
|
||||||
|
.and_then(|(_, rest)| rest.split(|c: char| !c.is_ascii_digit()).next())
|
||||||
|
.and_then(|n| n.parse::<u64>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
if let Some(uri) = lines.next().filter(|u| !u.starts_with('#'))
|
||||||
|
&& bandwidth >= best.as_ref().map(|(b, _)| *b).unwrap_or(0)
|
||||||
|
{
|
||||||
|
best = Some((bandwidth, uri.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let Some((_, uri)) = best else {
|
||||||
|
return Err("bsky video master playlist has no variants".to_string());
|
||||||
|
};
|
||||||
|
url::Url::parse(playlist_url)
|
||||||
|
.and_then(|base| base.join(&uri))
|
||||||
|
.map_err(|e| format!("bsky video variant URL: {e}"))?
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
playlist_url.to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
let variant = crate::site::download_media_limited(&playlist_url, 1_048_576)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("bsky video media playlist: {e}"))?;
|
||||||
|
let variant = String::from_utf8_lossy(&variant);
|
||||||
|
// Segment URIs: non-#, non-empty lines, resolved relative to the playlist.
|
||||||
|
let base = url::Url::parse(&playlist_url).map_err(|e| format!("bsky playlist URL: {e}"))?;
|
||||||
|
let segments: Vec<String> = variant
|
||||||
|
.lines()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|l| !l.is_empty() && !l.starts_with('#'))
|
||||||
|
.map(|l| base.join(l).map(|u| u.to_string()))
|
||||||
|
.collect::<Result<_, _>>()
|
||||||
|
.map_err(|e| format!("bsky segment URL: {e}"))?;
|
||||||
|
if segments.is_empty() {
|
||||||
|
return Err("bsky video playlist has no segments".to_string());
|
||||||
|
}
|
||||||
|
if segments.len() > 500 {
|
||||||
|
return Err("bsky video has too many segments".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let frames_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
|
let out_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
|
let mut total: u64 = 0;
|
||||||
|
let mut list = String::new();
|
||||||
|
for (i, seg) in segments.iter().enumerate() {
|
||||||
|
let bytes = crate::site::download_media_limited(seg, 20 * 1024 * 1024)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("bsky segment {i}: {e}"))?;
|
||||||
|
total += bytes.len() as u64;
|
||||||
|
if total > 256 * 1024 * 1024 {
|
||||||
|
return Err("bsky video exceeds total size cap".to_string());
|
||||||
|
}
|
||||||
|
let path = frames_dir.path().join(format!("seg_{i:04}.ts"));
|
||||||
|
std::fs::write(&path, &bytes).map_err(|e| e.to_string())?;
|
||||||
|
list.push_str(&format!("file '{}'\n", path.to_string_lossy()));
|
||||||
|
}
|
||||||
|
let list_path = frames_dir.path().join("list.txt");
|
||||||
|
std::fs::write(&list_path, &list).map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
let output = out_dir.path().join("video.mp4");
|
||||||
|
let list_str = list_path.to_string_lossy().into_owned();
|
||||||
|
let output_str = output.to_string_lossy().into_owned();
|
||||||
|
let status = tokio::task::spawn_blocking(move || {
|
||||||
|
std::process::Command::new("ffmpeg")
|
||||||
|
.args([
|
||||||
|
"-y",
|
||||||
|
"-f",
|
||||||
|
"concat",
|
||||||
|
"-safe",
|
||||||
|
"0",
|
||||||
|
"-i",
|
||||||
|
&list_str,
|
||||||
|
"-c",
|
||||||
|
"copy",
|
||||||
|
"-movflags",
|
||||||
|
"+faststart",
|
||||||
|
&output_str,
|
||||||
|
])
|
||||||
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::null())
|
||||||
|
.status()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("bsky remux worker panicked: {e}"))?;
|
||||||
|
match status {
|
||||||
|
Ok(s) if s.success() => Ok(Some((output, out_dir))),
|
||||||
|
Ok(s) => Err(format!("ffmpeg exited with {s}")),
|
||||||
|
Err(e) => Err(format!("ffmpeg spawn failed: {e}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetches a post thread by handle or DID (`at://` URIs work for both).
|
/// Fetches a post thread by handle or DID (`at://` URIs work for both).
|
||||||
@@ -30,8 +190,16 @@ pub async fn fetch(handle: &str, rkey: &str) -> Result<Post, FetchError> {
|
|||||||
])
|
])
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
|
// 404/410 = gone (permanent); 429/5xx = transient and retried by fetch.
|
||||||
|
let status = response.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
return match status.as_u16() {
|
||||||
|
404 | 410 => Err(FetchError::NotFound),
|
||||||
|
_ => Err(FetchError::Transient(format!("bsky status {status}"))),
|
||||||
|
};
|
||||||
|
}
|
||||||
let text = response.text().await?;
|
let text = response.text().await?;
|
||||||
Ok(Post::from_json(&text, rkey.to_string())?)
|
Post::from_json(&text, rkey.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -56,8 +224,8 @@ impl Post {
|
|||||||
pub fn caption(&self) -> String {
|
pub fn caption(&self) -> String {
|
||||||
format!(
|
format!(
|
||||||
"{url}\n<a href=\"{author_url}\">{author}</a>: {text}",
|
"{url}\n<a href=\"{author_url}\">{author}</a>: {text}",
|
||||||
url = self.url(),
|
url = encode_double_quoted_attribute(&self.url()),
|
||||||
author_url = self.author_url(),
|
author_url = encode_double_quoted_attribute(&self.author_url()),
|
||||||
author = encode_text(&self.author),
|
author = encode_text(&self.author),
|
||||||
text = encode_text(&self.text),
|
text = encode_text(&self.text),
|
||||||
)
|
)
|
||||||
@@ -197,7 +365,10 @@ mod tests {
|
|||||||
}));
|
}));
|
||||||
let post = Post::from_json(&raw.to_string(), "3xxxx".into()).unwrap();
|
let post = Post::from_json(&raw.to_string(), "3xxxx".into()).unwrap();
|
||||||
let fetched: Fetched = post.into();
|
let fetched: Fetched = post.into();
|
||||||
assert_eq!(fetched.source_url, "https://bsky.app/profile/user.bsky.social/post/3xxxx");
|
assert_eq!(
|
||||||
|
fetched.source_url,
|
||||||
|
"https://bsky.app/profile/user.bsky.social/post/3xxxx"
|
||||||
|
);
|
||||||
assert_eq!(fetched.title, "hello <world>");
|
assert_eq!(fetched.title, "hello <world>");
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
assert!(!fetched.sensitive);
|
assert!(!fetched.sensitive);
|
||||||
@@ -245,10 +416,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[ignore = "live network: requires outbound HTTPS to public.api.bsky.app"]
|
||||||
async fn live_fetch_with_photos() {
|
async fn live_fetch_with_photos() {
|
||||||
let fetched = fetch_from_url(
|
let fetched =
|
||||||
"https://bsky.app/profile/asagi0398.bsky.social/post/3mqkhrq5w6k2m",
|
fetch_from_url("https://bsky.app/profile/asagi0398.bsky.social/post/3mqkhrq5w6k2m")
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -259,8 +430,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[ignore = "live network: requires outbound HTTPS to public.api.bsky.app"]
|
||||||
async fn live_fetch_smoke() {
|
async fn live_fetch_smoke() {
|
||||||
let fetched = fetch_from_url("https://bsky.app/profile/fu-futa.bsky.social/post/3laoveufjv224")
|
let fetched =
|
||||||
|
fetch_from_url("https://bsky.app/profile/fu-futa.bsky.social/post/3laoveufjv224")
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
+211
-19
@@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
pub mod bsky;
|
pub mod bsky;
|
||||||
@@ -65,7 +66,8 @@ impl Fetched {
|
|||||||
/// HTML-escaped in full, then the (already-escaped) placeholder values
|
/// HTML-escaped in full, then the (already-escaped) placeholder values
|
||||||
/// are substituted — users can structure text but never inject raw HTML
|
/// are substituted — users can structure text but never inject raw HTML
|
||||||
/// or attributes. An empty/unknown format falls back to the built-in
|
/// or attributes. An empty/unknown format falls back to the built-in
|
||||||
/// caption.
|
/// caption. The result is truncated to [`MAX_CAPTION_CHARS`] (Telegram's
|
||||||
|
/// caption limit for HTML parse mode).
|
||||||
pub fn caption_with(&self, format: &str) -> String {
|
pub fn caption_with(&self, format: &str) -> String {
|
||||||
match (&self.render_data, format.is_empty()) {
|
match (&self.render_data, format.is_empty()) {
|
||||||
(Some(data), false) => caption_from_fields(
|
(Some(data), false) => caption_from_fields(
|
||||||
@@ -77,7 +79,7 @@ impl Fetched {
|
|||||||
&data.title,
|
&data.title,
|
||||||
&data.tags,
|
&data.tags,
|
||||||
),
|
),
|
||||||
_ => self.caption.clone(),
|
_ => truncate_caption(&self.caption),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,11 +96,47 @@ impl Fetched {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Hands over the temp dir keeping locally produced media (ugoira MP4,
|
||||||
|
/// bsky remux MP4) alive. The bot keeps it while its task may still be
|
||||||
|
/// retried by the queue, which runs after this [`Fetched`] is dropped and
|
||||||
|
/// its temp files would otherwise be gone. `None` when no such dir exists.
|
||||||
|
pub fn take_keep_alive(&mut self) -> Option<tempfile::TempDir> {
|
||||||
|
self._keep_alive.take()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Telegram's caption length limit (chars) for HTML parse mode; longer
|
||||||
|
/// captions are rejected with a 400.
|
||||||
|
pub const MAX_CAPTION_CHARS: usize = 1024;
|
||||||
|
|
||||||
|
/// Truncates a caption to at most [`MAX_CAPTION_CHARS`] chars, appending an
|
||||||
|
/// ellipsis when cut. Backs off to before an unclosed HTML entity (`&`
|
||||||
|
/// without its `;` would be malformed HTML and rejected by Telegram).
|
||||||
|
pub fn truncate_caption(caption: &str) -> String {
|
||||||
|
if caption.chars().count() <= MAX_CAPTION_CHARS {
|
||||||
|
return caption.to_string();
|
||||||
|
}
|
||||||
|
// Leave one char for the ellipsis; floor_char_boundary lands on a char
|
||||||
|
// edge (byte index ≤ MAX-1, so chars ≤ MAX-1).
|
||||||
|
let mut end = caption.floor_char_boundary(MAX_CAPTION_CHARS - 1);
|
||||||
|
// Don't split an entity: if the last '&' before `end` has no closing ';'
|
||||||
|
// inside the kept part, cut before it.
|
||||||
|
if let Some(amp) = caption[..end].rfind('&')
|
||||||
|
&& !caption[amp..end].contains(';')
|
||||||
|
{
|
||||||
|
end = amp;
|
||||||
|
}
|
||||||
|
let mut s = caption[..end].to_string();
|
||||||
|
s.push('…');
|
||||||
|
s
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Renders a user-supplied caption format from raw (already-escaped) field
|
/// Renders a user-supplied caption format from raw (already-escaped) field
|
||||||
/// values with the same escaping/substitution rules as
|
/// values with the same escaping/substitution rules as
|
||||||
/// [`Fetched::caption_with`]. An empty format returns `built_in` unchanged.
|
/// [`Fetched::caption_with`]. An empty format returns `built_in` unchanged.
|
||||||
|
/// The result is truncated to [`MAX_CAPTION_CHARS`] (Telegram's caption
|
||||||
|
/// limit for HTML parse mode).
|
||||||
pub fn caption_from_fields(
|
pub fn caption_from_fields(
|
||||||
format: &str,
|
format: &str,
|
||||||
built_in: &str,
|
built_in: &str,
|
||||||
@@ -109,15 +147,17 @@ pub fn caption_from_fields(
|
|||||||
tags: &str,
|
tags: &str,
|
||||||
) -> String {
|
) -> String {
|
||||||
if format.is_empty() {
|
if format.is_empty() {
|
||||||
return built_in.to_string();
|
return truncate_caption(built_in);
|
||||||
}
|
}
|
||||||
let escaped = html_escape::encode_text(format).into_owned();
|
let escaped = html_escape::encode_text(format).into_owned();
|
||||||
escaped
|
truncate_caption(
|
||||||
|
&escaped
|
||||||
.replace("{url}", url)
|
.replace("{url}", url)
|
||||||
.replace("{author}", author)
|
.replace("{author}", author)
|
||||||
.replace("{author_url}", author_url)
|
.replace("{author_url}", author_url)
|
||||||
.replace("{title}", title)
|
.replace("{title}", title)
|
||||||
.replace("{tags}", tags)
|
.replace("{tags}", tags),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stable per-post cache key derived from any supported URL, so variant
|
/// Stable per-post cache key derived from any supported URL, so variant
|
||||||
@@ -147,6 +187,13 @@ pub enum FetchError {
|
|||||||
/// The post exists but its content is withheld (twitter NSFW /
|
/// The post exists but its content is withheld (twitter NSFW /
|
||||||
/// age-restricted tweets come back as an empty `{}` from syndication).
|
/// age-restricted tweets come back as an empty `{}` from syndication).
|
||||||
Sensitive,
|
Sensitive,
|
||||||
|
/// A download exceeded the caller's size cap (see [`download_media_limited`]).
|
||||||
|
TooLarge,
|
||||||
|
/// A transient server-side failure (429 / 5xx); [`fetch`] retries these.
|
||||||
|
Transient(String),
|
||||||
|
/// A local I/O failure while streaming a download to disk
|
||||||
|
/// (see [`download_media_to_file`]).
|
||||||
|
Io(std::io::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl fmt::Display for FetchError {
|
impl fmt::Display for FetchError {
|
||||||
@@ -158,6 +205,9 @@ impl fmt::Display for FetchError {
|
|||||||
FetchError::NotFound => write!(f, "not found"),
|
FetchError::NotFound => write!(f, "not found"),
|
||||||
FetchError::Blocked => write!(f, "blocked"),
|
FetchError::Blocked => write!(f, "blocked"),
|
||||||
FetchError::Sensitive => write!(f, "content withheld (sensitive)"),
|
FetchError::Sensitive => write!(f, "content withheld (sensitive)"),
|
||||||
|
FetchError::TooLarge => write!(f, "media too large"),
|
||||||
|
FetchError::Transient(message) => write!(f, "transient: {message}"),
|
||||||
|
FetchError::Io(e) => write!(f, "io error: {e}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,6 +219,9 @@ impl std::error::Error for FetchError {
|
|||||||
FetchError::Json(e) => Some(e),
|
FetchError::Json(e) => Some(e),
|
||||||
FetchError::Pixiv(e) => Some(e),
|
FetchError::Pixiv(e) => Some(e),
|
||||||
FetchError::NotFound | FetchError::Blocked | FetchError::Sensitive => None,
|
FetchError::NotFound | FetchError::Blocked | FetchError::Sensitive => None,
|
||||||
|
FetchError::TooLarge => None,
|
||||||
|
FetchError::Transient(_) => None,
|
||||||
|
FetchError::Io(e) => Some(e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -179,7 +232,6 @@ impl From<reqwest::Error> for FetchError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
impl From<serde_json::Error> for FetchError {
|
impl From<serde_json::Error> for FetchError {
|
||||||
fn from(e: serde_json::Error) -> Self {
|
fn from(e: serde_json::Error) -> Self {
|
||||||
FetchError::Json(e)
|
FetchError::Json(e)
|
||||||
@@ -195,7 +247,22 @@ impl From<PixivError> for FetchError {
|
|||||||
/// Shared HTTP client (browser User-Agent) for twitter/bsky fetches and
|
/// Shared HTTP client (browser User-Agent) for twitter/bsky fetches and
|
||||||
/// [`download_media`].
|
/// [`download_media`].
|
||||||
pub(crate) static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
pub(crate) static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
||||||
let builder = reqwest::Client::builder().user_agent("Mozilla/5.0");
|
let mut builder = reqwest::Client::builder()
|
||||||
|
.user_agent("Mozilla/5.0")
|
||||||
|
// reqwest has no total timeout by default; a stalled connection
|
||||||
|
// would otherwise pin a fetch/handler forever.
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.connect_timeout(Duration::from_secs(10));
|
||||||
|
// Route site fetches through the same proxy the Bot API uses, so a
|
||||||
|
// network that needs TELOXIDE_PROXY (e.g. behind the GFW) does not
|
||||||
|
// leave site fetches dead while the bot itself works.
|
||||||
|
if let Some(proxy) = std::env::var("TELOXIDE_PROXY")
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
&& let Ok(p) = reqwest::Proxy::all(&proxy)
|
||||||
|
{
|
||||||
|
builder = builder.proxy(p);
|
||||||
|
}
|
||||||
// Each `#[tokio::test]` runs on its own runtime; the connection pool is
|
// Each `#[tokio::test]` runs on its own runtime; the connection pool is
|
||||||
// bound to the runtime that created it, so cross-runtime reuse of idle
|
// bound to the runtime that created it, so cross-runtime reuse of idle
|
||||||
// connections fails with DispatchGone. In test builds every request uses
|
// connections fails with DispatchGone. In test builds every request uses
|
||||||
@@ -205,13 +272,38 @@ pub(crate) static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
|||||||
builder.build().expect("failed to build HTTP client")
|
builder.build().expect("failed to build HTTP client")
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/// Whether a usable `ffmpeg` binary is on PATH (probed once). Shared by the
|
||||||
|
/// pixiv ugoira encoder and the bsky HLS remuxer.
|
||||||
|
static FFMPEG_AVAILABLE: LazyLock<bool> = LazyLock::new(|| {
|
||||||
|
std::process::Command::new("ffmpeg")
|
||||||
|
.arg("-version")
|
||||||
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::null())
|
||||||
|
.status()
|
||||||
|
.map(|s| s.success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
});
|
||||||
|
|
||||||
|
static FFMPEG_MISSING_LOGGED: AtomicBool = AtomicBool::new(false);
|
||||||
|
|
||||||
|
pub(crate) fn ffmpeg_available() -> bool {
|
||||||
|
*FFMPEG_AVAILABLE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn log_once_ffmpeg_missing() {
|
||||||
|
if !FFMPEG_MISSING_LOGGED.swap(true, Ordering::Relaxed) {
|
||||||
|
log::warn!("ffmpeg not found; ugoira and bsky video posts stay unsupported");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Fetches a post from its URL. Returns `Ok(None)` when no site pattern
|
/// Fetches a post from its URL. Returns `Ok(None)` when no site pattern
|
||||||
/// matches (unsupported links are silently ignored by the bot).
|
/// matches (unsupported links are silently ignored by the bot).
|
||||||
///
|
///
|
||||||
/// Transient network failures are retried: 3 total attempts with 1s then 2s
|
/// Transient network failures are retried: 3 total attempts with 1s then 2s
|
||||||
/// delays. Non-Http errors (Json/NotFound/Blocked/Pixiv) are not retried.
|
/// delays. Retried classes: bare HTTP errors, [`FetchError::Transient`]
|
||||||
|
/// (429/5xx from any site), and pixiv errors (its network failures arrive
|
||||||
|
/// wrapped as `PixivError`). Non-retried: Json/NotFound/Blocked/Sensitive.
|
||||||
pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||||
let mut last_http_error = None;
|
|
||||||
for attempt in 0..3u32 {
|
for attempt in 0..3u32 {
|
||||||
match fetch_once(url).await {
|
match fetch_once(url).await {
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(fetched)) => {
|
||||||
@@ -223,18 +315,17 @@ pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
|||||||
return Ok(Some(fetched));
|
return Ok(Some(fetched));
|
||||||
}
|
}
|
||||||
Ok(None) => return Ok(None),
|
Ok(None) => return Ok(None),
|
||||||
Err(FetchError::Http(e)) => {
|
Err(e @ (FetchError::Http(_) | FetchError::Transient(_) | FetchError::Pixiv(_))) => {
|
||||||
last_http_error = Some(e);
|
|
||||||
if attempt < 2 {
|
if attempt < 2 {
|
||||||
tokio::time::sleep(Duration::from_secs(1 << attempt)).await;
|
tokio::time::sleep(Duration::from_secs(1 << attempt)).await;
|
||||||
|
} else {
|
||||||
|
return Err(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(other) => return Err(other),
|
Err(other) => return Err(other),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(FetchError::Http(
|
unreachable!("retry loop always returns")
|
||||||
last_http_error.expect("retry loop always ran 3 attempts"),
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_once(url: &str) -> Result<Option<Fetched>, FetchError> {
|
async fn fetch_once(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||||
@@ -263,18 +354,74 @@ pub async fn media_size(url: &str) -> Result<Option<u64>, FetchError> {
|
|||||||
if lower.contains("pximg.net") {
|
if lower.contains("pximg.net") {
|
||||||
request = request.header("Referer", "https://www.pixiv.net/");
|
request = request.header("Referer", "https://www.pixiv.net/");
|
||||||
}
|
}
|
||||||
let response = request.send().await?;
|
let response = request.send().await?.error_for_status()?;
|
||||||
Ok(response.content_length())
|
Ok(response.content_length())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn download_media(url: &str) -> Result<bytes::Bytes, FetchError> {
|
/// Downloads a media file with a hard size cap: the body is streamed and the
|
||||||
|
/// download aborts with [`FetchError::TooLarge`] the moment the cap is
|
||||||
|
/// crossed (or when a declared Content-Length already exceeds it). Keeps the
|
||||||
|
/// bot from buffering arbitrarily large bodies into memory.
|
||||||
|
pub async fn download_media_limited(url: &str, max_bytes: u64) -> Result<bytes::Bytes, FetchError> {
|
||||||
let mut request = CLIENT.get(url);
|
let mut request = CLIENT.get(url);
|
||||||
let lower = url.to_ascii_lowercase();
|
let lower = url.to_ascii_lowercase();
|
||||||
if lower.contains("pximg.net") {
|
if lower.contains("pximg.net") {
|
||||||
request = request.header("Referer", "https://www.pixiv.net/");
|
request = request.header("Referer", "https://www.pixiv.net/");
|
||||||
}
|
}
|
||||||
let response = request.send().await?;
|
let response = request.send().await?.error_for_status()?;
|
||||||
Ok(response.bytes().await?)
|
if let Some(len) = response.content_length()
|
||||||
|
&& len > max_bytes
|
||||||
|
{
|
||||||
|
return Err(FetchError::TooLarge);
|
||||||
|
}
|
||||||
|
let mut response = response;
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
while let Some(chunk) = response.chunk().await? {
|
||||||
|
buf.extend_from_slice(&chunk);
|
||||||
|
if buf.len() as u64 > max_bytes {
|
||||||
|
return Err(FetchError::TooLarge);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(bytes::Bytes::from(buf))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn download_media(url: &str) -> Result<bytes::Bytes, FetchError> {
|
||||||
|
download_media_limited(url, u64::MAX).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Streams a download to `out`, aborting with [`FetchError::TooLarge`] the
|
||||||
|
/// moment the body crosses `max_bytes` (or when a declared Content-Length
|
||||||
|
/// already exceeds it). Unlike [`download_media_limited`] the body is never
|
||||||
|
/// buffered in memory — used for large files (e.g. the pixiv ugoira frame
|
||||||
|
/// zip, which can be hundreds of MB) that would otherwise spike RAM.
|
||||||
|
/// Returns the number of bytes written.
|
||||||
|
pub async fn download_media_to_file(
|
||||||
|
url: &str,
|
||||||
|
max_bytes: u64,
|
||||||
|
out: &mut std::fs::File,
|
||||||
|
) -> Result<u64, FetchError> {
|
||||||
|
use std::io::Write;
|
||||||
|
let mut request = CLIENT.get(url);
|
||||||
|
let lower = url.to_ascii_lowercase();
|
||||||
|
if lower.contains("pximg.net") {
|
||||||
|
request = request.header("Referer", "https://www.pixiv.net/");
|
||||||
|
}
|
||||||
|
let response = request.send().await?.error_for_status()?;
|
||||||
|
if let Some(len) = response.content_length()
|
||||||
|
&& len > max_bytes
|
||||||
|
{
|
||||||
|
return Err(FetchError::TooLarge);
|
||||||
|
}
|
||||||
|
let mut response = response;
|
||||||
|
let mut total: u64 = 0;
|
||||||
|
while let Some(chunk) = response.chunk().await? {
|
||||||
|
total += chunk.len() as u64;
|
||||||
|
if total > max_bytes {
|
||||||
|
return Err(FetchError::TooLarge);
|
||||||
|
}
|
||||||
|
out.write_all(&chunk).map_err(FetchError::Io)?;
|
||||||
|
}
|
||||||
|
Ok(total)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -330,6 +477,45 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_caption_keeps_short_text() {
|
||||||
|
assert_eq!(truncate_caption("short"), "short");
|
||||||
|
// Exactly at the limit: untouched.
|
||||||
|
let exact = "x".repeat(MAX_CAPTION_CHARS);
|
||||||
|
assert_eq!(truncate_caption(&exact), exact);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_caption_cuts_long_text_with_ellipsis() {
|
||||||
|
let long = "x".repeat(MAX_CAPTION_CHARS + 100);
|
||||||
|
let out = truncate_caption(&long);
|
||||||
|
assert!(
|
||||||
|
out.chars().count() <= MAX_CAPTION_CHARS,
|
||||||
|
"len {}",
|
||||||
|
out.chars().count()
|
||||||
|
);
|
||||||
|
assert!(out.ends_with('…'));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_caption_does_not_split_an_html_entity() {
|
||||||
|
// An entity crossing the cut must not be left half-open (& without ;).
|
||||||
|
let mut long = "a".repeat(MAX_CAPTION_CHARS - 4);
|
||||||
|
long.push_str("&bbbb");
|
||||||
|
let out = truncate_caption(&long);
|
||||||
|
assert!(out.chars().count() <= MAX_CAPTION_CHARS);
|
||||||
|
assert!(!out.contains("&"), "half entity left: {out:?}");
|
||||||
|
assert!(!out.ends_with('&'));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_caption_handles_multibyte_boundary() {
|
||||||
|
// Multi-byte chars near the cut must not panic (char-boundary cut).
|
||||||
|
let long = "界".repeat(MAX_CAPTION_CHARS + 10);
|
||||||
|
let out = truncate_caption(&long);
|
||||||
|
assert!(out.chars().count() <= MAX_CAPTION_CHARS);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn unsupported_url_returns_none() {
|
async fn unsupported_url_returns_none() {
|
||||||
let result = fetch("https://example.com/some/article").await;
|
let result = fetch("https://example.com/some/article").await;
|
||||||
@@ -346,7 +532,13 @@ mod tests {
|
|||||||
async fn download_media_pixiv_original_with_referer() {
|
async fn download_media_pixiv_original_with_referer() {
|
||||||
// Proves the Referer header is attached for i.pximg.net: a header-less
|
// Proves the Referer header is attached for i.pximg.net: a header-less
|
||||||
// GET to a pixiv original URL is rejected with 403.
|
// GET to a pixiv original URL is rejected with 403.
|
||||||
if std::env::var("PIXIV_REFRESH_TOKEN").is_err() {
|
// Empty-string check too: an unset CI secret arrives as "" (GitHub
|
||||||
|
// Actions), which would otherwise run the test tokenless and fail.
|
||||||
|
if std::env::var("PIXIV_REFRESH_TOKEN")
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
eprintln!("skipping: no PIXIV_REFRESH_TOKEN");
|
eprintln!("skipping: no PIXIV_REFRESH_TOKEN");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ use crate::media::Media;
|
|||||||
use crate::site::FetchError;
|
use crate::site::FetchError;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::io::{Cursor, Read};
|
use std::io::Read;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::{Duration, SystemTime};
|
use std::time::{Duration, SystemTime};
|
||||||
|
|
||||||
const AUTH_TOKEN_URL: &str = "https://oauth.secure.pixiv.net/auth/token";
|
const AUTH_TOKEN_URL: &str = "https://oauth.secure.pixiv.net/auth/token";
|
||||||
@@ -127,13 +127,18 @@ impl PixivAPI {
|
|||||||
pub async fn illust_detail(&self, illust_id: u64) -> Result<IllustrationModel, PixivError> {
|
pub async fn illust_detail(&self, illust_id: u64) -> Result<IllustrationModel, PixivError> {
|
||||||
let access_token = self.get_access_token().await?;
|
let access_token = self.get_access_token().await?;
|
||||||
let response = crate::site::CLIENT
|
let response = crate::site::CLIENT
|
||||||
.get(format!("{APP_API_URL}/v1/illust/detail?illust_id={illust_id}"))
|
.get(format!(
|
||||||
|
"{APP_API_URL}/v1/illust/detail?illust_id={illust_id}"
|
||||||
|
))
|
||||||
.header("app-os", "ios")
|
.header("app-os", "ios")
|
||||||
.header("app-os-version", "14.6")
|
.header("app-os-version", "14.6")
|
||||||
.header("User-Agent", APP_USER_AGENT)
|
.header("User-Agent", APP_USER_AGENT)
|
||||||
.bearer_auth(access_token)
|
.bearer_auth(access_token)
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
|
if !response.status().is_success() {
|
||||||
|
return Err(PixivError::Api(format!("status {}", response.status())));
|
||||||
|
}
|
||||||
let json: serde_json::Value = serde_json::from_str(&response.text().await?)?;
|
let json: serde_json::Value = serde_json::from_str(&response.text().await?)?;
|
||||||
if json.get("error").is_some() {
|
if json.get("error").is_some() {
|
||||||
let message = json
|
let message = json
|
||||||
@@ -175,13 +180,18 @@ impl PixivAPI {
|
|||||||
pub async fn ugoira_metadata(&self, illust_id: u64) -> Result<UgoiraMetadataModel, PixivError> {
|
pub async fn ugoira_metadata(&self, illust_id: u64) -> Result<UgoiraMetadataModel, PixivError> {
|
||||||
let access_token = self.get_access_token().await?;
|
let access_token = self.get_access_token().await?;
|
||||||
let response = crate::site::CLIENT
|
let response = crate::site::CLIENT
|
||||||
.get(format!("{APP_API_URL}/v1/ugoira/metadata?illust_id={illust_id}"))
|
.get(format!(
|
||||||
|
"{APP_API_URL}/v1/ugoira/metadata?illust_id={illust_id}"
|
||||||
|
))
|
||||||
.header("app-os", "ios")
|
.header("app-os", "ios")
|
||||||
.header("app-os-version", "14.6")
|
.header("app-os-version", "14.6")
|
||||||
.header("User-Agent", APP_USER_AGENT)
|
.header("User-Agent", APP_USER_AGENT)
|
||||||
.bearer_auth(access_token)
|
.bearer_auth(access_token)
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
|
if !response.status().is_success() {
|
||||||
|
return Err(PixivError::Api(format!("status {}", response.status())));
|
||||||
|
}
|
||||||
let json: serde_json::Value = serde_json::from_str(&response.text().await?)?;
|
let json: serde_json::Value = serde_json::from_str(&response.text().await?)?;
|
||||||
if json.get("error").is_some() {
|
if json.get("error").is_some() {
|
||||||
let message = json
|
let message = json
|
||||||
@@ -203,8 +213,8 @@ impl PixivAPI {
|
|||||||
&self,
|
&self,
|
||||||
illust_id: u64,
|
illust_id: u64,
|
||||||
) -> Result<Option<(String, tempfile::TempDir)>, PixivError> {
|
) -> Result<Option<(String, tempfile::TempDir)>, PixivError> {
|
||||||
if !ffmpeg_available() {
|
if !crate::site::ffmpeg_available() {
|
||||||
log_once_ffmpeg_missing();
|
crate::site::log_once_ffmpeg_missing();
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let metadata = self.ugoira_metadata(illust_id).await?;
|
let metadata = self.ugoira_metadata(illust_id).await?;
|
||||||
@@ -218,42 +228,77 @@ impl PixivAPI {
|
|||||||
let Some(zip_url) = zip_url else {
|
let Some(zip_url) = zip_url else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let zip_bytes = crate::site::download_media(&zip_url).await.map_err(|e| match e {
|
// Stream the frame zip to a temp file instead of buffering it in
|
||||||
|
// memory: ugoira zips can be hundreds of MB, and the old
|
||||||
|
// download_media_limited path spiked RAM up to the size cap.
|
||||||
|
let mut zip_file = tempfile::Builder::new()
|
||||||
|
.suffix(".zip")
|
||||||
|
.tempfile()
|
||||||
|
.map_err(|e| PixivError::Api(format!("temp zip failed: {e}")))?;
|
||||||
|
crate::site::download_media_to_file(&zip_url, 512 * 1024 * 1024, zip_file.as_file_mut())
|
||||||
|
.await
|
||||||
|
.map_err(|e| match e {
|
||||||
FetchError::Http(e) => PixivError::Http(e),
|
FetchError::Http(e) => PixivError::Http(e),
|
||||||
other => PixivError::Api(format!("frame zip download failed: {other}")),
|
other => PixivError::Api(format!("frame zip download failed: {other}")),
|
||||||
})?;
|
})?;
|
||||||
let frame_delays = metadata.frames.iter().map(|f| f.delay).collect::<Vec<_>>();
|
let frame_delays = metadata.frames.iter().map(|f| f.delay).collect::<Vec<_>>();
|
||||||
let result = tokio::task::spawn_blocking(
|
let result =
|
||||||
move || -> Result<(String, tempfile::TempDir), String> {
|
tokio::task::spawn_blocking(move || -> Result<(String, tempfile::TempDir), String> {
|
||||||
let frames_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
let frames_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
let out_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
let out_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
// Extract frames to canonical zero-padded names; pixiv ugoira
|
// Extract frames to canonical zero-padded names; pixiv ugoira
|
||||||
// frames are uniformly jpg or png per artwork.
|
// frames are uniformly jpg or png per artwork. The zip is read
|
||||||
let mut archive = zip::ZipArchive::new(Cursor::new(zip_bytes))
|
// from disk; `zip_file` stays alive for the whole extraction.
|
||||||
|
let mut archive = zip::ZipArchive::new(
|
||||||
|
std::fs::File::open(zip_file.path()).map_err(|e| e.to_string())?,
|
||||||
|
)
|
||||||
.map_err(|e| format!("unzip: {e}"))?;
|
.map_err(|e| format!("unzip: {e}"))?;
|
||||||
// pixiv ugoira frames are uniformly jpg or png per artwork; take
|
if archive.is_empty() {
|
||||||
// the extension from the first entry.
|
return Err("empty frame zip".to_string());
|
||||||
let extension = if archive.len() > 0 {
|
}
|
||||||
let first_name = archive
|
// Uniform jpg or png per artwork; sniff the first entry's
|
||||||
.by_index(0)
|
// magic bytes instead of trusting its filename.
|
||||||
.map_err(|e| e.to_string())?
|
let first = archive.by_index(0).map_err(|e| e.to_string())?;
|
||||||
.name()
|
let mut first_bytes = Vec::new();
|
||||||
.to_string();
|
first
|
||||||
first_name
|
.take(64 * 1024 * 1024 + 1)
|
||||||
.rsplit('.')
|
.read_to_end(&mut first_bytes)
|
||||||
.next()
|
.map_err(|e| e.to_string())?;
|
||||||
.unwrap_or("jpg")
|
if first_bytes.len() > 64 * 1024 * 1024 {
|
||||||
.to_string()
|
return Err("frame exceeds size cap".to_string());
|
||||||
|
}
|
||||||
|
let extension = if first_bytes.starts_with(&[0xFF, 0xD8]) {
|
||||||
|
"jpg"
|
||||||
|
} else if first_bytes.starts_with(b"\x89PNG") {
|
||||||
|
"png"
|
||||||
} else {
|
} else {
|
||||||
"jpg".to_string()
|
"jpg"
|
||||||
};
|
};
|
||||||
let mut count = 0usize;
|
let mut count = 0usize;
|
||||||
for i in 0..archive.len() {
|
{
|
||||||
let mut entry = archive.by_index(i).map_err(|e| e.to_string())?;
|
let path = frames_dir
|
||||||
|
.path()
|
||||||
|
.join(format!("img_{count:05}.{extension}"));
|
||||||
|
std::fs::write(&path, &first_bytes).map_err(|e| e.to_string())?;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
for i in 1..archive.len() {
|
||||||
|
let entry = archive.by_index(i).map_err(|e| e.to_string())?;
|
||||||
|
if entry.size() > 64 * 1024 * 1024 {
|
||||||
|
return Err(format!("frame {i} exceeds size cap"));
|
||||||
|
}
|
||||||
let mut bytes = Vec::new();
|
let mut bytes = Vec::new();
|
||||||
entry.read_to_end(&mut bytes).map_err(|e| e.to_string())?;
|
entry
|
||||||
let path = frames_dir.path().join(format!("img_{count:05}.{extension}"));
|
.take(64 * 1024 * 1024 + 1)
|
||||||
|
.read_to_end(&mut bytes)
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
if bytes.len() > 64 * 1024 * 1024 {
|
||||||
|
return Err(format!("frame {i} exceeds size cap"));
|
||||||
|
}
|
||||||
|
let path = frames_dir
|
||||||
|
.path()
|
||||||
|
.join(format!("img_{count:05}.{extension}"));
|
||||||
std::fs::write(&path, bytes).map_err(|e| e.to_string())?;
|
std::fs::write(&path, bytes).map_err(|e| e.to_string())?;
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
@@ -274,7 +319,10 @@ impl PixivAPI {
|
|||||||
"-framerate",
|
"-framerate",
|
||||||
&framerate.to_string(),
|
&framerate.to_string(),
|
||||||
"-i",
|
"-i",
|
||||||
&frames_dir.path().join(format!("img_%05d.{extension}")).to_string_lossy(),
|
&frames_dir
|
||||||
|
.path()
|
||||||
|
.join(format!("img_%05d.{extension}"))
|
||||||
|
.to_string_lossy(),
|
||||||
// libx264 needs even dimensions; pixiv ugoira frames can
|
// libx264 needs even dimensions; pixiv ugoira frames can
|
||||||
// be odd-sized (e.g. 277x405).
|
// be odd-sized (e.g. 277x405).
|
||||||
"-vf",
|
"-vf",
|
||||||
@@ -295,10 +343,12 @@ impl PixivAPI {
|
|||||||
return Err(format!("ffmpeg exited with {status}"));
|
return Err(format!("ffmpeg exited with {status}"));
|
||||||
}
|
}
|
||||||
Ok((output.to_string_lossy().into_owned(), out_dir))
|
Ok((output.to_string_lossy().into_owned(), out_dir))
|
||||||
},
|
})
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
.expect("ugoira encode worker panicked");
|
.map_err(|e| {
|
||||||
|
log::error!("ugoira encode worker panicked for {illust_id}: {e}");
|
||||||
|
PixivError::Api(format!("ugoira worker failed: {e}"))
|
||||||
|
})?;
|
||||||
match result {
|
match result {
|
||||||
Ok(pair) => Ok(Some(pair)),
|
Ok(pair) => Ok(Some(pair)),
|
||||||
Err(message) => {
|
Err(message) => {
|
||||||
@@ -309,32 +359,9 @@ impl PixivAPI {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static FFMPEG_AVAILABLE: LazyLock<bool> = LazyLock::new(|| {
|
|
||||||
std::process::Command::new("ffmpeg")
|
|
||||||
.arg("-version")
|
|
||||||
.stdout(std::process::Stdio::null())
|
|
||||||
.stderr(std::process::Stdio::null())
|
|
||||||
.status()
|
|
||||||
.map(|s| s.success())
|
|
||||||
.unwrap_or(false)
|
|
||||||
});
|
|
||||||
|
|
||||||
static FFMPEG_MISSING_LOGGED: AtomicBool = AtomicBool::new(false);
|
|
||||||
|
|
||||||
fn ffmpeg_available() -> bool {
|
|
||||||
*FFMPEG_AVAILABLE
|
|
||||||
}
|
|
||||||
|
|
||||||
fn log_once_ffmpeg_missing() {
|
|
||||||
if !FFMPEG_MISSING_LOGGED.swap(true, Ordering::Relaxed) {
|
|
||||||
log::warn!("ffmpeg not found; pixiv ugoira posts stay unsupported");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// pixiv3-rs replacement: `None` when `PIXIV_REFRESH_TOKEN` is unset.
|
/// pixiv3-rs replacement: `None` when `PIXIV_REFRESH_TOKEN` is unset.
|
||||||
static PIXIV_CLIENT: LazyLock<Option<PixivAPI>> = LazyLock::new(|| {
|
static PIXIV_CLIENT: LazyLock<Option<PixivAPI>> =
|
||||||
env::var("PIXIV_REFRESH_TOKEN").ok().map(PixivAPI::new)
|
LazyLock::new(|| env::var("PIXIV_REFRESH_TOKEN").ok().map(PixivAPI::new));
|
||||||
});
|
|
||||||
|
|
||||||
/// Set at startup when the login validation fails; pixiv stays disabled until
|
/// Set at startup when the login validation fails; pixiv stays disabled until
|
||||||
/// the next process start.
|
/// the next process start.
|
||||||
@@ -375,16 +402,31 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use dotenv::dotenv;
|
use dotenv::dotenv;
|
||||||
|
|
||||||
|
/// Skips when `PIXIV_REFRESH_TOKEN` is absent or empty (CI without the
|
||||||
|
/// secret must stay green; GitHub Actions exposes an unset secret as an
|
||||||
|
/// empty string, so `is_err()` alone is not enough).
|
||||||
|
fn require_pixiv_token() -> bool {
|
||||||
|
std::env::var("PIXIV_REFRESH_TOKEN")
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.is_some()
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_fetch() {
|
async fn test_fetch() {
|
||||||
dotenv().ok();
|
dotenv().ok();
|
||||||
|
if !require_pixiv_token() {
|
||||||
|
eprintln!("skipping: no PIXIV_REFRESH_TOKEN");
|
||||||
|
return;
|
||||||
|
}
|
||||||
let result = fetch(126839080).await;
|
let result = fetch(126839080).await;
|
||||||
assert!(result.is_ok());
|
assert!(result.is_ok());
|
||||||
println!("{:#?}", result);
|
println!("{:#?}", result);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn validate_with_bogus_token_fails() {
|
#[ignore = "live network: requires outbound HTTPS to oauth.secure.pixiv.net"]
|
||||||
|
async fn live_validate_with_bogus_token_fails() {
|
||||||
dotenv().ok();
|
dotenv().ok();
|
||||||
// A bogus token must surface as Api error (invalid_grant), not panic.
|
// A bogus token must surface as Api error (invalid_grant), not panic.
|
||||||
let client = PixivAPI::new("bogus_token_for_testing".to_string());
|
let client = PixivAPI::new("bogus_token_for_testing".to_string());
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
use super::model::{IllustrationModel, TypeModel};
|
use super::model::{IllustrationModel, TypeModel};
|
||||||
use crate::media::Media;
|
use crate::media::Media;
|
||||||
use crate::site::{FetchError, Fetched};
|
use crate::site::{FetchError, Fetched};
|
||||||
use html_escape::encode_text;
|
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
Regex::new(r"(?:www\.)?pixiv\.net/(?:en/)?(?:(?:i|artworks)/|member_illust\.php\?(?:mode=[a-z_]*&)?illust_id=)(\d+)").unwrap()
|
Regex::new(r"^(?:https?://)?(?:www\.)?pixiv\.net/(?:en/)?(?:(?:i|artworks)/|member_illust\.php\?(?:mode=[a-z_]*&)?illust_id=)(\d+)").unwrap()
|
||||||
});
|
});
|
||||||
|
|
||||||
pub fn enabled() -> bool {
|
pub fn enabled() -> bool {
|
||||||
@@ -48,9 +48,9 @@ impl Illustration {
|
|||||||
pub fn caption(&self) -> String {
|
pub fn caption(&self) -> String {
|
||||||
format!(
|
format!(
|
||||||
"<a href=\"{url}\">{title}</a> / <a href=\"{author_url}\">{author}</a>\n{tags}",
|
"<a href=\"{url}\">{title}</a> / <a href=\"{author_url}\">{author}</a>\n{tags}",
|
||||||
url = self.url(),
|
url = encode_double_quoted_attribute(&self.url()),
|
||||||
title = encode_text(&self.title),
|
title = encode_text(&self.title),
|
||||||
author_url = self.author_url(),
|
author_url = encode_double_quoted_attribute(&self.author_url()),
|
||||||
author = encode_text(&self.author),
|
author = encode_text(&self.author),
|
||||||
tags = encode_text(
|
tags = encode_text(
|
||||||
&self
|
&self
|
||||||
@@ -82,7 +82,10 @@ impl Illustration {
|
|||||||
// keeps media empty when encoding fails or ffmpeg is missing.
|
// keeps media empty when encoding fails or ffmpeg is missing.
|
||||||
} else if model.page_count > 1 {
|
} else if model.page_count > 1 {
|
||||||
media.extend(model.meta_pages.iter().filter_map(|page| {
|
media.extend(model.meta_pages.iter().filter_map(|page| {
|
||||||
page.image_urls.original.clone().map(|original| Media::Illustration {
|
page.image_urls
|
||||||
|
.original
|
||||||
|
.clone()
|
||||||
|
.map(|original| Media::Illustration {
|
||||||
title: None,
|
title: None,
|
||||||
url: original,
|
url: original,
|
||||||
thumbnail_url: Some(page.image_urls.medium.clone()),
|
thumbnail_url: Some(page.image_urls.medium.clone()),
|
||||||
@@ -147,8 +150,8 @@ impl From<Illustration> for Fetched {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
|
||||||
use super::super::model::IllustrationModel;
|
use super::super::model::IllustrationModel;
|
||||||
|
use super::*;
|
||||||
|
|
||||||
fn illust_json(
|
fn illust_json(
|
||||||
type_: &str,
|
type_: &str,
|
||||||
@@ -203,8 +206,14 @@ mod tests {
|
|||||||
("https://pixiv.net/artworks/123456", "123456"),
|
("https://pixiv.net/artworks/123456", "123456"),
|
||||||
("https://www.pixiv.net/en/artworks/123456", "123456"),
|
("https://www.pixiv.net/en/artworks/123456", "123456"),
|
||||||
("https://www.pixiv.net/i/123456", "123456"),
|
("https://www.pixiv.net/i/123456", "123456"),
|
||||||
("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=123456", "123456"),
|
(
|
||||||
("https://www.pixiv.net/en/member_illust.php?illust_id=123456", "123456"),
|
"https://www.pixiv.net/member_illust.php?mode=medium&illust_id=123456",
|
||||||
|
"123456",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"https://www.pixiv.net/en/member_illust.php?illust_id=123456",
|
||||||
|
"123456",
|
||||||
|
),
|
||||||
];
|
];
|
||||||
for (url, id) in cases {
|
for (url, id) in cases {
|
||||||
let caps = PATTERN.captures(url).unwrap_or_else(|| panic!("{url}"));
|
let caps = PATTERN.captures(url).unwrap_or_else(|| panic!("{url}"));
|
||||||
@@ -225,7 +234,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ugoira_yields_empty_media() {
|
fn ugoira_yields_empty_media() {
|
||||||
let v = illust_json("ugoira", 1, Some("https://i.pximg.net/orig.jpg"), None, vec![], 0);
|
let v = illust_json(
|
||||||
|
"ugoira",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/orig.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
0,
|
||||||
|
);
|
||||||
let illustration = parse(v);
|
let illustration = parse(v);
|
||||||
let fetched: Fetched = illustration.into();
|
let fetched: Fetched = illustration.into();
|
||||||
assert!(fetched.media.is_empty());
|
assert!(fetched.media.is_empty());
|
||||||
@@ -296,7 +312,12 @@ mod tests {
|
|||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
match &fetched.media[0] {
|
match &fetched.media[0] {
|
||||||
Media::Illustration { url, thumbnail_url, fallback_url, .. } => {
|
Media::Illustration {
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
fallback_url,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
assert_eq!(url, "https://i.pximg.net/p2.jpg");
|
assert_eq!(url, "https://i.pximg.net/p2.jpg");
|
||||||
assert_eq!(thumbnail_url.as_deref(), Some("m2.jpg"));
|
assert_eq!(thumbnail_url.as_deref(), Some("m2.jpg"));
|
||||||
assert_eq!(fallback_url.as_deref(), Some("l2.jpg"));
|
assert_eq!(fallback_url.as_deref(), Some("l2.jpg"));
|
||||||
@@ -307,7 +328,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn caption_with_escapes_format_and_substitutes() {
|
fn caption_with_escapes_format_and_substitutes() {
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], 0);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
0,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
// Format string is escaped in full, then placeholders substituted.
|
// Format string is escaped in full, then placeholders substituted.
|
||||||
let out = fetched.caption_with("{title} by {author} <script> {tags}");
|
let out = fetched.caption_with("{title} by {author} <script> {tags}");
|
||||||
@@ -330,7 +358,14 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn ai_work_gets_leading_ai_tag() {
|
fn ai_work_gets_leading_ai_tag() {
|
||||||
// illust_ai_type == 2 is the only AI marker.
|
// illust_ai_type == 2 is the only AI marker.
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], 2);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
2,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert!(
|
assert!(
|
||||||
fetched.caption.contains("#AI #tag1 #tag2"),
|
fetched.caption.contains("#AI #tag1 #tag2"),
|
||||||
@@ -338,14 +373,25 @@ mod tests {
|
|||||||
fetched.caption
|
fetched.caption
|
||||||
);
|
);
|
||||||
// The {tags} placeholder reflects the tag array too.
|
// The {tags} placeholder reflects the tag array too.
|
||||||
assert!(fetched.caption_with("{tags}").starts_with("#AI "), "got: {}", fetched.caption_with("{tags}"));
|
assert!(
|
||||||
|
fetched.caption_with("{tags}").starts_with("#AI "),
|
||||||
|
"got: {}",
|
||||||
|
fetched.caption_with("{tags}")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn non_ai_work_has_no_ai_tag() {
|
fn non_ai_work_has_no_ai_tag() {
|
||||||
// 1 = explicitly not AI, 0 = undefined: neither gets the #AI tag.
|
// 1 = explicitly not AI, 0 = undefined: neither gets the #AI tag.
|
||||||
for ai_type in [0, 1] {
|
for ai_type in [0, 1] {
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], ai_type);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
ai_type,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert!(
|
assert!(
|
||||||
!fetched.caption.contains("#AI"),
|
!fetched.caption.contains("#AI"),
|
||||||
@@ -357,7 +403,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn caption_escapes_and_links() {
|
fn caption_escapes_and_links() {
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], 0);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
0,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert!(
|
assert!(
|
||||||
fetched
|
fetched
|
||||||
@@ -367,9 +420,6 @@ mod tests {
|
|||||||
fetched.caption
|
fetched.caption
|
||||||
);
|
);
|
||||||
assert!(fetched.caption.contains("#tag1 #tag2"));
|
assert!(fetched.caption.contains("#tag1 #tag2"));
|
||||||
assert_eq!(
|
assert_eq!(fetched.source_url, "https://www.pixiv.net/artworks/123");
|
||||||
fetched.source_url,
|
|
||||||
"https://www.pixiv.net/artworks/123"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,4 +3,4 @@ mod interface;
|
|||||||
mod model;
|
mod model;
|
||||||
|
|
||||||
pub use api::{PixivAPI, PixivError, disable, fetch, validate};
|
pub use api::{PixivAPI, PixivError, disable, fetch, validate};
|
||||||
pub use interface::{PATTERN, Illustration, enabled, fetch_from_url};
|
pub use interface::{Illustration, PATTERN, enabled, fetch_from_url};
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use crate::site::FetchError;
|
use crate::site::FetchError;
|
||||||
|
|
||||||
@@ -40,8 +40,7 @@ static AUTH_TOKEN: LazyLock<Option<String>> = LazyLock::new(|| {
|
|||||||
});
|
});
|
||||||
|
|
||||||
/// Public "logged in" client token used by the x.com web app.
|
/// Public "logged in" client token used by the x.com web app.
|
||||||
const LOGGED_IN_BEARER: &str =
|
const LOGGED_IN_BEARER: &str = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
||||||
"Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
||||||
|
|
||||||
/// `TweetDetail` query id (from nazurin; still valid as of 2026-08,
|
/// `TweetDetail` query id (from nazurin; still valid as of 2026-08,
|
||||||
/// corroborated by the current FxEmbed build — see module caveats).
|
/// corroborated by the current FxEmbed build — see module caveats).
|
||||||
@@ -103,9 +102,7 @@ pub fn enabled() -> bool {
|
|||||||
/// Fetches a tweet as the logged-in user via the private GraphQL API.
|
/// Fetches a tweet as the logged-in user via the private GraphQL API.
|
||||||
/// Returns the syndication-shaped [`Tweet`] (media included for NSFW posts).
|
/// Returns the syndication-shaped [`Tweet`] (media included for NSFW posts).
|
||||||
pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
||||||
let token = AUTH_TOKEN
|
let token = AUTH_TOKEN.as_deref().ok_or(FetchError::Sensitive)?;
|
||||||
.as_deref()
|
|
||||||
.ok_or(FetchError::Sensitive)?;
|
|
||||||
// 16 random bytes as 32 hex chars: X rejects ct0 values of any other
|
// 16 random bytes as 32 hex chars: X rejects ct0 values of any other
|
||||||
// length with 403 code 353 ("matching csrf cookie and header").
|
// length with 403 code 353 ("matching csrf cookie and header").
|
||||||
let ct0: String = (0..16)
|
let ct0: String = (0..16)
|
||||||
@@ -129,18 +126,26 @@ pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
|||||||
.header("referer", "https://x.com/")
|
.header("referer", "https://x.com/")
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
if !response.status().is_success() {
|
// 404/410 = gone (permanent); 429/5xx = transient and retried by fetch.
|
||||||
log::warn!("twitter auth fetch {id}: HTTP {}", response.status());
|
let status = response.status();
|
||||||
return Err(FetchError::NotFound);
|
if !status.is_success() {
|
||||||
|
log::warn!("twitter auth fetch {id}: HTTP {status}");
|
||||||
|
return match status.as_u16() {
|
||||||
|
404 | 410 => Err(FetchError::NotFound),
|
||||||
|
_ => Err(FetchError::Transient(format!(
|
||||||
|
"twitter auth status {status}"
|
||||||
|
))),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
let text = response.text().await?;
|
let text = response.text().await?;
|
||||||
let json: Value = serde_json::from_str(&text)?;
|
let json: Value = serde_json::from_str(&text)?;
|
||||||
let result = parse_tweet_result(&json, id)?;
|
let result = parse_tweet_result(&json, id)?;
|
||||||
let syndication_shape = to_syndication_shape(&result)
|
let syndication_shape = to_syndication_shape(&result).ok_or_else(|| {
|
||||||
.ok_or_else(|| FetchError::Json(serde_json::Error::io(std::io::Error::new(
|
FetchError::Json(serde_json::Error::io(std::io::Error::new(
|
||||||
std::io::ErrorKind::InvalidData,
|
std::io::ErrorKind::InvalidData,
|
||||||
"missing tweet fields in GraphQL response",
|
"missing tweet fields in GraphQL response",
|
||||||
))))?;
|
)))
|
||||||
|
})?;
|
||||||
Tweet::from_syndication_json(&syndication_shape.to_string()).map_err(FetchError::Json)
|
Tweet::from_syndication_json(&syndication_shape.to_string()).map_err(FetchError::Json)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,7 +233,6 @@ fn to_syndication_shape(tweet: &Value) -> Option<Value> {
|
|||||||
"screen_name": user.get("screen_name"),
|
"screen_name": user.get("screen_name"),
|
||||||
},
|
},
|
||||||
"possibly_sensitive": legacy.get("possibly_sensitive"),
|
"possibly_sensitive": legacy.get("possibly_sensitive"),
|
||||||
"display_text_range": legacy.get("display_text_range"),
|
|
||||||
"entities": legacy.get("entities"),
|
"entities": legacy.get("entities"),
|
||||||
"mediaDetails": legacy.pointer("/extended_entities/media"),
|
"mediaDetails": legacy.pointer("/extended_entities/media"),
|
||||||
}))
|
}))
|
||||||
@@ -251,12 +255,12 @@ mod tests {
|
|||||||
"legacy": {
|
"legacy": {
|
||||||
"id_str": "2083868672721039569",
|
"id_str": "2083868672721039569",
|
||||||
"full_text": "nsfw content https://t.co/abc123",
|
"full_text": "nsfw content https://t.co/abc123",
|
||||||
"display_text_range": [0, 12],
|
|
||||||
"possibly_sensitive": true,
|
"possibly_sensitive": true,
|
||||||
"entities": {
|
"entities": {
|
||||||
"urls": [
|
// The appended media link lives in extended_entities.media,
|
||||||
{ "url": "https://t.co/abc123", "expanded_url": "https://example.com/x" }
|
// not entities.urls, so it has no expansion mapping and the
|
||||||
]
|
// content-based strip removes it.
|
||||||
|
"urls": []
|
||||||
},
|
},
|
||||||
"extended_entities": {
|
"extended_entities": {
|
||||||
"media": [
|
"media": [
|
||||||
@@ -318,8 +322,11 @@ mod tests {
|
|||||||
}
|
}
|
||||||
other => panic!("expected video, got {other:?}"),
|
other => panic!("expected video, got {other:?}"),
|
||||||
}
|
}
|
||||||
assert_eq!(fetched.source_url, "https://x.com/nsfw_author/status/2083868672721039569");
|
assert_eq!(
|
||||||
// display_text_range cuts the trailing t.co link.
|
fetched.source_url,
|
||||||
|
"https://x.com/nsfw_author/status/2083868672721039569"
|
||||||
|
);
|
||||||
|
// The appended media short link (no URL-entity mapping) is stripped.
|
||||||
assert_eq!(fetched.title, "nsfw content");
|
assert_eq!(fetched.title, "nsfw content");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -331,7 +338,10 @@ mod tests {
|
|||||||
let json = conversation(rt);
|
let json = conversation(rt);
|
||||||
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
||||||
assert!(result.pointer("/legacy/retweeted_status_result").is_none());
|
assert!(result.pointer("/legacy/retweeted_status_result").is_none());
|
||||||
assert_eq!(result.pointer("/legacy/id_str").unwrap(), "2083868672721039569");
|
assert_eq!(
|
||||||
|
result.pointer("/legacy/id_str").unwrap(),
|
||||||
|
"2083868672721039569"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use super::model;
|
use super::model;
|
||||||
use crate::media::Media;
|
use crate::media::Media;
|
||||||
use crate::site::{FetchError, Fetched};
|
use crate::site::{FetchError, Fetched};
|
||||||
use html_escape::encode_text;
|
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
@@ -35,9 +35,7 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log::info!(
|
log::info!("tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media");
|
||||||
"tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media"
|
|
||||||
);
|
|
||||||
Ok(empty_fetched(url))
|
Ok(empty_fetched(url))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -50,7 +48,9 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
|||||||
fn empty_fetched(url: &str) -> Fetched {
|
fn empty_fetched(url: &str) -> Fetched {
|
||||||
Fetched {
|
Fetched {
|
||||||
source_url: url.to_string(),
|
source_url: url.to_string(),
|
||||||
caption: url.to_string(),
|
// The raw user-supplied URL goes into an HTML caption; escape it so
|
||||||
|
// crafted links cannot break the parse (Telegram 400).
|
||||||
|
caption: encode_text(url).into_owned(),
|
||||||
title: String::new(),
|
title: String::new(),
|
||||||
media: vec![],
|
media: vec![],
|
||||||
sensitive: true,
|
sensitive: true,
|
||||||
@@ -70,8 +70,13 @@ pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
|||||||
))
|
))
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
if !response.status().is_success() {
|
// 404/410 = gone (permanent); 429/5xx = transient and retried by fetch.
|
||||||
return Err(FetchError::NotFound);
|
let status = response.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
return match status.as_u16() {
|
||||||
|
404 | 410 => Err(FetchError::NotFound),
|
||||||
|
_ => Err(FetchError::Transient(format!("twitter status {status}"))),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
let text = response.text().await?;
|
let text = response.text().await?;
|
||||||
// Deleted tweets answer with {"errors": [...]} instead of a tweet.
|
// Deleted tweets answer with {"errors": [...]} instead of a tweet.
|
||||||
@@ -148,8 +153,8 @@ impl Tweet {
|
|||||||
pub fn caption(&self) -> String {
|
pub fn caption(&self) -> String {
|
||||||
format!(
|
format!(
|
||||||
"{url}\n<a href=\"{author_url}\">{author}</a>: {text}",
|
"{url}\n<a href=\"{author_url}\">{author}</a>: {text}",
|
||||||
url = self.url(),
|
url = encode_double_quoted_attribute(&self.url()),
|
||||||
author_url = self.author_url(),
|
author_url = encode_double_quoted_attribute(&self.author_url()),
|
||||||
author = encode_text(&self.author),
|
author = encode_text(&self.author),
|
||||||
text = encode_text(&self.text),
|
text = encode_text(&self.text),
|
||||||
)
|
)
|
||||||
@@ -158,12 +163,10 @@ impl Tweet {
|
|||||||
pub fn from_syndication_json(raw_json: &str) -> Result<Self, serde_json::Error> {
|
pub fn from_syndication_json(raw_json: &str) -> Result<Self, serde_json::Error> {
|
||||||
let json: model::SyndicationTweet = serde_json::from_str(raw_json)?;
|
let json: model::SyndicationTweet = serde_json::from_str(raw_json)?;
|
||||||
let id = json.id_str;
|
let id = json.id_str;
|
||||||
// Strip the appended media short link first, then expand the remaining
|
// Expand the user's t.co short links to their real destinations and
|
||||||
// t.co short links (the user's own URLs) to their real destinations.
|
// strip the appended media short link, mirroring FxEmbed's linkFixer
|
||||||
let text = expand_links(
|
// (no display_text_range arithmetic — see expand_links).
|
||||||
&strip_trailing_short_links(&json.text, json.display_text_range),
|
let text = expand_links(&json.text, &json.entities.urls);
|
||||||
&json.entities.urls,
|
|
||||||
);
|
|
||||||
// `name` is the display name, `screen_name` the handle (Python's
|
// `name` is the display name, `screen_name` the handle (Python's
|
||||||
// vxtwitter mapping: author = display name, author_id = handle).
|
// vxtwitter mapping: author = display name, author_id = handle).
|
||||||
let author = json.user.name;
|
let author = json.user.name;
|
||||||
@@ -204,51 +207,48 @@ impl Tweet {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The raw syndication `text` ends with the appended media short link
|
/// Mirrors FxEmbed's `linkFixer` (link-fixer.ts): expand every t.co short
|
||||||
/// (" https://t.co/wmI8McgXul"). `display_text_range` marks the visible text;
|
/// link that has an entity mapping to its real destination, drop internal
|
||||||
/// a regex strips any remaining trailing t.co link when the range is absent
|
/// `x.com/i/web/status/…` plumbing links, then strip any remaining t.co
|
||||||
/// or a tweet ends in a URL short link.
|
/// short link (the appended media link and other unmapped short links).
|
||||||
///
|
/// Pure content matching — no `display_text_range` arithmetic, so the
|
||||||
/// X reports these indices in Unicode **code points**, not UTF-16 units
|
/// endpoint's inconsistent index units (UTF-16 vs code points, see the
|
||||||
/// (verified against GraphQL responses containing emoji: cutting an emoji
|
/// deleted `strip_trailing_short_links`) never matter.
|
||||||
/// tweet by UTF-16 units silently drops the character after the emoji).
|
|
||||||
fn strip_trailing_short_links(text: &str, display_text_range: Option<[usize; 2]>) -> String {
|
|
||||||
let mut out = match display_text_range {
|
|
||||||
Some([start, end]) if start < end => {
|
|
||||||
text.chars().skip(start).take(end - start).collect()
|
|
||||||
}
|
|
||||||
_ => text.to_string(),
|
|
||||||
};
|
|
||||||
while TRAILING_TCO.is_match(&out) {
|
|
||||||
out = TRAILING_TCO.replace(&out, "").into_owned();
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Trailing Twitter short link, optionally preceded by whitespace.
|
|
||||||
static TRAILING_TCO: LazyLock<Regex> = LazyLock::new(|| {
|
|
||||||
Regex::new(r"\s*https?://t\.co/[A-Za-z0-9]+$").unwrap()
|
|
||||||
});
|
|
||||||
|
|
||||||
/// Replaces every t.co short link that has an entity mapping with its
|
|
||||||
/// expanded URL. Short links without a mapping stay untouched.
|
|
||||||
fn expand_links(text: &str, urls: &[model::SyndicationEntityUrl]) -> String {
|
fn expand_links(text: &str, urls: &[model::SyndicationEntityUrl]) -> String {
|
||||||
let mut out = text.to_string();
|
let mut out = text.to_string();
|
||||||
for entity in urls {
|
for entity in urls {
|
||||||
if let Some(expanded) = &entity.expanded_url {
|
let Some(expanded) = &entity.expanded_url else {
|
||||||
out = out.replace(&entity.url, expanded);
|
continue;
|
||||||
|
};
|
||||||
|
let replacement = if WEB_STATUS_URL.is_match(expanded) {
|
||||||
|
""
|
||||||
|
} else {
|
||||||
|
expanded
|
||||||
|
};
|
||||||
|
out = out.replace(&entity.url, replacement);
|
||||||
}
|
}
|
||||||
}
|
TCO_LINK.replace_all(&out, "").into_owned()
|
||||||
out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Internal x.com page links (reply / quote plumbing) expand to
|
||||||
|
/// `x.com/i/web/status/<id>`; FxEmbed drops them — the tweet's own content
|
||||||
|
/// already carries the information.
|
||||||
|
static WEB_STATUS_URL: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"^https://(?:x\.com|twitter\.com)/i/web/status/\w+").unwrap());
|
||||||
|
|
||||||
|
/// A t.co short link, optionally preceded by a space. Any leftover
|
||||||
|
/// occurrence (unmapped — e.g. the appended media link) is removed,
|
||||||
|
/// mirroring FxEmbed. Real short-link codes are 10 alphanumerics; the
|
||||||
|
/// length-agnostic class keeps fixtures and hypothetical odd lengths safe.
|
||||||
|
static TCO_LINK: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r" ?https?://t\.co/[A-Za-z0-9]+").unwrap());
|
||||||
|
|
||||||
/// pbs.twimg.com serves a reduced default size without size params; `name=orig`
|
/// pbs.twimg.com serves a reduced default size without size params; `name=orig`
|
||||||
/// returns the original file (fxtwitter used to hand out the original
|
/// returns the original file (fxtwitter used to hand out the original
|
||||||
/// directly, the syndication API does not). Non-twimg URLs pass through
|
/// directly, the syndication API does not). Non-twimg URLs pass through
|
||||||
/// unchanged.
|
/// unchanged.
|
||||||
fn original_twimg_url(url: &str) -> String {
|
fn original_twimg_url(url: &str) -> String {
|
||||||
if url.starts_with("https://pbs.twimg.com/")
|
if url.starts_with("https://pbs.twimg.com/") && (url.ends_with(".jpg") || url.ends_with(".png"))
|
||||||
&& (url.ends_with(".jpg") || url.ends_with(".png"))
|
|
||||||
{
|
{
|
||||||
format!("{url}?name=orig")
|
format!("{url}?name=orig")
|
||||||
} else {
|
} else {
|
||||||
@@ -363,24 +363,23 @@ mod tests {
|
|||||||
match &fetched.media[0] {
|
match &fetched.media[0] {
|
||||||
Media::Illustration { url, .. } => {
|
Media::Illustration { url, .. } => {
|
||||||
// Photo URL is rewritten to request the original file.
|
// Photo URL is rewritten to request the original file.
|
||||||
assert_eq!(
|
assert_eq!(url, "https://pbs.twimg.com/media/photo.jpg?name=orig");
|
||||||
url,
|
|
||||||
"https://pbs.twimg.com/media/photo.jpg?name=orig"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
other => panic!("expected illustration, got {other:?}"),
|
other => panic!("expected illustration, got {other:?}"),
|
||||||
}
|
}
|
||||||
match &fetched.media[1] {
|
match &fetched.media[1] {
|
||||||
Media::Video { url, thumbnail_url, .. } => {
|
Media::Video {
|
||||||
|
url, thumbnail_url, ..
|
||||||
|
} => {
|
||||||
assert_eq!(url, "https://video.twimg.com/v.mp4");
|
assert_eq!(url, "https://video.twimg.com/v.mp4");
|
||||||
assert_eq!(thumbnail_url, "https://pbs.twimg.com/thumb.jpg");
|
assert_eq!(thumbnail_url, "https://pbs.twimg.com/thumb.jpg");
|
||||||
}
|
}
|
||||||
other => panic!("expected video, got {other:?}"),
|
other => panic!("expected video, got {other:?}"),
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
fetched
|
fetched.caption.contains(
|
||||||
.caption
|
"<a href=\"https://x.com/author_handle\">Display Name</a>: a & b <c>"
|
||||||
.contains("<a href=\"https://x.com/author_handle\">Display Name</a>: a & b <c>"),
|
),
|
||||||
"caption: {}",
|
"caption: {}",
|
||||||
fetched.caption
|
fetched.caption
|
||||||
);
|
);
|
||||||
@@ -411,13 +410,12 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_strips_trailing_media_short_link() {
|
fn syndication_text_strips_trailing_media_short_link() {
|
||||||
// Real syndication shape: the media short link sits after the visible
|
// Real syndication shape: the appended media short link sits after the
|
||||||
// text, and display_text_range marks where it begins.
|
// visible text; the unmapped t.co link is stripped by content.
|
||||||
let raw = serde_json::json!({
|
let raw = serde_json::json!({
|
||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": "hello world https://t.co/abc123",
|
"text": "hello world https://t.co/abc123",
|
||||||
"display_text_range": [0, 11],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"mediaDetails": []
|
"mediaDetails": []
|
||||||
});
|
});
|
||||||
@@ -427,8 +425,31 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_strips_trailing_short_link_without_range() {
|
fn syndication_text_strips_trailing_link_regardless_of_index_units() {
|
||||||
// No display_text_range: the regex fallback removes the trailing link.
|
// Real tweet 2084567054481571919: the visible text is 30 code points
|
||||||
|
// but 41 UTF-16 units, and the two endpoints historically reported
|
||||||
|
// display_text_range in different units (UTF-16 on syndication, code
|
||||||
|
// points on GraphQL). The FxEmbed-style content-based strip ignores
|
||||||
|
// the range entirely, so the appended media link is removed for any
|
||||||
|
// response shape.
|
||||||
|
let text = "妄想𝑨𝒅𝒅𝒊𝒄𝒕𝒊𝒐𝒏…🩷💚❤️\n#ゼンゼロ #zzzero https://t.co/XnIi83EkEB";
|
||||||
|
let visible = "妄想𝑨𝒅𝒅𝒊𝒄𝒕𝒊𝒐𝒏…🩷💚❤️\n#ゼンゼロ #zzzero";
|
||||||
|
let raw = serde_json::json!({
|
||||||
|
"__typename": "Tweet",
|
||||||
|
"id_str": "2084567054481571919",
|
||||||
|
"text": text,
|
||||||
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
|
"mediaDetails": []
|
||||||
|
});
|
||||||
|
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||||
|
assert_eq!(tweet.text, visible, "left a partial link");
|
||||||
|
assert!(!tweet.caption().contains("t.co"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn syndication_text_strips_trailing_short_link_without_entities() {
|
||||||
|
// No URL entities at all: the leftover t.co link is stripped by the
|
||||||
|
// content regex.
|
||||||
let raw = serde_json::json!({
|
let raw = serde_json::json!({
|
||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
@@ -449,7 +470,6 @@ mod tests {
|
|||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": "Test Tweet with @mentionThis $twtr https://t.co/RzmrQ6wAzD #hashtag https://t.co/9r69akA484",
|
"text": "Test Tweet with @mentionThis $twtr https://t.co/RzmrQ6wAzD #hashtag https://t.co/9r69akA484",
|
||||||
"display_text_range": [0, 67],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"entities": {
|
"entities": {
|
||||||
"urls": [{
|
"urls": [{
|
||||||
@@ -469,25 +489,47 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_keeps_unmapped_short_links() {
|
fn syndication_text_strips_unmapped_short_links() {
|
||||||
// No entity mapping for the embedded link: it stays as-is. Only the
|
// FxEmbed parity: short links without an entity mapping (appended
|
||||||
// trailing media link is stripped.
|
// media link, embedded unmapped links) are stripped, not kept.
|
||||||
let raw = serde_json::json!({
|
let raw = serde_json::json!({
|
||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": "check https://t.co/abc123 #tag https://t.co/def456",
|
"text": "check https://t.co/abc123 #tag https://t.co/def456",
|
||||||
"display_text_range": [0, 30],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"mediaDetails": []
|
"mediaDetails": []
|
||||||
});
|
});
|
||||||
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||||
assert_eq!(tweet.text, "check https://t.co/abc123 #tag");
|
assert_eq!(tweet.text, "check #tag");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_utf16_display_range_keeps_multibyte() {
|
fn syndication_text_drops_internal_web_status_links() {
|
||||||
// display_text_range is in UTF-16 units; a Japanese text must not be
|
// FxEmbed parity: a mapped link expanding to an internal
|
||||||
// sliced by UTF-8 bytes.
|
// x.com/i/web/status/... page (reply/quote plumbing) is removed
|
||||||
|
// instead of being shown.
|
||||||
|
let raw = serde_json::json!({
|
||||||
|
"__typename": "Tweet",
|
||||||
|
"id_str": "1",
|
||||||
|
"text": "see https://t.co/xyz1234567 for context",
|
||||||
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
|
"entities": {
|
||||||
|
"urls": [{
|
||||||
|
"url": "https://t.co/xyz1234567",
|
||||||
|
"expanded_url": "https://x.com/i/web/status/9876543210",
|
||||||
|
"display_url": "x.com/i/web/status/9876543210"
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
"mediaDetails": []
|
||||||
|
});
|
||||||
|
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||||
|
assert_eq!(tweet.text, "see for context");
|
||||||
|
assert!(!tweet.caption().contains("t.co"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn syndication_text_keeps_multibyte_text() {
|
||||||
|
// Text-only tweet: no short links, the multibyte text is untouched.
|
||||||
let text = "コミティア落ちたので、明日は行きません。🙏ごめんなさい";
|
let text = "コミティア落ちたので、明日は行きません。🙏ごめんなさい";
|
||||||
let units: Vec<u16> = text.encode_utf16().collect();
|
let units: Vec<u16> = text.encode_utf16().collect();
|
||||||
assert_eq!(units.len(), 28);
|
assert_eq!(units.len(), 28);
|
||||||
@@ -495,7 +537,6 @@ mod tests {
|
|||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": text,
|
"text": text,
|
||||||
"display_text_range": [0, 28],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"mediaDetails": []
|
"mediaDetails": []
|
||||||
});
|
});
|
||||||
@@ -532,21 +573,27 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[ignore = "live network: requires outbound HTTPS to cdn.syndication.twimg.com"]
|
||||||
async fn live_fetch_with_photos() {
|
async fn live_fetch_with_photos() {
|
||||||
let fetched = fetch("861627479294746624").await.unwrap();
|
let fetched = fetch("861627479294746624").await.unwrap();
|
||||||
assert_eq!(fetched.media.len(), 4);
|
assert_eq!(fetched.media.len(), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[ignore = "live network: requires outbound HTTPS to cdn.syndication.twimg.com"]
|
||||||
async fn live_fetch_text_only() {
|
async fn live_fetch_text_only() {
|
||||||
let fetched = fetch("1992471125734142256").await.unwrap();
|
let fetched = fetch("1992471125734142256").await.unwrap();
|
||||||
assert!(fetched.media.is_empty());
|
assert!(fetched.media.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[ignore = "live network: requires outbound HTTPS to cdn.syndication.twimg.com"]
|
||||||
async fn live_fetch_deleted_tweet_is_not_found() {
|
async fn live_fetch_deleted_tweet_is_not_found() {
|
||||||
// Deleted tweet: the syndication endpoint answers with errors.
|
// Deleted tweet: the syndication endpoint answers with errors.
|
||||||
let result = fetch("0").await;
|
let result = fetch("0").await;
|
||||||
assert!(matches!(result, Err(FetchError::NotFound)), "got {result:?}");
|
assert!(
|
||||||
|
matches!(result, Err(FetchError::NotFound)),
|
||||||
|
"got {result:?}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,10 +9,6 @@ pub struct SyndicationTweet {
|
|||||||
pub user: SyndicationUser,
|
pub user: SyndicationUser,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub possibly_sensitive: Option<bool>,
|
pub possibly_sensitive: Option<bool>,
|
||||||
/// Visible-text span; the raw `text` field has the appended media short
|
|
||||||
/// link after it. Indices are Unicode code points (not UTF-16 units).
|
|
||||||
#[serde(default, rename = "display_text_range")]
|
|
||||||
pub display_text_range: Option<[usize; 2]>,
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub entities: SyndicationEntities,
|
pub entities: SyndicationEntities,
|
||||||
#[serde(default, rename = "mediaDetails")]
|
#[serde(default, rename = "mediaDetails")]
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "xmedia-bot"
|
name = "xmedia-bot"
|
||||||
version = "1.0.6"
|
version = "1.2.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
teloxide = { version = "0.17", features = ["webhooks-axum", "macros"] }
|
teloxide = { version = "0.17", default-features = false, features = ["webhooks-axum", "macros", "rustls", "ctrlc_handler"] }
|
||||||
tokio = { version = "1.40", features = ["rt-multi-thread", "macros"] }
|
tokio = { version = "1.40", features = ["rt-multi-thread", "macros"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
@@ -12,10 +12,14 @@ log = "0.4"
|
|||||||
pretty_env_logger = "0.5"
|
pretty_env_logger = "0.5"
|
||||||
dotenv = "0.15"
|
dotenv = "0.15"
|
||||||
url = "2.5.2"
|
url = "2.5.2"
|
||||||
regex = "1.12"
|
|
||||||
html-escape = "0.2"
|
html-escape = "0.2"
|
||||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
|
bytes = "1"
|
||||||
|
png = "0.18"
|
||||||
|
zune-jpeg = "0.5"
|
||||||
|
fast_image_resize = "6"
|
||||||
|
jpeg-encoder = "0.7"
|
||||||
x-media = { path = "../x-media" }
|
x-media = { path = "../x-media" }
|
||||||
|
|||||||
@@ -23,37 +23,67 @@ pub struct Config {
|
|||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
pub fn load() -> Config {
|
pub fn load() -> Config {
|
||||||
let admin_ids = env::var("BOT_ADMIN")
|
// Fail-fast helpers: a misspelled value must not silently fall back
|
||||||
.ok()
|
// to a default and run with different behavior than the operator
|
||||||
.map(|s| {
|
// intended — log a loud warning naming the variable instead.
|
||||||
s.split(',')
|
fn parse_u64(name: &str, default: u64) -> u64 {
|
||||||
.filter_map(|part| part.trim().parse::<i64>().ok())
|
match env::var(name) {
|
||||||
|
Ok(v) => v.parse::<u64>().unwrap_or_else(|_| {
|
||||||
|
log::warn!("invalid {name}={v:?}; using default {default}");
|
||||||
|
default
|
||||||
|
}),
|
||||||
|
Err(_) => default,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let admin_ids = match env::var("BOT_ADMIN") {
|
||||||
|
Ok(s) => {
|
||||||
|
let (ids, bad): (Vec<_>, Vec<_>) = s
|
||||||
|
.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|part| !part.is_empty())
|
||||||
|
.partition(|part| part.parse::<i64>().is_ok());
|
||||||
|
if !bad.is_empty() {
|
||||||
|
log::warn!("BOT_ADMIN: ignoring non-numeric ids: {bad:?}");
|
||||||
|
}
|
||||||
|
ids.into_iter()
|
||||||
|
.filter_map(|p| p.parse::<i64>().ok())
|
||||||
.collect()
|
.collect()
|
||||||
})
|
}
|
||||||
.unwrap_or_default();
|
Err(_) => Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
let edit_message_ttl = env::var("EDIT_MESSAGE_TTL_SECONDS")
|
let edit_message_ttl =
|
||||||
.ok()
|
Duration::from_secs(parse_u64("EDIT_MESSAGE_TTL_SECONDS", 24 * 3600));
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
let link_cache_ttl =
|
||||||
.map(Duration::from_secs)
|
Duration::from_secs(parse_u64("LINK_CACHE_TTL_SECONDS", 7 * 24 * 3600));
|
||||||
.unwrap_or(Duration::from_secs(86400));
|
|
||||||
|
|
||||||
let link_cache_ttl = env::var("LINK_CACHE_TTL_SECONDS")
|
|
||||||
.ok()
|
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
|
||||||
.map(Duration::from_secs)
|
|
||||||
.unwrap_or(Duration::from_secs(7 * 24 * 3600));
|
|
||||||
|
|
||||||
let webhook_enabled = env::var("WEBHOOK")
|
let webhook_enabled = env::var("WEBHOOK")
|
||||||
.is_ok_and(|v| matches!(v.to_lowercase().as_str(), "true" | "yes" | "1"));
|
.is_ok_and(|v| matches!(v.to_lowercase().as_str(), "true" | "yes" | "1"));
|
||||||
let webhook_url = env::var("WEBHOOK_URL").ok().and_then(|s| s.parse().ok());
|
// The webhook settings are consumed by `.expect()` in main when
|
||||||
let webhook_listen = env::var("WEBHOOK_LISTEN").ok().and_then(|s| s.parse().ok());
|
// WEBHOOK=true, so an unparseable value fails fast at startup with a
|
||||||
let webhook_port = env::var("WEBHOOK_PORT").ok().and_then(|s| s.parse().ok());
|
// clear message; still log here for the WEBHOOK=false case.
|
||||||
|
let webhook_url = env::var("WEBHOOK_URL").ok().and_then(|s| {
|
||||||
|
s.parse::<url::Url>().ok().or_else(|| {
|
||||||
|
log::warn!("invalid WEBHOOK_URL={s:?}");
|
||||||
|
None
|
||||||
|
})
|
||||||
|
});
|
||||||
|
let webhook_listen = env::var("WEBHOOK_LISTEN").ok().and_then(|s| {
|
||||||
|
s.parse::<IpAddr>().ok().or_else(|| {
|
||||||
|
log::warn!("invalid WEBHOOK_LISTEN={s:?}");
|
||||||
|
None
|
||||||
|
})
|
||||||
|
});
|
||||||
|
let webhook_port = env::var("WEBHOOK_PORT").ok().and_then(|s| {
|
||||||
|
s.parse::<u16>().ok().or_else(|| {
|
||||||
|
log::warn!("invalid WEBHOOK_PORT={s:?}");
|
||||||
|
None
|
||||||
|
})
|
||||||
|
});
|
||||||
// Empty strings count as unset (e.g. `-e WEBHOOK_CERT=` to disable a
|
// Empty strings count as unset (e.g. `-e WEBHOOK_CERT=` to disable a
|
||||||
// value that would otherwise come from `.env`).
|
// value that would otherwise come from `.env`).
|
||||||
let webhook_cert = env::var("WEBHOOK_CERT")
|
let webhook_cert = env::var("WEBHOOK_CERT").ok().filter(|s| !s.is_empty());
|
||||||
.ok()
|
|
||||||
.filter(|s| !s.is_empty());
|
|
||||||
let webhook_secret_token = env::var("WEBHOOK_SECRET_TOKEN")
|
let webhook_secret_token = env::var("WEBHOOK_SECRET_TOKEN")
|
||||||
.ok()
|
.ok()
|
||||||
.filter(|s| !s.is_empty());
|
.filter(|s| !s.is_empty());
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
//! Shared SQLite plumbing for the three tables in `data/task_queue.db`
|
||||||
|
//! (`tasks` in queue.rs, `chat_state` in state.rs, `link_cache` in
|
||||||
|
//! link_cache.rs).
|
||||||
|
//!
|
||||||
|
//! All I/O runs inside `spawn_blocking` via [`DbPool::with_conn`] — rusqlite
|
||||||
|
//! connections are not Send-friendly to hold across an await point, and
|
||||||
|
//! blocking the async executor stalls every handler. Connections are reused
|
||||||
|
//! through a small per-store pool instead of opening a fresh connection per
|
||||||
|
//! operation: WAL lets readers run alongside writer leases, and the pool's
|
||||||
|
//! semaphore bounds how many DB operations run concurrently, giving natural
|
||||||
|
//! backpressure on hot paths (every message / URL / callback touches
|
||||||
|
//! chat_state or the link cache).
|
||||||
|
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
/// Upper bound on pooled (reused) connections and on concurrent DB
|
||||||
|
/// operations per store. Small on purpose: the queue's `BEGIN IMMEDIATE`
|
||||||
|
/// leases serialize writes anyway, and WAL readers rarely need more.
|
||||||
|
const POOL_SIZE: usize = 4;
|
||||||
|
|
||||||
|
/// A tiny connection pool for one SQLite file. Connections are checked out
|
||||||
|
/// on a blocking thread and returned afterwards; `acquire` opens a new
|
||||||
|
/// connection only when the idle list is empty, so the steady-state cost of
|
||||||
|
/// an operation is a list pop instead of a fresh open (+ busy timeout + WAL
|
||||||
|
/// pragma). The semaphore caps the number of concurrent operations, so a
|
||||||
|
/// burst of handlers queues up instead of opening unbounded connections.
|
||||||
|
pub struct DbPool {
|
||||||
|
// Arc so [`DbPool::with_conn`] can hand an owned handle to
|
||||||
|
// `spawn_blocking` without borrowing across the await point.
|
||||||
|
inner: Arc<PoolInner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PoolInner {
|
||||||
|
path: String,
|
||||||
|
permits: tokio::sync::Semaphore,
|
||||||
|
idle: Mutex<Vec<Connection>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DbPool {
|
||||||
|
pub fn new(path: &str) -> Self {
|
||||||
|
DbPool {
|
||||||
|
inner: Arc::new(PoolInner {
|
||||||
|
path: path.to_string(),
|
||||||
|
permits: tokio::sync::Semaphore::new(POOL_SIZE),
|
||||||
|
idle: Mutex::new(Vec::new()),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs `f` against a pooled connection on a blocking thread, returning
|
||||||
|
/// the closure's result. Owns the semaphore + `spawn_blocking` +
|
||||||
|
/// `expect` ceremony shared by every table access; the caller maps
|
||||||
|
/// errors to its own log line.
|
||||||
|
pub async fn with_conn<T, F>(&self, f: F) -> rusqlite::Result<T>
|
||||||
|
where
|
||||||
|
T: Send + 'static,
|
||||||
|
F: FnOnce(&mut Connection) -> rusqlite::Result<T> + Send + 'static,
|
||||||
|
{
|
||||||
|
let _permit = self
|
||||||
|
.inner
|
||||||
|
.permits
|
||||||
|
.acquire()
|
||||||
|
.await
|
||||||
|
.expect("db pool semaphore closed");
|
||||||
|
let inner = Arc::clone(&self.inner);
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let mut conn = inner.acquire()?;
|
||||||
|
let result = f(&mut conn);
|
||||||
|
inner.release(conn);
|
||||||
|
result
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("db worker panicked")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The database file this pool serves (used by tests that need a raw
|
||||||
|
/// connection, e.g. to seed rows directly).
|
||||||
|
#[cfg(test)]
|
||||||
|
pub fn path(&self) -> &str {
|
||||||
|
&self.inner.path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PoolInner {
|
||||||
|
/// Reuses an idle connection or opens a fresh one.
|
||||||
|
fn acquire(&self) -> rusqlite::Result<Connection> {
|
||||||
|
if let Some(conn) = self.idle.lock().pop() {
|
||||||
|
return Ok(conn);
|
||||||
|
}
|
||||||
|
open_db(&self.path)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a connection to the pool (dropped when the pool is full).
|
||||||
|
fn release(&self, conn: Connection) {
|
||||||
|
let mut idle = self.idle.lock();
|
||||||
|
if idle.len() < POOL_SIZE {
|
||||||
|
idle.push(conn);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opens the shared DB with a busy timeout.
|
||||||
|
pub fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
||||||
|
let conn = Connection::open(path)?;
|
||||||
|
conn.busy_timeout(Duration::from_secs(5))?;
|
||||||
|
// WAL lets readers run alongside writer leases instead of blocking on
|
||||||
|
// the rollback journal; the mode persists in the DB header, so the
|
||||||
|
// idempotent pragma here and in ensure_schema only needs to win once.
|
||||||
|
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unix timestamp in fractional seconds. Shared by the queue, chat store and
|
||||||
|
/// link cache (previously four private copies).
|
||||||
|
pub fn now_f64() -> f64 {
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs_f64())
|
||||||
|
.unwrap_or(0.0)
|
||||||
|
}
|
||||||
@@ -1,57 +1,105 @@
|
|||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
|
use crate::db::now_f64;
|
||||||
use crate::link_cache::{CachedMediaKind, CachedPost, LinkCache};
|
use crate::link_cache::{CachedMediaKind, CachedPost, LinkCache};
|
||||||
use crate::queue::PersistentTaskQueue;
|
use crate::queue::PersistentTaskQueue;
|
||||||
use crate::send::{self, MediaItemPayload, Task};
|
use crate::send::{self, MediaItemPayload, Task};
|
||||||
use crate::state::{ChatData, ChatStore, unix_now};
|
use crate::state::{ChatData, ChatStore, unix_now};
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
use teloxide::RequestError;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use tokio::sync::Semaphore;
|
|
||||||
use teloxide::types::{
|
use teloxide::types::{
|
||||||
CallbackQuery, ChatAction, ChatId, ChatKind, InlineQuery, InlineQueryResult,
|
CallbackQuery, ChatAction, ChatId, ChatKind, InlineQuery, InlineQueryResult,
|
||||||
InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, Message,
|
InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, Message,
|
||||||
MessageEntityKind, MessageId, ParseMode, Recipient, ReplyParameters,
|
MessageEntityKind, MessageId, ParseMode, Recipient, ReplyParameters,
|
||||||
};
|
};
|
||||||
use teloxide::utils::command::BotCommands;
|
use teloxide::utils::command::BotCommands;
|
||||||
use teloxide::RequestError;
|
|
||||||
use x_media::media::Media;
|
use x_media::media::Media;
|
||||||
|
|
||||||
pub static CHAT_STORE: LazyLock<ChatStore> = LazyLock::new(|| {
|
/// One URL job: bot handle + the message + the extracted URL.
|
||||||
ChatStore::open("data/task_queue.db").expect("failed to open chat store")
|
type UrlJob = (Bot, Message, String);
|
||||||
});
|
/// Bounded channel of URL jobs drained by [`start_url_workers`]. The bound
|
||||||
|
/// caps both queued memory and shutdown backlog; a full channel applies
|
||||||
|
/// backpressure to the per-chat handler instead of spawning unbounded tasks.
|
||||||
|
static URL_JOBS: LazyLock<parking_lot::Mutex<Option<tokio::sync::mpsc::Sender<UrlJob>>>> =
|
||||||
|
LazyLock::new(|| parking_lot::Mutex::new(None));
|
||||||
|
/// Set by main's shutdown sequence; workers stop pulling new jobs.
|
||||||
|
static URL_STOP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||||
|
|
||||||
|
/// Worker count draining URL jobs; keeps the old 8-permit concurrency cap
|
||||||
|
/// while bounding how many jobs can be queued at all.
|
||||||
|
const URL_WORKERS: usize = 8;
|
||||||
|
|
||||||
|
/// Starts the URL job workers (called once from main after the queue starts).
|
||||||
|
/// teloxide dispatches updates to a per-chat worker that handles them
|
||||||
|
/// sequentially, so a batch-forward of many messages would otherwise be
|
||||||
|
/// processed one at a time (fetch + send each, roughly a second per
|
||||||
|
/// message); the workers add throughput, and FIFO order preserves per-message
|
||||||
|
/// URL order.
|
||||||
|
pub async fn start_url_workers() {
|
||||||
|
let (tx, rx) = tokio::sync::mpsc::channel::<UrlJob>(256);
|
||||||
|
*URL_JOBS.lock() = Some(tx);
|
||||||
|
let rx = std::sync::Arc::new(tokio::sync::Mutex::new(rx));
|
||||||
|
for _ in 0..URL_WORKERS {
|
||||||
|
let rx = std::sync::Arc::clone(&rx);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
while !URL_STOP.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
|
let job = rx.lock().await.recv().await;
|
||||||
|
match job {
|
||||||
|
Some((bot, message, url)) => url_media(bot, &message, &url).await,
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stops URL workers (drains up to the 256 queued jobs, then exits).
|
||||||
|
pub fn stop_url_workers() {
|
||||||
|
URL_STOP.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub static CHAT_STORE: LazyLock<ChatStore> =
|
||||||
|
LazyLock::new(|| ChatStore::open("data/task_queue.db").expect("failed to open chat store"));
|
||||||
pub static TASK_QUEUE: LazyLock<PersistentTaskQueue> =
|
pub static TASK_QUEUE: LazyLock<PersistentTaskQueue> =
|
||||||
LazyLock::new(|| PersistentTaskQueue::new("data/task_queue.db"));
|
LazyLock::new(|| PersistentTaskQueue::new("data/task_queue.db"));
|
||||||
pub static LINK_CACHE: LazyLock<LinkCache> =
|
pub static LINK_CACHE: LazyLock<LinkCache> =
|
||||||
LazyLock::new(|| LinkCache::open("data/task_queue.db"));
|
LazyLock::new(|| LinkCache::open("data/task_queue.db"));
|
||||||
pub static CONFIG: LazyLock<Config> = LazyLock::new(Config::load);
|
pub static CONFIG: LazyLock<Config> = LazyLock::new(Config::load);
|
||||||
|
|
||||||
/// Cap on concurrent per-URL processing. teloxide dispatches updates to a
|
|
||||||
/// per-chat worker that handles them sequentially, so a batch-forward of many
|
|
||||||
/// messages would otherwise be processed one at a time (fetch + send each,
|
|
||||||
/// roughly a second per message). Moving the work into spawned tasks trades
|
|
||||||
/// per-chat reply ordering for throughput; the semaphore bounds how many run
|
|
||||||
/// at once so a big burst cannot hammer Telegram's rate limits.
|
|
||||||
static URL_TASKS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(8));
|
|
||||||
|
|
||||||
#[derive(BotCommands, Clone)]
|
#[derive(BotCommands, Clone)]
|
||||||
#[command(rename_rule = "snake_case", description = "")]
|
#[command(
|
||||||
|
rename_rule = "snake_case",
|
||||||
|
description = "Turn X/Pixiv/Bluesky links into media messages"
|
||||||
|
)]
|
||||||
enum Command {
|
enum Command {
|
||||||
#[command(description = "")]
|
#[command(description = "Get started")]
|
||||||
Start,
|
Start,
|
||||||
#[command(description = "")]
|
#[command(description = "Show command help")]
|
||||||
Help,
|
Help,
|
||||||
#[command(description = "", parse_with = "split")]
|
#[command(
|
||||||
|
description = "Set forward channel (@channel or ID)",
|
||||||
|
parse_with = "split"
|
||||||
|
)]
|
||||||
SetForwardChannel(String),
|
SetForwardChannel(String),
|
||||||
#[command(description = "")]
|
#[command(description = "Remove forward channel")]
|
||||||
RemoveForwardChannel,
|
RemoveForwardChannel,
|
||||||
#[command(description = "")]
|
#[command(description = "Toggle edit-before-forward")]
|
||||||
EditBeforeForward,
|
EditBeforeForward,
|
||||||
#[command(description = "", parse_with = "split")]
|
#[command(
|
||||||
|
description = "Reply with [] to save as template",
|
||||||
|
parse_with = "split"
|
||||||
|
)]
|
||||||
SetTemplate(String),
|
SetTemplate(String),
|
||||||
#[command(description = "")]
|
#[command(description = "Show chat state (debug)")]
|
||||||
BotDict,
|
BotDict,
|
||||||
#[command(description = "", parse_with = "split")]
|
#[command(description = "Set site caption format", parse_with = "split")]
|
||||||
SetFormat(String),
|
SetFormat(String),
|
||||||
|
#[command(
|
||||||
|
description = "Clear link cache (admin; optional URL, else all)",
|
||||||
|
parse_with = "split"
|
||||||
|
)]
|
||||||
|
ClearCache(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reply<T>(bot: Bot, message: Message, text: T) -> Result<Message, RequestError>
|
async fn reply<T>(bot: Bot, message: Message, text: T) -> Result<Message, RequestError>
|
||||||
@@ -63,13 +111,6 @@ where
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
fn now_f64() -> f64 {
|
|
||||||
std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_secs_f64())
|
|
||||||
.unwrap_or(0.0)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Extracts URL and text-link entities (text + caption), deduped in order.
|
/// Extracts URL and text-link entities (text + caption), deduped in order.
|
||||||
pub fn extract_urls(message: &Message) -> Vec<String> {
|
pub fn extract_urls(message: &Message) -> Vec<String> {
|
||||||
let mut urls = Vec::new();
|
let mut urls = Vec::new();
|
||||||
@@ -88,7 +129,10 @@ pub fn extract_urls(message: &Message) -> Vec<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
let mut seen = HashSet::new();
|
let mut seen = HashSet::new();
|
||||||
urls.retain(|url| seen.insert(url.clone()));
|
// Dedup by the normalized post id so variant URLs of the same post
|
||||||
|
// (/status/1 vs /status/1/photo/1) are sent once; unsupported URLs fall
|
||||||
|
// back to exact-string dedup.
|
||||||
|
urls.retain(|url| seen.insert(x_media::site::cache_key(url).unwrap_or_else(|| url.clone())));
|
||||||
urls
|
urls
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,7 +155,7 @@ async fn edit_message_handler(bot: &Bot, message: &Message) -> bool {
|
|||||||
};
|
};
|
||||||
let link = format!(
|
let link = format!(
|
||||||
"<a href=\"{0}\">{1}</a>",
|
"<a href=\"{0}\">{1}</a>",
|
||||||
edit.url,
|
html_escape::encode_double_quoted_attribute(&edit.url),
|
||||||
html_escape::encode_text(text)
|
html_escape::encode_text(text)
|
||||||
);
|
);
|
||||||
let new_text = if edit.template.is_empty() {
|
let new_text = if edit.template.is_empty() {
|
||||||
@@ -177,19 +221,31 @@ async fn set_forward_channel_handler(
|
|||||||
return Err(SetForwardChannelError::NotChannel);
|
return Err(SetForwardChannelError::NotChannel);
|
||||||
}
|
}
|
||||||
let channel_id = chat.id.0;
|
let channel_id = chat.id.0;
|
||||||
|
// The sender must be a channel administrator. Compare against the
|
||||||
|
// sender's user id, NOT the chat id (they only coincide in private
|
||||||
|
// chats, so the old check broke group usage).
|
||||||
|
let Some(sender) = message.from.as_ref() else {
|
||||||
|
return Err(SetForwardChannelError::NotAdmin);
|
||||||
|
};
|
||||||
match bot.get_chat_administrators(channel.clone()).await {
|
match bot.get_chat_administrators(channel.clone()).await {
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("Failed to get channel administrators {}: {}", channel, e);
|
log::error!("Failed to get channel administrators {}: {}", channel, e);
|
||||||
return Err(SetForwardChannelError::NotBotAdmin(e));
|
return Err(SetForwardChannelError::NotBotAdmin(e));
|
||||||
}
|
}
|
||||||
Ok(admins) => {
|
Ok(admins) => {
|
||||||
if !admins.iter().any(|admin| admin.user.id == message.chat.id) {
|
if !admins.iter().any(|admin| admin.user.id == sender.id) {
|
||||||
return Err(SetForwardChannelError::NotAdmin);
|
return Err(SetForwardChannelError::NotAdmin);
|
||||||
}
|
}
|
||||||
let bot_id = bot.get_me().await.expect("Failed get bot id").user.id;
|
// The bot itself must be an admin that can post; a missing
|
||||||
if let Some(bot_admin) = admins.iter().find(|admin| admin.user.id == bot_id)
|
// bot entry must not pass silently (copy would fail later).
|
||||||
&& !bot_admin.can_post_messages()
|
let bot_id = match bot.get_me().await {
|
||||||
{
|
Ok(me) => me.user.id,
|
||||||
|
Err(e) => return Err(SetForwardChannelError::NotBotAdmin(e)),
|
||||||
|
};
|
||||||
|
let bot_ok = admins
|
||||||
|
.iter()
|
||||||
|
.any(|admin| admin.user.id == bot_id && admin.can_post_messages());
|
||||||
|
if !bot_ok {
|
||||||
return Err(SetForwardChannelError::NotBotCanPost);
|
return Err(SetForwardChannelError::NotBotCanPost);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -197,7 +253,11 @@ async fn set_forward_channel_handler(
|
|||||||
Ok(channel_id)
|
Ok(channel_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Result<(), RequestError> {
|
async fn execute_command(
|
||||||
|
bot: &Bot,
|
||||||
|
message: &Message,
|
||||||
|
command: Command,
|
||||||
|
) -> Result<(), RequestError> {
|
||||||
match command {
|
match command {
|
||||||
Command::Start => {
|
Command::Start => {
|
||||||
bot.send_message(message.chat.id, "Hello!").await?;
|
bot.send_message(message.chat.id, "Hello!").await?;
|
||||||
@@ -209,13 +269,16 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
Command::SetForwardChannel(channel) => {
|
Command::SetForwardChannel(channel) => {
|
||||||
let result = match set_forward_channel_handler(bot, message, channel).await {
|
let result = match set_forward_channel_handler(bot, message, channel).await {
|
||||||
Ok(channel_id) => {
|
Ok(channel_id) => {
|
||||||
let mut chat_data = CHAT_STORE.get(message.chat.id.0).await;
|
CHAT_STORE
|
||||||
chat_data.forward_channel_id = Some(channel_id);
|
.update(message.chat.id.0, |data| {
|
||||||
CHAT_STORE.set(message.chat.id.0, &chat_data).await;
|
data.forward_channel_id = Some(channel_id);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
"Add successfully.".to_string()
|
"Add successfully.".to_string()
|
||||||
}
|
}
|
||||||
Err(SetForwardChannelError::EmptyParameter) => {
|
Err(SetForwardChannelError::EmptyParameter) => {
|
||||||
"Receive empty parameter.\nYou should enter a channel id or username".to_string()
|
"Receive empty parameter.\nYou should enter a channel id or username"
|
||||||
|
.to_string()
|
||||||
}
|
}
|
||||||
Err(SetForwardChannelError::NotChannel) => {
|
Err(SetForwardChannelError::NotChannel) => {
|
||||||
"Given id / username is not a channel".to_string()
|
"Given id / username is not a channel".to_string()
|
||||||
@@ -234,31 +297,34 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
}
|
}
|
||||||
Command::RemoveForwardChannel => {
|
Command::RemoveForwardChannel => {
|
||||||
let chat_id = message.chat.id.0;
|
let chat_id = message.chat.id.0;
|
||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
let text = CHAT_STORE
|
||||||
let text = if chat_data.forward_channel_id.is_some() {
|
.update(chat_id, |data| {
|
||||||
chat_data.forward_channel_id = None;
|
if data.forward_channel_id.is_some() {
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
data.forward_channel_id = None;
|
||||||
"Remove successfully.".to_string()
|
"Remove successfully.".to_string()
|
||||||
} else {
|
} else {
|
||||||
"No channel to remove.".to_string()
|
"No channel to remove.".to_string()
|
||||||
};
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
reply(bot.clone(), message.clone(), text).await?;
|
reply(bot.clone(), message.clone(), text).await?;
|
||||||
}
|
}
|
||||||
Command::EditBeforeForward => {
|
Command::EditBeforeForward => {
|
||||||
let chat_id = message.chat.id.0;
|
let chat_id = message.chat.id.0;
|
||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
let text = CHAT_STORE
|
||||||
let text = if chat_data.forward_channel_id.is_none() {
|
.update(chat_id, |data| {
|
||||||
|
if data.forward_channel_id.is_none() {
|
||||||
"Please enable forward channel first.".to_string()
|
"Please enable forward channel first.".to_string()
|
||||||
} else if chat_data.edit_before_forward {
|
} else if data.edit_before_forward {
|
||||||
chat_data.edit_before_forward = false;
|
data.edit_before_forward = false;
|
||||||
chat_data.edit_message.clear();
|
data.edit_message.clear();
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
|
||||||
"Disable edit before forward.".to_string()
|
"Disable edit before forward.".to_string()
|
||||||
} else {
|
} else {
|
||||||
chat_data.edit_before_forward = true;
|
data.edit_before_forward = true;
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
|
||||||
"Enable edit before forward.".to_string()
|
"Enable edit before forward.".to_string()
|
||||||
};
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
reply(bot.clone(), message.clone(), text).await?;
|
reply(bot.clone(), message.clone(), text).await?;
|
||||||
}
|
}
|
||||||
Command::SetTemplate(name) => {
|
Command::SetTemplate(name) => {
|
||||||
@@ -272,11 +338,14 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
} else if name.is_empty() {
|
} else if name.is_empty() {
|
||||||
"Please provide a name for the template.".to_string()
|
"Please provide a name for the template.".to_string()
|
||||||
} else {
|
} else {
|
||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
CHAT_STORE
|
||||||
chat_data
|
.update(chat_id, |data| {
|
||||||
.template
|
data.template.insert(
|
||||||
.insert(name, html_escape::encode_text(reply_text).into_owned());
|
name,
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
html_escape::encode_text(reply_text).into_owned(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
"Template set.".to_string()
|
"Template set.".to_string()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -292,7 +361,9 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
Command::SetFormat(arg) => {
|
Command::SetFormat(arg) => {
|
||||||
let chat_id = message.chat.id.0;
|
let chat_id = message.chat.id.0;
|
||||||
let (site, format) = match arg.split_once(char::is_whitespace) {
|
let (site, format) = match arg.split_once(char::is_whitespace) {
|
||||||
Some((site, format)) if !format.trim().is_empty() => (site.trim(), format.trim().to_string()),
|
Some((site, format)) if !format.trim().is_empty() => {
|
||||||
|
(site.trim(), format.trim().to_string())
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
reply(
|
reply(
|
||||||
bot.clone(),
|
bot.clone(),
|
||||||
@@ -312,12 +383,75 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
.await?;
|
.await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
CHAT_STORE
|
||||||
chat_data.message_format.insert(site.to_string(), format);
|
.update(chat_id, |data| {
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
data.message_format.insert(site.to_string(), format);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
reply(bot.clone(), message.clone(), "Format set.").await?;
|
reply(bot.clone(), message.clone(), "Format set.").await?;
|
||||||
}
|
}
|
||||||
|
Command::ClearCache(arg) => {
|
||||||
|
let sender_id = message
|
||||||
|
.from
|
||||||
|
.as_ref()
|
||||||
|
.map(|user| user.id.0 as i64)
|
||||||
|
.unwrap_or(-1);
|
||||||
|
if !CONFIG.admin_ids.contains(&sender_id) {
|
||||||
|
reply(bot.clone(), message.clone(), "Admin only.").await?;
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
let arg = arg.trim();
|
||||||
|
if arg.is_empty() {
|
||||||
|
let removed = LINK_CACHE.clear(None).await;
|
||||||
|
log::info!("cache cleared by {sender_id}: {removed} entries");
|
||||||
|
reply(
|
||||||
|
bot.clone(),
|
||||||
|
message.clone(),
|
||||||
|
format!("Cleared {removed} cached entr{}.", plural(removed)),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
} else {
|
||||||
|
let key = match x_media::site::cache_key(arg) {
|
||||||
|
Some(key) => key,
|
||||||
|
None => {
|
||||||
|
reply(
|
||||||
|
bot.clone(),
|
||||||
|
message.clone(),
|
||||||
|
"Unrecognized link. Use a twitter/x, pixiv or bsky post URL.",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let removed = LINK_CACHE.clear(Some(&key)).await;
|
||||||
|
log::info!("cache entry cleared by {sender_id}: {key} ({removed} rows)");
|
||||||
|
reply(
|
||||||
|
bot.clone(),
|
||||||
|
message.clone(),
|
||||||
|
format!(
|
||||||
|
"Cleared cache for {arg} ({} entr{}).",
|
||||||
|
removed,
|
||||||
|
plural(removed)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `"y"` for one, `"ies"` for anything else — "1 entry" / "2 entries".
|
||||||
|
fn plural(n: usize) -> &'static str {
|
||||||
|
if n == 1 { "y" } else { "ies" }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Registers the bot's command list with Telegram so clients show it in the
|
||||||
|
/// `/` menu (Bot API `setMyCommands`).
|
||||||
|
pub async fn register_commands(bot: &Bot) -> Result<(), RequestError> {
|
||||||
|
let commands = Command::bot_commands();
|
||||||
|
bot.set_my_commands(commands.clone()).await?;
|
||||||
|
log::info!("registered {} commands", commands.len());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,8 +516,13 @@ async fn dispatch_send(bot: Bot, message: &Message, task: &Task, url: &str) {
|
|||||||
Ok(message_ids) => {
|
Ok(message_ids) => {
|
||||||
log::info!("sent {} message(s) for {url}", message_ids.len());
|
log::info!("sent {} message(s) for {url}", message_ids.len());
|
||||||
send::post_send_actions(&bot, task, message_ids).await;
|
send::post_send_actions(&bot, task, message_ids).await;
|
||||||
|
// The task settled: drop any keep-alive temp media.
|
||||||
|
send::release_keep_alive(task);
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable { delay_seconds, task }) => {
|
Err(send::SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
log::info!("send for {url} failed, queued for retry in {delay_seconds:.1}s");
|
log::info!("send for {url} failed, queued for retry in {delay_seconds:.1}s");
|
||||||
enqueue_retry(task, delay_seconds).await;
|
enqueue_retry(task, delay_seconds).await;
|
||||||
let _ = reply(bot, message.clone(), "Send failed. Task queued for retry.").await;
|
let _ = reply(bot, message.clone(), "Send failed. Task queued for retry.").await;
|
||||||
@@ -393,6 +532,7 @@ async fn dispatch_send(bot: Bot, message: &Message, task: &Task, url: &str) {
|
|||||||
task,
|
task,
|
||||||
}) => {
|
}) => {
|
||||||
send::invalidate_cache(&task).await;
|
send::invalidate_cache(&task).await;
|
||||||
|
send::release_keep_alive(&task);
|
||||||
log::error!("send for {url} failed permanently: {err_message}");
|
log::error!("send for {url} failed permanently: {err_message}");
|
||||||
let _ = reply(bot, message.clone(), format!("Send failed: {err_message}")).await;
|
let _ = reply(bot, message.clone(), format!("Send failed: {err_message}")).await;
|
||||||
}
|
}
|
||||||
@@ -429,7 +569,9 @@ fn build_send_task(
|
|||||||
chat_id,
|
chat_id,
|
||||||
reply_to_message_id: message.id.0 as i64,
|
reply_to_message_id: message.id.0 as i64,
|
||||||
caption,
|
caption,
|
||||||
media_batches: send::chunk_media_items(items),
|
// Photos first so a mixed photo+video group starts with a photo
|
||||||
|
// (Telegram's sendMediaGroup rule); order within each kind is kept.
|
||||||
|
media_batches: send::chunk_media_items(send::photos_first(items)),
|
||||||
batch_index: 0,
|
batch_index: 0,
|
||||||
sent_message_ids: vec![],
|
sent_message_ids: vec![],
|
||||||
source_url,
|
source_url,
|
||||||
@@ -444,7 +586,10 @@ fn build_send_task(
|
|||||||
|
|
||||||
async fn url_media(bot: Bot, message: &Message, url: &str) {
|
async fn url_media(bot: Bot, message: &Message, url: &str) {
|
||||||
let chat_id = message.chat.id.0;
|
let chat_id = message.chat.id.0;
|
||||||
if let Err(e) = bot.send_chat_action(ChatId(chat_id), ChatAction::Typing).await {
|
if let Err(e) = bot
|
||||||
|
.send_chat_action(ChatId(chat_id), ChatAction::Typing)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log::error!("send_chat_action failed: {e}");
|
log::error!("send_chat_action failed: {e}");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -463,7 +608,7 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let caption = if format.is_empty() {
|
let caption = if format.is_empty() {
|
||||||
cached.caption.clone()
|
x_media::site::truncate_caption(&cached.caption)
|
||||||
} else {
|
} else {
|
||||||
x_media::site::caption_from_fields(
|
x_media::site::caption_from_fields(
|
||||||
&format,
|
&format,
|
||||||
@@ -520,9 +665,14 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
// Retries exhausted: notify the user (Rust-only requirement 3).
|
// Retries exhausted: notify the user (Rust-only requirement 3).
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("fetch {url}: {e}");
|
log::error!("fetch {url}: {e}");
|
||||||
let _ = reply(bot, message.clone(), "Failed to fetch media from this link.").await;
|
let _ = reply(
|
||||||
|
bot,
|
||||||
|
message.clone(),
|
||||||
|
"Failed to fetch media from this link.",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(mut fetched)) => {
|
||||||
if fetched.media.is_empty() {
|
if fetched.media.is_empty() {
|
||||||
let _ = reply(
|
let _ = reply(
|
||||||
bot,
|
bot,
|
||||||
@@ -542,8 +692,9 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
let caption = fetched.caption_with(&format);
|
let caption = fetched.caption_with(&format);
|
||||||
// Raw render data for the link cache; the send fills in the
|
// Raw render data for the link cache; the send fills in the
|
||||||
// Telegram file ids and persists the entry.
|
// Telegram file ids and persists the entry.
|
||||||
let cache_data = fetched.render_fields().map(|(author, author_url, title, tags)| {
|
let cache_data = fetched
|
||||||
CachedPost {
|
.render_fields()
|
||||||
|
.map(|(author, author_url, title, tags)| CachedPost {
|
||||||
url: fetched.source_url.clone(),
|
url: fetched.source_url.clone(),
|
||||||
caption: fetched.caption.clone(),
|
caption: fetched.caption.clone(),
|
||||||
title: title.to_string(),
|
title: title.to_string(),
|
||||||
@@ -552,7 +703,6 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
tags: tags.to_string(),
|
tags: tags.to_string(),
|
||||||
sensitive: fetched.sensitive,
|
sensitive: fetched.sensitive,
|
||||||
media: vec![],
|
media: vec![],
|
||||||
}
|
|
||||||
});
|
});
|
||||||
let items: Vec<MediaItemPayload> = fetched
|
let items: Vec<MediaItemPayload> = fetched
|
||||||
.media
|
.media
|
||||||
@@ -567,6 +717,13 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
items,
|
items,
|
||||||
cache_data,
|
cache_data,
|
||||||
);
|
);
|
||||||
|
// Hand the keep-alive temp dir (ugoira / bsky remux MP4) to the
|
||||||
|
// retry registry: a queued retry runs after this function returns
|
||||||
|
// and the fetch's own TempDir is dropped, so without this the
|
||||||
|
// local file would be gone by the time the retry sends it.
|
||||||
|
if let Some(dir) = fetched.take_keep_alive() {
|
||||||
|
send::KEEP_ALIVE.lock().push(dir);
|
||||||
|
}
|
||||||
dispatch_send(bot, message, &task, url).await;
|
dispatch_send(bot, message, &task, url).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -581,9 +738,15 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
|||||||
.unwrap_or_else(|| "unknown".to_string());
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
let text_preview = message
|
let text_preview = message
|
||||||
.text()
|
.text()
|
||||||
.map(|t| if t.len() > 120 { &t[..120] } else { t })
|
.map(|t| {
|
||||||
|
let end = t.floor_char_boundary(120.min(t.len()));
|
||||||
|
&t[..end]
|
||||||
|
})
|
||||||
.unwrap_or("<no text>");
|
.unwrap_or("<no text>");
|
||||||
log::info!("message from {sender} in {} (private={is_private}): {text_preview}", message.chat.id);
|
log::info!(
|
||||||
|
"message from {sender} in {} (private={is_private}): {text_preview}",
|
||||||
|
message.chat.id
|
||||||
|
);
|
||||||
// URL/edit flows only run in private chats; commands run in any chat.
|
// URL/edit flows only run in private chats; commands run in any chat.
|
||||||
if is_private && edit_message_handler(&bot, &message).await {
|
if is_private && edit_message_handler(&bot, &message).await {
|
||||||
return respond(());
|
return respond(());
|
||||||
@@ -601,26 +764,104 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
|||||||
log::info!("extracted {} URL(s): {urls:?}", urls.len());
|
log::info!("extracted {} URL(s): {urls:?}", urls.len());
|
||||||
}
|
}
|
||||||
for url in urls {
|
for url in urls {
|
||||||
let bot = bot.clone();
|
// Clone out of the lock: the parking_lot guard is !Send and must
|
||||||
let message = message.clone();
|
// not be held across the await below.
|
||||||
tokio::spawn(async move {
|
let Some(tx) = URL_JOBS.lock().clone() else {
|
||||||
// Held for the whole task; the semaphore is never closed.
|
log::warn!("url workers not started; dropping link");
|
||||||
let _permit = URL_TASKS.acquire().await.expect("URL semaphore closed");
|
break;
|
||||||
url_media(bot, &message, &url).await;
|
};
|
||||||
});
|
let _ = tx.send((bot.clone(), message.clone(), url)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
respond(())
|
respond(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Debounce window for inline queries: Telegram fires an inline query on
|
||||||
|
/// every keystroke, and each prefix of a pasted URL (e.g. `.../status/12`,
|
||||||
|
/// `.../status/123`, ...) already matches the site patterns. Without a
|
||||||
|
/// debounce every keystroke triggers a fetch (3 attempts!) of a half-typed
|
||||||
|
/// post id. Only answer once the query has been stable for this long.
|
||||||
|
const INLINE_DEBOUNCE: std::time::Duration = std::time::Duration::from_millis(800);
|
||||||
|
|
||||||
|
/// Last seen inline query and whether it was already answered. Guards the
|
||||||
|
/// debounce timer: a repeat of an answered query is served by Telegram's
|
||||||
|
/// inline cache (see `cache_time`), not by another fetch.
|
||||||
|
struct InlineDebounceState {
|
||||||
|
query: String,
|
||||||
|
answered: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
static INLINE_DEBOUNCE_STATE: LazyLock<parking_lot::Mutex<Option<InlineDebounceState>>> =
|
||||||
|
LazyLock::new(|| parking_lot::Mutex::new(None));
|
||||||
|
|
||||||
pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), RequestError> {
|
pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), RequestError> {
|
||||||
if query.query.is_empty() {
|
if query.query.is_empty() {
|
||||||
return respond(());
|
return respond(());
|
||||||
}
|
}
|
||||||
|
// Only run a fetch for something that is actually a supported post URL.
|
||||||
|
if x_media::site::cache_key(&query.query).is_none() {
|
||||||
|
return respond(());
|
||||||
|
}
|
||||||
|
// Debounce: record the query and answer only after it has been stable for
|
||||||
|
// INLINE_DEBOUNCE (the timer below). An already-answered repeat of the
|
||||||
|
// same query is left to Telegram's inline cache instead of re-fetching.
|
||||||
|
{
|
||||||
|
let mut state = INLINE_DEBOUNCE_STATE.lock();
|
||||||
|
if let Some(prev) = state.as_ref()
|
||||||
|
&& prev.query == query.query
|
||||||
|
&& prev.answered
|
||||||
|
{
|
||||||
|
return respond(());
|
||||||
|
}
|
||||||
|
*state = Some(InlineDebounceState {
|
||||||
|
query: query.query.clone(),
|
||||||
|
answered: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let query_text = query.query.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
tokio::time::sleep(INLINE_DEBOUNCE).await;
|
||||||
|
// Only the last query of a typing burst survives: earlier timers see
|
||||||
|
// the query changed and give up without answering.
|
||||||
|
{
|
||||||
|
let mut state = INLINE_DEBOUNCE_STATE.lock();
|
||||||
|
let Some(state) = state.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if state.query != query_text || state.answered {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Claim the answer so a repeat of the same query cannot start a
|
||||||
|
// second fetch; reset below when no answer was produced.
|
||||||
|
state.answered = true;
|
||||||
|
}
|
||||||
|
match answer_inline_query(bot, query).await {
|
||||||
|
Ok(true) => {}
|
||||||
|
// No results produced (or nothing to answer): let a repeat of the
|
||||||
|
// same query retry the fetch.
|
||||||
|
Ok(false) | Err(_) => {
|
||||||
|
let mut state = INLINE_DEBOUNCE_STATE.lock();
|
||||||
|
if let Some(state) = state.as_mut()
|
||||||
|
&& state.query == query_text
|
||||||
|
{
|
||||||
|
state.answered = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
respond(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetches the post behind an inline query and answers it. The caller has
|
||||||
|
/// already applied the debounce. Returns `true` when an answer was sent.
|
||||||
|
async fn answer_inline_query(bot: Bot, query: InlineQuery) -> Result<bool, RequestError> {
|
||||||
log::info!("inline query: {}", query.query);
|
log::info!("inline query: {}", query.query);
|
||||||
match x_media::site::fetch(&query.query).await {
|
match x_media::site::fetch(&query.query).await {
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(fetched)) => {
|
||||||
let mut results: Vec<InlineQueryResult> = Vec::new();
|
let mut results: Vec<InlineQueryResult> = Vec::new();
|
||||||
|
// Inline results have the same 1024-char caption limit as regular
|
||||||
|
// messages; truncate once here for all items.
|
||||||
|
let caption = x_media::site::truncate_caption(&fetched.caption);
|
||||||
for (i, media) in fetched.media.iter().enumerate() {
|
for (i, media) in fetched.media.iter().enumerate() {
|
||||||
let id = format!("{i}");
|
let id = format!("{i}");
|
||||||
let Some(url) = url::Url::parse(media.url()).ok() else {
|
let Some(url) = url::Url::parse(media.url()).ok() else {
|
||||||
@@ -630,7 +871,7 @@ pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), Re
|
|||||||
.thumbnail_url()
|
.thumbnail_url()
|
||||||
.and_then(|t| url::Url::parse(t).ok())
|
.and_then(|t| url::Url::parse(t).ok())
|
||||||
.unwrap_or_else(|| url.clone());
|
.unwrap_or_else(|| url.clone());
|
||||||
let caption = fetched.caption.clone();
|
let caption = caption.clone();
|
||||||
let result = match media {
|
let result = match media {
|
||||||
Media::Illustration { .. } => {
|
Media::Illustration { .. } => {
|
||||||
// Inline photo results have their own (smaller) size
|
// Inline photo results have their own (smaller) size
|
||||||
@@ -665,13 +906,18 @@ pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), Re
|
|||||||
results.push(result);
|
results.push(result);
|
||||||
}
|
}
|
||||||
if !results.is_empty() {
|
if !results.is_empty() {
|
||||||
bot.answer_inline_query(query.id, results).await?;
|
// Explicit cache window: repeats of the same query within 5
|
||||||
|
// minutes are served by Telegram without hitting the bot.
|
||||||
|
bot.answer_inline_query(query.id, results)
|
||||||
|
.cache_time(300)
|
||||||
|
.await?;
|
||||||
|
return Ok(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(None) => {}
|
Ok(None) => {}
|
||||||
Err(e) => log::error!("inline fetch {}: {e}", query.query),
|
Err(e) => log::error!("inline fetch {}: {e}", query.query),
|
||||||
}
|
}
|
||||||
respond(())
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<(), RequestError> {
|
pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<(), RequestError> {
|
||||||
@@ -683,10 +929,13 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
let chat_id = message.chat().id.0;
|
let chat_id = message.chat().id.0;
|
||||||
let prompt_message_id = message.id().0 as i64;
|
let prompt_message_id = message.id().0 as i64;
|
||||||
let ttl_secs = CONFIG.edit_message_ttl.as_secs() as i64;
|
let ttl_secs = CONFIG.edit_message_ttl.as_secs() as i64;
|
||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
let chat_data = CHAT_STORE.get(chat_id).await;
|
||||||
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
||||||
let Some(edit) = edit else {
|
let Some(edit) = edit else {
|
||||||
log::info!("callback from {}: no edit record for prompt {prompt_message_id}", chat_id);
|
log::info!(
|
||||||
|
"callback from {}: no edit record for prompt {prompt_message_id}",
|
||||||
|
chat_id
|
||||||
|
);
|
||||||
bot.answer_callback_query(callback_query_id)
|
bot.answer_callback_query(callback_query_id)
|
||||||
.text("Expired")
|
.text("Expired")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -694,8 +943,11 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
};
|
};
|
||||||
// Lazy expiry: a stale record (past the TTL, not yet swept) is dropped.
|
// Lazy expiry: a stale record (past the TTL, not yet swept) is dropped.
|
||||||
if edit.created_at + ttl_secs <= unix_now() {
|
if edit.created_at + ttl_secs <= unix_now() {
|
||||||
chat_data.edit_message.remove(&prompt_message_id);
|
CHAT_STORE
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
.update(chat_id, |data| {
|
||||||
|
data.edit_message.remove(&prompt_message_id);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
bot.answer_callback_query(callback_query_id)
|
bot.answer_callback_query(callback_query_id)
|
||||||
.text("Expired")
|
.text("Expired")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -705,7 +957,10 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
let Some(data) = data else {
|
let Some(data) = data else {
|
||||||
return respond(());
|
return respond(());
|
||||||
};
|
};
|
||||||
log::info!("callback from {} on prompt {prompt_message_id}: {data}", chat_id);
|
log::info!(
|
||||||
|
"callback from {} on prompt {prompt_message_id}: {data}",
|
||||||
|
chat_id
|
||||||
|
);
|
||||||
if data == "forward" {
|
if data == "forward" {
|
||||||
match chat_data.forward_channel_id {
|
match chat_data.forward_channel_id {
|
||||||
Some(channel_id) => {
|
Some(channel_id) => {
|
||||||
@@ -728,10 +983,16 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
let _ = bot
|
let _ = bot
|
||||||
.delete_message(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
.delete_message(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
||||||
.await;
|
.await;
|
||||||
chat_data.edit_message.remove(&prompt_message_id);
|
CHAT_STORE
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
.update(chat_id, |data| {
|
||||||
|
data.edit_message.remove(&prompt_message_id);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable { delay_seconds, task }) => {
|
Err(send::SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
log::info!("forward queued for retry in {delay_seconds:.1}s");
|
log::info!("forward queued for retry in {delay_seconds:.1}s");
|
||||||
enqueue_retry(task, delay_seconds).await;
|
enqueue_retry(task, delay_seconds).await;
|
||||||
bot.answer_callback_query(callback_query_id)
|
bot.answer_callback_query(callback_query_id)
|
||||||
@@ -765,10 +1026,13 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
.caption(template_html)
|
.caption(template_html)
|
||||||
.parse_mode(ParseMode::Html)
|
.parse_mode(ParseMode::Html)
|
||||||
.await;
|
.await;
|
||||||
if let Some(entry) = chat_data.edit_message.get_mut(&prompt_message_id) {
|
CHAT_STORE
|
||||||
|
.update(chat_id, |data| {
|
||||||
|
if let Some(entry) = data.edit_message.get_mut(&prompt_message_id) {
|
||||||
entry.template = name.to_string();
|
entry.template = name.to_string();
|
||||||
}
|
}
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
})
|
||||||
|
.await;
|
||||||
log::info!("template '{name}' applied to prompt {prompt_message_id}");
|
log::info!("template '{name}' applied to prompt {prompt_message_id}");
|
||||||
}
|
}
|
||||||
bot.answer_callback_query(callback_query_id).await?;
|
bot.answer_callback_query(callback_query_id).await?;
|
||||||
|
|||||||
@@ -8,7 +8,8 @@
|
|||||||
//! [`Config::link_cache_ttl`]; a stale entry is dropped lazily on read and
|
//! [`Config::link_cache_ttl`]; a stale entry is dropped lazily on read and
|
||||||
//! by the periodic prune in `main`.
|
//! by the periodic prune in `main`.
|
||||||
|
|
||||||
use rusqlite::{params, Connection};
|
use crate::db::now_f64;
|
||||||
|
use rusqlite::{Connection, params};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -46,13 +47,7 @@ pub struct CachedPost {
|
|||||||
/// SQLite-backed cache sharing `data/task_queue.db` with the queue and chat
|
/// SQLite-backed cache sharing `data/task_queue.db` with the queue and chat
|
||||||
/// state (same `open_db` pattern: busy timeout, `spawn_blocking` I/O).
|
/// state (same `open_db` pattern: busy timeout, `spawn_blocking` I/O).
|
||||||
pub struct LinkCache {
|
pub struct LinkCache {
|
||||||
db_path: String,
|
pool: crate::db::DbPool,
|
||||||
}
|
|
||||||
|
|
||||||
fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
|
||||||
let conn = Connection::open(path)?;
|
|
||||||
conn.busy_timeout(Duration::from_secs(5))?;
|
|
||||||
Ok(conn)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LinkCache {
|
impl LinkCache {
|
||||||
@@ -66,18 +61,18 @@ impl LinkCache {
|
|||||||
log::error!("failed to initialize link cache schema: {e}");
|
log::error!("failed to initialize link cache schema: {e}");
|
||||||
}
|
}
|
||||||
Self {
|
Self {
|
||||||
db_path: db_path.to_string(),
|
pool: crate::db::DbPool::new(db_path),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the cached post if present and not expired; a stale entry is
|
/// Returns the cached post if present and not expired; a stale entry is
|
||||||
/// removed on the spot.
|
/// removed on the spot.
|
||||||
pub async fn get(&self, key: &str, ttl: Duration) -> Option<CachedPost> {
|
pub async fn get(&self, key: &str, ttl: Duration) -> Option<CachedPost> {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
let ttl = ttl.as_secs_f64();
|
let ttl = ttl.as_secs_f64();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<CachedPost>> {
|
let result = self
|
||||||
let conn = open_db(&db_path)?;
|
.pool
|
||||||
|
.with_conn(move |conn| {
|
||||||
let mut stmt =
|
let mut stmt =
|
||||||
conn.prepare("SELECT payload, created_at FROM link_cache WHERE url = ?1")?;
|
conn.prepare("SELECT payload, created_at FROM link_cache WHERE url = ?1")?;
|
||||||
let mut rows = stmt.query(params![key])?;
|
let mut rows = stmt.query(params![key])?;
|
||||||
@@ -90,74 +85,93 @@ impl LinkCache {
|
|||||||
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
serde_json::from_str(&payload).map(Some).map_err(|e| {
|
Ok(Some(serde_json::from_str::<CachedPost>(&payload).map_err(
|
||||||
rusqlite::Error::ToSqlConversionFailure(Box::new(e))
|
|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)),
|
||||||
|
)?))
|
||||||
})
|
})
|
||||||
})
|
.await;
|
||||||
.await
|
match result {
|
||||||
.expect("link cache read worker panicked")
|
Ok(v) => v,
|
||||||
.unwrap_or_else(|e| {
|
Err(e) => {
|
||||||
log::error!("link cache read failed: {e}");
|
log::error!("link cache read failed: {e}");
|
||||||
None
|
None
|
||||||
})
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn put(&self, key: &str, post: &CachedPost) {
|
pub async fn put(&self, key: &str, post: &CachedPost) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
let payload = serde_json::to_string(post).expect("cached post serializes");
|
let payload = serde_json::to_string(post).expect("cached post serializes");
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = self
|
||||||
let conn = open_db(&db_path)?;
|
.pool
|
||||||
|
.with_conn(move |conn| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO link_cache (url, payload, created_at) VALUES (?1, ?2, ?3)",
|
"INSERT OR REPLACE INTO link_cache (url, payload, created_at) VALUES (?1, ?2, ?3)",
|
||||||
params![key, payload, now_f64()],
|
params![key, payload, now_f64()],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("link cache write worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("link cache write failed: {e}"));
|
log::error!("link cache write failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drops an entry (e.g. a cached file id that turned out invalid).
|
/// Drops an entry (e.g. a cached file id that turned out invalid).
|
||||||
pub async fn remove(&self, key: &str) {
|
pub async fn remove(&self, key: &str) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = self
|
||||||
let conn = open_db(&db_path)?;
|
.pool
|
||||||
|
.with_conn(move |conn| {
|
||||||
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("link cache delete worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("link cache delete failed: {e}"));
|
log::error!("link cache delete failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes expired entries; returns how many were deleted.
|
/// Removes expired entries; returns how many were deleted.
|
||||||
pub async fn prune(&self, ttl: Duration) -> usize {
|
pub async fn prune(&self, ttl: Duration) -> usize {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let cutoff = now_f64() - ttl.as_secs_f64();
|
let cutoff = now_f64() - ttl.as_secs_f64();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<usize> {
|
let result = self
|
||||||
let conn = open_db(&db_path)?;
|
.pool
|
||||||
|
.with_conn(move |conn| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM link_cache WHERE created_at < ?1",
|
"DELETE FROM link_cache WHERE created_at < ?1",
|
||||||
params![cutoff],
|
params![cutoff],
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("link cache prune worker panicked")
|
match result {
|
||||||
.unwrap_or_else(|e| {
|
Ok(n) => n,
|
||||||
|
Err(e) => {
|
||||||
log::error!("link cache prune failed: {e}");
|
log::error!("link cache prune failed: {e}");
|
||||||
0
|
0
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn now_f64() -> f64 {
|
/// Deletes one entry (by normalized cache key) or the whole cache when
|
||||||
std::time::SystemTime::now()
|
/// `key` is `None`. Returns how many rows were removed.
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
pub async fn clear(&self, key: Option<&str>) -> usize {
|
||||||
.map(|d| d.as_secs_f64())
|
let key = key.map(str::to_string);
|
||||||
.unwrap_or(0.0)
|
let result = self
|
||||||
|
.pool
|
||||||
|
.with_conn(move |conn| match &key {
|
||||||
|
Some(key) => conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key]),
|
||||||
|
None => conn.execute("DELETE FROM link_cache", []),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
match result {
|
||||||
|
Ok(n) => n,
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("link cache clear failed: {e}");
|
||||||
|
0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -200,14 +214,21 @@ mod tests {
|
|||||||
// Force the row into the past so a 1s TTL expires it.
|
// Force the row into the past so a 1s TTL expires it.
|
||||||
{
|
{
|
||||||
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
||||||
conn.execute(
|
conn.execute("UPDATE link_cache SET created_at = created_at - 100", [])
|
||||||
"UPDATE link_cache SET created_at = created_at - 100",
|
|
||||||
[],
|
|
||||||
)
|
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
assert!(cache.get("twitter:1", Duration::from_secs(1)).await.is_none());
|
assert!(
|
||||||
assert!(cache.get("twitter:1", Duration::from_secs(3600)).await.is_none());
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(1))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -217,14 +238,60 @@ mod tests {
|
|||||||
cache.put("twitter:1", &entry()).await;
|
cache.put("twitter:1", &entry()).await;
|
||||||
cache.put("pixiv:2", &entry()).await;
|
cache.put("pixiv:2", &entry()).await;
|
||||||
cache.remove("twitter:1").await;
|
cache.remove("twitter:1").await;
|
||||||
assert!(cache.get("twitter:1", Duration::from_secs(3600)).await.is_none());
|
assert!(
|
||||||
assert!(cache.get("pixiv:2", Duration::from_secs(3600)).await.is_some());
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
{
|
{
|
||||||
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
||||||
conn.execute("UPDATE link_cache SET created_at = created_at - 100", [])
|
conn.execute("UPDATE link_cache SET created_at = created_at - 100", [])
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
assert_eq!(cache.prune(Duration::from_secs(1)).await, 1);
|
assert_eq!(cache.prune(Duration::from_secs(1)).await, 1);
|
||||||
assert!(cache.get("pixiv:2", Duration::from_secs(3600)).await.is_none());
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn clear_one_entry_or_all() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let cache = LinkCache::open(dir.path().join("c.db").to_str().unwrap());
|
||||||
|
cache.put("twitter:1", &entry()).await;
|
||||||
|
cache.put("pixiv:2", &entry()).await;
|
||||||
|
// By key: only the matching row is removed.
|
||||||
|
assert_eq!(cache.clear(Some("twitter:1")).await, 1);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
// Whole cache: nothing left; removing an absent key deletes 0 rows.
|
||||||
|
assert_eq!(cache.clear(None).await, 1);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(cache.clear(None).await, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
use dotenv::dotenv;
|
use dotenv::dotenv;
|
||||||
use teloxide::dptree::endpoint;
|
use teloxide::dptree::endpoint;
|
||||||
|
use teloxide::prelude::*;
|
||||||
use teloxide::stop::StopToken;
|
use teloxide::stop::StopToken;
|
||||||
use teloxide::types::{ChatId, InputFile, MessageId};
|
use teloxide::types::{ChatId, InputFile, MessageId};
|
||||||
use teloxide::update_listeners::{self, webhooks, UpdateListener};
|
use teloxide::update_listeners::{self, UpdateListener, webhooks};
|
||||||
use teloxide::prelude::*;
|
|
||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
use x_media::site;
|
use x_media::site;
|
||||||
|
|
||||||
mod config;
|
mod config;
|
||||||
|
mod db;
|
||||||
mod handlers;
|
mod handlers;
|
||||||
mod link_cache;
|
mod link_cache;
|
||||||
|
mod photo;
|
||||||
mod queue;
|
mod queue;
|
||||||
mod send;
|
mod send;
|
||||||
mod state;
|
mod state;
|
||||||
@@ -41,6 +43,14 @@ async fn main() {
|
|||||||
log::info!("Starting bot");
|
log::info!("Starting bot");
|
||||||
|
|
||||||
let bot = Bot::from_env();
|
let bot = Bot::from_env();
|
||||||
|
// Force the queue workers' shared Bot to initialize now so a missing
|
||||||
|
// token fails at startup, not on the first queued task.
|
||||||
|
let _ = &*send::BOT;
|
||||||
|
|
||||||
|
// Register the command list with Telegram (client `/` menu).
|
||||||
|
if let Err(e) = handlers::register_commands(&bot).await {
|
||||||
|
log::warn!("failed to register commands: {e}");
|
||||||
|
}
|
||||||
|
|
||||||
log::info!(
|
log::info!(
|
||||||
"config: {} admin(s), edit-message TTL {}s",
|
"config: {} admin(s), edit-message TTL {}s",
|
||||||
@@ -55,6 +65,10 @@ async fn main() {
|
|||||||
.await;
|
.await;
|
||||||
log::info!("task queue worker started");
|
log::info!("task queue worker started");
|
||||||
|
|
||||||
|
// URL job workers: bounded channel + fixed pool for per-URL work.
|
||||||
|
handlers::start_url_workers().await;
|
||||||
|
log::info!("url workers started");
|
||||||
|
|
||||||
// Pixiv login validation (user request): a failed login notifies the
|
// Pixiv login validation (user request): a failed login notifies the
|
||||||
// admin and disables pixiv for this process.
|
// admin and disables pixiv for this process.
|
||||||
if site::pixiv::enabled() {
|
if site::pixiv::enabled() {
|
||||||
@@ -73,7 +87,10 @@ async fn main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Edit-expiry sweep: clears the prompt's buttons once the record expires.
|
// Edit-expiry sweep: clears the prompt's buttons once the record expires.
|
||||||
log::info!("edit-expiry sweep: every 300s, ttl {}", CONFIG.edit_message_ttl.as_secs());
|
log::info!(
|
||||||
|
"edit-expiry sweep: every 300s, ttl {}",
|
||||||
|
CONFIG.edit_message_ttl.as_secs()
|
||||||
|
);
|
||||||
let (stop_tx, stop_rx) = watch::channel(false);
|
let (stop_tx, stop_rx) = watch::channel(false);
|
||||||
{
|
{
|
||||||
let bot = bot.clone();
|
let bot = bot.clone();
|
||||||
@@ -94,7 +111,10 @@ async fn main() {
|
|||||||
// If the prompt was already deleted, this fails with a
|
// If the prompt was already deleted, this fails with a
|
||||||
// 400 "message to edit not found" — log and ignore.
|
// 400 "message to edit not found" — log and ignore.
|
||||||
if let Err(e) = bot
|
if let Err(e) = bot
|
||||||
.edit_message_reply_markup(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
.edit_message_reply_markup(
|
||||||
|
ChatId(chat_id),
|
||||||
|
MessageId(prompt_message_id as i32),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
log::info!("edit-expiry sweep: prompt message gone: {e}");
|
log::info!("edit-expiry sweep: prompt message gone: {e}");
|
||||||
@@ -116,10 +136,7 @@ async fn main() {
|
|||||||
|
|
||||||
if CONFIG.webhook_enabled {
|
if CONFIG.webhook_enabled {
|
||||||
log::info!("running in webhook mode");
|
log::info!("running in webhook mode");
|
||||||
let url = CONFIG
|
let url = CONFIG.webhook_url.clone().expect("WEBHOOK_URL is not set");
|
||||||
.webhook_url
|
|
||||||
.clone()
|
|
||||||
.expect("WEBHOOK_URL is not set");
|
|
||||||
// `webhooks::axum` calls set_webhook itself (with the full options,
|
// `webhooks::axum` calls set_webhook itself (with the full options,
|
||||||
// secret token included) — no explicit registration here.
|
// secret token included) — no explicit registration here.
|
||||||
let listen = CONFIG.webhook_listen.expect("WEBHOOK_LISTEN is not set");
|
let listen = CONFIG.webhook_listen.expect("WEBHOOK_LISTEN is not set");
|
||||||
@@ -161,12 +178,25 @@ async fn main() {
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Graceful stop (Ctrl+C / SIGTERM): stop the sweep, notify the admin, drain the queue.
|
// Graceful stop (Ctrl+C / SIGTERM): stop the sweep, notify the admin,
|
||||||
|
// drain the queue. Bounded: a worker mid-download (30 s timeout) or a
|
||||||
|
// long ugoira encode must not hold the shutdown hostage forever.
|
||||||
log::info!("Stopping bot");
|
log::info!("Stopping bot");
|
||||||
|
const SHUTDOWN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||||
|
let shutdown = async {
|
||||||
let _ = stop_tx.send(true);
|
let _ = stop_tx.send(true);
|
||||||
|
handlers::stop_url_workers();
|
||||||
if let Some(admin) = CONFIG.admin_ids.first() {
|
if let Some(admin) = CONFIG.admin_ids.first() {
|
||||||
let _ = bot.send_message(ChatId(*admin), "Shutting down...").await;
|
let _ = bot.send_message(ChatId(*admin), "Shutting down...").await;
|
||||||
}
|
}
|
||||||
TASK_QUEUE.stop().await;
|
TASK_QUEUE.stop().await;
|
||||||
|
};
|
||||||
|
if tokio::time::timeout(SHUTDOWN_TIMEOUT, shutdown)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
log::warn!("graceful shutdown timed out after {SHUTDOWN_TIMEOUT:?}; exiting");
|
||||||
|
} else {
|
||||||
log::info!("Bot stopped");
|
log::info!("Bot stopped");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,529 @@
|
|||||||
|
//! Pure-Rust photo processing: brings a downloaded photo within Telegram's
|
||||||
|
//! limits (width + height ≤ 10000 px, bytes ≤ 10 MiB) without ffmpeg.
|
||||||
|
//!
|
||||||
|
//! Stack: `png` (image-png) for PNG decode/encode, `zune-jpeg` for JPEG
|
||||||
|
//! decode, `fast_image_resize` (Lanczos3) for downsampling, `jpeg-encoder`
|
||||||
|
//! for JPEG output.
|
||||||
|
//!
|
||||||
|
//! Bit-depth rule: a PNG above 24 bits (32-bit RGBA or 16-bit per channel)
|
||||||
|
//! is reduced to 24-bit RGB; 24-bit and lower depths are left untouched —
|
||||||
|
//! gray stays gray, never upconverted. The only upconversion is palette
|
||||||
|
//! expansion, which resampling requires. Alpha is flattened onto white (JPEG
|
||||||
|
//! and 24-bit RGB have no alpha channel).
|
||||||
|
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
use fast_image_resize as fir;
|
||||||
|
use tempfile::NamedTempFile;
|
||||||
|
|
||||||
|
/// Telegram rejects photos whose width + height exceed this limit
|
||||||
|
/// (PHOTO_INVALID_DIMENSIONS). Verified empirically: 6300x3730 (sum 10030)
|
||||||
|
/// fails, 6100x3900 (sum 10000) passes.
|
||||||
|
pub const PHOTO_MAX_DIMENSION_SUM: u32 = 10000;
|
||||||
|
/// Resize target with a safety margin so rounding cannot cross the cap.
|
||||||
|
pub const PHOTO_TARGET_DIMENSION_SUM: u32 = 9900;
|
||||||
|
/// Upload cap (bytes): files above this are not uploaded; the bot falls back
|
||||||
|
/// to a smaller media URL instead.
|
||||||
|
pub const MAX_UPLOAD_BYTES: u64 = 10 * 1024 * 1024;
|
||||||
|
/// Decode budget (bytes): a larger intermediate buffer is not worth the peak
|
||||||
|
/// memory; the photo degrades to the smaller URL instead. Also the cap for
|
||||||
|
/// downloading photos in the send fallback (they must be downloaded whole).
|
||||||
|
pub(crate) const MAX_DECODE_BYTES: u64 = 512 * 1024 * 1024;
|
||||||
|
/// JPEG output quality (1-100).
|
||||||
|
const JPEG_QUALITY: u8 = 90;
|
||||||
|
|
||||||
|
/// What to upload for a downloaded photo.
|
||||||
|
pub enum PhotoPrep {
|
||||||
|
/// Upload this file (the original when within limits, else the processed
|
||||||
|
/// copy).
|
||||||
|
Upload(NamedTempFile),
|
||||||
|
/// The photo cannot be brought within Telegram's limits — the caller
|
||||||
|
/// falls back to the item's smaller URL.
|
||||||
|
UseFallback,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A decoded image buffer tagged with its channel layout.
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum PixBuf {
|
||||||
|
Gray(Vec<u8>),
|
||||||
|
GrayAlpha(Vec<u8>),
|
||||||
|
Rgb(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PixBuf {
|
||||||
|
fn pixel_type(&self) -> fir::PixelType {
|
||||||
|
match self {
|
||||||
|
PixBuf::Gray(_) => fir::PixelType::U8,
|
||||||
|
PixBuf::GrayAlpha(_) => fir::PixelType::U8x2,
|
||||||
|
PixBuf::Rgb(_) => fir::PixelType::U8x3,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn into_vec(self) -> Vec<u8> {
|
||||||
|
match self {
|
||||||
|
PixBuf::Gray(v) | PixBuf::GrayAlpha(v) | PixBuf::Rgb(v) => v,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Entry point: detects the format and processes the photo if needed.
|
||||||
|
/// The caller hands in the already-downloaded bytes (they are in memory from
|
||||||
|
/// the download anyway; re-reading the temp file would double the I/O).
|
||||||
|
pub fn prepare_photo(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||||
|
if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||||
|
prepare_png(file, bytes)
|
||||||
|
} else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||||||
|
prepare_jpeg(file, bytes)
|
||||||
|
} else {
|
||||||
|
log::warn!("photo in unsupported format; falling back to smaller media");
|
||||||
|
Ok(PhotoPrep::UseFallback)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses the PNG IHDR (bytes 8..26: signature + length + "IHDR" + width +
|
||||||
|
/// height + bit depth + color type).
|
||||||
|
fn parse_png_header(bytes: &[u8]) -> Option<(u32, u32, png::BitDepth, png::ColorType)> {
|
||||||
|
if !bytes.starts_with(b"\x89PNG\r\n\x1a\n") || bytes.len() < 26 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let w = u32::from_be_bytes(bytes.get(16..20)?.try_into().ok()?);
|
||||||
|
let h = u32::from_be_bytes(bytes.get(20..24)?.try_into().ok()?);
|
||||||
|
let depth = match *bytes.get(24)? {
|
||||||
|
1 => png::BitDepth::One,
|
||||||
|
2 => png::BitDepth::Two,
|
||||||
|
4 => png::BitDepth::Four,
|
||||||
|
8 => png::BitDepth::Eight,
|
||||||
|
16 => png::BitDepth::Sixteen,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let color = match *bytes.get(25)? {
|
||||||
|
0 => png::ColorType::Grayscale,
|
||||||
|
2 => png::ColorType::Rgb,
|
||||||
|
3 => png::ColorType::Indexed,
|
||||||
|
4 => png::ColorType::GrayscaleAlpha,
|
||||||
|
6 => png::ColorType::Rgba,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
Some((w, h, depth, color))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Output channels of a decoded frame for the given color type (post
|
||||||
|
/// STRIP_16; palette expands to RGB).
|
||||||
|
fn output_channels(color: png::ColorType) -> usize {
|
||||||
|
match color {
|
||||||
|
png::ColorType::Grayscale => 1,
|
||||||
|
png::ColorType::GrayscaleAlpha => 2,
|
||||||
|
png::ColorType::Rgb | png::ColorType::Indexed => 3,
|
||||||
|
png::ColorType::Rgba => 4,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The 32→24 rule: RGBA (32-bit) becomes RGB with alpha composited onto
|
||||||
|
/// white; 16-bit per channel was already stripped to 8-bit at decode.
|
||||||
|
fn flatten_rgba_to_rgb(rgba: &[u8]) -> Vec<u8> {
|
||||||
|
let mut rgb = Vec::with_capacity(rgba.len() / 4 * 3);
|
||||||
|
for px in rgba.chunks_exact(4) {
|
||||||
|
let a = px[3] as u32;
|
||||||
|
for v in &px[..3] {
|
||||||
|
// Over white: C = C*a/255 + 255*(1 - a/255).
|
||||||
|
let v = (*v as u32 * a + 255 * (255 - a)) / 255;
|
||||||
|
rgb.push(v.min(255) as u8);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rgb
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lanczos3 downsampling via fast_image_resize.
|
||||||
|
fn resize_pix(pix: PixBuf, w: u32, h: u32, nw: u32, nh: u32) -> Result<PixBuf, String> {
|
||||||
|
let pixel_type = pix.pixel_type();
|
||||||
|
let src = fir::images::Image::from_vec_u8(w, h, pix.into_vec(), pixel_type)
|
||||||
|
.map_err(|e| format!("resize input: {e}"))?;
|
||||||
|
let mut dst = fir::images::Image::new(nw, nh, pixel_type);
|
||||||
|
let mut resizer = fir::Resizer::new();
|
||||||
|
let options = fir::ResizeOptions::default()
|
||||||
|
.resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::Lanczos3));
|
||||||
|
resizer
|
||||||
|
.resize(&src, &mut dst, &options)
|
||||||
|
.map_err(|e| format!("resize: {e}"))?;
|
||||||
|
let buf = dst.into_vec();
|
||||||
|
Ok(match pixel_type {
|
||||||
|
fir::PixelType::U8 => PixBuf::Gray(buf),
|
||||||
|
fir::PixelType::U8x2 => PixBuf::GrayAlpha(buf),
|
||||||
|
_ => PixBuf::Rgb(buf),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_png(out: &mut Vec<u8>, pix: &PixBuf, w: u32, h: u32) -> Result<(), png::EncodingError> {
|
||||||
|
let (color, buf) = match pix {
|
||||||
|
PixBuf::Gray(v) => (png::ColorType::Grayscale, v.as_slice()),
|
||||||
|
PixBuf::GrayAlpha(v) => (png::ColorType::GrayscaleAlpha, v.as_slice()),
|
||||||
|
PixBuf::Rgb(v) => (png::ColorType::Rgb, v.as_slice()),
|
||||||
|
};
|
||||||
|
let mut encoder = png::Encoder::new(out, w, h);
|
||||||
|
encoder.set_color(color);
|
||||||
|
encoder.set_depth(png::BitDepth::Eight);
|
||||||
|
let mut writer = encoder.write_header()?;
|
||||||
|
writer.write_image_data(buf)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_jpeg(pix: &PixBuf, w: u32, h: u32) -> Result<Vec<u8>, String> {
|
||||||
|
use jpeg_encoder::{ColorType, Encoder};
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let encoder = Encoder::new(&mut out, JPEG_QUALITY);
|
||||||
|
match pix {
|
||||||
|
PixBuf::Gray(v) => encoder
|
||||||
|
.encode(v, w as u16, h as u16, ColorType::Luma)
|
||||||
|
.map_err(|e| format!("jpeg encode: {e}"))?,
|
||||||
|
PixBuf::GrayAlpha(v) => {
|
||||||
|
// JPEG has no alpha: composite onto white, output as gray.
|
||||||
|
let gray: Vec<u8> = v
|
||||||
|
.chunks_exact(2)
|
||||||
|
.map(|px| {
|
||||||
|
let (g, a) = (px[0] as u32, px[1] as u32);
|
||||||
|
((g * a + 255 * (255 - a)) / 255).min(255) as u8
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
encoder
|
||||||
|
.encode(&gray, w as u16, h as u16, ColorType::Luma)
|
||||||
|
.map_err(|e| format!("jpeg encode: {e}"))?;
|
||||||
|
}
|
||||||
|
PixBuf::Rgb(v) => encoder
|
||||||
|
.encode(v, w as u16, h as u16, ColorType::Rgb)
|
||||||
|
.map_err(|e| format!("jpeg encode: {e}"))?,
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_temp(bytes: &[u8], ext: &str) -> Result<NamedTempFile, String> {
|
||||||
|
let mut file = tempfile::Builder::new()
|
||||||
|
.suffix(&format!(".{ext}"))
|
||||||
|
.tempfile()
|
||||||
|
.map_err(|e| format!("temp file failed: {e}"))?;
|
||||||
|
file.as_file_mut()
|
||||||
|
.write_all(bytes)
|
||||||
|
.map_err(|e| format!("temp file write failed: {e}"))?;
|
||||||
|
Ok(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn target_dims(w: u32, h: u32) -> (u32, u32) {
|
||||||
|
let scale = PHOTO_TARGET_DIMENSION_SUM as f64 / (w + h) as f64;
|
||||||
|
(
|
||||||
|
((w as f64 * scale).round() as u32).max(1),
|
||||||
|
((h as f64 * scale).round() as u32).max(1),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// PNG branch: decode (16→8, palette→RGB; gray/GA stay), flatten RGBA to
|
||||||
|
/// RGB, Lanczos-downscale beyond the dimension cap, encode PNG — a PNG still
|
||||||
|
/// over the upload cap afterwards becomes JPEG.
|
||||||
|
fn prepare_png(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||||
|
let (w, h, _bit_depth, color_type) = parse_png_header(bytes).ok_or("invalid PNG header")?;
|
||||||
|
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
||||||
|
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
||||||
|
return Ok(PhotoPrep::Upload(file));
|
||||||
|
}
|
||||||
|
log::info!(
|
||||||
|
"photo {w}x{h} ({_bit_depth:?} {color_type:?}, {} bytes) needs processing",
|
||||||
|
bytes.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
let channels = output_channels(color_type);
|
||||||
|
if (w as u64) * (h as u64) * channels as u64 > MAX_DECODE_BYTES {
|
||||||
|
log::warn!("photo decode buffer exceeds the memory budget; falling back to smaller media");
|
||||||
|
return Ok(PhotoPrep::UseFallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
// STRIP_16 drops 16-bit to 8-bit (the depth-reduction step); palette
|
||||||
|
// expands to RGB (resampling requires it). Gray and gray-alpha are kept.
|
||||||
|
let transforms = match color_type {
|
||||||
|
png::ColorType::Indexed => png::Transformations::EXPAND,
|
||||||
|
_ => png::Transformations::STRIP_16,
|
||||||
|
};
|
||||||
|
let mut decoder = png::Decoder::new(std::io::Cursor::new(bytes));
|
||||||
|
decoder.set_transformations(transforms);
|
||||||
|
let mut reader = decoder
|
||||||
|
.read_info()
|
||||||
|
.map_err(|e| format!("png decode: {e}"))?;
|
||||||
|
let out_w = reader.info().width;
|
||||||
|
let out_h = reader.info().height;
|
||||||
|
let mut buf = vec![
|
||||||
|
0u8;
|
||||||
|
reader
|
||||||
|
.output_buffer_size()
|
||||||
|
.ok_or("png output buffer size")?
|
||||||
|
];
|
||||||
|
reader
|
||||||
|
.next_frame(&mut buf)
|
||||||
|
.map_err(|e| format!("png frame: {e}"))?;
|
||||||
|
|
||||||
|
let mut pix = match color_type {
|
||||||
|
png::ColorType::Rgba => PixBuf::Rgb(flatten_rgba_to_rgb(&buf)),
|
||||||
|
png::ColorType::Grayscale => PixBuf::Gray(buf),
|
||||||
|
png::ColorType::GrayscaleAlpha => PixBuf::GrayAlpha(buf),
|
||||||
|
png::ColorType::Rgb | png::ColorType::Indexed => PixBuf::Rgb(buf),
|
||||||
|
};
|
||||||
|
|
||||||
|
let (mut w, mut h) = (out_w, out_h);
|
||||||
|
if w + h > PHOTO_MAX_DIMENSION_SUM {
|
||||||
|
let (nw, nh) = target_dims(w, h);
|
||||||
|
pix = resize_pix(pix, w, h, nw, nh)?;
|
||||||
|
(w, h) = (nw, nh);
|
||||||
|
log::info!("downscaled photo to {w}x{h} (Lanczos3)");
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut png_bytes = Vec::new();
|
||||||
|
encode_png(&mut png_bytes, &pix, w, h).map_err(|e| format!("png encode: {e}"))?;
|
||||||
|
if png_bytes.len() as u64 <= MAX_UPLOAD_BYTES {
|
||||||
|
return Ok(PhotoPrep::Upload(write_temp(&png_bytes, "png")?));
|
||||||
|
}
|
||||||
|
log::info!("PNG still over the upload cap after processing; transcoding to JPEG");
|
||||||
|
let jpeg_bytes = encode_jpeg(&pix, w, h)?;
|
||||||
|
if jpeg_bytes.len() as u64 <= MAX_UPLOAD_BYTES {
|
||||||
|
return Ok(PhotoPrep::Upload(write_temp(&jpeg_bytes, "jpg")?));
|
||||||
|
}
|
||||||
|
log::warn!("processed photo still exceeds the upload cap; falling back to smaller media");
|
||||||
|
Ok(PhotoPrep::UseFallback)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JPEG branch: zune-jpeg decode → Lanczos downscale → jpeg-encoder output.
|
||||||
|
fn prepare_jpeg(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||||
|
let mut decoder = zune_jpeg::JpegDecoder::new(std::io::Cursor::new(bytes));
|
||||||
|
// Decodes to RGB by default. Headers first so dimensions are known before
|
||||||
|
// the (potentially huge) pixel decode.
|
||||||
|
decoder
|
||||||
|
.decode_headers()
|
||||||
|
.map_err(|e| format!("jpeg headers: {e}"))?;
|
||||||
|
let info = decoder.info().ok_or("jpeg info unavailable")?;
|
||||||
|
let (w, h) = (info.width as u32, info.height as u32);
|
||||||
|
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
||||||
|
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
||||||
|
return Ok(PhotoPrep::Upload(file));
|
||||||
|
}
|
||||||
|
if (w as u64) * (h as u64) * 3 > MAX_DECODE_BYTES {
|
||||||
|
log::warn!("photo decode buffer exceeds the memory budget; falling back to smaller media");
|
||||||
|
return Ok(PhotoPrep::UseFallback);
|
||||||
|
}
|
||||||
|
let pixels = decoder.decode().map_err(|e| format!("jpeg decode: {e}"))?;
|
||||||
|
let mut pix = PixBuf::Rgb(pixels);
|
||||||
|
let (mut w, mut h) = (w, h);
|
||||||
|
if w + h > PHOTO_MAX_DIMENSION_SUM {
|
||||||
|
let (nw, nh) = target_dims(w, h);
|
||||||
|
pix = resize_pix(pix, w, h, nw, nh)?;
|
||||||
|
(w, h) = (nw, nh);
|
||||||
|
log::info!("downscaled jpeg to {w}x{h} (Lanczos3)");
|
||||||
|
}
|
||||||
|
let jpeg_bytes = encode_jpeg(&pix, w, h)?;
|
||||||
|
if jpeg_bytes.len() as u64 <= MAX_UPLOAD_BYTES {
|
||||||
|
return Ok(PhotoPrep::Upload(write_temp(&jpeg_bytes, "jpg")?));
|
||||||
|
}
|
||||||
|
log::warn!("processed photo still exceeds the upload cap; falling back to smaller media");
|
||||||
|
Ok(PhotoPrep::UseFallback)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn png_header(w: u32, h: u32, depth: u8, color: u8) -> Vec<u8> {
|
||||||
|
let mut bytes = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR".to_vec();
|
||||||
|
bytes.extend(w.to_be_bytes());
|
||||||
|
bytes.extend(h.to_be_bytes());
|
||||||
|
bytes.extend([depth, color, 0, 0, 0]);
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_png_header() {
|
||||||
|
let bytes = png_header(8979, 5316, 16, 6); // 16-bit RGBA
|
||||||
|
let (w, h, depth, color) = parse_png_header(&bytes).unwrap();
|
||||||
|
assert_eq!((w, h), (8979, 5316));
|
||||||
|
assert_eq!(depth, png::BitDepth::Sixteen);
|
||||||
|
assert_eq!(color, png::ColorType::Rgba);
|
||||||
|
|
||||||
|
let (_, _, depth, color) = parse_png_header(&png_header(10, 10, 8, 0)).unwrap();
|
||||||
|
assert_eq!(depth, png::BitDepth::Eight);
|
||||||
|
assert_eq!(color, png::ColorType::Grayscale);
|
||||||
|
|
||||||
|
assert!(parse_png_header(b"not a png").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn flatten_rgba_to_rgb_composites_over_white() {
|
||||||
|
// opaque red stays red
|
||||||
|
assert_eq!(flatten_rgba_to_rgb(&[255, 0, 0, 255]), vec![255, 0, 0]);
|
||||||
|
// fully transparent → white
|
||||||
|
assert_eq!(flatten_rgba_to_rgb(&[0, 0, 0, 0]), vec![255, 255, 255]);
|
||||||
|
// half alpha red → (255+255)/2 = 255, (0*128 + 255*127)/255 = 127
|
||||||
|
let out = flatten_rgba_to_rgb(&[255, 0, 0, 128]);
|
||||||
|
assert_eq!(out[0], 255);
|
||||||
|
assert_eq!(out[1], 127);
|
||||||
|
assert_eq!(out[2], 127);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn target_dims_stay_under_the_cap() {
|
||||||
|
for (w, h) in [(12000u32, 7000u32), (10000, 10000), (8979, 5316)] {
|
||||||
|
let (nw, nh) = target_dims(w, h);
|
||||||
|
assert!(nw + nh <= PHOTO_MAX_DIMENSION_SUM, "{w}x{h} -> {nw}x{nh}");
|
||||||
|
assert!(nw >= 1 && nh >= 1);
|
||||||
|
}
|
||||||
|
// already within limits: no change expected from the caller, but the
|
||||||
|
// helper must not produce zero dimensions.
|
||||||
|
let (nw, nh) = target_dims(500, 400);
|
||||||
|
assert!(nw >= 1 && nh >= 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resize_pix_changes_dimensions() {
|
||||||
|
// 300x200 RGB → 100x66
|
||||||
|
let buf: Vec<u8> = (0..300 * 200 * 3).map(|i| (i % 251) as u8).collect();
|
||||||
|
let resized = resize_pix(PixBuf::Rgb(buf), 300, 200, 100, 66).unwrap();
|
||||||
|
match resized {
|
||||||
|
PixBuf::Rgb(v) => assert_eq!(v.len(), 100 * 66 * 3),
|
||||||
|
other => panic!("expected rgb, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn png_encode_roundtrip_keeps_gray() {
|
||||||
|
let gray = vec![128u8; 4 * 4];
|
||||||
|
let mut out = Vec::new();
|
||||||
|
encode_png(&mut out, &PixBuf::Gray(gray), 4, 4).unwrap();
|
||||||
|
assert!(!out.is_empty());
|
||||||
|
let (_, _, depth, color) = parse_png_header(&out).unwrap();
|
||||||
|
assert_eq!(depth, png::BitDepth::Eight);
|
||||||
|
assert_eq!(color, png::ColorType::Grayscale);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn jpeg_encode_produces_bytes() {
|
||||||
|
let rgb = vec![128u8; 8 * 8 * 3];
|
||||||
|
let out = encode_jpeg(&PixBuf::Rgb(rgb), 8, 8).unwrap();
|
||||||
|
assert!(out.len() > 100);
|
||||||
|
assert!(out.starts_with(&[0xFF, 0xD8]));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a small dimension-oversized PNG (9999x2 → sum 10001) to a temp
|
||||||
|
/// file and runs the full pipeline.
|
||||||
|
fn run_pipeline(w: u32, h: u32, color: png::ColorType, fill: u8) -> Result<PhotoPrep, String> {
|
||||||
|
let (channels, data): (usize, Vec<u8>) = match color {
|
||||||
|
png::ColorType::Grayscale => (1, vec![fill; (w * h) as usize]),
|
||||||
|
png::ColorType::Rgb => (3, vec![fill; (w * h * 3) as usize]),
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
{
|
||||||
|
let mut encoder = png::Encoder::new(&mut bytes, w, h);
|
||||||
|
encoder.set_color(color);
|
||||||
|
encoder.set_depth(png::BitDepth::Eight);
|
||||||
|
let mut writer = encoder.write_header().unwrap();
|
||||||
|
writer.write_image_data(&data).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(data.len(), channels * (w * h) as usize);
|
||||||
|
|
||||||
|
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||||
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
|
prepare_photo(file, &bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pipeline_downscales_oversized_png_keeping_format() {
|
||||||
|
let prep = run_pipeline(9999, 2, png::ColorType::Rgb, 128).unwrap();
|
||||||
|
match prep {
|
||||||
|
PhotoPrep::Upload(file) => {
|
||||||
|
let out = std::fs::read(file.path()).unwrap();
|
||||||
|
let (w, h, depth, color) = parse_png_header(&out).unwrap();
|
||||||
|
assert!(w + h <= PHOTO_MAX_DIMENSION_SUM, "{w}x{h}");
|
||||||
|
assert_eq!(depth, png::BitDepth::Eight);
|
||||||
|
assert_eq!(color, png::ColorType::Rgb);
|
||||||
|
}
|
||||||
|
PhotoPrep::UseFallback => panic!("over-dimension PNG should have been resized"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pipeline_keeps_gray_png_gray() {
|
||||||
|
let prep = run_pipeline(9999, 2, png::ColorType::Grayscale, 200).unwrap();
|
||||||
|
match prep {
|
||||||
|
PhotoPrep::Upload(file) => {
|
||||||
|
let out = std::fs::read(file.path()).unwrap();
|
||||||
|
let (_, _, _, color) = parse_png_header(&out).unwrap();
|
||||||
|
assert_eq!(color, png::ColorType::Grayscale, "gray must not upconvert");
|
||||||
|
}
|
||||||
|
PhotoPrep::UseFallback => panic!("over-dimension gray PNG should have been resized"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pipeline_resizes_oversized_jpeg() {
|
||||||
|
// Build a small over-dimension JPEG with jpeg-encoder.
|
||||||
|
let (w, h) = (9999u16, 2u16);
|
||||||
|
let rgb = vec![90u8; (w as usize) * (h as usize) * 3];
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
{
|
||||||
|
let encoder = jpeg_encoder::Encoder::new(&mut bytes, 90);
|
||||||
|
encoder
|
||||||
|
.encode(&rgb, w, h, jpeg_encoder::ColorType::Rgb)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let mut file = tempfile::Builder::new().suffix(".jpg").tempfile().unwrap();
|
||||||
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
|
match prepare_photo(file, &bytes).unwrap() {
|
||||||
|
PhotoPrep::Upload(file) => {
|
||||||
|
let out = std::fs::read(file.path()).unwrap();
|
||||||
|
assert!(out.starts_with(&[0xFF, 0xD8]), "output must stay jpeg");
|
||||||
|
// 9999x2 downscaled: the buffer length tells the new dims.
|
||||||
|
assert!(out.len() > 100);
|
||||||
|
}
|
||||||
|
PhotoPrep::UseFallback => panic!("over-dimension JPEG should have been resized"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "heavy: generates a >10 MiB PNG (run explicitly)"]
|
||||||
|
fn pipeline_transcodes_oversized_png_to_jpeg() {
|
||||||
|
// 6000x4000 (sum 10000 — under the dimension cap) smooth gradient with
|
||||||
|
// small per-pixel noise: PNG-incompressible (delta filters defeated)
|
||||||
|
// but JPEG-friendly (DCT smooths the small noise). Verified with
|
||||||
|
// ffmpeg: 8000x6000 amp-5 variant is a 59 MB PNG / 3.3 MB JPEG.
|
||||||
|
let (w, h) = (6000u32, 4000u32);
|
||||||
|
let mut rng = 0x1234_5678_9abc_def0u64;
|
||||||
|
let mut data = Vec::with_capacity((w * h * 3) as usize);
|
||||||
|
for y in 0..h {
|
||||||
|
for x in 0..w {
|
||||||
|
let base = (x + y) * 255 / (w + h);
|
||||||
|
rng = rng
|
||||||
|
.wrapping_mul(6364136223846793005)
|
||||||
|
.wrapping_add(1442695040888963407);
|
||||||
|
let n = ((rng >> 33) % 11) as i32 - 5; // noise in [-5, 5]
|
||||||
|
let v = (base as i32 + n).clamp(0, 255) as u8;
|
||||||
|
data.extend_from_slice(&[v, v, v]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
{
|
||||||
|
let mut encoder = png::Encoder::new(&mut bytes, w, h);
|
||||||
|
encoder.set_color(png::ColorType::Rgb);
|
||||||
|
encoder.set_depth(png::BitDepth::Eight);
|
||||||
|
let mut writer = encoder.write_header().unwrap();
|
||||||
|
writer.write_image_data(&data).unwrap();
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
bytes.len() as u64 > MAX_UPLOAD_BYTES,
|
||||||
|
"test needs a >10MiB PNG, got {}",
|
||||||
|
bytes.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||||
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
|
match prepare_photo(file, &bytes).unwrap() {
|
||||||
|
PhotoPrep::Upload(file) => {
|
||||||
|
let out = std::fs::read(file.path()).unwrap();
|
||||||
|
assert!(out.starts_with(&[0xFF, 0xD8]), "must transcode to JPEG");
|
||||||
|
assert!(out.len() as u64 <= MAX_UPLOAD_BYTES);
|
||||||
|
}
|
||||||
|
PhotoPrep::UseFallback => panic!("PNG over the byte cap must transcode to JPEG"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+172
-86
@@ -5,13 +5,14 @@
|
|||||||
//! flow. The Python dict-mutation hack (attempts inside the payload) is
|
//! flow. The Python dict-mutation hack (attempts inside the payload) is
|
||||||
//! replaced by dedicated columns.
|
//! replaced by dedicated columns.
|
||||||
|
|
||||||
|
use crate::db::now_f64;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::{params, Connection, TransactionBehavior};
|
use rusqlite::{Connection, TransactionBehavior, params};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
use std::time::Duration;
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
@@ -29,15 +30,9 @@ const QUEUE_WORKERS: usize = 4;
|
|||||||
pub enum QueueError {
|
pub enum QueueError {
|
||||||
/// Reschedule with the given delay; after `MAX_RETRIES` attempts the task
|
/// Reschedule with the given delay; after `MAX_RETRIES` attempts the task
|
||||||
/// is dead-lettered instead.
|
/// is dead-lettered instead.
|
||||||
Retryable {
|
Retryable { delay_seconds: f64, payload: Value },
|
||||||
delay_seconds: f64,
|
|
||||||
payload: Value,
|
|
||||||
},
|
|
||||||
/// Give up now.
|
/// Give up now.
|
||||||
Permanent {
|
Permanent { message: String, payload: Value },
|
||||||
message: String,
|
|
||||||
payload: Value,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||||
@@ -45,7 +40,7 @@ type Handler = dyn Fn(Value) -> BoxFuture<'static, Result<(), QueueError>> + Sen
|
|||||||
type DeadLetter = dyn Fn(Value, String) -> BoxFuture<'static, ()> + Send + Sync;
|
type DeadLetter = dyn Fn(Value, String) -> BoxFuture<'static, ()> + Send + Sync;
|
||||||
|
|
||||||
pub struct PersistentTaskQueue {
|
pub struct PersistentTaskQueue {
|
||||||
db_path: String,
|
pool: std::sync::Arc<crate::db::DbPool>,
|
||||||
notify: Arc<Notify>,
|
notify: Arc<Notify>,
|
||||||
stop: Arc<AtomicBool>,
|
stop: Arc<AtomicBool>,
|
||||||
worker: Mutex<Vec<JoinHandle<()>>>,
|
worker: Mutex<Vec<JoinHandle<()>>>,
|
||||||
@@ -59,35 +54,40 @@ struct LeasedRow {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Owned worker state so the spawned loop does not borrow the queue handle.
|
/// Owned worker state so the spawned loop does not borrow the queue handle.
|
||||||
|
#[derive(Clone)]
|
||||||
struct QueueWorker {
|
struct QueueWorker {
|
||||||
db_path: String,
|
pool: std::sync::Arc<crate::db::DbPool>,
|
||||||
notify: Arc<Notify>,
|
notify: Arc<Notify>,
|
||||||
stop: Arc<AtomicBool>,
|
stop: Arc<AtomicBool>,
|
||||||
handler: Arc<Handler>,
|
handler: Arc<Handler>,
|
||||||
dead_letter: Arc<DeadLetter>,
|
dead_letter: Arc<DeadLetter>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn now_f64() -> f64 {
|
/// Resets rows left `in_progress` with an expired lock TTL back to `pending`
|
||||||
SystemTime::now()
|
/// so they can be leased again (crash/panic recovery).
|
||||||
.duration_since(UNIX_EPOCH)
|
fn recover_update(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
|
||||||
.map(|d| d.as_secs_f64())
|
conn.execute(
|
||||||
.unwrap_or(0.0)
|
"UPDATE tasks SET status='pending', locked_until=0 WHERE status='in_progress' AND locked_until < ?1",
|
||||||
|
params![now_f64()],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Opens the queue DB with a busy timeout. Handler tasks enqueue while
|
/// Base delay × 2^attempts (attempts = retries already done), capped at 300s.
|
||||||
/// workers lease/update rows concurrently; without the timeout a concurrent
|
/// Applied at the queue layer so the attempt count actually reaches the
|
||||||
/// write fails immediately with SQLITE_BUSY and the operation is lost.
|
/// backoff computation; Telegram `RetryAfter` delays get the same treatment
|
||||||
fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
/// (conservatively larger wait, no API change needed).
|
||||||
let conn = Connection::open(path)?;
|
fn scaled_retry_delay(base: f64, attempts: i32) -> f64 {
|
||||||
conn.busy_timeout(Duration::from_secs(5))?;
|
(base * 2f64.powi(attempts)).min(300.0)
|
||||||
Ok(conn)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ensure_schema(conn: &Connection) -> rusqlite::Result<()> {
|
fn ensure_schema(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
"PRAGMA journal_mode=WAL; \
|
||||||
|
CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
||||||
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
||||||
locked_until REAL NOT NULL, created_at REAL NOT NULL);",
|
locked_until REAL NOT NULL, created_at REAL NOT NULL); \
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tasks_pending ON tasks(status, run_after);",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -108,7 +108,7 @@ impl PersistentTaskQueue {
|
|||||||
log::error!("failed to initialize queue schema: {e}");
|
log::error!("failed to initialize queue schema: {e}");
|
||||||
}
|
}
|
||||||
Self {
|
Self {
|
||||||
db_path: db_path.to_string(),
|
pool: std::sync::Arc::new(crate::db::DbPool::new(db_path)),
|
||||||
notify: Arc::new(Notify::new()),
|
notify: Arc::new(Notify::new()),
|
||||||
stop: Arc::new(AtomicBool::new(false)),
|
stop: Arc::new(AtomicBool::new(false)),
|
||||||
worker: Mutex::new(Vec::new()),
|
worker: Mutex::new(Vec::new()),
|
||||||
@@ -129,17 +129,43 @@ impl PersistentTaskQueue {
|
|||||||
let dead_letter: Arc<DeadLetter> =
|
let dead_letter: Arc<DeadLetter> =
|
||||||
Arc::new(move |payload, message| Box::pin(dead_letter(payload, message)));
|
Arc::new(move |payload, message| Box::pin(dead_letter(payload, message)));
|
||||||
self.recover_stale().await;
|
self.recover_stale().await;
|
||||||
let mut handles = Vec::with_capacity(QUEUE_WORKERS);
|
let mut handles = Vec::with_capacity(QUEUE_WORKERS + 1);
|
||||||
for _ in 0..QUEUE_WORKERS {
|
for _ in 0..QUEUE_WORKERS {
|
||||||
let worker = QueueWorker {
|
let worker = QueueWorker {
|
||||||
db_path: self.db_path.clone(),
|
pool: std::sync::Arc::clone(&self.pool),
|
||||||
notify: Arc::clone(&self.notify),
|
notify: Arc::clone(&self.notify),
|
||||||
stop: Arc::clone(&self.stop),
|
stop: Arc::clone(&self.stop),
|
||||||
handler: Arc::clone(&handler),
|
handler: Arc::clone(&handler),
|
||||||
dead_letter: Arc::clone(&dead_letter),
|
dead_letter: Arc::clone(&dead_letter),
|
||||||
};
|
};
|
||||||
handles.push(tokio::spawn(worker.run_loop()));
|
handles.push(tokio::spawn(worker.run_loop_supervised()));
|
||||||
}
|
}
|
||||||
|
// Periodic lease-expiry sweep: recovers rows a crashed/panicked
|
||||||
|
// worker left `in_progress` (the lock TTL bounds the wait). Woken by
|
||||||
|
// the same notify as the workers, so enqueue and stop interrupt the
|
||||||
|
// sleep; the first interval tick fires immediately (harmless extra
|
||||||
|
// recovery at startup).
|
||||||
|
let sweep_pool = std::sync::Arc::clone(&self.pool);
|
||||||
|
let sweep_notify = Arc::clone(&self.notify);
|
||||||
|
let sweep_stop = Arc::clone(&self.stop);
|
||||||
|
handles.push(tokio::spawn(async move {
|
||||||
|
let mut interval = tokio::time::interval(Duration::from_secs(30));
|
||||||
|
loop {
|
||||||
|
let notified = sweep_notify.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
tokio::select! {
|
||||||
|
_ = &mut notified => {}
|
||||||
|
_ = interval.tick() => {}
|
||||||
|
}
|
||||||
|
if sweep_stop.load(Ordering::Relaxed) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let result = sweep_pool.with_conn(move |conn| recover_update(conn)).await;
|
||||||
|
if let Err(e) = result {
|
||||||
|
log::error!("queue sweep failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}));
|
||||||
*self.worker.lock() = handles;
|
*self.worker.lock() = handles;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,10 +188,8 @@ impl PersistentTaskQueue {
|
|||||||
self.counter.fetch_add(1, Ordering::Relaxed)
|
self.counter.fetch_add(1, Ordering::Relaxed)
|
||||||
);
|
);
|
||||||
let payload = payload.to_string();
|
let payload = payload.to_string();
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
log::info!("enqueued {id} (run_after {run_after:.1})");
|
log::info!("enqueued {id} (run_after {run_after:.1})");
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
self.pool.with_conn(move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO tasks (id, payload, run_after, attempts, status, locked_until, created_at) \
|
"INSERT OR REPLACE INTO tasks (id, payload, run_after, attempts, status, locked_until, created_at) \
|
||||||
VALUES (?1, ?2, ?3, 0, 'pending', 0, ?4)",
|
VALUES (?1, ?2, ?3, 0, 'pending', 0, ?4)",
|
||||||
@@ -173,36 +197,46 @@ impl PersistentTaskQueue {
|
|||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await?;
|
||||||
.expect("queue insert worker panicked")?;
|
// `notify_one` stores a permit when no worker is registered, so a
|
||||||
// Wake every sleeping worker: with several workers the one that finds
|
// notification fired between a worker's DB reads and its `notified()`
|
||||||
// nothing due must not starve the newly inserted row.
|
// registration is not lost (notify_waiters would drop it). The
|
||||||
self.notify.notify_waiters();
|
// awakened worker re-leases and finds the new row.
|
||||||
|
self.notify.notify_one();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn recover_stale(&self) {
|
async fn recover_stale(&self) {
|
||||||
let db_path = self.db_path.clone();
|
self.recover_sweep().await;
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
}
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
async fn recover_sweep(&self) {
|
||||||
"UPDATE tasks SET status='pending', locked_until=0 WHERE status='in_progress' AND locked_until < ?1",
|
let result = self.pool.with_conn(move |conn| recover_update(conn)).await;
|
||||||
params![now_f64()],
|
if let Err(e) = result {
|
||||||
)?;
|
log::error!("queue recovery failed: {e}");
|
||||||
Ok(())
|
}
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("queue recovery worker panicked")
|
|
||||||
.unwrap_or_else(|e| log::error!("queue recovery failed: {e}"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl QueueWorker {
|
impl QueueWorker {
|
||||||
|
/// Supervised worker: the inner loop runs in its own task so a panic
|
||||||
|
/// (e.g. inside a handler or a DB closure) kills only that task; the
|
||||||
|
/// supervisor respawns it until stop is set. The row a dead worker had
|
||||||
|
/// leased is recovered by the periodic sweep once its lock TTL expires.
|
||||||
|
async fn run_loop_supervised(self) {
|
||||||
|
while !self.stop.load(Ordering::Relaxed) {
|
||||||
|
let worker = self.clone();
|
||||||
|
if let Err(e) = tokio::spawn(async move { worker.run_loop().await }).await {
|
||||||
|
log::error!("queue worker panicked, restarting: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn run_loop(self) {
|
async fn run_loop(self) {
|
||||||
while !self.stop.load(Ordering::Relaxed) {
|
while !self.stop.load(Ordering::Relaxed) {
|
||||||
match self.lease_next().await {
|
match self.lease_next().await {
|
||||||
Some(row) => self.process(row).await,
|
Ok(Some(row)) => self.process(row).await,
|
||||||
None => {
|
Ok(None) => {
|
||||||
let wait_until = self.earliest_run_after().await;
|
let wait_until = self.earliest_run_after().await;
|
||||||
let notified = self.notify.notified();
|
let notified = self.notify.notified();
|
||||||
tokio::pin!(notified);
|
tokio::pin!(notified);
|
||||||
@@ -219,15 +253,20 @@ impl QueueWorker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A lease failure while rows are due would otherwise loop
|
||||||
|
// with sleep(0) and hammer SQLite; back off briefly.
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("queue lease failed: {e}");
|
||||||
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Leases the oldest due row (sets it `in_progress` with a lock TTL).
|
/// Leases the oldest due row (sets it `in_progress` with a lock TTL).
|
||||||
async fn lease_next(&self) -> Option<LeasedRow> {
|
/// Errors are surfaced so the caller can back off instead of spinning.
|
||||||
let db_path = self.db_path.clone();
|
async fn lease_next(&self) -> Result<Option<LeasedRow>, rusqlite::Error> {
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<LeasedRow>> {
|
self.pool.with_conn(|conn| {
|
||||||
let mut conn = open_db(&db_path)?;
|
|
||||||
// BEGIN IMMEDIATE: with several workers, a deferred transaction
|
// BEGIN IMMEDIATE: with several workers, a deferred transaction
|
||||||
// that read before another worker's lease commit would fail with
|
// that read before another worker's lease commit would fail with
|
||||||
// SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes
|
// SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes
|
||||||
@@ -266,30 +305,28 @@ impl QueueWorker {
|
|||||||
}))
|
}))
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("queue lease worker panicked")
|
|
||||||
.unwrap_or_else(|e| {
|
|
||||||
log::error!("queue lease failed: {e}");
|
|
||||||
None
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn earliest_run_after(&self) -> Option<f64> {
|
async fn earliest_run_after(&self) -> Option<f64> {
|
||||||
let db_path = self.db_path.clone();
|
let result = self
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<f64>> {
|
.pool
|
||||||
let conn = open_db(&db_path)?;
|
.with_conn(|conn| {
|
||||||
let mut stmt = conn.prepare("SELECT MIN(run_after) FROM tasks WHERE status='pending'")?;
|
let mut stmt =
|
||||||
|
conn.prepare("SELECT MIN(run_after) FROM tasks WHERE status='pending'")?;
|
||||||
let mut rows = stmt.query([])?;
|
let mut rows = stmt.query([])?;
|
||||||
match rows.next()? {
|
match rows.next()? {
|
||||||
Some(row) => Ok(row.get::<_, Option<f64>>(0)?),
|
Some(row) => Ok(row.get::<_, Option<f64>>(0)?),
|
||||||
None => Ok(None),
|
None => Ok(None),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue timing worker panicked")
|
match result {
|
||||||
.unwrap_or_else(|e| {
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
log::error!("queue timing query failed: {e}");
|
log::error!("queue timing query failed: {e}");
|
||||||
None
|
None
|
||||||
})
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process(&self, row: LeasedRow) {
|
async fn process(&self, row: LeasedRow) {
|
||||||
@@ -318,12 +355,13 @@ impl QueueWorker {
|
|||||||
self.delete_row(&row.id).await;
|
self.delete_row(&row.id).await;
|
||||||
(self.dead_letter)(payload, message).await;
|
(self.dead_letter)(payload, message).await;
|
||||||
} else {
|
} else {
|
||||||
|
let delay = scaled_retry_delay(delay_seconds, row.attempts);
|
||||||
log::info!(
|
log::info!(
|
||||||
"task {} rescheduled in {delay_seconds:.1}s (attempt {})",
|
"task {} rescheduled in {delay:.1}s (attempt {})",
|
||||||
row.id,
|
row.id,
|
||||||
row.attempts + 1
|
row.attempts + 1
|
||||||
);
|
);
|
||||||
self.reschedule(&row.id, payload, delay_seconds, row.attempts + 1)
|
self.reschedule(&row.id, payload, delay, row.attempts + 1)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -336,34 +374,35 @@ impl QueueWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_row(&self, id: &str) {
|
async fn delete_row(&self, id: &str) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let id = id.to_string();
|
let id = id.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = self
|
||||||
let conn = open_db(&db_path)?;
|
.pool
|
||||||
|
.with_conn(move |conn| {
|
||||||
conn.execute("DELETE FROM tasks WHERE id = ?1", params![id])?;
|
conn.execute("DELETE FROM tasks WHERE id = ?1", params![id])?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue delete worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("queue delete failed: {e}"));
|
log::error!("queue delete failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reschedule(&self, id: &str, payload: Value, delay_seconds: f64, attempts: i32) {
|
async fn reschedule(&self, id: &str, payload: Value, delay_seconds: f64, attempts: i32) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let id = id.to_string();
|
let id = id.to_string();
|
||||||
let payload = payload.to_string();
|
let payload = payload.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = self.pool.with_conn(move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE tasks SET payload=?1, run_after=?2, attempts=?3, status='pending', locked_until=0 WHERE id=?4",
|
"UPDATE tasks SET payload=?1, run_after=?2, attempts=?3, status='pending', locked_until=0 WHERE id=?4",
|
||||||
params![payload, now_f64() + delay_seconds, attempts, id],
|
params![payload, now_f64() + delay_seconds, attempts, id],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue reschedule worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("queue reschedule failed: {e}"));
|
log::error!("queue reschedule failed: {e}");
|
||||||
self.notify.notify_waiters();
|
}
|
||||||
|
// Same permit semantics as enqueue: never lose the wakeup.
|
||||||
|
self.notify.notify_one();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -372,6 +411,16 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
|
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scaled_retry_delay_scales_and_caps() {
|
||||||
|
assert_eq!(scaled_retry_delay(1.0, 0), 1.0);
|
||||||
|
assert_eq!(scaled_retry_delay(1.0, 1), 2.0);
|
||||||
|
assert_eq!(scaled_retry_delay(1.0, 2), 4.0);
|
||||||
|
assert_eq!(scaled_retry_delay(1.5, 1), 3.0);
|
||||||
|
assert_eq!(scaled_retry_delay(1.0, 10), 300.0, "capped at 300s");
|
||||||
|
assert_eq!(scaled_retry_delay(300.0, 0), 300.0);
|
||||||
|
}
|
||||||
|
|
||||||
async fn new_queue() -> (PersistentTaskQueue, tempfile::TempDir) {
|
async fn new_queue() -> (PersistentTaskQueue, tempfile::TempDir) {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
let path = dir.path().join("queue.db");
|
let path = dir.path().join("queue.db");
|
||||||
@@ -510,4 +559,41 @@ mod tests {
|
|||||||
assert_eq!(calls.load(AtomicOrdering::SeqCst), 1);
|
assert_eq!(calls.load(AtomicOrdering::SeqCst), 1);
|
||||||
queue.stop().await;
|
queue.stop().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn runtime_sweep_recovers_expired_lease() {
|
||||||
|
let (queue, _dir) = new_queue().await;
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let c = calls.clone();
|
||||||
|
queue
|
||||||
|
.start(
|
||||||
|
move |payload| {
|
||||||
|
assert_eq!(payload["s"], 1);
|
||||||
|
c.fetch_add(1, AtomicOrdering::SeqCst);
|
||||||
|
async { Ok(()) }
|
||||||
|
},
|
||||||
|
|_payload, _message| async {},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
// Insert a stale leased row AFTER startup: without a runtime sweep it
|
||||||
|
// would stay `in_progress` forever (only start() used to recover).
|
||||||
|
{
|
||||||
|
let conn = Connection::open(queue.pool.path()).unwrap();
|
||||||
|
ensure_schema(&conn).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO tasks (id, payload, run_after, attempts, status, locked_until, created_at) \
|
||||||
|
VALUES ('task_stale_runtime', '{\"s\":1}', 0, 0, 'in_progress', ?1, 0)",
|
||||||
|
params![now_f64() - 1000.0],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
queue.recover_sweep().await;
|
||||||
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||||
|
assert_eq!(
|
||||||
|
calls.load(AtomicOrdering::SeqCst),
|
||||||
|
1,
|
||||||
|
"expired lease must be recovered and processed exactly once"
|
||||||
|
);
|
||||||
|
queue.stop().await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+549
-142
File diff suppressed because it is too large
Load Diff
+116
-27
@@ -2,10 +2,11 @@
|
|||||||
//! `data/task_queue.db`, shared with the task queue).
|
//! `data/task_queue.db`, shared with the task queue).
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::{params, Connection};
|
use rusqlite::params;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Default, Clone, Debug)]
|
#[derive(Serialize, Deserialize, Default, Clone, Debug)]
|
||||||
@@ -34,7 +35,10 @@ pub struct EditMessage {
|
|||||||
pub struct ChatStore {
|
pub struct ChatStore {
|
||||||
/// In-memory cache; the DB is the source of truth on first access.
|
/// In-memory cache; the DB is the source of truth on first access.
|
||||||
cache: Mutex<HashMap<i64, ChatData>>,
|
cache: Mutex<HashMap<i64, ChatData>>,
|
||||||
db_path: String,
|
/// Per-chat async locks serializing get→mutate→set so concurrent handler
|
||||||
|
/// tasks (batch-forwards, callbacks) cannot clobber each other's writes.
|
||||||
|
locks: Mutex<HashMap<i64, Arc<tokio::sync::Mutex<()>>>>,
|
||||||
|
pool: crate::db::DbPool,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn unix_now() -> i64 {
|
pub fn unix_now() -> i64 {
|
||||||
@@ -45,7 +49,9 @@ pub fn unix_now() -> i64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ChatStore {
|
impl ChatStore {
|
||||||
/// Creates the parent directory and both tables (idempotent).
|
/// Creates the parent directory and the `chat_state` table (idempotent).
|
||||||
|
/// The shared `tasks` / `link_cache` tables are owned by `queue.rs` and
|
||||||
|
/// `link_cache.rs` respectively.
|
||||||
pub fn open(path: &str) -> rusqlite::Result<Self> {
|
pub fn open(path: &str) -> rusqlite::Result<Self> {
|
||||||
if let Some(parent) = Path::new(path).parent()
|
if let Some(parent) = Path::new(path).parent()
|
||||||
&& !parent.as_os_str().is_empty()
|
&& !parent.as_os_str().is_empty()
|
||||||
@@ -53,17 +59,15 @@ impl ChatStore {
|
|||||||
std::fs::create_dir_all(parent)
|
std::fs::create_dir_all(parent)
|
||||||
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?;
|
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?;
|
||||||
}
|
}
|
||||||
let conn = Connection::open(path)?;
|
let conn = crate::db::open_db(path)?;
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
"CREATE TABLE IF NOT EXISTS chat_state (chat_id TEXT PRIMARY KEY, payload TEXT NOT NULL);",
|
||||||
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
|
||||||
locked_until REAL NOT NULL, created_at REAL NOT NULL); \
|
|
||||||
CREATE TABLE IF NOT EXISTS chat_state (chat_id TEXT PRIMARY KEY, payload TEXT NOT NULL);",
|
|
||||||
)?;
|
)?;
|
||||||
drop(conn);
|
drop(conn);
|
||||||
Ok(ChatStore {
|
Ok(ChatStore {
|
||||||
cache: Mutex::new(HashMap::new()),
|
cache: Mutex::new(HashMap::new()),
|
||||||
db_path: path.to_string(),
|
locks: Mutex::new(HashMap::new()),
|
||||||
|
pool: crate::db::DbPool::new(path),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,22 +75,21 @@ impl ChatStore {
|
|||||||
if let Some(data) = self.cache.lock().get(&chat_id) {
|
if let Some(data) = self.cache.lock().get(&chat_id) {
|
||||||
return data.clone();
|
return data.clone();
|
||||||
}
|
}
|
||||||
let db_path = self.db_path.clone();
|
let chat_key = chat_id.to_string();
|
||||||
let payload = tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<String>> {
|
let payload = self
|
||||||
let conn = Connection::open(&db_path)?;
|
.pool
|
||||||
|
.with_conn(move |conn| {
|
||||||
// Concurrent handler tasks (batch-forwards) may write chat_state
|
// Concurrent handler tasks (batch-forwards) may write chat_state
|
||||||
// while this read runs; without a busy timeout a write lock
|
// while this read runs; the shared busy timeout handles the
|
||||||
// collision fails the query immediately.
|
// write-lock collision instead of failing the query.
|
||||||
conn.busy_timeout(std::time::Duration::from_secs(5))?;
|
|
||||||
let mut stmt = conn.prepare("SELECT payload FROM chat_state WHERE chat_id = ?1")?;
|
let mut stmt = conn.prepare("SELECT payload FROM chat_state WHERE chat_id = ?1")?;
|
||||||
let mut rows = stmt.query(params![chat_id.to_string()])?;
|
let mut rows = stmt.query(params![chat_key])?;
|
||||||
match rows.next()? {
|
match rows.next()? {
|
||||||
Some(row) => Ok(Some(row.get(0)?)),
|
Some(row) => Ok(Some(row.get::<_, String>(0)?)),
|
||||||
None => Ok(None),
|
None => Ok(None),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("chat_state worker panicked")
|
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
log::error!("chat_state read failed: {e}");
|
log::error!("chat_state read failed: {e}");
|
||||||
None
|
None
|
||||||
@@ -101,19 +104,40 @@ impl ChatStore {
|
|||||||
pub async fn set(&self, chat_id: i64, data: &ChatData) {
|
pub async fn set(&self, chat_id: i64, data: &ChatData) {
|
||||||
self.cache.lock().insert(chat_id, data.clone());
|
self.cache.lock().insert(chat_id, data.clone());
|
||||||
let payload = serde_json::to_string(data).expect("chat state serializes");
|
let payload = serde_json::to_string(data).expect("chat state serializes");
|
||||||
let db_path = self.db_path.clone();
|
let chat_id = chat_id.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = self
|
||||||
let conn = Connection::open(&db_path)?;
|
.pool
|
||||||
conn.busy_timeout(std::time::Duration::from_secs(5))?;
|
.with_conn(move |conn| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO chat_state (chat_id, payload) VALUES (?1, ?2)",
|
"INSERT OR REPLACE INTO chat_state (chat_id, payload) VALUES (?1, ?2)",
|
||||||
params![chat_id.to_string(), payload],
|
params![chat_id, payload],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("chat_state worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("chat_state write failed: {e}"));
|
log::error!("chat_state write failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serializes a get→mutate→set cycle per chat: concurrent handler tasks
|
||||||
|
/// (the batch-forward design spawns several per chat) each snapshot the
|
||||||
|
/// same `ChatData` and last-writer-wins would silently drop mutations,
|
||||||
|
/// e.g. a second `edit_message` record. The per-chat lock makes the
|
||||||
|
/// cycle atomic. Returns the closure's result.
|
||||||
|
pub async fn update<R>(&self, chat_id: i64, f: impl FnOnce(&mut ChatData) -> R) -> R {
|
||||||
|
let lock = {
|
||||||
|
let mut locks = self.locks.lock();
|
||||||
|
locks
|
||||||
|
.entry(chat_id)
|
||||||
|
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||||
|
.clone()
|
||||||
|
};
|
||||||
|
let _guard = lock.lock().await;
|
||||||
|
let mut data = self.get(chat_id).await;
|
||||||
|
let r = f(&mut data);
|
||||||
|
self.set(chat_id, &data).await;
|
||||||
|
r
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes edit-before-forward records whose `created_at + ttl` is in the
|
/// Removes edit-before-forward records whose `created_at + ttl` is in the
|
||||||
@@ -123,6 +147,10 @@ impl ChatStore {
|
|||||||
let now = unix_now();
|
let now = unix_now();
|
||||||
let ttl_secs = ttl.as_secs() as i64;
|
let ttl_secs = ttl.as_secs() as i64;
|
||||||
let mut removed = Vec::new();
|
let mut removed = Vec::new();
|
||||||
|
// Chats with no live edit records: evicted from the cache (and their
|
||||||
|
// per-chat lock) so the cache stays bounded to active prompts. The DB
|
||||||
|
// keeps the row; the next get() reloads it.
|
||||||
|
let mut evicted_chats = Vec::new();
|
||||||
let changed: Vec<(i64, ChatData)> = {
|
let changed: Vec<(i64, ChatData)> = {
|
||||||
let mut cache = self.cache.lock();
|
let mut cache = self.cache.lock();
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
@@ -139,18 +167,79 @@ impl ChatStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if kept.len() != data.edit_message.len() {
|
if kept.len() != data.edit_message.len() {
|
||||||
|
// Persist the pruned row (removes expired records from
|
||||||
|
// the DB too, not just the cache).
|
||||||
data.edit_message = kept;
|
data.edit_message = kept;
|
||||||
out.push((*chat_id, data.clone()));
|
out.push((*chat_id, data.clone()));
|
||||||
}
|
}
|
||||||
|
if data.edit_message.is_empty() {
|
||||||
|
evicted_chats.push(*chat_id);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
// Lock order: update() takes the per-chat lock before the cache
|
||||||
|
// lock, so prune must not hold the cache lock while taking locks.
|
||||||
|
drop(cache);
|
||||||
out
|
out
|
||||||
};
|
};
|
||||||
for (chat_id, data) in changed {
|
for (chat_id, data) in changed {
|
||||||
self.set(chat_id, &data).await;
|
self.set(chat_id, &data).await;
|
||||||
}
|
}
|
||||||
|
if !evicted_chats.is_empty() {
|
||||||
|
let mut cache = self.cache.lock();
|
||||||
|
let mut locks = self.locks.lock();
|
||||||
|
for chat_id in &evicted_chats {
|
||||||
|
cache.remove(chat_id);
|
||||||
|
locks.remove(chat_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
if !removed.is_empty() {
|
if !removed.is_empty() {
|
||||||
log::info!("pruned {} expired edit-before-forward record(s)", removed.len());
|
log::info!(
|
||||||
|
"pruned {} expired edit-before-forward record(s)",
|
||||||
|
removed.len()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
removed
|
removed
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn concurrent_updates_do_not_lose_edit_records() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = std::sync::Arc::new(
|
||||||
|
ChatStore::open(dir.path().join("s.db").to_str().unwrap()).unwrap(),
|
||||||
|
);
|
||||||
|
let mut handles = Vec::new();
|
||||||
|
for i in 0..4 {
|
||||||
|
let store = Arc::clone(&store);
|
||||||
|
handles.push(tokio::spawn(async move {
|
||||||
|
store
|
||||||
|
.update(1001, |data| {
|
||||||
|
data.edit_message.insert(
|
||||||
|
i,
|
||||||
|
EditMessage {
|
||||||
|
url: format!("https://x.com/u/status/{i}"),
|
||||||
|
chat_id: 1001,
|
||||||
|
forward_message_ids: vec![i],
|
||||||
|
template: String::new(),
|
||||||
|
created_at: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
for h in handles {
|
||||||
|
h.await.unwrap();
|
||||||
|
}
|
||||||
|
let data = store.get(1001).await;
|
||||||
|
assert_eq!(
|
||||||
|
data.edit_message.len(),
|
||||||
|
4,
|
||||||
|
"concurrent get→mutate→set must not drop records"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+23
-19
@@ -5,30 +5,25 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- '80:80'
|
- '80:80'
|
||||||
- '443:443'
|
- '443:443'
|
||||||
environment:
|
|
||||||
# Bare-IP access only.
|
|
||||||
# DEFAULT_HOST: 'bot.example.com'
|
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/tmp/docker.sock:ro
|
- /var/run/docker.sock:/tmp/docker.sock:ro
|
||||||
- ./nginx-certs:/etc/nginx/certs:ro
|
- certs:/etc/nginx/certs:ro
|
||||||
- ./nginx-vhost.d:/etc/nginx/vhost.d:ro
|
- html:/usr/share/nginx/html:ro
|
||||||
- ./nginx-html:/usr/share/nginx/html:ro
|
|
||||||
networks: [proxy]
|
networks: [proxy]
|
||||||
labels:
|
labels:
|
||||||
- 'com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy=true'
|
- 'com.github.nginx-proxy.nginx'
|
||||||
container_name: nginx-proxy
|
container_name: nginx-proxy
|
||||||
|
|
||||||
acme-companion:
|
acme-companion:
|
||||||
image: nginxproxy/acme-companion
|
image: nginxproxy/acme-companion
|
||||||
restart: always
|
restart: always
|
||||||
environment:
|
environment:
|
||||||
DEFAULT_EMAIL: 'admin@yoursfunny.top'
|
DEFAULT_EMAIL: ''
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
- ./nginx-certs:/etc/nginx/certs:rw
|
- certs:/etc/nginx/certs:rw
|
||||||
- ./nginx-vhost.d:/etc/nginx/vhost.d:rw
|
- html:/usr/share/nginx/html:rw
|
||||||
- ./nginx-html:/usr/share/nginx/html:rw
|
- acme:/etc/acme.sh
|
||||||
- ./nginx-acme:/etc/acme.sh
|
|
||||||
networks: [proxy]
|
networks: [proxy]
|
||||||
container_name: acme-companion
|
container_name: acme-companion
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -42,21 +37,17 @@ services:
|
|||||||
TELOXIDE_TOKEN: ''
|
TELOXIDE_TOKEN: ''
|
||||||
BOT_ADMIN: ''
|
BOT_ADMIN: ''
|
||||||
PIXIV_REFRESH_TOKEN: ''
|
PIXIV_REFRESH_TOKEN: ''
|
||||||
# Optional: x.com session cookie (auth_token) — fetches NSFW tweets
|
|
||||||
# that the public syndication endpoint withholds.
|
|
||||||
TWITTER_AUTH_TOKEN: ''
|
TWITTER_AUTH_TOKEN: ''
|
||||||
EDIT_MESSAGE_TTL_SECONDS: '86400'
|
EDIT_MESSAGE_TTL_SECONDS: '86400'
|
||||||
# Link-result cache TTL (default 604800 = 7 days).
|
|
||||||
LINK_CACHE_TTL_SECONDS: '604800'
|
LINK_CACHE_TTL_SECONDS: '604800'
|
||||||
RUST_LOG: 'info'
|
RUST_LOG: 'info'
|
||||||
VIRTUAL_HOST: 'bot.example.com'
|
VIRTUAL_HOST: '<YOUR_DOMAIN>'
|
||||||
VIRTUAL_PORT: '8443'
|
VIRTUAL_PORT: '8443'
|
||||||
# LETSENCRYPT_HOST: 'bot.example.com'
|
# ACME_HOST: 'your.domain.com'
|
||||||
WEBHOOK: 'true'
|
WEBHOOK: 'true'
|
||||||
WEBHOOK_LISTEN: '0.0.0.0'
|
WEBHOOK_LISTEN: '0.0.0.0'
|
||||||
WEBHOOK_PORT: '8443'
|
WEBHOOK_PORT: '8443'
|
||||||
WEBHOOK_URL: 'https://bot.example.com/'
|
WEBHOOK_URL: 'https://<YOUR_DOMAIN>/'
|
||||||
# WEBHOOK_CERT: './cert/cert.pem'
|
|
||||||
WEBHOOK_SECRET_TOKEN: ''
|
WEBHOOK_SECRET_TOKEN: ''
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
@@ -64,6 +55,19 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
- nginx-proxy
|
- nginx-proxy
|
||||||
container_name: tgxmb
|
container_name: tgxmb
|
||||||
|
# Webhook mode only: the bot listens on WEBHOOK_PORT; nginx-proxy shows
|
||||||
|
# 502s while this is down, so surface it to the orchestrator.
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8443'"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
certs:
|
||||||
|
html:
|
||||||
|
acme:
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
proxy:
|
proxy:
|
||||||
|
|||||||
+14
-3
@@ -5,9 +5,20 @@ if [ "$(id -u)" -eq '0' ]
|
|||||||
then
|
then
|
||||||
USER_ID=${LOCAL_USER_ID:-9001}
|
USER_ID=${LOCAL_USER_ID:-9001}
|
||||||
|
|
||||||
useradd --shell /bin/bash -u ${USER_ID} -o -c "" -m user > /dev/null 2>&1
|
# `docker compose restart` / `docker restart` reuse the same container, so
|
||||||
usermod -a -G root user > /dev/null 2>&1
|
# the overlay fs keeps the user created on first boot. A second `useradd`
|
||||||
chown -R `id -u user`:`id -u user` /app > /dev/null 2>&1
|
# then fails with exit code 9, which would trip `set -e` and kill the
|
||||||
|
# container on every restart. Create only if missing; align the UID
|
||||||
|
# otherwise so LOCAL_USER_ID changes still apply.
|
||||||
|
if ! id user > /dev/null 2>&1
|
||||||
|
then
|
||||||
|
useradd --shell /bin/bash -u ${USER_ID} -o -c "" -m user > /dev/null 2>&1 || true
|
||||||
|
else
|
||||||
|
usermod -u ${USER_ID} -o user > /dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
# Bind-mounted volumes may not support chown; a failure here must not kill
|
||||||
|
# the container either.
|
||||||
|
chown -R `id -u user`:`id -u user` /app > /dev/null 2>&1 || true
|
||||||
|
|
||||||
export HOME=/home/user
|
export HOME=/home/user
|
||||||
# setpriv (util-linux, present in bookworm-slim) replaces gosu: drop to the
|
# setpriv (util-linux, present in bookworm-slim) replaces gosu: drop to the
|
||||||
|
|||||||
Reference in New Issue
Block a user