From 12a065846c3fd6b1b69e9837d48930afbb808a93 Mon Sep 17 00:00:00 2001 From: YoursFunny Date: Sun, 16 Aug 2026 17:07:33 +0800 Subject: [PATCH] feat(statics): make the SQLite path configurable via DATA_DIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DB file was hardcoded to CWD-relative data/task_queue.db — a footgun for systemd/cron deployments and a confusing startup failure when the data/ dir did not exist (SQLite never creates parent dirs). db_path() now reads DATA_DIR (default data, CWD-relative, unchanged for local runs and the docker-compose ./data mount) and creates the directory automatically. README/README.en.md env tables and AGENTS.md document the new variable. --- AGENTS.md | 8 ++++---- README.en.md | 1 + README.md | 1 + crates/xmedia-bot/src/handlers/statics.rs | 20 ++++++++++++++++++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e9074e0..ba19782 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ The `x-media` library: `site::fetch(url)` dispatches through the `SITES` registr | `crates/x-media/src/site//` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, `cache_key`/`is_retryable`/`media_headers`, unit struct `Site` implementing `site::Site`, `From for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`). Twitter's `from_syndication_json` HTML-decodes the API text — syndication and GraphQL `full_text` both arrive pre-escaped (`>` `<` `&` `'`) — so the stored text is raw and the caption escapes exactly once | | `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, command registration (`register_commands`), shared `send::BOT` force-init, queue worker start, site login validation (`site::validate_all`), 300 s edit-expiry sweep, dptree handler tree, webhook vs polling dispatch | | `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` | -| `crates/xmedia-bot/src/db.rs` | `DbPool`: per-store SQLite connection pool (`POOL_SIZE = 4`, WAL, busy_timeout) over `data/task_queue.db`; `open_store` creates file + schema, `with_conn` runs all rusqlite I/O in `spawn_blocking` | +| `crates/xmedia-bot/src/db.rs` | `DbPool`: per-store SQLite connection pool (`POOL_SIZE = 4`, WAL, busy_timeout) over `$DATA_DIR/task_queue.db` (default `data/`); `open_store` creates file + schema, `with_conn` runs all rusqlite I/O in `spawn_blocking` | | `crates/xmedia-bot/src/handlers/` | Handler modules: `mod.rs` (message entry point, `reply`, `log_key`), `commands.rs` (teloxide `BotCommands` enum + command executor, incl. the `/test ` parse-only debug command), `urls.rs` (URL extraction + bounded job channel (256) drained by `URL_WORKERS = 8` workers (`start_url_workers`) — backpressure instead of unbounded spawns; teloxide's per-chat workers are sequential — batch-forwards need concurrency), `inline.rs`/`callback.rs` (inline queries / edit-before-forward buttons), `statics.rs` (global statics) | | `crates/xmedia-bot/src/state.rs` | `ChatStore`: parking_lot `Mutex` cache + SQLite write-through (`chat_state` table) | | `crates/xmedia-bot/src/link_cache.rs` | `LinkCache`: SQLite-backed cache (`link_cache` table) of successfully sent posts — raw caption fields + Telegram `file_id`s; repeat links re-send locally (no fetch/upload), TTL + prune, invalidated on permanent send failure | @@ -72,7 +72,7 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi | File | Why it matters | |---|---| | `crates/xmedia-bot/src/main.rs` | Startup sequence, webhook vs polling, graceful shutdown (SIGINT via teloxide ctrlc / SIGTERM via `stop_token` for docker, → sweep stop → admin msg → queue stop) | -| `crates/xmedia-bot/src/handlers/` | `statics.rs` = `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `data/task_queue.db` **relative to CWD**); `commands.rs` = command dispatch (incl. the `/test ` parse-only debug command); `urls.rs` = URL extraction + retry enqueue; `inline.rs`/`callback.rs` = inline queries / edit-before-forward buttons | +| `crates/xmedia-bot/src/handlers/` | `statics.rs` = `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `$DATA_DIR/task_queue.db`, default `data/` **relative to CWD**, dir auto-created); `commands.rs` = command dispatch (incl. the `/test ` parse-only debug command); `urls.rs` = URL extraction + retry enqueue; `inline.rs`/`callback.rs` = inline queries / edit-before-forward buttons | | `crates/xmedia-bot/src/send.rs` | Constants `MAX_MEDIA_GROUP = 9`; fallback chain; `classify_request_error`; download-and-reupload fallback triggered only by Telegram API errors (`is_media_fetch_failure` / `is_size_error`) | | `crates/xmedia-bot/src/photo.rs` | Pure-Rust photo processing (no ffmpeg): `png` (image-png) decode/encode + `zune-jpeg` decode + `fast_image_resize` Lanczos3 downscale + `jpeg-encoder`. Photos over Telegram's limits (width + height > 10000 px → `PHOTO_INVALID_DIMENSIONS`; bytes > 10 MiB) are decoded, downscaled keeping the format, PNG bit depth > 24 (RGBA 32-bit / 16-bit per channel) reduced to 24-bit RGB with alpha flattened white (≤24-bit untouched, never upconverted), and transcoded to JPEG only if still over the cap; memory budget guarded, otherwise the item's smaller fallback URL | | `crates/x-media/src/site/mod.rs` | Dispatcher, `Fetched`/`FetchError`, shared `CLIENT`, `download_media` (adds `Referer: https://www.pixiv.net/` for `pximg.net` hotlink protection) | @@ -89,8 +89,8 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi - Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently. - **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls"]` (webpki-roots baked in, so the image ships no CA bundle). One reqwest 0.12.28 in the lock. - **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build). -- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only). -- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `data/task_queue.db` is CWD-relative — run from the workspace root, or `/app` in Docker. Mount `./data` and `./cert` volumes. +- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `DATA_DIR` (default `data`, CWD-relative; the SQLite dir, auto-created), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only). +- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `$DATA_DIR/task_queue.db` (default `data/task_queue.db`, CWD-relative — run from the workspace root, or `/app` in Docker; set `DATA_DIR` to pin state anywhere). Mount `./data` and `./cert` volumes. - `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`. - Docs are in Chinese (README, AGENTS.md); user-facing bot strings are in English. Keep that split when editing user-facing strings and docs. diff --git a/README.en.md b/README.en.md index 45d2c28..1f68053 100644 --- a/README.en.md +++ b/README.en.md @@ -84,6 +84,7 @@ Telegram only accepts ports 443/80/88/8443. | `BOT_ADMIN` | Admin chat IDs, comma-separated; receives start/stop notifications | | `EDIT_MESSAGE_TTL_SECONDS` | Edit-before-forward record expiry in seconds, default 86400 | | `LINK_CACHE_TTL_SECONDS` | Link-result cache expiry in seconds, default 604800 (7 days) | +| `DATA_DIR` | Data directory (where the SQLite `task_queue.db` lives), default `data` (relative to the working directory, created automatically) | | `RUST_LOG` | Log level | | `TELOXIDE_PROXY` | HTTP proxy (e.g. `http://127.0.0.1:10808`); applies to both the Telegram Bot API and site fetches — required on restricted networks (e.g. behind the GFW) | | `LOCAL_USER_ID` | UID the container runs as, default 9001 | diff --git a/README.md b/README.md index 8bf8513..98163f4 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Telegram 只接受 443/80/88/8443 端口。 | `BOT_ADMIN` | 管理员聊天 ID，逗号分隔；接收启动/停止通知 | | `EDIT_MESSAGE_TTL_SECONDS` | 转发前编辑记录过期秒数，默认 86400 | | `LINK_CACHE_TTL_SECONDS` | 链接结果缓存过期秒数，默认 604800（7 天） | +| `DATA_DIR` | 数据目录（SQLite 数据库 `task_queue.db` 所在目录），默认 `data`（相对工作目录，会自动创建） | | `RUST_LOG` | 日志级别 | | `TELOXIDE_PROXY` | HTTP 代理（如 `http://127.0.0.1:10808`）；同时作用于 Telegram Bot API 与站点抓取请求，网络受限环境（如 GFW）必需 | | `LOCAL_USER_ID` | 容器内运行用户 UID，默认 9001 | diff --git a/crates/xmedia-bot/src/handlers/statics.rs b/crates/xmedia-bot/src/handlers/statics.rs index 57dc291..28984e3 100644 --- a/crates/xmedia-bot/src/handlers/statics.rs +++ b/crates/xmedia-bot/src/handlers/statics.rs @@ -12,8 +12,24 @@ use std::sync::{Arc, LazyLock}; /// cache): a single pool bounds concurrent DB work on `data/task_queue.db` /// instead of three independent pools competing for the same file. The schema /// for all three tables is initialized once, here. -static DB: LazyLock> = - LazyLock::new(|| db::open_store("data/task_queue.db").expect("failed to open database")); +static DB: LazyLock> = LazyLock::new(|| { + let path = db_path(); + db::open_store(&path.to_string_lossy()).expect("failed to open database") +}); + +/// DB file location: `$DATA_DIR/task_queue.db` (default `data`, relative to +/// the working directory — keeps the docker-compose `./data` mount and local +/// runs unchanged). The directory is created if missing: SQLite does not +/// create parent dirs, so the old hardcoded `data/task_queue.db` failed with +/// a confusing error when started from a directory without `data/`, and a +/// CWD-relative path is a footgun for systemd / cron deployments — `DATA_DIR` +/// lets them pin the state anywhere. +fn db_path() -> std::path::PathBuf { + let dir = std::env::var("DATA_DIR").unwrap_or_else(|_| "data".to_string()); + let dir_path = std::path::Path::new(&dir); + std::fs::create_dir_all(dir_path).expect("failed to create data directory"); + dir_path.join("task_queue.db") +} pub static CHAT_STORE: LazyLock = LazyLock::new(|| ChatStore::new(Arc::clone(&DB))); pub static TASK_QUEUE: LazyLock =